From b7a22146a0c2ea4e0a2c4bcb0324ecefa42ce077 Mon Sep 17 00:00:00 2001 From: Hare Date: Tue, 20 Oct 2026 06:00:58 +0900 Subject: [PATCH 01/28] fix: recover confirmed worker restore failures --- crates/worker-runtime/src/execution.rs | 19 + crates/worker-runtime/src/runtime.rs | 16 +- crates/worker-runtime/src/worker_backend.rs | 1510 ++++++++++++----- .../worker_backend_restore_failure_tests.rs | 945 +++++++++++ crates/worker/src/bootstrap.rs | 6 +- crates/worker/src/controller.rs | 20 +- crates/worker/src/lib.rs | 11 +- .../worker/src/runtime/worker_allocation.rs | 5 +- .../src/runtime/worker_allocation/error.rs | 20 + .../runtime/worker_allocation/lifecycle.rs | 557 +++++- .../src/runtime/worker_allocation/mutate.rs | 25 +- .../src/runtime/worker_allocation/table.rs | 9 + crates/worker/src/worker.rs | 788 ++++++++- crates/worker/tests/controller_test.rs | 205 +++ .../tests/restored_parent_control_tests.rs | 5 +- .../src/server_assignment_attention_tests.rs | 11 +- .../src/server_assignment_delivery_tests.rs | 11 +- .../src/server_workspace_config_wip_tests.rs | 5 +- 18 files changed, 3621 insertions(+), 547 deletions(-) create mode 100644 crates/worker-runtime/src/worker_backend_restore_failure_tests.rs diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index f15d5ca4e..56892006b 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -630,6 +630,16 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { Ok(()) } + /// Completed resource-owner evidence retained while Runtime settlement may + /// still need retry. This is not a new request or execution authorization. + fn completed_restore_failure( + &self, + _operation_id: WorkerLifecycleOperationId, + _worker_ref: &WorkerRef, + ) -> Option { + None + } + fn reconcile_restore( &self, request: WorkerExecutionRestoreRequest, @@ -898,6 +908,15 @@ impl WorkerExecutionBackendRef { self.backend.preflight_restore(request) } + pub(crate) fn completed_restore_failure( + &self, + operation_id: WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Option { + self.backend + .completed_restore_failure(operation_id, worker_ref) + } + pub(crate) fn reconcile_restore( &self, request: WorkerExecutionRestoreRequest, diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 2b671c6e5..9f41c41f4 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -2489,6 +2489,14 @@ impl Runtime { .map(|result| (Some(result), None)) } + #[cfg(all(test, feature = "fs-store"))] + pub(crate) fn test_runtime_store(&self) -> FsRuntimeStore { + match &self.lock().unwrap().persistence { + RuntimePersistence::Fs(store) => store.clone(), + RuntimePersistence::Memory => panic!("expected filesystem Runtime"), + } + } + #[cfg(test)] pub(crate) fn test_restore_request( &self, @@ -2944,7 +2952,11 @@ impl Runtime { return Err(error); } - let restore_result = if reconcile { + let completed_failure = backend.completed_restore_failure(operation_id, worker_ref); + let has_completed_failure = completed_failure.is_some(); + let restore_result = if let Some(result) = completed_failure { + result + } else if reconcile { backend.reconcile_restore(request) } else { backend.restore_worker(request) @@ -3069,7 +3081,7 @@ impl Runtime { } } WorkerExecutionSpawnResult::Rejected(result) => { - if reconcile { + if reconcile && !has_completed_failure { return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_reconciliation_rejected", diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 74044cde9..55617d312 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -175,6 +175,163 @@ pub struct RuntimeWorkerController { pub workspace_client: Arc, } +/// Failure facts supplied by the resource owner, never inferred from error text. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimeRestoreStage { + Preflight, + Metadata, + Reconstruction, + Binding, + Feature, + Controller, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimeRestoreResources { + None, + WorkerStartup, + WorkdirBinding, + Unproven, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum RuntimeRestoreCleanupFailure { + #[error("restore resource ownership unavailable")] + Ownership, + #[error("restore Workdir close failed")] + Workdir, + #[error("restore run artifact cleanup failed")] + Artifacts, + #[error("restore allocation release unconfirmed")] + Allocation, + #[error("restore Feature task settlement unconfirmed")] + Feature, + #[error("factory cannot confirm restore cleanup")] + Unsupported, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RuntimeRestoreCleanup { + NotNeeded, + Required, + Confirmed, + Failed(RuntimeRestoreCleanupFailure), +} + +/// Diagnostics are fixed owner-selected causes, not arbitrary provider messages, +/// paths, configuration, or credentials. Cleanup cannot overwrite the first cause. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeRestoreFailure { + pub stage: RuntimeRestoreStage, + pub resources: RuntimeRestoreResources, + cause: &'static str, + cleanup: RuntimeRestoreCleanup, + io_kind: Option, + cleanup_failure: Option, +} + +impl RuntimeRestoreFailure { + pub fn refused(stage: RuntimeRestoreStage, cause: &'static str) -> Self { + Self { + stage, + resources: RuntimeRestoreResources::None, + cause, + cleanup: RuntimeRestoreCleanup::NotNeeded, + io_kind: None, + cleanup_failure: None, + } + } + + pub fn unproven(stage: RuntimeRestoreStage, cause: &'static str) -> Self { + Self { + stage, + resources: RuntimeRestoreResources::Unproven, + cause, + cleanup: RuntimeRestoreCleanup::Required, + io_kind: None, + cleanup_failure: None, + } + } + + fn started(stage: RuntimeRestoreStage, cause: &'static str) -> Self { + Self { + stage, + resources: RuntimeRestoreResources::WorkerStartup, + cause, + cleanup: RuntimeRestoreCleanup::Required, + io_kind: None, + cleanup_failure: None, + } + } + + fn needs_cleanup(&self) -> bool { + matches!( + self.cleanup, + RuntimeRestoreCleanup::Required | RuntimeRestoreCleanup::Failed(_) + ) + } +} + +impl std::fmt::Display for RuntimeRestoreFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let cause: String = self.cause.chars().take(160).collect(); + write!( + f, + "restore {:?}: {}; io={:?}; resources={:?}; cleanup={:?}; prior_cleanup={:?}", + self.stage, cause, self.io_kind, self.resources, self.cleanup, self.cleanup_failure + ) + } +} + +fn restore_worker_cause(error: &WorkerError) -> &'static str { + match error { + WorkerError::ConstructionCleanup { source, .. } => restore_worker_cause(source), + WorkerError::SegmentEmpty { .. } => "saved segment empty", + WorkerError::SegmentSystemPromptMissing { .. } => "saved segment system prompt missing", + WorkerError::WorkerMetadataPending { .. } => "saved Worker metadata pending", + WorkerError::WorkerMetadataMissing { .. } => "saved Worker metadata missing", + WorkerError::WorkerMetadataInactive { .. } => "saved Worker metadata inactive", + WorkerError::WorkerMetadataManifestSnapshotMissing { .. } => { + "saved manifest snapshot missing" + } + WorkerError::ScopeLock(_) => "Worker allocation acquisition failed", + WorkerError::Store(_) => "saved segment read failed", + WorkerError::WorkerStore(_) => "Worker metadata read failed", + WorkerError::Provider(_) => "saved model provider initialization failed", + WorkerError::SubjektivSessionAttribution { .. } => "restored attribution failed", + WorkerError::FeatureInstall(_) | WorkerError::FeatureLifecycle(_) => { + "restored Feature installation failed" + } + _ => "saved Worker reconstruction failed", + } +} + +async fn finish_restore_failure( + factory: &F, + request: &WorkerExecutionRestoreRequest, + mut failure: RuntimeRestoreFailure, +) -> RuntimeRestoreFailure { + if failure.needs_cleanup() { + let cleanup = if failure.resources == RuntimeRestoreResources::WorkdirBinding { + // Mutable materialization is outside the factory's cleanup authority. + // Preserve the exact accepted operation for its resource owner; never + // infer rollback from a factory that did not acquire these bindings. + Err(RuntimeRestoreCleanupFailure::Unsupported) + } else { + factory.cleanup_failed_restore(request).await + }; + failure.cleanup = match cleanup { + Ok(()) => RuntimeRestoreCleanup::Confirmed, + Err(error) => { + failure.cleanup_failure.get_or_insert(error); + RuntimeRestoreCleanup::Failed(error) + } + }; + } + tracing::warn!(failure = %failure, "Worker restore failed; operation-owned cleanup evidence retained"); + failure +} + /// Factory seam used by [`WorkerRuntimeExecutionBackend`] to construct a real /// controller-backed Worker for a Runtime catalog entry. #[async_trait] @@ -235,7 +392,7 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { async fn restore_controller( &self, request: WorkerExecutionRestoreRequest, - ) -> Result; + ) -> Result; /// Prove cleanup of a restore future that completed without returning a /// controller. An error is not absence evidence; unsupported factories keep @@ -243,8 +400,8 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { async fn cleanup_failed_restore( &self, _request: &WorkerExecutionRestoreRequest, - ) -> Result<(), String> { - Err("Runtime Worker factory cannot prove failed restore cleanup".to_string()) + ) -> Result<(), RuntimeRestoreCleanupFailure> { + Err(RuntimeRestoreCleanupFailure::Unsupported) } /// Verify/release factory-owned resources when the backend registry has no @@ -508,20 +665,73 @@ pub struct ProfileRuntimeWorkerFactory { workspace_request_clients: Arc>, embedded_worker_mutation_dispatcher: Option>, controller_transport: WorkerControllerTransport, - failed_restore_sessions: Arc< - Mutex< - HashMap< - WorkerRef, - ( - crate::execution::WorkerLifecycleOperationId, - Arc, - ), - >, - >, - >, + failed_restore_resources: Arc>>, + #[cfg(test)] + restore_faults: Arc, +} + +#[derive(Clone)] +struct FailedRestoreResources { + operation_id: crate::execution::WorkerLifecycleOperationId, + sessions: Vec>, + allocation: Option, + feature_tasks: Option, +} + +#[cfg(test)] +#[derive(Default)] +struct RestoreFaults { + fail_at: Mutex>, + executions: std::sync::atomic::AtomicUsize, + cleanup_fail: AtomicBool, + socket_failure: AtomicBool, + factory_gate: Mutex>>, + factory_entered: Mutex>>, + cleanup_gate: Mutex>>, + cleanup_entered: Mutex>>, } impl ProfileRuntimeWorkerFactory { + fn retain_construction_failure( + &self, + request: &WorkerExecutionRestoreRequest, + error: &WorkerError, + ) { + let mut resources = self + .failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(owner) = resources.get_mut(&request.worker_ref) else { + return; + }; + if owner.operation_id != request.operation_id { + return; + } + match error { + WorkerError::ConstructionCleanup { + allocation, + sessions, + .. + } => { + owner.allocation = allocation.clone(); + owner.sessions.push(sessions.clone()); + } + WorkerError::ScopeLock(error) => owner.allocation = error.allocation_cleanup_handle(), + _ => {} + } + } + + #[cfg(test)] + fn restore_checkpoint(&self, stage: RuntimeRestoreStage) -> Result<(), String> { + let mut failure = self.restore_faults.fail_at.lock().unwrap(); + if *failure == Some(stage) { + failure.take(); + Err("injected restore boundary failure".into()) + } else { + Ok(()) + } + } + pub fn new(profile_base_dir: impl Into) -> Self { let profile_base_dir = profile_base_dir.into(); Self { @@ -536,7 +746,9 @@ impl ProfileRuntimeWorkerFactory { workspace_request_clients: Arc::new(HashMap::new()), embedded_worker_mutation_dispatcher: None, controller_transport: WorkerControllerTransport::InProcess, - failed_restore_sessions: Arc::new(Mutex::new(HashMap::new())), + failed_restore_resources: Arc::new(Mutex::new(HashMap::new())), + #[cfg(test)] + restore_faults: Arc::new(RestoreFaults::default()), } } @@ -1708,265 +1920,403 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { async fn restore_controller( &self, request: WorkerExecutionRestoreRequest, - ) -> Result { - self.preflight_restore(&request).await?; - let worker_name = Self::runtime_worker_name_for_ref(&request.worker_ref); - let only_binding = (request.workdir_attachments.len() == 1) - .then(|| request.workdir_attachments.values().next()) - .flatten(); - let filesystem_authority = only_binding - .map(|binding| { - WorkerFilesystemAuthority::local( - binding.root().to_path_buf(), - binding.cwd().to_path_buf(), - ) - }) - .unwrap_or(WorkerFilesystemAuthority::None); - let workspace_backend_ref = RuntimeWorkspaceBackendRef::from_worker_request( - &request.request, - self.runtime_id.as_deref(), - ); - let observation_runtime_id = self.runtime_id.clone(); - let observation_workspace_id = request - .request - .workspace_api - .as_ref() - .map(|api| api.workspace_id.clone()); - let observation_grants = request.request.worker_observation_grants.clone(); - let observation_enabled = request.request.worker_observation_enabled; - let workspace_request_client = request - .request - .workspace_api - .as_ref() - .and_then(|api| self.workspace_request_clients.get(&api.workspace_id)); - let workspace_context = workspace_backend_ref.worker_context( - &request.worker_ref, - request.workspace_scope.as_ref(), - self.worker_mutation_identity.as_ref(), - workspace_request_client, - self.embedded_worker_mutation_dispatcher.as_ref(), - Some(self.prompt_projection_cache.clone()), - ); - - let worker_aggregate_dir = self.worker_aggregate_dir(&request.worker_ref)?; - let session_dir = worker_aggregate_dir.join("session"); - let session_store = - WorkerSessionStore::new_with_retention(&session_dir, self.session_retention()?) - .map_err(|err| { - format!( - "failed to initialize canonical Worker Session store at {}: {err}", - session_dir.display() - ) - })?; - let worker_metadata_store = - WorkerAggregateStore::new(&worker_aggregate_dir, worker_name.clone()).map_err( - |err| { - format!( - "failed to initialize canonical Worker metadata store at {}: {err}", - worker_aggregate_dir.display() - ) - }, - )?; - let metadata = worker_metadata_store - .read_by_name(&worker_name) - .map_err(|error| format!("failed to read Worker metadata: {error}"))? - .ok_or_else(|| "Persisted Worker metadata is unavailable".to_string())?; - let (manifest, loader) = Self::manifest_for_restore(&metadata, &request.request)?; - let store = CombinedStore::new(session_store, worker_metadata_store); - - let mut worker = match Worker::restore_from_worker_metadata_with_context( - &worker_name, - manifest.clone(), - store.clone(), - loader.clone(), - workspace_context.clone(), - filesystem_authority.clone(), - ) - .await - { - Ok(worker) => worker, - Err(WorkerError::WorkerMetadataPending { .. }) - if request.request.initial_input.is_none() => - { - let pending_loader = if workspace_context.workspace_id().is_some() { - let bundle = request.config_bundle.as_ref().ok_or_else(|| { - "pending Workspace Worker restore requires operation-owned launch material; generic restore must not reconstruct it from current Workspace config" - .to_string() - })?; - let resolution = self - .observe_bundle_prompt_projection( - bundle, - observation_workspace_id.as_deref(), - )? - .ok_or_else(|| { - "pending Workspace Worker restore requires a saved Workspace Prompt projection" - .to_string() - })?; - loader - .clone() - .with_effective_catalog(resolution.projection.catalog.clone()) - } else { - loader.clone() - }; - Worker::restore_pending_from_worker_metadata_with_context( - &worker_name, - manifest.clone(), - store, - pending_loader, - workspace_context, - filesystem_authority, - ) - .await - .map_err(|err| format!("failed to recreate pending Worker from metadata: {err}"))? - } - Err(err) => return Err(format!("failed to restore Worker from metadata: {err}")), - }; - if let Some(binding) = request.request.backend_job.as_ref() { - worker - .bind_backend_job(binding.clone()) - .map_err(str::to_string)?; - } - validate_worker_memory_settings(worker.manifest(), &request.request)?; - worker - .finalize_subjektiv_session_attribution( - request.request.subjektiv_attached, - SubjektivSessionAttributionLifecycle::RestoredSession, + ) -> Result { + self.preflight_restore(&request).await.map_err(|_| { + RuntimeRestoreFailure::refused( + RuntimeRestoreStage::Preflight, + "restore preflight refused", ) - .map_err(|error| format!("finalize restored Worker Session attribution: {error}"))?; - let flow_transition_enabled = worker.manifest().feature.flow.enabled; - let workdir_sessions = restored_workdir_router( - &request.workdir_attachments, - &request.logical_workdir_attachments, - &request.request.workdir_attachments, - worker.scope().clone(), - worker.workspace_client_handle(), - )?; - worker.bind_workdir_sessions(workdir_sessions); - if let (Some(runtime_id), Some(workspace_id)) = - (observation_runtime_id, observation_workspace_id.clone()) - && observation_enabled - { - let mut providers: Vec> = vec![Arc::new( - WorkspaceClientWorkerObservationProvider::new(worker.workspace_client_handle()), - )]; - if !observation_grants.is_empty() { - providers.push(Arc::new(RuntimeGrantedWorkerObservationProvider { - runtime_id, - workspace_id, - grants: observation_grants.into_iter().take(100).collect(), - hub: self.observation_hub.clone(), - })); - } - worker.bind_worker_observation_provider(Some(Arc::new( - CompositeWorkerObservationProvider::new(providers), - ))); - } - if flow_transition_enabled { - let report = worker - .install_runtime_flow_transition_feature() - .map_err(|error| format!("install Flow transition feature: {error}"))?; - if report.reports.iter().any(|report| !report.installed) { - return Err(format!( - "install Flow transition feature failed: {:?}", - report.reports - )); - } - } - - let workspace_client = worker.workspace_client_handle(); - let run_dir = self.worker_restore_run_dir(&request.worker_ref, request.operation_id)?; - let bash_output_dir = bash_output_dir_for_worker_id(&request.worker_ref.worker_id); - let cleanup_sessions = worker.workdir_sessions(); - let started = PreparedWorker::new( - worker, - WorkerBootstrapLayout::RuntimeManagedRun { - run_dir: run_dir.clone(), - bash_output_dir, - }, - self.controller_transport, - ) - .start() - .await - .map_err(|error| match error { - WorkerBootstrapError::Worker(source) => { - format!("failed to prepare restored Worker: {source}") - } - WorkerBootstrapError::Controller { - source, - cleanup_failed, - } => { - if cleanup_failed { - self.failed_restore_sessions - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert( - request.worker_ref.clone(), - (request.operation_id, cleanup_sessions), - ); - } - format!( - "failed to spawn restored Worker controller in {}: {source}", - run_dir.display() - ) - } })?; - let (handle, shutdown_rx, controller_task) = - (started.handle, started.shutdown, started.controller_task); - if flow_transition_enabled { - handle.shared_state.enable_flow_transition(); + #[cfg(test)] + self.restore_faults + .executions + .fetch_add(1, Ordering::SeqCst); + let mut stage = RuntimeRestoreStage::Metadata; + let mut io_kind = None; + // Reconstruction can write before failing; only the resource owner can + // confirm cleanup, even when no Worker value was returned. + let result: Result = async { + let worker_name = Self::runtime_worker_name_for_ref(&request.worker_ref); + let only_binding = (request.workdir_attachments.len() == 1) + .then(|| request.workdir_attachments.values().next()) + .flatten(); + let filesystem_authority = only_binding + .map(|binding| { + WorkerFilesystemAuthority::local( + binding.root().to_path_buf(), + binding.cwd().to_path_buf(), + ) + }) + .unwrap_or(WorkerFilesystemAuthority::None); + let workspace_backend_ref = RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + self.runtime_id.as_deref(), + ); + let observation_runtime_id = self.runtime_id.clone(); + let observation_workspace_id = request + .request + .workspace_api + .as_ref() + .map(|api| api.workspace_id.clone()); + let observation_grants = request.request.worker_observation_grants.clone(); + let observation_enabled = request.request.worker_observation_enabled; + let workspace_request_client = request + .request + .workspace_api + .as_ref() + .and_then(|api| self.workspace_request_clients.get(&api.workspace_id)); + let workspace_context = workspace_backend_ref.worker_context( + &request.worker_ref, + request.workspace_scope.as_ref(), + self.worker_mutation_identity.as_ref(), + workspace_request_client, + self.embedded_worker_mutation_dispatcher.as_ref(), + Some(self.prompt_projection_cache.clone()), + ); + + let worker_aggregate_dir = self + .worker_aggregate_dir(&request.worker_ref) + .map_err(|_| "Worker aggregate root unavailable")?; + let session_dir = worker_aggregate_dir.join("session"); + let session_store = WorkerSessionStore::new_with_retention( + &session_dir, + self.session_retention() + .map_err(|_| "Session retention unavailable")?, + ) + .map_err(|_| "canonical Session store initialization failed")?; + let worker_metadata_store = + WorkerAggregateStore::new(&worker_aggregate_dir, worker_name.clone()) + .map_err(|_| "canonical metadata store initialization failed")?; + let metadata = worker_metadata_store + .read_by_name(&worker_name) + .map_err(|_| "Worker metadata read failed")? + .ok_or("saved Worker metadata unavailable")?; + let (manifest, loader) = Self::manifest_for_restore(&metadata, &request.request) + .map_err(|_| "saved manifest resolution failed")?; + let store = CombinedStore::new(session_store, worker_metadata_store); + + stage = RuntimeRestoreStage::Reconstruction; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: Vec::new(), + allocation: None, + feature_tasks: None, + }, + ); + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + let mut worker = match Worker::restore_from_worker_metadata_with_context( + &worker_name, + manifest.clone(), + store.clone(), + loader.clone(), + workspace_context.clone(), + filesystem_authority.clone(), + ) + .await + { + Ok(worker) => worker, + Err(WorkerError::WorkerMetadataPending { .. }) + if request.request.initial_input.is_none() => + { + let pending_loader = if workspace_context.workspace_id().is_some() { + let bundle = request + .config_bundle + .as_ref() + .ok_or("pending restore launch material missing")?; + let resolution = self + .observe_bundle_prompt_projection( + bundle, + observation_workspace_id.as_deref(), + ) + .map_err(|_| "pending restore Prompt projection invalid")? + .ok_or("pending restore Prompt projection missing")?; + loader + .clone() + .with_effective_catalog(resolution.projection.catalog.clone()) + } else { + loader.clone() + }; + Worker::restore_pending_from_worker_metadata_with_context( + &worker_name, + manifest.clone(), + store, + pending_loader, + workspace_context, + filesystem_authority, + ) + .await + .map_err(|error| { + self.retain_construction_failure(&request, &error); + restore_worker_cause(&error) + })? + } + Err(error) => { + self.retain_construction_failure(&request, &error); + return Err(restore_worker_cause(&error)); + } + }; + let feature_tasks = worker.retain_startup_feature_cleanup(); + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: vec![worker.workdir_sessions()], + allocation: worker.restore_allocation_cleanup_handle(), + feature_tasks: Some(feature_tasks), + }, + ); + #[cfg(test)] + { + let gate = self.restore_faults.factory_gate.lock().unwrap().take(); + if let Some(gate) = gate { + if let Some(entered) = + self.restore_faults.factory_entered.lock().unwrap().take() + { + let _ = entered.send(()); + } + gate.notified().await; + } + } + stage = RuntimeRestoreStage::Binding; + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + if let Some(binding) = request.request.backend_job.as_ref() { + worker + .bind_backend_job(binding.clone()) + .map_err(|_| "restored Backend Job binding failed")?; + } + validate_worker_memory_settings(worker.manifest(), &request.request) + .map_err(|_| "saved memory settings mismatch")?; + worker + .finalize_subjektiv_session_attribution( + request.request.subjektiv_attached, + SubjektivSessionAttributionLifecycle::RestoredSession, + ) + .map_err(|error| restore_worker_cause(&error))?; + let flow_transition_enabled = worker.manifest().feature.flow.enabled; + let workdir_sessions = restored_workdir_router( + &request.workdir_attachments, + &request.logical_workdir_attachments, + &request.request.workdir_attachments, + worker.scope().clone(), + worker.workspace_client_handle(), + ) + .map_err(|_| "restored Workdir binding failed")?; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get_mut(&request.worker_ref) + .expect("retained restore owner") + .sessions + .push(workdir_sessions.clone()); + worker + .workdir_sessions() + .close_all() + .await + .map_err(|_| "initial Workdir close failed")?; + worker.bind_workdir_sessions(workdir_sessions); + if let (Some(runtime_id), Some(workspace_id)) = + (observation_runtime_id, observation_workspace_id.clone()) + && observation_enabled + { + let mut providers: Vec> = vec![Arc::new( + WorkspaceClientWorkerObservationProvider::new(worker.workspace_client_handle()), + )]; + if !observation_grants.is_empty() { + providers.push(Arc::new(RuntimeGrantedWorkerObservationProvider { + runtime_id, + workspace_id, + grants: observation_grants.into_iter().take(100).collect(), + hub: self.observation_hub.clone(), + })); + } + worker.bind_worker_observation_provider(Some(Arc::new( + CompositeWorkerObservationProvider::new(providers), + ))); + } + stage = RuntimeRestoreStage::Feature; + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + if flow_transition_enabled { + let report = worker + .install_runtime_flow_transition_feature() + .map_err(|_| "Flow transition installation failed")?; + if report.reports.iter().any(|report| !report.installed) { + return Err("Flow transition installation refused"); + } + } + + stage = RuntimeRestoreStage::Controller; + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + let workspace_client = worker.workspace_client_handle(); + let run_dir = self + .worker_restore_run_dir(&request.worker_ref, request.operation_id) + .map_err(|_| "restore run directory unavailable")?; + let bash_output_dir = bash_output_dir_for_worker_id(&request.worker_ref.worker_id); + let transport = self.controller_transport; + #[cfg(test)] + let transport = if self.restore_faults.socket_failure.load(Ordering::SeqCst) { + WorkerControllerTransport::UnixSocket + } else { + transport + }; + let started = PreparedWorker::new( + worker, + WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: run_dir.clone(), + bash_output_dir, + }, + transport, + ) + .start() + .await + .map_err(|error| match error { + WorkerBootstrapError::Worker(error) => restore_worker_cause(&error), + WorkerBootstrapError::Controller { source, .. } => { + io_kind = Some(source.kind()); + "restored Worker controller startup failed" + } + })?; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&request.worker_ref); + let (handle, shutdown_rx, controller_task) = + (started.handle, started.shutdown, started.controller_task); + if flow_transition_enabled { + handle.shared_state.enable_flow_transition(); + } + Ok(RuntimeWorkerController { + handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), + controller_task, + workspace_client, + }) } - Ok(RuntimeWorkerController { - handle, - shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), - controller_task, - workspace_client, + .await; + result.map_err(|cause| { + let mut failure = RuntimeRestoreFailure::started(stage, cause); + failure.io_kind = io_kind; + failure }) } async fn cleanup_failed_restore( &self, request: &WorkerExecutionRestoreRequest, - ) -> Result<(), String> { + ) -> Result<(), RuntimeRestoreCleanupFailure> { + #[cfg(test)] + { + let gate = self.restore_faults.cleanup_gate.lock().unwrap().take(); + if let Some(gate) = gate { + if let Some(entered) = self.restore_faults.cleanup_entered.lock().unwrap().take() { + let _ = entered.send(()); + } + gate.notified().await; + } + if self.restore_faults.cleanup_fail.load(Ordering::SeqCst) { + return Err(RuntimeRestoreCleanupFailure::Workdir); + } + } let retained = self - .failed_restore_sessions + .failed_restore_resources .lock() - .map_err(|_| "failed restore sessions lock is poisoned".to_string())? + .map_err(|_| RuntimeRestoreCleanupFailure::Ownership)? .get(&request.worker_ref) .cloned(); - if let Some((operation_id, sessions)) = retained { - if operation_id != request.operation_id { - return Err("failed restore sessions belong to another operation".to_string()); + if let Some(resources) = retained { + if resources.operation_id != request.operation_id { + return Err(RuntimeRestoreCleanupFailure::Ownership); + } + if let Some(feature_tasks) = resources.feature_tasks { + feature_tasks + .shutdown() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Feature)?; + } + for sessions in resources.sessions { + sessions + .close_all() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; + } + if let Some(allocation) = resources.allocation { + allocation + .release() + .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; + } else { + // Reconstruction may fail before returning its allocation guard. + // Absence is checked at the allocation authority, not inferred from Err. + let path = worker::runtime::worker_allocation::default_allocation_path() + .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; + let table = worker::runtime::worker_allocation::LockFileGuard::open(&path) + .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; + if table + .data() + .find(&Self::runtime_worker_name_for_ref(&request.worker_ref)) + .is_some() + { + return Err(RuntimeRestoreCleanupFailure::Allocation); + } } - sessions - .close_all() - .await - .map_err(|error| format!("failed restore session cleanup: {error}"))?; } // restore_controller starts the Controller only as its final fallible // step. A completed error leaves no live Controller; remove only this // operation's disposable run artifacts, never Session/metadata authority. - self.stopped_restore_run_dirs(&request.worker_ref)?; - let run_dir = self.worker_restore_run_dir(&request.worker_ref, request.operation_id)?; - Self::remove_restore_run_artifacts(&run_dir)?; - self.failed_restore_sessions + self.stopped_restore_run_dirs(&request.worker_ref) + .map_err(|_| RuntimeRestoreCleanupFailure::Artifacts)?; + let run_dir = self + .worker_restore_run_dir(&request.worker_ref, request.operation_id) + .map_err(|_| RuntimeRestoreCleanupFailure::Artifacts)?; + Self::remove_restore_run_artifacts(&run_dir) + .map_err(|_| RuntimeRestoreCleanupFailure::Artifacts)?; + self.failed_restore_resources .lock() - .map_err(|_| "failed restore sessions lock is poisoned".to_string())? + .map_err(|_| RuntimeRestoreCleanupFailure::Ownership)? .remove(&request.worker_ref); + self.observation_hub.get(&request.worker_ref); Ok(()) } async fn reconcile_stopped_worker(&self, worker_ref: &WorkerRef) -> Result<(), String> { let retained = self - .failed_restore_sessions + .failed_restore_resources .lock() .map_err(|_| "failed restore sessions lock is poisoned".to_string())? .get(worker_ref) .cloned(); - if let Some((_, sessions)) = retained.as_ref() { - sessions - .close_all() - .await - .map_err(|error| format!("failed restore session cleanup: {error}"))?; + if let Some(resources) = retained.as_ref() { + if let Some(feature_tasks) = resources.feature_tasks.as_ref() { + feature_tasks + .shutdown() + .await + .map_err(|_| "failed restore Feature cleanup".to_string())?; + } + for sessions in &resources.sessions { + sessions + .close_all() + .await + .map_err(|_| "failed restore Workdir cleanup".to_string())?; + } + if let Some(allocation) = resources.allocation.as_ref() { + allocation + .release() + .map_err(|_| "failed restore allocation cleanup".to_string())?; + } } // A restart loses the map, not the operation-owned filesystem artifacts. // Validate every candidate before deleting any, and retain map/journal @@ -1975,7 +2325,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { Self::remove_restore_run_artifacts(&run_dir)?; } if retained.is_some() { - self.failed_restore_sessions + self.failed_restore_resources .lock() .map_err(|_| "failed restore sessions lock is poisoned".to_string())? .remove(worker_ref); @@ -2310,7 +2660,7 @@ struct RuntimeWorkerExecution { struct PendingRuntimeRestore { request: WorkerExecutionRestoreRequest, task: tokio::sync::Mutex>>, - result: Mutex>>, + result: Mutex>>, failure: Mutex>, } @@ -2332,8 +2682,14 @@ impl PendingRuntimeRestore { } Ok(Err(error)) => { task.take(); - let message = - format!("restore factory task failed: {error}; cleanup is unproven"); + let message = format!( + "restore factory task failed ({}); cleanup is unproven", + if error.is_panic() { + "panic" + } else { + "cancelled" + } + ); *self .failure .lock() @@ -2363,6 +2719,18 @@ pub struct WorkerRuntimeExecutionBackend { runtime: Mutex>, workers: Mutex>, pending_restores: Mutex>>, + // One completed execution receipt per Worker, separate from live resources. + // Runtime owns durable request replay; retain this until the next admitted + // operation so a failed canonical settlement cannot re-execute the factory. + completed_restore_failures: Mutex< + HashMap< + WorkerRef, + ( + crate::execution::WorkerLifecycleOperationId, + RuntimeRestoreFailure, + ), + >, + >, worker_locks: Mutex>>>, spawn_restore_timeout: Duration, subject_reservations: Mutex>, @@ -2386,6 +2754,7 @@ where runtime: Mutex::new(Some(runtime)), workers: Mutex::new(HashMap::new()), pending_restores: Mutex::new(HashMap::new()), + completed_restore_failures: Mutex::new(HashMap::new()), worker_locks: Mutex::new(HashMap::new()), spawn_restore_timeout: SPAWN_RESTORE_TASK_TIMEOUT, subject_reservations: Mutex::new(HashMap::new()), @@ -2519,6 +2888,82 @@ where .map_err(|err| format!("worker adapter task did not complete: {err}"))? } + fn preflight_restore_inputs( + &self, + request: &WorkerExecutionRestoreRequest, + ) -> Result<(), WorkerExecutionResult> { + if !request.previous_workdir_attachments.is_empty() { + let materializer = self + .working_directory_materializer + .as_ref() + .ok_or_else(|| { + WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "Persisted Worker Workdir attachments cannot be restored by this Runtime", + ) + })?; + for previous in &request.previous_workdir_attachments { + let relative_cwd = request + .request + .workdir_attachments + .iter() + .find(|claim| claim.alias == previous.alias) + .and_then(|claim| claim.relative_cwd.as_deref()); + // Recovery attachments remain CleanupPending across restart. + // Validate current bindability/identity, not the old state name. + // Preflight is read-only; actual restore binds again below. + materializer.preflight_bind_working_directory( + &previous.working_directory.summary.working_directory_id, + relative_cwd, + ).map_err(|message| { + WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + format!("Persisted Worker Workdir attachment `{}` is unavailable: {message}", previous.alias), + ) + })?; + } + } + if request.previous_workdir_attachments.is_empty() + && !request.request.workdir_attachment_requests.is_empty() + { + return Err(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "Persisted Worker Workdir allocations are unavailable", + )); + } + if request.previous_workdir_attachments.is_empty() + && !request.request.workdir_attachments.is_empty() + && self.working_directory_materializer.is_none() + { + return Err(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "Persisted Worker Workdir claims cannot be restored by this Runtime", + )); + } + if let Some(workspace_api) = request.request.workspace_api.as_ref() { + let Some(scope) = request.workspace_scope.as_ref() else { + return Err(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "Persisted Workspace restore authorization is unavailable", + )); + }; + if scope.workspace_id != workspace_api.workspace_id { + return Err(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "Persisted Workspace restore authorization does not match Worker authority", + )); + } + } + + let factory = Arc::clone(&self.factory); + let request = request.clone(); + self.run_on_adapter_runtime(async move { factory.preflight_restore(&request).await }) + .map_err(|message| { + WorkerExecutionResult::rejected(WorkerExecutionOperation::Restore, message) + })?; + Ok(()) + } + fn pending_restore( &self, worker_ref: &WorkerRef, @@ -2534,10 +2979,94 @@ where pending: &Arc, timeout: Duration, ) -> Result<(), String> { - let pending = Arc::clone(pending); + let owner = Arc::clone(pending); self.run_cancellable_on_adapter_runtime(timeout + Duration::from_secs(1), async move { - pending.wait(timeout).await + owner.wait(timeout).await }) + .map_err(|message| { + match pending.result.lock().ok().and_then(|result| { + result + .as_ref() + .and_then(|result| result.as_ref().err()) + .cloned() + }) { + Some(failure) => format!("{failure}; {message}"), + None => message, + } + }) + } + + // Binding failures cannot borrow the factory's cleanup proof. Retain the + // exact operation and any already acquired bindings for their actual owner. + fn retain_restore_binding_failure( + &self, + mut request: WorkerExecutionRestoreRequest, + bindings: BTreeMap, + ) -> WorkerExecutionSpawnResult { + request.workdir_attachments = bindings; + let mut failure = RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Binding, + "restore Workdir binding failed; materializer reconciliation required", + ); + failure.resources = RuntimeRestoreResources::WorkdirBinding; + failure.cleanup = RuntimeRestoreCleanup::Failed(RuntimeRestoreCleanupFailure::Unsupported); + failure.cleanup_failure = Some(RuntimeRestoreCleanupFailure::Unsupported); + self.pending_restores + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + Arc::new(PendingRuntimeRestore { + request, + task: tokio::sync::Mutex::new(None), + result: Mutex::new(Some(Err(failure.clone()))), + failure: Mutex::new(None), + }), + ); + WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + failure.to_string(), + ), + worker_state: None, + workdir_attachments: Vec::new(), + } + } + + // Called only after joining the previous owned task, under the Worker lock. + // A caller timeout never cancels this close/join future. + fn retry_pending_restore_cleanup( + &self, + pending: &Arc, + ) -> Result<(), String> { + let failure = pending + .result + .lock() + .map_err(|_| "restore result lock is poisoned")? + .as_ref() + .and_then(|result| result.as_ref().err()) + .filter(|failure| failure.needs_cleanup()) + .cloned(); + if let Some(failure) = failure { + let owner = Arc::clone(pending); + let factory = Arc::clone(&self.factory); + let mut task = pending + .task + .try_lock() + .map_err(|_| "restore task is still owned")?; + if task.is_some() { + return Err("restore task has not been joined".into()); + } + *task = Some(self.spawn_on_adapter_runtime(async move { + let failure = + finish_restore_failure(factory.as_ref(), &owner.request, failure).await; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Err(failure)); + })?); + } + Ok(()) } fn remove_pending_restore(&self, worker_ref: &WorkerRef) -> Result<(), String> { @@ -3230,6 +3759,12 @@ where if let Err(message) = self.wait_pending_restore(&pending, Duration::from_secs(5)) { return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } + if let Err(message) = self + .retry_pending_restore_cleanup(&pending) + .and_then(|()| self.wait_pending_restore(&pending, Duration::from_secs(5))) + { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } if self.workers.lock().is_err() { return WorkerExecutionResult::errored( WorkerExecutionOperation::Stop, @@ -3264,17 +3799,14 @@ where Vec::new(), ); } - Some(Err(_)) => { - let factory = Arc::clone(&self.factory); - let request = pending.request.clone(); - if let Err(message) = self - .run_cancellable_on_adapter_runtime(RUNTIME_TASK_TIMEOUT, async move { - factory.cleanup_failed_restore(&request).await - }) - { + Some(Err(failure)) => { + if !matches!( + failure.cleanup, + RuntimeRestoreCleanup::NotNeeded | RuntimeRestoreCleanup::Confirmed + ) { return WorkerExecutionResult::errored( WorkerExecutionOperation::Stop, - message, + failure.to_string(), ); } } @@ -4067,76 +4599,29 @@ where } drop(workers); - if !request.previous_workdir_attachments.is_empty() { - let materializer = self - .working_directory_materializer - .as_ref() - .ok_or_else(|| { - WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - "Persisted Worker Workdir attachments cannot be restored by this Runtime", - ) - })?; - for previous in &request.previous_workdir_attachments { - let relative_cwd = request - .request - .workdir_attachments - .iter() - .find(|claim| claim.alias == previous.alias) - .and_then(|claim| claim.relative_cwd.as_deref()); - // Recovery attachments remain CleanupPending across restart. - // Validate current bindability/identity, not the old state name. - // Preflight is read-only; actual restore binds again below. - materializer.preflight_bind_working_directory( - &previous.working_directory.summary.working_directory_id, - relative_cwd, - ).map_err(|message| { - WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - format!("Persisted Worker Workdir attachment `{}` is unavailable: {message}", previous.alias), - ) - })?; - } - } - if request.previous_workdir_attachments.is_empty() - && !request.request.workdir_attachment_requests.is_empty() - { - return Err(WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - "Persisted Worker Workdir allocations are unavailable", - )); - } - if request.previous_workdir_attachments.is_empty() - && !request.request.workdir_attachments.is_empty() - && self.working_directory_materializer.is_none() - { - return Err(WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - "Persisted Worker Workdir claims cannot be restored by this Runtime", - )); - } - if let Some(workspace_api) = request.request.workspace_api.as_ref() { - let Some(scope) = request.workspace_scope.as_ref() else { - return Err(WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - "Persisted Workspace restore authorization is unavailable", - )); - }; - if scope.workspace_id != workspace_api.workspace_id { - return Err(WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - "Persisted Workspace restore authorization does not match Worker authority", - )); - } - } + self.preflight_restore_inputs(request) + } - let factory = Arc::clone(&self.factory); - let request = request.clone(); - self.run_on_adapter_runtime(async move { factory.preflight_restore(&request).await }) - .map_err(|message| { - WorkerExecutionResult::rejected(WorkerExecutionOperation::Restore, message) - })?; - Ok(()) + fn completed_restore_failure( + &self, + operation_id: crate::execution::WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Option { + let completed = self + .completed_restore_failures + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let (completed_id, failure) = completed.get(worker_ref)?; + if *completed_id != operation_id { + return None; + } + let result = + WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, failure.to_string()); + Some(if failure.cleanup == RuntimeRestoreCleanup::NotNeeded { + WorkerExecutionSpawnResult::Rejected(result) + } else { + WorkerExecutionSpawnResult::RolledBack(result) + }) } fn restore_worker( @@ -4162,6 +4647,11 @@ where )); } }; + if let Some(result) = + self.completed_restore_failure(request.operation_id, &request.worker_ref) + { + return result; + } let workers = self.workers.lock().map_err(|_| { WorkerExecutionResult::errored( WorkerExecutionOperation::Restore, @@ -4206,93 +4696,138 @@ where } drop(workers); - let mut workdir_attachments = BTreeMap::new(); - if !request.previous_workdir_attachments.is_empty() { - let Some(materializer) = self.working_directory_materializer.as_ref() else { - return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - "persisted Worker has Workdir attachments, but no materializer is configured for this Runtime backend", - )); - }; - for status in request.previous_workdir_attachments.clone() { - let relative_cwd = request + // An already accepted operation owns its bindings. Re-observing mutable + // Workdir materialization must not turn an unknown operation into a new + // read-only refusal or allocate another candidate. + let admitted_pending = match self.pending_restore(&request.worker_ref) { + Ok(Some(pending)) if pending.request.operation_id == request.operation_id => { + Some(pending) + } + Ok(Some(_)) => { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::busy( + WorkerExecutionOperation::Restore, + "another restore operation still owns pending resources", + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + Ok(None) => None, + Err(message) => { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + message, + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + }; + if let Some(pending) = admitted_pending.as_ref() { + request = pending.request.clone(); + } else { + if let Err(result) = self.preflight_restore_inputs(&request) { + return WorkerExecutionSpawnResult::Rejected(result); + } + let aliases = if request.previous_workdir_attachments.is_empty() { + request .request .workdir_attachments .iter() - .find(|claim| claim.alias == status.alias) - .and_then(|claim| claim.relative_cwd.as_deref()); - match materializer.bind_working_directory( - &status.working_directory.summary.working_directory_id, - relative_cwd, - ) { - Ok(binding) => { - if workdir_attachments - .insert(status.alias.clone(), binding) - .is_some() - { - return WorkerExecutionSpawnResult::Rejected( - WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - format!( - "duplicate persisted Workdir attachment alias `{}`", - status.alias - ), - ), - ); - } - } - Err(error) => { - return WorkerExecutionSpawnResult::Rejected( - WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - error.to_string(), - ), - ); - } - } - } - } else if !request.request.workdir_attachments.is_empty() { - let Some(materializer) = self.working_directory_materializer.as_ref() else { + .map(|claim| &claim.alias) + .collect::>() + } else { + request + .previous_workdir_attachments + .iter() + .map(|status| &status.alias) + .collect::>() + }; + if aliases + .iter() + .collect::>() + .len() + != aliases.len() + { return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( WorkerExecutionOperation::Restore, - "persisted Worker has Workdir claims, but no materializer is configured for this Runtime backend", + "duplicate restore Workdir attachment alias", )); - }; - for claim in request.request.workdir_attachments.clone() { - match materializer.bind_working_directory( - &claim.working_directory_id, - claim.relative_cwd.as_deref(), - ) { - Ok(binding) => { - if workdir_attachments - .insert(claim.alias.clone(), binding) - .is_some() - { - return WorkerExecutionSpawnResult::Rejected( - WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - format!("duplicate Workdir attachment alias `{}`", claim.alias), - ), - ); + } + let mut workdir_attachments = BTreeMap::new(); + if !request.previous_workdir_attachments.is_empty() { + let Some(materializer) = self.working_directory_materializer.as_ref() else { + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "persisted Worker has Workdir attachments, but no materializer is configured for this Runtime backend", + )); + }; + for status in request.previous_workdir_attachments.clone() { + let relative_cwd = request + .request + .workdir_attachments + .iter() + .find(|claim| claim.alias == status.alias) + .and_then(|claim| claim.relative_cwd.as_deref()); + match materializer.bind_working_directory( + &status.working_directory.summary.working_directory_id, + relative_cwd, + ) { + Ok(binding) => { + if workdir_attachments + .insert(status.alias.clone(), binding) + .is_some() + { + return self + .retain_restore_binding_failure(request, workdir_attachments); + } + } + Err(_) => { + return self + .retain_restore_binding_failure(request, workdir_attachments); } } - Err(error) => { - return WorkerExecutionSpawnResult::Rejected( - WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - error.to_string(), - ), - ); + } + } else if !request.request.workdir_attachments.is_empty() { + let Some(materializer) = self.working_directory_materializer.as_ref() else { + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "persisted Worker has Workdir claims, but no materializer is configured for this Runtime backend", + )); + }; + for claim in request.request.workdir_attachments.clone() { + match materializer.bind_working_directory( + &claim.working_directory_id, + claim.relative_cwd.as_deref(), + ) { + Ok(binding) => { + if workdir_attachments + .insert(claim.alias.clone(), binding) + .is_some() + { + return self + .retain_restore_binding_failure(request, workdir_attachments); + } + } + Err(_) => { + return self + .retain_restore_binding_failure(request, workdir_attachments); + } } } } + request.workdir_attachments = workdir_attachments.clone(); } - request.workdir_attachments = workdir_attachments.clone(); - let worker_ref = request.worker_ref.clone(); let operation_id = request.operation_id; + let mut existing_pending = false; let pending = match self.pending_restore(&worker_ref) { - Ok(Some(pending)) if pending.request.operation_id == operation_id => pending, + Ok(Some(pending)) if pending.request.operation_id == operation_id => { + existing_pending = true; + pending + } Ok(Some(_)) => { return WorkerExecutionSpawnResult::ReconciliationRequired { result: WorkerExecutionResult::busy( @@ -4304,16 +4839,10 @@ where }; } Ok(None) => { - let factory = Arc::clone(&self.factory); - let preflight_request = request.clone(); - if let Err(message) = self.run_on_adapter_runtime(async move { - factory.preflight_restore(&preflight_request).await - }) { - return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - message, - )); - } + self.completed_restore_failures + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&worker_ref); let pending = Arc::new(PendingRuntimeRestore { request: request.clone(), task: tokio::sync::Mutex::new(None), @@ -4335,7 +4864,23 @@ where let factory = Arc::clone(&self.factory); let result_owner = Arc::clone(&pending); let task = self.spawn_on_adapter_runtime(async move { - let result = factory.restore_controller(request).await; + let result = match factory.restore_controller(request.clone()).await { + Ok(controller) => Ok(controller), + Err(mut failure) => { + if failure.resources == RuntimeRestoreResources::None + && !request.workdir_attachments.is_empty() + { + failure.resources = RuntimeRestoreResources::WorkdirBinding; + failure.cleanup = RuntimeRestoreCleanup::Required; + } + *result_owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure.clone())); + Err(finish_restore_failure(factory.as_ref(), &request, failure).await) + } + }; let mut retained = result_owner .result .lock() @@ -4377,6 +4922,21 @@ where workdir_attachments: Vec::new(), }; } + if existing_pending { + if let Err(message) = self + .retry_pending_restore_cleanup(&pending) + .and_then(|()| self.wait_pending_restore(&pending, self.spawn_restore_timeout)) + { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + message, + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + } let controller = { let mut retained = match pending.result.lock() { Ok(retained) => retained, @@ -4396,12 +4956,39 @@ where Ok(controller) => controller, Err(_) => unreachable!(), }, - Some(Err(message)) => { + Some(Err(failure)) => { + let result = WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + failure.to_string(), + ); + let cleanup = failure.cleanup; + if matches!( + cleanup, + RuntimeRestoreCleanup::NotNeeded | RuntimeRestoreCleanup::Confirmed + ) { + self.completed_restore_failures + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(worker_ref.clone(), (operation_id, failure.clone())); + drop(retained); + if let Err(message) = self.remove_pending_restore(&worker_ref) { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + message, + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + return if cleanup == RuntimeRestoreCleanup::NotNeeded { + WorkerExecutionSpawnResult::Rejected(result) + } else { + WorkerExecutionSpawnResult::RolledBack(result) + }; + } return WorkerExecutionSpawnResult::ReconciliationRequired { - result: WorkerExecutionResult::errored( - WorkerExecutionOperation::Restore, - message.clone(), - ), + result, worker_state: None, workdir_attachments: Vec::new(), }; @@ -6508,7 +7095,7 @@ mod tests { } Err(error) => error, }; - assert!(error.contains(expected), "{error}"); + assert_eq!(error.stage, RuntimeRestoreStage::Preflight); assert_eq!(persisted_files(&runtime_store), before); assert!( factory @@ -6647,10 +7234,7 @@ mod tests { } Err(error) => error, }; - assert!( - error.contains("subjektiv settings snapshot mismatch"), - "{error}" - ); + assert_eq!(error.stage, RuntimeRestoreStage::Preflight); assert_eq!(persisted_files(&runtime_store), before); assert!(factory.observation_hub.workers.lock().unwrap().is_empty()); let run_dir = factory @@ -7421,7 +8005,7 @@ mod tests { async fn restore_controller( &self, _request: WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result { panic!("create timeout must not restore a Worker") } } @@ -7584,7 +8168,7 @@ mod tests { async fn restore_controller( &self, request: WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result { let request = WorkerExecutionSpawnRequest { worker_ref: request.worker_ref, request: request.request, @@ -7593,7 +8177,12 @@ mod tests { workdir_attachments: request.workdir_attachments, config_bundle: request.config_bundle, }; - self.spawn_controller(request).await + self.spawn_controller(request).await.map_err(|_| { + RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "mock controller startup failed", + ) + }) } } @@ -7901,8 +8490,11 @@ mod tests { async fn restore_controller( &self, _request: WorkerExecutionRestoreRequest, - ) -> Result { - Err("restore failed".to_string()) + ) -> Result { + Err(RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Reconstruction, + "restore failed", + )) } } @@ -7926,7 +8518,7 @@ mod tests { async fn restore_controller( &self, request: WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result { *self.restore_id.lock().unwrap() = Some(request.operation_id); let runs = self .runtime_store @@ -7940,14 +8532,17 @@ mod tests { b"incomplete restore artifact", ) .unwrap(); - Err("injected restore failure with persisted artifacts".to_string()) + Err(RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "injected restore failure with persisted artifacts", + )) } async fn cleanup_failed_restore( &self, _: &WorkerExecutionRestoreRequest, - ) -> Result<(), String> { - Err("injected restore artifact cleanup failure".to_string()) + ) -> Result<(), RuntimeRestoreCleanupFailure> { + Err(RuntimeRestoreCleanupFailure::Artifacts) } } @@ -8054,7 +8649,7 @@ mod tests { assert!( backend .factory - .failed_restore_sessions + .failed_restore_resources .lock() .unwrap() .is_empty() @@ -8107,16 +8702,18 @@ mod tests { fs::create_dir_all(run.parent().unwrap()).unwrap(); fs::write(&run, "invalid restore artifact").unwrap(); let sessions = Arc::new(WorkdirSessionRouter::new()); - factory.failed_restore_sessions.lock().unwrap().insert( + factory.failed_restore_resources.lock().unwrap().insert( worker_ref.clone(), - (request.operation_id, Arc::clone(&sessions)), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: vec![Arc::clone(&sessions)], + allocation: None, + feature_tasks: None, + }, ); assert!( - factory - .cleanup_failed_restore(&request) - .await - .unwrap_err() - .contains("unsafe restore artifact") + factory.cleanup_failed_restore(&request).await.unwrap_err() + == RuntimeRestoreCleanupFailure::Artifacts ); assert!( factory @@ -8126,20 +8723,20 @@ mod tests { .contains("unsafe restore artifact") ); let retained = factory - .failed_restore_sessions + .failed_restore_resources .lock() .unwrap() .get(&worker_ref) .cloned() .unwrap(); - assert_eq!(retained.0, request.operation_id); - assert!(Arc::ptr_eq(&retained.1, &sessions)); + assert_eq!(retained.operation_id, request.operation_id); + assert!(Arc::ptr_eq(&retained.sessions[0], &sessions)); fs::remove_file(&run).unwrap(); fs::create_dir(&run).unwrap(); fs::write(run.join("worker.out.log"), "stale restore artifact").unwrap(); factory.reconcile_stopped_worker(&worker_ref).await.unwrap(); assert!(!run.exists()); - assert!(factory.failed_restore_sessions.lock().unwrap().is_empty()); + assert!(factory.failed_restore_resources.lock().unwrap().is_empty()); } #[cfg(unix)] @@ -8206,6 +8803,9 @@ mod tests { ); } + #[cfg(feature = "fs-store")] + include!("worker_backend_restore_failure_tests.rs"); + struct PendingFailureFactory { restore_calls: Arc, cleanup_calls: Arc, @@ -8223,20 +8823,23 @@ mod tests { async fn restore_controller( &self, _request: WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result { self.restore_calls.fetch_add(1, Ordering::SeqCst); tokio::time::sleep(Duration::from_millis(50)).await; - Err("injected uncertain restore side effect".to_string()) + Err(RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Reconstruction, + "injected uncertain restore side effect", + )) } async fn cleanup_failed_restore( &self, _request: &WorkerExecutionRestoreRequest, - ) -> Result<(), String> { + ) -> Result<(), RuntimeRestoreCleanupFailure> { self.cleanup_calls.fetch_add(1, Ordering::SeqCst); if self.cleanup_allowed.load(Ordering::SeqCst) { Ok(()) } else { - Err("injected pending restore cleanup failure".to_string()) + Err(RuntimeRestoreCleanupFailure::Workdir) } } } @@ -8307,12 +8910,7 @@ mod tests { failed.outcome, crate::execution::WorkerExecutionOutcome::Errored ); - assert!( - failed - .message - .unwrap() - .contains("pending restore cleanup failure") - ); + assert!(failed.message.unwrap().contains("Workdir")); assert_eq!( backend .pending_restore(&worker_ref) @@ -8330,7 +8928,7 @@ mod tests { "cleanup then catalog-commit retry must converge" ); assert_eq!(restore_calls.load(Ordering::SeqCst), 1); - assert_eq!(cleanup_calls.load(Ordering::SeqCst), 2); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 3); } struct LateControllerFactory { @@ -8351,7 +8949,7 @@ mod tests { async fn restore_controller( &self, request: WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result { let first = self.restores.fetch_add(1, Ordering::SeqCst) == 0; if first { self.release.notified().await; @@ -8831,7 +9429,7 @@ mod tests { Ok(_) => panic!("legacy Workspace Worker restore unexpectedly succeeded"), Err(error) => error, }; - assert!(error.contains("replacement Worker is required"), "{error}"); + assert_eq!(error.stage, RuntimeRestoreStage::Preflight); } #[test] diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs new file mode 100644 index 000000000..af39d3619 --- /dev/null +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -0,0 +1,945 @@ +// Integration coverage through EmbeddedRuntime -> real Profile factory -> real +// Worker/bootstrap, local Git Workdir provider, and allocation table. Model I/O +// is not invoked; this is a deterministic ordinary suite, not product E2E. + +struct ProfileFailureFixture { + runtime: EmbeddedRuntime, + backend: Arc, + worker: crate::catalog::WorkerDetail, + _repo: tempfile::TempDir, + root: tempfile::TempDir, +} + +impl ProfileFailureFixture { + fn new(timeout: Duration) -> Self { + let root = tempfile::tempdir().unwrap(); + let repo = create_clean_repo(); + let store = root.path().join("private-restore-test/runtime"); + let backend = Arc::new( + WorkerRuntimeExecutionBackend::new( + ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store), + ) + .unwrap() + .with_spawn_restore_timeout(timeout) + .with_working_directory_materializer(RuntimeGitMaterializer::new( + root.path().join("workdirs"), + )), + ); + let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( + crate::fs_store::FsRuntimeStoreOptions { + root: store, + runtime_id: "restore-failure-test".into(), + display_name: None, + }, + backend.clone(), + ) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let mut request = create_request("restore failure recovery"); + request.profile = ProfileSelector::Builtin("default".into()); + request.workdir_attachment_requests = vec![WorkingDirectoryAttachmentRequest { + alias: WorkdirAttachmentAlias::new("checkout").unwrap(), + working_directory: working_directory_request(repo.path()), + }]; + let worker = runtime.create_worker(request).unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + Self { + root, + _repo: repo, + runtime, + backend, + worker, + } + } + + fn fail(&self, stage: RuntimeRestoreStage) { + *self.backend.factory.restore_faults.fail_at.lock().unwrap() = Some(stage); + } + + fn assert_released(&self) { + assert!(self.backend.pending_restores.lock().unwrap().is_empty()); + assert!(self.backend.workers.lock().unwrap().is_empty()); + assert!( + self.backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty() + ); + assert!( + self.backend + .factory + .observation_hub + .workers + .lock() + .unwrap() + .is_empty() + ); + let table = worker::runtime::worker_allocation::LockFileGuard::open( + &worker::runtime::worker_allocation::default_allocation_path().unwrap(), + ) + .unwrap(); + assert!( + table + .data() + .find(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref( + &self.worker.worker_ref + )) + .is_none() + ); + let aggregate = self + .backend + .factory + .worker_aggregate_dir(&self.worker.worker_ref) + .unwrap(); + assert!(fs::read_dir(aggregate.join("runs")).unwrap().all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("restore-") + })); + assert_eq!( + self.runtime + .worker_detail(&self.worker.worker_ref) + .unwrap() + .status, + crate::catalog::WorkerStatus::Stopped + ); + } + + fn fresh_restore_and_old_replay( + &self, + request: runtime_api::WorkerRestoreRequest, + failed: crate::runtime::RuntimeWorkerRestoreResult, + ) { + let calls = self + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst); + let replay = self + .runtime + .restore_worker_operation(&self.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + serde_json::to_value(&replay).unwrap(), + serde_json::to_value(&failed).unwrap() + ); + assert_eq!( + self.backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + calls + ); + let fresh = self.runtime.test_restore_request(&self.worker.worker_ref); + assert_ne!(fresh.request_id, request.request_id); + assert_ne!( + fresh.expected_observation_token, + request.expected_observation_token + ); + let restored = self + .runtime + .restore_worker_operation(&self.worker.worker_ref, fresh) + .unwrap(); + assert_eq!( + restored.state, + server_api::WorkerRestoreState::Accepted, + "{restored:?}" + ); + assert_eq!( + self.backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + calls + 1 + ); + assert_eq!(self.backend.workers.lock().unwrap().len(), 1); + let replay = self + .runtime + .restore_worker_operation(&self.worker.worker_ref, request) + .unwrap(); + assert_eq!( + serde_json::to_value(&replay).unwrap(), + serde_json::to_value(&failed).unwrap() + ); + assert_eq!( + self.backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + calls + 1 + ); + // Only tear down after the fresh explicit Restore has already succeeded. + self.runtime + .stop_worker(&self.worker.worker_ref, None) + .unwrap(); + } +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() { + for stage in [ + RuntimeRestoreStage::Reconstruction, + RuntimeRestoreStage::Binding, + RuntimeRestoreStage::Feature, + RuntimeRestoreStage::Controller, + ] { + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + fixture.fail(stage); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{stage:?}: {result:?}" + ); + let diagnostic = serde_json::to_string(&result).unwrap(); + assert!( + diagnostic.contains("injected restore boundary failure"), + "{diagnostic}" + ); + assert!(diagnostic.contains(&format!("{stage:?}")), "{diagnostic}"); + assert!(!diagnostic.contains(&fixture.root.path().to_string_lossy().to_string())); + assert!(diagnostic.len() < 1024); + fixture.assert_released(); + eprintln!( + "restore-fault stage={stage:?} outcome=RolledBack pending=0 controllers=0 allocations=0" + ); + fixture.fresh_restore_and_old_replay(request, result); + } +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_cleanup_failure_retains_owner_and_original_cause_until_same_operation_recovers() +{ + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + fixture.fail(RuntimeRestoreStage::Feature); + fixture + .backend + .factory + .restore_faults + .cleanup_fail + .store(true, Ordering::SeqCst); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let failed = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + failed.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + let message = pending + .result + .lock() + .unwrap() + .as_ref() + .unwrap() + .as_ref() + .err() + .unwrap() + .to_string(); + assert!( + message.contains("injected restore boundary failure"), + "{message}" + ); + assert!(message.contains("Workdir"), "{message}"); + let owned = fixture + .backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .get(&fixture.worker.worker_ref) + .cloned() + .unwrap(); + assert_eq!(owned.operation_id, pending.request.operation_id); + assert!(owned.allocation.is_some()); + assert!( + owned + .sessions + .iter() + .any(|sessions| !sessions.aliases().is_empty()) + ); + let repeat = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + repeat.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + assert!(Arc::ptr_eq( + &pending, + &fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap() + )); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + fixture + .backend + .factory + .restore_faults + .cleanup_fail + .store(false, Ordering::SeqCst); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{result:?}" + ); + assert!( + serde_json::to_string(&result) + .unwrap() + .contains("injected restore boundary failure") + ); + assert!( + owned + .sessions + .iter() + .all(|sessions| sessions.aliases().is_empty()) + ); + fixture.assert_released(); + eprintln!( + "restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=RolledBack" + ); + fixture.fresh_restore_and_old_replay(request, result); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconciles_without_execution() + { + let fixture = ProfileFailureFixture::new(Duration::from_secs(2)); + fixture.fail(RuntimeRestoreStage::Feature); + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .cleanup_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .cleanup_entered + .lock() + .unwrap() = Some(entered_tx); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + entered.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + assert!( + pending + .result + .lock() + .unwrap() + .as_ref() + .unwrap() + .as_ref() + .err() + .unwrap() + .to_string() + .contains("injected restore boundary failure") + ); + assert!(pending.task.try_lock().unwrap().is_some()); + for _ in 0..2 { + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + assert!(Arc::ptr_eq( + &pending, + &fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap() + )); + } + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + release.notify_one(); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{result:?}" + ); + fixture.assert_released(); + eprintln!( + "restore-fault cleanup=timeout retained-task=true late-completion=joined executions=1" + ); + fixture.fresh_restore_and_old_replay(request, result); +} + +#[cfg(unix)] +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation() { + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + fixture + .backend + .factory + .restore_faults + .socket_failure + .store(true, Ordering::SeqCst); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{result:?}" + ); + let diagnostic = serde_json::to_string(&result).unwrap(); + assert!( + diagnostic.contains("controller startup failed"), + "{diagnostic}" + ); + assert!(diagnostic.contains("InvalidInput"), "{diagnostic}"); + assert!(!diagnostic.contains("private-restore-test")); + fixture.assert_released(); + fixture + .backend + .factory + .restore_faults + .socket_failure + .store(false, Ordering::SeqCst); + eprintln!( + "restore-fault stage=socket-start real-bootstrap=true partial-run=removed allocation=released" + ); + fixture.fresh_restore_and_old_replay(request, result); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_read_only_refusal_replays_and_fresh_observation_can_restore() { + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + let aggregate = fixture + .backend + .factory + .worker_aggregate_dir(&fixture.worker.worker_ref) + .unwrap(); + fs::rename(aggregate.join("session"), aggregate.join("saved-session")).unwrap(); + let before = persisted_files(&aggregate); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::Rejected, + "{result:?}" + ); + let mut after = persisted_files(&aggregate); + let mut before = before; + // The Runtime rejection receipt is expected to commit; source authority is not. + before.remove(Path::new("worker.json")); + after.remove(Path::new("worker.json")); + assert_eq!(after, before); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 0 + ); + fs::rename(aggregate.join("saved-session"), aggregate.join("session")).unwrap(); + // Creation's stopped observation entry is weak; querying it evicts it. + assert!( + fixture + .backend + .factory + .observation_hub + .get(&fixture.worker.worker_ref) + .is_none() + ); + fixture.assert_released(); + fixture.fresh_restore_and_old_replay(request, result); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_real_artifact_cleanup_failure_retains_same_owner_and_cause() { + let fixture = ProfileFailureFixture::new(Duration::from_secs(2)); + fixture.fail(RuntimeRestoreStage::Binding); + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .factory_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .factory_entered + .lock() + .unwrap() = Some(entered_tx); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + entered.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + let run = fixture + .backend + .factory + .worker_restore_run_dir(&fixture.worker.worker_ref, pending.request.operation_id) + .unwrap(); + fs::write(&run, "invalid disposable run artifact").unwrap(); + release.notify_one(); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{result:?}" + ); + let cause = pending + .result + .lock() + .unwrap() + .as_ref() + .unwrap() + .as_ref() + .err() + .unwrap() + .clone(); + assert_eq!(cause.stage, RuntimeRestoreStage::Binding); + assert_eq!( + cause.cleanup_failure, + Some(RuntimeRestoreCleanupFailure::Artifacts) + ); + assert!( + fixture + .backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .contains_key(&fixture.worker.worker_ref) + ); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + fs::remove_file(&run).unwrap(); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{result:?}" + ); + assert!(result.message.as_ref().unwrap().contains("Artifacts")); + fixture.assert_released(); + eprintln!( + "restore-fault cleanup=real-artifact-failure late-factory=true owner=retained original-cause=preserved" + ); + fixture.fresh_restore_and_old_replay(request, result); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_stop_race_joins_late_factory_failure_without_second_controller() { + let fixture = ProfileFailureFixture::new(Duration::from_secs(2)); + fixture.fail(RuntimeRestoreStage::Binding); + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .factory_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .factory_entered + .lock() + .unwrap() = Some(entered_tx); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request) + .unwrap(); + entered.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + assert!(pending.result.lock().unwrap().is_none()); + let runtime = fixture.runtime.clone(); + let worker_ref = fixture.worker.worker_ref.clone(); + let (started_tx, started) = mpsc::channel(); + let stopping = std::thread::spawn(move || { + started_tx.send(()).unwrap(); + runtime.stop_worker(&worker_ref, None) + }); + started.recv_timeout(Duration::from_secs(5)).unwrap(); + // The factory still owns the allocation and cannot attest stopped yet. + assert!( + !pending + .task + .try_lock() + .map(|task| task.as_ref().is_some_and(|task| task.is_finished())) + .unwrap_or(false) + ); + release.notify_one(); + stopping.join().unwrap().unwrap(); + fixture.assert_released(); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + let fresh = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, fresh) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::Accepted, + "{result:?}" + ); + assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); + fixture + .runtime + .stop_worker(&fixture.worker.worker_ref, None) + .unwrap(); + eprintln!("restore-fault race=Stop late-factory=joined controllers-at-most=1"); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_failure_settlement_write_retry_preserves_completed_cause_without_execution() { + for fault in [ + crate::fs_store::AtomicWriteFault::BeforeRename, + crate::fs_store::AtomicWriteFault::AfterRename, + ] { + let fixture = ProfileFailureFixture::new(Duration::from_secs(2)); + fixture.fail(RuntimeRestoreStage::Binding); + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .factory_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .factory_entered + .lock() + .unwrap() = Some(entered_tx); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + entered.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let operation_id = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap() + .request + .operation_id; + fixture + .runtime + .test_runtime_store() + .fail_next_worker_write(fault); + release.notify_one(); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{result:?}" + ); + assert_eq!( + result.reason_code.as_deref(), + Some("worker_restore_rollback_commit_failed") + ); + assert!( + fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .is_none() + ); + assert!( + fixture + .backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty() + ); + let saved = fixture + .backend + .completed_restore_failure(operation_id, &fixture.worker.worker_ref) + .unwrap(); + assert!( + matches!(saved, WorkerExecutionSpawnResult::RolledBack(ref failure) + if failure.message.as_ref().unwrap().contains("injected restore boundary failure")) + ); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{result:?}" + ); + assert!( + result + .message + .as_ref() + .unwrap() + .contains("injected restore boundary failure") + ); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + fixture.assert_released(); + eprintln!( + "restore-fault settlement={fault:?} completed-cause=retained replay-executions=1" + ); + fixture.fresh_restore_and_old_replay(request, result); + } +} + +struct RacedBindingFactory { + changed_checkout: PathBuf, + preflights: AtomicUsize, +} + +#[async_trait] +impl RuntimeWorkerFactory for RacedBindingFactory { + async fn spawn_controller( + &self, + _: WorkerExecutionSpawnRequest, + ) -> Result { + panic!("binding-only failure must not create a controller") + } + async fn preflight_restore(&self, _: &WorkerExecutionRestoreRequest) -> Result<(), String> { + // Inject another owner's checkout change AFTER the real materializer's + // read-only checks, but before its mutable bind. Recovery stays real. + if self.preflights.fetch_add(1, Ordering::SeqCst) == 0 { + fs::remove_dir_all(&self.changed_checkout).unwrap(); + } + Ok(()) + } + async fn restore_controller( + &self, + _: WorkerExecutionRestoreRequest, + ) -> Result { + panic!("factory execution must not run after partial binding failed") + } +} + +#[test] +fn partial_restore_binding_failure_keeps_exact_operation_without_factory_cleanup_claim() { + let root = tempfile::tempdir().unwrap(); + let repo = create_clean_repo(); + let materializer = RuntimeGitMaterializer::new(root.path()); + let first = materializer + .create(&working_directory_request(repo.path())) + .unwrap(); + let second = materializer + .create(&working_directory_request(repo.path())) + .unwrap(); + let mut request = direct_restore_request(WorkerRef::new(WorkerId::now_v7())); + request.previous_workdir_attachments = vec![ + WorkingDirectoryAttachmentStatus { + alias: WorkdirAttachmentAlias::new("first").unwrap(), + working_directory: first.status(), + }, + WorkingDirectoryAttachmentStatus { + alias: WorkdirAttachmentAlias::new("second").unwrap(), + working_directory: second.status(), + }, + ]; + let backend = WorkerRuntimeExecutionBackend::new(RacedBindingFactory { + changed_checkout: second.root().to_path_buf(), + preflights: AtomicUsize::new(0), + }) + .unwrap() + .with_working_directory_materializer(materializer); + assert!(matches!( + backend.restore_worker(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + let pending = backend + .pending_restore(&request.worker_ref) + .unwrap() + .unwrap(); + assert_eq!(pending.request.operation_id, request.operation_id); + assert_eq!(pending.request.workdir_attachments.len(), 1); + assert_eq!( + pending + .request + .workdir_attachments + .values() + .next() + .unwrap() + .root(), + first.root() + ); + for _ in 0..2 { + assert!(matches!( + backend.reconcile_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + } + assert_eq!(backend.factory.preflights.load(Ordering::SeqCst), 1); + assert!(!backend.stop_worker(&request.worker_ref).is_accepted()); + assert!(Arc::ptr_eq( + &pending, + &backend + .pending_restore(&request.worker_ref) + .unwrap() + .unwrap() + )); + assert!(backend.workers.lock().unwrap().is_empty()); + eprintln!( + "restore-fault stage=mutable-binding outcome=ReconciliationRequired partial-bindings=retained factory-executions=0" + ); + // Fixture teardown drops the retained passive local binding. This is not + // evidence that credential/policy materialization has been rolled back. +} diff --git a/crates/worker/src/bootstrap.rs b/crates/worker/src/bootstrap.rs index d027aa8c9..ba8afef39 100644 --- a/crates/worker/src/bootstrap.rs +++ b/crates/worker/src/bootstrap.rs @@ -271,6 +271,7 @@ where St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { let cleanup_sessions = worker.workdir_sessions(); + let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some(); let controller = match layout { WorkerBootstrapLayout::Direct { runtime_base, @@ -305,7 +306,10 @@ where controller_task, }), Err(source) => { - let cleanup_failed = cleanup_sessions.close_all().await.is_err(); + // Factory-owned task scopes must settle before sessions close. + // Cleanup remains unconfirmed here; preserve the startup source. + let cleanup_failed = + factory_owned_cleanup || cleanup_sessions.close_all().await.is_err(); Err(WorkerBootstrapError::Controller { source, cleanup_failed, diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index d8be6da1a..8c4640bf9 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -854,6 +854,7 @@ impl WorkerController { St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { let sessions = worker.workdir_sessions(); + let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some(); let result = Self::spawn_initialized( worker, runtime_base, @@ -863,7 +864,10 @@ impl WorkerController { transport, ) .await; + // An opted-in factory must quiesce its captured Feature tasks before + // closing their Workdir sessions; this wrapper cannot confirm that order. if result.is_err() + && !factory_owned_cleanup && let Err(error) = sessions.close_all().await { tracing::warn!(%error, "Workdir sessions close after controller startup failure failed"); @@ -1092,6 +1096,7 @@ impl WorkerController { let pause_tx = worker.engine_mut().pause_sender(); let notify_buffer = worker.notify_buffer_handle(); + worker.finish_startup_feature_cleanup_capture(); let controller_task = tokio::spawn(controller_loop( worker, method_rx, @@ -4424,21 +4429,6 @@ mod tests { assert!(feature_install < materialize_gate); assert!(materialize_gate < materialize); assert!(materialize < exposure); - let observer = startup.find("let command_observer = fs_for_view").unwrap(); - let recover = startup.find(".recover_unfinished_compaction()").unwrap(); - let files = startup - .find("runtime_dir.write_status(&shared_state).await?") - .unwrap(); - let loop_start = startup.find("tokio::spawn(controller_loop(").unwrap(); - assert!(recover < observer && files < observer && observer < loop_start); - let socket_startup = &startup[observer..loop_start]; - let abort = socket_startup.find("observer.abort()").unwrap(); - let join = socket_startup.find("observer.await").unwrap(); - let failure = socket_startup.find("return Err(error)").unwrap(); - assert!( - abort < join && join < failure, - "startup cannot detach its observer on error" - ); assert!( startup[materialize_gate..materialize] .contains("needs_initial_session_head_materialization") diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index 10cf5a6c5..54145dbf9 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -69,9 +69,10 @@ pub use session_history::{ pub use shared_state::WorkerSharedState; pub use worker::{ LocalWorkingDirectory, SubjektivSessionAttributionLifecycle, Worker, WorkerError, - WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, WorkspaceBinaryRequest, - WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError, - WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse, - WorkspaceServerOperation, WorkspaceWorkerDiscoveryRequest, apply_worker_manifest, - marker_workspace_client, unavailable_workspace_client, + WorkerFilesystemAuthority, WorkerRunResult, WorkerStartupFeatureCleanup, + WorkerWorkspaceContext, WorkspaceBinaryRequest, WorkspaceBinaryResponse, WorkspaceClient, + WorkspaceClientError, WorkspaceId, WorkspaceIdError, WorkspacePromptCatalogResolution, + WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse, WorkspaceServerOperation, + WorkspaceWorkerDiscoveryRequest, apply_worker_manifest, marker_workspace_client, + unavailable_workspace_client, }; diff --git a/crates/worker/src/runtime/worker_allocation.rs b/crates/worker/src/runtime/worker_allocation.rs index c7445cdaa..ab6bc3455 100644 --- a/crates/worker/src/runtime/worker_allocation.rs +++ b/crates/worker/src/runtime/worker_allocation.rs @@ -21,8 +21,9 @@ pub use error::ScopeLockError; #[cfg(test)] pub(crate) use lifecycle::install_top_level_at_for_test; pub use lifecycle::{ - ScopeAllocationGuard, SegmentActivationCommitError, SegmentActivationGuard, SegmentLockInfo, - adopt_allocation, install_top_level, install_top_level_with_deny, lookup_segment, + ScopeAllocationCleanup, ScopeAllocationGuard, SegmentActivationCommitError, + SegmentActivationGuard, SegmentLockInfo, adopt_allocation, install_top_level, + install_top_level_with_deny, lookup_segment, }; pub use mutate::{ delegate_scope, reclaim_delegated_scope, reclaim_stale, reclaim_stale_with, register_worker, diff --git a/crates/worker/src/runtime/worker_allocation/error.rs b/crates/worker/src/runtime/worker_allocation/error.rs index 601202693..334e0125f 100644 --- a/crates/worker/src/runtime/worker_allocation/error.rs +++ b/crates/worker/src/runtime/worker_allocation/error.rs @@ -11,6 +11,14 @@ use session_store::SegmentId; pub enum ScopeLockError { #[error("I/O error on workers.json: {0}")] Io(#[from] io::Error), + #[error("worker `{worker_name}` allocation still has a live guard")] + AllocationGuardAttached { worker_name: String }, + #[error("allocation acquisition was published but its save failed: {source}")] + AcquisitionSaveFailed { + #[source] + source: io::Error, + cleanup: super::lifecycle::ScopeAllocationCleanup, + }, #[error("worker `{0}` is already allocated")] DuplicateWorkerName(String), #[error("requested scope `{}` conflicts with worker allocation `{competitor}` rule `{}`", .rule.target.display(), .competitor_rule.target.display())] @@ -46,3 +54,15 @@ pub enum ScopeLockError { actual: Option, }, } + +impl ScopeLockError { + /// Retain the release authority for an owned acquisition whose table rename + /// succeeded before save returned an error. Refusals and unpublished saves + /// do not confer ownership of any preexisting allocation. + pub fn allocation_cleanup_handle(&self) -> Option { + match self { + Self::AcquisitionSaveFailed { cleanup, .. } => Some(cleanup.clone()), + _ => None, + } + } +} diff --git a/crates/worker/src/runtime/worker_allocation/lifecycle.rs b/crates/worker/src/runtime/worker_allocation/lifecycle.rs index cab09ec18..ba6505008 100644 --- a/crates/worker/src/runtime/worker_allocation/lifecycle.rs +++ b/crates/worker/src/runtime/worker_allocation/lifecycle.rs @@ -3,6 +3,7 @@ //! up on drop. use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; use manifest::ScopeRule; use session_store::SegmentId; @@ -11,13 +12,82 @@ use super::error::ScopeLockError; use super::mutate::release_worker; use super::table::{LockFileGuard, default_allocation_path}; -/// Owned allocation: on drop, opens the lock file and releases this -/// Worker's entry. The guard keeps only the name + lock-file path; it -/// does not hold the `flock` for the Worker's lifetime. +/// Owned allocation shared with any retained cleanup handles. Only the final +/// owner's drop attempts best-effort release; no `flock` is held for its lifetime. #[derive(Debug)] pub struct ScopeAllocationGuard { + owner: Arc, +} + +/// Retains an allocation's release authority even after its Worker is consumed +/// or dropped. A successful controller transfer should drop this handle without +/// explicitly releasing the live Worker's allocation. +#[derive(Debug, Clone)] +pub struct ScopeAllocationCleanup { + owner: Arc, +} + +#[derive(Debug)] +struct ScopeAllocationOwner { worker_name: String, lock_path: PathBuf, + state: Mutex, +} + +#[derive(Debug)] +struct ScopeAllocationOwnerState { + guard_attached: bool, + released: bool, +} + +impl ScopeAllocationOwner { + fn release(&self) -> Result<(), ScopeLockError> { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + if state.guard_attached { + return Err(ScopeLockError::AllocationGuardAttached { + worker_name: self.worker_name.clone(), + }); + } + if state.released { + return Ok(()); + } + let mut guard = LockFileGuard::open(&self.lock_path)?; + if guard.data().find(&self.worker_name).is_some() { + release_worker(&mut guard, &self.worker_name)?; + } else { + // A previous save may have removed the entry but failed after rename. + // Complete the durable save before reporting release as confirmed. + guard.save()?; + } + state.released = true; + Ok(()) + } +} + +impl Drop for ScopeAllocationOwner { + fn drop(&mut self) { + let _ = self.release(); + } +} + +impl Drop for ScopeAllocationGuard { + fn drop(&mut self) { + self.owner + .state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .guard_attached = false; + } +} + +impl ScopeAllocationCleanup { + /// Release only after the factory has returned an error and no live + /// Controller can still use this allocation. An attached guard rejects + /// release. Success is idempotent across all clones; failure retains the + /// authority for an explicit retry. + pub fn release(&self) -> Result<(), ScopeLockError> { + self.owner.release() + } } /// Exclusive authority to activate a replacement Segment for one allocated Worker. @@ -56,12 +126,58 @@ impl SegmentActivationCommitError { } impl ScopeAllocationGuard { + fn new(worker_name: String, lock_path: PathBuf) -> Self { + Self { + owner: Arc::new(ScopeAllocationOwner { + worker_name, + lock_path, + state: Mutex::new(ScopeAllocationOwnerState { + guard_attached: true, + released: false, + }), + }), + } + } + + fn commit_acquisition(self, table: &mut LockFileGuard) -> Result { + let mut published = false; + match table.save_with_publication(&mut published) { + Ok(()) => Ok(self), + Err(source) => { + let cleanup = if published { + Some(self.cleanup_handle()) + } else { + // A staged entry is not ours until rename. Disarm before + // detaching so Drop cannot remove a preexisting allocation + // (or reopen the table lock still held by this caller). + self.owner + .state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .released = true; + None + }; + drop(self); + match cleanup { + Some(cleanup) => Err(ScopeLockError::AcquisitionSaveFailed { source, cleanup }), + None => Err(ScopeLockError::Io(source)), + } + } + } + } + pub fn worker_name(&self) -> &str { - &self.worker_name + &self.owner.worker_name } pub fn lock_path(&self) -> &Path { - &self.lock_path + &self.owner.lock_path + } + + pub fn cleanup_handle(&self) -> ScopeAllocationCleanup { + ScopeAllocationCleanup { + owner: Arc::clone(&self.owner), + } } /// Lock the machine-wide table and validate a Segment replacement before @@ -71,24 +187,25 @@ impl ScopeAllocationGuard { expected: SegmentId, replacement: SegmentId, ) -> Result { - let guard = LockFileGuard::open(&self.lock_path)?; + let guard = LockFileGuard::open(self.lock_path())?; let actual = guard .data() .allocations .iter() - .find(|allocation| allocation.worker_name == self.worker_name) + .find(|allocation| allocation.worker_name == self.worker_name()) .and_then(|allocation| allocation.segment_id); if actual != Some(expected) { return Err(ScopeLockError::SegmentChanged { - worker_name: self.worker_name.clone(), + worker_name: self.worker_name().to_owned(), expected, actual, }); } if let Some(existing) = guard.data().allocations.iter().find(|allocation| { - allocation.segment_id == Some(replacement) && allocation.worker_name != self.worker_name + allocation.segment_id == Some(replacement) + && allocation.worker_name != self.worker_name() }) { return Err(ScopeLockError::SegmentConflict { segment_id: replacement, @@ -99,7 +216,7 @@ impl ScopeAllocationGuard { Ok(SegmentActivationGuard { guard, - worker_name: self.worker_name.clone(), + worker_name: self.worker_name().to_owned(), expected, replacement, }) @@ -145,14 +262,6 @@ impl SegmentActivationGuard { } } -impl Drop for ScopeAllocationGuard { - fn drop(&mut self) { - if let Ok(mut guard) = LockFileGuard::open(&self.lock_path) { - let _ = release_worker(&mut guard, &self.worker_name); - } - } -} - /// Open the default lock file, register a top-level Worker, and return a /// guard that will release the allocation on drop. pub fn install_top_level( @@ -185,7 +294,7 @@ pub fn install_top_level_with_deny( ) -> Result { let lock_path = default_allocation_path()?; let mut guard = LockFileGuard::open(&lock_path)?; - super::mutate::register_worker_with_deny( + super::mutate::prepare_worker_with_deny( &mut guard, worker_name.clone(), pid, @@ -194,10 +303,7 @@ pub fn install_top_level_with_deny( scope_deny, segment_id, )?; - Ok(ScopeAllocationGuard { - worker_name, - lock_path, - }) + ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard) } #[cfg(test)] @@ -210,7 +316,7 @@ pub(crate) fn install_top_level_at_for_test( segment_id: SegmentId, ) -> Result { let mut guard = LockFileGuard::open(&lock_path)?; - super::mutate::register_worker_with_deny( + super::mutate::prepare_worker_with_deny( &mut guard, worker_name.clone(), pid, @@ -219,10 +325,7 @@ pub(crate) fn install_top_level_at_for_test( Vec::new(), segment_id, )?; - Ok(ScopeAllocationGuard { - worker_name, - lock_path, - }) + ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard) } /// Take ownership of an existing allocation that was pre-registered by @@ -240,6 +343,15 @@ pub fn adopt_allocation( segment_id: SegmentId, ) -> Result { let lock_path = default_allocation_path()?; + adopt_allocation_at(lock_path, worker_name, new_pid, segment_id) +} + +fn adopt_allocation_at( + lock_path: PathBuf, + worker_name: String, + new_pid: u32, + segment_id: SegmentId, +) -> Result { let mut guard = LockFileGuard::open(&lock_path)?; let alloc = guard .data_mut() @@ -247,11 +359,7 @@ pub fn adopt_allocation( .ok_or_else(|| ScopeLockError::UnknownWorker(worker_name.clone()))?; alloc.pid = new_pid; alloc.segment_id = Some(segment_id); - guard.save()?; - Ok(ScopeAllocationGuard { - worker_name, - lock_path, - }) + ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard) } /// Information about a Worker that currently holds an allocation for a @@ -329,6 +437,388 @@ mod tests { } } + #[test] + fn retained_cleanup_handle_defers_release_until_final_owner_drop() { + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let guard = install_top_level_at_for_test( + lock_path.clone(), + "retained".into(), + std::process::id(), + sock("retained"), + Vec::new(), + sid(), + ) + .unwrap(); + let cleanup = guard.cleanup_handle(); + let retained = cleanup.clone(); + + drop(guard); + drop(cleanup); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("retained").is_some()); + } + drop(retained); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("retained").is_none()); + } + + #[test] + fn dropping_cleanup_handle_after_transfer_preserves_live_guard() { + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let segment = sid(); + let guard = install_top_level_at_for_test( + lock_path.clone(), + "transferred".into(), + std::process::id(), + sock("transferred"), + Vec::new(), + segment, + ) + .unwrap(); + + drop(guard.cleanup_handle()); + // A successful controller transfer leaves the guard usable, including + // the existing activation protocol, without retaining a cleanup handle. + let replacement = sid(); + guard + .begin_segment_activation(segment, replacement) + .unwrap() + .commit() + .unwrap(); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert_eq!( + table.data().find("transferred").unwrap().segment_id, + Some(replacement) + ); + } + drop(guard); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("transferred").is_none()); + } + + #[test] + fn cleanup_rejects_live_guard_then_releases_only_detached_allocation() { + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let guard = install_top_level_at_for_test( + lock_path.clone(), + "live".into(), + std::process::id(), + sock("live"), + Vec::new(), + sid(), + ) + .unwrap(); + let _competitor = install_top_level_at_for_test( + lock_path.clone(), + "competitor".into(), + std::process::id(), + sock("competitor"), + Vec::new(), + sid(), + ) + .unwrap(); + let cleanup = guard.cleanup_handle(); + assert!(matches!( + cleanup.release(), + Err(ScopeLockError::AllocationGuardAttached { worker_name }) if worker_name == "live" + )); + drop(cleanup.clone()); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("live").is_some()); + assert!(table.data().find("competitor").is_some()); + } + + drop(guard); + cleanup.release().unwrap(); + cleanup.clone().release().unwrap(); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("live").is_none()); + assert!(table.data().find("competitor").is_some()); + } + + #[test] + fn published_install_failure_retains_cleanup_across_error_drop_and_retry() { + use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test}; + + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let _competitor = install_top_level_at_for_test( + lock_path.clone(), + "competitor".into(), + std::process::id(), + sock("competitor"), + Vec::new(), + sid(), + ) + .unwrap(); + let segment = sid(); + fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename); + let error = install_top_level_at_for_test( + lock_path.clone(), + "failed".into(), + std::process::id(), + sock("failed"), + Vec::new(), + segment, + ) + .unwrap_err(); + assert!(matches!( + &error, + ScopeLockError::AcquisitionSaveFailed { .. } + )); + let cleanup = error.allocation_cleanup_handle().unwrap(); + let retry = error.allocation_cleanup_handle().unwrap(); + drop(error); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert_eq!( + table.data().find("failed").unwrap().segment_id, + Some(segment) + ); + assert!(table.data().find("competitor").is_some()); + } + + fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename); + assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_)))); + drop(cleanup); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("failed").is_some()); + } + retry.release().unwrap(); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("failed").is_none()); + assert!(table.data().find("competitor").is_some()); + } + + #[test] + fn refused_install_has_no_cleanup_authority_over_existing_allocation() { + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let segment = sid(); + let _competitor = install_top_level_at_for_test( + lock_path.clone(), + "competitor".into(), + std::process::id(), + sock("competitor"), + vec![write_rule("/shared", true)], + segment, + ) + .unwrap(); + let cases = [ + ("competitor", sid(), Vec::new()), + ("other", segment, Vec::new()), + ("other", sid(), vec![write_rule("/shared", true)]), + ]; + for (name, requested_segment, scope) in cases { + let error = install_top_level_at_for_test( + lock_path.clone(), + name.into(), + std::process::id(), + sock("refused"), + scope, + requested_segment, + ) + .unwrap_err(); + assert!(matches!( + &error, + ScopeLockError::DuplicateWorkerName(_) + | ScopeLockError::SegmentConflict { .. } + | ScopeLockError::WriteConflict { .. } + )); + assert!(error.allocation_cleanup_handle().is_none()); + drop(error); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert_eq!(table.data().allocations.len(), 1); + assert_eq!( + table.data().find("competitor").unwrap().socket, + sock("competitor") + ); + assert_eq!( + table.data().find("competitor").unwrap().segment_id, + Some(segment) + ); + } + } + + #[test] + fn unpublished_install_failure_has_no_cleanup_authority() { + use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test}; + + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let _competitor = install_top_level_at_for_test( + lock_path.clone(), + "competitor".into(), + std::process::id(), + sock("competitor"), + Vec::new(), + sid(), + ) + .unwrap(); + fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename); + let error = install_top_level_at_for_test( + lock_path.clone(), + "failed".into(), + std::process::id(), + sock("failed"), + Vec::new(), + sid(), + ) + .unwrap_err(); + assert!(matches!(&error, ScopeLockError::Io(_))); + assert!(error.allocation_cleanup_handle().is_none()); + drop(error); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("failed").is_none()); + assert!(table.data().find("competitor").is_some()); + } + + #[test] + fn published_adoption_failure_retains_cleanup_for_retry() { + use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test}; + + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + { + let mut table = LockFileGuard::open(&lock_path).unwrap(); + delegate_placeholder(&mut table, "child", std::process::id()); + delegate_placeholder(&mut table, "competitor", std::process::id()); + } + let segment = sid(); + fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename); + let error = adopt_allocation_at( + lock_path.clone(), + "child".into(), + std::process::id(), + segment, + ) + .unwrap_err(); + let cleanup = error.allocation_cleanup_handle().unwrap(); + drop(error); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert_eq!( + table.data().find("child").unwrap().segment_id, + Some(segment) + ); + } + fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename); + assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_)))); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("child").is_none()); + assert!(table.data().find("competitor").is_some()); + } + cleanup.release().unwrap(); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("child").is_none()); + assert!(table.data().find("competitor").is_some()); + } + + #[test] + fn unpublished_adoption_failure_preserves_placeholder_without_cleanup_authority() { + use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test}; + + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let placeholder_pid = std::process::id(); + { + let mut table = LockFileGuard::open(&lock_path).unwrap(); + delegate_placeholder(&mut table, "child", placeholder_pid); + } + fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename); + let error = adopt_allocation_at( + lock_path.clone(), + "child".into(), + placeholder_pid.wrapping_add(1), + sid(), + ) + .unwrap_err(); + assert!(matches!(&error, ScopeLockError::Io(_))); + assert!(error.allocation_cleanup_handle().is_none()); + drop(error); + let table = LockFileGuard::open(&lock_path).unwrap(); + let allocation = table.data().find("child").unwrap(); + assert_eq!(allocation.pid, placeholder_pid); + assert_eq!(allocation.segment_id, None); + } + + #[test] + fn failed_cleanup_release_retains_authority_for_retry() { + use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test}; + + for point in [SaveFaultPoint::BeforeRename, SaveFaultPoint::AfterRename] { + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let guard = install_top_level_at_for_test( + lock_path.clone(), + "failed".into(), + std::process::id(), + sock("failed"), + vec![read_rule("/shared", true)], + sid(), + ) + .unwrap(); + let _other = install_top_level_at_for_test( + lock_path.clone(), + "other".into(), + std::process::id(), + sock("other"), + vec![read_rule("/shared", true)], + sid(), + ) + .unwrap(); + let cleanup = guard.cleanup_handle(); + let retry = cleanup.clone(); + drop(guard); + fail_next_save_at_for_test(&lock_path, point); + + assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_)))); + drop(cleanup); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert_eq!( + table.data().find("failed").is_some(), + point == SaveFaultPoint::BeforeRename + ); + assert!(table.data().find("other").is_some()); + } + retry.release().unwrap(); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("failed").is_none()); + assert!(table.data().find("other").is_some()); + } + + // Success disarms every clone and final Drop, even if the name is + // subsequently allocated to a different Worker. + let _replacement = install_top_level_at_for_test( + lock_path.clone(), + "failed".into(), + std::process::id(), + sock("replacement"), + vec![read_rule("/shared", true)], + sid(), + ) + .unwrap(); + retry.clone().release().unwrap(); + drop(retry); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert_eq!( + table.data().find("failed").unwrap().socket, + sock("replacement") + ); + assert!(table.data().find("other").is_some()); + } + } + #[test] fn adopt_allocation_rewrites_pid_and_releases_on_drop() { let dir = TempDir::new().unwrap(); @@ -358,6 +848,7 @@ mod tests { let dir = TempDir::new().unwrap(); let _sandbox = RuntimeDirSandbox::new(dir.path()); let err = adopt_allocation("ghost".into(), 42, sid()).unwrap_err(); + assert!(err.allocation_cleanup_handle().is_none()); assert!(matches!(err, ScopeLockError::UnknownWorker(ref n) if n == "ghost")); } diff --git a/crates/worker/src/runtime/worker_allocation/mutate.rs b/crates/worker/src/runtime/worker_allocation/mutate.rs index 846adca98..9ba22f057 100644 --- a/crates/worker/src/runtime/worker_allocation/mutate.rs +++ b/crates/worker/src/runtime/worker_allocation/mutate.rs @@ -57,6 +57,30 @@ pub fn register_worker_with_deny( scope_allow: Vec, scope_deny: Vec, segment_id: SegmentId, +) -> Result<(), ScopeLockError> { + prepare_worker_with_deny( + guard, + worker_name, + pid, + socket, + scope_allow, + scope_deny, + segment_id, + )?; + guard.save()?; + Ok(()) +} + +/// Admit and stage a registration without publishing it. Owned installation +/// constructs its cleanup authority after admission and before committing. +pub(super) fn prepare_worker_with_deny( + guard: &mut LockFileGuard, + worker_name: String, + pid: u32, + socket: PathBuf, + scope_allow: Vec, + scope_deny: Vec, + segment_id: SegmentId, ) -> Result<(), ScopeLockError> { reclaim_stale(guard); if guard.data().find(&worker_name).is_some() { @@ -101,7 +125,6 @@ pub fn register_worker_with_deny( delegated_from: None, segment_id: Some(segment_id), }); - guard.save()?; Ok(()) } diff --git a/crates/worker/src/runtime/worker_allocation/table.rs b/crates/worker/src/runtime/worker_allocation/table.rs index b9d0a16ae..a8420551d 100644 --- a/crates/worker/src/runtime/worker_allocation/table.rs +++ b/crates/worker/src/runtime/worker_allocation/table.rs @@ -245,6 +245,14 @@ impl LockFileGuard { /// Persist with atomic replacement while the separate allocation lock stays /// held. A crash exposes either the old complete table or the new one. pub fn save(&mut self) -> io::Result<()> { + let mut published = false; + self.save_with_publication(&mut published) + } + + /// Report this save's rename while the caller still holds the table lock. + /// In-memory mutation alone is not evidence that an acquisition took place. + pub(super) fn save_with_publication(&mut self, published: &mut bool) -> io::Result<()> { + *published = false; let json = serde_json::to_vec_pretty(&self.data).map_err(io::Error::other)?; let mut temp_os = self.data_path.as_os_str().to_owned(); temp_os.push(".tmp"); @@ -264,6 +272,7 @@ impl LockFileGuard { )); } fs::rename(&temp_path, &self.data_path)?; + *published = true; #[cfg(test)] if take_save_fault(&self.data_path, SaveFaultPoint::AfterRename) { return Err(io::Error::other( diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 6af8d337b..92a7e7047 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -533,7 +533,7 @@ use crate::prompt::source::PromptCatalogSource; use crate::prompt::system::{SystemPromptContext, SystemPromptError, SystemPromptTemplate}; use crate::runtime::dir; use crate::runtime::worker_allocation::{ - self, ScopeAllocationGuard, ScopeLockError, SegmentActivationGuard, + self, ScopeAllocationCleanup, ScopeAllocationGuard, ScopeLockError, SegmentActivationGuard, }; use crate::skill::{SkillActivationResponse, SkillClientError}; #[cfg(test)] @@ -2951,6 +2951,69 @@ struct PendingCompactionCleanup { compaction_id: String, } +/// Retains startup task scopes after a consuming controller startup fails. +/// +/// Call `shutdown` only after startup has failed with no live Controller. Keep +/// this handle on errors or timeout and retry; it owns any in-flight settlement +/// future, including join handles drained from the registries. On successful +/// controller transfer, drop the factory's handle without shutting it down. +#[derive(Clone)] +pub struct WorkerStartupFeatureCleanup { + inner: Arc, +} + +struct StartupFeatureCleanupInner { + registries: Mutex>, + settlement: tokio::sync::Mutex< + Option>>, + >, +} + +impl WorkerStartupFeatureCleanup { + fn new(registry: FeatureBackgroundTaskRegistry) -> Self { + Self { + inner: Arc::new(StartupFeatureCleanupInner { + registries: Mutex::new(vec![registry]), + settlement: tokio::sync::Mutex::new(None), + }), + } + } + + fn retain(&self, registry: FeatureBackgroundTaskRegistry) { + self.inner + .registries + .lock() + .expect("startup Feature cleanup poisoned") + .push(registry); + } + + /// Stops and joins every captured registry. Cancelling this call preserves + /// its settlement future for the next call, rather than dropping task joins. + pub async fn shutdown(&self) -> Result<(), crate::hook::HookError> { + let mut settlement = self.inner.settlement.lock().await; + let pending = settlement.get_or_insert_with(|| { + let registries = self + .inner + .registries + .lock() + .expect("startup Feature cleanup poisoned") + .clone(); + Box::pin(async move { + let mut failure = None; + for registry in registries { + if let Err(error) = registry.shutdown().await { + failure.get_or_insert(error); + } + } + failure.map_or(Ok(()), Err) + }) + }); + let result = pending.as_mut().await; + *settlement = None; + result + } +} + /// An independent agent execution unit. /// /// Holds a [`Engine`] directly and persists session state via @@ -2997,6 +3060,8 @@ pub struct Worker { hook_registry: Option>, /// Executable background tasks registered by successfully installed features. feature_background_tasks: FeatureBackgroundTaskRegistry, + /// Present only while an opted-in factory is capturing startup task scopes. + startup_feature_cleanup: Option, /// Enabled chat invocation metadata and executable handlers. feature_invocations: FeatureInvocationRegistry, /// Internal Workers install an explicit Feature composition and disable @@ -3087,9 +3152,8 @@ pub struct Worker { /// Workers built via `from_manifest` / `from_manifest_spawned` / /// `restore_from_manifest` (production paths); `None` for the /// low-level `Worker::new` constructor used in tests, which bypasses - /// the registry. Kept purely for its `Drop` impl, which releases - /// the allocation when the Worker is dropped. - #[allow(dead_code)] + /// the registry. Cleanup handles can retain the allocation beyond Worker + /// drop; only the final shared owner attempts best-effort release. scope_allocation: Option, /// Socket path of the spawning Worker. `Some` only for Workers built via /// `from_manifest_spawned`. Consumed by the controller to fire @@ -3155,7 +3219,7 @@ pub struct Worker { impl Drop for Worker { fn drop(&mut self) { // A failed activation authority nests the same machine-wide lock used by - // `ScopeAllocationGuard::drop`. Release the nested guard first so Worker + // allocation owner's final Drop. Release the nested guard first so Worker // teardown cannot self-deadlock while removing its outer allocation. let authority = match self.failed_activation_authority.get_mut() { Ok(authority) => authority.take(), @@ -3165,6 +3229,47 @@ impl Drop for Worker { } } +impl Worker { + /// Opt in before a consuming startup call. The handle captures the current + /// task registry and each subsequently installed registry until successful + /// controller transfer ends capture. Repeated calls share the same capture. + pub fn retain_startup_feature_cleanup(&mut self) -> WorkerStartupFeatureCleanup { + self.startup_feature_cleanup + .get_or_insert_with(|| { + WorkerStartupFeatureCleanup::new(self.feature_background_tasks.clone()) + }) + .clone() + } + + /// Observes explicit factory cleanup ownership without enabling capture. + pub fn startup_feature_cleanup_handle(&self) -> Option { + self.startup_feature_cleanup.clone() + } + + /// End factory capture immediately before transferring to a live Controller. + /// The factory must also drop its handle after startup returns success. + pub fn finish_startup_feature_cleanup_capture(&mut self) { + self.startup_feature_cleanup = None; + } + + /// Retain cleanup authority before passing the reconstructed Worker into a + /// consuming factory. Explicit release is only valid after factory failure + /// with no live Controller; drop the handle on successful controller transfer. + pub fn restore_allocation_cleanup_handle(&self) -> Option { + self.scope_allocation + .as_ref() + .map(ScopeAllocationGuard::cleanup_handle) + } + + fn construction_failure(&self, error: WorkerError) -> WorkerError { + WorkerError::ConstructionCleanup { + source: Box::new(error), + allocation: self.restore_allocation_cleanup_handle(), + sessions: self.workdir_sessions.clone(), + } + } +} + impl Worker { pub async fn stop_feature_runtime(&mut self, reason: impl Into) { if let Some(hooks) = self.hook_registry.clone() { @@ -3352,6 +3457,7 @@ impl Worker { hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + startup_feature_cleanup: None, feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: true, interceptor_installed: false, @@ -3850,6 +3956,9 @@ impl Worker { return report; } self.feature_background_tasks = report.background_tasks.clone(); + if let Some(cleanup) = &self.startup_feature_cleanup { + cleanup.retain(self.feature_background_tasks.clone()); + } self.feature_invocations = report.chat_invocations.clone(); for instruction in report.installed_instruction_contributions() { self.register_feature_instruction(instruction); @@ -7906,7 +8015,7 @@ where let workdir_sessions = workdir_sessions_from_authority(&common.filesystem_authority, &scope); - let mut worker = Self { + let worker = Self { backend_job: std::sync::OnceLock::new(), manifest, engine: Some(worker), @@ -7925,6 +8034,7 @@ where hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + startup_feature_cleanup: None, feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: true, interceptor_installed: false, @@ -7966,10 +8076,15 @@ where log_writer: None, notification_committer: None, }; - worker.apply_permissions_from_manifest(); - worker.apply_prune_from_manifest(); - worker.write_worker_metadata_pending()?; - Ok(worker) + worker.finish_pending_construction() + } + + fn finish_pending_construction(mut self) -> Result { + self.apply_permissions_from_manifest(); + self.apply_prune_from_manifest(); + self.write_worker_metadata_pending() + .map_err(|error| self.construction_failure(error))?; + Ok(self) } /// Complete the Host-owned Session-to-Subject attribution boundary before a @@ -8306,6 +8421,7 @@ where hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + startup_feature_cleanup: None, feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: false, interceptor_installed: false, @@ -8438,6 +8554,7 @@ where hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + startup_feature_cleanup: None, feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: true, interceptor_installed: false, @@ -8751,6 +8868,11 @@ where if state.system_prompt.is_none() { return Err(WorkerError::SegmentSystemPromptMissing { segment_id }); } + let restored_history_entries = + restore_history_entries(session_id, segment_id, &raw_entries).map_err(|error| { + WorkerError::InvalidState(format!("restore typed Worker session history: {error}")) + })?; + let flow_runtime_state = restored_flow_runtime_state(&state.extensions)?; let mirror_entries: Vec = raw_entries.clone(); let scope_config = effective_restore_scope_config(&store, &manifest)?; @@ -8797,10 +8919,6 @@ where // (the Worker's one and only write path that prepends a summary at // history[0]). Restoring the anchor lets Anthropic re-use a // stable cache prefix for long-lived restored sessions. - let restored_history_entries = - restore_history_entries(session_id, segment_id, &raw_entries).map_err(|error| { - WorkerError::InvalidState(format!("restore typed Worker session history: {error}")) - })?; let restored_history = restored_history_entries .iter() .map(|entry| entry.item.clone()) @@ -8839,14 +8957,13 @@ where filesystem_authority: common.filesystem_authority, workdir_sessions, workspace_context: common.workspace_context, - flow_runtime_state: Arc::new(Mutex::new(restored_flow_runtime_state( - &state.extensions, - )?)), + flow_runtime_state: Arc::new(Mutex::new(flow_runtime_state)), flow_feature_enabled: false, scope, hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + startup_feature_cleanup: None, feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: true, interceptor_installed: false, @@ -8896,17 +9013,24 @@ where worker .session .restore_pending_activations(&state.extensions); - worker - .pending_submission_handle() - .reconcile_uploaded_file_pins()?; - worker.apply_permissions_from_manifest(); - worker.apply_prune_from_manifest(); - worker.write_worker_metadata_active(SegmentLocation { - session_id, - segment_id, - })?; - worker.reconcile_restored_delegations().await?; - Ok(worker) + worker.finish_restored_construction().await + } + + async fn finish_restored_construction(mut self) -> Result { + self.pending_submission_handle() + .reconcile_uploaded_file_pins() + .map_err(|error| self.construction_failure(error.into()))?; + self.apply_permissions_from_manifest(); + self.apply_prune_from_manifest(); + self.write_worker_metadata_active(SegmentLocation { + session_id: self.session_id(), + segment_id: self.segment_id(), + }) + .map_err(|error| self.construction_failure(error))?; + self.reconcile_restored_delegations() + .await + .map_err(|error| self.construction_failure(error))?; + Ok(self) } async fn reconcile_restored_delegations(&mut self) -> Result<(), WorkerError> { @@ -9745,6 +9869,16 @@ fn compact_failure_category(error: &WorkerError) -> CompactFailureCategory { #[derive(Debug, thiserror::Error)] pub enum WorkerError { + /// A constructor failed after acquiring resources. Retain these handles until + /// explicit cleanup succeeds; the Worker has dropped before this is returned. + #[error("{source}")] + ConstructionCleanup { + #[source] + source: Box, + allocation: Option, + sessions: Arc, + }, + #[error("invalid durable Worker state: {0}")] InvalidState(String), @@ -12224,6 +12358,604 @@ mod build_summary_prompt_tests { assert_eq!(restored.active_run_turn_count, None); } + #[tokio::test] + async fn restore_cleanup_handle_retains_allocation_after_worker_drop() { + let dir = tempfile::tempdir().unwrap(); + let store = session_store::FsStore::new(dir.path().join("sessions")).unwrap(); + let mut worker = Worker::new( + minimal_manifest(), + Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient), + store, + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + Scope::empty(), + ) + .await + .unwrap(); + assert!(worker.restore_allocation_cleanup_handle().is_none()); + let lock_path = dir.path().join("workers.json"); + worker.scope_allocation = Some( + worker_allocation::install_top_level_at_for_test( + lock_path.clone(), + "test-worker".into(), + std::process::id(), + dir.path().join("worker.sock"), + Vec::new(), + worker.segment_id(), + ) + .unwrap(), + ); + let cleanup = worker.restore_allocation_cleanup_handle().unwrap(); + + drop(worker); + { + let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("test-worker").is_some()); + } + cleanup.release().unwrap(); + let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("test-worker").is_none()); + } + + // Constructor finalization uses real Worker, allocation, Workdir, and storage + // implementations. Only the model and metadata-write failure boundary are substituted. + mod construction_cleanup_tests { + use super::*; + + type ConstructionStore = + session_store::CombinedStore; + + async fn constructed_worker( + temp: &tempfile::TempDir, + ) -> Worker, ConstructionStore> { + let store = session_store::CombinedStore::new( + session_store::FsStore::new(temp.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(temp.path().join("workers")).unwrap(), + ); + let mut worker = Worker::new( + minimal_manifest(), + Engine::, Mutable, SessionHistoryMetadata>::new_annotated( + Box::new(NoopClient), + ), + store, + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::local( + temp.path().to_path_buf(), + temp.path().to_path_buf(), + ), + Scope::writable(temp.path()).unwrap(), + ) + .await + .unwrap(); + worker.scope_allocation = Some( + worker_allocation::install_top_level_at_for_test( + temp.path().join("workers.json"), + worker.manifest.worker.name.clone(), + std::process::id(), + temp.path().join("worker.sock"), + Vec::new(), + worker.segment_id(), + ) + .unwrap(), + ); + worker + } + + #[tokio::test] + async fn post_construction_failures_retain_resources_through_failed_release_retry() { + for stage in [ + "pending_metadata", + "restore_pins", + "restore_metadata", + "restore_delegations", + ] { + let temp = tempfile::tempdir().unwrap(); + let mut worker = constructed_worker(&temp).await; + if stage != "pending_metadata" { + worker.ensure_segment_head().await.unwrap(); + } + let name = worker.manifest.worker.name.clone(); + let lock_path = temp.path().join("workers.json"); + let sessions = Arc::downgrade(&worker.workdir_sessions()); + let session = worker.workdir_sessions().resolve(None).unwrap().session; + let writes = Arc::new(AtomicUsize::new(0)); + let writer_calls = writes.clone(); + let metadata_path = temp + .path() + .join("workers") + .join(&name) + .join("metadata.json"); + worker.worker_metadata_writer = Some(Arc::new(move |_| { + writer_calls.fetch_add(1, Ordering::SeqCst); + if stage == "restore_delegations" { + // A successful publication followed by an unreadable retained + // record forces the subsequent delegation reconciliation to fail. + std::fs::create_dir_all(&metadata_path)?; + Ok(()) + } else { + Err(WorkerStoreError::Io(std::io::Error::other( + "metadata write refused", + ))) + } + })); + if stage == "restore_pins" { + let pins_dir = temp + .path() + .join("sessions") + .join(worker.session_id().to_string()) + .join("artifacts") + .join("paste"); + std::fs::create_dir_all(pins_dir.join(".aggregate.lock")).unwrap(); + } + let result = if stage == "pending_metadata" { + worker.finish_pending_construction() + } else { + worker.finish_restored_construction().await + }; + let error = match result { + Err(error) => error, + Ok(_) => panic!("{stage} must fail after Worker construction"), + }; + let WorkerError::ConstructionCleanup { + source, + allocation, + sessions: retained, + } = &error + else { + panic!("{stage} lost construction cleanup authority: {error:?}"); + }; + if stage == "restore_pins" { + assert!(matches!( + source.as_ref(), + WorkerError::Store(StoreError::Io(_)) + )); + assert_eq!(writes.load(Ordering::SeqCst), 0); + } else { + assert!(matches!( + source.as_ref(), + WorkerError::WorkerStore(WorkerStoreError::Io(_)) + )); + assert_eq!(writes.load(Ordering::SeqCst), 1); + } + assert_eq!(error.to_string(), source.to_string()); + assert_eq!( + std::error::Error::source(&error).unwrap().to_string(), + source.to_string() + ); + assert!(Arc::ptr_eq(&sessions.upgrade().unwrap(), retained)); + let allocation = allocation.clone().expect("constructed Worker's allocation"); + let retained = retained.clone(); + drop(error); + { + let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap(); + assert!( + table.data().find(&name).is_some(), + "{stage} released during Worker Drop" + ); + } + assert!( + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root() + }) + .await + .is_ok() + ); + retained.close_all().await.unwrap(); + assert!(retained.is_empty()); + assert!( + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root() + }) + .await + .is_err() + ); + + worker_allocation::fail_next_save_for_test(&lock_path); + assert!(matches!(allocation.release(), Err(ScopeLockError::Io(_)))); + { + let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap(); + assert!( + table.data().find(&name).is_some(), + "{stage} lost failed release" + ); + } + allocation.release().unwrap(); + allocation.clone().release().unwrap(); + let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find(&name).is_none()); + } + } + + #[tokio::test] + async fn malformed_restore_state_is_rejected_before_filesystem_preparation() { + for corrupt_flow in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let store = session_store::CombinedStore::new( + session_store::FsStore::new(temp.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(temp.path().join("workers")).unwrap(), + ); + let session_id = session_store::new_session_id(); + let segment_id = session_store::new_segment_id(); + let mut entries = vec![LogEntry::AnnotatedSegmentStart { + ts: 0, + session_id, + system_prompt: Some("saved prompt".into()), + config: RequestConfig::default(), + history: Vec::new(), + forked_from: None, + compacted_from: None, + }]; + if corrupt_flow { + entries.push(LogEntry::Extension { + ts: 1, + domain: FLOW_RUNTIME_EXTENSION_DOMAIN.into(), + payload: serde_json::Value::Null, + }); + } else { + entries.push(LogEntry::ToolResultCorrected { + ts: 1, + entry: to_logged_history_entry(&history_entry( + Item::user_message("correction without a target"), + WorkerHistoryProvenance::LegacyUnknown, + )), + }); + } + store + .create_segment(session_id, segment_id, &entries) + .unwrap(); + let result = Worker::, _>::restore_from_manifest_with_context_and_model_client( + session_id, + segment_id, + minimal_manifest(), + store, + PromptCatalogSource::builtins_only(), + WorkerWorkspaceContext::no_workspace(), + // If preparation runs first, it must reject this missing root, + // without touching the process-global allocation table. + WorkerFilesystemAuthority::local(temp.path().join("missing"), temp.path().join("missing")), + Some(Box::new(NoopClient)), + ).await; + let error = match result { + Err(error) => error, + Ok(_) => panic!("corrupt restore state must be rejected"), + }; + let WorkerError::InvalidState(message) = error else { + panic!("restore parsed state too late: {error:?}"); + }; + assert!(message.contains(if corrupt_flow { + "Flow runtime state" + } else { + "typed Worker session history" + })); + } + } + } + + // Real Feature installation, task registries, Workdir provider/router, and + // PreparedWorker startup; only the model and task completion barrier are test implementations. + mod startup_feature_cleanup_tests { + use super::*; + use crate::feature::background::{ + BackgroundTaskCancellation, BackgroundTaskContext, BackgroundTaskSpec, + FeatureBackgroundTask, + }; + use crate::feature::{ + BackgroundTaskDeclaration, FeatureDescriptor, FeatureId, FeatureInstallContext, + FeatureInstallError, FeatureModule, + }; + use crate::hook::{HookError, HookErrorCategory}; + use tokio::sync::Notify; + + type StartupStore = + session_store::CombinedStore; + + #[derive(Default)] + struct TaskSignals { + entered: Notify, + cancelled: AtomicBool, + release: Notify, + exited: AtomicBool, + session: Mutex>, + } + + #[derive(Clone)] + struct BarrierTask { + signals: Arc, + _lifetime: Arc<()>, + panic_on_release: bool, + } + + #[async_trait] + impl FeatureBackgroundTask for BarrierTask { + async fn run( + &self, + _context: BackgroundTaskContext, + cancellation: BackgroundTaskCancellation, + ) -> Result<(), HookError> { + self.signals.entered.notify_one(); + cancellation.cancelled().await; + self.signals.cancelled.store(true, Ordering::Release); + self.signals.release.notified().await; + assert!(!self.panic_on_release, "injected startup task join failure"); + let session = self.signals.session.lock().unwrap().clone(); + if let Some(session) = session { + assert!( + session_open(&session).await, + "Feature task must retain Workdir access until it is quiescent" + ); + } + self.signals.exited.store(true, Ordering::Release); + Ok(()) + } + } + + struct BarrierFeature(BarrierTask); + + impl FeatureModule for BarrierFeature { + fn descriptor(&self) -> FeatureDescriptor { + FeatureDescriptor::builtin("startup-barrier", "Startup barrier") + .with_background_task(BackgroundTaskDeclaration::worker_managed( + "barrier", + "Startup task completion barrier", + )) + } + + fn install( + &self, + context: &mut FeatureInstallContext<'_>, + ) -> Result<(), FeatureInstallError> { + context.background_tasks().register( + BackgroundTaskSpec::single_flight( + BackgroundTaskDeclaration::worker_managed( + "barrier", + "Startup task completion barrier", + ), + Duration::from_secs(60), + ), + self.0.clone(), + ) + } + } + + async fn worker(temp: &tempfile::TempDir) -> Worker { + let store = session_store::CombinedStore::new( + session_store::FsStore::new(temp.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(temp.path().join("workers")).unwrap(), + ); + Worker::new( + minimal_manifest(), + Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient), + store, + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + Scope::empty(), + ) + .await + .unwrap() + } + + fn install( + worker: &mut Worker, + panic_on_release: bool, + ) -> (Arc, std::sync::Weak<()>) { + let signals = Arc::new(TaskSignals::default()); + let lifetime = Arc::new(()); + let weak = Arc::downgrade(&lifetime); + let report = worker.install_features(FeatureRegistryBuilder::new().with_module( + BarrierFeature(BarrierTask { + signals: signals.clone(), + _lifetime: lifetime, + panic_on_release, + }), + )); + assert!(!report.has_errors()); + (signals, weak) + } + + async fn start(worker: &Worker, signals: &TaskSignals) { + worker + .feature_background_tasks + .start( + &FeatureId::builtin("startup-barrier"), + "barrier", + HookInvocationContext { + workspace_id: None, + worker_id: "startup-owner".into(), + session_id: worker.session_id().to_string(), + run_id: None, + turn_index: None, + call_id: None, + }, + ) + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), signals.entered.notified()) + .await + .unwrap(); + } + + fn bind_session( + worker: &mut Worker, + temp: &tempfile::TempDir, + ) -> workdir::WorkdirSessionHandle { + let session: workdir::WorkdirSessionHandle = + Arc::new(workdir::LocalWorkdirSession::materialized_bound( + workdir::Workdir::new("startup-task-cleanup"), + temp.path().to_path_buf(), + temp.path().to_path_buf(), + SharedScope::new(Scope::writable(temp.path()).unwrap()), + workdir::WorkdirSessionCapabilities::ALL, + )); + worker.bind_single_workdir_session(Some(session.clone())); + session + } + + async fn session_open(session: &workdir::WorkdirSessionHandle) -> bool { + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root(), + }) + .await + .is_ok() + } + + async fn fail_consuming_start( + worker: Worker, + temp: &tempfile::TempDir, + ) { + let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some(); + let blocked_parent = temp.path().join("blocked-runtime"); + std::fs::write(&blocked_parent, "not a directory").unwrap(); + let result = crate::PreparedWorker::new( + worker, + crate::WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: blocked_parent.join("1"), + bash_output_dir: temp.path().join("spill"), + }, + crate::WorkerControllerTransport::InProcess, + ) + .start() + .await; + let Err(crate::WorkerBootstrapError::Controller { + source, + cleanup_failed, + }) = result + else { + panic!("runtime parent file must reject controller startup"); + }; + assert!(matches!( + source.kind(), + std::io::ErrorKind::NotADirectory | std::io::ErrorKind::AlreadyExists + )); + assert_eq!(cleanup_failed, factory_owned_cleanup); + } + + #[tokio::test] + async fn consumed_worker_failure_retains_all_task_joins_across_cleanup_timeout() { + let temp = tempfile::tempdir().unwrap(); + let mut worker = worker(&temp).await; + let session = bind_session(&mut worker, &temp); + let sessions = worker.workdir_sessions(); + let (first, first_lifetime) = install(&mut worker, false); + *first.session.lock().unwrap() = Some(session.clone()); + start(&worker, &first).await; + let cleanup = worker.retain_startup_feature_cleanup(); + let retry_owner = cleanup.clone(); + let (second, second_lifetime) = install(&mut worker, false); + *second.session.lock().unwrap() = Some(session.clone()); + start(&worker, &second).await; + fail_consuming_start(worker, &temp).await; + + assert!(!sessions.aliases().is_empty()); + assert!(session_open(&session).await); + assert!(first_lifetime.upgrade().is_some()); + assert!(second_lifetime.upgrade().is_some()); + assert!( + tokio::time::timeout(Duration::from_millis(50), cleanup.shutdown()) + .await + .is_err() + ); + assert!(first.cancelled.load(Ordering::Acquire)); + drop(cleanup); + + // A fresh shutdown of the drained registry would falsely succeed + // once the second task finishes, abandoning the first task's join. + second.release.notify_one(); + assert!( + tokio::time::timeout(Duration::from_millis(50), retry_owner.shutdown()) + .await + .is_err() + ); + assert!(!first.exited.load(Ordering::Acquire)); + assert!(session_open(&session).await); + first.release.notify_one(); + tokio::time::timeout(Duration::from_secs(5), retry_owner.shutdown()) + .await + .unwrap() + .unwrap(); + assert!(first.exited.load(Ordering::Acquire)); + assert!(second.exited.load(Ordering::Acquire)); + assert!(session_open(&session).await); + sessions.close_all().await.unwrap(); + assert!(sessions.aliases().is_empty()); + assert!(!session_open(&session).await); + drop(retry_owner); + assert!(first_lifetime.upgrade().is_none()); + assert!(second_lifetime.upgrade().is_none()); + } + + #[tokio::test] + async fn startup_without_factory_cleanup_owner_still_closes_sessions_on_error() { + let temp = tempfile::tempdir().unwrap(); + let mut worker = worker(&temp).await; + let session = bind_session(&mut worker, &temp); + let sessions = worker.workdir_sessions(); + assert!(session_open(&session).await); + fail_consuming_start(worker, &temp).await; + assert!(sessions.aliases().is_empty()); + assert!(!session_open(&session).await); + } + + #[tokio::test] + async fn cleanup_error_settles_other_registries_and_keeps_owner_for_retry() { + let temp = tempfile::tempdir().unwrap(); + let mut worker = worker(&temp).await; + let (first, first_lifetime) = install(&mut worker, true); + start(&worker, &first).await; + let cleanup = worker.retain_startup_feature_cleanup(); + let (second, second_lifetime) = install(&mut worker, false); + start(&worker, &second).await; + fail_consuming_start(worker, &temp).await; + first.release.notify_one(); + second.release.notify_one(); + + let error = tokio::time::timeout(Duration::from_secs(5), cleanup.shutdown()) + .await + .unwrap() + .unwrap_err(); + assert_eq!(error.category, HookErrorCategory::Internal); + assert!(second.exited.load(Ordering::Acquire)); + assert!(first_lifetime.upgrade().is_some()); + assert!(second_lifetime.upgrade().is_some()); + tokio::time::timeout(Duration::from_secs(5), cleanup.shutdown()) + .await + .unwrap() + .unwrap(); + drop(cleanup); + assert!(first_lifetime.upgrade().is_none()); + assert!(second_lifetime.upgrade().is_none()); + } + + #[tokio::test] + async fn successful_controller_transfer_releases_old_startup_task_scopes() { + let temp = tempfile::tempdir().unwrap(); + let mut worker = worker(&temp).await; + let (_signals, old_scope) = install(&mut worker, false); + let cleanup = worker.retain_startup_feature_cleanup(); + let started = crate::PreparedWorker::new( + worker, + crate::WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: temp.path().join("runs/1"), + bash_output_dir: temp.path().join("spill"), + }, + crate::WorkerControllerTransport::InProcess, + ) + .start() + .await + .unwrap(); + + assert!(old_scope.upgrade().is_some()); + drop(cleanup); + let released = old_scope.upgrade().is_none(); + stop_body_publication_controller(started).await; + assert!( + released, + "live Controller must not retain factory startup capture" + ); + } + } + #[tokio::test] async fn post_metadata_allocation_failure_fences_appends_and_restart_converges() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/worker/tests/controller_test.rs b/crates/worker/tests/controller_test.rs index 1938f0b18..e4b8d7041 100644 --- a/crates/worker/tests/controller_test.rs +++ b/crates/worker/tests/controller_test.rs @@ -759,6 +759,211 @@ async fn controller_startup_failure_closes_bound_workdir_session() { )); } +mod socket_startup_failure { + //! Connect the real controller startup, command observer, socket bind and session rollback. + //! Only the Workdir telemetry boundary and LLM are substituted. This ordinary controller + //! integration target uses an isolated directory at the socket path to reject the real bind. + use super::*; + use std::path::PathBuf; + use tokio::sync::broadcast; + use workdir::WorkdirSession; + + #[derive(Debug)] + struct ObservedSession { + local: LocalWorkdirSession, + socket_path: PathBuf, + events: broadcast::Sender, + subscriptions: AtomicUsize, + receivers_at_close: Mutex>, + } + + #[async_trait] + impl WorkdirSession for ObservedSession { + fn workdir(&self) -> &Workdir { + self.local.workdir() + } + + fn capabilities(&self) -> WorkdirSessionCapabilities { + self.local.capabilities() + } + + async fn stat( + &self, + request: workdir::StatRequest, + ) -> Result { + self.local.stat(request).await + } + + async fn read( + &self, + request: workdir::ReadRequest, + ) -> Result { + self.local.read(request).await + } + + async fn write( + &self, + request: workdir::WriteRequest, + ) -> Result { + self.local.write(request).await + } + + async fn edit( + &self, + request: workdir::EditRequest, + ) -> Result { + self.local.edit(request).await + } + + async fn list( + &self, + request: workdir::ListRequest, + ) -> Result { + self.local.list(request).await + } + + async fn glob( + &self, + request: workdir::GlobRequest, + ) -> Result { + self.local.glob(request).await + } + + async fn grep( + &self, + request: workdir::GrepRequest, + ) -> Result { + self.local.grep(request).await + } + + async fn start_command( + &self, + request: CommandRequest, + ) -> Result { + self.local.start_command(request).await + } + + async fn command_status( + &self, + handle: workdir::CommandHandle, + ) -> Result { + self.local.command_status(handle).await + } + + async fn command_output( + &self, + request: CommandOutputRequest, + ) -> Result { + self.local.command_output(request).await + } + + async fn cancel_command(&self, handle: workdir::CommandHandle) -> Result<(), WorkdirError> { + self.local.cancel_command(handle).await + } + + fn subscribe_command_events(&self) -> Option> { + // Runtime creation and initial file writes must already have succeeded. A directory + // survives SocketServer's stale-file removal and makes the subsequent Unix bind fail. + std::fs::create_dir(&self.socket_path).unwrap(); + self.subscriptions.fetch_add(1, Ordering::SeqCst); + Some(self.events.subscribe()) + } + + async fn close(&self) -> Result<(), WorkdirError> { + // Record before any await: aborting without joining must not pass by getting a + // scheduler turn during the provider's asynchronous close. The sender stays alive + // even after close, so channel closure cannot accidentally dispose of the observer. + self.receivers_at_close + .lock() + .unwrap() + .push(self.events.receiver_count()); + self.local.close().await + } + } + + #[tokio::test(flavor = "current_thread")] + async fn socket_bind_failure_joins_command_observer_before_closing_session() { + let root = tempfile::tempdir().unwrap(); + let run_dir = root.path().join("runs/1"); + let socket_path = run_dir.join("worker.sock"); + let client = MockClient::new(simple_text_events()); + let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap(); + let store = CombinedStore::new( + FsStore::new(root.path().join("sessions")).unwrap(), + FsWorkerStore::new(root.path().join("workers")).unwrap(), + ); + let engine = + Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated( + client.clone(), + ); + let mut worker = Worker::new( + manifest, + engine, + store, + WorkerWorkspaceContext::local_filesystem(None), + WorkerFilesystemAuthority::local(root.path().into(), root.path().into()), + manifest::Scope::writable(root.path()).unwrap(), + ) + .await + .unwrap(); + let (events, unused_receiver) = broadcast::channel(8); + drop(unused_receiver); + let session = Arc::new(ObservedSession { + local: LocalWorkdirSession::materialized_bound( + Workdir::new("socket-startup-failure"), + root.path().into(), + root.path().into(), + worker.scope().clone(), + WorkdirSessionCapabilities::ALL, + ), + socket_path: socket_path.clone(), + events, + subscriptions: AtomicUsize::new(0), + receivers_at_close: Mutex::new(Vec::new()), + }); + worker.bind_single_workdir_session(Some(session.clone())); + let sessions = worker.workdir_sessions(); + + let result = tokio::time::timeout( + std::time::Duration::from_secs(5), + WorkerController::spawn_runtime_managed_run( + worker, + &run_dir, + &root.path().join("bash-output"), + ), + ) + .await + .expect("socket startup and rollback must settle"); + let error = match result { + Err(error) => error, + Ok(_) => panic!("a directory at the Unix socket path must reject controller startup"), + }; + assert_eq!(error.kind(), std::io::ErrorKind::AddrInUse); + assert_eq!(session.subscriptions.load(Ordering::SeqCst), 1); + assert_eq!(*session.receivers_at_close.lock().unwrap(), vec![0]); + assert_eq!(session.events.receiver_count(), 0); + assert!(sessions.aliases().is_empty()); + // No controller retains the Worker/Engine after the failed startup. + assert_eq!(Arc::strong_count(&client.call_count), 1); + assert_eq!(client.call_count.load(Ordering::SeqCst), 0); + assert!(socket_path.is_dir()); + assert!(tokio::net::UnixStream::connect(&socket_path).await.is_err()); + assert!(matches!( + session + .start_command(CommandRequest { + command: "printf unreachable".to_owned(), + timeout_secs: 5, + output_limit: 1024, + cwd: workdir::WorkdirPath::root(), + spill_dir: None, + tool_call_id: None, + }) + .await, + Err(WorkdirError::Unavailable(_)) + )); + } +} + async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) { let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); loop { diff --git a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs index 0c5ca25dc..8e5c4d779 100644 --- a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs +++ b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs @@ -85,7 +85,10 @@ impl RuntimeWorkerFactory for CapturingFactory { async fn restore_controller( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, - ) -> std::result::Result { + ) -> std::result::Result< + RuntimeWorkerController, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { let worker_ref = request.worker_ref.clone(); let controller = self.inner.restore_controller(request).await?; self.captures diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs index 2f5c36ca8..da46ba67d 100644 --- a/crates/workspace-server/src/server_assignment_attention_tests.rs +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -494,9 +494,18 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFac async fn restore_controller( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, - ) -> std::result::Result { + ) -> std::result::Result< + worker_runtime::worker_backend::RuntimeWorkerController, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { self.controller(request.worker_ref.worker_id.to_string(), true) .await + .map_err(|_| { + worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( + worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, + "attention fixture reconstruction failed", + ) + }) } } diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs index 5d397814a..ff5adfca2 100644 --- a/crates/workspace-server/src/server_assignment_delivery_tests.rs +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -832,9 +832,18 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFact async fn restore_controller( &self, request: delivery_execution::WorkerExecutionRestoreRequest, - ) -> std::result::Result { + ) -> std::result::Result< + worker_runtime::worker_backend::RuntimeWorkerController, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { self.controller(request.worker_ref.worker_id.to_string(), true) .await + .map_err(|_| { + worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( + worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, + "delivery fixture reconstruction failed", + ) + }) } } diff --git a/crates/workspace-server/src/server_workspace_config_wip_tests.rs b/crates/workspace-server/src/server_workspace_config_wip_tests.rs index 1e989edd8..0714b0f22 100644 --- a/crates/workspace-server/src/server_workspace_config_wip_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_wip_tests.rs @@ -584,7 +584,10 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto async fn restore_controller( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, - ) -> std::result::Result { + ) -> std::result::Result< + worker_runtime::worker_backend::RuntimeWorkerController, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { self.inner.restore_controller(request).await } } From b5c105545c2fddd1849150bf0355832d1e9a301c Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 04:55:38 +0900 Subject: [PATCH 02/28] fix: close unattached deferred sessions in restore rollback Logical Workdir routing can use a Workspace session instead of the operation's raw Deferred handle, and reconstruction can fail before routing. Close the exact handles frozen in the accepted operation request after router cleanup and before releasing allocation ownership. Do not resolve aliases against a new execution or consult current Host grants for cleanup. Reproduced on 61c20b099 with diagnostic-only test output: rollback emptied routers with provider opens 0, but raw read succeeded and opened the provider (1). Retain the SessionClosed and zero-open regression, add pre-router Metadata/Reconstruction/Binding recovery and operation-mismatch/same-alias isolation coverage. Focused Profile and deferred cases pass; full exact-source validation follows this commit before publication. --- crates/worker-runtime/src/worker_backend.rs | 21 +- .../worker_backend_restore_failure_tests.rs | 179 ++++++++++++++++-- 2 files changed, 176 insertions(+), 24 deletions(-) diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 5583848ad..af4933bc3 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -2051,6 +2051,9 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { // Reconstruction can write before failing; only the resource owner can // confirm cleanup, even when no Worker value was returned. let result: Result = async { + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; let worker_name = Self::runtime_worker_name_for_ref(&request.worker_ref); let only_binding = (request.workdir_attachments.len() == 1) .then(|| request.workdir_attachments.values().next()) @@ -2351,22 +2354,34 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .map_err(|_| RuntimeRestoreCleanupFailure::Ownership)? .get(&request.worker_ref) .cloned(); - if let Some(resources) = retained { + if let Some(resources) = retained.as_ref() { if resources.operation_id != request.operation_id { return Err(RuntimeRestoreCleanupFailure::Ownership); } - if let Some(feature_tasks) = resources.feature_tasks { + if let Some(feature_tasks) = resources.feature_tasks.as_ref() { feature_tasks .shutdown() .await .map_err(|_| RuntimeRestoreCleanupFailure::Feature)?; } - for sessions in resources.sessions { + for sessions in &resources.sessions { sessions .close_all() .await .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; } + } + // Logical attachments can route through a different Workspace session, + // and reconstruction can fail before any router exists. Close the exact + // raw handles retained by this operation, never a current alias lookup. + // An unopened Deferred session closes without contacting its provider. + for session in request.deferred_workdir_sessions.values() { + session + .close() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; + } + if let Some(resources) = retained { if let Some(allocation) = resources.allocation { allocation .release() diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs index f7c021837..8b870cd62 100644 --- a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -216,6 +216,31 @@ impl ProfileFailureFixture { ); } + fn assert_closed_deferred_sessions(&self, pending: &PendingRuntimeRestore) { + let sessions = pending.request.deferred_workdir_sessions.clone(); + assert!(!sessions.is_empty()); + self.backend + .run_on_adapter_runtime(async move { + for (alias, session) in sessions { + let read = session + .read(workdir::ReadRequest { + path: workdir::WorkdirPath::new("README.md").unwrap(), + offset: 0, + limit: 10, + max_bytes: 1024, + }) + .await; + assert!( + matches!(read, Err(workdir::WorkdirError::SessionClosed)), + "raw deferred attachment {alias} remained usable: {read:?}" + ); + } + Ok(()) + }) + .unwrap(); + assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0); + } + fn fresh_restore_and_old_replay( &self, request: runtime_api::WorkerRestoreRequest, @@ -464,33 +489,145 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer .all(|sessions| sessions.aliases().is_empty()) ); fixture.assert_released(); - let unopened = pending.request.deferred_workdir_sessions.clone(); - fixture - .backend - .run_on_adapter_runtime(async move { - for session in unopened.values() { - assert!(matches!( - session - .read(workdir::ReadRequest { - path: workdir::WorkdirPath::new("README.md").unwrap(), - offset: 0, - limit: 10, - max_bytes: 1024, - }) - .await, - Err(workdir::WorkdirError::SessionClosed) - )); - } - Ok(()) - }) - .unwrap(); - assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0); + fixture.assert_closed_deferred_sessions(&pending); eprintln!( "restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=RolledBack" ); fixture.fresh_restore_and_old_replay(request, result); } +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_failure_before_router_binding_closes_unattached_raw_sessions() { + for stage in [ + RuntimeRestoreStage::Metadata, + RuntimeRestoreStage::Reconstruction, + RuntimeRestoreStage::Binding, + ] { + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + fixture.fail(stage); + fixture + .backend + .factory + .restore_faults + .cleanup_fail + .store(true, Ordering::SeqCst); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let failed = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + failed.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + assert!(pending.request.workdir_attachments.is_empty()); + assert!(!pending.request.logical_workdir_attachments.is_empty()); + assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0); + fixture + .backend + .factory + .restore_faults + .cleanup_fail + .store(false, Ordering::SeqCst); + let rolled_back = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + rolled_back.state, + server_api::WorkerRestoreState::RolledBack, + "{stage:?}: {rolled_back:?}" + ); + fixture.assert_released(); + fixture.assert_closed_deferred_sessions(&pending); + fixture.fresh_restore_and_old_replay(request, rolled_back); + // Retained old cleanup authority must not become the fresh execution's + // same-alias connection, even after the new execution has stopped. + fixture.assert_closed_deferred_sessions(&pending); + } +} + +#[tokio::test] +#[serial_test::serial(worker_allocation)] +async fn failed_restore_raw_cleanup_is_operation_bound_and_does_not_follow_reused_alias() { + let root = restore_failure_tempdir(); + let factory = ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(root.path()); + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let alias = WorkdirAttachmentAlias::new("checkout").unwrap(); + let opens = Arc::new(AtomicUsize::new(0)); + let raw = |opens: Arc| { + deferred_workdir::DeferredWorkdirSession::handle( + "same-workdir", + workdir::WorkdirSessionCapabilities::READ_ONLY, + move || { + opens.fetch_add(1, Ordering::SeqCst); + Err("offline fixture".into()) + }, + ) + }; + let mut old = direct_restore_request(worker_ref.clone()); + old.deferred_workdir_sessions + .insert(alias.clone(), raw(opens.clone())); + let mut current = direct_restore_request(worker_ref.clone()); + current + .deferred_workdir_sessions + .insert(alias.clone(), raw(opens.clone())); + factory.failed_restore_resources.lock().unwrap().insert( + worker_ref.clone(), + FailedRestoreResources { + operation_id: current.operation_id, + sessions: Vec::new(), + allocation: None, + feature_tasks: None, + }, + ); + assert_eq!( + factory.cleanup_failed_restore(&old).await, + Err(RuntimeRestoreCleanupFailure::Ownership) + ); + assert_eq!(opens.load(Ordering::SeqCst), 0); + assert_eq!( + factory.failed_restore_resources.lock().unwrap()[&worker_ref].operation_id, + current.operation_id + ); + factory.cleanup_failed_restore(¤t).await.unwrap(); + assert_eq!(opens.load(Ordering::SeqCst), 0); + // Old raw ownership was not routed into, or closed with, the newer operation. + let request = workdir::ReadRequest { + path: workdir::WorkdirPath::new("README.md").unwrap(), + offset: 0, + limit: 10, + max_bytes: 1024, + }; + assert!(matches!( + current.deferred_workdir_sessions[&alias] + .read(request.clone()) + .await, + Err(workdir::WorkdirError::SessionClosed) + )); + assert!(matches!( + old.deferred_workdir_sessions[&alias] + .read(request.clone()) + .await, + Err(workdir::WorkdirError::Unavailable(_)) + )); + assert_eq!(opens.load(Ordering::SeqCst), 1); + factory.cleanup_failed_restore(&old).await.unwrap(); + assert!(matches!( + old.deferred_workdir_sessions[&alias].read(request).await, + Err(workdir::WorkdirError::SessionClosed) + )); + assert_eq!(opens.load(Ordering::SeqCst), 1); +} + #[test] #[serial_test::serial(worker_allocation)] fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconciles_without_execution() From 73a3e674c9656f419632a7d8be970ff5440a917e Mon Sep 17 00:00:00 2001 From: Hare Date: Thu, 22 Oct 2026 22:58:57 +0900 Subject: [PATCH 03/28] feat(worker): separate restored candidates, repair, and activation --- .../session-store/src/worker_session_store.rs | 15 + crates/worker-runtime/src/fs_store.rs | 21 +- crates/worker-runtime/src/lib.rs | 3 + .../src/retention/runtime_v5_tests.rs | 6 +- crates/worker-runtime/src/runtime.rs | 926 ++++++---- crates/worker-runtime/src/worker_backend.rs | 1141 ++++++++---- .../src/worker_backend/restore_recovery.rs | 173 ++ .../worker_backend_restore_candidate_tests.rs | 898 ++++++++++ .../worker_backend_restore_failure_tests.rs | 340 +++- .../worker-runtime/src/working_directory.rs | 27 +- crates/worker/src/bootstrap.rs | 33 + crates/worker/src/controller.rs | 29 + crates/worker/src/discovery.rs | 11 +- crates/worker/src/lib.rs | 3 + crates/worker/src/worker.rs | 1034 ++++++++++- crates/worker/tests/controller_test.rs | 37 +- crates/workspace-server/src/lib.rs | 3 + .../tests/restored_parent_control_tests.rs | 90 +- .../subjektiv_journal_connected_tests.rs | 1580 ++++++++++++++++- .../src/server_assignment_attention_tests.rs | 60 +- .../src/server_assignment_delivery_tests.rs | 142 +- .../src/server_workspace_config_wip_tests.rs | 89 +- deno.json | 4 +- tests/support/restore_environment.rs | 129 ++ tools/test.test.ts | 369 +++- tools/test.ts | 216 ++- 26 files changed, 6592 insertions(+), 787 deletions(-) create mode 100644 crates/worker-runtime/src/worker_backend/restore_recovery.rs create mode 100644 crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs create mode 100644 tests/support/restore_environment.rs diff --git a/crates/session-store/src/worker_session_store.rs b/crates/session-store/src/worker_session_store.rs index a19f1b2db..3a5dad1e8 100644 --- a/crates/session-store/src/worker_session_store.rs +++ b/crates/session-store/src/worker_session_store.rs @@ -194,6 +194,21 @@ impl WorkerSessionStore { Ok(store) } + /// Open a current retained Session without creating or migrating anything. + /// The live constraint is checked here and again by every later publication; + /// attaching the control store itself does not acquire publication authority. + pub fn open_existing_with_retention( + root: impl Into, + retention: crate::retention::SessionRetentionStore, + ) -> Result { + let mut store = Self::open_read_only(root)?; + if let Some(session) = *store.session_id.lock().unwrap() { + retention.ensure_live(session)?; + } + store.retention = Some(retention); + Ok(store) + } + fn lock_publication( &self, session: SessionId, diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index de799b323..21ecfeca8 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -2301,9 +2301,28 @@ impl WorkerAggregateRecord { }); } if let Some(request) = request { + let mut validation_request = request.clone(); + // Pending admission may supply a missing Workspace connection + // without applying it to the saved execution. Validate that + // exact owner's intent against identity; do not widen the + // Available-record rule or mutate the canonical request. + if validation_request.workspace_api.is_none() + && let PersistedWorkerLifecycleOperation::Restore(restore) = &operation + && let Some(api) = restore_guard + .pending_preparation(restore.operation_id) + .and_then(|preparation| preparation.workspace_api.as_ref()) + { + validation_request.workspace_api = + Some( + serde_json::from_value(serde_json::to_value(api).map_err( + |error| runtime_store_corrupt(path, error.to_string()), + )?) + .map_err(|error| runtime_store_corrupt(path, error.to_string()))?, + ); + } WorkerExecutionRecord { schema_version: SCHEMA_VERSION, - request: request.clone(), + request: validation_request, binding: binding.clone(), restore_intent, } diff --git a/crates/worker-runtime/src/lib.rs b/crates/worker-runtime/src/lib.rs index 1e160a6f5..f4a5b02ff 100644 --- a/crates/worker-runtime/src/lib.rs +++ b/crates/worker-runtime/src/lib.rs @@ -16,6 +16,9 @@ pub mod management; pub mod observation; pub mod profile_archive; pub mod resource; +#[cfg(test)] +#[path = "../../../tests/support/restore_environment.rs"] +mod restore_test_environment; #[cfg(feature = "fs-store")] pub mod retention; mod runtime; diff --git a/crates/worker-runtime/src/retention/runtime_v5_tests.rs b/crates/worker-runtime/src/retention/runtime_v5_tests.rs index d9a3bccdd..47d75c940 100644 --- a/crates/worker-runtime/src/retention/runtime_v5_tests.rs +++ b/crates/worker-runtime/src/retention/runtime_v5_tests.rs @@ -682,8 +682,10 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() { #[cfg(unix)] #[test] fn current_runtime_session_migration_still_rejects_live_worker_sockets() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().join("runtime"); + // The same TempDir owns the relative bind/connect reference. No cwd change + // or alias is needed, and the real live-socket migration refusal stays tested. + let temp = tempfile::tempdir_in(".").unwrap(); + let root = Path::new(temp.path().file_name().unwrap()).join("runtime"); let (worker, _, _) = runtime_fixture(&root, 12); let before = tree_bytes(&root); let runs = root.join("workers").join(worker.to_string()).join("runs"); diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 19c17b053..eaec092dc 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -1886,70 +1886,7 @@ impl Runtime { let previous = state.worker(worker_ref)?.clone(); { let worker = state.worker_mut(worker_ref)?; - if !attachments.is_empty() && worker.workspace_id.is_none() { - return Err(RuntimeError::InvalidRequest( - "logical Workdir attachments require Workspace authority".to_string(), - )); - } - if worker.request.is_none() { - return Err(RuntimeError::WorkerExecutionUnavailable { - worker_id: worker.worker_id, - message: "persisted Worker restore request is unavailable".to_string(), - }); - } - let logical_by_alias = attachments - .iter() - .map(|attachment| { - ( - attachment.alias.clone(), - attachment.working_directory_id.as_str(), - ) - }) - .collect::>(); - for local in &worker.workdir_attachments { - let local_id = local - .working_directory - .summary - .working_directory_id - .as_str(); - if let Some(logical_id) = logical_by_alias.get(&local.alias) - && *logical_id != local_id - { - return Err(RuntimeError::InvalidRequest(format!( - "logical Workdir attachment alias `{}` conflicts with Runtime-local Workdir {}", - local.alias, local_id - ))); - } - } - let retained_local = worker - .workdir_attachments - .iter() - .filter_map(|local| { - let local_id = local - .working_directory - .summary - .working_directory_id - .as_str(); - (logical_by_alias.get(&local.alias).copied() == Some(local_id)) - .then(|| (local.alias.clone(), local_id.to_string())) - }) - .collect::>(); - worker - .workdir_attachments - .retain(|local| retained_local.contains_key(&local.alias)); - let request = worker - .request - .as_mut() - .expect("restore request existence checked above"); - request - .workdir_attachment_requests - .retain(|local| retained_local.contains_key(&local.alias)); - request.workdir_attachments.retain(|local| { - retained_local - .get(&local.alias) - .is_some_and(|workdir_id| workdir_id == &local.working_directory_id) - }); - worker.logical_workdir_attachments = attachments; + worker.apply_logical_workdir_attachments(attachments)?; } if let Err(error) = state.persist_worker(&worker_ref.worker_id) { state.workers.insert(worker_ref.worker_id, previous); @@ -2417,54 +2354,6 @@ impl Runtime { return Ok(result); } } - if !owner.preparation_applied { - let uncertain = |message: String| { - RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::ReconciliationRequired, - "worker_restore_preparation_reconciliation_required", - message, - ) - }; - // Apply reference updates under the original pending_restore ID; no provider is opened. - let mut effects_started = false; - if let Err(error) = self.apply_restore_preparation_under_lock( - worker_ref, - owner.request.preparation, - owner.operation_id, - &mut effects_started, - ) { - if effects_started - || matches!(error, RuntimeError::StoreCommitOutcomeUnknown { .. }) - { - return Ok(uncertain(sanitize_worker_create_failure_message( - &error.to_string(), - ))); - } - let result = RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::Rejected, - "worker_restore_preparation_rejected", - sanitize_worker_create_failure_message(&error.to_string()), - ); - self.finish_restore_receipt(worker_ref, &result)?; - return Ok(result); - } - let mut state = self.lock()?; - let mut candidate = state.worker(worker_ref)?.clone(); - candidate - .restore_guard - .owners - .get_mut(&owner.request.request_id) - .expect("admitted owner") - .preparation_applied = true; - if let Err(error) = state.persist_worker_record(&candidate) { - // Keep the started (not applied) record in memory. Even a - // known failed completion write follows possible effects. - return Ok(uncertain(sanitize_worker_create_failure_message( - &error.to_string(), - ))); - } - state.workers.insert(worker_ref.worker_id, candidate); - } } let result = self.restore_worker_under_lock(worker_ref, mode)?; self.finish_restore_receipt(worker_ref, &result)?; @@ -2481,95 +2370,18 @@ impl Runtime { } let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); - if candidate.restore_guard.active_request_id.is_some() { + if candidate.restore_guard.active_request_id.is_some() + || candidate.pending_restore.is_some() + { candidate.restore_guard.finish(result.clone()); candidate.pending_restore = None; + candidate.restore_candidate_context = None; state.persist_worker_record(&candidate)?; state.workers.insert(worker_ref.worker_id, candidate); - } - Ok(()) - } - - fn apply_restore_preparation_under_lock( - &self, - worker_ref: &WorkerRef, - preparation: Option, - operation_id: WorkerLifecycleOperationId, - effects_started: &mut bool, - ) -> Result<(), RuntimeError> { - let Some(preparation) = preparation else { - return Ok(()); - }; - // Validate all read-only structure before any binding effect. - if let Some(attachments) = &preparation.workdir_attachments { - let attachments: Vec = serde_json::from_value( - serde_json::to_value(attachments) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, - ) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; - validate_logical_workdir_attachments(&attachments)?; - } - // DTO conversion and every Runtime preparation effect occur AFTER admission. - if let Some(api) = preparation.workspace_api { - let api: WorkspaceApiRef = serde_json::from_value( - serde_json::to_value(api) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, - ) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; - if self.lock()?.worker(worker_ref)?.workspace_id.as_deref() - != Some(api.workspace_id.as_str()) - { - return Err(RuntimeError::InvalidRequest( - "Prepared Workspace API does not match Worker Workspace identity".into(), - )); - } - let mut state = self.lock()?; - let mut candidate = state.worker(worker_ref)?.clone(); - if candidate.pending_stop.is_some() - || candidate - .pending_restore - .is_none_or(|pending| pending.operation_id != operation_id) - { - return Err(RuntimeError::RestoreObservationConflict { - worker_id: worker_ref.worker_id, - }); - } - let persisted = candidate.request.as_mut().ok_or_else(|| { - RuntimeError::InvalidRequest("Restore specification is missing".into()) - })?; - if persisted.workspace_api.as_ref().is_some_and(|existing| { - existing.workspace_id != api.workspace_id - || existing.base_url.trim_end_matches('/') != api.base_url.trim_end_matches('/') - }) { - return Err(RuntimeError::InvalidRequest( - "Workspace API preparation cannot change Worker identity or base URL".into(), - )); - } - persisted.workspace_api = Some(api); - *effects_started = true; - match state.persist_worker_record(&candidate) { - Ok(()) => { - state.workers.insert(worker_ref.worker_id, candidate); - } - Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { - state.workers.insert(worker_ref.worker_id, candidate); - return Err(error); - } - Err(error) => return Err(error), + if result.state == WorkerRestoreState::Accepted { + state.publish_worker_upsert(worker_ref.worker_id)?; } } - if let Some(attachments) = preparation.workdir_attachments { - let attachments: Vec = serde_json::from_value( - serde_json::to_value(attachments) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, - ) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; - validate_logical_workdir_attachments(&attachments)?; - *effects_started = true; - self.replace_worker_workdir_attachments_under_lock(worker_ref, attachments, true)?; - } - // Retained legacy SSH preparation is not a prerequisite or an effect - // of restoring a Worker. Sessions resolve live access when used. Ok(()) } @@ -2621,6 +2433,28 @@ impl Runtime { { return Ok(RuntimeWorkerRestoreResult::accepted(worker.detail())); } + let preparation = worker + .restore_guard + .active_request_id + .as_ref() + .and_then(|id| worker.restore_guard.owners.get(id)) + .filter(|owner| !owner.preparation_applied) + .and_then(|owner| owner.request.preparation.as_ref()); + let prepared_worker = match worker.prepared_restore_record(preparation) { + Ok(prepared) => prepared, + Err(error) => { + return Ok(RuntimeWorkerRestoreResult::failed( + if reconcile { + WorkerRestoreState::ReconciliationRequired + } else { + WorkerRestoreState::Rejected + }, + "worker_restore_preparation_rejected", + sanitize_worker_create_failure_message(&error.to_string()), + )); + } + }; + let worker = &prepared_worker; let Some(worker_request) = worker.request.clone() else { return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::Rejected, @@ -2645,6 +2479,16 @@ impl Runtime { .restore_candidate_context .clone() .unwrap_or_else(|| self.restore_candidate_execution_context(worker_ref.clone())); + // First-head repair needs the original launch Prompt material. Use + // only the exact cached snapshot, never current Profile resolution; + // ordinary saved-head restore still works without that cache. + let config_bundle = worker_request.config_bundle.as_ref().and_then(|reference| { + state + .config_bundles + .get(&reference.id) + .filter(|bundle| bundle.metadata.digest == reference.digest) + .cloned() + }); let request = WorkerExecutionRestoreRequest { operation_id, worker_ref: worker_ref.clone(), @@ -2655,7 +2499,7 @@ impl Runtime { logical_workdir_attachments: worker.logical_workdir_attachments.clone(), workdir_attachments: BTreeMap::new(), deferred_workdir_sessions: BTreeMap::new(), - config_bundle: None, + config_bundle, }; ( backend, @@ -2746,11 +2590,15 @@ impl Runtime { if let Err(error) = backend.activate_restored_worker(operation_id, worker_ref) { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %error, - "Worker restore committed but Backend observation activation failed" - ); + self.retain_restore_reconciliation_pending( + worker_ref, + candidate_context.clone(), + )?; + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_activation_pending", + sanitize_worker_create_failure_message(&error), + )); } if let Err(error) = self.activate_restore_candidate_context(worker_ref, &candidate_context) @@ -2766,13 +2614,21 @@ impl Runtime { sanitize_worker_create_failure_message(&error.to_string()), )); } - Ok(RuntimeWorkerRestoreResult::accepted(worker)) + let result = RuntimeWorkerRestoreResult::accepted(worker); + if let Err(error) = self.finish_restore_receipt(worker_ref, &result) { + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_receipt_commit_pending", + sanitize_worker_create_failure_message(&error.to_string()), + )); + } + Ok(result) } - Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + Err(error) => { tracing::error!( worker_id = %worker_ref.worker_id, error = %error, - "Worker restore commit outcome is unknown; preserving execution evidence" + "Worker restore activated but commit did not confirm success; preserving operation and execution" ); if let Err(retain_error) = self.retain_restore_execution_evidence( worker_ref, @@ -2783,70 +2639,15 @@ impl Runtime { tracing::error!( worker_id = %worker_ref.worker_id, error = %retain_error, - "failed to persist restore execution evidence after an uncertain commit" + "failed to persist restore execution evidence; original operation remains pending" ); } Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, - "worker_restore_commit_outcome_unknown", - "Worker restore commit outcome is uncertain; reread or retry the Worker restore", + "worker_restore_commit_reconciliation_required", + sanitize_worker_create_failure_message(&error.to_string()), )) } - Err(error) => match self.cleanup_failed_restore(&backend, worker_ref) { - Ok(()) => { - self.discard_restore_candidate_context(worker_ref, &candidate_context)?; - tracing::warn!( - worker_id = %worker_ref.worker_id, - error = %error, - "Worker restore commit failed and was rolled back" - ); - let receipt = RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::RolledBack, - "worker_restore_commit_rolled_back", - "Worker restore commit failed and cleanup completed", - ); - if let Err(settle_error) = - self.settle_restore_rollback(worker_ref, &receipt) - { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %settle_error, - "Worker restore cleanup completed but its durable operation could not be settled" - ); - return Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::ReconciliationRequired, - "worker_restore_rollback_commit_failed", - "Worker restore cleanup completed but durable reconciliation is still required", - )); - } - Ok(receipt) - } - Err(cleanup_error) => { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %error, - cleanup_error = %cleanup_error, - "Worker restore requires reconciliation after commit and cleanup failures" - ); - if let Err(retain_error) = self.retain_restore_execution_evidence( - worker_ref, - worker_state, - workdir_attachments, - candidate_context.clone(), - ) { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %retain_error, - "failed to persist restore execution evidence; in-process reconciliation remains required" - ); - } - Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::ReconciliationRequired, - "worker_restore_reconciliation_required", - "Worker restore result is uncertain; reread or retry the Worker restore", - )) - } - }, } } WorkerExecutionSpawnResult::Rejected(result) => { @@ -2962,13 +2763,11 @@ impl Runtime { fn activate_restore_candidate_context( &self, - worker_ref: &WorkerRef, + _worker_ref: &WorkerRef, context: &crate::execution::WorkerExecutionContext, ) -> Result<(), RuntimeError> { + // Retain the publication owner until terminal receipt persistence succeeds. context.activate_candidate()?; - self.lock()? - .worker_mut(worker_ref)? - .restore_candidate_context = None; Ok(()) } @@ -4581,13 +4380,43 @@ impl Runtime { ) -> Result { let mut state = self.lock()?; state.ensure_worker_ref(worker_ref)?; - let mut candidate = state.worker(worker_ref)?.clone(); + let previous = state.worker(worker_ref)?; + if previous.pending_stop.is_some() { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + let mut candidate = if let Some(id) = &previous.restore_guard.active_request_id { + let owner = previous.restore_guard.owners.get(id).ok_or_else(|| { + RuntimeError::InvalidRequest("Active Restore request has no persisted owner".into()) + })?; + if previous + .pending_restore + .is_none_or(|pending| pending.operation_id != owner.operation_id) + { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + let mut candidate = + previous.prepared_restore_record(owner.request.preparation.as_ref())?; + candidate + .restore_guard + .owners + .get_mut(id) + .expect("admitted owner") + .preparation_applied = true; + candidate + } else { + previous.clone() + }; candidate.restore_guard.rotate(); candidate.execution_metadata_available = true; candidate.execution_bound = true; - candidate.pending_restore = None; + // Keep pending/owner until buffered observations also activate. Otherwise + // a retry could replay Accepted after a failed publication activation. + // Activation follows this pending decision; terminal success is still absent. candidate.pending_stop = None; - // Keep the actual publication owner until buffered observations activate. candidate.status = status; let mut worker_state = worker_state; if worker_state.last_finished_submission_request_id.is_none() { @@ -4600,6 +4429,9 @@ impl Runtime { candidate.last_finished_submission_request_id.clone(); } let _ = candidate.apply_worker_state(&worker_state); + if let Some(pending) = candidate.pending_restore.as_mut() { + pending.last_settled_status = candidate.status; + } candidate.restore_intent = restore_intent_for_status(candidate.status); candidate.workdir_attachments = workdir_attachments; @@ -4607,17 +4439,14 @@ impl Runtime { // subsequent publish uses the same pure projection while the Worker // operation lock keeps this candidate stable. candidate.restore_guard.rotate(); - let accepted = RuntimeWorkerRestoreResult::accepted(candidate.detail()); - candidate.restore_guard.finish(accepted); let _subscription_projection = state.subscription_worker(&candidate)?; - // Persist the complete candidate before making it observable. A failed - // write therefore leaves both the in-memory catalog and subscriptions - // on the pre-restore record, so cleanup can produce a true rollback. + // Persist the execution projection before publishing observations. Terminal + // receipt settlement follows successful observation activation separately. state.persist_worker_record(&candidate)?; let detail = candidate.detail(); state.workers.insert(worker_ref.worker_id, candidate); - state.publish_worker_upsert(worker_ref.worker_id)?; + if let Err(error) = state.persist_runtime_snapshot() { tracing::warn!( worker_id = %worker_ref.worker_id, @@ -4723,27 +4552,6 @@ impl Runtime { Ok(()) } - fn cleanup_failed_restore( - &self, - backend: &WorkerExecutionBackendRef, - worker_ref: &WorkerRef, - ) -> Result<(), RuntimeError> { - let result = backend.stop_worker(worker_ref); - if !result.is_accepted() { - return Err(RuntimeError::WorkerExecutionRejected { - worker_id: worker_ref.worker_id, - operation: result.operation, - outcome: result.outcome, - message: format!( - "restored execution cleanup failed after catalog commit failure: {}", - result.message_or_default() - ), - result, - }); - } - Ok(()) - } - /// Bind the Backend registry identity once. Retention evidence fails closed /// until the Runtime host supplies this trusted configuration. pub fn bind_runtime_identity(&self, runtime_id: &str) -> Result<(), RuntimeError> { @@ -6494,6 +6302,21 @@ impl Default for RestoreGuard { } impl RestoreGuard { + pub(crate) fn pending_preparation( + &self, + operation_id: WorkerLifecycleOperationId, + ) -> Option<&runtime_api::WorkerRestorePreparation> { + let id = self.active_request_id.as_ref()?; + let owner = self.owners.get(id)?; + if owner.operation_id != operation_id + || owner.receipt.is_some() + || owner.request.request_id != *id + { + return None; + } + owner.request.preparation.as_ref() + } + fn rotate(&mut self) { self.observation_token = uuid::Uuid::now_v7().to_string(); } @@ -6535,6 +6358,123 @@ struct WorkerRecord { } impl WorkerRecord { + fn apply_logical_workdir_attachments( + &mut self, + mut attachments: Vec, + ) -> Result<(), RuntimeError> { + validate_logical_workdir_attachments(&attachments)?; + attachments.sort_by(|left, right| left.alias.cmp(&right.alias)); + if !attachments.is_empty() && self.workspace_id.is_none() { + return Err(RuntimeError::InvalidRequest( + "logical Workdir attachments require Workspace authority".to_string(), + )); + } + if self.request.is_none() { + return Err(RuntimeError::WorkerExecutionUnavailable { + worker_id: self.worker_id, + message: "persisted Worker restore request is unavailable".to_string(), + }); + } + let logical_by_alias = attachments + .iter() + .map(|attachment| { + ( + attachment.alias.clone(), + attachment.working_directory_id.as_str(), + ) + }) + .collect::>(); + for local in &self.workdir_attachments { + let local_id = local + .working_directory + .summary + .working_directory_id + .as_str(); + if let Some(logical_id) = logical_by_alias.get(&local.alias) + && *logical_id != local_id + { + return Err(RuntimeError::InvalidRequest(format!( + "logical Workdir attachment alias `{}` conflicts with Runtime-local Workdir {}", + local.alias, local_id + ))); + } + } + let retained_local = self + .workdir_attachments + .iter() + .filter_map(|local| { + let local_id = local + .working_directory + .summary + .working_directory_id + .as_str(); + (logical_by_alias.get(&local.alias).copied() == Some(local_id)) + .then(|| (local.alias.clone(), local_id.to_string())) + }) + .collect::>(); + self.workdir_attachments + .retain(|local| retained_local.contains_key(&local.alias)); + let request = self + .request + .as_mut() + .expect("restore request existence checked above"); + request + .workdir_attachment_requests + .retain(|local| retained_local.contains_key(&local.alias)); + request.workdir_attachments.retain(|local| { + retained_local + .get(&local.alias) + .is_some_and(|workdir_id| workdir_id == &local.working_directory_id) + }); + self.logical_workdir_attachments = attachments; + Ok(()) + } + + // Overlay preparation on an unpublished clone, never on Runtime authority. + // The same validation is repeated at the effect boundary under the Worker lock. + fn prepared_restore_record( + &self, + preparation: Option<&runtime_api::WorkerRestorePreparation>, + ) -> Result { + let mut candidate = self.clone(); + let Some(preparation) = preparation else { + return Ok(candidate); + }; + if let Some(api) = &preparation.workspace_api { + let api: WorkspaceApiRef = serde_json::from_value( + serde_json::to_value(api) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + if candidate.workspace_id.as_deref() != Some(api.workspace_id.as_str()) { + return Err(RuntimeError::InvalidRequest( + "Prepared Workspace API does not match Worker Workspace identity".into(), + )); + } + let persisted = candidate.request.as_mut().ok_or_else(|| { + RuntimeError::InvalidRequest("Restore specification is missing".into()) + })?; + if persisted.workspace_api.as_ref().is_some_and(|existing| { + existing.workspace_id != api.workspace_id + || existing.base_url.trim_end_matches('/') != api.base_url.trim_end_matches('/') + }) { + return Err(RuntimeError::InvalidRequest( + "Workspace API preparation cannot change Worker identity or base URL".into(), + )); + } + persisted.workspace_api = Some(api); + } + if let Some(attachments) = &preparation.workdir_attachments { + let attachments: Vec = serde_json::from_value( + serde_json::to_value(attachments) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + candidate.apply_logical_workdir_attachments(attachments)?; + } + Ok(candidate) + } + fn apply_worker_state(&mut self, incoming: &protocol::WorkerStateSnapshot) { if self.worker_state.as_ref().map(|state| &state.state) != Some(&incoming.state) { self.restore_guard.rotate(); @@ -8201,6 +8141,10 @@ mod tests { restore_gate: Mutex>>, restore_count: Mutex, restore_operation_ids: Mutex>, + activation_result: Mutex>>, + activation_gate: Mutex>>, + activation_operation_ids: Mutex>, + restore_requests: Mutex>, #[cfg(feature = "ws-server")] restore_event: Mutex>, config_bundles: Mutex>>, @@ -8448,6 +8392,7 @@ mod tests { &self, request: WorkerExecutionRestoreRequest, ) -> WorkerExecutionSpawnResult { + self.restore_requests.lock().unwrap().push(request.clone()); self.job_bindings .lock() .unwrap() @@ -8493,6 +8438,54 @@ mod tests { } } + fn reconcile_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + let known = self + .restore_requests + .lock() + .unwrap() + .iter() + .any(|previous| { + previous.operation_id == request.operation_id + && previous.worker_ref == request.worker_ref + }); + if known + && self + .contexts + .lock() + .unwrap() + .contains_key(&request.worker_ref.worker_id) + { + return WorkerExecutionSpawnResult::Connected { + worker_state: protocol::WorkerStatus::Idle.into(), + workdir_attachments: request.previous_workdir_attachments, + }; + } + self.restore_worker(request) + } + + fn activate_restored_worker( + &self, + operation_id: WorkerLifecycleOperationId, + _worker_ref: &WorkerRef, + ) -> Result<(), String> { + self.activation_operation_ids + .lock() + .unwrap() + .push(operation_id); + let gate = self.activation_gate.lock().unwrap().clone(); + if let Some(gate) = gate { + gate.enter_and_wait(); + } + self.activation_result + .lock() + .unwrap() + .clone() + .unwrap_or(Ok(())) + } + fn worker_completions( &self, _worker_ref: &WorkerRef, @@ -9755,7 +9748,7 @@ mod tests { } #[test] - fn synced_config_bundle_is_stored_checked_and_used_for_worker_creation() { + fn synced_config_bundle_is_stored_checked_and_used_for_worker_creation_and_restore() { let backend = Arc::new(TestExecutionBackend::default()); let runtime = Runtime::with_execution_backend(RuntimeOptions::default(), backend.clone()).unwrap(); @@ -9807,7 +9800,7 @@ mod tests { .unwrap(); assert_eq!( backend.config_bundles.lock().unwrap().as_slice(), - &[Some(bundle), None] + &[Some(bundle.clone()), Some(bundle)] ); } @@ -11725,6 +11718,343 @@ mod tests { } } + #[test] + fn restore_activation_failure_never_records_or_replays_accepted() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("activation failure")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let stops = *backend.stop_count.lock().unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + *backend.activation_result.lock().unwrap() = Some(Err("partial activation".into())); + let mut operation_id = None; + for _ in 0..2 { + let result = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(result.state, WorkerRestoreState::ReconciliationRequired); + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + let pending = record.pending_restore.unwrap(); + assert_eq!( + *operation_id.get_or_insert(pending.operation_id), + pending.operation_id + ); + assert!(record.restore_candidate_context.is_some()); + let owner = &record.restore_guard.owners[&request.request_id]; + assert_eq!(owner.operation_id, pending.operation_id); + assert!(owner.receipt.is_none()); + assert_eq!( + record.restore_guard.active_request_id.as_ref(), + Some(&request.request_id) + ); + assert_eq!(*backend.stop_count.lock().unwrap(), stops); + } + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + *backend.activation_result.lock().unwrap() = None; + let accepted = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(accepted.state, WorkerRestoreState::Accepted); + let activations = backend.activation_operation_ids.lock().unwrap().len(); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + accepted + ); + assert_eq!( + backend.activation_operation_ids.lock().unwrap().len(), + activations + ); + assert!( + backend + .activation_operation_ids + .lock() + .unwrap() + .iter() + .all(|id| Some(*id) == operation_id) + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + } + + #[cfg(feature = "fs-store")] + #[test] + fn activated_restore_commit_failure_preserves_pending_owner_without_stopping_execution() { + for fault in [ + crate::fs_store::AtomicWriteFault::BeforeRename, + crate::fs_store::AtomicWriteFault::AfterRename, + ] { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Arc::new( + Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("activation-commit"), + backend.clone(), + ) + .unwrap(), + ); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("activated commit failure")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let stops = *backend.stop_count.lock().unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let gate = Arc::new(RestoreGate::default()); + *backend.activation_gate.lock().unwrap() = Some(gate.clone()); + let restoring_runtime = runtime.clone(); + let restoring_ref = worker.worker_ref.clone(); + let restoring_request = request.clone(); + let restoring = std::thread::spawn(move || { + restoring_runtime.restore_worker_operation(&restoring_ref, restoring_request) + }); + assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + runtime_store(&runtime).fail_next_worker_write(fault); + gate.release(); + assert_eq!( + restoring.join().unwrap().unwrap().state, + WorkerRestoreState::ReconciliationRequired + ); + { + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert_eq!(record.pending_restore.unwrap().operation_id, operation_id); + assert!(record.restore_candidate_context.is_some()); + assert!( + record.restore_guard.owners[&request.request_id] + .receipt + .is_none() + ); + } + let persisted: serde_json::Value = serde_json::from_slice( + &std::fs::read( + dir.path() + .join("runtime/workers") + .join(worker.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap(); + let saved_receipt = + &persisted["restore_guard"]["owners"][&request.request_id]["receipt"]; + if matches!(fault, crate::fs_store::AtomicWriteFault::BeforeRename) { + assert!(saved_receipt.is_null()); + } else { + // Activation was confirmed before this uncertain terminal write. + assert_eq!(saved_receipt["state"], "accepted"); + } + assert_eq!(*backend.stop_count.lock().unwrap(), stops); + *backend.activation_gate.lock().unwrap() = None; + let accepted = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(accepted.state, WorkerRestoreState::Accepted); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + accepted + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!(*backend.stop_count.lock().unwrap(), stops); + } + } + + #[cfg(all(feature = "fs-store", feature = "ws-server"))] + #[test] + fn buffered_restore_state_write_failure_keeps_valid_pending_owner_and_replays_once() { + for fault in [ + crate::fs_store::AtomicWriteFault::BeforeRename, + crate::fs_store::AtomicWriteFault::AfterRename, + ] { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Arc::new( + Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("buffered-state"), + backend.clone(), + ) + .unwrap(), + ); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("buffered live state")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let gate = Arc::new(RestoreGate::default()); + *backend.activation_gate.lock().unwrap() = Some(gate.clone()); + *backend.restore_event.lock().unwrap() = Some(protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Running.into(), + }); + let restoring_runtime = runtime.clone(); + let restoring_ref = worker.worker_ref.clone(); + let restoring_request = request.clone(); + let restoring = std::thread::spawn(move || { + restoring_runtime.restore_worker_operation(&restoring_ref, restoring_request) + }); + assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + // The pending execution projection is committed, but its live + // Running observation is buffered until backend activation returns. + runtime_store(&runtime).fail_next_worker_write(fault); + gate.release(); + assert_eq!( + restoring.join().unwrap().unwrap().state, + WorkerRestoreState::ReconciliationRequired + ); + let persisted = runtime_store(&runtime).load_runtime_state().unwrap(); + let record = &persisted.workers[&worker.worker_id]; + assert_eq!(record.status, WorkerStatus::Idle); + let PersistedWorkerExecutionState::ReconciliationRequired( + PersistedWorkerLifecycleOperation::Restore(pending), + ) = &record.execution_state + else { + panic!("pending owner disappeared") + }; + assert_eq!(pending.last_settled_status, record.status); + assert_eq!(pending.operation_id, operation_id); + assert!( + record.restore_guard.owners[&request.request_id] + .receipt + .is_none() + ); + // Live protocol state and settled catalog state are deliberately + // different during activation. Persistence must remain readable. + assert_eq!( + runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .worker_state + .unwrap() + .catalog_status(), + protocol::WorkerStatus::Running + ); + *backend.activation_gate.lock().unwrap() = None; + let accepted = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(accepted.state, WorkerRestoreState::Accepted); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + accepted + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert!( + backend + .activation_operation_ids + .lock() + .unwrap() + .iter() + .all(|id| *id == operation_id) + ); + assert!(matches!( + runtime_store(&runtime) + .load_runtime_state() + .unwrap() + .workers[&worker.worker_id] + .execution_state, + PersistedWorkerExecutionState::Available(_) + )); + } + } + + #[test] + fn restore_candidate_receives_prepared_bindings_without_mutating_runtime_authority() { + let (runtime, backend) = runtime_and_backend(); + let runtime = Arc::new(runtime); + let workspace_scope = scope("workspace-a", "server-a"); + let mut create = scoped_task_request("read-only candidate bindings", "workspace-a"); + create.workspace_api = None; + let worker = runtime + .create_worker_scoped(&workspace_scope, create) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "workspace-a", "base_url": "https://prepared.example" }, + "workdir_attachments": [], "repository_access": [] + })).unwrap()); + let gate = Arc::new(RestoreGate::default()); + *backend.restore_gate.lock().unwrap() = Some(gate.clone()); + let restoring_runtime = runtime.clone(); + let restoring_ref = worker.worker_ref.clone(); + let restoring_scope = workspace_scope.clone(); + let restoring = std::thread::spawn(move || { + restoring_runtime.restore_worker_operation_scoped( + &restoring_scope, + &restoring_ref, + request, + ) + }); + assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); + { + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert!(record.request.as_ref().unwrap().workspace_api.is_none()); + assert!( + !record + .restore_guard + .owners + .values() + .next() + .unwrap() + .preparation_applied + ); + } + assert_eq!( + backend.restore_requests.lock().unwrap()[0] + .request + .workspace_api + .as_ref() + .unwrap() + .base_url, + "https://prepared.example" + ); + assert!(backend.activation_operation_ids.lock().unwrap().is_empty()); + gate.release(); + assert_eq!( + restoring.join().unwrap().unwrap().state, + WorkerRestoreState::Accepted + ); + let state = runtime.lock().unwrap(); + assert_eq!( + state + .worker(&worker.worker_ref) + .unwrap() + .request + .as_ref() + .unwrap() + .workspace_api + .as_ref() + .unwrap() + .base_url, + "https://prepared.example" + ); + } + #[cfg(feature = "ws-server")] #[test] fn committed_restore_retries_retained_observation_activation() { diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index af4933bc3..b5ace780f 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -16,6 +16,7 @@ //! Manifests, credentials, and handles are not passed to browser/API projections. mod deferred_workdir; +mod restore_recovery; use std::collections::{BTreeMap, HashMap}; use std::future::Future; @@ -176,12 +177,25 @@ pub struct RuntimeWorkerController { pub workspace_client: Arc, } +/// An unpublished restore resource owner. No protocol endpoint is available +/// until activation succeeds; discard settles the actual candidate resources +/// directly, without sending Shutdown to an ordinary Controller. +#[async_trait] +pub trait RuntimeRestoreCandidate: Send { + fn initial_state(&self) -> protocol::WorkerStateSnapshot; + + async fn activate(&mut self) -> Result; + + async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure>; +} + /// Failure facts supplied by the resource owner, never inferred from error text. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RuntimeRestoreStage { Preflight, Metadata, Reconstruction, + Recovery, Binding, Feature, Controller, @@ -228,6 +242,7 @@ pub struct RuntimeRestoreFailure { cleanup: RuntimeRestoreCleanup, io_kind: Option, cleanup_failure: Option, + durable_effects_possible: bool, } impl RuntimeRestoreFailure { @@ -239,6 +254,7 @@ impl RuntimeRestoreFailure { cleanup: RuntimeRestoreCleanup::NotNeeded, io_kind: None, cleanup_failure: None, + durable_effects_possible: false, } } @@ -250,6 +266,7 @@ impl RuntimeRestoreFailure { cleanup: RuntimeRestoreCleanup::Required, io_kind: None, cleanup_failure: None, + durable_effects_possible: false, } } @@ -261,6 +278,7 @@ impl RuntimeRestoreFailure { cleanup: RuntimeRestoreCleanup::Required, io_kind: None, cleanup_failure: None, + durable_effects_possible: false, } } @@ -285,7 +303,8 @@ impl std::fmt::Display for RuntimeRestoreFailure { fn restore_worker_cause(error: &WorkerError) -> &'static str { match error { - WorkerError::ConstructionCleanup { source, .. } => restore_worker_cause(source), + WorkerError::ConstructionCleanup { source, .. } + | WorkerError::RestoreActivationEffects { source } => restore_worker_cause(source), WorkerError::SegmentEmpty { .. } => "saved segment empty", WorkerError::SegmentSystemPromptMissing { .. } => "saved segment system prompt missing", WorkerError::WorkerMetadataPending { .. } => "saved Worker metadata pending", @@ -325,6 +344,52 @@ async fn finish_restore_failure( failure } +async fn prepare_restore_operation( + factory: &F, + request: WorkerExecutionRestoreRequest, +) -> Result, RuntimeRestoreFailure> { + factory.preflight_restore(&request).await.map_err(|_| { + RuntimeRestoreFailure::refused(RuntimeRestoreStage::Preflight, "restore preflight refused") + })?; + let repaired = factory.recover_restore_source(&request).await?; + let candidate = factory + .prepare_restore(request) + .await + .map_err(|mut failure| { + failure.durable_effects_possible |= repaired; + failure + })?; + if repaired { + Ok(Box::new(RepairedRestoreCandidate { candidate })) + } else { + Ok(candidate) + } +} + +// A completed repair is a canonical effect even when later activation fails +// before its own mutation boundary. Carry that evidence with the resource owner. +struct RepairedRestoreCandidate { + candidate: Box, +} + +#[async_trait] +impl RuntimeRestoreCandidate for RepairedRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.candidate.initial_state() + } + + async fn activate(&mut self) -> Result { + self.candidate.activate().await.map_err(|mut failure| { + failure.durable_effects_possible = true; + failure + }) + } + + async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure> { + self.candidate.discard().await + } +} + /// Factory seam used by [`WorkerRuntimeExecutionBackend`] to construct a real /// controller-backed Worker for a Runtime catalog entry. #[async_trait] @@ -382,10 +447,28 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { Ok(()) } + /// Separate idempotent recovery, bound to the accepted operation and its + /// canonical Session. A true result records that preparation's no-mutation + /// guarantee cannot be used to describe this operation as a whole. + async fn recover_restore_source( + &self, + _request: &WorkerExecutionRestoreRequest, + ) -> Result { + Ok(false) + } + + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure>; + async fn restore_controller( &self, request: WorkerExecutionRestoreRequest, - ) -> Result; + ) -> Result { + let mut candidate = prepare_restore_operation(self, request).await?; + candidate.activate().await + } /// Prove cleanup of a restore future that completed without returning a /// controller. An error is not absence evidence; unsupported factories keep @@ -667,6 +750,7 @@ pub struct ProfileRuntimeWorkerFactory { struct FailedRestoreResources { operation_id: crate::execution::WorkerLifecycleOperationId, sessions: Vec>, + candidate_workdir_cleanup: Option, allocation: Option, feature_tasks: Option, } @@ -690,6 +774,10 @@ impl ProfileRuntimeWorkerFactory { request: &WorkerExecutionRestoreRequest, error: &WorkerError, ) { + if let WorkerError::RestoreActivationEffects { source } = error { + self.retain_construction_failure(request, source); + return; + } let mut resources = self .failed_restore_resources .lock() @@ -2032,10 +2120,22 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { Ok(()) } - async fn restore_controller( + async fn recover_restore_source( + &self, + request: &WorkerExecutionRestoreRequest, + ) -> Result { + self.recover_saved_restore_source(request) + .await + .map_err(|mut failure| { + failure.durable_effects_possible = true; + failure + }) + } + + async fn prepare_restore( &self, request: WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result, RuntimeRestoreFailure> { self.preflight_restore(&request).await.map_err(|_| { RuntimeRestoreFailure::refused( RuntimeRestoreStage::Preflight, @@ -2047,10 +2147,9 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .executions .fetch_add(1, Ordering::SeqCst); let mut stage = RuntimeRestoreStage::Metadata; - let mut io_kind = None; - // Reconstruction can write before failing; only the resource owner can - // confirm cleanup, even when no Worker value was returned. - let result: Result = async { + // Reconstruction owns read-only resources; cleanup confirmation must + // come from that owner even when it cannot return a candidate. + let result: Result, &'static str> = async { #[cfg(test)] self.restore_checkpoint(stage) .map_err(|_| "injected restore boundary failure")?; @@ -2070,14 +2169,6 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { &request.request, self.runtime_id.as_deref(), ); - let observation_runtime_id = self.runtime_id.clone(); - let observation_workspace_id = request - .request - .workspace_api - .as_ref() - .map(|api| api.workspace_id.clone()); - let observation_grants = request.request.worker_observation_grants.clone(); - let observation_enabled = request.request.worker_observation_enabled; let workspace_request_client = request .request .workspace_api @@ -2096,14 +2187,14 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .worker_aggregate_dir(&request.worker_ref) .map_err(|_| "Worker aggregate root unavailable")?; let session_dir = worker_aggregate_dir.join("session"); - let session_store = WorkerSessionStore::new_with_retention( + let session_store = WorkerSessionStore::open_existing_with_retention( &session_dir, self.session_retention() .map_err(|_| "Session retention unavailable")?, ) .map_err(|_| "canonical Session store initialization failed")?; let worker_metadata_store = - WorkerAggregateStore::new(&worker_aggregate_dir, worker_name.clone()) + WorkerAggregateStore::open_read_only(&worker_aggregate_dir, worker_name.clone()) .map_err(|_| "canonical metadata store initialization failed")?; let metadata = worker_metadata_store .read_by_name(&worker_name) @@ -2122,6 +2213,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { FailedRestoreResources { operation_id: request.operation_id, sessions: Vec::new(), + candidate_workdir_cleanup: None, allocation: None, feature_tasks: None, }, @@ -2129,70 +2221,29 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { #[cfg(test)] self.restore_checkpoint(stage) .map_err(|_| "injected restore boundary failure")?; - let mut worker = match Worker::restore_from_worker_metadata_with_context( - &worker_name, - manifest.clone(), - store.clone(), - loader.clone(), - workspace_context.clone(), - filesystem_authority.clone(), - ) - .await - { - Ok(worker) => worker, - Err(WorkerError::WorkerMetadataPending { .. }) - if request.request.initial_input.is_none() => - { - let pending_loader = if workspace_context.workspace_id().is_some() { - let bundle = request - .config_bundle - .as_ref() - .ok_or("pending restore launch material missing")?; - let resolution = self - .observe_bundle_prompt_projection( - bundle, - observation_workspace_id.as_deref(), - ) - .map_err(|_| "pending restore Prompt projection invalid")? - .ok_or("pending restore Prompt projection missing")?; - loader - .clone() - .with_effective_catalog(resolution.projection.catalog.clone()) - } else { - loader.clone() - }; - Worker::restore_pending_from_worker_metadata_with_context( - &worker_name, - manifest.clone(), - store, - pending_loader, - workspace_context, - filesystem_authority, - ) - .await - .map_err(|error| { - self.retain_construction_failure(&request, &error); - restore_worker_cause(&error) - })? - } - Err(error) => { + let candidate = + Worker::prepare_restored_from_worker_metadata_with_context_and_model_client( + &worker_name, + manifest, + store, + loader, + workspace_context, + filesystem_authority, + self.spawn_model_client_factory + .as_ref() + .map(|factory| factory()), + ) + .await + .map_err(|error| { self.retain_construction_failure(&request, &error); - return Err(restore_worker_cause(&error)); - } - }; - let feature_tasks = worker.retain_startup_feature_cleanup(); + restore_worker_cause(&error) + })?; self.failed_restore_resources .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert( - request.worker_ref.clone(), - FailedRestoreResources { - operation_id: request.operation_id, - sessions: vec![worker.workdir_sessions()], - allocation: worker.restore_allocation_cleanup_handle(), - feature_tasks: Some(feature_tasks), - }, - ); + .get_mut(&request.worker_ref) + .expect("retained restore owner") + .candidate_workdir_cleanup = Some(candidate.workdir_cleanup()); #[cfg(test)] { let gate = self.restore_faults.factory_gate.lock().unwrap().take(); @@ -2209,126 +2260,16 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { #[cfg(test)] self.restore_checkpoint(stage) .map_err(|_| "injected restore boundary failure")?; - if let Some(binding) = request.request.backend_job.as_ref() { - worker - .bind_backend_job(binding.clone()) - .map_err(|_| "restored Backend Job binding failed")?; - } - validate_worker_memory_settings(worker.manifest(), &request.request) + validate_worker_memory_settings(candidate.manifest(), &request.request) .map_err(|_| "saved memory settings mismatch")?; - worker - .finalize_subjektiv_session_attribution( - request.request.subjektiv_attached, - SubjektivSessionAttributionLifecycle::RestoredSession, - ) - .map_err(|error| restore_worker_cause(&error))?; - let flow_transition_enabled = worker.manifest().feature.flow.enabled; - let workdir_sessions = restored_workdir_router( - &request.workdir_attachments, - &request.logical_workdir_attachments, - &request.deferred_workdir_sessions, - &request.request.workdir_attachments, - worker.workspace_client_handle(), - ) - .map_err(|_| "restored Workdir binding failed")?; - self.failed_restore_resources - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .get_mut(&request.worker_ref) - .expect("retained restore owner") - .sessions - .push(workdir_sessions.clone()); - worker - .workdir_sessions() - .close_all() - .await - .map_err(|_| "initial Workdir close failed")?; - worker.bind_workdir_sessions(workdir_sessions); - if let (Some(runtime_id), Some(workspace_id)) = - (observation_runtime_id, observation_workspace_id.clone()) - && observation_enabled - { - let mut providers: Vec> = vec![Arc::new( - WorkspaceClientWorkerObservationProvider::new(worker.workspace_client_handle()), - )]; - if !observation_grants.is_empty() { - providers.push(Arc::new(RuntimeGrantedWorkerObservationProvider { - runtime_id, - workspace_id, - grants: observation_grants.into_iter().take(100).collect(), - hub: self.observation_hub.clone(), - })); - } - worker.bind_worker_observation_provider(Some(Arc::new( - CompositeWorkerObservationProvider::new(providers), - ))); - } - stage = RuntimeRestoreStage::Feature; - #[cfg(test)] - self.restore_checkpoint(stage) - .map_err(|_| "injected restore boundary failure")?; - if flow_transition_enabled { - let report = worker - .install_runtime_flow_transition_feature() - .map_err(|_| "Flow transition installation failed")?; - if report.reports.iter().any(|report| !report.installed) { - return Err("Flow transition installation refused"); - } - } - - stage = RuntimeRestoreStage::Controller; - #[cfg(test)] - self.restore_checkpoint(stage) - .map_err(|_| "injected restore boundary failure")?; - let workspace_client = worker.workspace_client_handle(); - let run_dir = self - .worker_restore_run_dir(&request.worker_ref, request.operation_id) - .map_err(|_| "restore run directory unavailable")?; - let transport = self.controller_transport; - #[cfg(test)] - let transport = if self.restore_faults.socket_failure.load(Ordering::SeqCst) { - WorkerControllerTransport::UnixSocket - } else { - transport - }; - let started = PreparedWorker::new( - worker, - WorkerBootstrapLayout::RuntimeManagedRun { - run_dir: run_dir.clone(), - }, - transport, - ) - .start() - .await - .map_err(|error| match error { - WorkerBootstrapError::Worker(error) => restore_worker_cause(&error), - WorkerBootstrapError::Controller { source, .. } => { - io_kind = Some(source.kind()); - "restored Worker controller startup failed" - } - })?; - self.failed_restore_resources - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(&request.worker_ref); - let (handle, shutdown_rx, controller_task) = - (started.handle, started.shutdown, started.controller_task); - if flow_transition_enabled { - handle.shared_state.enable_flow_transition(); - } - Ok(RuntimeWorkerController { - handle, - shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), - controller_task, - workspace_client, - }) + Ok(Box::new(ProfileRestoreCandidate { + prepared: Some(candidate), + factory: self.clone(), + request: request.clone(), + }) as Box) } .await; - result.map_err(|cause| { - let mut failure = RuntimeRestoreFailure::started(stage, cause); - failure.io_kind = io_kind; - failure - }) + result.map_err(|cause| RuntimeRestoreFailure::started(stage, cause)) } async fn cleanup_failed_restore( @@ -2364,6 +2305,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .await .map_err(|_| RuntimeRestoreCleanupFailure::Feature)?; } + if let Some(cleanup) = resources.candidate_workdir_cleanup.as_ref() { + cleanup + .close() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; + } for sessions in &resources.sessions { sessions .close_all() @@ -2434,6 +2381,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .await .map_err(|_| "failed restore Feature cleanup".to_string())?; } + if let Some(cleanup) = resources.candidate_workdir_cleanup.as_ref() { + cleanup + .close() + .await + .map_err(|_| "failed candidate Workdir cleanup".to_string())?; + } for sessions in &resources.sessions { sessions .close_all() @@ -2472,6 +2425,216 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { } } +type ProfileRestoreStore = CombinedStore; + +struct ProfileRestoreCandidate { + prepared: Option>, + factory: ProfileRuntimeWorkerFactory, + request: WorkerExecutionRestoreRequest, +} + +#[async_trait] +impl RuntimeRestoreCandidate for ProfileRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + protocol::WorkerStateSnapshot::initial() + } + + async fn activate(&mut self) -> Result { + let prepared = self.prepared.take().ok_or_else(|| { + RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "restore candidate already consumed", + ) + })?; + let worker = prepared.activate().await.map_err(|error| { + self.factory + .retain_construction_failure(&self.request, &error); + let mut failure = RuntimeRestoreFailure::started( + RuntimeRestoreStage::Reconstruction, + restore_worker_cause(&error), + ); + failure.durable_effects_possible = + matches!(error, WorkerError::RestoreActivationEffects { .. }); + failure + })?; + self.factory + .activate_prepared_restore(worker, &self.request) + .await + .map_err(|mut failure| { + failure.durable_effects_possible = true; + failure + }) + } + + async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure> { + if let Some(prepared) = self.prepared.as_mut() { + prepared + .discard() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; + } + self.prepared = None; + self.factory.cleanup_failed_restore(&self.request).await + } +} + +impl ProfileRuntimeWorkerFactory { + async fn activate_prepared_restore( + &self, + mut worker: Worker< + Box, + CombinedStore, + >, + request: &WorkerExecutionRestoreRequest, + ) -> Result { + let mut stage = RuntimeRestoreStage::Binding; + let mut io_kind = None; + let result: Result = async { + let feature_tasks = worker.retain_startup_feature_cleanup(); + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: vec![worker.workdir_sessions()], + candidate_workdir_cleanup: None, + allocation: worker.restore_allocation_cleanup_handle(), + feature_tasks: Some(feature_tasks), + }, + ); + let observation_runtime_id = self.runtime_id.clone(); + let observation_workspace_id = request + .request + .workspace_api + .as_ref() + .map(|api| api.workspace_id.clone()); + let observation_grants = request.request.worker_observation_grants.clone(); + let observation_enabled = request.request.worker_observation_enabled; + if let Some(binding) = request.request.backend_job.as_ref() { + worker + .bind_backend_job(binding.clone()) + .map_err(|_| "restored Backend Job binding failed")?; + } + validate_worker_memory_settings(worker.manifest(), &request.request) + .map_err(|_| "saved memory settings mismatch")?; + let workdir_sessions = restored_workdir_router( + &request.workdir_attachments, + &request.logical_workdir_attachments, + &request.deferred_workdir_sessions, + &request.request.workdir_attachments, + worker.workspace_client_handle(), + ) + .map_err(|_| "restored Workdir binding failed")?; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get_mut(&request.worker_ref) + .expect("retained restore owner") + .sessions + .push(workdir_sessions.clone()); + worker + .workdir_sessions() + .close_all() + .await + .map_err(|_| "initial Workdir close failed")?; + worker.bind_workdir_sessions(workdir_sessions); + if let (Some(runtime_id), Some(workspace_id)) = + (observation_runtime_id, observation_workspace_id.clone()) + && observation_enabled + { + let mut providers: Vec> = vec![Arc::new( + WorkspaceClientWorkerObservationProvider::new(worker.workspace_client_handle()), + )]; + if !observation_grants.is_empty() { + providers.push(Arc::new(RuntimeGrantedWorkerObservationProvider { + runtime_id, + workspace_id, + grants: observation_grants.into_iter().take(100).collect(), + hub: self.observation_hub.clone(), + })); + } + worker.bind_worker_observation_provider(Some(Arc::new( + CompositeWorkerObservationProvider::new(providers), + ))); + } + worker + .finalize_subjektiv_session_attribution( + request.request.subjektiv_attached, + SubjektivSessionAttributionLifecycle::RestoredSession, + ) + .map_err(|error| restore_worker_cause(&error))?; + let flow_transition_enabled = worker.manifest().feature.flow.enabled; + stage = RuntimeRestoreStage::Feature; + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + if flow_transition_enabled { + let report = worker + .install_runtime_flow_transition_feature() + .map_err(|_| "Flow transition installation failed")?; + if report.reports.iter().any(|report| !report.installed) { + return Err("Flow transition installation refused"); + } + } + + stage = RuntimeRestoreStage::Controller; + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + let workspace_client = worker.workspace_client_handle(); + let run_dir = self + .worker_restore_run_dir(&request.worker_ref, request.operation_id) + .map_err(|_| "restore run directory unavailable")?; + let transport = self.controller_transport; + #[cfg(test)] + let transport = if self.restore_faults.socket_failure.load(Ordering::SeqCst) { + WorkerControllerTransport::UnixSocket + } else { + transport + }; + let started = PreparedWorker::new( + worker, + WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: run_dir.clone(), + }, + transport, + ) + .start() + .await + .map_err(|error| match error { + WorkerBootstrapError::Worker(error) => restore_worker_cause(&error), + WorkerBootstrapError::Controller { source, .. } => { + io_kind = Some(source.kind()); + "restored Worker controller startup failed" + } + })?; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&request.worker_ref); + let (handle, shutdown_rx, controller_task) = + (started.handle, started.shutdown, started.controller_task); + if flow_transition_enabled { + handle.shared_state.enable_flow_transition(); + } + Ok(RuntimeWorkerController { + handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), + controller_task, + workspace_client, + }) + } + .await; + result.map_err(|cause| { + let mut failure = RuntimeRestoreFailure::started(stage, cause); + failure.io_kind = io_kind; + failure + }) + } +} + #[derive(Clone)] struct RuntimeExecutionTaskScope { tasks: Arc>>>, @@ -2788,7 +2951,11 @@ struct RuntimeWorkerExecution { struct PendingRuntimeRestore { request: WorkerExecutionRestoreRequest, task: tokio::sync::Mutex>>, - result: Mutex>>, + result: Mutex, RuntimeRestoreFailure>>>, + activated: Mutex>>, + activation_started: AtomicBool, + discard_started: AtomicBool, + discard_result: Mutex>>, failure: Mutex>, } @@ -3081,6 +3248,7 @@ where fn retry_pending_restore_cleanup( &self, pending: &Arc, + resume_activation: bool, ) -> Result<(), String> { let failure = pending .result @@ -3088,7 +3256,12 @@ where .map_err(|_| "restore result lock is poisoned")? .as_ref() .and_then(|result| result.as_ref().err()) - .filter(|failure| failure.needs_cleanup()) + .filter(|failure| { + failure.needs_cleanup() + || (resume_activation + && (failure.durable_effects_possible + || pending.activation_started.load(Ordering::Acquire))) + }) .cloned(); if let Some(failure) = failure { let owner = Arc::clone(pending); @@ -3101,8 +3274,43 @@ where return Err("restore task has not been joined".into()); } *task = Some(self.spawn_on_adapter_runtime(async move { + if resume_activation + && !failure.needs_cleanup() + && (failure.durable_effects_possible + || owner.activation_started.load(Ordering::Acquire)) + { + let result = + match prepare_restore_operation(factory.as_ref(), owner.request.clone()) + .await + { + Ok(candidate) => Ok(candidate), + Err(failure) => Err(finish_restore_failure( + factory.as_ref(), + &owner.request, + failure, + ) + .await), + }; + owner.activation_started.store(false, Ordering::Release); + *owner + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = None; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(result); + return; + } let failure = finish_restore_failure(factory.as_ref(), &owner.request, failure).await; + if owner.activation_started.load(Ordering::Acquire) { + *owner + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure.clone())); + } *owner .result .lock() @@ -3794,64 +4002,148 @@ where return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } if let Err(message) = self - .retry_pending_restore_cleanup(&pending) + .retry_pending_restore_cleanup(&pending, false) .and_then(|()| self.wait_pending_restore(&pending, Duration::from_secs(5))) { return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } - if self.workers.lock().is_err() { + if let Some(Err(failure)) = pending + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + && failure.needs_cleanup() + { return WorkerExecutionResult::errored( WorkerExecutionOperation::Stop, - "worker adapter registry lock is poisoned", + failure.to_string(), ); } - let mut retained = match pending.result.lock() { - Ok(retained) => retained, - Err(_) => { - return WorkerExecutionResult::errored( - WorkerExecutionOperation::Stop, - "restore result lock is poisoned", - ); - } - }; - match retained.as_ref() { - Some(Ok(_)) => { - let controller = match retained.take().unwrap() { - Ok(controller) => controller, - Err(_) => unreachable!(), - }; - let state = Arc::new(RwLock::new(controller.handle.shared_state.snapshot())); - self.retain_uncertain_unconnected_controller( - worker_ref, - controller.handle, - controller.shutdown, - Arc::new(AtomicBool::new(false)), - RuntimeExecutionTaskScope::new(controller.controller_task), - state, - Some(controller.workspace_client), - Some(pending.request.operation_id), - Vec::new(), - ); - } - Some(Err(failure)) => { - if !matches!( - failure.cleanup, - RuntimeRestoreCleanup::NotNeeded | RuntimeRestoreCleanup::Confirmed - ) { + if pending.activation_started.load(Ordering::Acquire) { + let mut activated = match pending.activated.lock() { + Ok(result) => result, + Err(_) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "restore activation result lock is poisoned", + ); + } + }; + match activated.as_ref() { + Some(Ok(_)) => { + let controller = + activated.take().unwrap().unwrap_or_else(|_| unreachable!()); + let state = + Arc::new(RwLock::new(controller.handle.shared_state.snapshot())); + self.retain_uncertain_unconnected_controller( + worker_ref, + controller.handle, + controller.shutdown, + Arc::new(AtomicBool::new(false)), + RuntimeExecutionTaskScope::new(controller.controller_task), + state, + Some(controller.workspace_client), + Some(pending.request.operation_id), + Vec::new(), + ); + } + Some(Err(failure)) + if matches!( + failure.cleanup, + RuntimeRestoreCleanup::Confirmed | RuntimeRestoreCleanup::NotNeeded + ) => {} + Some(Err(failure)) => { return WorkerExecutionResult::errored( WorkerExecutionOperation::Stop, failure.to_string(), ); } + None => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "restore activation ownership is unproven", + ); + } } - None => { - return WorkerExecutionResult::errored( - WorkerExecutionOperation::Stop, - "restore factory has no completion evidence; cleanup is unproven", - ); + } else { + if !pending.discard_started.swap(true, Ordering::AcqRel) { + let owner = pending.clone(); + let task = match self.spawn_on_adapter_runtime(async move { + let result = owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let cleanup = match result { + Some(Ok(mut candidate)) => { + let result = candidate.discard().await; + if result.is_err() { + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Ok(candidate)); + } + result + } + Some(Err(failure)) => { + let result = if matches!( + failure.cleanup, + RuntimeRestoreCleanup::Confirmed + | RuntimeRestoreCleanup::NotNeeded + ) { + Ok(()) + } else { + Err(RuntimeRestoreCleanupFailure::Ownership) + }; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure)); + result + } + None => Err(RuntimeRestoreCleanupFailure::Ownership), + }; + *owner + .discard_result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(cleanup); + }) { + Ok(task) => task, + Err(message) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + message, + ); + } + }; + *pending.task.try_lock().expect("joined candidate task") = Some(task); + } + if let Err(message) = self.wait_pending_restore(&pending, Duration::from_secs(5)) { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + let cleanup = *pending + .discard_result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match cleanup { + Some(Ok(())) => {} + Some(Err(error)) => { + pending.discard_started.store(false, Ordering::Release); + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + error.to_string(), + ); + } + None => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "candidate discard has no completion evidence", + ); + } } } - drop(retained); if let Err(message) = self.remove_pending_restore(worker_ref) { return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } @@ -4729,9 +5021,10 @@ where } drop(workers); - // An already accepted operation owns its bindings. Re-observing mutable - // Workdir materialization must not turn an unknown operation into a new - // read-only refusal or allocate another candidate. + // The operation retains its resources across timeouts, but its saved + // bindings never override a later authoritative attachment/owner change. + // Compare before substituting the owned request or retrying any effects; + // drift retains the same owner for reconciliation/Stop, not a new intent. let admitted_pending = match self.pending_restore(&request.worker_ref) { Ok(Some(pending)) if pending.request.operation_id == request.operation_id => { Some(pending) @@ -4759,6 +5052,22 @@ where } }; if let Some(pending) = admitted_pending.as_ref() { + if request.request != pending.request.request + || request.workspace_scope != pending.request.workspace_scope + || request.logical_workdir_attachments + != pending.request.logical_workdir_attachments + || request.previous_workdir_attachments + != pending.request.previous_workdir_attachments + { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "current Worker authority differs from the owned restore candidate", + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } request = pending.request.clone(); } else { if let Err(result) = self.preflight_restore_inputs(&request) { @@ -4806,6 +5115,10 @@ where request: request.clone(), task: tokio::sync::Mutex::new(None), result: Mutex::new(None), + activated: Mutex::new(None), + activation_started: AtomicBool::new(false), + discard_started: AtomicBool::new(false), + discard_result: Mutex::new(None), failure: Mutex::new(None), }); let mut resources = match self.pending_restores.lock() { @@ -4823,7 +5136,9 @@ where let factory = Arc::clone(&self.factory); let result_owner = Arc::clone(&pending); let task = self.spawn_on_adapter_runtime(async move { - let result = match factory.restore_controller(request.clone()).await { + let result = match prepare_restore_operation(factory.as_ref(), request.clone()) + .await + { Ok(controller) => Ok(controller), Err(failure) => { *result_owner @@ -4877,7 +5192,7 @@ where } if existing_pending { if let Err(message) = self - .retry_pending_restore_cleanup(&pending) + .retry_pending_restore_cleanup(&pending, true) .and_then(|()| self.wait_pending_restore(&pending, self.spawn_restore_timeout)) { return WorkerExecutionSpawnResult::ReconciliationRequired { @@ -4890,8 +5205,30 @@ where }; } } - let controller = { - let mut retained = match pending.result.lock() { + if pending.activation_started.load(Ordering::Acquire) { + let activated = pending + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(Ok(controller)) = activated.as_ref() { + return WorkerExecutionSpawnResult::Connected { + worker_state: controller.handle.shared_state.snapshot(), + workdir_attachments: pending.request.previous_workdir_attachments.clone(), + }; + } + let message = match activated.as_ref() { + Some(Err(failure)) => failure.to_string(), + _ => "restore activation is owned by the same operation; reconcile activation" + .to_string(), + }; + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message), + worker_state: None, + workdir_attachments: pending.request.previous_workdir_attachments.clone(), + }; + } + let worker_state = { + let retained = match pending.result.lock() { Ok(retained) => retained, Err(_) => { return WorkerExecutionSpawnResult::ReconciliationRequired { @@ -4905,16 +5242,23 @@ where } }; match retained.as_ref() { - Some(Ok(_)) => match retained.take().unwrap() { - Ok(controller) => controller, - Err(_) => unreachable!(), - }, + Some(Ok(candidate)) => candidate.initial_state(), Some(Err(failure)) => { let result = WorkerExecutionResult::errored( WorkerExecutionOperation::Restore, failure.to_string(), ); let cleanup = failure.cleanup; + if failure.durable_effects_possible { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result, + worker_state: None, + workdir_attachments: pending + .request + .previous_workdir_attachments + .clone(), + }; + } if matches!( cleanup, RuntimeRestoreCleanup::NotNeeded | RuntimeRestoreCleanup::Confirmed @@ -4959,25 +5303,10 @@ where } }; - let result = self.connect_handle( - WorkerExecutionOperation::Restore, - worker_ref.clone(), - pending.request.context.clone(), - controller.handle, - controller.shutdown, - controller.controller_task, - Some(operation_id), - pending.request.previous_workdir_attachments.clone(), - Some(controller.workspace_client), - ); - if let Err(message) = self.remove_pending_restore(&worker_ref) { - return WorkerExecutionSpawnResult::ReconciliationRequired { - result: WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message), - worker_state: None, - workdir_attachments: Vec::new(), - }; + WorkerExecutionSpawnResult::Connected { + worker_state, + workdir_attachments: pending.request.previous_workdir_attachments.clone(), } - result } fn activate_restored_worker( @@ -4996,15 +5325,109 @@ where return Err(message); } }; + if let Some(pending) = self.pending_restore(worker_ref)? { + if pending.request.operation_id != operation_id { + return Err("restored Worker operation identity does not match".into()); + } + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + if !pending.activation_started.swap(true, Ordering::AcqRel) { + let owner = pending.clone(); + let factory = self.factory.clone(); + let task = self.spawn_on_adapter_runtime(async move { + let candidate = owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let result = match candidate { + Some(Ok(mut candidate)) => { + let result = candidate.activate().await; + match result { + Ok(controller) => Ok(controller), + Err(mut failure) => { + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure.clone())); + match candidate.discard().await { + Ok(()) => { + failure.cleanup = RuntimeRestoreCleanup::Confirmed + } + Err(error) => { + failure.cleanup_failure.get_or_insert(error); + failure.cleanup = RuntimeRestoreCleanup::Failed(error); + } + } + let failure = finish_restore_failure( + factory.as_ref(), + &owner.request, + failure, + ) + .await; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure.clone())); + Err(failure) + } + } + } + Some(Err(failure)) => Err(failure), + None => Err(RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "restore candidate ownership unavailable", + )), + }; + *owner + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(result); + })?; + *pending + .task + .try_lock() + .map_err(|_| "restore activation task is already owned")? = Some(task); + } + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + let controller = { + let mut result = pending + .activated + .lock() + .map_err(|_| "restore activation result lock is poisoned")?; + match result.as_ref() { + Some(Ok(_)) => result.take().unwrap().map_err(|error| error.to_string())?, + Some(Err(failure)) => return Err(failure.to_string()), + None => return Err("restore activation has no completion evidence".into()), + } + }; + match self.connect_handle( + WorkerExecutionOperation::Restore, + worker_ref.clone(), + pending.request.context.clone(), + controller.handle, + controller.shutdown, + controller.controller_task, + Some(operation_id), + pending.request.previous_workdir_attachments.clone(), + Some(controller.workspace_client), + ) { + WorkerExecutionSpawnResult::Connected { .. } => { + self.remove_pending_restore(worker_ref)? + } + other => return Err(format!("restore activation connection failed: {other:?}")), + } + } let workers = self .workers .lock() - .map_err(|_| "worker adapter registry lock is poisoned".to_string())?; + .map_err(|_| "worker adapter registry lock is poisoned")?; let execution = workers .get(worker_ref) - .ok_or_else(|| "restored Worker execution is not registered".to_string())?; + .ok_or("restored Worker execution is not registered")?; if execution.restore_operation_id != Some(operation_id) { - return Err("restored Worker operation identity does not match".to_string()); + return Err("restored Worker operation identity does not match".into()); } self.factory.activate_restored_controller( worker_ref, @@ -5702,6 +6125,7 @@ mod tests { root: &Path, materialize_head: bool, ) -> WorkerExecutionRestoreRequest { + require_restore_test_environment(); let worker_ref = WorkerRef::new(WorkerId::now_v7()); let worker_name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref); let aggregate_dir = root.join("workers").join(worker_ref.worker_id.to_string()); @@ -5901,10 +6325,17 @@ mod tests { }) .unwrap(); backend.preflight_restore(&request).unwrap(); - let expected = match backend.restore_worker(request.clone()) { + let _prepared = match backend.restore_worker(request.clone()) { WorkerExecutionSpawnResult::Connected { worker_state, .. } => worker_state, other => panic!("real Profile backend restore failed: {other:?}"), }; + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); + let expected = backend.workers.lock().unwrap()[&request.worker_ref] + .handle + .shared_state + .snapshot(); assert!(matches!( backend.worker_session(WorkerSessionObservationRequest { worker_ref: request.worker_ref.clone(), @@ -6035,6 +6466,9 @@ mod tests { backend.restore_worker(request.clone()), WorkerExecutionSpawnResult::Connected { .. } )); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); let mut transport_a = Arc::clone(&backend) .attach_worker_protocol(&request.worker_ref) .unwrap(); @@ -6072,6 +6506,9 @@ mod tests { backend.restore_worker(request.clone()), WorkerExecutionSpawnResult::Connected { .. } )); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); let current_b = backend .workers .lock() @@ -6166,6 +6603,9 @@ mod tests { backend.restore_worker(request.clone()), WorkerExecutionSpawnResult::Connected { .. } )); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); let execution = backend.workers.lock().unwrap()[&request.worker_ref].clone(); let mut transport = Arc::clone(&backend) .attach_worker_protocol(&request.worker_ref) @@ -6227,6 +6667,9 @@ mod tests { backend.restore_worker(request.clone()), WorkerExecutionSpawnResult::Connected { .. } )); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); (backend, request) } @@ -6748,6 +7191,10 @@ mod tests { request: direct_restore_request(worker_ref.clone()), task: tokio::sync::Mutex::new(None), result: Mutex::new(None), + activated: Mutex::new(None), + activation_started: AtomicBool::new(false), + discard_started: AtomicBool::new(false), + discard_result: Mutex::new(None), failure: Mutex::new(Some("task completion unknown".into())), }), ); @@ -6787,6 +7234,10 @@ mod tests { request: request.clone(), task: tokio::sync::Mutex::new(Some(failed)), result: Mutex::new(None), + activated: Mutex::new(None), + activation_started: AtomicBool::new(false), + discard_started: AtomicBool::new(false), + discard_result: Mutex::new(None), failure: Mutex::new(None), }); backend @@ -7072,6 +7523,9 @@ mod tests { matches!(restored, WorkerExecutionSpawnResult::Connected { .. }), "{restored:?}" ); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); let session = backend.workers.lock().unwrap()[&request.worker_ref] .handle .shared_state @@ -8139,6 +8593,23 @@ mod tests { ); } + struct TestRestoreCandidate(Option); + #[async_trait] + impl RuntimeRestoreCandidate for TestRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.0.as_ref().unwrap().handle.shared_state.snapshot() + } + async fn activate(&mut self) -> Result { + Ok(self.0.take().unwrap()) + } + async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure> { + if let Some(controller) = self.0.take() { + shutdown_profile_controller(controller).await; + } + Ok(()) + } + } + struct PendingCreateFactory { started: mpsc::Sender<()>, dropped: mpsc::Sender<()>, @@ -8154,6 +8625,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for PendingCreateFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, _request: WorkerExecutionSpawnRequest, @@ -8324,6 +8804,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for MockFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, request: WorkerExecutionSpawnRequest, @@ -8738,6 +9227,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for FailingFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, _request: WorkerExecutionSpawnRequest, @@ -8766,6 +9264,15 @@ mod tests { #[cfg(feature = "fs-store")] #[async_trait] impl RuntimeWorkerFactory for RestartArtifactFailureFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, request: WorkerExecutionSpawnRequest, @@ -8807,7 +9314,9 @@ mod tests { #[cfg(feature = "fs-store")] #[test] fn fs_restart_stop_cleans_prior_restore_artifacts_and_retains_journal_on_failure() { - let root = tempfile::tempdir().unwrap(); + require_restore_test_environment(); + let root = tempfile::tempdir_in(".").unwrap(); + let socket_base = Path::new(root.path().file_name().unwrap()).join("controllers"); let runtime_store = root.path().join("runtime"); let options = crate::fs_store::FsRuntimeStoreOptions { root: runtime_store.clone(), @@ -8819,7 +9328,7 @@ mod tests { WorkerRuntimeExecutionBackend::new(RestartArtifactFailureFactory { inner: MockFactory { client: MockClient::sequential(vec![]), - runtime_base: root.path().join("controllers"), + runtime_base: socket_base, cwd: root.path().to_path_buf(), store_dir: root.path().join("sessions"), worker_metadata_dir: root.path().join("metadata"), @@ -8965,6 +9474,7 @@ mod tests { FailedRestoreResources { operation_id: request.operation_id, sessions: vec![Arc::clone(&sessions)], + candidate_workdir_cleanup: None, allocation: None, feature_tasks: None, }, @@ -9063,6 +9573,7 @@ mod tests { #[cfg(feature = "fs-store")] include!("worker_backend_restore_failure_tests.rs"); + include!("worker_backend_restore_candidate_tests.rs"); struct PendingFailureFactory { restore_calls: Arc, @@ -9072,6 +9583,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for PendingFailureFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, _request: WorkerExecutionSpawnRequest, @@ -9199,6 +9719,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for LateControllerFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, request: WorkerExecutionSpawnRequest, @@ -9812,67 +10341,6 @@ mod tests { assert!(!socket_path.exists()); } - #[test] - fn profile_runtime_factory_uses_shared_worker_bootstrap_seams() { - let source = include_str!("worker_backend.rs"); - let production = source - .split_once("#[cfg(test)]\nmod tests") - .map(|(production, _)| production) - .expect("worker backend test module marker"); - let factory = production - .split_once("impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory") - .map(|(_, factory)| factory) - .expect("profile runtime factory implementation"); - let (fresh, restore) = factory - .split_once("async fn restore_controller") - .expect("fresh and restore factory paths"); - let assert_in_order = |path: &str, markers: &[&str]| { - let mut offset = 0; - for marker in markers { - let relative = path[offset..] - .find(marker) - .unwrap_or_else(|| panic!("missing ordered factory marker {marker}")); - offset += relative + marker.len(); - } - }; - assert_in_order( - fresh, - &[ - "WorkerBootstrap::new(", - ".prepare()", - "finalize_subjektiv_session_attribution(", - "worker.bind_workdir_sessions(", - "worker.bind_worker_observation_provider(", - "install_runtime_flow_transition_feature()", - "prepared.start()", - ], - ); - assert_in_order( - restore, - &[ - "Worker::restore_from_worker_metadata_with_context(", - "finalize_subjektiv_session_attribution(", - "worker.bind_workdir_sessions(", - "worker.bind_worker_observation_provider(", - "install_runtime_flow_transition_feature()", - "PreparedWorker::new(", - ".start()", - ], - ); - assert!( - production.contains("WorkerBootstrap::new("), - "fresh runtime Workers must use the shared construction bootstrap" - ); - assert!( - production.contains("PreparedWorker::new("), - "restored runtime Workers must use the shared pre-exposure lifecycle" - ); - assert!( - !production.contains("WorkerController::spawn_runtime_managed_run_with_transport"), - "runtime factory paths must not bypass the shared controller lifecycle" - ); - } - #[test] #[serial_test::serial(worker_allocation)] fn shared_bootstrap_preserves_in_process_transport_for_fresh_and_restored_runtime_workers() { @@ -10018,6 +10486,9 @@ mod tests { factory_calls_after_restore, "reconciliation must not construct a second controller" ); + backend + .activate_restored_worker(operation_id, &worker.worker_ref) + .unwrap(); let workers = backend.workers.lock().unwrap(); let execution = workers.get(&worker.worker_ref).unwrap(); assert_eq!(execution.restore_operation_id, Some(operation_id)); diff --git a/crates/worker-runtime/src/worker_backend/restore_recovery.rs b/crates/worker-runtime/src/worker_backend/restore_recovery.rs new file mode 100644 index 000000000..8dd47ee15 --- /dev/null +++ b/crates/worker-runtime/src/worker_backend/restore_recovery.rs @@ -0,0 +1,173 @@ +//! Explicit, operation-owned repair of saved Session material. This is not +//! read-only candidate preparation: the canonical Session/attribution is the +//! receipt for replay after a lost response, and effects are reported as such. + +use super::*; + +impl ProfileRuntimeWorkerFactory { + pub(super) async fn recover_saved_restore_source( + &self, + request: &WorkerExecutionRestoreRequest, + ) -> Result { + let cause = |cause| RuntimeRestoreFailure::started(RuntimeRestoreStage::Recovery, cause); + let name = Self::runtime_worker_name_for_ref(&request.worker_ref); + let root = self + .worker_aggregate_dir(&request.worker_ref) + .map_err(|_| cause("restore recovery aggregate unavailable"))?; + let metadata_store = WorkerAggregateStore::open_read_only(&root, &name) + .map_err(|_| cause("restore recovery metadata open failed"))?; + let metadata = metadata_store + .read_by_name(&name) + .map_err(|_| cause("restore recovery metadata read failed"))? + .ok_or_else(|| cause("restore recovery metadata missing"))?; + let active = metadata + .active + .as_ref() + .ok_or_else(|| cause("restore recovery Session missing"))?; + let pending = active.segment_id.is_none(); + let incomplete = request.request.subjektiv_attached + && !matches!( + metadata.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Confirmed { .. }) + ); + if !pending && !incomplete { + return Ok(false); + } + // The accepted Runtime operation already owns this Session. Its stable + // Session identity and sole committed seed, not a new restore request, + // are used to reconcile an interrupted first-head publication. + let session_store = WorkerSessionStore::new_with_retention( + root.join("session"), + self.session_retention() + .map_err(|_| cause("restore recovery retention unavailable"))?, + ) + .map_err(|_| cause("restore recovery Session open failed"))?; + let (manifest, mut loader) = Self::manifest_for_restore(&metadata, &request.request) + .map_err(|_| cause("restore recovery saved manifest invalid"))?; + let workspace = RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + self.runtime_id.as_deref(), + ); + let request_client = request + .request + .workspace_api + .as_ref() + .and_then(|api| self.workspace_request_clients.get(&api.workspace_id)); + let context = workspace.worker_context( + &request.worker_ref, + request.workspace_scope.as_ref(), + self.worker_mutation_identity.as_ref(), + request_client, + self.embedded_worker_mutation_dispatcher.as_ref(), + Some(self.prompt_projection_cache.clone()), + ); + if pending && context.workspace_id().is_some() { + let bundle = request + .config_bundle + .as_ref() + .ok_or_else(|| cause("pending recovery launch material missing"))?; + let projection = self + .observe_bundle_prompt_projection( + bundle, + request + .request + .workspace_api + .as_ref() + .map(|api| api.workspace_id.as_str()), + ) + .map_err(|_| cause("pending recovery Prompt projection invalid"))? + .ok_or_else(|| cause("pending recovery Prompt projection missing"))?; + loader = loader.with_effective_catalog(projection.projection.catalog.clone()); + } + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: Vec::new(), + candidate_workdir_cleanup: None, + allocation: None, + feature_tasks: None, + }, + ); + let store = CombinedStore::new(session_store, metadata_store); + let worker_result = Worker::restore_pending_from_worker_metadata_with_context( + &name, + manifest, + store, + loader, + context, + WorkerFilesystemAuthority::None, + ) + .await; + let mut worker = worker_result.map_err(|error| { + self.retain_construction_failure(request, &error); + cause(restore_worker_cause(&error)) + })?; + let tasks = worker.retain_startup_feature_cleanup(); + let sessions = worker.workdir_sessions(); + let allocation = worker.restore_allocation_cleanup_handle(); + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: vec![sessions.clone()], + candidate_workdir_cleanup: None, + allocation: allocation.clone(), + feature_tasks: Some(tasks.clone()), + }, + ); + let result = async { + if let Some(binding) = request.request.backend_job.as_ref() { + worker + .bind_backend_job(binding.clone()) + .map_err(|_| cause("recovery Backend Job binding failed"))?; + } + worker + .finalize_subjektiv_session_attribution( + request.request.subjektiv_attached, + SubjektivSessionAttributionLifecycle::RestoredSession, + ) + .map_err(|error| cause(restore_worker_cause(&error)))?; + if pending { + worker::WorkerController::recover_pending_head(&mut worker, &root) + .await + .map_err(|_| cause("pending Session head recovery failed"))?; + } + Ok::<_, RuntimeRestoreFailure>(()) + } + .await; + // Never release allocation while the Worker guard is attached. Retained + // handles in the same operation remain available on every cleanup error. + drop(worker); + let cleanup = async { + tasks + .shutdown() + .await + .map_err(|_| cause("recovery Feature settlement failed"))?; + sessions + .close_all() + .await + .map_err(|_| cause("recovery Workdir close failed"))?; + if let Some(allocation) = allocation { + allocation + .release() + .map_err(|_| cause("recovery allocation release failed"))?; + } + Ok::<_, RuntimeRestoreFailure>(()) + } + .await; + result?; + cleanup?; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&request.worker_ref); + Ok(true) + } +} diff --git a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs new file mode 100644 index 000000000..a5b0312f9 --- /dev/null +++ b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs @@ -0,0 +1,898 @@ +// Included inside worker_backend::tests, alongside the restore failure suite. +// These integration tests connect the real Profile factory, canonical stores, +// restore owner, and Controller. Only model I/O and the Workspace HTTP transport +// are substituted. The loopback Host is not an authentication/policy test. +// Run in the ordinary worker-runtime lib suite with the default product features +// and the filter `restore_candidate::`. No live service is used. Like the +// failure suite, this requires the existing Yoi path settings to resolve into +// target/restore-test-environment; the common fixture rejects unsafe fallbacks +// before any factory, Git, or allocation-table operation. + +#[cfg(feature = "http-server")] +mod restore_candidate { + use super::*; + + const FIRST: &str = "candidate-saved-first"; + const SECOND: &str = "candidate-saved-second"; + const NOTICE: &str = "candidate-saved-notice"; + const FRESH: &str = "candidate-fresh-input"; + const WAIT: Duration = Duration::from_secs(5); + + struct HostProbe { + calls: Arc>>, + task: Option>, + } + + impl Drop for HostProbe { + fn drop(&mut self) { + if let Some(task) = self.task.take() { + task.abort(); + } + } + } + + // Aborting on unwind prevents a failed setup assertion from leaving its + // hanging model turn alive. Successful setup uses the normal shutdown/join. + struct SeedController(Option); + + impl Drop for SeedController { + fn drop(&mut self) { + if let Some(controller) = self.0.as_ref() { + controller.controller_task.abort(); + } + } + } + + async fn receive_matching( + events: &mut tokio::sync::broadcast::Receiver, + phase: &str, + mut matches: impl FnMut(&Event) -> bool, + ) -> Event { + let mut seen = Vec::new(); + tokio::time::timeout(WAIT, async { + loop { + let event = events.recv().await.expect("Controller event stream"); + seen.push(format!("{event:?}")); + if matches(&event) { + return event; + } + assert!( + !matches!( + event, + Event::Error { .. } + | Event::SubmissionRejected { .. } + | Event::NotificationRejected { .. } + ), + "unexpected Controller rejection: {event:?}" + ); + } + }) + .await + .unwrap_or_else(|_| { + panic!("Controller acknowledgement deadline during {phase}; observed {seen:?}") + }) + } + + async fn submit(handle: &WorkerHandle, id: &str) -> Event { + let mut events = handle.subscribe(); + handle + .send(Method::Submit { + submission_request_id: id.into(), + input: vec![protocol::Segment::Text { content: id.into() }], + }) + .await + .unwrap(); + receive_matching(&mut events, id, |event| { + matches!(event, Event::SubmissionAccepted { submission_request_id, .. } + if submission_request_id == id) + }) + .await + } + + async fn notify(handle: &WorkerHandle) { + let mut events = handle.subscribe(); + handle + .send(Method::Notify { + notification_request_id: NOTICE.into(), + message: NOTICE.into(), + }) + .await + .unwrap(); + receive_matching(&mut events, "notification acceptance", |event| { + matches!(event, Event::NotificationAccepted { notification_request_id } + if notification_request_id == NOTICE) + }) + .await; + } + + async fn pending(handle: &WorkerHandle) -> protocol::PendingSubmissionsSnapshot { + let mut events = handle.subscribe(); + handle.send(Method::ListPendingSubmissions).await.unwrap(); + match receive_matching(&mut events, "pending queue snapshot", |event| { + matches!(event, Event::PendingSubmissionsChanged { .. }) + }) + .await + { + Event::PendingSubmissionsChanged { pending } => pending, + _ => unreachable!(), + } + } + + async fn finished(handle: &WorkerHandle, id: &str) { + let mut events = handle.subscribe(); + let is_finished = |snapshot: &protocol::WorkerStateSnapshot| { + snapshot.last_finished_submission_request_id.as_deref() == Some(id) + && snapshot.state == protocol::WorkerState::Idle + }; + if !is_finished(&handle.shared_state.snapshot()) { + receive_matching( + &mut events, + id, + |event| matches!(event, Event::WorkerState { snapshot } if is_finished(snapshot)), + ) + .await; + } + } + + fn queue_checkpoint(entries: &[LogEntry]) -> serde_json::Value { + entries + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == "worker.pending_activations.v1" => Some(payload.clone()), + _ => None, + }) + .expect("real Controller must have committed its pending activation checkpoint") + } + + struct Fixture { + backend: Arc, + request: WorkerExecutionRestoreRequest, + model: MockClient, + session: WorkerSessionStore, + session_id: session_store::SessionId, + segment_id: session_store::SegmentId, + receipts: Vec, + host: HostProbe, + _root: tempfile::TempDir, + } + + impl Fixture { + fn new() -> Self { + Self::with_transport(WorkerControllerTransport::InProcess) + } + + fn with_transport(transport: WorkerControllerTransport) -> Self { + let root = restore_failure_tempdir(); + let model = MockClient::sequential(vec![ + MockResponse::Complete(vec![ + LlmEvent::tool_use_start(0, "candidate-task-probe", "TaskCreate"), + LlmEvent::tool_input_delta( + 0, + r#"{"subject":"Activated candidate","description":"Execution probe"}"#, + ), + LlmEvent::tool_use_stop(0), + LlmEvent::Status(StatusEvent { + status: ResponseStatus::Completed, + }), + ]), + MockResponse::Complete(simple_text_events()), + MockResponse::Complete(simple_text_events()), + MockResponse::Complete(simple_text_events()), + ]); + let injected = model.clone(); + let mut factory = ProfileRuntimeWorkerFactory::new(root.path()) + .with_runtime_store_dir(root.path()) + .with_runtime_id("candidate-runtime") + .with_spawn_model_client_factory(move || Box::new(injected.clone())); + factory.controller_transport = transport; + let catalog = worker::PromptCatalog::builtins_only() + .unwrap() + .projection() + .clone(); + factory + .observe_workspace_prompt_projection( + worker::WorkspacePromptProjection::new( + "workspace-saved-subjektiv", + "candidate-fixture", + catalog.catalog_digest.clone(), + catalog, + ) + .unwrap(), + ) + .unwrap(); + let backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap()); + let fixture_root = root.path().to_path_buf(); + let (request, session, session_id, segment_id, receipts, host) = backend + .run_on_adapter_runtime(async move { + let calls = Arc::new(Mutex::new(Vec::new())); + let authorization_calls = calls.clone(); + let other_calls = calls.clone(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let authorization_path = + "/api/w/workspace-saved-subjektiv/authorization/check"; + let app = axum::Router::new() + .route( + authorization_path, + axum::routing::post(move |axum::Json(input): axum::Json< + server_api::authorization::OperationAuthorizationRequest, + >| { + let calls = authorization_calls.clone(); + async move { + calls.lock().unwrap().push(( + "POST".into(), authorization_path.into(), + )); + let response = if input.operation == "task" + && input.resource == server_api::authorization::PermissionResource::Feature + { + server_api::authorization::OperationAuthorizationResponse { + scope: Some(server_api::authorization::PermissionResource::Feature), + } + } else { + saved_subjektiv_authorization_fixture(input) + .expect("only the saved fixture Feature grants are available") + }; + axum::Json(response) + } + }), + ) + .fallback(move |request: axum::extract::Request| { + let calls = other_calls.clone(); + async move { + calls.lock().unwrap().push(( + request.method().to_string(), request.uri().path().to_owned(), + )); + axum::http::StatusCode::FORBIDDEN + } + }); + let host = HostProbe { + calls, + task: Some(tokio::spawn(async move { axum::serve(listener, app).await.unwrap() })), + }; + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let (mut manifest, mut create) = + saved_subjektiv_policy(&fixture_root, &worker_ref); + // Normal restore deliberately excludes Subject repair. Keep + // a real Workspace client so accidental Host effects are visible. + manifest.feature.subjektiv = manifest::ResolvedSubjektivFeatureConfig::default(); + manifest.feature.task.enabled = true; + create.subjektiv_attached = false; + create.workspace_api.as_mut().unwrap().base_url = base_url; + let aggregate = fixture_root.join("workers").join(worker_ref.worker_id.to_string()); + let session = WorkerSessionStore::new(aggregate.join("session")).unwrap(); + let metadata = WorkerAggregateStore::new(&aggregate, &manifest.worker.name).unwrap(); + let mut engine = Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated( + MockClient::sequential(vec![MockResponse::Hang(simple_text_events().into_iter().take(2).collect())]), + ); + engine.set_system_prompt("Committed restore candidate fixture prompt"); + let mut worker = Worker::new( + manifest.clone(), + engine, + CombinedStore::new(session.clone(), metadata.clone()), + RuntimeWorkspaceBackendRef::from_worker_request(&create, Some("candidate-runtime")) + .worker_context(&worker_ref, Some(&crate::runtime::RuntimeWorkspaceScope::new( + "workspace-saved-subjektiv", "server-main", + )), None, None, None, None), + WorkerFilesystemAuthority::None, + Scope::empty(), + ).await.unwrap(); + worker.enable_worker_metadata_write_through().unwrap(); + worker.materialize_durable_session_head().await.unwrap(); + let (session_id, segment_id) = (worker.session_id(), worker.segment_id()); + let workspace_client = worker.workspace_client_handle(); + let started = PreparedWorker::new( + worker, + WorkerBootstrapLayout::RuntimeManagedRun { run_dir: fixture_root.join("seed") }, + WorkerControllerTransport::InProcess, + ).start().await.unwrap(); + let mut seed = SeedController(Some(RuntimeWorkerController { + handle: started.handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), + controller_task: started.controller_task, + workspace_client, + })); + let handle = seed.0.as_ref().unwrap().handle.clone(); + let mut started_events = handle.subscribe(); + submit(&handle, "candidate-setup-turn").await; + // Running and streaming observations can arrive in either + // order. Do not discard TextDelta while awaiting WorkerState. + let mut running = handle.shared_state.catalog_status() == WorkerStatus::Running; + let mut streaming = false; + receive_matching(&mut started_events, "seed running and streaming", |event| { + if let Event::WorkerState { snapshot } = event { + running |= snapshot.catalog_status() == protocol::WorkerStatus::Running; + } + streaming |= matches!(event, Event::TextDelta { .. }); + running && streaming + }).await; + let state = Arc::new(RwLock::new(handle.shared_state.snapshot())); + let mut events = handle.subscribe(); + handle.send(Method::Pause { command: next_internal_command(&state).unwrap() }) + .await.unwrap(); + receive_matching(&mut events, "seed pause acknowledgement", |event| { + matches!(event, Event::WorkerState { snapshot } + if snapshot.catalog_status() == protocol::WorkerStatus::Paused) + }).await; + let mut receipts = Vec::new(); + for id in [FIRST, SECOND] { + let receipt = submit(&handle, id).await; + assert!(matches!(receipt, Event::SubmissionAccepted { + disposition: protocol::SubmissionDisposition::Queued, .. + })); + receipts.push(serde_json::to_value(receipt).unwrap()); + if id == FIRST { notify(&handle).await; } + } + assert_eq!(pending(&handle).await.submissions.len(), 2); + shutdown_profile_controller(seed.0.take().unwrap()).await; + // This orphan pin would be removed by the normal activation + // reconciliation. Preparation and discard must leave it alone. + let orphan = session.write_uploaded_file( + session_id, "orphan.txt", "text/plain", b"retained orphan", + session_store::UploadedFileLimits::default(), + ).unwrap(); + session.pin_uploaded_file(session_id, &orphan, "uncommitted-acceptance").unwrap(); + let mut saved = metadata.read_by_name(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref)) + .unwrap().unwrap(); + saved.resolved_manifest_snapshot = Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap()); + saved.workspace_root = Some(fixture_root.join("stale-derived-root")); + metadata.write(&saved).unwrap(); + let mut request = direct_restore_request(worker_ref); + request.request = create; + request.workspace_scope = Some(crate::runtime::RuntimeWorkspaceScope::new( + "workspace-saved-subjektiv", "server-main", + )); + Ok((request, session, session_id, segment_id, receipts, host)) + }) + .unwrap(); + let fixture = Self { + backend, + request, + model, + session, + session_id, + segment_id, + receipts, + host, + _root: root, + }; + let checkpoint = queue_checkpoint(&fixture.entries()); + assert_eq!(checkpoint["pending"].as_array().unwrap().len(), 2); + assert_eq!( + checkpoint["pending_notifications"] + .as_array() + .unwrap() + .len(), + 1 + ); + fixture.assert_quiet(); + fixture + } + + fn entries(&self) -> Vec { + self.session + .read_all(self.session_id, self.segment_id) + .unwrap() + } + + fn canonical_files(&self) -> BTreeMap> { + let aggregate = self + .backend + .factory + .worker_aggregate_dir(&self.request.worker_ref) + .unwrap(); + let mut files = persisted_files(&aggregate); + files.retain(|path, _| !path.components().any(|part| part.as_os_str() == "runs")); + files + } + + fn assert_no_modifying_host_calls(&self) { + assert!( + self.host + .calls + .lock() + .unwrap() + .iter() + .all(|(method, path)| { + method == "GET" + || method == "HEAD" + || (method == "POST" + && path == "/api/w/workspace-saved-subjektiv/authorization/check") + }), + "unexpected modifying Host call" + ); + } + + fn assert_quiet(&self) { + assert_eq!(self.model.call_count.load(Ordering::SeqCst), 0); + assert!(self.model.captured.lock().unwrap().is_empty()); + self.assert_no_modifying_host_calls(); + assert!( + session_store::collect_state(&self.entries()) + .annotated_history + .iter() + .all(|entry| { + !matches!( + entry.item, + LoggedItem::ToolCall { .. } | LoggedItem::ToolResult { .. } + ) + }), + "prepare committed tool execution" + ); + } + + fn prepare(&self) { + assert!(matches!( + self.backend.restore_worker(self.request.clone()), + WorkerExecutionSpawnResult::Connected { .. } + )); + // Connected here is the backend's prepared snapshot, not publication. + assert!( + self.backend + .pending_restore(&self.request.worker_ref) + .unwrap() + .is_some() + ); + assert!(self.backend.workers.lock().unwrap().is_empty()); + assert!( + self.backend + .factory + .observation_hub + .get(&self.request.worker_ref) + .is_none() + ); + assert!( + !self + .backend + .factory + .worker_restore_run_dir(&self.request.worker_ref, self.request.operation_id,) + .unwrap() + .exists() + ); + let table = worker::runtime::worker_allocation::LockFileGuard::open( + &worker::runtime::worker_allocation::default_allocation_path().unwrap(), + ) + .unwrap(); + assert!( + table + .data() + .find(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref( + &self.request.worker_ref, + )) + .is_none() + ); + } + + fn assert_no_allocation(&self) { + let table = worker::runtime::worker_allocation::LockFileGuard::open( + &worker::runtime::worker_allocation::default_allocation_path().unwrap(), + ) + .unwrap(); + assert!( + table + .data() + .find(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref( + &self.request.worker_ref, + )) + .is_none() + ); + } + + fn assert_released(&self) { + self.assert_no_allocation(); + assert!( + self.backend + .pending_restore(&self.request.worker_ref) + .unwrap() + .is_none() + ); + assert!(self.backend.workers.lock().unwrap().is_empty()); + assert!( + self.backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty() + ); + assert!( + self.backend + .factory + .observation_hub + .get(&self.request.worker_ref) + .is_none() + ); + } + } + + impl Drop for Fixture { + fn drop(&mut self) { + // Keep the storage and loopback transport alive until owned tasks stop. + let _ = self.backend.stop_worker(&self.request.worker_ref); + if let Some(task) = self.host.task.take() { + task.abort(); + let _ = self.backend.run_on_adapter_runtime(async move { + let _ = task.await; + Ok(()) + }); + } + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn prepare_saved_inputs_rejects_new_input_and_discard_preserves_authority() { + // Exercise the configured socket transport without ever publishing or + // binding it; activation coverage below uses the normal InProcess path. + let mut fixture = Fixture::with_transport(WorkerControllerTransport::UnixSocket); + let opens = Arc::new(AtomicUsize::new(0)); + let counted = opens.clone(); + let raw = deferred_workdir::DeferredWorkdirSession::handle( + "candidate-discard-resource", + workdir::WorkdirSessionCapabilities::READ_ONLY, + move || { + counted.fetch_add(1, Ordering::SeqCst); + Err("not opened during prepare".into()) + }, + ); + fixture.request.deferred_workdir_sessions.insert( + WorkdirAttachmentAlias::new("checkout").unwrap(), + raw.clone(), + ); + let before = fixture.canonical_files(); + fixture.prepare(); + for input in [ + WorkerInput::user("must-not-accept"), + WorkerInput::notify("must-not-notify"), + ] { + assert!( + !fixture + .backend + .dispatch_input(&fixture.request.worker_ref, input) + .is_accepted() + ); + } + #[cfg(feature = "ws-server")] + assert!( + fixture + .backend + .clone() + .attach_worker_protocol(&fixture.request.worker_ref) + .is_err() + ); + assert!(!matches!( + fixture + .backend + .worker_session(WorkerSessionObservationRequest { + worker_ref: fixture.request.worker_ref.clone(), + }), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before); + assert!( + fixture + .backend + .factory + .failed_restore_resources + .lock() + .unwrap()[&fixture.request.worker_ref] + .candidate_workdir_cleanup + .is_some(), + "retain the real prepared Worker's close-only resource owner" + ); + assert!( + fixture + .backend + .stop_worker(&fixture.request.worker_ref) + .is_accepted() + ); + fixture.assert_released(); + + let factory = fixture.backend.factory.clone(); + let request = fixture.request.clone(); + fixture + .backend + .run_on_adapter_runtime(async move { + // The adapter resolves and owns its own raw handles. Separately + // exercise the factory seam with the exact supplied cleanup handle. + let mut candidate = factory + .prepare_restore(request) + .await + .map_err(|error| error.to_string())?; + candidate + .discard() + .await + .map_err(|error| error.to_string())?; + assert!(matches!( + raw.read(workdir::ReadRequest { + path: workdir::WorkdirPath::new("probe.txt").unwrap(), + offset: 0, + limit: 1, + max_bytes: 32, + }) + .await, + Err(workdir::WorkdirError::SessionClosed) + )); + Ok(()) + }) + .unwrap(); + assert_eq!(opens.load(Ordering::SeqCst), 0); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before); + assert!( + fixture + .backend + .stop_worker(&fixture.request.worker_ref) + .is_accepted() + ); + assert_eq!(fixture.canonical_files(), before); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn prepare_failure_at_each_pre_activation_checkpoint_preserves_saved_authority() { + // Feature and Controller are activation checkpoints, not preparation. + for stage in [ + RuntimeRestoreStage::Metadata, + RuntimeRestoreStage::Reconstruction, + RuntimeRestoreStage::Binding, + ] { + let fixture = Fixture::new(); + let before = fixture.canonical_files(); + *fixture + .backend + .factory + .restore_faults + .fail_at + .lock() + .unwrap() = Some(stage); + let result = fixture.backend.restore_worker(fixture.request.clone()); + assert!( + matches!(result, WorkerExecutionSpawnResult::RolledBack(_)), + "{stage:?}: {result:?}" + ); + fixture.assert_released(); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before, "{stage:?}"); + let completed = fixture.backend.completed_restore_failures.lock().unwrap(); + let (_, failure) = &completed[&fixture.request.worker_ref]; + assert_eq!(failure.stage, stage); + assert_eq!(failure.cleanup, RuntimeRestoreCleanup::Confirmed); + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn activate_saved_inputs_drains_fifo_once_and_replays_durable_receipts() { + let fixture = Fixture::new(); + let before = fixture.canonical_files(); + fixture.prepare(); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before); + fixture + .backend + .activate_restored_worker(fixture.request.operation_id, &fixture.request.worker_ref) + .unwrap(); + let handle = fixture.backend.workers.lock().unwrap()[&fixture.request.worker_ref] + .handle + .clone(); + let model = fixture.model.clone(); + let receipts = fixture.receipts.clone(); + fixture + .backend + .run_on_adapter_runtime(async move { + finished(&handle, SECOND).await; + assert_eq!(model.call_count.load(Ordering::SeqCst), 3); + { + let requests = model.captured.lock().unwrap(); + let first = serde_json::to_string(&requests[0].items).unwrap(); + let second = serde_json::to_string(&requests[2].items).unwrap(); + assert!( + requests[0] + .tools + .iter() + .any(|tool| tool.name == "TaskCreate") + ); + assert!(first.contains(FIRST)); + assert!(!first.contains(SECOND), "second Submit overtook the first"); + assert_eq!(first.matches(NOTICE).count(), 1); + assert!(second.find(FIRST).unwrap() < second.find(SECOND).unwrap()); + assert_eq!(second.matches(NOTICE).count(), 1); + } + let committed = serde_json::to_value(handle.committed_entries()).unwrap(); + for (id, mut expected) in [FIRST, SECOND].into_iter().zip(receipts) { + // Activation advances the durable receipt from Queued to Started; + // replay must preserve its original submission identity. + expected["data"]["disposition"] = serde_json::json!("started"); + assert_eq!( + serde_json::to_value(submit(&handle, id).await).unwrap(), + expected + ); + } + notify(&handle).await; + assert!(pending(&handle).await.submissions.is_empty()); + assert_eq!(model.call_count.load(Ordering::SeqCst), 3); + assert_eq!( + serde_json::to_value(handle.committed_entries()).unwrap(), + committed + ); + Ok(()) + }) + .unwrap(); + let history = session_store::collect_state(&fixture.entries()).annotated_history; + assert!( + history.iter().any(|entry| matches!( + &entry.item, + LoggedItem::ToolResult { call_id, is_error: false, .. } + if call_id == "candidate-task-probe" + )), + "activation must execute the real TaskCreate tool, not just decode a call" + ); + for marker in [FIRST, SECOND, NOTICE] { + let count = history + .iter() + .filter(|entry| { + match &entry.item { + LoggedItem::Message { content, .. } => content.iter().any(|part| { + matches!(part, LoggedContentPart::Text { text } if text.contains(marker)) + }), + _ => false, + } + }) + .count(); + assert_eq!(count, 1, "{marker} must commit exactly once"); + } + let mut fresh = WorkerInput::user(FRESH); + fresh.submission_request_id = Some(FRESH.into()); + assert!( + fixture + .backend + .dispatch_input(&fixture.request.worker_ref, fresh) + .is_accepted() + ); + let handle = fixture.backend.workers.lock().unwrap()[&fixture.request.worker_ref] + .handle + .clone(); + fixture + .backend + .run_on_adapter_runtime(async move { + finished(&handle, FRESH).await; + assert!(pending(&handle).await.submissions.is_empty()); + Ok(()) + }) + .unwrap(); + assert_eq!(fixture.model.call_count.load(Ordering::SeqCst), 4); + assert!( + !fixture.host.calls.lock().unwrap().is_empty(), + "activation must reach the configured Host transport" + ); + fixture.assert_no_modifying_host_calls(); + assert!( + fixture + .backend + .stop_worker(&fixture.request.worker_ref) + .is_accepted() + ); + fixture.assert_released(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn explicit_repair_reconciles_lost_attribution_response_without_reseeding() { + for materialized in [true, false] { + let root = restore_failure_tempdir(); + let model = MockClient::new(Vec::new()); + let injected = model.clone(); + let backend = WorkerRuntimeExecutionBackend::new( + ProfileRuntimeWorkerFactory::new(root.path()) + .with_runtime_store_dir(root.path()) + .with_runtime_id("repair-runtime") + .with_spawn_model_client_factory(move || Box::new(injected.clone())), + ) + .unwrap(); + let factory = backend.factory.clone(); + let fixture_root = root.path().to_path_buf(); + backend.run_on_adapter_runtime(async move { + let mut request = saved_subjektiv_restore_fixture(&fixture_root, materialized).await; + let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&request.worker_ref); + let aggregate = fixture_root.join("workers").join(request.worker_ref.worker_id.to_string()); + let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap(); + let mut saved = metadata.read_by_name(&name).unwrap().unwrap(); + let session_id = saved.active.as_ref().unwrap().session_id; + saved.subjektiv_session_attribution = Some(session_store::SubjektivSessionAttributionState::OutcomeUnknown { session_id, diagnostic: "saved interrupted attribution".into() }); + metadata.write(&saved).unwrap(); + // Explicit repair may acquire artifact locks; compare canonical + // contents separately from that lock-file side effect. + let session_files = || { + let mut files = persisted_files(&aggregate.join("session")); + files.remove(Path::new("artifacts/paste/.aggregate.lock")); + files + }; + let original = session_files(); + let mut bundle = test_bundle(); + bundle.metadata.workspace_id = "workspace-saved-subjektiv".into(); + bundle.prompt_catalog = Some(worker::PromptCatalog::builtins_only().unwrap().projection().clone()); + request.config_bundle = Some(bundle.with_computed_digest()); + + // Substitute only the Host transport. A committed attribution + // with an undecodable response models lost acknowledgement; + // its durable Session-keyed receipt survives the retry. This + // does not test the real Host's signature or Subject policy. + let attribution_path = fixture_root.join("host-attribution.json"); + let stored = attribution_path.clone(); + let calls = Arc::new(AtomicUsize::new(0)); + let record_calls = calls.clone(); + let startup_calls = Arc::new(AtomicUsize::new(0)); + let startups = startup_calls.clone(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + request.request.workspace_api.as_mut().unwrap().base_url = format!("http://{}", listener.local_addr().unwrap()); + let app = axum::Router::new() + .route("/api/w/workspace-saved-subjektiv/authorization/check", axum::routing::post( + |axum::Json(input): axum::Json| async move { + axum::Json(saved_subjektiv_authorization_fixture(input).unwrap()) + })) + .route("/api/w/workspace-saved-subjektiv/subjektiv/sessions", axum::routing::post( + move |axum::Json(input): axum::Json| { + let stored = stored.clone(); + let calls = record_calls.clone(); + async move { + assert_eq!(input.session_id, session_id.to_string()); + let ordinal = calls.fetch_add(1, Ordering::SeqCst); + if !stored.exists() { + assert!(input.create_if_missing); + fs::write(&stored, serde_json::to_vec(&server_api::SubjektivRecordSessionResponse { + session_id: input.session_id, subject_id: "saved-subject".into(), + }).unwrap()).unwrap(); + } + (axum::http::StatusCode::OK, [(axum::http::header::CONTENT_TYPE, "application/json")], + if ordinal == 0 { "{}".to_string() } else { fs::read_to_string(stored).unwrap() }) + } + })) + .route("/api/w/workspace-saved-subjektiv/subjektiv/journal/startup", axum::routing::get(move || { + let startups = startups.clone(); + async move { + startups.fetch_add(1, Ordering::SeqCst); + axum::Json(serde_json::json!({ "subject_id": "saved-subject", "journals": [], "omitted": false })) + } + })); + let host = HostProbe { calls: Arc::new(Mutex::new(Vec::new())), task: Some(tokio::spawn(async move { axum::serve(listener, app).await.unwrap() })) }; + // Same real factory and same operation for both repairs. No + // candidate/controller is constructed by the test itself. + assert!(factory.recover_saved_restore_source(&request).await.is_err()); + assert!(attribution_path.exists(), "Host committed before response loss"); + assert_eq!(session_files(), original); + let failed = metadata.read_by_name(&name).unwrap().unwrap(); + assert!(matches!(failed.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::OutcomeUnknown { session_id: id, .. }) if id == session_id)); + assert_eq!(startup_calls.load(Ordering::SeqCst), 0); + factory.cleanup_failed_restore(&request).await.unwrap(); + assert!(factory.recover_saved_restore_source(&request).await.unwrap()); + let repaired = metadata.read_by_name(&name).unwrap().unwrap(); + assert_eq!(repaired.active.as_ref().unwrap().session_id, session_id); + assert!(matches!(repaired.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Confirmed { session_id: id, .. }) if id == session_id)); + let segments = WorkerSessionStore::new(aggregate.join("session")).unwrap() + .list_segments(session_id).unwrap(); + assert_eq!(segments.len(), 1, "retry must retain exactly one seed"); + if materialized { + assert_eq!(session_files(), original); + assert_eq!(startup_calls.load(Ordering::SeqCst), 0); + } else { + assert_eq!(startup_calls.load(Ordering::SeqCst), 1); + } + let after_repair = persisted_files(&aggregate); + let calls_after_repair = calls.load(Ordering::SeqCst); + assert!(!factory.recover_saved_restore_source(&request).await.unwrap()); + assert_eq!(persisted_files(&aggregate), after_repair); + assert_eq!(calls.load(Ordering::SeqCst), calls_after_repair); + assert!(factory.failed_restore_resources.lock().unwrap().is_empty()); + drop(host); + Ok(()) + }).unwrap(); + assert_eq!(model.call_count.load(Ordering::SeqCst), 0); + } + } +} diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs index 8b870cd62..40de8168c 100644 --- a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -1,8 +1,11 @@ // Integration coverage through EmbeddedRuntime -> real Profile factory -> real -// Worker/bootstrap, local Git Workdir provider, and allocation table. Model I/O -// is not invoked; this is a deterministic ordinary suite, not product E2E. +// Worker/bootstrap, local Git Workdir provider, and allocation table. External +// model I/O is replaced for saved-input coverage; this is not product E2E. + +use crate::restore_test_environment::require_restore_test_environment; fn restore_failure_tempdir() -> tempfile::TempDir { + require_restore_test_environment(); // Explicitly stay inside this checkout, even when TMPDIR points elsewhere. // Keep fixture components short; no symlink/alias to evade socket limits. let target = Path::new(env!("CARGO_MANIFEST_DIR")) @@ -102,11 +105,16 @@ struct ProfileFailureFixture { backend: Arc, worker: crate::catalog::WorkerDetail, provider_opens: Arc, + seed_handle: Option, root: tempfile::TempDir, } impl ProfileFailureFixture { fn new(timeout: Duration) -> Self { + Self::with_saved_inputs(timeout, None) + } + + fn with_saved_inputs(timeout: Duration, model: Option) -> Self { let root = restore_failure_tempdir(); let repo = root.path().join("g"); fs::create_dir(&repo).unwrap(); @@ -118,16 +126,20 @@ impl ProfileFailureFixture { git(&repo, &["commit", "-m", "init"]); let store = root.path().join("s"); let provider_opens = Arc::new(AtomicUsize::new(0)); + let mut factory = + ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store); + let queue_inputs = model.is_some(); + if let Some(model) = model { + factory = factory.with_spawn_model_client_factory(move || Box::new(model.clone())); + } let backend = Arc::new( - WorkerRuntimeExecutionBackend::new( - ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store), - ) - .unwrap() - .with_spawn_restore_timeout(timeout) - .with_working_directory_materializer(RestoreOpenCountingMaterializer { - inner: RuntimeGitMaterializer::new(root.path().join("w")), - opens: provider_opens.clone(), - }), + WorkerRuntimeExecutionBackend::new(factory) + .unwrap() + .with_spawn_restore_timeout(timeout) + .with_working_directory_materializer(RestoreOpenCountingMaterializer { + inner: RuntimeGitMaterializer::new(root.path().join("w")), + opens: provider_opens.clone(), + }), ); let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( crate::fs_store::FsRuntimeStoreOptions { @@ -146,12 +158,41 @@ impl ProfileFailureFixture { working_directory: working_directory_request(&repo), }]; let worker = runtime.create_worker(request).unwrap(); + let seed_handle = queue_inputs.then(|| { + backend.workers.lock().unwrap()[&worker.worker_ref] + .handle + .clone() + }); + if let Some(handle) = seed_handle.clone() { + backend.run_on_adapter_runtime(async move { + async fn event(events: &mut tokio::sync::broadcast::Receiver, predicate: impl Fn(&Event) -> bool) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let next = events.recv().await.unwrap(); + assert!(!matches!(next, Event::Error { .. } | Event::SubmissionRejected { .. } | Event::NotificationRejected { .. }), "{next:?}"); + if predicate(&next) { break; } + } + }).await.expect("saved-input fixture acknowledgement"); + } + let mut events = handle.subscribe(); + handle.send(Method::submit_text("revocation-seed", "hold for saved input setup")).await.unwrap(); + event(&mut events, |next| matches!(next, Event::TextDelta { .. })).await; + handle.send(Method::Pause { command: protocol::WorkerCommandEnvelope::new(handle.shared_state.snapshot().last_command_id + 1) }).await.unwrap(); + event(&mut events, |next| matches!(next, Event::WorkerState { snapshot } if snapshot.catalog_status() == protocol::WorkerStatus::Paused)).await; + handle.send(Method::submit_text("revocation-saved-submit", "must stay queued")).await.unwrap(); + event(&mut events, |next| matches!(next, Event::SubmissionAccepted { submission_request_id, disposition: protocol::SubmissionDisposition::Queued, .. } if submission_request_id == "revocation-saved-submit")).await; + handle.send(Method::Notify { notification_request_id: "revocation-saved-notice".into(), message: "must also stay queued".into() }).await.unwrap(); + event(&mut events, |next| matches!(next, Event::NotificationAccepted { notification_request_id } if notification_request_id == "revocation-saved-notice")).await; + Ok(()) + }).unwrap(); + } runtime.stop_worker(&worker.worker_ref, None).unwrap(); // Count restore provider access, not the initial create/stop setup. provider_opens.store(0, Ordering::SeqCst); Self { root, provider_opens, + seed_handle, runtime, backend, worker, @@ -241,6 +282,68 @@ impl ProfileFailureFixture { assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0); } + fn resume_same_operation( + &self, + request: runtime_api::WorkerRestoreRequest, + operation_id: crate::execution::WorkerLifecycleOperationId, + ) { + let result = self + .runtime + .restore_worker_operation(&self.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::Accepted, + "{result:?}" + ); + assert_eq!( + self.backend.workers.lock().unwrap()[&self.worker.worker_ref].restore_operation_id, + Some(operation_id) + ); + let calls = self + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst); + let replay = self + .runtime + .restore_worker_operation(&self.worker.worker_ref, request) + .unwrap(); + assert_eq!( + serde_json::to_value(replay).unwrap(), + serde_json::to_value(result).unwrap() + ); + assert_eq!( + self.backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + calls + ); + self.runtime + .stop_worker(&self.worker.worker_ref, None) + .unwrap(); + assert!( + self.backend + .factory + .observation_hub + .get(&self.worker.worker_ref) + .is_none() + ); + assert!(self.backend.pending_restores.lock().unwrap().is_empty()); + assert!(self.backend.workers.lock().unwrap().is_empty()); + assert!( + self.backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty() + ); + } + fn fresh_restore_and_old_replay( &self, request: runtime_api::WorkerRestoreRequest, @@ -319,6 +422,171 @@ impl ProfileFailureFixture { } } +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_timeout_attachment_revocation_cannot_reactivate_old_candidate() { + let model = MockClient::sequential(vec![MockResponse::Hang( + simple_text_events().into_iter().take(2).collect(), + )]); + let fixture = + ProfileFailureFixture::with_saved_inputs(Duration::from_secs(2), Some(model.clone())); + let body_calls = model.call_count.load(Ordering::SeqCst); + assert_eq!(body_calls, 1); + let aggregate = fixture + .backend + .factory + .worker_aggregate_dir(&fixture.worker.worker_ref) + .unwrap(); + let session_before = persisted_files(&aggregate.join("session")); + let metadata = WorkerAggregateStore::open_read_only( + &aggregate, + ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&fixture.worker.worker_ref), + ) + .unwrap(); + let saved = metadata + .read_by_name(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref( + &fixture.worker.worker_ref, + )) + .unwrap() + .unwrap(); + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .factory_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .factory_entered + .lock() + .unwrap() = Some(entered_tx); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let first = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + entered.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + assert!(!pending.request.logical_workdir_attachments.is_empty()); + let updated = fixture + .runtime + .replace_worker_workdir_attachments(&fixture.worker.worker_ref, Vec::new()) + .unwrap(); + assert!(updated.workdir_attachments.is_empty()); + let authority = || -> serde_json::Value { + serde_json::from_slice( + &fs::read( + fixture + .root + .path() + .join("s/workers") + .join(fixture.worker.worker_ref.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap() + }; + assert!(authority().get("logical_workdir_attachments").is_none()); + release.notify_one(); + let replay = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "current attachment revocation must fence the old candidate: {replay:?}" + ); + assert!( + replay + .message + .as_deref() + .unwrap_or_default() + .contains("current Worker authority differs"), + "{replay:?}" + ); + let record = authority(); + let owner = &record["restore_guard"]["owners"][&request.request_id]; + assert_eq!( + owner["operation_id"], + serde_json::to_value(pending.request.operation_id).unwrap() + ); + assert!(owner["receipt"].is_null()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!(Arc::ptr_eq( + &pending, + &fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap() + )); + assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0); + assert!(!pending.activation_started.load(Ordering::SeqCst)); + assert_eq!(model.call_count.load(Ordering::SeqCst), body_calls); + assert_eq!(persisted_files(&aggregate.join("session")), session_before); + assert_eq!( + metadata.read_by_name(&saved.worker_name).unwrap().unwrap(), + saved + ); + let old = fixture.seed_handle.clone().unwrap(); + fixture + .backend + .run_on_adapter_runtime(async move { + assert!(old.protocol_is_closed()); + let old_command = Method::Resume { + command: protocol::WorkerCommandEnvelope::new( + old.shared_state.snapshot().last_command_id + 1, + ), + }; + assert!( + old.send(old_command).await.is_err(), + "old execution command must not enter the restore candidate" + ); + Ok(()) + }) + .unwrap(); + assert!(authority().get("logical_workdir_attachments").is_none()); + let input = fixture.runtime.send_input( + &fixture.worker.worker_ref, + WorkerInput::user("must-not-enter-revoked-candidate"), + ); + assert!( + input.is_err(), + "pending candidate must not admit fresh input: {input:?}" + ); + // Stop owns cleanup of the original operation, never a newly rebound alias. + fixture + .runtime + .stop_worker(&fixture.worker.worker_ref, None) + .unwrap(); + fixture.assert_closed_deferred_sessions(&pending); + assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0); + assert!(fixture.backend.pending_restores.lock().unwrap().is_empty()); + let old = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request) + .unwrap(); + assert_ne!(old.state, server_api::WorkerRestoreState::Accepted); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); +} + #[test] #[serial_test::serial(worker_allocation)] fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() { @@ -327,8 +595,6 @@ fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() { for stage in [ RuntimeRestoreStage::Reconstruction, RuntimeRestoreStage::Binding, - RuntimeRestoreStage::Feature, - RuntimeRestoreStage::Controller, ] { let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); fixture.fail(stage); @@ -474,7 +740,7 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer .unwrap(); assert_eq!( result.state, - server_api::WorkerRestoreState::RolledBack, + server_api::WorkerRestoreState::ReconciliationRequired, "{result:?}" ); assert!( @@ -488,12 +754,11 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer .iter() .all(|sessions| sessions.aliases().is_empty()) ); - fixture.assert_released(); fixture.assert_closed_deferred_sessions(&pending); eprintln!( - "restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=RolledBack" + "restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=ReconciliationRequired" ); - fixture.fresh_restore_and_old_replay(request, result); + fixture.resume_same_operation(request, pending.request.operation_id); } #[test] @@ -585,6 +850,7 @@ async fn failed_restore_raw_cleanup_is_operation_bound_and_does_not_follow_reuse FailedRestoreResources { operation_id: current.operation_id, sessions: Vec::new(), + candidate_workdir_cleanup: None, allocation: None, feature_tasks: None, }, @@ -715,14 +981,14 @@ fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconcile .unwrap(); assert_eq!( result.state, - server_api::WorkerRestoreState::RolledBack, + server_api::WorkerRestoreState::Accepted, "{result:?}" ); - fixture.assert_released(); + assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); eprintln!( "restore-fault cleanup=timeout retained-task=true late-completion=joined executions=1" ); - fixture.fresh_restore_and_old_replay(request, result); + fixture.resume_same_operation(request, pending.request.operation_id); } #[cfg(unix)] @@ -745,7 +1011,7 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( .unwrap(); assert_eq!( result.state, - server_api::WorkerRestoreState::RolledBack, + server_api::WorkerRestoreState::ReconciliationRequired, "{result:?}" ); let diagnostic = serde_json::to_string(&result).unwrap(); @@ -757,8 +1023,12 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( // This case deliberately exercises the real Unix socket path-limit error; // positive restore cases retain the factory's normal InProcess transport. // Short fixture roots must not accidentally turn it into a successful bind. - let operation_id = - fixture.backend.completed_restore_failures.lock().unwrap()[&fixture.worker.worker_ref].0; + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + let operation_id = pending.request.operation_id; let socket = fixture .backend .factory @@ -767,7 +1037,25 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( .join("worker.sock"); assert!(socket.as_os_str().as_encoded_bytes().len() >= 108); assert!(!diagnostic.contains(&fixture.root.path().to_string_lossy().to_string())); - fixture.assert_released(); + assert!(!socket.exists()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!( + fixture + .backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty() + ); + assert!( + fixture + .backend + .completed_restore_failures + .lock() + .unwrap() + .is_empty() + ); fixture .backend .factory @@ -777,7 +1065,7 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( eprintln!( "restore-fault stage=socket-start real-bootstrap=true partial-run=removed allocation=released" ); - fixture.fresh_restore_and_old_replay(request, result); + fixture.resume_same_operation(request, operation_id); } #[test] @@ -1178,6 +1466,8 @@ fn profile_restore_missing_committed_system_prompt_rolls_back_and_replays_classi "#, ) .unwrap(); + let mut manifest = manifest; + manifest.worker.name = worker_name.clone(); metadata.resolved_manifest_snapshot = Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap()); metadata_store.write(&metadata).unwrap(); diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index a541ca2e1..97cdff43a 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -3676,8 +3676,12 @@ mod tests { #[test] fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() { + crate::restore_test_environment::require_restore_test_environment(); let repo = create_clean_repo(); - let runtime_root = tempfile::tempdir().unwrap(); + // Both ends use a short reference into this TempDir-owned tree. Keep + // real agent/broker sockets and the existing bind/cleanup assertions. + let runtime_root = tempfile::tempdir_in(".").unwrap(); + let runtime_path = Path::new(runtime_root.path().file_name().unwrap()); let key_root = tempfile::tempdir().unwrap(); let key_path = key_root.path().join("id_ed25519"); let fallback_key_path = key_root.path().join("id_ed25519_fallback"); @@ -3689,7 +3693,7 @@ mod tests { .unwrap(); assert!(status.success()); } - let materializer = RuntimeGitMaterializer::new(runtime_root.path()); + let materializer = RuntimeGitMaterializer::new(runtime_path); let mut request = request(repo.path()); request.materialization = Some(crate::catalog::RepositoryMaterializationContext { workspace_id: "workspace-1".to_string(), @@ -3731,7 +3735,7 @@ mod tests { kind: server_api::RepositorySourceKind::Ssh, uri: "ssh://git@example.test/repo.git".to_string(), }; - let command_access = RepositoryCommandAccess::prepare(runtime_root.path(), &ssh_operation) + let command_access = RepositoryCommandAccess::prepare(runtime_path, &ssh_operation) .unwrap() .unwrap(); assert!(!command_access.root.join("identity").exists()); @@ -3873,18 +3877,17 @@ mod tests { .unwrap(); // Restore preflight must validate without consuming this one-shot SSH // delivery or creating agent/session resources needed by actual bind. - let record_before = - fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(); + let record_before = fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap(); materializer .preflight_bind_working_directory(&id, None) .unwrap(); assert_eq!( - fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(), + fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap(), record_before ); assert_eq!(materializer.list_working_directories().unwrap().len(), 1); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 0 @@ -3898,7 +3901,7 @@ mod tests { assert!(ssh_policy.contains("__repository-ssh")); assert!(!ssh_policy.contains("SSH_AUTH_SOCK")); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 1 @@ -3916,7 +3919,7 @@ mod tests { "consumed delivery expiry must not revoke a live Workdir session" ); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 1, @@ -3925,7 +3928,7 @@ mod tests { drop(binding); assert!(!ssh_command.exists()); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 0 @@ -3971,7 +3974,7 @@ mod tests { ], &read_only_command_policy, )); - let broker_socket = fs::read_dir(runtime_root.path().join(REPOSITORY_ACCESS_DIR)) + let broker_socket = fs::read_dir(runtime_path.join(REPOSITORY_ACCESS_DIR)) .unwrap() .filter_map(Result::ok) .map(|entry| entry.path().join("b.sock")) @@ -4104,7 +4107,7 @@ mod tests { .code, "working_directory_repository_access_expired" ); - let restored = RuntimeGitMaterializer::new(runtime_root.path()); + let restored = RuntimeGitMaterializer::new(runtime_path); assert_eq!( restored.bind_working_directory(&id, None).unwrap_err().code, "working_directory_remote_repository_access_required" diff --git a/crates/worker/src/bootstrap.rs b/crates/worker/src/bootstrap.rs index daf86ee25..00e6c5ddf 100644 --- a/crates/worker/src/bootstrap.rs +++ b/crates/worker/src/bootstrap.rs @@ -9,6 +9,7 @@ use crate::PromptCatalogSource; use crate::controller::{ ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle, }; +pub use crate::worker::{PreparedRestoredWorker, RestoreCandidateWorkdirCleanup}; use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext}; use manifest::WorkerManifest; @@ -168,6 +169,38 @@ where Ok(PreparedWorker::new(worker, self.layout, self.transport)) } + /// Reconstruct a saved materialized Worker without allocation, attribution + /// repair, Feature installation, or controller exposure. The caller owns the + /// later controller layout/transport and must activate before starting it. + pub async fn prepare_restored_candidate( + self, + worker_name: &str, + ) -> Result, WorkerBootstrapError> { + let mut candidate = + Worker::prepare_restored_from_worker_metadata_with_context_and_model_client( + worker_name, + self.manifest, + self.store, + self.prompt_catalog, + self.workspace_context, + self.filesystem_authority, + self.model_client, + ) + .await + .map_err(WorkerBootstrapError::Worker)?; + if let Some(workdir_session) = self.workdir_session { + candidate + .worker_mut() + .map_err(WorkerBootstrapError::Worker)? + .bind_single_workdir_session(Some(workdir_session)); + } + candidate + .worker_mut() + .map_err(WorkerBootstrapError::Worker)? + .bind_subjektiv_host(self.subjektiv_host); + Ok(candidate) + } + pub async fn prepare_restored( self, worker_name: &str, diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index 3ff170dd9..6721753d6 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -689,6 +689,35 @@ pub enum WorkerControllerTransport { pub struct WorkerController; impl WorkerController { + /// Explicit recovery of an unmaterialized Session head. This installs the + /// saved Feature composition needed to produce the seed, but creates no + /// Controller, input channel, socket, observer, or queue consumer. The caller + /// owns the same operation's Feature/session/allocation cleanup on every exit. + pub async fn recover_pending_head( + worker: &mut Worker, + runtime_base: &Path, + ) -> std::io::Result<()> + where + C: LlmClient + Clone + 'static, + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + if worker.needs_initial_session_head_materialization() { + worker + .validate_subjektiv_body_publication() + .map_err(std::io::Error::other)?; + let registry = SpawnedWorkerRegistry::new_internal( + worker.manifest().worker.name.clone(), + worker.scope().clone(), + ); + register_worker_tools(worker, runtime_base.to_path_buf(), registry, None, None).await?; + worker + .materialize_durable_session_head() + .await + .map_err(std::io::Error::other)?; + } + Ok(()) + } + pub async fn spawn( worker: Worker, runtime_base: &Path, diff --git a/crates/worker/src/discovery.rs b/crates/worker/src/discovery.rs index dd7ecb8e8..e8354a499 100644 --- a/crates/worker/src/discovery.rs +++ b/crates/worker/src/discovery.rs @@ -1066,7 +1066,8 @@ mod tests { use tokio::net::UnixListener; use crate::runtime::dir::RuntimeDir; - use crate::runtime::worker_allocation::test_util::RuntimeDirSandbox; + + use crate::restore_test_environment::require_restore_test_environment; #[derive(Clone)] struct FailTargetPeerStore { @@ -1110,11 +1111,13 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn state_backed_visibility_and_restore_planning() { - let root = TempDir::new().unwrap(); + require_restore_test_environment(); + let root = tempfile::tempdir_in(".").unwrap(); let store_dir = root.path().join("store"); - let runtime_base = root.path().join("runtime"); + // Bind, durable child metadata, and discovery probes all use this same + // short reference into the tree still owned by root. No cwd mutation. + let runtime_base = Path::new(root.path().file_name().unwrap()).join("runtime"); std::fs::create_dir_all(&runtime_base).unwrap(); - let _runtime_sandbox = RuntimeDirSandbox::new(&runtime_base); let store = FsWorkerStore::new(&store_dir).unwrap(); let session_id = new_session_id(); diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index e04ec36a4..2f3df7f9b 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -20,6 +20,9 @@ mod metadata_lifecycle; pub mod model_client; mod paste_artifact_tool; pub mod prompt; +#[cfg(test)] +#[path = "../../../tests/support/restore_environment.rs"] +mod restore_test_environment; pub mod runtime; pub mod runtime_command; pub mod segment_log_sink; diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 1ef93cd0e..59eb439a7 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -3386,6 +3386,208 @@ impl WorkerStartupFeatureCleanup { } } +/// A close-only candidate resource owner, not filesystem execution authority. +#[derive(Clone)] +pub struct RestoreCandidateWorkdirCleanup { + sessions: Arc, +} + +impl RestoreCandidateWorkdirCleanup { + pub async fn close(&self) -> Result<(), WorkerError> { + self.sessions + .close_all() + .await + .map_err(|error| WorkerError::FeatureLifecycle(error.to_string())) + } +} + +/// An unpublished, read-only reconstruction of an existing materialized Worker. +/// +/// The host must hold its per-Worker operation lock through activation. Snapshot +/// checks reject drift; they do not replace that lock or current host authorization. +/// No controller, allocation, Feature installation, or repair runs during prepare. +/// The public candidate exposes no Worker execution or mutation handles. Binding +/// inside this crate is limited to the open candidate; once discard begins, only +/// cleanup retry remains valid, even if its previous attempt failed or was cancelled. +/// Use `discard` for confirmed cleanup; Drop only schedules best-effort cleanup on +/// the current Tokio runtime and cannot attest that asynchronous cleanup completed. +pub struct PreparedRestoredWorker { + worker: Option, St>>, + closing: bool, + metadata: Option, + entries: serde_json::Value, + manifest: serde_json::Value, +} + +impl PreparedRestoredWorker +where + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, +{ + pub fn manifest(&self) -> &WorkerManifest { + &self.worker().manifest + } + + /// Retain the same Workdir resource owner for explicit cleanup, including + /// after discard fails. This does not grant Worker input or execution access. + pub fn workdir_cleanup(&self) -> RestoreCandidateWorkdirCleanup { + RestoreCandidateWorkdirCleanup { + sessions: self.worker().workdir_sessions.clone(), + } + } + + #[cfg(test)] + fn workdir_sessions(&self) -> Arc { + self.worker().workdir_sessions.clone() + } + + fn worker(&self) -> &Worker, St> { + self.worker + .as_ref() + .expect("restore candidate already discarded") + } + + /// Bootstrap may bind host services before returning an open candidate. + /// Never reopen a candidate whose cleanup has begun. + pub(crate) fn worker_mut( + &mut self, + ) -> Result<&mut Worker, St>, WorkerError> { + if self.closing { + return Err(WorkerError::RestoreCandidateClosing); + } + self.worker + .as_mut() + .ok_or(WorkerError::RestoreCandidateClosing) + } + + fn revalidate(&self, worker: &Worker, St>) -> Result<(), WorkerError> { + let changed = |component| WorkerError::RestoreCandidateChanged { + worker_name: worker.manifest.worker.name.clone(), + component, + }; + if worker.store.read_by_name(&worker.manifest.worker.name)? != self.metadata { + return Err(changed("metadata")); + } + if restore_snapshot_value(&worker.manifest)? != self.manifest { + return Err(changed("manifest")); + } + let entries = worker + .store + .read_all(worker.session_id(), worker.segment_id())?; + if restore_snapshot_value(&entries)? != self.entries { + return Err(changed("segment")); + } + Ok(()) + } + + /// Acquire writer allocation only after exact source validation, validate + /// again under that allocation, then apply the existing restore mutations. + /// An error may retain partial durable effects; ConstructionCleanup carries + /// resource ownership and is not evidence of rollback. + pub async fn activate(self) -> Result, St>, WorkerError> { + self.activate_with_allocation(|worker| { + let socket_path = dir::default_base() + .map_err(ScopeLockError::from)? + .join(&worker.manifest.worker.name) + .join("sock"); + Ok(worker_allocation::install_top_level_with_deny( + worker.manifest.worker.name.clone(), + std::process::id(), + socket_path, + worker.scope.snapshot().allow_rules(), + worker.scope.snapshot().deny_rules(), + worker.segment_id(), + )?) + }) + .await + } + + async fn activate_with_allocation( + mut self, + allocate: impl FnOnce( + &Worker, St>, + ) -> Result, + ) -> Result, St>, WorkerError> { + let mut worker = self + .worker + .take() + .ok_or(WorkerError::RestoreCandidateClosing)?; + if self.closing { + // Preserve the same cleanup owner without validating or acquiring + // allocation for a partially closed candidate. + return Err(worker.construction_failure(WorkerError::RestoreCandidateClosing)); + } + self.revalidate(&worker) + .map_err(|error| worker.construction_failure(error))?; + worker.scope_allocation = + Some(allocate(&worker).map_err(|error| worker.construction_failure(error))?); + self.revalidate(&worker) + .map_err(|error| worker.construction_failure(error))?; + worker + .finish_restored_construction() + .await + .map_err(|source| WorkerError::RestoreActivationEffects { + source: Box::new(source), + }) + } + + /// Directly settle candidate resources, without Controller Shutdown. Failed + /// cleanup retains the candidate so the same owner can retry it. + pub async fn discard(&mut self) -> Result<(), WorkerError> { + self.closing = true; + if let Some(worker) = self.worker.as_mut() { + worker + .retain_startup_feature_cleanup() + .shutdown() + .await + .map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?; + worker + .workdir_sessions + .close_all() + .await + .map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?; + // An unpublished candidate has no allocation guard. Activation moves + // the Worker out and uses ConstructionCleanup on failure; never try + // releasing a cleanup handle while its guard is still attached. + debug_assert!(worker.scope_allocation.is_none()); + } + self.worker = None; + Ok(()) + } +} + +impl Drop for PreparedRestoredWorker { + fn drop(&mut self) { + let Some(worker) = self.worker.as_mut() else { + return; + }; + let sessions = worker.workdir_sessions.clone(); + let features = worker.retain_startup_feature_cleanup(); + // Candidates do not acquire allocation until consuming activation. + debug_assert!(worker.scope_allocation.is_none()); + match tokio::runtime::Handle::try_current() { + Ok(runtime) => { + runtime.spawn(async move { + if let Err(error) = features.shutdown().await { + tracing::warn!(%error, "restore candidate Feature cleanup unconfirmed"); + return; + } + if let Err(error) = sessions.close_all().await { + tracing::warn!(%error, "restore candidate Workdir cleanup unconfirmed"); + } + }); + } + Err(error) => { + tracing::warn!(%error, "restore candidate dropped without async cleanup runtime") + } + } + } +} + +fn restore_snapshot_value(value: &impl serde::Serialize) -> Result { + serde_json::to_value(value) + .map_err(|error| WorkerError::InvalidState(format!("encode restore snapshot: {error}"))) +} + /// An independent agent execution unit. /// /// Holds a [`Engine`] directly and persists session state via @@ -3405,6 +3607,7 @@ pub struct Worker { /// constructors install this from the same FsStore that owns the session /// logs; low-level `Worker::new` tests leave it absent. worker_metadata_writer: Option, + persist_restored_manifest: bool, worker_metadata_segment_cas: Option, /// Shared session pointer. Source of truth for the Worker's current /// `segment_id` and append tally. `self.segment_id()` is a thin @@ -3820,6 +4023,7 @@ impl Worker { last_run_interrupted: false, store, worker_metadata_writer: None, + persist_restored_manifest: false, worker_metadata_segment_cas: None, segment_state: SegmentState::new(session_id, segment_id, 0), filesystem_authority, @@ -4534,14 +4738,21 @@ impl Worker { } fn worker_metadata(&self, active: Option) -> WorkerMetadata { - worker_metadata_for_manifest( + let mut metadata = worker_metadata_for_manifest( &self.manifest, self.workspace_id(), self.filesystem_authority .as_local() .map(|local| local.root.as_path()), active, - ) + ); + // A repaired/restored Session must not lose the saved execution policy + // merely because its current composition also permits legacy local use. + if self.persist_restored_manifest { + metadata.resolved_manifest_snapshot = + manifest::write_persisted_worker_manifest_snapshot(&self.manifest).ok(); + } + metadata } fn write_worker_metadata_pending(&self) -> Result<(), WorkerError> { @@ -8445,6 +8656,7 @@ where last_run_interrupted: false, store, worker_metadata_writer, + persist_restored_manifest: existing_session_id.is_some(), worker_metadata_segment_cas, segment_state: SegmentState::new(session_id, segment_id, 0), filesystem_authority: common.filesystem_authority, @@ -8852,6 +9064,7 @@ where last_run_interrupted: false, store, worker_metadata_writer: None, + persist_restored_manifest: false, worker_metadata_segment_cas: None, segment_state: SegmentState::new(session_id, segment_id, 0), filesystem_authority: common.filesystem_authority, @@ -8987,6 +9200,7 @@ where last_run_interrupted: false, store, worker_metadata_writer, + persist_restored_manifest: true, worker_metadata_segment_cas, segment_state: SegmentState::new(session_id, segment_id, 0), filesystem_authority: common.filesystem_authority, @@ -9080,6 +9294,33 @@ where workspace_context: WorkerWorkspaceContext, filesystem_authority: WorkerFilesystemAuthority, ) -> Result { + Self::prepare_restored_from_worker_metadata_with_context_and_model_client( + worker_name, + manifest, + store, + loader, + workspace_context, + filesystem_authority, + None, + ) + .await? + .activate() + .await + } + + /// Prepare only a materialized saved Worker. Pending Session creation and + /// Subject attribution repair must be performed explicitly by the host + /// before entering this read-only path. + #[allow(clippy::too_many_arguments)] + pub async fn prepare_restored_from_worker_metadata_with_context_and_model_client( + worker_name: &str, + fallback: WorkerManifest, + store: St, + loader: PromptCatalogSource, + workspace_context: WorkerWorkspaceContext, + filesystem_authority: WorkerFilesystemAuthority, + model_client: Option>, + ) -> Result, WorkerError> { let metadata = store .read_by_name(worker_name)? @@ -9093,11 +9334,13 @@ where worker_name: worker_name.to_string(), }); } - let active = metadata - .active - .ok_or_else(|| WorkerError::WorkerMetadataInactive { - worker_name: worker_name.to_string(), - })?; + let active = + metadata + .active + .as_ref() + .ok_or_else(|| WorkerError::WorkerMetadataInactive { + worker_name: worker_name.to_string(), + })?; let segment_id = active .segment_id .ok_or_else(|| WorkerError::WorkerMetadataPending { @@ -9106,10 +9349,15 @@ where })?; let manifest = restore_manifest_from_worker_metadata_snapshot( worker_name, - metadata.resolved_manifest_snapshot, - manifest, + metadata.resolved_manifest_snapshot.clone(), + fallback, )?; - Self::restore_from_manifest_with_context( + if manifest.worker.name != worker_name { + return Err(WorkerError::InvalidState( + "restore manifest Worker name mismatch".into(), + )); + } + Self::prepare_restored_from_manifest_snapshot( active.session_id, segment_id, manifest, @@ -9117,6 +9365,8 @@ where loader, workspace_context, filesystem_authority, + model_client, + Some(metadata), ) .await } @@ -9303,10 +9553,41 @@ where filesystem_authority: WorkerFilesystemAuthority, model_client: Option>, ) -> Result { + let metadata = store.read_by_name(&manifest.worker.name)?; + Self::prepare_restored_from_manifest_snapshot( + session_id, + segment_id, + manifest, + store, + loader, + workspace_context, + filesystem_authority, + model_client, + metadata, + ) + .await? + .activate() + .await + } + + #[allow(clippy::too_many_arguments)] + async fn prepare_restored_from_manifest_snapshot( + session_id: SessionId, + segment_id: SegmentId, + manifest: WorkerManifest, + store: St, + loader: PromptCatalogSource, + workspace_context: WorkerWorkspaceContext, + filesystem_authority: WorkerFilesystemAuthority, + model_client: Option>, + metadata: Option, + ) -> Result, WorkerError> { // Read raw entries once so we can both reconstruct state and // seed the broadcast sink's mirror with the same prefix that // sits on disk. let raw_entries = store.read_all(session_id, segment_id)?; + let entries_snapshot = restore_snapshot_value(&raw_entries)?; + let manifest_snapshot = restore_snapshot_value(&manifest)?; let state = session_store::collect_state(&raw_entries); if state.entries_count == 0 { return Err(WorkerError::SegmentEmpty { segment_id }); @@ -9332,24 +9613,6 @@ where model_client, )?; - // Atomic: register_worker inside install_top_level rejects when - // another live allocation already holds `segment_id`. Wrapping - // the lookup + install inside a single `LockFileGuard` is what - // makes "no two live Workers write to the same session log" - // actually structural rather than a hopeful pre-check. - let socket_path = dir::default_base() - .map_err(ScopeLockError::from)? - .join(&manifest.worker.name) - .join("sock"); - let scope_allocation = worker_allocation::install_top_level_with_deny( - manifest.worker.name.clone(), - std::process::id(), - socket_path, - common.scope.allow_rules(), - common.scope.deny_rules(), - segment_id, - )?; - // Build the worker and apply the manifest defaults first, then // overwrite the pieces the session log is authoritative for. let mut worker = @@ -9398,6 +9661,7 @@ where last_run_interrupted: state.last_run_interrupted, store, worker_metadata_writer, + persist_restored_manifest: true, worker_metadata_segment_cas, segment_state: SegmentState::new(session_id, segment_id, state.entries_count), filesystem_authority: common.filesystem_authority, @@ -9436,7 +9700,7 @@ where pending_attachments: Arc::new(Mutex::new(Vec::::new())), pending_committed_history: Arc::new(Mutex::new(std::collections::VecDeque::new())), failed_activation_authority: Mutex::new(None), - scope_allocation: Some(scope_allocation), + scope_allocation: None, callback_socket: None, runtime_ticket_role: None, prompts: common.prompts, @@ -9461,7 +9725,20 @@ where worker .session .restore_pending_activations(&state.extensions); - worker.finish_restored_construction().await + worker.apply_permissions_from_manifest(); + worker.apply_prune_from_manifest(); + let mut candidate = PreparedRestoredWorker { + worker: Some(worker), + closing: false, + metadata, + entries: entries_snapshot, + manifest: manifest_snapshot, + }; + if let Err(error) = candidate.revalidate(candidate.worker()) { + let worker = candidate.worker.take().expect("candidate just constructed"); + return Err(worker.construction_failure(error)); + } + Ok(candidate) } async fn finish_restored_construction(mut self) -> Result { @@ -10329,6 +10606,23 @@ pub enum WorkerError { sessions: Arc, }, + /// Activation entered canonical pin/metadata/delegation updates. Cleanup + /// ownership in the source does not establish rollback of those effects. + #[error("{source}")] + RestoreActivationEffects { + #[source] + source: Box, + }, + + #[error("restore candidate is closing; only cleanup retry is allowed")] + RestoreCandidateClosing, + + #[error("restore candidate source changed for Worker {worker_name}: {component}")] + RestoreCandidateChanged { + worker_name: String, + component: &'static str, + }, + #[error("invalid durable Worker state: {0}")] InvalidState(String), @@ -12998,6 +13292,686 @@ mod build_summary_prompt_tests { assert!(table.data().find("test-worker").is_none()); } + // Real Session/metadata stores, allocation and Workdir providers; only the + // model transport is substituted. Profile/Runtime publication belongs to the + // factory integration suite, not this reconstruction component suite. + mod restored_candidate_tests { + use super::*; + + type CandidateStore = + session_store::CombinedStore; + + #[derive(Clone)] + struct CountedClient(Arc); + + #[async_trait] + impl LlmClient for CountedClient { + async fn stream( + &self, + request: agen::llm_client::Request, + ) -> Result< + std::pin::Pin< + Box< + dyn futures::Stream< + Item = Result< + agen::llm_client::event::Event, + agen::llm_client::ClientError, + >, + > + Send, + >, + >, + agen::llm_client::ClientError, + > { + self.0.fetch_add(1, Ordering::SeqCst); + NoopClient.stream(request).await + } + + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + } + + #[derive(Debug)] + struct FailCloseOnce { + inner: workdir::WorkdirSessionHandle, + attempts: AtomicUsize, + } + + #[async_trait] + impl workdir::WorkdirSession for FailCloseOnce { + fn workdir(&self) -> &workdir::Workdir { + self.inner.workdir() + } + fn capabilities(&self) -> workdir::WorkdirSessionCapabilities { + self.inner.capabilities() + } + async fn stat( + &self, + request: workdir::StatRequest, + ) -> Result { + self.inner.stat(request).await + } + async fn read( + &self, + request: workdir::ReadRequest, + ) -> Result { + self.inner.read(request).await + } + async fn write( + &self, + request: workdir::WriteRequest, + ) -> Result { + self.inner.write(request).await + } + async fn edit( + &self, + request: workdir::EditRequest, + ) -> Result { + self.inner.edit(request).await + } + async fn list( + &self, + request: workdir::ListRequest, + ) -> Result { + self.inner.list(request).await + } + async fn glob( + &self, + request: workdir::GlobRequest, + ) -> Result { + self.inner.glob(request).await + } + async fn grep( + &self, + request: workdir::GrepRequest, + ) -> Result { + self.inner.grep(request).await + } + async fn start_command( + &self, + request: workdir::CommandRequest, + ) -> Result { + self.inner.start_command(request).await + } + async fn command_status( + &self, + handle: workdir::CommandHandle, + ) -> Result { + self.inner.command_status(handle).await + } + async fn command_output( + &self, + request: workdir::CommandOutputRequest, + ) -> Result { + self.inner.command_output(request).await + } + async fn cancel_command( + &self, + handle: workdir::CommandHandle, + ) -> Result<(), workdir::WorkdirError> { + self.inner.cancel_command(handle).await + } + async fn close(&self) -> Result<(), workdir::WorkdirError> { + self.inner.close().await?; + if self.attempts.fetch_add(1, Ordering::SeqCst) == 0 { + // The real provider has closed, but its first response is + // uncertain: the candidate must not become activatable again. + Err(workdir::WorkdirError::Unavailable( + "close response lost".into(), + )) + } else { + Ok(()) + } + } + } + + struct Fixture { + temp: tempfile::TempDir, + store: CandidateStore, + manifest: WorkerManifest, + session_id: SessionId, + segment_id: SegmentId, + orphan: protocol::UploadedFileRef, + model_calls: Arc, + } + + impl Fixture { + async fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let store = session_store::CombinedStore::new( + session_store::FsStore::new(temp.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(temp.path().join("workers")).unwrap(), + ); + let mut manifest = minimal_manifest(); + manifest.scope = ScopeConfig { + allow: vec![ScopeRule { + target: temp.path().canonicalize().unwrap(), + permission: Permission::Write, + recursive: true, + symlink_policy: Default::default(), + }], + deny: Vec::new(), + }; + let model_calls = Arc::new(AtomicUsize::new(0)); + let mut worker = Worker::new( + manifest.clone(), + Engine::, Mutable, SessionHistoryMetadata>::new_annotated( + Box::new(CountedClient(model_calls.clone())), + ), + store.clone(), + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + Scope::empty(), + ) + .await + .unwrap(); + worker + .engine_mut() + .set_system_prompt("saved candidate prompt"); + worker.ensure_segment_head().await.unwrap(); + let session_id = worker.session_id(); + let segment_id = worker.segment_id(); + let queue = worker.pending_submission_handle(); + queue + .accept( + "saved-submit".into(), + vec![Segment::Text { + content: "saved input".into(), + }], + false, + ) + .unwrap(); + queue + .accept_notification("saved-notification".into(), "saved notice".into()) + .unwrap(); + let orphan = store + .write_uploaded_file( + session_id, + "orphan.txt", + "text/plain", + b"orphan", + session_store::UploadedFileLimits::default(), + ) + .unwrap(); + store + .pin_uploaded_file(session_id, &orphan, "acceptance-without-checkpoint") + .unwrap(); + let mut metadata = WorkerMetadata::new( + &manifest.worker.name, + Some(WorkerActiveSegmentRef { + session_id, + segment_id: Some(segment_id), + }), + ); + // Activation replaces this stale derived path; preparation must not. + metadata.workspace_root = Some(temp.path().join("old-root")); + metadata.spawned_children.push(WorkerSpawnedChild { + worker_name: "missing-child".into(), + socket_path: temp.path().join("missing-child.sock"), + scope_delegated: Vec::new(), + callback_address: temp.path().join("unused-parent.sock"), + }); + metadata.resolved_manifest_snapshot = + Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap()); + store.write(&metadata).unwrap(); + drop(worker); + Self { + temp, + store, + manifest, + session_id, + segment_id, + orphan, + model_calls, + } + } + + async fn prepare(&self) -> PreparedRestoredWorker { + Worker::prepare_restored_from_worker_metadata_with_context_and_model_client( + &self.manifest.worker.name, + self.manifest.clone(), + self.store.clone(), + PromptCatalogSource::builtins_only(), + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::local( + self.temp.path().to_path_buf(), + self.temp.path().to_path_buf(), + ), + Some(Box::new(CountedClient(self.model_calls.clone()))), + ) + .await + .unwrap() + } + + fn allocate( + &self, + worker: &Worker, CandidateStore>, + ) -> Result { + Ok(worker_allocation::install_top_level_at_for_test( + self.temp.path().join("allocation.json"), + worker.manifest.worker.name.clone(), + std::process::id(), + self.temp.path().join("worker.sock"), + worker.scope.snapshot().allow_rules(), + worker.segment_id(), + )?) + } + + fn snapshot(&self) -> std::collections::BTreeMap> { + fn visit( + root: &Path, + path: &Path, + files: &mut std::collections::BTreeMap>, + ) { + for entry in std::fs::read_dir(path).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + visit(root, &path, files); + } else { + files.insert( + path.strip_prefix(root).unwrap().to_path_buf(), + std::fs::read(path).unwrap(), + ); + } + } + } + let mut files = std::collections::BTreeMap::new(); + for name in ["sessions", "workers"] { + visit(self.temp.path(), &self.temp.path().join(name), &mut files); + } + files + } + } + + #[tokio::test] + async fn prepare_and_discard_preserve_session_queue_pins_and_metadata() { + let fixture = Fixture::new().await; + let before = fixture.snapshot(); + let mut candidate = fixture.prepare().await; + assert!( + candidate + .worker() + .restore_allocation_cleanup_handle() + .is_none() + ); + assert!(!fixture.temp.path().join("worker.sock").exists()); + assert_eq!( + candidate.worker().pending_submissions().submissions.len(), + 1 + ); + assert_eq!( + candidate + .worker() + .session + .pending_activations + .lock() + .unwrap() + .pending_notifications + .len(), + 1 + ); + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + let session = candidate + .worker() + .workdir_sessions() + .resolve(None) + .unwrap() + .session; + assert!( + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root() + }) + .await + .is_ok() + ); + candidate.discard().await.unwrap(); + candidate.discard().await.unwrap(); + assert!( + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root() + }) + .await + .is_err() + ); + assert_eq!(fixture.snapshot(), before); + } + + #[tokio::test] + async fn failed_discard_allows_only_cleanup_retry_not_binding_or_activation() { + for retry_discard in [true, false] { + let fixture = Fixture::new().await; + let before = fixture.snapshot(); + let mut candidate = fixture.prepare().await; + let provider = Arc::new(FailCloseOnce { + inner: candidate.workdir_sessions().resolve(None).unwrap().session, + attempts: AtomicUsize::new(0), + }); + candidate + .worker_mut() + .unwrap() + .bind_single_workdir_session(Some(provider.clone())); + let cleanup = candidate.workdir_sessions(); + assert!(matches!( + candidate.discard().await, + Err(WorkerError::FeatureLifecycle(_)) + )); + assert_eq!(provider.attempts.load(Ordering::SeqCst), 1); + assert!(!cleanup.is_empty()); + assert_eq!(fixture.snapshot(), before); + assert!(matches!( + candidate.worker_mut(), + Err(WorkerError::RestoreCandidateClosing) + )); + assert_eq!( + candidate.manifest().worker.name, + fixture.manifest.worker.name + ); + assert!(Arc::ptr_eq(&candidate.workdir_sessions(), &cleanup)); + if retry_discard { + candidate.discard().await.unwrap(); + assert!(cleanup.is_empty()); + assert_eq!(provider.attempts.load(Ordering::SeqCst), 2); + assert!(matches!( + candidate.worker_mut(), + Err(WorkerError::RestoreCandidateClosing) + )); + candidate.discard().await.unwrap(); + } else { + let allocations = AtomicUsize::new(0); + let error = match candidate + .activate_with_allocation(|worker| { + allocations.fetch_add(1, Ordering::SeqCst); + fixture.allocate(worker) + }) + .await + { + Err(error) => error, + Ok(_) => panic!("partially closed candidate was activated"), + }; + let WorkerError::ConstructionCleanup { + source, + allocation, + sessions, + } = error + else { + panic!("cleanup ownership lost"); + }; + assert!(matches!(*source, WorkerError::RestoreCandidateClosing)); + assert!(allocation.is_none()); + assert_eq!(allocations.load(Ordering::SeqCst), 0); + assert_eq!(provider.attempts.load(Ordering::SeqCst), 1); + assert!(Arc::ptr_eq(&sessions, &cleanup)); + sessions.close_all().await.unwrap(); + assert!(cleanup.is_empty()); + assert_eq!(provider.attempts.load(Ordering::SeqCst), 2); + } + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + } + } + + #[tokio::test] + async fn dropping_candidate_closes_workdir_without_consuming_saved_inputs() { + let fixture = Fixture::new().await; + let before = fixture.snapshot(); + let candidate = fixture.prepare().await; + let sessions = candidate.worker().workdir_sessions(); + let session = sessions.resolve(None).unwrap().session; + drop(candidate); + tokio::time::timeout(Duration::from_secs(5), async { + while !sessions.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("candidate Drop must finish direct close"); + assert!( + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root() + }) + .await + .is_err() + ); + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn activation_alone_reconciles_orphan_pins_and_updates_active_metadata() { + let fixture = Fixture::new().await; + let before_metadata = fixture + .store + .read_by_name(&fixture.manifest.worker.name) + .unwrap() + .unwrap(); + let before_entries = restore_snapshot_value( + &fixture + .store + .read_all(fixture.session_id, fixture.segment_id) + .unwrap(), + ) + .unwrap(); + let candidate = fixture.prepare().await; + assert!(matches!( + fixture + .store + .delete_uploaded_file(fixture.session_id, &fixture.orphan.artifact_id), + Err(StoreError::ArtifactAlreadyCommitted) + )); + let worker = candidate + .activate_with_allocation(|worker| fixture.allocate(worker)) + .await + .unwrap(); + assert!(worker.restore_allocation_cleanup_handle().is_some()); + let metadata = fixture + .store + .read_by_name(&fixture.manifest.worker.name) + .unwrap() + .unwrap(); + assert_ne!(metadata.workspace_root, before_metadata.workspace_root); + assert_eq!(metadata.active, before_metadata.active); + assert_eq!(before_metadata.spawned_children.len(), 1); + assert!(metadata.spawned_children.is_empty()); + assert_eq!(metadata.reclaimed_children.len(), 1); + assert!( + fixture + .store + .delete_uploaded_file(fixture.session_id, &fixture.orphan.artifact_id) + .unwrap() + ); + assert_eq!(worker.pending_submissions().submissions.len(), 1); + assert_eq!( + worker + .session + .pending_activations + .lock() + .unwrap() + .pending_notifications + .len(), + 1 + ); + assert_eq!( + restore_snapshot_value( + &fixture + .store + .read_all(fixture.session_id, fixture.segment_id) + .unwrap() + ) + .unwrap(), + before_entries + ); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + worker.workdir_sessions().close_all().await.unwrap(); + let allocation = worker.restore_allocation_cleanup_handle().unwrap(); + assert!(matches!( + allocation.release(), + Err(ScopeLockError::AllocationGuardAttached { .. }) + )); + drop(worker); + allocation.release().unwrap(); + } + + #[tokio::test] + async fn metadata_or_same_entry_count_history_drift_rejects_before_allocation() { + for component in ["metadata", "segment", "manifest"] { + let fixture = Fixture::new().await; + let mut candidate = fixture.prepare().await; + match component { + "metadata" => { + fixture + .store + .update_by_name(&fixture.manifest.worker.name, |metadata| { + metadata.workspace_id = Some("changed-workspace".into()); + }) + .unwrap(); + } + "segment" => { + let mut entries = fixture + .store + .read_all(fixture.session_id, fixture.segment_id) + .unwrap(); + let LogEntry::AnnotatedSegmentStart { system_prompt, .. } = &mut entries[0] + else { + panic!("fixture seed"); + }; + *system_prompt = Some("different same-length history".into()); + fixture + .store + .create_segment(fixture.session_id, fixture.segment_id, &entries) + .unwrap(); + } + "manifest" => { + candidate.worker_mut().unwrap().manifest.model.model_id = + Some("changed-model".into()); + } + _ => unreachable!(), + } + let before = fixture.snapshot(); + let allocations = AtomicUsize::new(0); + let error = match candidate + .activate_with_allocation(|worker| { + allocations.fetch_add(1, Ordering::SeqCst); + fixture.allocate(worker) + }) + .await + { + Err(error) => error, + Ok(_) => panic!("stale candidate accepted"), + }; + let WorkerError::ConstructionCleanup { + source, + allocation, + sessions, + } = error + else { + panic!("cleanup ownership lost"); + }; + assert!( + matches!(*source, WorkerError::RestoreCandidateChanged { component: changed, .. } if changed == component) + ); + assert!(allocation.is_none()); + assert_eq!(allocations.load(Ordering::SeqCst), 0); + sessions.close_all().await.unwrap(); + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + } + } + + #[tokio::test] + async fn post_allocation_metadata_drift_retains_cleanup_without_mutating_pins() { + let fixture = Fixture::new().await; + let candidate = fixture.prepare().await; + let error = match candidate + .activate_with_allocation(|worker| { + let guard = fixture.allocate(worker)?; + fixture + .store + .update_by_name(&fixture.manifest.worker.name, |metadata| { + metadata.workspace_id = Some("changed-under-allocation".into()); + })?; + Ok(guard) + }) + .await + { + Err(error) => error, + Ok(_) => panic!("stale candidate accepted"), + }; + let before = fixture.snapshot(); + let WorkerError::ConstructionCleanup { + source, + allocation, + sessions, + } = error + else { + panic!("cleanup ownership lost"); + }; + assert!(matches!( + *source, + WorkerError::RestoreCandidateChanged { + component: "metadata", + .. + } + )); + let allocation = + allocation.expect("allocation retained after second validation failure"); + assert!( + worker_allocation::LockFileGuard::open( + &fixture.temp.path().join("allocation.json") + ) + .unwrap() + .data() + .find(&fixture.manifest.worker.name) + .is_some() + ); + assert!(matches!( + fixture + .store + .delete_uploaded_file(fixture.session_id, &fixture.orphan.artifact_id), + Err(StoreError::ArtifactAlreadyCommitted) + )); + sessions.close_all().await.unwrap(); + worker_allocation::fail_next_save_for_test( + &fixture.temp.path().join("allocation.json"), + ); + assert!(allocation.release().is_err()); + allocation.release().unwrap(); + assert_eq!(fixture.snapshot(), before); + } + + #[tokio::test] + async fn pending_session_is_rejected_without_repair_or_session_creation() { + let fixture = Fixture::new().await; + fixture + .store + .update_by_name(&fixture.manifest.worker.name, |metadata| { + metadata.active.as_mut().unwrap().segment_id = None; + }) + .unwrap(); + let before = fixture.snapshot(); + let result = + Worker::prepare_restored_from_worker_metadata_with_context_and_model_client( + &fixture.manifest.worker.name, + fixture.manifest.clone(), + fixture.store.clone(), + PromptCatalogSource::builtins_only(), + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + Some(Box::new(CountedClient(fixture.model_calls.clone()))), + ) + .await; + assert!(matches!( + result, + Err(WorkerError::WorkerMetadataPending { .. }) + )); + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + } + } + // Constructor finalization uses real Worker, allocation, Workdir, and storage // implementations. Only the model and metadata-write failure boundary are substituted. mod construction_cleanup_tests { diff --git a/crates/worker/tests/controller_test.rs b/crates/worker/tests/controller_test.rs index 1009b79fd..2fe26e3d6 100644 --- a/crates/worker/tests/controller_test.rs +++ b/crates/worker/tests/controller_test.rs @@ -1,3 +1,4 @@ +use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -150,6 +151,19 @@ impl LlmClient for MockClient { // Helpers // --------------------------------------------------------------------------- +#[path = "../../../tests/support/restore_environment.rs"] +mod restore_test_environment; +use restore_test_environment::require_restore_test_environment; + +fn owned_relative_runtime_base() -> (tempfile::TempDir, PathBuf) { + require_restore_test_environment(); + let temp = tempfile::tempdir_in(".").unwrap(); + // The relative reference and the TempDir name the same owned tree, without + // putting the checkout prefix into sockaddr_un or changing process cwd. + let runtime_base = Path::new(temp.path().file_name().unwrap()).join("runtime"); + (temp, runtime_base) +} + fn simple_text_events() -> Vec { vec![ LlmEvent::text_block_start(0), @@ -295,6 +309,7 @@ async fn make_worker_with_manifest_and_workspace_context( manifest: WorkerManifest, workspace_context: WorkerWorkspaceContext, ) -> (Worker, std::path::PathBuf) { + require_restore_test_environment(); let store_tmp = tempfile::tempdir().unwrap(); let store = CombinedStore::new( FsStore::new(store_tmp.path()).unwrap(), @@ -320,18 +335,21 @@ async fn make_worker_with_manifest_and_workspace_context( } async fn spawn_controller(worker: Worker) -> WorkerHandle { - let tmp = tempfile::tempdir().unwrap(); - let runtime_base = tmp.path().to_owned(); - std::mem::forget(tmp); - let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base) + let (tmp, runtime_base) = owned_relative_runtime_base(); + let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base) .await .unwrap(); + // Retain ownership until controller shutdown (or test-runtime teardown). + tokio::spawn(async move { + let _ = shutdown_rx.await; + drop(tmp); + }); handle } #[tokio::test] async fn canonical_worker_handle_uploads_and_submits_attachment() { - let root = tempfile::tempdir().unwrap(); + let (root, runtime_base) = owned_relative_runtime_base(); let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap(); let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap(); let store = CombinedStore::new( @@ -353,7 +371,7 @@ async fn canonical_worker_handle_uploads_and_submits_attachment() { ) .await .unwrap(); - let (handle, shutdown_rx) = WorkerController::spawn(worker, &root.path().join("runtime")) + let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base) .await .unwrap(); let mut rx = handle.subscribe(); @@ -1761,11 +1779,12 @@ permission = "write" "#; let client = MockClient::new(simple_text_events()); let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0; - let tmp = tempfile::tempdir().unwrap(); - let result = WorkerController::spawn(worker, tmp.path()).await; + let (_tmp, runtime_base) = owned_relative_runtime_base(); + let result = WorkerController::spawn(worker, &runtime_base).await; assert!( result.is_ok(), - "feature exposure must not imply delegation authority" + "feature exposure must not imply delegation authority: {:?}", + result.as_ref().err() ); } diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index 02c40beb4..dd117ae7b 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -19,6 +19,9 @@ pub mod repositories; pub mod repository_access; pub mod repository_source; pub mod resource_broker; +#[cfg(test)] +#[path = "../../../tests/support/restore_environment.rs"] +mod restore_test_environment; pub mod retention; pub mod runtime_settings; pub mod runtime_subscription; diff --git a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs index 91093f5fc..d1a4d6c61 100644 --- a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs +++ b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs @@ -11,6 +11,20 @@ use worker_runtime::worker_backend::{ WorkerRuntimeExecutionBackend, }; +pub(super) fn connected_restore_tempdir() -> tempfile::TempDir { + crate::restore_test_environment::require_restore_test_environment(); + let checkout = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap(); + tempfile::Builder::new() + .prefix("c") + .rand_bytes(3) + .tempdir_in(checkout.join("target")) + .unwrap() +} + const WATCHDOG: Duration = Duration::from_secs(45); #[derive(Clone)] @@ -82,27 +96,91 @@ impl RuntimeWorkerFactory for CapturingFactory { ) -> std::result::Result<(), String> { self.inner.preflight_restore(request).await } - async fn restore_controller( + async fn recover_restore_source( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.inner.recover_restore_source(request).await + } + + async fn prepare_restore( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + let worker_ref = request.worker_ref.clone(); + let inner = self.inner.prepare_restore(request).await?; + Ok(Box::new(CapturingCandidate { + inner, + worker_ref, + captures: self.captures.clone(), + })) + } + + async fn cleanup_failed_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.cleanup_failed_restore(request).await + } + + async fn reconcile_stopped_worker( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.reconcile_stopped_worker(worker_ref).await + } + + fn activate_restored_controller( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + workspace_id: Option<&str>, + handle: &worker::WorkerHandle, + ) { + self.inner + .activate_restored_controller(worker_ref, workspace_id, handle); + } +} + +struct CapturingCandidate { + inner: Box, + worker_ref: worker_runtime::identity::WorkerRef, + captures: Arc, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for CapturingCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.inner.initial_state() + } + + async fn activate( + &mut self, ) -> std::result::Result< RuntimeWorkerController, worker_runtime::worker_backend::RuntimeRestoreFailure, > { - let worker_ref = request.worker_ref.clone(); - let controller = self.inner.restore_controller(request).await?; + let controller = self.inner.activate().await?; self.captures .handles .lock() .unwrap() - .push((worker_ref.clone(), controller.handle.clone())); + .push((self.worker_ref.clone(), controller.handle.clone())); self.captures .restored_clients .lock() .unwrap() - .push((worker_ref, controller.workspace_client.clone())); + .push((self.worker_ref.clone(), controller.workspace_client.clone())); Ok(controller) } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.discard().await + } } struct Resources { @@ -262,7 +340,7 @@ async fn call(tools: &BTreeMap>, name: &str, args: Value) #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn restored_runtime_parent_lists_and_inputs_existing_child() { - let root = tempfile::tempdir().unwrap(); + let root = connected_restore_tempdir(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let base = format!("http://{}", listener.local_addr().unwrap()); let mut config = test_server_config(root.path()); diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs index 4cc4b5c24..3b9215113 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs @@ -11,6 +11,8 @@ use agen::llm_client::{ use serde_json::{Value, json}; use std::sync::{Arc, Mutex}; +use super::super::restored_parent_control_tests::connected_restore_tempdir; + #[derive(Clone, Default)] struct ConnectedJournalModel { dialogs: Arc>>, @@ -320,7 +322,7 @@ async fn connected_wait_for_recording( #[tokio::test(flavor = "multi_thread", worker_threads = 6)] async fn backend_real_runtime_journal_release_and_next_body_persisted_startup() { - let dir = tempfile::tempdir().unwrap(); + let dir = connected_restore_tempdir(); init_clean_git_workspace(dir.path()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let mut config = test_server_config(dir.path()); @@ -896,3 +898,1579 @@ fn connected_host_reader( ) .unwrap() } + +// This wrapper changes timing only. Production owns reconstruction, recovery, +// activation and cleanup; the real SQLite trust binding supplies the denial. +#[derive(Default)] +struct RestoreAuthProbe { + prepared: tokio::sync::Notify, + release: tokio::sync::Notify, + prepares: std::sync::atomic::AtomicUsize, + activations: std::sync::atomic::AtomicUsize, + publications: std::sync::atomic::AtomicUsize, + handles: Mutex>, + controller_tasks: Mutex>, + operations: Mutex>, + repairs: Mutex>, + repair_operations: Mutex>, + gate_repair: std::sync::atomic::AtomicBool, + repairing: tokio::sync::Notify, + release_repair: tokio::sync::Notify, +} + +struct RestoreAuthFactory { + inner: worker_runtime::worker_backend::ProfileRuntimeWorkerFactory, + probe: Arc, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeWorkerFactory for RestoreAuthFactory { + fn observe_workspace_prompt_projection( + &self, + projection: worker::WorkspacePromptProjection, + ) -> std::result::Result<(), String> { + self.inner.observe_workspace_prompt_projection(projection) + } + + async fn fetch_workspace_config( + &self, + request: worker_runtime::execution::WorkspaceConfigFetchRequest, + ) -> std::result::Result { + self.inner.fetch_workspace_config(request).await + } + + fn retained_session_snapshot( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result< + session_store::RetainedSessionSnapshot, + session_store::RetainedSnapshotReadError, + > { + self.inner.retained_session_snapshot(worker_ref) + } + + fn retained_session_history_page( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + cursor: Option<&str>, + limit: Option, + ) -> std::result::Result + { + self.inner + .retained_session_history_page(worker_ref, cursor, limit) + } + + fn retained_session_attachment( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + session_id: &str, + attachment_id: &str, + ) -> std::result::Result< + session_store::RetainedSessionAttachment, + session_store::RetainedAttachmentReadError, + > { + self.inner + .retained_session_attachment(worker_ref, session_id, attachment_id) + } + + async fn spawn_controller( + &self, + mut request: worker_runtime::execution::WorkerExecutionSpawnRequest, + ) -> std::result::Result { + if request.request.backend_job.is_some() { + return self.inner.spawn_controller(request).await; + } + // Save a no-auth model too, so a restore regression cannot reach developer + // credentials. Only the external model stream is replaced, not Host I/O. + let archive = worker_runtime::profile_archive::ProfileSourceArchive::build( + worker_runtime::profile_archive::ProfileSourceArchiveInput { + id: "profile-source-archive:restore-host-auth".into(), + entrypoints: BTreeMap::from([("builtin:companion".into(), "profiles/body.dcdl".into())]), + imports: BTreeMap::new(), + sources: BTreeMap::from([("profiles/body.dcdl".into(), r#"{ + slug = "body"; description = "Connected restore Host authority"; scope = "workspace_read"; + model = { scheme = "anthropic"; model_id = "test-model"; auth = { kind = "none"; }; }; + engine = { max_tokens = 100; }; + feature = { subjektiv = { enabled = true; extraction = { enabled = false; }; }; }; + }"#.into())]), + }, + ).map_err(|error| error.to_string())?; + request.request.profile_source = + worker_runtime::catalog::ProfileSourceArchiveSource::Embedded { archive }; + let controller = self.inner.spawn_controller(request).await?; + self.probe + .controller_tasks + .lock() + .unwrap() + .push(controller.controller_task.abort_handle()); + self.probe + .handles + .lock() + .unwrap() + .push(controller.handle.clone()); + Ok(controller) + } + + async fn preflight_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), String> { + self.inner.preflight_restore(request).await + } + + async fn recover_restore_source( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.probe + .repair_operations + .lock() + .unwrap() + .push(serde_json::to_value(request.operation_id).unwrap()); + if self + .probe + .gate_repair + .swap(false, std::sync::atomic::Ordering::SeqCst) + { + self.probe.repairing.notify_one(); + self.probe.release_repair.notified().await; + } + let repaired = self.inner.recover_restore_source(request).await?; + self.probe.repairs.lock().unwrap().push(repaired); + Ok(repaired) + } + + async fn prepare_restore( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + let operation = serde_json::to_value(request.operation_id).unwrap(); + let inner = self.inner.prepare_restore(request).await?; + self.probe.operations.lock().unwrap().push(operation); + if self + .probe + .prepares + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + == 0 + { + self.probe.prepared.notify_one(); + self.probe.release.notified().await; + } + Ok(Box::new(RestoreAuthCandidate { + inner, + probe: self.probe.clone(), + })) + } + + async fn cleanup_failed_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.cleanup_failed_restore(request).await + } + + async fn reconcile_stopped_worker( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.reconcile_stopped_worker(worker_ref).await + } + + fn activate_restored_controller( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + workspace_id: Option<&str>, + handle: &worker::WorkerHandle, + ) { + self.inner + .activate_restored_controller(worker_ref, workspace_id, handle); + } +} + +struct RestoreAuthCandidate { + inner: Box, + probe: Arc, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for RestoreAuthCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.inner.initial_state() + } + + async fn activate( + &mut self, + ) -> std::result::Result< + worker_runtime::worker_backend::RuntimeWorkerController, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + self.probe + .activations + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let controller = self.inner.activate().await?; + self.probe + .publications + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.probe + .controller_tasks + .lock() + .unwrap() + .push(controller.controller_task.abort_handle()); + self.probe + .handles + .lock() + .unwrap() + .push(controller.handle.clone()); + Ok(controller) + } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.discard().await + } +} + +#[derive(Clone, Default)] +struct RestoreAuthModel { + calls: Arc, + jobs: ConnectedJournalModel, +} + +#[async_trait::async_trait] +impl LlmClient for RestoreAuthModel { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + + async fn stream( + &self, + request: Request, + ) -> std::result::Result { + use futures::StreamExt; + if request.tools.iter().any(|tool| { + matches!( + tool.name.as_str(), + "ReadJournalState" | "SubmitBackendJobResult" | "SubmitMemorySurface" + ) + }) { + return self.jobs.stream(request).await; + } + // Measure the saved body, not independent Journal jobs admitted by its + // seed turn before Stop; those have a different execution owner. + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(Box::pin( + futures::stream::iter([ + Ok(Event::text_block_start(0)), + Ok(Event::text_delta( + 0, + "fixture turn held for durable queue setup", + )), + ]) + .chain(futures::stream::pending()), + )) + } +} + +struct ConnectedRestoreServer { + server: tokio::task::JoinHandle<()>, + probe: Arc, +} + +impl Drop for ConnectedRestoreServer { + fn drop(&mut self) { + // Unblock a prepared candidate even if an assertion unwinds at the gate. + self.probe.release.notify_one(); + self.probe.release_repair.notify_one(); + self.server.abort(); + for task in self.probe.controller_tasks.lock().unwrap().iter() { + task.abort(); + } + } +} + +async fn restore_auth_event( + events: &mut tokio::sync::broadcast::Receiver, + predicate: impl Fn(&protocol::Event) -> bool, +) -> protocol::Event { + tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let event = events.recv().await.expect("real Controller event stream"); + assert!( + !matches!( + event, + protocol::Event::Error { .. } + | protocol::Event::SubmissionRejected { .. } + | protocol::Event::NotificationRejected { .. } + ), + "{event:?}" + ); + if predicate(&event) { + return event; + } + } + }) + .await + .expect("real Controller acknowledgement timed out") +} + +// Only initial creation is substituted for the pending-head case: it writes a +// genuine first-Session Worker, but publishes no Controller/head. Every restore, +// repair, Host authorization, activation and Stop delegates to production. +struct PendingHeadSeedBackend { + inner: Arc>, + seed: std::sync::atomic::AtomicBool, + root: std::path::PathBuf, + model: RestoreAuthModel, +} + +impl worker_runtime::execution::WorkerExecutionBackend for PendingHeadSeedBackend { + fn backend_id(&self) -> &str { + self.inner.backend_id() + } + fn spawn_worker( + &self, + request: worker_runtime::execution::WorkerExecutionSpawnRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + if !self.seed.swap(false, std::sync::atomic::Ordering::SeqCst) { + return self.inner.spawn_worker(request); + } + let root = self + .root + .join("workers") + .join(request.worker_ref.worker_id.to_string()); + let name = format!("worker-runtime-{}", request.worker_ref.worker_id); + let model = self.model.clone(); + let settings = request.request.memory_settings.clone().unwrap(); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + use session_store::WorkerMetadataStore as _; + let mut manifest = manifest::WorkerManifest::from_toml(&format!( + r#" +[worker] +name = "{name}" +pwd = "./" +[scope] +allow = [] +[model] +scheme = "anthropic" +model_id = "test-model" +auth = {{ kind = "none" }} +[engine] +max_tokens = 100 +[feature.subjektiv.profile] +enabled = true +[feature.subjektiv.profile.extraction] +enabled = false +"# + )) + .unwrap(); + manifest + .feature + .subjektiv + .bind_workspace_settings(settings) + .unwrap(); + let sessions = + session_store::WorkerSessionStore::new(root.join("session")).unwrap(); + let metadata = session_store::WorkerAggregateStore::new(&root, &name).unwrap(); + let engine = agen::Engine::< + _, + agen::state::Mutable, + worker::SessionHistoryMetadata, + >::new_annotated(model); + let mut head = worker::Worker::new( + manifest.clone(), + engine, + session_store::CombinedStore::new(sessions.clone(), metadata.clone()), + worker::WorkerWorkspaceContext::unavailable( + Some(worker::WorkspaceId::new(TEST_WORKSPACE_ID).unwrap()), + "initial creation interrupted before first head", + ), + worker::WorkerFilesystemAuthority::None, + manifest::Scope::empty(), + ) + .await + .unwrap(); + head.enable_worker_metadata_write_through().unwrap(); + let mut saved = metadata.read_by_name(&name).unwrap().unwrap(); + assert!(saved.active.as_ref().unwrap().segment_id.is_none()); + saved.resolved_manifest_snapshot = Some( + manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap(), + ); + metadata.write(&saved).unwrap(); + drop(head); + }); + }) + .join() + .unwrap(); + worker_runtime::execution::WorkerExecutionSpawnResult::Connected { + worker_state: protocol::WorkerStateSnapshot::initial(), + workdir_attachments: Vec::new(), + } + } + fn fetch_workspace_config( + &self, + request: worker_runtime::execution::WorkspaceConfigFetchRequest, + ) -> std::result::Result { + self.inner.fetch_workspace_config(request) + } + fn observe_workspace_prompt_projection( + &self, + projection: worker::WorkspacePromptProjection, + ) -> std::result::Result<(), String> { + self.inner.observe_workspace_prompt_projection(projection) + } + fn preflight_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), worker_runtime::execution::WorkerExecutionResult> { + self.inner.preflight_restore(request) + } + fn completed_restore_failure( + &self, + id: worker_runtime::execution::WorkerLifecycleOperationId, + worker: &worker_runtime::identity::WorkerRef, + ) -> Option { + self.inner.completed_restore_failure(id, worker) + } + fn restore_worker( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.inner.restore_worker(request) + } + fn reconcile_restore( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.inner.reconcile_restore(request) + } + fn activate_restored_worker( + &self, + id: worker_runtime::execution::WorkerLifecycleOperationId, + worker: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.activate_restored_worker(id, worker) + } + fn dispatch_input( + &self, + worker: &worker_runtime::identity::WorkerRef, + input: worker_runtime::interaction::WorkerInput, + ) -> worker_runtime::execution::WorkerExecutionResult { + self.inner.dispatch_input(worker, input) + } + fn dispatch_method( + &self, + worker: &worker_runtime::identity::WorkerRef, + method: protocol::Method, + ) -> worker_runtime::execution::WorkerExecutionResult { + self.inner.dispatch_method(worker, method) + } + fn stop_worker_operation( + &self, + request: worker_runtime::execution::WorkerExecutionStopRequest, + ) -> worker_runtime::execution::WorkerExecutionResult { + self.inner.stop_worker_operation(request) + } + fn worker_snapshot( + &self, + worker: &worker_runtime::identity::WorkerRef, + ) -> Option { + self.inner.worker_snapshot(worker) + } +} + +// One real connected seed and stopped queue setup shared by the publication +// denial and explicit-repair regressions. Only timing and response delivery are +// intercepted: every callback still passes the production signer/router/store. +struct ConnectedRestoreFixture { + api: WorkspaceApi, + execution: + Arc>, + probe: Arc, + model: RestoreAuthModel, + worker: RuntimeWorkerRef, + subject_id: String, + requests: Arc>>, + callbacks: Arc>>, + lose_response: Arc, + _server: ConnectedRestoreServer, + _dir: tempfile::TempDir, +} + +async fn connected_restore_fixture() -> ConnectedRestoreFixture { + connected_restore_fixture_with_pending_head(false).await +} + +async fn connected_restore_fixture_with_pending_head( + pending_head: bool, +) -> ConnectedRestoreFixture { + use std::sync::atomic::Ordering::SeqCst; + + // Guard the resolved global paths before factory/config/allocation access. + // No Git or Workdir attachment is needed for this connected regression. + let dir = connected_restore_tempdir(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut config = test_server_config(dir.path()); + config.repositories.clear(); + config.backend_base_url = Some(format!("http://{}", listener.local_addr().unwrap())); + let control = Arc::new(test_control_store(&config)); + seed_test_registered_workspace(control.as_ref(), &config) + .await + .unwrap(); + let broker = BackendResourceBroker::default(); + let dispatcher = Arc::new( + crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new( + config.clone(), + control.clone(), + ), + ); + let model = RestoreAuthModel::default(); + let model_factory = model.clone(); + let audience = embedded_runtime_request_audience(&config).unwrap(); + let probe = Arc::new(RestoreAuthProbe::default()); + let factory = worker_runtime::worker_backend::ProfileRuntimeWorkerFactory::new( + config.workspace_execution_root.clone(), + ) + .with_controller_transport(worker::WorkerControllerTransport::InProcess) + .with_embedded_worker_mutation_dispatcher(EMBEDDED_WORKER_RUNTIME_ID, dispatcher.clone()) + .with_workspace_request_client( + worker_runtime::workspace_request::RuntimeWorkspaceRequestClient::new( + TEST_WORKSPACE_ID, + audience.clone(), + EMBEDDED_WORKER_RUNTIME_ID, + ) + .with_runtime_request_source(&EMBEDDED_RUNTIME_REQUEST_IDENTITY, audience), + ) + .with_runtime_store_dir(config.embedded_runtime_store_root.clone()) + .with_resource_client(Arc::new(broker.clone())) + .with_spawn_model_client_factory(move || Box::new(model_factory.clone())); + let execution = Arc::new( + worker_runtime::worker_backend::WorkerRuntimeExecutionBackend::new(RestoreAuthFactory { + inner: factory, + probe: probe.clone(), + }) + .unwrap(), + ); + let seeded_execution = Arc::new(PendingHeadSeedBackend { + inner: execution.clone(), + seed: std::sync::atomic::AtomicBool::new(pending_head), + root: config.embedded_runtime_store_root.clone(), + model: model.clone(), + }); + let api = WorkspaceApi::new_with_execution_backend_and_broker( + config, + control, + seeded_execution, + broker, + HostExecutors::default(), + Some(dispatcher), + ) + .await + .unwrap(); + seed_test_delivery_binding(&api); + let requests = Arc::new(Mutex::new(Vec::new())); + let observed = requests.clone(); + let callbacks = Arc::new(Mutex::new(Vec::new())); + let recorded = callbacks.clone(); + let lose_response = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let lose_once = lose_response.clone(); + let app = build_router(api.clone()).layer(axum::middleware::from_fn( + move |request: axum::extract::Request, next: axum::middleware::Next| { + let observed = observed.clone(); + let recorded = recorded.clone(); + let lose_once = lose_once.clone(); + async move { + let path = request.uri().path().to_owned(); + let is_session = path.ends_with("/subjektiv/sessions"); + let request = if is_session { + let (parts, body) = request.into_parts(); + let bytes = to_bytes(body, usize::MAX).await.unwrap(); + recorded + .lock() + .unwrap() + .push(serde_json::from_slice(&bytes).unwrap()); + axum::http::Request::from_parts(parts, Body::from(bytes)) + } else { + request + }; + let response = next.run(request).await; + observed.lock().unwrap().push((path, response.status())); + if is_session + && response.status() == StatusCode::OK + && lose_once.swap(false, SeqCst) + { + // The real handler has already persisted its immutable SQLite + // attribution. Lose only its acknowledgement, never fake a Host. + axum::http::Response::builder() + .status(StatusCode::BAD_GATEWAY) + .body(Body::from("fixture lost committed Host response")) + .unwrap() + } else { + response + } + } + }, + )); + let server = ConnectedRestoreServer { + server: tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }), + probe: probe.clone(), + }; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject(crate::subjektiv::SubjectRole::new("restore-host-auth").unwrap()) + .unwrap(); + let owner = + seed_test_workspace_owner_token(api.store.as_ref(), "restore-host-auth-owner").await; + let update = server_api::authorization::SubjectPermissionsUpdate { + operation_id: "restore-host-auth-permissions".into(), + expected_change_id: None, + execution: [ + "task", + "subjektiv", + "web", + "image", + "sub_worker", + "worker", + "workspace_worker_discovery", + "objective", + "manage_workdir", + "workdir_catalog", + "drive", + "ticket", + ] + .into_iter() + .map(|name| server_api::authorization::ScopedPermission { + operation: name.into(), + resource: server_api::authorization::PermissionResource::Feature, + }) + .collect(), + delegation: vec![], + }; + let response = build_router(api.clone()) + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(format!( + "/api/w/{TEST_WORKSPACE_ID}/subjektiv/subjects/{}/permissions", + subject.id + )) + .header("authorization", format!("Bearer {owner}")) + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_vec(&update).unwrap())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let Json(created) = scoped_create_workspace_worker( + State(api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.into(), + }), + HeaderMap::new(), + Json(CreateWorkspaceWorkerRequest { + runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(), + display_name: "restore-auth-body".into(), + singleton_key: None, + profile: Some("builtin:companion".into()), + ticket_assignment: None, + initial_submit: vec![], + workdir_attachments: vec![], + feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest { + subjektiv: Some(server_api::SubjektivWorkerConnectionRequest { + subject_id: subject.id.clone(), + }), + }, + control_operation_id: None, + execution_grant: None, + }), + ) + .await + .unwrap(); + let worker = RuntimeWorkerRef::new(created.runtime_id, created.worker_id); + if pending_head { + assert_eq!( + api.runtime + .stop_worker(&worker, WorkerLifecycleRequest::default()) + .unwrap() + .state, + InternalWorkerOperationState::Accepted + ); + } else { + let handle = probe.handles.lock().unwrap()[0].clone(); + let mut events = handle.subscribe(); + handle + .send(protocol::Method::submit_text( + "setup-running", + "hold for queue setup", + )) + .await + .unwrap(); + restore_auth_event(&mut events, |event| { + matches!(event, protocol::Event::TextDelta { .. }) + }) + .await; + let command = protocol::WorkerCommandEnvelope::new( + handle.shared_state.snapshot().last_command_id + 1, + ); + handle + .send(protocol::Method::Pause { command }) + .await + .unwrap(); + restore_auth_event(&mut events, |event| { + matches!(event, protocol::Event::WorkerState { snapshot } + if snapshot.catalog_status() == protocol::WorkerStatus::Paused) + }) + .await; + handle + .send(protocol::Method::submit_text( + "saved-submit", + "must remain queued", + )) + .await + .unwrap(); + restore_auth_event(&mut events, |event| { + matches!(event, protocol::Event::SubmissionAccepted { + submission_request_id, disposition: protocol::SubmissionDisposition::Queued, .. + } if submission_request_id == "saved-submit") + }) + .await; + handle + .send(protocol::Method::Notify { + notification_request_id: "saved-notice".into(), + message: "must remain queued too".into(), + }) + .await + .unwrap(); + restore_auth_event(&mut events, |event| { + matches!(event, protocol::Event::NotificationAccepted { + notification_request_id + } if notification_request_id == "saved-notice") + }) + .await; + let stopped = api + .runtime + .stop_worker(&worker, WorkerLifecycleRequest::default()) + .unwrap(); + assert_eq!(stopped.state, InternalWorkerOperationState::Accepted); + assert!(handle.protocol_is_closed()); + // Pause/Shutdown can admit real Journal work for the seed turn. Finish that + // owned work before measuring the restore window; it is not restore I/O. + let attr = store + .subject_session_attributions(&subject.id) + .unwrap() + .into_iter() + .find(|attr| attr.worker_id == worker.worker_id) + .unwrap(); + let host = JournalHostContext::validated_session(&store, attr.clone()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(30), async { + let mut ticks = tokio::time::interval(std::time::Duration::from_millis(25)); + loop { + ticks.tick().await; + let jobs = api + .store + .list_backend_journal_jobs(TEST_WORKSPACE_ID, &attr.session_id, None, 100) + .unwrap(); + let closed = store + .journal_for_session(&host) + .unwrap() + .is_none_or(|journal| { + store + .active_journal_generation(&host, &journal) + .unwrap() + .is_none() + }); + if closed + && !store.journal_retention_required(&host).unwrap() + && jobs + .iter() + .all(|job| job.state == BackendJobState::Completed) + { + break; + } + } + }) + .await + .expect("seed Journal work must settle before measuring restore"); + } + + ConnectedRestoreFixture { + api, + execution, + probe, + model, + worker, + subject_id: subject.id, + requests, + callbacks, + lose_response, + _server: server, + _dir: dir, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn restore_after_real_prepare_rejects_revoked_host_trust_without_publication_or_queue_consumption() + { + use session_store::WorkerMetadataStore as _; + use std::sync::atomic::Ordering::SeqCst; + use worker_runtime::execution::WorkerExecutionBackend as _; + + let fixture = connected_restore_fixture().await; + let api = &fixture.api; + let execution = &fixture.execution; + let probe = &fixture.probe; + let model = &fixture.model; + let worker = fixture.worker.clone(); + let requests = &fixture.requests; + let store = open_subjektiv_store(api).unwrap(); + let subject_id = &fixture.subject_id; + let aggregate = api + .config + .embedded_runtime_store_root + .join("workers") + .join(&worker.worker_id); + let name = format!("worker-runtime-{}", worker.worker_id); + let metadata = session_store::WorkerAggregateStore::open_read_only(&aggregate, &name).unwrap(); + let saved = metadata.read_by_name(&name).unwrap().unwrap(); + assert!(matches!( + saved.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Confirmed { .. }) + )); + let active = saved.active.as_ref().unwrap(); + let session = + session_store::WorkerSessionStore::open_read_only(aggregate.join("session")).unwrap(); + let entries = || { + session + .read_all_read_only(active.session_id, active.segment_id.unwrap()) + .unwrap() + }; + let before = serde_json::to_value(entries()).unwrap(); + let checkpoint = entries() + .into_iter() + .rev() + .find_map(|entry| match entry { + session_store::LogEntry::Extension { + domain, payload, .. + } if domain == "worker.pending_activations.v1" => Some(payload), + _ => None, + }) + .expect("real Controller persisted pending activation checkpoint"); + assert_eq!( + checkpoint["pending"][0]["submission_request_id"], + "saved-submit" + ); + assert_eq!( + checkpoint["pending_notifications"][0]["notification_request_id"], + "saved-notice" + ); + let attributions = store.subject_session_attributions(subject_id).unwrap(); + let key = crate::subjektiv::subject_worker_singleton_key(subject_id).unwrap(); + let singleton = api + .store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &key) + .unwrap(); + let model_before = model.calls.load(SeqCst); + let request = runtime_api::WorkerRestoreRequest { + expected_observation_token: api + .runtime + .worker(&worker) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: "restore-revoked-after-real-prepare".into(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }; + assert!( + requests.lock().unwrap().iter().any( + |(path, status)| path.ends_with("/subjektiv/sessions") && *status == StatusCode::OK + ), + "the same production signer must be accepted before trust revocation" + ); + requests.lock().unwrap().clear(); + let runtime = api.runtime.clone(); + let target = worker.clone(); + let intent = request.clone(); + let restoring = tokio::task::spawn_blocking(move || runtime.restore_worker(&target, intent)); + tokio::time::timeout( + std::time::Duration::from_secs(10), + probe.prepared.notified(), + ) + .await + .expect("real prepare must complete before revocation"); + assert_eq!(probe.activations.load(SeqCst), 0); + assert_eq!(model.calls.load(SeqCst), model_before); + assert_eq!( + serde_json::to_value(entries()).unwrap(), + before, + "preparation must not consume saved input or append history" + ); + assert_eq!(metadata.read_by_name(&name).unwrap().unwrap(), saved); + assert!( + requests + .lock() + .unwrap() + .iter() + .all(|(path, _)| !path.ends_with("/subjektiv/sessions")), + "publication authorization belongs after candidate preparation" + ); + let prepared_record: Value = + serde_json::from_slice(&std::fs::read(aggregate.join("worker.json")).unwrap()).unwrap(); + let operation = prepared_record["restore_guard"]["owners"]["restore-revoked-after-real-prepare"]["operation_id"].clone(); + assert!( + !operation.is_null(), + "admission must persist the operation owner before prepare" + ); + assert!( + api.config_store + .revoke_workspace_runtime_binding( + TEST_WORKSPACE_ID, + EMBEDDED_WORKER_RUNTIME_ID, + &now_registry_timestamp(), + ) + .unwrap() + ); + probe.release.notify_one(); + let first = tokio::time::timeout(std::time::Duration::from_secs(20), restoring) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{first:?}" + ); + assert!( + requests + .lock() + .unwrap() + .iter() + .any(|(path, status)| path.ends_with("/subjektiv/sessions") + && *status == StatusCode::UNAUTHORIZED), + "actual signed Host publication must see revoked persisted trust" + ); + assert_eq!(probe.prepares.load(SeqCst), 1); + assert_eq!(probe.activations.load(SeqCst), 1); + let runtime = api.runtime.clone(); + let target = worker.clone(); + let replay = tokio::time::timeout( + std::time::Duration::from_secs(20), + tokio::task::spawn_blocking(move || runtime.restore_worker(&target, request)), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{replay:?}" + ); + // Reconciliation may reconstruct after confirmed cleanup, but must never + // invent a new operation or repair an already confirmed attribution. + assert!( + probe + .operations + .lock() + .unwrap() + .iter() + .all(|id| *id == operation) + ); + assert!( + probe + .repairs + .lock() + .unwrap() + .iter() + .all(|repaired| !repaired) + ); + let worker_ref = worker_runtime::identity::WorkerRef::try_from(&worker).unwrap(); + assert!( + execution.worker_snapshot(&worker_ref).is_none(), + "no live Controller may be published" + ); + assert!( + requests + .lock() + .unwrap() + .iter() + .all(|(path, _)| !path.ends_with("/authorization/check")), + "restoring saved Features must not re-admit them" + ); + assert_eq!(probe.publications.load(SeqCst), 0); + assert_eq!( + probe.handles.lock().unwrap().len(), + 1, + "only stopped seed Controller existed" + ); + assert_eq!(model.calls.load(SeqCst), model_before); + assert_eq!( + serde_json::to_value(entries()).unwrap(), + before, + "Host denial must not consume either saved queue or execute Tools" + ); + assert_eq!( + metadata + .read_by_name(&name) + .unwrap() + .unwrap() + .subjektiv_session_attribution, + saved.subjektiv_session_attribution + ); + assert_eq!( + store.subject_session_attributions(subject_id).unwrap(), + attributions + ); + assert_eq!( + api.store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &key) + .unwrap(), + singleton + ); + // Activation may already reconcile metadata/pins/allocation. Denial is not a + // fictitious rollback/Stopped: the same operation must retain its owner. + let record: Value = + serde_json::from_slice(&std::fs::read(aggregate.join("worker.json")).unwrap()).unwrap(); + assert_eq!( + record["restore_guard"]["active_request_id"], + "restore-revoked-after-real-prepare" + ); + let owner = &record["restore_guard"]["owners"]["restore-revoked-after-real-prepare"]; + assert_eq!( + owner["operation_id"], operation, + "the durable operation owner must survive denial and replay" + ); + assert_eq!( + record["execution_state"]["state"], + "reconciliation_required" + ); + assert_eq!( + record["execution_state"]["execution"]["operation"], + "restore" + ); + assert_eq!( + record["execution_state"]["execution"]["intent"]["operation_id"], + operation + ); + assert!( + owner["receipt"].is_null(), + "no saved Accepted receipt may escape on replay" + ); + // Only this fixture's retained operation is cleaned up, after its durable + // evidence has been checked. Cleanup is not the restore outcome above. + let cleanup = execution.stop_worker(&worker_ref); + assert!( + cleanup.is_accepted(), + "fixture restore cleanup: {cleanup:?}" + ); + shutdown_workspace_feature_storage(&api).await.unwrap(); +} + +#[derive(Clone, Copy)] +enum ConnectedRepair { + IncompleteAttribution, + PendingHead, + RevokedAttribution, +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn explicit_repair_existing_attribution_replays_lost_real_host_response_once() { + connected_explicit_repair(ConnectedRepair::IncompleteAttribution).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn explicit_repair_pending_head_creates_one_session_after_lost_real_host_response() { + connected_explicit_repair(ConnectedRepair::PendingHead).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn explicit_repair_replay_rejects_revoked_real_host_trust_without_false_accepted() { + connected_explicit_repair(ConnectedRepair::RevokedAttribution).await; +} + +async fn connected_explicit_repair(scenario: ConnectedRepair) { + use session_store::{Store as _, WorkerMetadataStore as _}; + use std::sync::atomic::Ordering::SeqCst; + use worker_runtime::execution::WorkerExecutionBackend as _; + + let pending = matches!(scenario, ConnectedRepair::PendingHead); + let fixture = connected_restore_fixture_with_pending_head(pending).await; + let api = &fixture.api; + let probe = &fixture.probe; + let worker = &fixture.worker; + let revoked = matches!(scenario, ConnectedRepair::RevokedAttribution); + let aggregate = api + .config + .embedded_runtime_store_root + .join("workers") + .join(&worker.worker_id); + let name = format!("worker-runtime-{}", worker.worker_id); + let metadata = session_store::WorkerAggregateStore::new(&aggregate, &name).unwrap(); + let mut saved = metadata.read_by_name(&name).unwrap().unwrap(); + let original_active = saved.active.clone().unwrap(); + // The genuine pending fixture has no Session manifest yet. Open its empty + // store before measuring repair; after publication always reread authority + // rather than the original Store's cached empty lineage. + let pending_store = + pending.then(|| session_store::WorkerSessionStore::new(aggregate.join("session")).unwrap()); + let sessions = + || match session_store::WorkerSessionStore::open_read_only(aggregate.join("session")) { + Ok(store) => store, + Err(session_store::StoreError::Io(error)) + if pending && error.kind() == std::io::ErrorKind::NotFound => + { + pending_store.as_ref().unwrap().clone() + } + Err(error) => panic!("read repaired Session: {error}"), + }; + let original_entries = match original_active.segment_id { + Some(segment_id) => serde_json::to_value( + sessions() + .read_all_read_only(original_active.session_id, segment_id) + .unwrap(), + ) + .unwrap(), + None => { + assert!(pending); + assert!( + sessions() + .list_segments(original_active.session_id) + .unwrap() + .is_empty() + ); + serde_json::json!([]) + } + }; + let store = open_subjektiv_store(api).unwrap(); + let initial_attributions = store + .subject_session_attributions(&fixture.subject_id) + .unwrap(); + let singleton_key = + crate::subjektiv::subject_worker_singleton_key(&fixture.subject_id).unwrap(); + let singleton = api + .store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &singleton_key) + .unwrap(); + + let session_id = saved.active.as_ref().unwrap().session_id; + assert_eq!( + initial_attributions + .iter() + .filter(|attr| attr.session_id == session_id.to_string()) + .count(), + usize::from(!pending), + "existing attribution repair is not first creation; pending-head repair is" + ); + // Deliberate initial interruption fixture, before admitting any restore. + // Subsequent transitions, including OutcomeUnknown, are production writes. + saved.subjektiv_session_attribution = Some( + session_store::SubjektivSessionAttributionState::OutcomeUnknown { + session_id, + diagnostic: "fixture interrupted Session attribution acknowledgement".into(), + }, + ); + metadata.write(&saved).unwrap(); + let model_before = fixture.model.calls.load(SeqCst); + fixture.requests.lock().unwrap().clear(); + fixture.callbacks.lock().unwrap().clear(); + fixture.lose_response.store(true, SeqCst); + let request = runtime_api::WorkerRestoreRequest { + expected_observation_token: api + .runtime + .worker(worker) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: "connected-explicit-repair".into(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }; + let runtime = api.runtime.clone(); + let target = worker.clone(); + let intent = request.clone(); + let first = tokio::time::timeout( + std::time::Duration::from_secs(20), + tokio::task::spawn_blocking(move || runtime.restore_worker(&target, intent)), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{first:?}" + ); + assert!( + !fixture.lose_response.load(SeqCst), + "lose acknowledgement only after real Host OK" + ); + assert_eq!( + probe.prepares.load(SeqCst), + 0, + "repair precedes candidate construction" + ); + assert_eq!(probe.activations.load(SeqCst), 0); + assert_eq!(probe.publications.load(SeqCst), 0); + assert_eq!(fixture.model.calls.load(SeqCst), model_before); + let failed = metadata.read_by_name(&name).unwrap().unwrap(); + assert_eq!( + failed.active, saved.active, + "response loss cannot publish a first head" + ); + assert!(matches!(failed.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::OutcomeUnknown { session_id: id, .. }) if id == session_id)); + let committed_attributions = store + .subject_session_attributions(&fixture.subject_id) + .unwrap(); + assert_eq!( + committed_attributions.len(), + initial_attributions.len() + usize::from(pending) + ); + assert_eq!( + committed_attributions + .iter() + .filter(|attr| attr.session_id == session_id.to_string() + && attr.worker_id == worker.worker_id) + .count(), + 1, + "actual SQLite attribution persisted before response loss" + ); + if !pending { + assert_eq!( + committed_attributions, initial_attributions, + "reconcile the existing immutable row" + ); + } + assert_eq!(fixture.callbacks.lock().unwrap().len(), 1); + { + let callbacks = fixture.callbacks.lock().unwrap(); + assert_eq!(callbacks[0]["session_id"], session_id.to_string()); + assert_eq!(callbacks[0]["create_if_missing"], true); + } + let record = || -> Value { + serde_json::from_slice(&std::fs::read(aggregate.join("worker.json")).unwrap()).unwrap() + }; + let first_record = record(); + let operation = + first_record["restore_guard"]["owners"]["connected-explicit-repair"]["operation_id"] + .clone(); + assert!(!operation.is_null()); + assert!( + first_record["restore_guard"]["owners"]["connected-explicit-repair"]["receipt"].is_null() + ); + assert_eq!( + first_record["restore_guard"]["active_request_id"], + "connected-explicit-repair" + ); + assert_eq!( + first_record["execution_state"]["state"], + "reconciliation_required" + ); + assert!( + fixture + .requests + .lock() + .unwrap() + .iter() + .any( + |(path, status)| path.ends_with("/subjektiv/sessions") && *status == StatusCode::OK + ) + ); + assert!( + !fixture + .requests + .lock() + .unwrap() + .iter() + .any(|(path, _)| path.ends_with("/subjektiv/journal/startup")), + "no startup before attribution acknowledgement" + ); + if pending { + assert!(sessions().list_segments(session_id).unwrap().is_empty()); + } + + // Re-enter the same public Runtime intent and durable operation. The repair + // gate changes timing only; trust revocation is the real persisted binding. + if revoked { + probe.gate_repair.store(true, SeqCst); + } + let mut repaired_seed = None; + let runtime = api.runtime.clone(); + let target = worker.clone(); + let intent = request.clone(); + let retry = tokio::task::spawn_blocking(move || runtime.restore_worker(&target, intent)); + if revoked { + tokio::time::timeout( + std::time::Duration::from_secs(10), + probe.repairing.notified(), + ) + .await + .unwrap(); + assert!( + api.config_store + .revoke_workspace_runtime_binding( + TEST_WORKSPACE_ID, + EMBEDDED_WORKER_RUNTIME_ID, + &now_registry_timestamp(), + ) + .unwrap() + ); + probe.release_repair.notify_one(); + } else { + tokio::time::timeout( + std::time::Duration::from_secs(10), + probe.prepared.notified(), + ) + .await + .unwrap(); + assert_eq!(probe.activations.load(SeqCst), 0); + assert_eq!(fixture.model.calls.load(SeqCst), model_before); + assert_eq!(*probe.repairs.lock().unwrap(), vec![true]); + let repaired = metadata.read_by_name(&name).unwrap().unwrap(); + assert_eq!(repaired.active.as_ref().unwrap().session_id, session_id); + assert!(matches!(repaired.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Confirmed { session_id: id, ref subject_id }) + if id == session_id && subject_id == &fixture.subject_id)); + assert_eq!( + sessions().list_segments(session_id).unwrap().len(), + 1, + "exactly one durable seed" + ); + let segment_id = repaired.active.unwrap().segment_id.unwrap(); + let entries = sessions() + .read_all_read_only(session_id, segment_id) + .unwrap(); + let seeds = entries + .iter() + .filter(|entry| matches!(entry, session_store::LogEntry::AnnotatedSegmentStart { .. })) + .collect::>(); + assert_eq!(seeds.len(), 1); + repaired_seed = Some((segment_id, serde_json::to_value(seeds[0]).unwrap())); + if !pending { + assert_eq!( + serde_json::to_value(entries).unwrap(), + original_entries, + "attribution-only repair and normal prepare cannot reseed or consume saved input" + ); + } + assert_eq!( + fixture + .requests + .lock() + .unwrap() + .iter() + .filter(|(path, _)| path.ends_with("/subjektiv/journal/startup")) + .count(), + usize::from(pending) + ); + probe.release.notify_one(); + } + let replay = tokio::time::timeout(std::time::Duration::from_secs(20), retry) + .await + .unwrap() + .unwrap() + .unwrap(); + if revoked { + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{replay:?}" + ); + assert!( + fixture + .requests + .lock() + .unwrap() + .iter() + .any(|(path, status)| path.ends_with("/subjektiv/sessions") + && *status == StatusCode::UNAUTHORIZED) + ); + assert_eq!(probe.prepares.load(SeqCst), 0); + assert_eq!(probe.activations.load(SeqCst), 0); + assert_eq!(probe.publications.load(SeqCst), 0); + assert_eq!(fixture.model.calls.load(SeqCst), model_before); + assert!( + matches!(metadata.read_by_name(&name).unwrap().unwrap().subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Failed { session_id: id, .. }) if id == session_id) + ); + let denied = record(); + assert_eq!( + denied["restore_guard"]["active_request_id"], + "connected-explicit-repair" + ); + assert_eq!( + denied["restore_guard"]["owners"]["connected-explicit-repair"]["operation_id"], + operation + ); + assert!( + denied["restore_guard"]["owners"]["connected-explicit-repair"]["receipt"].is_null() + ); + assert_eq!( + denied["execution_state"]["state"], + "reconciliation_required" + ); + assert_eq!( + serde_json::to_value( + sessions() + .read_all_read_only( + original_active.session_id, + original_active.segment_id.unwrap(), + ) + .unwrap() + ) + .unwrap(), + original_entries + ); + } else { + assert_eq!( + replay.state, + server_api::WorkerRestoreState::Accepted, + "{replay:?}" + ); + let callbacks_after = fixture.callbacks.lock().unwrap().len(); + let repairs_after = probe.repair_operations.lock().unwrap().len(); + let runtime = api.runtime.clone(); + let target = worker.clone(); + let accepted = tokio::time::timeout( + std::time::Duration::from_secs(20), + tokio::task::spawn_blocking(move || runtime.restore_worker(&target, request)), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + accepted.state, + server_api::WorkerRestoreState::Accepted, + "{accepted:?}" + ); + assert_eq!(probe.prepares.load(SeqCst), 1); + assert_eq!(probe.publications.load(SeqCst), 1); + assert_eq!(probe.repair_operations.lock().unwrap().len(), repairs_after); + assert_eq!( + fixture.callbacks.lock().unwrap().len(), + callbacks_after, + "receipt replay performs no Host callback" + ); + assert_eq!(sessions().list_segments(session_id).unwrap().len(), 1); + let repaired = metadata.read_by_name(&name).unwrap().unwrap(); + let active = repaired.active.unwrap(); + let (segment_id, seed) = repaired_seed.unwrap(); + assert_eq!(active.session_id, session_id); + assert_eq!(active.segment_id, Some(segment_id)); + let entries = sessions() + .read_all_read_only(session_id, segment_id) + .unwrap(); + let seeds = entries + .iter() + .filter(|entry| matches!(entry, session_store::LogEntry::AnnotatedSegmentStart { .. })) + .collect::>(); + assert_eq!(seeds.len(), 1); + assert_eq!( + serde_json::to_value(seeds[0]).unwrap(), + seed, + "activation and receipt replay preserve the exact repaired seed" + ); + } + if pending { + assert_eq!( + fixture.model.calls.load(SeqCst), + model_before, + "pending first-head repair has no body input to execute" + ); + assert_eq!( + session_id, original_active.session_id, + "first head belongs to the originally pending Session" + ); + assert!(original_active.segment_id.is_none()); + } + assert_eq!( + store + .subject_session_attributions(&fixture.subject_id) + .unwrap(), + committed_attributions + ); + assert_eq!( + api.store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &singleton_key) + .unwrap(), + singleton + ); + assert!( + probe + .repair_operations + .lock() + .unwrap() + .iter() + .all(|id| *id == operation) + ); + assert!( + probe + .operations + .lock() + .unwrap() + .iter() + .all(|id| *id == operation) + ); + assert_eq!( + probe.repair_operations.lock().unwrap().len(), + 2, + "one failed repair and one same-operation replay" + ); + assert!( + fixture + .requests + .lock() + .unwrap() + .iter() + .all(|(path, _)| !path.ends_with("/authorization/check")), + "saved Features must not be re-admitted" + ); + { + let callbacks = fixture.callbacks.lock().unwrap(); + assert!( + callbacks + .iter() + .all(|callback| callback["session_id"] == session_id.to_string()) + ); + assert_eq!( + callbacks + .iter() + .filter(|callback| callback["create_if_missing"] == true) + .count(), + 2, + "repair retries the Session-keyed receipt; publication only verifies it" + ); + } + let worker_ref = worker_runtime::identity::WorkerRef::try_from(worker).unwrap(); + if revoked { + assert!(fixture.execution.worker_snapshot(&worker_ref).is_none()); + assert!( + fixture.execution.stop_worker(&worker_ref).is_accepted(), + "fixture-owned repair resource cleanup" + ); + } else { + assert!(fixture.execution.worker_snapshot(&worker_ref).is_some()); + // Teardown is outside the measured restore window: Stop may admit an + // independent Journal job, not a second execution of the restored body. + assert_eq!( + api.runtime + .stop_worker(worker, WorkerLifecycleRequest::default()) + .unwrap() + .state, + InternalWorkerOperationState::Accepted + ); + } + shutdown_workspace_feature_storage(api).await.unwrap(); +} diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs index dece1c3f4..cb88a2636 100644 --- a/crates/workspace-server/src/server_assignment_attention_tests.rs +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -490,21 +490,61 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFac self.controller(request.worker_ref.worker_id.to_string(), false) .await } - async fn restore_controller( + async fn prepare_restore( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + Ok(Box::new(AttentionRestoreCandidate { + inputs: Some(( + AttentionWorkerFactory { + root: self.root.clone(), + }, + request.worker_ref.worker_id.to_string(), + )), + })) + } +} + +// Preparation owns only fixture inputs; reconstruction and Controller startup +// remain lazy so discard never needs to shut down a started Worker. +struct AttentionRestoreCandidate { + inputs: Option<(AttentionWorkerFactory, String)>, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for AttentionRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + protocol::WorkerStateSnapshot::initial() + } + + async fn activate( + &mut self, ) -> std::result::Result< worker_runtime::worker_backend::RuntimeWorkerController, worker_runtime::worker_backend::RuntimeRestoreFailure, > { - self.controller(request.worker_ref.worker_id.to_string(), true) - .await - .map_err(|_| { - worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( - worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, - "attention fixture reconstruction failed", - ) - }) + let (factory, worker_id) = self.inputs.take().ok_or_else(|| { + worker_runtime::worker_backend::RuntimeRestoreFailure::refused( + worker_runtime::worker_backend::RuntimeRestoreStage::Controller, + "attention fixture candidate already consumed", + ) + })?; + factory.controller(worker_id, true).await.map_err(|_| { + worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( + worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, + "attention fixture reconstruction failed", + ) + }) + } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inputs.take(); + Ok(()) } } @@ -611,7 +651,7 @@ async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() { async fn real_attention_worker_start_restore(lose_response: bool) { use session_store::{LogEntry, Store, WorkerSessionStore}; - let dir = tempfile::tempdir().unwrap(); + let dir = restored_parent_control_tests::connected_restore_tempdir(); init_clean_git_workspace(dir.path()); let config = test_server_config(dir.path()); let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs index 43bddd74f..25031c8f6 100644 --- a/crates/workspace-server/src/server_assignment_delivery_tests.rs +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -14,17 +14,37 @@ async fn notification_diagnostics_http_auth_scope_and_readonly_evidence() { seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID); let ticket = delivery_ticket(&api, "Notification diagnostics auth fixture"); delivery_assign(&api, ticket.id.as_str(), "diagnostics-assignment").await; - let path = format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",ticket.id.as_str()); - let page = delivery_http(&api,"GET",&path,None,StatusCode::OK).await; - assert_eq!(page["items"].as_array().unwrap().len(),1); - assert_eq!(page["items"][0]["model_processing_confirmed"],false); + let path = format!( + "/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1", + ticket.id.as_str() + ); + let page = delivery_http(&api, "GET", &path, None, StatusCode::OK).await; + assert_eq!(page["items"].as_array().unwrap().len(), 1); + assert_eq!(page["items"][0]["model_processing_confirmed"], false); assert!(page["items"][0].get("content").is_none()); - let again=delivery_http(&api,"GET",&path,None,StatusCode::OK).await; - assert_eq!(again,page); - let response=build_router(api.clone()).oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap()).await.unwrap(); - assert_eq!(response.status(),StatusCode::UNAUTHORIZED); - delivery_http(&api,"GET","/api/w/other-workspace/notification-diagnostics",None,StatusCode::NOT_FOUND).await; - delivery_http(&api,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),None,StatusCode::BAD_REQUEST).await; + let again = delivery_http(&api, "GET", &path, None, StatusCode::OK).await; + assert_eq!(again, page); + let response = build_router(api.clone()) + .oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + delivery_http( + &api, + "GET", + "/api/w/other-workspace/notification-diagnostics", + None, + StatusCode::NOT_FOUND, + ) + .await; + delivery_http( + &api, + "GET", + &format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"), + None, + StatusCode::BAD_REQUEST, + ) + .await; } fn delivery_assignment_path(ticket: &str) -> String { @@ -828,21 +848,62 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFact self.controller(request.worker_ref.worker_id.to_string(), false) .await } - async fn restore_controller( + async fn prepare_restore( &self, request: delivery_execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + Ok(Box::new(DeliveryRestoreCandidate { + inputs: Some(( + DeliveryWorkerFactory { + root: self.root.clone(), + client: self.client.clone(), + }, + request.worker_ref.worker_id.to_string(), + )), + })) + } +} + +// Retain the same scripted client/receipt store without opening either store or +// starting a Controller until activation. Discard drops only these owned inputs. +struct DeliveryRestoreCandidate { + inputs: Option<(DeliveryWorkerFactory, String)>, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for DeliveryRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + protocol::WorkerStateSnapshot::initial() + } + + async fn activate( + &mut self, ) -> std::result::Result< worker_runtime::worker_backend::RuntimeWorkerController, worker_runtime::worker_backend::RuntimeRestoreFailure, > { - self.controller(request.worker_ref.worker_id.to_string(), true) - .await - .map_err(|_| { - worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( - worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, - "delivery fixture reconstruction failed", - ) - }) + let (factory, worker_id) = self.inputs.take().ok_or_else(|| { + worker_runtime::worker_backend::RuntimeRestoreFailure::refused( + worker_runtime::worker_backend::RuntimeRestoreStage::Controller, + "delivery fixture candidate already consumed", + ) + })?; + factory.controller(worker_id, true).await.map_err(|_| { + worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( + worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, + "delivery fixture reconstruction failed", + ) + }) + } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inputs.take(); + Ok(()) } } @@ -941,6 +1002,7 @@ async fn delivery_real_fixture( Arc, DeliveryScriptedClient, ) { + let _environment = restored_parent_control_tests::connected_restore_tempdir(); let config = test_server_config(root); let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) @@ -1146,9 +1208,22 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi "uncertain delivery changed request id or payload: {attempts:?}" ); assert_eq!(delivery_receipts(&api, &worker).len(), 1); - let unknown = api.store.notification_diagnostics(TEST_WORKSPACE_ID, &server_api::NotificationDiagnosticsQuery { - assignment_id: Some(assignment.into()), ..Default::default() - }).unwrap().items.into_iter().find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown).expect("saved unknown acceptance must remain visible while Runtime already has its receipt"); + let unknown = api + .store + .notification_diagnostics( + TEST_WORKSPACE_ID, + &server_api::NotificationDiagnosticsQuery { + assignment_id: Some(assignment.into()), + ..Default::default() + }, + ) + .unwrap() + .items + .into_iter() + .find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown) + .expect( + "saved unknown acceptance must remain visible while Runtime already has its receipt", + ); assert!(!unknown.model_processing_confirmed); // Change mutable rendering context while the acknowledgement is unknown. // A new self-report is not a new input and cannot change the frozen retry. @@ -1209,11 +1284,24 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi } delivery_wait_idle(&reopened, &worker, 2).await; assert_eq!(delivery_receipts(&reopened, &worker).len(), 2); - let evidence = delivery_http(&reopened,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",unknown.source_id),None,StatusCode::OK).await; - assert_eq!(evidence["items"][0]["stage"],"runtime_accepted"); - assert_eq!(evidence["items"][0]["delivery_id"],json!(unknown.delivery_id)); - assert_eq!(evidence["items"][0]["model_processing_confirmed"],false); - assert_eq!(evidence["items"][0]["retry_candidate"],false); + let evidence = delivery_http( + &reopened, + "GET", + &format!( + "/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}", + unknown.source_id + ), + None, + StatusCode::OK, + ) + .await; + assert_eq!(evidence["items"][0]["stage"], "runtime_accepted"); + assert_eq!( + evidence["items"][0]["delivery_id"], + json!(unknown.delivery_id) + ); + assert_eq!(evidence["items"][0]["model_processing_confirmed"], false); + assert_eq!(evidence["items"][0]["retry_candidate"], false); assert_eq!( delivery_commits(&reopened, &worker) .iter() diff --git a/crates/workspace-server/src/server_workspace_config_wip_tests.rs b/crates/workspace-server/src/server_workspace_config_wip_tests.rs index 6c3fd0fd6..9e970e17d 100644 --- a/crates/workspace-server/src/server_workspace_config_wip_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_wip_tests.rs @@ -187,7 +187,14 @@ async fn config_call( runtime .invoke( path.into(), - if matches!(operation,"edit"|"write") {wip_protocol::InterfaceReference {scope:"/".into(),name:format!("yoi.file/{operation}-text/v1")}} else {config_interface(path)}, + if matches!(operation, "edit" | "write") { + wip_protocol::InterfaceReference { + scope: "/".into(), + name: format!("yoi.file/{operation}-text/v1"), + } + } else { + config_interface(path) + }, operation.into(), args, agen::tool::ToolExecutionContext::direct(), @@ -581,14 +588,88 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto *self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle()); Ok(controller) } - async fn restore_controller( + async fn preflight_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), String> { + self.inner.preflight_restore(request).await + } + + async fn recover_restore_source( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.inner.recover_restore_source(request).await + } + + async fn prepare_restore( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + let inner = self.inner.prepare_restore(request).await?; + Ok(Box::new(ConfigCaptureCandidate { + inner, + client: self.client.clone(), + controller_abort: self.controller_abort.clone(), + })) + } + + async fn cleanup_failed_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.cleanup_failed_restore(request).await + } + + async fn reconcile_stopped_worker( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.reconcile_stopped_worker(worker_ref).await + } + + fn activate_restored_controller( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + workspace_id: Option<&str>, + handle: &worker::WorkerHandle, + ) { + self.inner + .activate_restored_controller(worker_ref, workspace_id, handle); + } +} + +struct ConfigCaptureCandidate { + inner: Box, + client: Arc>>>, + controller_abort: Arc>>, +} + +#[async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for ConfigCaptureCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.inner.initial_state() + } + + async fn activate( + &mut self, ) -> std::result::Result< worker_runtime::worker_backend::RuntimeWorkerController, worker_runtime::worker_backend::RuntimeRestoreFailure, > { - self.inner.restore_controller(request).await + let controller = self.inner.activate().await?; + *self.client.lock().unwrap() = Some(controller.workspace_client.clone()); + *self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle()); + Ok(controller) + } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.discard().await } } @@ -717,6 +798,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION, decode_runtime_request_source_claims, }; + let root = restored_parent_control_tests::connected_restore_tempdir(); // Remote Runtime-owned HTTP adapter, including its real request signer. let mut fixture = manual_worker_assignment_fixture().await; let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap(); @@ -796,7 +878,6 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w // Actual embedded Runtime factory -> real Worker controller -> captured // production WorkspaceClient -> signed HTTP -> production Backend middleware. - let root = tempfile::tempdir().unwrap(); init_clean_git_workspace(root.path()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); diff --git a/deno.json b/deno.json index d10675008..b006ccf53 100644 --- a/deno.json +++ b/deno.json @@ -1,8 +1,8 @@ { "tasks": { - "test": "deno run --allow-read --allow-write=target --allow-run --allow-env tools/test.ts", + "test": "deno run --allow-read --allow-write=target --allow-run --allow-env --allow-sys=uid tools/test.ts", "test:ticket:real-api": "cargo test -p yoi-workspace-server --lib ticket_web_roundtrip_tests -- --ignored", - "test:runner": "deno test --allow-read --allow-write --allow-run --allow-env tools/test.test.ts" + "test:runner": "deno test --allow-read --allow-write --allow-run --allow-env --allow-sys=uid tools/test.test.ts" }, "fmt": { "proseWrap": "never" diff --git a/tests/support/restore_environment.rs b/tests/support/restore_environment.rs new file mode 100644 index 000000000..c045a2ed5 --- /dev/null +++ b/tests/support/restore_environment.rs @@ -0,0 +1,129 @@ +//! Fail-closed preflight shared by restore fixtures across Worker, Runtime and Server. +//! +//! The normal runner supplies a separate namespace per process; a manually +//! configured invocation may use the checkout-owned base itself. This module +//! never sets process-wide environment or creates directories/tables. + +use std::fs; +use std::path::{Component, Path, PathBuf}; + +fn validate_restore_test_paths( + base: &Path, + resolved: [Option; 5], +) -> Result { + let root = resolved[0] + .as_deref() + .filter(|config| config.file_name().is_some_and(|name| name == "config")) + .and_then(Path::parent) + .ok_or("config")?; + let namespace = root.strip_prefix(base).map_err(|_| "config")?; + if namespace + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err("config"); + } + for ((label, suffix), actual) in [ + ("config", "config"), + ("data", "data"), + ("secrets", "data"), + ("runtime", "run"), + ("allocation", "run/workers.json"), + ] + .into_iter() + .zip(resolved.iter()) + { + if actual.as_deref() != Some(root.join(suffix).as_path()) { + return Err(label); + } + } + Ok(root.to_path_buf()) +} + +fn require_owned_directory(checkout: &Path, directory: &Path) { + let mut checked = checkout.to_path_buf(); + for part in directory.strip_prefix(checkout).unwrap().components() { + assert!(matches!(part, Component::Normal(_))); + checked.push(part); + let metadata = fs::symlink_metadata(&checked) + .expect("create isolated restore test directories before launching the test process"); + assert!( + metadata.is_dir() && !metadata.file_type().is_symlink(), + "restore test directory must not traverse symlinks" + ); + } +} + +pub(crate) fn require_restore_test_environment() { + let checkout = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap(); + let base = checkout.join("target/restore-test-environment"); + // Compare all resolver results before inspecting even an absent allocation + // table (its lookup can create a lock). Never inspect a live fallback path. + let root = validate_restore_test_paths( + &base, + [ + manifest::paths::config_dir(), + manifest::paths::data_dir(), + manifest::paths::secret_data_dir(), + manifest::paths::runtime_dir(), + manifest::paths::worker_allocation_path(), + ], + ) + .unwrap_or_else(|label| { + panic!( + "restore fixture requires isolated {label}: run deno task test, or configure YOI_CONFIG_DIR=/target/restore-test-environment/config, YOI_DATA_DIR=/target/restore-test-environment/data, YOI_RUNTIME_DIR=/target/restore-test-environment/run before launch" + ) + }); + assert_eq!(fs::canonicalize(checkout).unwrap(), checkout); + for directory in [ + root.clone(), + root.join("config"), + root.join("data"), + root.join("run"), + ] { + require_owned_directory(checkout, &directory); + } + for name in ["workers.json", "workers.json.lock"] { + match fs::symlink_metadata(root.join("run").join(name)) { + Ok(metadata) => assert!( + metadata.is_file() && !metadata.file_type().is_symlink(), + "owned allocation paths must be regular files, not symlinks" + ), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => panic!("inspect owned restore allocation path: {error}"), + } + } +} + +#[test] +fn restore_fixture_path_validation_rejects_fallback_before_resource_access() { + let base = Path::new("/owned/test-environment"); + for root in [base.to_path_buf(), base.join("run-1/worker-shard-2")] { + let safe = ["config", "data", "data", "run", "run/workers.json"] + .map(|suffix| Some(root.join(suffix))); + assert_eq!( + validate_restore_test_paths(base, safe.clone()), + Ok(root.clone()) + ); + for index in 0..safe.len() { + for fallback in [None, Some(PathBuf::from("/not-a-fixture"))] { + let mut unsafe_paths = safe.clone(); + unsafe_paths[index] = fallback; + assert!(validate_restore_test_paths(base, unsafe_paths).is_err()); + } + } + } + for root in [ + base.join("../escape"), + base.join("safe/../../escape"), + PathBuf::from("/outside"), + ] { + let paths = ["config", "data", "data", "run", "run/workers.json"] + .map(|suffix| Some(root.join(suffix))); + assert!(validate_restore_test_paths(base, paths).is_err()); + } +} diff --git a/tools/test.test.ts b/tools/test.test.ts index aa8c69d12..bad5ee0c1 100644 --- a/tools/test.test.ts +++ b/tools/test.test.ts @@ -6,6 +6,7 @@ import { mapConcurrent, parseJobs, parseRustListing, + prepareTestIsolation, resultSummary, rustTestCommands, shardRustCommand, @@ -202,6 +203,39 @@ Deno.test("round-robin shards retain every listed name exactly once at threshold assertThrows(() => shardRustCommand(task, ["same", "same"])); }); +Deno.test("global runtime-path packages serialize every normal target without losing names", () => { + for (const pkg of ["worker", "worker-runtime", "yoi-workspace-server"]) { + for (const target of ["lib", "controller_restore_test"]) { + const task = { + name: `${pkg}/${target}`, + command: "/test-binary", + args: [], + cwd: "/pkg", + }; + for (const count of [0, 1, 59, 72, 73, 1135]) { + const names = listedNames(count); + const shards = shardRustCommand(task, names); + assert( + shards.flatMap((shard) => shard.selectedNames).sort().join() === + [...names].sort().join(), + ); + for (const shard of shards) { + const args = shard.task.args; + assert(args[args.indexOf("--test-threads") + 1] === "1"); + if (count > 72) { + assert(args[0] === "--exact"); + assert(args.slice(3).join() === shard.selectedNames.join()); + } else { + assert(args.join() === "--test-threads,1"); + } + assert(!args.includes("--ignored") && !args.includes("--skip")); + } + } + assert(task.args.length === 0, "do not mutate discovered commands"); + } + } +}); + Deno.test("scheduler drains active workers before propagating a callback error", async () => { let drained = false; let failed = false; @@ -266,6 +300,276 @@ function emergencyKill(pid: number) { } } +async function assertRejects( + run: () => Promise, + message: string, +) { + try { + await run(); + } catch (error) { + assert(String(error).includes(message), String(error)); + return; + } + throw new Error(`expected rejection containing ${message}`); +} + +Deno.test({ + name: + "isolated real children use bounded disjoint tables and sanitized Git/compiler environments", + ignore: Deno.build.os !== "linux", + async fn() { + const checkout = await Deno.realPath(await Deno.makeTempDir()); + const environmentSnapshot = () => + JSON.stringify( + Object.entries(process.env).sort(([a], [b]) => a.localeCompare(b)), + ); + const before = environmentSnapshot(); + try { + // An ancestor repository must not make an uninitialized tempfile look like + // a repository. No developer repository or configuration is borrowed. + for (const dir of [".git/objects", ".git/refs"]) { + await Deno.mkdir(`${checkout}/${dir}`, { recursive: true }); + } + await Deno.writeTextFile( + `${checkout}/.git/HEAD`, + "ref: refs/heads/main\n", + ); + const [first, second] = await Promise.all([ + prepareTestIsolation(checkout, "same/output/id"), + prepareTestIsolation(checkout, "same/output/id"), + ]); + assert( + first.root !== second.root, + "concurrent runner ids must not collide", + ); + const task = { + ...fixtureTask("isolation-env", checkout), + env: { + CARGO_MANIFEST_DIR: `${checkout}/crates/example`, + CARGO_HOME: `${checkout}/compiler-cache`, + RUSTUP_HOME: `${checkout}/toolchain-cache`, + YOI_HOME: "/live/yoi", + YOI_CONFIG_DIR: "/live/config", + YOI_DATA_DIR: "/live/data", + YOI_RUNTIME_DIR: "/live/run", + HOME: "/live/home", + TMPDIR: "/deleted/compiler-tmp", + GIT_DIR: "/live/git", + GIT_WORK_TREE: "/live/tree", + GIT_CONFIG: "/live/gitconfig", + GIT_CONFIG_GLOBAL: "/live/gitconfig", + GIT_CONFIG_SYSTEM: "/live/system-gitconfig", + GIT_TEMPLATE_DIR: "/live/template", + GIT_CONFIG_PARAMETERS: "'core.hooksPath'='/live/hooks'", + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "core.hooksPath", + GIT_CONFIG_VALUE_0: "/live/hooks", + HTTP_PROXY: "http://live.invalid", + https_proxy: "http://live.invalid", + ALL_PROXY: "http://live.invalid", + NO_PROXY: "live.invalid", + RUSTC_WRAPPER: "/live/sccache", + RUSTC_WORKSPACE_WRAPPER: "/live/wrapper", + SCCACHE_DISABLE: "0", + }, + }; + const taskBefore = JSON.stringify(task); + const commands = await Promise.all([ + first.wrapCommand(task, "pkg/a"), + first.wrapCommand(task, "pkg?a"), // same sanitized label + first.wrapCommand(task, "pkg/a"), // same exact label + second.wrapCommand(task, "pkg/a"), + ]); + assert(JSON.stringify(task) === taskBefore); + assert( + environmentSnapshot() === before, + "host environment changed", + ); + const results = await Promise.all( + commands.map((command, index) => + execute(command, { + output: checkout, + label: `environment-${index}`, + timeoutMs: 10_000, + }) + ), + ); + const tables = new Set(); + for (const [index, result] of results.entries()) { + assert(result.code === 0 && !result.error, JSON.stringify(result)); + const observed = JSON.parse(result.stdout); + const env = observed.env as Record; + assert(observed.cwd === checkout && commands[index].cwd === task.cwd); + assert(commands[index].args === task.args); + assert(env.CARGO_MANIFEST_DIR === task.env.CARGO_MANIFEST_DIR); + assert(env.CARGO_HOME === task.env.CARGO_HOME); + assert(env.RUSTUP_HOME === task.env.RUSTUP_HOME); + const namespace = env.YOI_RUNTIME_DIR.replace(/\/run$/, ""); + const prefix = `${checkout}/target/restore-test-environment/`; + assert(namespace.startsWith(prefix)); + const suffix = namespace.slice(prefix.length); + assert( + suffix.split("/").every((part) => /^[a-zA-Z0-9_-]+$/.test(part)), + ); + assert(env.YOI_CONFIG_DIR === `${namespace}/config`); + assert(env.YOI_DATA_DIR === `${namespace}/data`); + assert( + env.YOI_HOME === env.YOI_DATA_DIR, + "secret data must equal data", + ); + assert(env.HOME === `${namespace}/home`); + for ( + const dir of [ + namespace, + env.YOI_CONFIG_DIR, + env.YOI_DATA_DIR, + env.YOI_RUNTIME_DIR, + env.HOME, + ] + ) { + assert((await Deno.lstat(dir)).isDirectory); + assert(await Deno.realPath(dir) === dir); + } + for (const key of ["TMPDIR", "TMP", "TEMP"]) { + assert(env[key] === `${checkout}/target/t`); + } + for ( + const key of [ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_CONFIG", + "GIT_CONFIG_PARAMETERS", + "GIT_CONFIG_COUNT", + "GIT_CONFIG_KEY_0", + "GIT_CONFIG_VALUE_0", + "HTTP_PROXY", + "https_proxy", + "ALL_PROXY", + "NO_PROXY", + ] + ) { + assert(!(key in env), `${key} leaked to child`); + } + assert(env.GIT_CONFIG_GLOBAL === "/dev/null"); + assert(env.GIT_CONFIG_SYSTEM === "/dev/null"); + assert(env.GIT_CONFIG_NOSYSTEM === "1"); + assert(env.GIT_CEILING_DIRECTORIES === checkout); + assert(env.GIT_TEMPLATE_DIR === `${namespace}/git-template`); + assert(observed.globalConfig === ""); + assert(observed.ancestorDiscovered === false); + assert(observed.hooks.length === 0 && observed.info.length === 0); + assert(observed.author.includes("normal-tests@example.invalid")); + assert(env.GIT_AUTHOR_NAME === env.GIT_COMMITTER_NAME); + assert(env.GIT_AUTHOR_EMAIL === env.GIT_COMMITTER_EMAIL); + assert(env.RUSTC_WRAPPER === "" && env.RUSTC_WORKSPACE_WRAPPER === ""); + assert(env.SCCACHE_DISABLE === "1"); + const table = `${env.YOI_RUNTIME_DIR}/workers.json`; + tables.add(table); + assert(await Deno.readTextFile(table) === String(observed.pid)); + } + assert( + tables.size === commands.length, + "independent commands shared a table", + ); + assert(environmentSnapshot() === before); + } finally { + await Deno.remove(checkout, { recursive: true }); + } + }, +}); + +Deno.test("isolation refuses lexical aliases and symlinked owned paths before external writes", async () => { + for ( + const unsafe of ["target", "target/restore-test-environment", "target/t"] + ) { + const checkout = await Deno.realPath(await Deno.makeTempDir()); + const outside = await Deno.realPath(await Deno.makeTempDir()); + try { + await Deno.writeTextFile(`${outside}/sentinel`, "untouched"); + if (unsafe !== "target") await Deno.mkdir(`${checkout}/target`); + await Deno.symlink(outside, `${checkout}/${unsafe}`); + await assertRejects( + () => prepareTestIsolation(checkout, "run"), + "symlink", + ); + assert( + Array.from(Deno.readDirSync(outside)).map((entry) => entry.name) + .join() === + "sentinel", + ); + assert(await Deno.readTextFile(`${outside}/sentinel`) === "untouched"); + } finally { + await Deno.remove(checkout, { recursive: true }); + await Deno.remove(outside, { recursive: true }); + } + } + const checkout = await Deno.realPath(await Deno.makeTempDir()); + try { + await Deno.symlink(checkout, `${checkout}/alias`); + await assertRejects( + () => prepareTestIsolation(`${checkout}/alias`, "run"), + "symlink", + ); + for (const suffix of ["/../escape", "/./child", "//child"]) { + await assertRejects( + () => prepareTestIsolation(`${checkout}${suffix}`, "run"), + "normal-only", + ); + } + assert( + Array.from(Deno.readDirSync(checkout)).map((entry) => entry.name) + .join() === + "alias", + ); + } finally { + await Deno.remove(checkout, { recursive: true }); + } +}); + +Deno.test("command wrapping rechecks owned ancestors and refuses symlinked namespace children", async () => { + for ( + const unsafe of [ + "ancestor", + "tmp", + "config", + "data", + "run", + "home", + "git-template", + "git-template/info", + "git-template/hooks", + ] + ) { + const checkout = await Deno.realPath(await Deno.makeTempDir()); + const outside = await Deno.realPath(await Deno.makeTempDir()); + try { + const isolation = await prepareTestIsolation(checkout, "run"); + let path: string; + if (unsafe === "ancestor") { + path = isolation.root; + await Deno.remove(path); + } else if (unsafe === "tmp") { + path = `${checkout}/target/t`; + await Deno.remove(path); + } else { + const namespace = `${isolation.root}/unsafe-0`; + await Deno.mkdir(namespace); + if (unsafe.includes("/")) await Deno.mkdir(`${namespace}/git-template`); + path = `${namespace}/${unsafe}`; + } + await Deno.symlink(outside, path); + await assertRejects( + () => isolation.wrapCommand(fixtureTask("success", checkout), "unsafe"), + "symlink", + ); + assert(Array.from(Deno.readDirSync(outside)).length === 0); + } finally { + await Deno.remove(checkout, { recursive: true }); + await Deno.remove(outside, { recursive: true }); + } + } +}); + for (const [mode, code] of [["success", 0], ["nonzero", 7]] as const) { Deno.test({ name: `execute streams real ${mode} child output and records its exit`, @@ -436,15 +740,28 @@ Deno.test({ "runtime listing persists complete name inventory for large and small binaries", ignore: Deno.build.os !== "linux", async fn() { - const output = await Deno.makeTempDir(); + const output = await Deno.realPath(await Deno.makeTempDir()); try { + const isolation = await prepareTestIsolation(output, "discovery"); + const wrappedTables = new Set(); const discovery = await discoverRustCommands( [ fixtureTask("list-large", output), fixtureTask("list-small", output), ], 2, - { output }, + { + output, + run: async (task, label) => { + const wrapped = await isolation.wrapCommand(task, label); + wrappedTables.add(`${wrapped.env!.YOI_RUNTIME_DIR}/workers.json`); + return await execute(wrapped, { output, label, timeoutMs: 10_000 }); + }, + }, + ); + assert( + wrappedTables.size === 2, + "all discovery launches must use the callback", ); assert(discovery.inventory.complete); assert( @@ -635,6 +952,54 @@ if (import.meta.main) { ); } Deno.exit(mode === "list-nonzero" ? 7 : 0); + } else if (mode === "isolation-env") { + const env = Deno.env.toObject(); + const scratch = await Deno.makeTempDir({ dir: env.TMPDIR }); + const text = (bytes: Uint8Array) => new TextDecoder().decode(bytes); + async function git(args: string[]) { + return await new Deno.Command("git", { + args, + cwd: scratch, + stdout: "piped", + stderr: "piped", + }).output(); + } + try { + const ownedRepo = await git([ + "-C", + output, + "rev-parse", + "--show-toplevel", + ]); + assert( + ownedRepo.success && text(ownedRepo.stdout).trim() === output, + "owned ancestor repository fixture must be discoverable at its root", + ); + const ancestor = await git(["rev-parse", "--show-toplevel"]); + const init = await git(["init", "--quiet"]); + assert(init.success, text(init.stderr)); + const global = await git(["config", "--global", "--list"]); + assert(global.success, text(global.stderr)); + const author = await git(["var", "GIT_AUTHOR_IDENT"]); + assert(author.success, text(author.stderr)); + await Deno.writeTextFile( + `${env.YOI_RUNTIME_DIR}/workers.json`, + String(Deno.pid), + ); + console.log(JSON.stringify({ + env, + cwd: Deno.cwd(), + pid: Deno.pid, + ancestorDiscovered: ancestor.success, + globalConfig: text(global.stdout), + author: text(author.stdout), + hooks: Array.from(Deno.readDirSync(`${scratch}/.git/hooks`)), + info: Array.from(Deno.readDirSync(`${scratch}/.git/info`)), + })); + } finally { + await Deno.remove(scratch, { recursive: true }); + } + Deno.exit(0); } else if (mode === "runner-parent") { const cancellation = installCancellationHandlers(); try { diff --git a/tools/test.ts b/tools/test.ts index 14cb347d4..35bf8c769 100644 --- a/tools/test.ts +++ b/tools/test.ts @@ -1,9 +1,10 @@ -// Bound top-level suites/shards, not total processes; large shards use two test threads. +// Bound top-level suites/shards, not total processes; large shards normally use two threads. // Cargo selects targets/features. Retain all listed names; never skip cached successes. import { spawn } from "node:child_process"; import process from "node:process"; import { availableParallelism } from "node:os"; import { fileURLToPath } from "node:url"; +import { posix } from "node:path"; // Safety limits, NOT the separate 60s performance goal. export const SUITE_TIMEOUT_MS = 180_000; @@ -65,7 +66,19 @@ export function shardRustCommand( if (new Set(names).size !== names.length) { throw new Error("cannot shard duplicate test names"); } - if (names.length <= 72) return [{ task, selectedNames: [...names] }]; + // RuntimeDirSandbox changes process-global paths. Do not overlap its writes + // with restore guards in the same process, including small integration targets. + const serial = ["worker", "worker-runtime", "yoi-workspace-server"].includes( + task.name.split("/")[0], + ); + if (names.length <= 72) { + return [{ + task: serial + ? { ...task, args: [...task.args, "--test-threads", "1"] } + : task, + selectedNames: [...names], + }]; + } const count = Math.min(32, Math.ceil(names.length / 36)); const shards: string[][] = Array.from({ length: count }, () => []); names.forEach((name, index) => shards[index % count].push(name)); @@ -75,7 +88,13 @@ export function shardRustCommand( name: `${task.name}/shard-${ String(index + 1).padStart(2, "0") }-of-${count}`, - args: [...task.args, "--exact", "--test-threads", "2", ...selectedNames], + args: [ + ...task.args, + "--exact", + "--test-threads", + serial ? "1" : "2", + ...selectedNames, + ], }, selectedNames, })); @@ -84,18 +103,27 @@ export function shardRustCommand( export async function discoverRustCommands( commands: TestCommand[], jobs: number, - options: { output: string; signal?: AbortSignal }, + options: { + output: string; + signal?: AbortSignal; + run?: (task: TestCommand, label: string) => Promise; + }, ) { const binaries = await mapConcurrent(commands, jobs, async (task, index) => { - const listing = await execute({ + const listingTask = { ...task, name: `${task.name}/list`, args: [...task.args, "--list", "--format", "terse"], - }, { - ...options, - label: `list-${String(index).padStart(3, "0")}`, - timeoutMs: SUITE_TIMEOUT_MS, - }); + }; + const label = `list-${String(index).padStart(3, "0")}`; + const listing = await (options.run + ? options.run(listingTask, label) + : execute(listingTask, { + output: options.output, + signal: options.signal, + label, + timeoutMs: SUITE_TIMEOUT_MS, + })); let shards: RustShard[] = []; let error: string | undefined; try { @@ -124,7 +152,8 @@ export async function discoverRustCommands( error, selectedCases: error ? null - : shards.reduce((sum, shard) => sum + shard.selectedNames.length, 0), + : shards.reduce((sum, shard) => + sum + shard.selectedNames.length, 0), shards, }; }); @@ -426,12 +455,149 @@ export interface TestCommand { command: string; args: string[]; cwd: string; - env?: Record; + env?: Record; // Exact link from each execution result into inventory.json, even when two // Cargo targets happen to share the same package/target display name. inventoryRef?: { binaryIndex: number; shardIndex: number }; } +// Walk with lstat before creating any child: recursive mkdir/realPath alone +// would follow an existing symlink first. Reject aliases rather than silently +// canonicalizing a caller's checkout. As with the Rust guard, this assumes the +// owned checkout is not concurrently replaced by an adversarial filesystem actor. +async function ownedTestDirectory(checkout: string, relative: string) { + if ( + !posix.isAbsolute(checkout) || checkout === "/" || + posix.normalize(checkout) !== checkout + ) throw new Error("test checkout must be an absolute normal-only path"); + + async function checkDirectory(path: string, owned: boolean) { + const stat = await Deno.lstat(path); + if (!stat.isDirectory || stat.isSymlink) { + throw new Error( + `test isolation refuses non-directory or symlink: ${path}`, + ); + } + if (owned && process.getuid && stat.uid !== process.getuid()) { + throw new Error( + `test isolation directory is not owned by this user: ${path}`, + ); + } + if (await Deno.realPath(path) !== path) { + throw new Error( + `test isolation refuses non-canonical directory: ${path}`, + ); + } + } + let checkoutPath = ""; + for (const part of checkout.slice(1).split("/")) { + checkoutPath += `/${part}`; + await checkDirectory(checkoutPath, false); + } + let path = checkout; + for (const part of relative.split("/")) { + if (!part || part === "." || part === "..") { + throw new Error("invalid owned test directory component"); + } + path += `/${part}`; + try { + await Deno.mkdir(path, { mode: 0o700 }); + } catch (error) { + if (!(error instanceof Deno.errors.AlreadyExists)) throw error; + } + await checkDirectory(path, true); + } + return path; +} + +/** Prepare child-only isolation for the maintained normal-test entrypoint. + * Each wrap gets a fresh allocation table, even for repeated/sanitized labels or + * concurrent runners using the same output id. Resources stay under target for + * diagnosis; callers with temporary checkouts own their recursive cleanup. + */ +export async function prepareTestIsolation(checkout: string, runId: string) { + const ownedDirectory = (relative: string) => + ownedTestDirectory(checkout, relative); + const prefix = "target/restore-test-environment"; + await ownedDirectory(prefix); + // Keep tempfile Unix-socket references short. tempfile owns unique resources + // here; neither runners nor suites remove/replace this shared directory. + const tmp = await ownedDirectory("target/t"); + const digest = new Uint8Array( + await crypto.subtle.digest("SHA-256", new TextEncoder().encode(runId)), + ); + const id = Array.from( + digest.slice(0, 6), + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""); + const relativeRoot = `${prefix}/runner-${id}-${crypto.randomUUID()}`; + const root = await ownedDirectory(relativeRoot); + let next = 0; + return { + root, + async wrapCommand(task: TestCommand, label: string): Promise { + const safeLabel = label.replaceAll(/[^a-zA-Z0-9_-]/g, "-").slice(0, 40) || + "command"; + const relative = `${relativeRoot}/${safeLabel}-${next++}`; + // Recheck shared ancestors as well as the command namespace on every wrap. + await ownedDirectory("target/t"); + const namespace = await ownedDirectory(relative); + for ( + const dir of [ + "config", + "data", + "run", + "home", + "git-template/info", + "git-template/hooks", + ] + ) { + await ownedDirectory(`${relative}/${dir}`); + } + const inherited = { ...process.env, ...task.env }; + const env: NonNullable = { ...task.env }; + // Undefined shadows inherited entries in execute's merge; node spawn omits + // them. Strip indexed Git config and less common Git path overrides too. + for (const key of Object.keys(inherited)) { + if (/^GIT_/i.test(key) || /(^|_)proxy$/i.test(key)) { + env[key] = undefined; + } + } + Object.assign(env, { + HOME: `${namespace}/home`, + YOI_HOME: `${namespace}/data`, + YOI_CONFIG_DIR: `${namespace}/config`, + YOI_DATA_DIR: `${namespace}/data`, + YOI_RUNTIME_DIR: `${namespace}/run`, + TMPDIR: tmp, + TMP: tmp, + TEMP: tmp, + GIT_TEMPLATE_DIR: `${namespace}/git-template`, + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_SYSTEM: "/dev/null", + GIT_CONFIG_NOSYSTEM: "1", + GIT_ATTR_NOSYSTEM: "1", + GIT_CEILING_DIRECTORIES: checkout, + GIT_AUTHOR_NAME: "Normal test runner", + GIT_AUTHOR_EMAIL: "normal-tests@example.invalid", + GIT_COMMITTER_NAME: "Normal test runner", + GIT_COMMITTER_EMAIL: "normal-tests@example.invalid", + RUSTC_WRAPPER: "", + RUSTC_WORKSPACE_WRAPPER: "", + SCCACHE_DISABLE: "1", + }); + // HOME is isolated, but toolchain/cache discovery must keep its ordinary + // host locations (these are compiler settings, not live product state). + for (const key of ["CARGO_HOME", "RUSTUP_HOME"] as const) { + const fallback = key === "CARGO_HOME" ? ".cargo" : ".rustup"; + env[key] = inherited[key] || + (process.env.HOME ? `${process.env.HOME}/${fallback}` : undefined); + } + return { ...task, env }; + }, + }; +} + export function rustTestCommands( artifacts: CargoArtifact[], packages: { id: string; name: string; manifest_path: string }[], @@ -497,16 +663,26 @@ async function main() { "", ); const jobs = parseJobs(Deno.args); - const output = `${root}/target/normal-tests/${ + const outputId = `${ new Date().toISOString().replaceAll(":", "-") - }`; - await Deno.mkdir(output, { recursive: true }); + }-${crypto.randomUUID()}`; + const isolation = await prepareTestIsolation(root, outputId); + const output = await ownedTestDirectory( + root, + `target/normal-tests/${outputId}`, + ); const cancellation = installCancellationHandlers(); - const run = ( + const run = async ( task: TestCommand, label: string, timeoutMs = SUITE_TIMEOUT_MS, - ) => execute(task, { output, label, timeoutMs, signal: cancellation.signal }); + ) => + execute(await isolation.wrapCommand(task, label), { + output, + label, + timeoutMs, + signal: cancellation.signal, + }); const summary: Record = { phase: "metadata", failed: false }; let executionStart: number | undefined; const writeSummary = () => @@ -516,7 +692,7 @@ async function main() { { jobs, concurrency: - "top-level suites/shards; large Rust shards use 2 test threads, other suites retain internal parallelism", + "top-level suites/shards; worker, worker-runtime and yoi-workspace-server use 1 test thread; other large Rust shards use 2, other suites retain internal parallelism", platform: "linux", suiteTimeoutMs: SUITE_TIMEOUT_MS, buildTimeoutMs: BUILD_TIMEOUT_MS, @@ -600,6 +776,7 @@ async function main() { const discovery = await discoverRustCommands(rust, jobs, { output, signal: cancellation.signal, + run, }); summary.discoverySeconds = (performance.now() - discoveryStart) / 1000; summary.inventory = "inventory.json"; @@ -655,6 +832,7 @@ async function main() { "--allow-write", "--allow-run", "--allow-env", + "--allow-sys=uid", "tools/test.test.ts", "tools/check-state-contract-names.test.ts", ], @@ -663,7 +841,7 @@ async function main() { ...discovery.tasks, ]; console.log( - `Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (large Rust shards: 2 test threads; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`, + `Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (worker/worker-runtime/yoi-workspace-server: 1 test thread; other large Rust shards: 2; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`, ); const results: ReturnType[] = []; summary.results = results; From 907392cce8d7915913c2ab785056867ebd76246d Mon Sep 17 00:00:00 2001 From: Hare Date: Fri, 23 Oct 2026 00:21:02 +0900 Subject: [PATCH 04/28] fix: preserve restore effects and fence prepared attachment replay --- crates/worker-runtime/src/runtime.rs | 112 +++- crates/worker-runtime/src/worker_backend.rs | 69 +- .../worker_backend_restore_candidate_tests.rs | 206 ++++++ .../worker_backend_restore_failure_tests.rs | 595 +++++++++++++++++- 4 files changed, 952 insertions(+), 30 deletions(-) diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index eaec092dc..fc53df360 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -2150,6 +2150,16 @@ impl Runtime { { return Err(conflict()); } + if let Err(error) = owner.validate_workdir_preparation_authority(&candidate) { + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_authority_changed", + sanitize_worker_create_failure_message(&error.to_string()), + )), + None, + )); + } } else { // Startup/older persisted pending operations have a lifecycle // owner but no public request receipt. Bind recovery to that @@ -2215,12 +2225,18 @@ impl Runtime { candidate.pending_restore = Some(pending); candidate.restore_guard.rotate(); candidate.restore_guard.active_request_id = Some(request.request_id.clone()); + let workdir_attachment_preparation_base = request + .preparation + .as_ref() + .and_then(|preparation| preparation.workdir_attachments.as_ref()) + .map(|_| candidate.logical_workdir_attachments.clone()); candidate.restore_guard.owners.insert( request.request_id.clone(), RestoreRequestOwner { fingerprint, operation_id, request, + workdir_attachment_preparation_base, preparation_applied: recovering_pending, execution_started: recovering_pending, receipt: None, @@ -2433,14 +2449,15 @@ impl Runtime { { return Ok(RuntimeWorkerRestoreResult::accepted(worker.detail())); } - let preparation = worker + let owner = worker .restore_guard .active_request_id .as_ref() - .and_then(|id| worker.restore_guard.owners.get(id)) - .filter(|owner| !owner.preparation_applied) - .and_then(|owner| owner.request.preparation.as_ref()); - let prepared_worker = match worker.prepared_restore_record(preparation) { + .and_then(|id| worker.restore_guard.owners.get(id)); + let prepared_worker = match owner.map_or_else( + || Ok(worker.clone()), + |owner| owner.prepared_worker_record(worker), + ) { Ok(prepared) => prepared, Err(error) => { return Ok(RuntimeWorkerRestoreResult::failed( @@ -4398,8 +4415,7 @@ impl Runtime { worker_id: worker_ref.worker_id, }); } - let mut candidate = - previous.prepared_restore_record(owner.request.preparation.as_ref())?; + let mut candidate = owner.prepared_worker_record(previous)?; candidate .restore_guard .owners @@ -6277,12 +6293,48 @@ pub(crate) struct RestoreRequestOwner { fingerprint: String, operation_id: WorkerLifecycleOperationId, request: runtime_api::WorkerRestoreRequest, + // This is the admission comparison input, not a second attachment authority. + // Without it an old preparation could hide a later canonical revocation. + // Older owners lacking this fact cannot safely reapply attachment overrides. + #[serde(default, skip_serializing_if = "Option::is_none")] + workdir_attachment_preparation_base: Option>, preparation_applied: bool, #[serde(default)] execution_started: bool, receipt: Option, } +impl RestoreRequestOwner { + fn validate_workdir_preparation_authority( + &self, + worker: &WorkerRecord, + ) -> Result<(), RuntimeError> { + if !self.preparation_applied + && self + .request + .preparation + .as_ref() + .is_some_and(|preparation| preparation.workdir_attachments.is_some()) + && self.workdir_attachment_preparation_base.as_ref() + != Some(&worker.logical_workdir_attachments) + { + return Err(RuntimeError::InvalidRequest( + "current Worker authority differs from admitted Restore attachment preparation" + .into(), + )); + } + Ok(()) + } + + fn prepared_worker_record(&self, worker: &WorkerRecord) -> Result { + if self.preparation_applied { + return Ok(worker.clone()); + } + self.validate_workdir_preparation_authority(worker)?; + worker.prepared_restore_record(self.request.preparation.as_ref()) + } +} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct RestoreGuard { @@ -11995,7 +12047,7 @@ mod tests { let mut request = runtime.test_restore_request(&worker.worker_ref); request.preparation = Some(serde_json::from_value(serde_json::json!({ "workspace_api": { "workspace_id": "workspace-a", "base_url": "https://prepared.example" }, - "workdir_attachments": [], "repository_access": [] + "workdir_attachments": [{ "alias": "checkout", "working_directory_id": "prepared-logical-workdir" }], "repository_access": [] })).unwrap()); let gate = Arc::new(RestoreGate::default()); *backend.restore_gate.lock().unwrap() = Some(gate.clone()); @@ -12014,6 +12066,26 @@ mod tests { let state = runtime.lock().unwrap(); let record = state.worker(&worker.worker_ref).unwrap(); assert!(record.request.as_ref().unwrap().workspace_api.is_none()); + assert!(record.logical_workdir_attachments.is_empty()); + let owner = record.restore_guard.owners.values().next().unwrap(); + let mut saved_owner = serde_json::to_value(owner).unwrap(); + let decoded: RestoreRequestOwner = serde_json::from_value(saved_owner.clone()).unwrap(); + assert!( + decoded + .validate_workdir_preparation_authority(record) + .is_ok() + ); + saved_owner + .as_object_mut() + .unwrap() + .remove("workdir_attachment_preparation_base"); + let legacy: RestoreRequestOwner = serde_json::from_value(saved_owner).unwrap(); + assert!( + legacy + .validate_workdir_preparation_authority(record) + .is_err(), + "a missing historical comparison input must not be invented from current authority" + ); assert!( !record .restore_guard @@ -12033,6 +12105,11 @@ mod tests { .base_url, "https://prepared.example" ); + assert_eq!( + backend.restore_requests.lock().unwrap()[0].logical_workdir_attachments[0] + .working_directory_id, + "prepared-logical-workdir" + ); assert!(backend.activation_operation_ids.lock().unwrap().is_empty()); gate.release(); assert_eq!( @@ -12053,6 +12130,25 @@ mod tests { .base_url, "https://prepared.example" ); + let record = state.worker(&worker.worker_ref).unwrap(); + assert_eq!( + record.logical_workdir_attachments[0].working_directory_id, + "prepared-logical-workdir" + ); + let owner = record.restore_guard.owners.values().next().unwrap(); + assert!(owner.preparation_applied); + let mut revoked = record.clone(); + revoked + .apply_logical_workdir_attachments(Vec::new()) + .unwrap(); + assert!( + owner + .prepared_worker_record(&revoked) + .unwrap() + .logical_workdir_attachments + .is_empty(), + "a committed preparation is not a perpetual grant" + ); } #[cfg(feature = "ws-server")] diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index b5ace780f..695a03379 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -360,20 +360,21 @@ async fn prepare_restore_operation( failure })?; if repaired { - Ok(Box::new(RepairedRestoreCandidate { candidate })) + Ok(Box::new(EffectRetainingRestoreCandidate { candidate })) } else { Ok(candidate) } } -// A completed repair is a canonical effect even when later activation fails -// before its own mutation boundary. Carry that evidence with the resource owner. -struct RepairedRestoreCandidate { +// Completed repair or an earlier activation may have changed canonical state. +// Keep that operation-owned evidence through reconstruction and even an +// activation failure before the replacement candidate's own mutation boundary. +struct EffectRetainingRestoreCandidate { candidate: Box, } #[async_trait] -impl RuntimeRestoreCandidate for RepairedRestoreCandidate { +impl RuntimeRestoreCandidate for EffectRetainingRestoreCandidate { fn initial_state(&self) -> protocol::WorkerStateSnapshot { self.candidate.initial_state() } @@ -2953,6 +2954,9 @@ struct PendingRuntimeRestore { task: tokio::sync::Mutex>>, result: Mutex, RuntimeRestoreFailure>>>, activated: Mutex>>, + // Monotonic for this operation, not a claim that its current candidate has + // started. A reconstructed candidate has a new result and no completion in + // `activated`; prior activation/effect evidence must survive that handoff. activation_started: AtomicBool, discard_started: AtomicBool, discard_result: Mutex>>, @@ -2960,6 +2964,16 @@ struct PendingRuntimeRestore { } impl PendingRuntimeRestore { + fn owns_prepared_candidate(&self) -> bool { + matches!( + self.result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref(), + Some(Ok(_)) + ) + } + async fn wait(&self, timeout: Duration) -> Result<(), String> { let mut task = self.task.lock().await; if let Some(message) = self @@ -3279,23 +3293,40 @@ where && (failure.durable_effects_possible || owner.activation_started.load(Ordering::Acquire)) { + let prior_effects = failure.durable_effects_possible + || owner.activation_started.load(Ordering::Acquire); let result = match prepare_restore_operation(factory.as_ref(), owner.request.clone()) .await { - Ok(candidate) => Ok(candidate), - Err(failure) => Err(finish_restore_failure( - factory.as_ref(), - &owner.request, - failure, - ) - .await), + Ok(candidate) => { + Ok(Box::new(EffectRetainingRestoreCandidate { candidate }) + as Box) + } + Err(mut failure) => { + failure.durable_effects_possible |= prior_effects; + Err(finish_restore_failure( + factory.as_ref(), + &owner.request, + failure, + ) + .await) + } }; - owner.activation_started.store(false, Ordering::Release); + // A completed reprepare replaces attempt resources, never + // the operation's effect history. Only a ready candidate + // may clear the old activation completion for a new attempt. + let activated = match &result { + Ok(_) => None, + Err(failure) if owner.activation_started.load(Ordering::Acquire) => { + Some(Err(failure.clone())) + } + Err(_) => None, + }; *owner .activated .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = None; + .unwrap_or_else(std::sync::PoisonError::into_inner) = activated; *owner .result .lock() @@ -4019,7 +4050,9 @@ where failure.to_string(), ); } - if pending.activation_started.load(Ordering::Acquire) { + if pending.activation_started.load(Ordering::Acquire) + && !pending.owns_prepared_candidate() + { let mut activated = match pending.activated.lock() { Ok(result) => result, Err(_) => { @@ -5205,7 +5238,8 @@ where }; } } - if pending.activation_started.load(Ordering::Acquire) { + if pending.activation_started.load(Ordering::Acquire) && !pending.owns_prepared_candidate() + { let activated = pending .activated .lock() @@ -5330,7 +5364,8 @@ where return Err("restored Worker operation identity does not match".into()); } self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; - if !pending.activation_started.swap(true, Ordering::AcqRel) { + if pending.owns_prepared_candidate() { + pending.activation_started.store(true, Ordering::Release); let owner = pending.clone(); let factory = self.factory.clone(); let task = self.spawn_on_adapter_runtime(async move { diff --git a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs index a5b0312f9..443fbac6f 100644 --- a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs @@ -895,4 +895,210 @@ mod restore_candidate { assert_eq!(model.call_count.load(Ordering::SeqCst), 0); } } + + #[test] + #[serial_test::serial(worker_allocation)] + fn attachment_preparation_replay_cannot_hide_logical_authority_revocation() { + use crate::fs_store::{ + FsRuntimeStoreOptions, PersistedWorkerExecution, PersistedWorkerExecutionState, + PersistedWorkerRecord, + }; + let mut fixture = Fixture::new(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_secs(2); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let options = FsRuntimeStoreOptions { + root: fixture._root.path().join("catalog"), + runtime_id: "candidate-runtime".into(), + display_name: None, + }; + let attachment = crate::catalog::LogicalWorkdirAttachment { + alias: WorkdirAttachmentAlias::new("checkout").unwrap(), + working_directory_id: "workspace-only-workdir".into(), + capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY, + }; + // Only the initial Runtime catalog is seeded. The saved Profile, queue, + // restore admission, revocation, replay and Stop use their real owners. + let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( + options.clone(), + fixture.backend.clone(), + ) + .unwrap(); + let create = fixture.request.request.clone(); + runtime + .test_runtime_store() + .write_worker_record(&PersistedWorkerRecord { + restore_guard: crate::runtime::RestoreGuard::default(), + worker_ref: worker_ref.clone(), + worker_id: worker_ref.worker_id, + profile: create.profile.clone(), + display_name: None, + profile_source: create.profile_source.reference(), + config_bundle: create.config_bundle.clone(), + created_at_ms: None, + status: crate::catalog::WorkerStatus::Stopped, + last_finished_submission_request_id: None, + execution_state: PersistedWorkerExecutionState::Available( + PersistedWorkerExecution { + request: create, + binding: None, + restore_intent: crate::catalog::WorkerRestoreIntent::Explicit, + }, + ), + workspace_id: Some(scope.workspace_id.clone()), + workdir_attachments: Vec::new(), + logical_workdir_attachments: vec![attachment.clone()], + }) + .unwrap(); + drop(runtime); + let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( + options.clone(), + fixture.backend.clone(), + ) + .unwrap(); + assert!( + runtime + .worker_detail(&worker_ref) + .unwrap() + .workdir_attachments + .is_empty() + ); + let before = fixture.canonical_files(); + let authority = || -> serde_json::Value { + serde_json::from_slice( + &fs::read( + options + .root + .join("workers") + .join(worker_ref.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap() + }; + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .factory_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .factory_entered + .lock() + .unwrap() = Some(entered_tx); + let mut request = runtime.test_restore_request(&worker_ref); + request.preparation.as_mut().unwrap().workdir_attachments = + Some(serde_json::from_value(serde_json::to_value(vec![attachment]).unwrap()).unwrap()); + let first = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request.clone()) + .unwrap(); + entered.recv_timeout(WAIT).unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{first:?}" + ); + let pending = fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .unwrap(); + assert!(!pending.request.logical_workdir_attachments.is_empty()); + assert!(pending.request.previous_workdir_attachments.is_empty()); + assert!(pending.request.deferred_workdir_sessions.is_empty()); + runtime + .replace_worker_workdir_attachments_scoped(&scope, &worker_ref, Vec::new()) + .unwrap(); + assert!(authority().get("logical_workdir_attachments").is_none()); + release.notify_one(); + let replay = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "revocation hidden by saved preparation: {replay:?}" + ); + assert!( + replay + .message + .as_deref() + .unwrap_or_default() + .contains("current Worker authority differs"), + "{replay:?}" + ); + let record = authority(); + assert!(record.get("logical_workdir_attachments").is_none()); + let owner = &record["restore_guard"]["owners"][&request.request_id]; + assert_eq!( + owner["operation_id"], + serde_json::to_value(pending.request.operation_id).unwrap() + ); + assert!(owner["receipt"].is_null()); + assert_eq!( + owner["workdir_attachment_preparation_base"] + .as_array() + .unwrap() + .len(), + 1 + ); + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + let durable_guard = + serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap(); + assert_eq!(durable_guard["owners"][&request.request_id], *owner); + let (coordinated, preparation) = runtime + .coordinate_worker_restore_operation( + &worker_ref, + runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: None, + }, + Some(&scope), + ) + .unwrap(); + assert_eq!( + coordinated.unwrap().state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + assert!( + preparation.is_none(), + "coordinator must not export the revoked preparation again" + ); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!(Arc::ptr_eq( + &pending, + &fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .unwrap() + )); + assert!(!pending.activation_started.load(Ordering::SeqCst)); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before); + runtime + .stop_worker_scoped(&scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before); + assert!(authority().get("logical_workdir_attachments").is_none()); + assert_ne!( + runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request) + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + } } diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs index 40de8168c..91530ab28 100644 --- a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -100,6 +100,148 @@ impl WorkingDirectoryMaterializer for RestoreOpenCountingMaterializer { } } +// Substitute only initial creation: a new aggregate/Session is left pending +// before its first head, without ever materializing or rewinding a saved head. +// All Restore preflight, repair, retry, activation and Stop use the real backend. +struct PendingProfileSeedBackend { + inner: Arc, +} + +impl WorkerExecutionBackend for PendingProfileSeedBackend { + fn backend_id(&self) -> &str { + self.inner.backend_id() + } + + fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { + let aggregate = self + .inner + .factory + .worker_aggregate_dir(&request.worker_ref) + .unwrap(); + let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&request.worker_ref); + self.inner + .run_on_adapter_runtime(async move { + assert!( + !aggregate.join("session").exists(), + "pending fixture requires a new Session aggregate" + ); + let manifest = manifest::WorkerManifest::from_toml(&format!( + r#" +[worker] +name = "{name}" +pwd = "./" +[scope] +allow = [] +[model] +scheme = "anthropic" +model_id = "test-model" +auth = {{ kind = "none" }} +[engine] +max_tokens = 100 +"# + )) + .unwrap(); + let sessions = WorkerSessionStore::new(aggregate.join("session")).unwrap(); + let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap(); + assert!(metadata.read_by_name(&name).unwrap().is_none()); + let engine = + Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated( + MockClient::new(Vec::new()), + ); + let mut head = Worker::new( + manifest.clone(), + engine, + CombinedStore::new(sessions.clone(), metadata.clone()), + WorkerWorkspaceContext::unavailable( + None, + "initial creation interrupted before first head", + ), + WorkerFilesystemAuthority::None, + Scope::empty(), + ) + .await + .unwrap(); + head.enable_worker_metadata_write_through().unwrap(); + let mut saved = metadata.read_by_name(&name).unwrap().unwrap(); + let active = saved.active.as_ref().unwrap(); + assert!(active.segment_id.is_none()); + assert!(sessions.list_segments(active.session_id).unwrap().is_empty()); + saved.resolved_manifest_snapshot = + Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap()); + metadata.write(&saved).unwrap(); + drop(head); + Ok(()) + }) + .unwrap(); + WorkerExecutionSpawnResult::Connected { + worker_state: protocol::WorkerStateSnapshot::initial(), + workdir_attachments: Vec::new(), + } + } + + fn fetch_workspace_config( + &self, + request: WorkspaceConfigFetchRequest, + ) -> Result { + self.inner.fetch_workspace_config(request) + } + + fn observe_workspace_prompt_projection( + &self, + projection: worker::WorkspacePromptProjection, + ) -> Result<(), String> { + self.inner.observe_workspace_prompt_projection(projection) + } + + fn preflight_restore( + &self, + request: &WorkerExecutionRestoreRequest, + ) -> Result<(), WorkerExecutionResult> { + self.inner.preflight_restore(request) + } + + fn completed_restore_failure( + &self, + operation_id: crate::execution::WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Option { + self.inner + .completed_restore_failure(operation_id, worker_ref) + } + + fn restore_worker(&self, request: WorkerExecutionRestoreRequest) -> WorkerExecutionSpawnResult { + self.inner.restore_worker(request) + } + + fn reconcile_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + self.inner.reconcile_restore(request) + } + + fn activate_restored_worker( + &self, + operation_id: crate::execution::WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Result<(), String> { + self.inner + .activate_restored_worker(operation_id, worker_ref) + } + + fn dispatch_input(&self, worker_ref: &WorkerRef, input: WorkerInput) -> WorkerExecutionResult { + self.inner.dispatch_input(worker_ref, input) + } + + fn dispatch_method(&self, worker_ref: &WorkerRef, method: Method) -> WorkerExecutionResult { + self.inner.dispatch_method(worker_ref, method) + } + + fn stop_worker_operation(&self, request: WorkerExecutionStopRequest) -> WorkerExecutionResult { + self.inner.stop_worker_operation(request) + } +} + struct ProfileFailureFixture { runtime: EmbeddedRuntime, backend: Arc, @@ -115,6 +257,14 @@ impl ProfileFailureFixture { } fn with_saved_inputs(timeout: Duration, model: Option) -> Self { + Self::with_seed(timeout, model, false) + } + + fn with_pending_head(timeout: Duration) -> Self { + Self::with_seed(timeout, None, true) + } + + fn with_seed(timeout: Duration, model: Option, pending_head: bool) -> Self { let root = restore_failure_tempdir(); let repo = root.path().join("g"); fs::create_dir(&repo).unwrap(); @@ -141,22 +291,31 @@ impl ProfileFailureFixture { opens: provider_opens.clone(), }), ); + let execution: Arc = if pending_head { + Arc::new(PendingProfileSeedBackend { + inner: backend.clone(), + }) + } else { + backend.clone() + }; let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( crate::fs_store::FsRuntimeStoreOptions { root: store, runtime_id: "r".into(), display_name: None, }, - backend.clone(), + execution, ) .unwrap(); runtime.store_config_bundle(test_bundle()).unwrap(); let mut request = create_request("restore failure recovery"); request.profile = ProfileSelector::Builtin("default".into()); - request.workdir_attachment_requests = vec![WorkingDirectoryAttachmentRequest { - alias: WorkdirAttachmentAlias::new("checkout").unwrap(), - working_directory: working_directory_request(&repo), - }]; + if !pending_head { + request.workdir_attachment_requests = vec![WorkingDirectoryAttachmentRequest { + alias: WorkdirAttachmentAlias::new("checkout").unwrap(), + working_directory: working_directory_request(&repo), + }]; + } let worker = runtime.create_worker(request).unwrap(); let seed_handle = queue_inputs.then(|| { backend.workers.lock().unwrap()[&worker.worker_ref] @@ -203,6 +362,127 @@ impl ProfileFailureFixture { *self.backend.factory.restore_faults.fail_at.lock().unwrap() = Some(stage); } + fn pin_unclaimed_restore_upload(&self) -> (PathBuf, BTreeMap>) { + let aggregate = self + .backend + .factory + .worker_aggregate_dir(&self.worker.worker_ref) + .unwrap(); + let name = + ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&self.worker.worker_ref); + let metadata = WorkerAggregateStore::open_read_only(&aggregate, name.clone()).unwrap(); + let session_id = metadata + .read_by_name(&name) + .unwrap() + .unwrap() + .active + .unwrap() + .session_id; + let session = WorkerSessionStore::new(aggregate.join("session")).unwrap(); + let orphan = session + .write_uploaded_file( + session_id, + "orphan.txt", + "text/plain", + b"restore effect evidence", + session_store::UploadedFileLimits::default(), + ) + .unwrap(); + session + .pin_uploaded_file(session_id, &orphan, "uncommitted-restore-input") + .unwrap(); + let artifacts = aggregate.join("session/artifacts/paste"); + let pinned = persisted_files(&artifacts); + assert!(!pinned.is_empty()); + (artifacts, pinned) + } + + fn assert_retained_restore_effects( + &self, + request: &runtime_api::WorkerRestoreRequest, + pending: &Arc, + stage: RuntimeRestoreStage, + ) { + let retained = self + .backend + .pending_restore(&self.worker.worker_ref) + .unwrap() + .expect("resource cleanup must not remove the operation's effect owner"); + assert!(Arc::ptr_eq(pending, &retained)); + { + let result = retained.result.lock().unwrap(); + let failure = result.as_ref().unwrap().as_ref().err().unwrap(); + assert_eq!(failure.stage, stage); + assert_eq!(failure.cleanup, RuntimeRestoreCleanup::Confirmed); + assert!( + failure.durable_effects_possible, + "resource cleanup cannot erase prior canonical effects: {failure:?}" + ); + } + let record: serde_json::Value = serde_json::from_slice( + &fs::read( + self.root + .path() + .join("s/workers") + .join(self.worker.worker_ref.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!( + record["restore_guard"]["active_request_id"], + request.request_id + ); + let operation = serde_json::to_value(pending.request.operation_id).unwrap(); + let owner = &record["restore_guard"]["owners"][&request.request_id]; + assert_eq!(owner["operation_id"], operation); + assert!( + owner["receipt"].is_null(), + "false terminal receipt: {owner:?}" + ); + assert_eq!( + record["execution_state"]["state"], + "reconciliation_required" + ); + assert_eq!( + record["execution_state"]["execution"]["operation"], + "restore" + ); + assert_eq!( + record["execution_state"]["execution"]["intent"]["operation_id"], + operation + ); + assert!( + self.backend + .completed_restore_failures + .lock() + .unwrap() + .is_empty() + ); + assert!(self.backend.workers.lock().unwrap().is_empty()); + assert!( + self.backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty(), + "cleanup confirmed resources, not rollback of canonical effects" + ); + assert!( + self.backend + .factory + .observation_hub + .get(&self.worker.worker_ref) + .is_none() + ); + if !pending.request.deferred_workdir_sessions.is_empty() { + self.assert_closed_deferred_sessions(pending); + } + assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0); + } + fn assert_released(&self) { assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0); assert!(self.backend.pending_restores.lock().unwrap().is_empty()); @@ -422,6 +702,311 @@ impl ProfileFailureFixture { } } +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_prior_effects_survive_read_only_reprepare_failure() { + for (activation_stage, read_stage) in [ + (RuntimeRestoreStage::Feature, RuntimeRestoreStage::Metadata), + ( + RuntimeRestoreStage::Controller, + RuntimeRestoreStage::Reconstruction, + ), + ] { + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload(); + fixture.fail(activation_stage); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let first = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{activation_stage:?}: {first:?}" + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + assert!(pending.activation_started.load(Ordering::SeqCst)); + fixture.assert_retained_restore_effects(&request, &pending, activation_stage); + let effected = persisted_files(&artifacts); + assert_ne!( + effected, pinned, + "activation must actually reconcile the orphan pin" + ); + + // The next attempt needs no repair, and fails before any new mutation. + // Resource cleanup is confirmed, but the first attempt's pin change + // was not rolled back. Repeated sends must retain the same owner. + for _ in 0..2 { + fixture.fail(read_stage); + let replay = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{activation_stage:?} then {read_stage:?}: {replay:?}" + ); + fixture.assert_retained_restore_effects(&request, &pending, read_stage); + assert!(pending.activation_started.load(Ordering::SeqCst)); + assert_eq!(persisted_files(&artifacts), effected); + } + fixture.resume_same_operation(request, pending.request.operation_id); + } +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_prior_effects_from_pending_head_repair_survive_read_only_reprepare_failure() { + let fixture = ProfileFailureFixture::with_pending_head(Duration::from_secs(5)); + let aggregate = fixture + .backend + .factory + .worker_aggregate_dir(&fixture.worker.worker_ref) + .unwrap(); + let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&fixture.worker.worker_ref); + let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap(); + let initial = metadata.read_by_name(&name).unwrap().unwrap(); + let pending_session = initial.active.as_ref().unwrap().session_id; + assert!(initial.active.as_ref().unwrap().segment_id.is_none()); + { + let before = WorkerSessionStore::new(aggregate.join("session")).unwrap(); + assert!(before.list_sessions().unwrap().is_empty()); + assert!(before.list_segments(pending_session).unwrap().is_empty()); + } + // The independent new Worker has never published a head. Its first segment + // and active pointer must be created by production repair, not fixture + // rewinding/deletion of a materialized Session. + fixture.fail(RuntimeRestoreStage::Metadata); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let first = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{first:?}" + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + assert!(!pending.activation_started.load(Ordering::SeqCst)); + assert!(pending.request.deferred_workdir_sessions.is_empty()); + fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Metadata); + let failure = pending + .result + .lock() + .unwrap() + .as_ref() + .unwrap() + .as_ref() + .err() + .unwrap() + .clone(); + let repaired = WorkerAggregateStore::open_read_only(&aggregate, &name) + .unwrap() + .read_by_name(&name) + .unwrap() + .unwrap(); + eprintln!( + "pending-head repair first={first:?} failure={failure:?} active={:?}", + repaired.active + ); + let active = repaired + .active + .as_ref() + .expect("real repair must retain the pending Session's active pointer"); + assert_eq!(active.session_id, pending_session); + // Store handles opened before the first Manifest retain session_id=None; + // they cannot observe that another owner has since published this Session. + // Reopen read-only to require the canonical Manifest as well as its logs. + let session = WorkerSessionStore::open_read_only(aggregate.join("session")) + .unwrap_or_else(|error| { + panic!( + "repair Manifest reread failed: {error}; first={first:?}; failure={failure:?}; active={active:?}" + ) + }); + assert_eq!(session.list_sessions().unwrap(), vec![pending_session]); + let segments = session.list_segments(pending_session).unwrap(); + assert_eq!( + segments.len(), + 1, + "real repair must commit exactly one seed; failure={failure:?}; active={active:?}" + ); + assert_eq!(active.segment_id, Some(segments[0])); + let entries = session + .read_all_read_only(pending_session, segments[0]) + .unwrap(); + assert!(matches!( + entries.first(), + Some(LogEntry::AnnotatedSegmentStart { .. }) + )); + assert!( + session_store::collect_state(&entries) + .system_prompt + .is_some() + ); + assert_ne!( + repaired, initial, + "repair must publish the first active head" + ); + let repaired_session = persisted_files(&aggregate.join("session")); + + for _ in 0..2 { + fixture.fail(RuntimeRestoreStage::Reconstruction); + let replay = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "repair effect cannot become a rollback on the next read failure: {replay:?}" + ); + fixture.assert_retained_restore_effects( + &request, + &pending, + RuntimeRestoreStage::Reconstruction, + ); + let current_metadata = WorkerAggregateStore::open_read_only(&aggregate, &name).unwrap(); + assert_eq!( + current_metadata.read_by_name(&name).unwrap().unwrap(), + repaired + ); + let current_session = + WorkerSessionStore::open_read_only(aggregate.join("session")).unwrap(); + assert_eq!( + current_session.list_sessions().unwrap(), + vec![pending_session] + ); + assert_eq!( + current_session.list_segments(pending_session).unwrap(), + segments + ); + assert_eq!( + persisted_files(&aggregate.join("session")), + repaired_session + ); + } + fixture.resume_same_operation(request, pending.request.operation_id); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_prior_effects_survive_rebuilt_candidate_pre_effect_activation_failure() { + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload(); + fixture.fail(RuntimeRestoreStage::Feature); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let first = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{first:?}" + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Feature); + let effected = persisted_files(&artifacts); + assert_ne!(effected, pinned); + + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .factory_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .factory_entered + .lock() + .unwrap() = Some(entered_tx); + let runtime = fixture.runtime.clone(); + let worker_ref = fixture.worker.worker_ref.clone(); + let resend = request.clone(); + let retry = std::thread::spawn(move || runtime.restore_worker_operation(&worker_ref, resend)); + let reached = entered.recv_timeout(Duration::from_secs(5)); + // This gate is after real read-only reconstruction. Change only saved + // metadata, so the rebuilt candidate's first activation revalidation fails + // before allocation/pin/metadata/delegation mutation. No factory or repair + // implementation is replaced by a fake. + let aggregate = fixture + .backend + .factory + .worker_aggregate_dir(&fixture.worker.worker_ref) + .unwrap(); + let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&fixture.worker.worker_ref); + let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap(); + let mut changed = metadata.read_by_name(&name).unwrap().unwrap(); + changed.workspace_root = Some(fixture.root.path().join("changed-after-reconstruction")); + metadata.write(&changed).unwrap(); + release.notify_one(); + let rebuilt = retry.join().unwrap().unwrap(); + reached.expect("same-operation retry must reach the real reconstructed candidate"); + assert_eq!( + rebuilt.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{rebuilt:?}" + ); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 2 + ); + assert!(pending.activation_started.load(Ordering::SeqCst)); + fixture.assert_retained_restore_effects( + &request, + &pending, + RuntimeRestoreStage::Reconstruction, + ); + assert_eq!(metadata.read_by_name(&name).unwrap().unwrap(), changed); + assert_eq!(persisted_files(&artifacts), effected); + + fixture.fail(RuntimeRestoreStage::Metadata); + let replay = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "successful reconstruction must not erase the older effect: {replay:?}" + ); + fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Metadata); + assert_eq!(metadata.read_by_name(&name).unwrap().unwrap(), changed); + assert_eq!(persisted_files(&artifacts), effected); + fixture.resume_same_operation(request, pending.request.operation_id); +} + #[test] #[serial_test::serial(worker_allocation)] fn profile_restore_timeout_attachment_revocation_cannot_reactivate_old_candidate() { From 85f70d4bc1916b2c35a02a3f5856796910be9464 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 06:20:22 +0900 Subject: [PATCH 05/28] test: adapt fixed Worker restore fixtures to integration lane --- Cargo.lock | 6 + crates/worker-runtime/Cargo.toml | 2 + crates/worker-runtime/README.md | 31 + crates/worker-runtime/src/fs_store.rs | 4 + crates/worker-runtime/src/lib.rs | 2 + crates/worker-runtime/src/test_support.rs | 859 +++ .../worker-runtime/src/test_support_tests.rs | 885 +++ crates/worker-runtime/src/worker_backend.rs | 9 + crates/workspace-server/Cargo.toml | 4 + crates/workspace-server/src/lib.rs | 2 + crates/workspace-server/src/store.rs | 42 +- crates/workspace-server/src/test_support.rs | 688 ++ tests/e2e/Cargo.toml | 12 + tests/e2e/README.md | 38 + .../fixtures/worker-codex-v1/expected.json | 52 + .../fixtures/worker-codex-v1/manifest.json | 34 + .../worker-codex-v1/worker-template.json | 5605 +++++++++++++++++ .../worker-codex-v1/workspace-config.dcdl | 22 + tests/e2e/src/fixture_source.rs | 336 + tests/e2e/src/lib.rs | 40 +- tests/e2e/src/worker_environment.rs | 1178 +++- tests/e2e/src/worker_session.rs | 284 + tests/e2e/tests/panel.rs | 10 +- tests/e2e/tests/worker_restore.rs | 274 + tests/e2e/tests/worker_restore/fixture.rs | 257 + tests/e2e/tests/worker_restore/seed.rs | 482 ++ tools/state-contract-name-exceptions.json | 33 + 27 files changed, 11036 insertions(+), 155 deletions(-) create mode 100644 crates/worker-runtime/src/test_support.rs create mode 100644 crates/worker-runtime/src/test_support_tests.rs create mode 100644 crates/workspace-server/src/test_support.rs create mode 100644 tests/e2e/fixtures/worker-codex-v1/expected.json create mode 100644 tests/e2e/fixtures/worker-codex-v1/manifest.json create mode 100644 tests/e2e/fixtures/worker-codex-v1/worker-template.json create mode 100644 tests/e2e/fixtures/worker-codex-v1/workspace-config.dcdl create mode 100644 tests/e2e/src/fixture_source.rs create mode 100644 tests/e2e/src/worker_session.rs create mode 100644 tests/e2e/tests/worker_restore.rs create mode 100644 tests/e2e/tests/worker_restore/fixture.rs create mode 100644 tests/e2e/tests/worker_restore/seed.rs diff --git a/Cargo.lock b/Cargo.lock index f2901cb1f..c88fc1d23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6707,15 +6707,21 @@ name = "yoi-e2e" version = "0.0.0" dependencies = [ "chrono", + "config-source", "libc", + "manifest", + "protocol", "reqwest", "serde", "serde_json", "server-api", + "session-store", "sha2 0.11.0", "tempfile", + "tungstenite 0.29.0", "webauthn-authenticator-rs", "webauthn-rs-proto", + "worker-runtime", "yoi-workspace-server", ] diff --git a/crates/worker-runtime/Cargo.toml b/crates/worker-runtime/Cargo.toml index 0eba6a589..32f6ea537 100644 --- a/crates/worker-runtime/Cargo.toml +++ b/crates/worker-runtime/Cargo.toml @@ -19,6 +19,8 @@ required-features = ["ws-server", "fs-store"] [features] default = ["ws-server", "fs-store"] fs-store = ["dep:fs4"] +# Offline fixture tooling only; never enabled by the product defaults. +test-support = ["fs-store", "ws-server"] http-server = ["dep:axum", "dep:tower", "dep:reqwest"] ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"] diff --git a/crates/worker-runtime/README.md b/crates/worker-runtime/README.md index 4984c2e18..e4f041745 100644 --- a/crates/worker-runtime/README.md +++ b/crates/worker-runtime/README.md @@ -8,3 +8,34 @@ Manages Worker creation, stopping, and Restore, execution connections within the - [`execution`] — execution backends and operation-specific results. - [`working_directory`] — repository materialization, session resources, and safe deletion. - [`workspace_issuer`] — Workspace signature verification and replay prevention. + +## Offline stopped-Worker fixtures + +The nondefault `test-support` feature exposes `test_support::StoppedWorkerTemplate`. +It is intentionally limited to current-schema, stopped, attachment-free artificial +conversations with one Segment, text history, and no pending execution/activation. +It does not add a product endpoint or change create/restore authorization. + +1. With the source Runtime offline, call `StoppedWorkerTemplate::extract(worker_dir)`. + The source must have its existing Runtime owner lock and identity. Extraction is + read-only; the returned serde template excludes source auth, saved Manifest, + request/config references, run directories, and Runtime operation history. +2. Initialize **new** Runtime identity and Workspace trust in a fresh destination + using normal setup. Preserve the template's Runtime and Workspace IDs. +3. Call `template.install(&options, request, &resolved_manifest)` before starting + the Runtime. Supply a fresh typed `CreateWorkerRequest` (including config/profile + references and trusted memory settings) and `manifest::WorkerManifest` resolved + for that environment. Its Worker name must equal `template.worker_name()`. +4. Start normally and use the ordinary explicit restore operation. Installation + itself creates no execution and leaves the Worker stopped with a fresh guard. + +The template preserves the original Session manifest/JSONL bytes, including stable +entry IDs, timestamps, and settled submission receipts. It rejects resumable queues +and Feature invocation history. A saved system prompt is required for restore; +checkpoint receipts must match the Worker's typed receipt schema. Preserved records +must use canonical writer field shapes: unknown/duplicate fields (including nested +fields), discarded optional fields, and opaque Feature history payloads are rejected, +not stripped or rewritten. Portability/secret-marker checks do not replace +review of the supplied artificial conversation or its independently verified hashes. +The installer refuses any destination content other than initialized `auth.toml`; +interrupted publication is fail-closed, not an overwrite/recovery facility. diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index 21ecfeca8..dcfb1b42c 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -39,6 +39,10 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +#[cfg(feature = "test-support")] +#[path = "test_support.rs"] +pub mod test_support; + const SCHEMA_VERSION: u32 = 12; // Frozen source schemas: do not move their meaning when adding a migration. const SOURCE_SCHEMA_VERSION: u32 = 11; diff --git a/crates/worker-runtime/src/lib.rs b/crates/worker-runtime/src/lib.rs index f4a5b02ff..bb72c71e3 100644 --- a/crates/worker-runtime/src/lib.rs +++ b/crates/worker-runtime/src/lib.rs @@ -35,6 +35,8 @@ pub mod workspace_issuer; #[cfg(feature = "http-server")] pub mod workspace_request; +#[cfg(feature = "test-support")] +pub use fs_store::test_support; #[cfg(feature = "fs-store")] pub use fs_store::{FsRuntimeStore, FsRuntimeStoreOptions}; pub use management::RuntimeOptions; diff --git a/crates/worker-runtime/src/test_support.rs b/crates/worker-runtime/src/test_support.rs new file mode 100644 index 000000000..fb0f32693 --- /dev/null +++ b/crates/worker-runtime/src/test_support.rs @@ -0,0 +1,859 @@ +//! Offline support for the approved, attachment-free stopped Worker fixture. +//! +//! Enabled only by `test-support`. This is not an import API: extraction accepts a +//! current-schema, settled, single-Segment aggregate; installation accepts only a +//! fresh Runtime directory containing its newly initialized `auth.toml`. No +//! Runtime or execution backend is constructed. Original Session bytes are never +//! rewritten. The caller must separately review/attest the artificial conversation; +//! structural checks and secret-marker checks cannot prove arbitrary prose safe. + +use super::*; +use crate::catalog::ProfileSourceArchiveSource; +use crate::workspace_issuer::{ + WorkspaceIssuerTrustRecord, WorkspaceIssuerTrustState, validate_workspace_issuer_trust_records, +}; +use session_store::{ + SegmentId, SessionId, WorkerActiveSegmentRef, WorkerMetadata, WorkerSessionStore, +}; + +#[cfg(test)] +#[path = "test_support_tests.rs"] +mod tests; + +const TEMPLATE_VERSION: u32 = 1; +const MAX_SESSION_BYTES: usize = 8 * 1024 * 1024; + +/// Portable fixture, intentionally without Debug (Session material is private). +/// Fields are private and every deserialized template is validated on install. +/// Serialize this value to retain the fixture; do not serialize its source aggregate. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StoppedWorkerTemplate { + template_version: u32, + runtime_schema_version: u32, + runtime_id: String, + workspace_id: String, + worker_id: WorkerId, + created_at_ms: Option, + session_id: SessionId, + segment_id: SegmentId, + session_manifest: Vec, + segment_jsonl: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct SessionManifest { + schema_version: u32, + session_id: SessionId, +} + +// Read-only projection: never retain source key material in a template. +#[derive(Deserialize)] +struct AuthIdentity { + identity_id: String, +} + +#[derive(Deserialize)] +struct SourceAuth { + identity: AuthIdentity, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct InitializedAuth { + identity: crate::auth::RuntimeIdentityMaterial, + workspace_issuers: Vec, + #[serde(default)] + archived_legacy_issuers: Vec, +} + +#[derive(Deserialize)] +struct GuardState { + active_request_id: Option, + owners: BTreeMap, +} + +#[derive(Deserialize)] +struct GuardOwner { + receipt: Option, +} + +fn reject(message: &'static str) -> RuntimeError { + RuntimeError::InvalidRequest(format!("stopped Worker fixture: {message}")) +} + +fn io(result: std::io::Result) -> Result { + result.map_err(|_| reject("filesystem operation failed")) +} + +fn decode(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(|_| reject("invalid typed document")) +} + +fn no_symlink_components(path: &Path) -> Result<(), RuntimeError> { + let mut current = PathBuf::new(); + for part in path.components() { + if matches!(part, std::path::Component::ParentDir) { + return Err(reject("parent traversal is unsupported")); + } + current.push(part); + let metadata = io(fs::symlink_metadata(¤t))?; + if metadata.file_type().is_symlink() { + return Err(reject("symlinks are unsupported")); + } + } + Ok(()) +} + +fn check_tree(path: &Path, budget: &mut usize) -> Result<(), RuntimeError> { + if *budget == 0 { + return Err(reject("file count exceeds fixture limit")); + } + *budget -= 1; + let metadata = io(fs::symlink_metadata(path))?; + if metadata.is_dir() { + for entry in io(fs::read_dir(path))? { + check_tree(&io(entry)?.path(), budget)?; + } + } else if !metadata.is_file() { + return Err(reject("symlinks and special files are unsupported")); + } + Ok(()) +} + +fn regular_file(path: &Path) -> Result<(), RuntimeError> { + no_symlink_components(path)?; + if !io(fs::symlink_metadata(path))?.is_file() { + return Err(reject("expected a regular file")); + } + Ok(()) +} + +fn bytes(path: &Path) -> Result, RuntimeError> { + regular_file(path)?; + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOATIME | libc::O_NOFOLLOW); + } + let file = io(options.open(path))?; + let mut result = Vec::new(); + io(file + .take(MAX_SESSION_BYTES as u64 + 1) + .read_to_end(&mut result))?; + if result.len() > MAX_SESSION_BYTES { + return Err(reject("file exceeds fixture limit")); + } + Ok(result) +} + +fn text(path: &Path) -> Result { + String::from_utf8(bytes(path)?).map_err(|_| reject("invalid UTF-8 document")) +} + +fn portable_id(id: &str) -> bool { + !id.is_empty() + && id.len() <= 256 + && id + .bytes() + .all(|c| c.is_ascii_alphanumeric() || b"-_".contains(&c)) +} + +impl StoppedWorkerTemplate { + pub fn runtime_id(&self) -> &str { + &self.runtime_id + } + pub fn workspace_id(&self) -> &str { + &self.workspace_id + } + pub fn worker_id(&self) -> WorkerId { + self.worker_id + } + pub fn worker_name(&self) -> String { + runtime_worker_name(self.worker_id) + } + pub fn session_id(&self) -> SessionId { + self.session_id + } + pub fn segment_id(&self) -> SegmentId { + self.segment_id + } + + /// Extract without migration, rewriting metadata, or copying credentials, + /// resolved manifests, execution-operation receipts, or run directories. + /// The source Runtime must be offline; its existing owner lock is held for + /// the entire read. The path must be `/workers/`. + pub fn extract(worker_dir: &Path) -> Result { + no_symlink_components(worker_dir)?; + let workers = worker_dir + .parent() + .ok_or_else(|| reject("invalid aggregate path"))?; + if workers.file_name().and_then(|s| s.to_str()) != Some(WORKERS_DIR) { + return Err(reject("invalid aggregate path")); + } + let root = workers + .parent() + .ok_or_else(|| reject("invalid Runtime path"))?; + // Require the existing lock, rather than creating a file in the source. + let lock_path = root.with_file_name(format!( + ".{}.runtime-owner.lock", + root.file_name() + .and_then(|s| s.to_str()) + .ok_or_else(|| reject("invalid Runtime path"))? + )); + regular_file(&lock_path)?; + let lock = io(OpenOptions::new().read(true).open(lock_path))?; + if !io(FileExt::try_lock_shared(&lock))? { + return Err(reject("source Runtime is not offline")); + } + let _lock = RuntimeStoreOwnerLock { file: lock }; + check_tree(worker_dir, &mut 256)?; + let snapshot: RuntimeSnapshot = decode(&bytes(&root.join(RUNTIME_FILE))?)?; + snapshot + .validate(&root.join(RUNTIME_FILE)) + .map_err(|_| reject("unsupported Runtime schema"))?; + if !snapshot.subject_create_recovery_fences.is_empty() + || !snapshot.subject_releases.is_empty() + { + return Err(reject("source has lifecycle operations")); + } + let record: WorkerAggregateRecord = decode(&bytes(&worker_dir.join(WORKER_FILE))?)?; + let record = record + .validate(&worker_dir.join(WORKER_FILE)) + .map_err(|_| reject("invalid Worker schema or identity"))?; + let guard: GuardState = decode( + &serde_json::to_vec(&record.restore_guard) + .map_err(|_| reject("invalid restore guard"))?, + )?; + if record.status != WorkerStatus::Stopped + || guard.active_request_id.is_some() + || guard.owners.values().any(|owner| owner.receipt.is_none()) + { + return Err(reject("Worker is not settled and stopped")); + } + let PersistedWorkerExecutionState::Available(execution) = &record.execution_state else { + return Err(reject("pending or unavailable execution metadata")); + }; + // A stopped aggregate may retain the empty historical binding marker. + // It is deliberately not copied to the installed Worker. + if execution.restore_intent != WorkerRestoreIntent::Explicit { + return Err(reject("Worker requires explicit restore")); + } + if !record.workdir_attachments.is_empty() || !record.logical_workdir_attachments.is_empty() + { + return Err(reject("attachments are unsupported")); + } + check_request(&execution.request, false)?; + let workspace_id = record + .workspace_id + .ok_or_else(|| reject("missing Workspace identity"))?; + let metadata: WorkerMetadata = decode(&bytes(&worker_dir.join(WORKER_METADATA_FILE))?)?; + crate::worker_backend::validate_fixture_restore_metadata(&metadata, &execution.request) + .map_err(|_| reject("invalid source restore Manifest or settings"))?; + if metadata.worker_name != runtime_worker_name(record.worker_id) + || metadata.workspace_id.as_deref() != Some(&workspace_id) + || !metadata.spawned_children.is_empty() + || !metadata.reclaimed_children.is_empty() + || !metadata.peers.is_empty() + || metadata.subjektiv_session_attribution.is_some() + { + return Err(reject("unsupported or inconsistent Worker metadata")); + } + let active = metadata + .active + .ok_or_else(|| reject("missing active Session"))?; + let segment_id = active + .segment_id + .ok_or_else(|| reject("unmaterialized Session"))?; + let source_auth: SourceAuth = toml::from_str(&text(&root.join("auth.toml"))?) + .map_err(|_| reject("missing Runtime identity"))?; + let session = worker_dir.join("session"); + let fixture = Self { + template_version: TEMPLATE_VERSION, + runtime_schema_version: SCHEMA_VERSION, + runtime_id: source_auth.identity.identity_id, + workspace_id, + worker_id: record.worker_id, + created_at_ms: record.created_at_ms, + session_id: active.session_id, + segment_id, + session_manifest: bytes(&session.join("session.json"))?, + segment_jsonl: bytes(&session.join("segments").join(format!("{segment_id}.jsonl")))?, + }; + fixture.validate_shape()?; + fixture.validate_session(&session)?; + Ok(fixture) + } + + /// Install after normal identity/trust initialization, before Runtime start. + /// `request` and `manifest` supply NEW backend/config/profile references and + /// freshly resolved policy, not a launch. The saved source Manifest is never + /// reused. `manifest.worker.name` must equal `worker_name()`. + /// Only `auth.toml` may already exist. The owner lock fences live Runtimes; + /// staged files are synced and published without overwriting any Worker. + /// An interrupted publication fails closed as an incomplete Runtime layout. + pub fn install( + &self, + options: &FsRuntimeStoreOptions, + request: CreateWorkerRequest, + manifest: &manifest::WorkerManifest, + ) -> Result<(), RuntimeError> { + self.validate_shape()?; + check_request(&request, true)?; + if manifest.worker.name != self.worker_name() { + return Err(reject("resolved Manifest Worker name mismatch")); + } + let mut metadata = WorkerMetadata::new( + self.worker_name(), + Some(WorkerActiveSegmentRef { + session_id: self.session_id, + segment_id: Some(self.segment_id), + }), + ) + .with_workspace_id(&self.workspace_id); + metadata.resolved_manifest_snapshot = Some( + manifest::write_persisted_worker_manifest_snapshot(manifest) + .map_err(|_| reject("invalid resolved Manifest"))?, + ); + crate::worker_backend::validate_fixture_restore_metadata(&metadata, &request) + .map_err(|_| reject("resolved Manifest does not match fresh request settings"))?; + if options.runtime_id != self.runtime_id + || request.worker_id != self.worker_id + || request + .workspace_api + .as_ref() + .map(|api| api.workspace_id.as_str()) + != Some(&self.workspace_id) + { + return Err(reject("installation identity mismatch")); + } + no_symlink_components(&options.root)?; + check_tree(&options.root, &mut 256)?; + // Validate the lock location before the ordinary owner-lock opener. + let lock_path = options.root.with_file_name(format!( + ".{}.runtime-owner.lock", + options + .root + .file_name() + .and_then(|s| s.to_str()) + .ok_or_else(|| reject("invalid Runtime path"))? + )); + if fs::symlink_metadata(&lock_path).is_ok() { + regular_file(&lock_path)?; + } + let lock = acquire_runtime_store_owner_lock(&options.root)?; + validate_fresh_runtime_store_layout(&options.root) + .map_err(|_| reject("destination is not fresh; refusing overwrite"))?; + let auth: InitializedAuth = toml::from_str(&text(&options.root.join("auth.toml"))?) + .map_err(|_| reject("initialize Runtime identity and trust before install"))?; + if auth.identity.identity_id != self.runtime_id + || !auth.archived_legacy_issuers.is_empty() + || auth.identity.signing_key().is_err() + { + return Err(reject("invalid destination Runtime identity")); + } + validate_workspace_issuer_trust_records(&auth.workspace_issuers) + .map_err(|_| reject("invalid destination Workspace trust"))?; + let api = request + .workspace_api + .as_ref() + .expect("validated Workspace binding"); + if !auth.workspace_issuers.iter().any(|trust| { + trust.workspace_id == self.workspace_id + && trust.backend_url == api.base_url + && trust.state == WorkspaceIssuerTrustState::Active + }) { + return Err(reject("request does not match initialized Workspace trust")); + } + let stage = options + .root + .join(format!(".fixture-{}", uuid::Uuid::now_v7())); + io(fs::create_dir(&stage))?; + let store = FsRuntimeStore { + root: stage.clone(), + _owner_lock: Some(lock), + worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), + }; + let result = (|| { + let worker = PersistedWorkerRecord { + restore_guard: crate::runtime::RestoreGuard::default(), + worker_ref: WorkerRef::new(self.worker_id), + worker_id: self.worker_id, + profile: request.profile.clone(), + display_name: request.display_name.clone(), + profile_source: request.profile_source.reference(), + config_bundle: request.config_bundle.clone(), + created_at_ms: self.created_at_ms, + status: WorkerStatus::Stopped, + last_finished_submission_request_id: None, + workspace_id: Some(self.workspace_id.clone()), + workdir_attachments: Vec::new(), + logical_workdir_attachments: Vec::new(), + execution_state: PersistedWorkerExecutionState::Available( + PersistedWorkerExecution { + request, + binding: None, + restore_intent: WorkerRestoreIntent::Explicit, + }, + ), + }; + WorkerAggregateRecord::from_persisted(&worker) + .validate(&store.worker_dir(&self.worker_id).join(WORKER_FILE)) + .map_err(|_| reject("invalid installation request"))?; + store.write_worker_record(&worker)?; + let worker_dir = store.worker_dir(&self.worker_id); + atomic_write_json( + &worker_dir.join(WORKER_METADATA_FILE), + &metadata, + "write fixture metadata", + )?; + let session = worker_dir.join("session"); + io(fs::create_dir_all(session.join("segments")))?; + write_new(&session.join("session.json"), &self.session_manifest)?; + write_new( + &session + .join("segments") + .join(format!("{}.jsonl", self.segment_id)), + &self.segment_jsonl, + )?; + self.validate_session(&session)?; + store.write_runtime_snapshot(&PersistedRuntimeState { + subject_create_recovery_fences: BTreeMap::new(), + subject_releases: Vec::new(), + // Match normal store initialization: this is catalog availability, + // not a process or Worker execution start. + display_name: options.display_name.clone(), + status: RuntimeStatus::Running, + next_diagnostic_id: 1, + workers: BTreeMap::new(), + workspace_owners: BTreeMap::new(), + diagnostics: Vec::new(), + })?; + let loaded = store.load_runtime_state()?; + if loaded.workers.len() != 1 || !loaded.diagnostics.is_empty() { + return Err(reject("staged aggregate failed ordinary store roundtrip")); + } + sync_runtime_tree(&stage)?; + // No destination existed when checked under the owner lock. Hard-link + // the snapshot last with create-new semantics; never replace a file. + // Reserve the destination catalog with create-new semantics. Even + // an empty preexisting workers directory must never be replaced. + let destination_workers = options.root.join(WORKERS_DIR); + io(fs::create_dir(&destination_workers))?; + io(fs::rename( + store.worker_dir(&self.worker_id), + destination_workers.join(self.worker_id.to_string()), + ))?; + sync_directory(&destination_workers, "publish stopped Worker aggregate")?; + io(fs::hard_link( + stage.join(RUNTIME_FILE), + options.root.join(RUNTIME_FILE), + ))?; + sync_directory(&options.root, "publish stopped fixture")?; + Ok(()) + })(); + // Only remove our private staging directory, never existing destination data. + let cleanup = fs::remove_dir_all(&stage); + result?; + io(cleanup)?; + sync_directory(&options.root, "finish stopped fixture") + } + + fn validate_shape(&self) -> Result<(), RuntimeError> { + if self.template_version != TEMPLATE_VERSION + || self.runtime_schema_version != SCHEMA_VERSION + { + return Err(reject("unsupported template schema")); + } + if !portable_id(&self.runtime_id) + || !portable_id(&self.workspace_id) + || self.session_id.get_version_num() != 7 + || self.segment_id.get_version_num() != 7 + || self.session_manifest.len() > MAX_SESSION_BYTES + || self.segment_jsonl.len() > MAX_SESSION_BYTES + { + return Err(reject("invalid portable identity or size")); + } + let manifest: SessionManifest = decode(&self.session_manifest)?; + if manifest.session_id != self.session_id + || manifest.schema_version == 0 + || !self.segment_jsonl.ends_with(b"\n") + { + return Err(reject("Session identity or committed boundary mismatch")); + } + // Read-only JSON traversal, never mutation/reconstruction of Session bytes. + // Reject known secret/host markers even in otherwise valid typed records. + for line in self + .segment_jsonl + .split(|b| *b == b'\n') + .filter(|line| !line.is_empty()) + { + let _: PortableJson = decode(line)?; + validate_preserved_record(line)?; + } + Ok(()) + } + + fn validate_history_entry( + &self, + entry: &session_store::LoggedHistoryEntry, + ids: &mut std::collections::BTreeSet, + ) -> Result<(), RuntimeError> { + if !portable_id(&entry.metadata.entry_id.0) + || !ids.insert(entry.metadata.entry_id.0.clone()) + { + return Err(reject("duplicate or invalid entry identity")); + } + if entry + .metadata + .input_metadata + .as_ref() + .is_some_and(|metadata| { + metadata.feature_payload().is_some() || metadata.public_item().is_some() + }) + { + // These are opaque owner-specific payloads, not part of the narrow + // artificial conversation contract. Never preserve unchecked JSON. + return Err(reject("Feature history payloads are unsupported")); + } + if !matches!( + entry.item, + session_store::LoggedItem::Message { .. } + | session_store::LoggedItem::AssistantResponseBoundary { .. } + ) { + return Err(reject("only text conversation records are supported")); + } + if let session_store::LoggedSessionHistoryOrigin::ModelOutput { worker } = + &entry.metadata.origin + { + // Session producer IDs identify the original execution, not the + // Workspace catalog Worker. Preserve them; do not rebind/rewrite them. + if !portable_id(&worker.worker_id) + || worker + .workspace_id + .as_ref() + .is_some_and(|id| id != &self.workspace_id) + || worker + .runtime_id + .as_ref() + .is_some_and(|id| id != &self.runtime_id) + { + return Err(reject("Session origin identity mismatch")); + } + } + Ok(()) + } + + fn validate_session(&self, root: &Path) -> Result<(), RuntimeError> { + for entry in io(fs::read_dir(root))? { + let entry = io(entry)?; + match entry.file_name().to_str() { + Some("session.json" | "segments") => {} + Some("artifacts") => check_empty_artifacts(&entry.path())?, + _ => return Err(reject("unsupported Session files")), + } + } + let entries = io(fs::read_dir(root.join("segments")))?.collect::, _>>(); + let entries = io(entries)?; + if entries.len() != 1 + || entries[0].file_name() != format!("{}.jsonl", self.segment_id).as_str() + { + return Err(reject("only the fixed single-Segment fixture is supported")); + } + let store = WorkerSessionStore::open_read_only(root) + .map_err(|_| reject("unsupported Session schema"))?; + let logs = store + .read_all_read_only_bounded(self.session_id, self.segment_id, MAX_SESSION_BYTES as u64) + .map_err(|_| reject("invalid canonical Session records"))? + .0; + if !matches!(logs.first(), Some(session_store::LogEntry::AnnotatedSegmentStart { + session_id, system_prompt: Some(_), forked_from: None, compacted_from: None, .. + }) if *session_id == self.session_id) + { + return Err(reject( + "missing system prompt, invalid Segment seed, or unsupported lineage", + )); + } + let mut ids = std::collections::BTreeSet::new(); + for (index, log) in logs.iter().enumerate() { + use session_store::LogEntry; + match log { + LogEntry::AnnotatedSegmentStart { history, .. } + if index == 0 && history.is_empty() => {} + LogEntry::AnnotatedUserInput { + history, + extensions, + .. + } => { + for extension in extensions { + validate_settled_activation(extension)?; + } + for entry in history { + self.validate_history_entry(entry, &mut ids)?; + } + } + LogEntry::AnnotatedAssistantItem { entry, .. } => { + self.validate_history_entry(entry, &mut ids)? + } + LogEntry::RunCompleted { + entry_id: Some(id), .. + } => { + if !portable_id(&id.0) || !ids.insert(id.0.clone()) { + return Err(reject("duplicate or invalid entry identity")); + } + } + LogEntry::Invoke { .. } | LogEntry::LlmUsage { .. } | LogEntry::TurnEnd { .. } => {} + _ => { + return Err(reject( + "unsupported conversation record or pending operation", + )); + } + } + } + let state = session_store::collect_state(&logs); + if state.last_run_interrupted || state.active_run_turn_count.is_some() { + return Err(reject("Session has pending execution")); + } + for segments in &state.user_segments { + if segments + .iter() + .any(|segment| !matches!(segment, protocol::Segment::Text { .. })) + { + return Err(reject("Session attachments are unsupported")); + } + } + Ok(()) + } +} + +// Compare *fields*, not re-encoded bytes or scalar values (e.g. f32 JSON +// normalization). Serde's permissive/untagged/flattened compatibility readers +// can drop unknown data without surfacing it to an ignored-field callback. A +// retained field must survive the owner's typed projection at every depth. +// This intentionally accepts only canonical writer shapes: even an explicit +// empty optional field omitted by that writer is rejected, not repaired. +// The projection is validation-only; installation always writes original bytes. +fn validate_preserved_record(bytes: &[u8]) -> Result<(), RuntimeError> { + let original: serde_json::Value = decode(bytes)?; + let record: session_store::LogEntry = decode(bytes)?; + let projection = + serde_json::to_value(&record).map_err(|_| reject("invalid typed Session record"))?; + require_preserved_fields(&original, &projection) +} + +fn require_preserved_fields( + original: &serde_json::Value, + projection: &serde_json::Value, +) -> Result<(), RuntimeError> { + use serde_json::Value; + match (original, projection) { + (Value::Object(original), Value::Object(projection)) => { + for (key, value) in original { + let projected = projection + .get(key) + .ok_or_else(|| reject("unknown or noncanonical Session field"))?; + require_preserved_fields(value, projected)?; + } + } + (Value::Array(original), Value::Array(projection)) + if original.len() == projection.len() => + { + for (value, projected) in original.iter().zip(projection) { + require_preserved_fields(value, projected)?; + } + } + (Value::Null, Value::Null) + | (Value::Bool(_), Value::Bool(_)) + | (Value::Number(_), Value::Number(_)) + | (Value::String(_), Value::String(_)) => {} + _ => return Err(reject("noncanonical Session field shape")), + } + Ok(()) +} + +// Exact receipt wire fields owned by worker::PendingActivationState. Keep these +// fixture-only mirrors strict; do not change the production compatibility reader. +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct SubmissionReceipt { + submission_request_id: String, + source_namespace: String, + submission_id: String, + payload_digest: String, + disposition: protocol::SubmissionDisposition, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct NotificationReceipt { + notification_request_id: String, + source_namespace: String, + payload_digest: String, +} + +// The approved conversation has committed submission checkpoints in its original +// Session records. Keep their receipt bytes, but reject any resumable queue or +// Feature invocation history. This projection confers no execution authority. +fn validate_settled_activation( + extension: &session_store::SessionExtension, +) -> Result<(), RuntimeError> { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct Checkpoint { + #[serde(rename = "next_activation_sequence")] + _next_activation_sequence: u64, + activating: Option, + activating_notification: Vec, + pending: Vec, + pending_notifications: Vec, + #[serde(rename = "receipts")] + _receipts: Vec, + #[serde(rename = "notification_receipts")] + _notification_receipts: Vec, + invocation_receipts: Vec, + invocation_recovery_blocked: bool, + } + if extension.domain != "worker.pending_activations.v1" { + return Err(reject("unsupported Session extension")); + } + let checkpoint: Checkpoint = serde_json::from_value(extension.payload.clone()) + .map_err(|_| reject("invalid Session activation checkpoint"))?; + if checkpoint.activating.is_some() + || !checkpoint.activating_notification.is_empty() + || !checkpoint.pending.is_empty() + || !checkpoint.pending_notifications.is_empty() + || !checkpoint.invocation_receipts.is_empty() + || checkpoint.invocation_recovery_blocked + { + return Err(reject( + "Session has pending activation or invocation history", + )); + } + Ok(()) +} + +fn check_request(request: &CreateWorkerRequest, install: bool) -> Result<(), RuntimeError> { + if !request.workdir_attachment_requests.is_empty() + || !request.workdir_attachments.is_empty() + || request.backend_job.is_some() + || request.subjektiv_attached + || request.worker_observation_enabled + || !request.worker_observation_grants.is_empty() + || (install && request.initial_input.is_some()) + { + return Err(reject( + "unsupported capabilities, attachments, or initial input", + )); + } + if install + && (request.create_fingerprint.trim().is_empty() + || request.config_bundle.is_none() + || !matches!( + request.profile_source, + ProfileSourceArchiveSource::WorkspaceConfig { .. } + )) + { + return Err(reject( + "fresh config/profile references and create fingerprint are required", + )); + } + Ok(()) +} + +fn check_empty_artifacts(root: &Path) -> Result<(), RuntimeError> { + for entry in io(fs::read_dir(root))? { + let entry = io(entry)?; + let metadata = io(entry.metadata())?; + if metadata.is_dir() { + check_empty_artifacts(&entry.path())?; + } else if entry.file_name() != ".aggregate.lock" || metadata.len() != 0 { + return Err(reject("Session artifacts are unsupported")); + } + } + Ok(()) +} + +fn write_new(path: &Path, bytes: &[u8]) -> Result<(), RuntimeError> { + let mut file = io(OpenOptions::new().write(true).create_new(true).open(path))?; + io(file.write_all(bytes))?; + io(file.sync_all()) +} + +// A streaming validation visitor, not a generic document available to callers. +struct PortableJson; +impl<'de> Deserialize<'de> for PortableJson { + fn deserialize>(deserializer: D) -> Result { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = PortableJson; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("portable fixture JSON") + } + fn visit_bool(self, _: bool) -> Result { + Ok(PortableJson) + } + fn visit_i64(self, _: i64) -> Result { + Ok(PortableJson) + } + fn visit_u64(self, _: u64) -> Result { + Ok(PortableJson) + } + fn visit_f64(self, _: f64) -> Result { + Ok(PortableJson) + } + fn visit_unit(self) -> Result { + Ok(PortableJson) + } + fn visit_str(self, value: &str) -> Result { + let lower = value.to_ascii_lowercase(); + if [ + "://", + "private key", + "bearer ", + "sk-", + "api_key", + "access_token", + "refresh_token", + "private_key", + "authorization", + ] + .iter() + .any(|marker| lower.contains(marker)) + || value + .split_whitespace() + .any(|word| word.starts_with('/') || word.contains(":\\")) + { + return Err(E::custom("nonportable or sensitive fixture material")); + } + Ok(PortableJson) + } + fn visit_seq>( + self, + mut seq: A, + ) -> Result { + while seq.next_element::()?.is_some() {} + Ok(PortableJson) + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut keys = std::collections::BTreeSet::new(); + while let Some(key) = map.next_key::()? { + // A duplicate key can hide discarded data even inside an + // otherwise opaque extension payload. Never retain it. + if !keys.insert(key.clone()) { + return Err(serde::de::Error::custom("duplicate fixture JSON field")); + } + serde::de::Visitor::visit_str::(Visitor, &key)?; + map.next_value::()?; + } + Ok(PortableJson) + } + } + deserializer.deserialize_any(Visitor) + } +} diff --git a/crates/worker-runtime/src/test_support_tests.rs b/crates/worker-runtime/src/test_support_tests.rs new file mode 100644 index 000000000..203434258 --- /dev/null +++ b/crates/worker-runtime/src/test_support_tests.rs @@ -0,0 +1,885 @@ +use super::*; +use crate::auth::RuntimeIdentityMaterial; +use crate::catalog::{ProfileSelector, WorkspaceApiRef}; +use crate::profile_archive::ProfileSourceGraphSummary; +use session_store::{LogEntry, Store}; +use sha2::{Digest, Sha256}; + +const RUNTIME: &str = "fixture-runtime"; +const WORKSPACE: &str = "fixture-workspace"; +const NEW_URL: &str = "http://127.0.0.1:34567"; + +#[derive(Serialize)] +#[serde(untagged)] +enum ReceiptFixture { + Valid(T), + Malformed(u8), +} + +#[derive(Default, Serialize)] +struct CheckpointFixture { + next_activation_sequence: u64, + activating: Option, + activating_notification: Vec, + pending: Vec, + pending_notifications: Vec, + receipts: Vec>, + notification_receipts: Vec>, + invocation_receipts: Vec, + invocation_recovery_blocked: bool, +} + +fn settled_checkpoint() -> CheckpointFixture { + CheckpointFixture { + next_activation_sequence: 2, + receipts: vec![ReceiptFixture::Valid(SubmissionReceipt { + submission_request_id: "completed-submission-request".into(), + source_namespace: "fixture-source".into(), + submission_id: "completed-submission".into(), + payload_digest: "fixture-payload-digest".into(), + disposition: protocol::SubmissionDisposition::Started, + })], + notification_receipts: vec![ReceiptFixture::Valid(NotificationReceipt { + notification_request_id: "completed-notification-request".into(), + source_namespace: "fixture-source".into(), + payload_digest: "fixture-notification-digest".into(), + })], + ..Default::default() + } +} + +fn checkpoint_extension(checkpoint: &CheckpointFixture) -> session_store::SessionExtension { + session_store::SessionExtension::new( + "worker.pending_activations.v1", + serde_json::to_value(checkpoint).unwrap(), + ) +} + +fn worker_id() -> WorkerId { + "01a1657d-b762-7791-87ae-a49f28cadbce".parse().unwrap() +} + +fn resolved() -> manifest::WorkerManifest { + let mut config = manifest::WorkerManifestConfig::resolution_defaults(); + config.worker.name = Some(runtime_worker_name(worker_id())); + manifest::WorkerManifest::try_from(config).unwrap() +} + +fn request() -> CreateWorkerRequest { + CreateWorkerRequest { + worker_id: worker_id(), + create_fingerprint: "new-create-fingerprint".into(), + profile: ProfileSelector::Builtin("default".into()), + display_name: Some("fixture".into()), + profile_source: ProfileSourceArchiveSource::WorkspaceConfig { + archive: ProfileSourceArchiveRef { + id: "new-profile".into(), + digest: "new-profile-digest".into(), + size_bytes: 1, + source_graph: ProfileSourceGraphSummary { + source_count: 1, + total_source_bytes: 1, + entrypoints: BTreeMap::from([("builtin:default".into(), "default.dcl".into())]), + import_count: 0, + }, + }, + }, + config_bundle: Some(ConfigBundleRef { + id: "new-config".into(), + digest: "new-config-digest".into(), + }), + workspace_api: Some(WorkspaceApiRef { + workspace_id: WORKSPACE.into(), + base_url: NEW_URL.into(), + }), + initial_input: None, + workdir_attachment_requests: Vec::new(), + workdir_attachments: Vec::new(), + worker_observation_enabled: false, + worker_observation_grants: Vec::new(), + memory_settings: Some(manifest::WorkspaceMemorySettingsSnapshot { + workspace_id: WORKSPACE.into(), + language: "en".into(), + }), + subjektiv_attached: false, + backend_job: None, + } +} + +fn initialize(root: &Path, runtime_id: &str, workspace_id: &str, url: &str) { + #[derive(Serialize)] + struct Auth { + identity: RuntimeIdentityMaterial, + workspace_issuers: Vec, + } + let issuer = RuntimeIdentityMaterial::generate("fixture-issuer").unwrap(); + let digest = Sha256::digest(crate::auth::decode_public_key(&issuer.public_key).unwrap()); + let fingerprint = format!( + "sha256:{}", + digest + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + ); + let trust = WorkspaceIssuerTrustRecord::from_bundle( + server_api::WorkspacePublicIdentityBundle { + workspace_id: workspace_id.into(), + backend_url: url.into(), + key_id: "fixture-issuer".into(), + algorithm: "ed25519".into(), + public_key: issuer.public_key, + public_key_fingerprint: fingerprint, + }, + "fixture-trust".into(), + 100, + ) + .unwrap(); + fs::create_dir_all(root).unwrap(); + fs::write( + root.join("auth.toml"), + toml::to_string(&Auth { + identity: RuntimeIdentityMaterial::generate(runtime_id).unwrap(), + workspace_issuers: vec![trust], + }) + .unwrap(), + ) + .unwrap(); +} + +fn record(request: CreateWorkerRequest) -> PersistedWorkerRecord { + PersistedWorkerRecord { + restore_guard: crate::runtime::RestoreGuard::default(), + worker_ref: WorkerRef::new(request.worker_id), + worker_id: request.worker_id, + profile: request.profile.clone(), + display_name: request.display_name.clone(), + profile_source: request.profile_source.reference(), + config_bundle: request.config_bundle.clone(), + created_at_ms: Some(123456), + status: WorkerStatus::Stopped, + last_finished_submission_request_id: Some("old-submission-operation".into()), + workspace_id: Some(WORKSPACE.into()), + workdir_attachments: vec![], + logical_workdir_attachments: vec![], + execution_state: PersistedWorkerExecutionState::Available(PersistedWorkerExecution { + request, + binding: Some(PersistedWorkerExecutionBinding {}), + restore_intent: WorkerRestoreIntent::Explicit, + }), + } +} + +fn source() -> (tempfile::TempDir, PathBuf, StoppedWorkerTemplate) { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("runtime"); + initialize(&root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456"); + drop( + crate::Runtime::with_fs_store(FsRuntimeStoreOptions::new(&root).with_runtime_id(RUNTIME)) + .unwrap(), + ); + let opened = FsRuntimeStore::open_or_create(root.clone(), RUNTIME).unwrap(); + let mut old = request(); + old.workspace_api.as_mut().unwrap().base_url = "http://127.0.0.1:23456".into(); + old.config_bundle.as_mut().unwrap().id = "old-config".into(); + opened.store.write_worker_record(&record(old)).unwrap(); + let worker = opened.store.worker_dir(&worker_id()); + let session_id = "01a1657d-b800-7777-8888-111111111111".parse().unwrap(); + let segment_id = "01a1657d-b818-77c1-bd73-176172a4bd82".parse().unwrap(); + let mut metadata = WorkerMetadata::new( + runtime_worker_name(worker_id()), + Some(WorkerActiveSegmentRef { + session_id, + segment_id: Some(segment_id), + }), + ) + .with_workspace_id(WORKSPACE) + .with_workspace_root(PathBuf::from("/old/host/path")); + metadata.resolved_manifest_snapshot = + Some(manifest::write_persisted_worker_manifest_snapshot(&resolved()).unwrap()); + atomic_write_json( + &worker.join(WORKER_METADATA_FILE), + &metadata, + "test metadata", + ) + .unwrap(); + let sessions = WorkerSessionStore::new(worker.join("session")).unwrap(); + let history = session_store::LoggedHistoryEntry { + item: session_store::LoggedItem::Message { + role: session_store::LoggedRole::Assistant, + content: vec![session_store::LoggedContentPart::Text { + text: "artificial fixture reply".into(), + }], + }, + metadata: session_store::LoggedSessionHistoryMetadata { + entry_id: session_store::LoggedSessionHistoryEntryId("fixed-entry-id".into()), + origin: session_store::LoggedSessionHistoryOrigin::ModelOutput { + worker: session_store::LoggedWorkerSubject { + workspace_id: Some(WORKSPACE.into()), + runtime_id: Some(RUNTIME.into()), + worker_id: worker_id().to_string(), + }, + }, + derivation: None, + input_metadata: None, + }, + }; + sessions + .create_segment( + session_id, + segment_id, + &[ + LogEntry::AnnotatedSegmentStart { + ts: 123, + session_id, + system_prompt: Some("Harmless artificial fixture instructions.".into()), + config: Default::default(), + history: vec![], + forked_from: None, + compacted_from: None, + }, + LogEntry::Invoke { + ts: 124, + trigger: protocol::InvokeKind::UserSend, + }, + LogEntry::AnnotatedUserInput { + ts: 124, + segments: vec![protocol::Segment::text( + "Harmless artificial fixture input.", + )], + extensions: vec![checkpoint_extension(&settled_checkpoint())], + history: vec![session_store::LoggedHistoryEntry { + item: session_store::LoggedItem::Message { + role: session_store::LoggedRole::User, + content: vec![session_store::LoggedContentPart::Text { + text: "Harmless artificial fixture input.".into(), + }], + }, + metadata: session_store::LoggedSessionHistoryMetadata { + entry_id: session_store::LoggedSessionHistoryEntryId( + "fixed-user-id".into(), + ), + origin: session_store::LoggedSessionHistoryOrigin::BackendInstruction { + operation_id: None, + }, + derivation: None, + input_metadata: None, + }, + }], + }, + LogEntry::AnnotatedAssistantItem { + ts: 125, + entry: history, + }, + LogEntry::LlmUsage { + ts: 125, + history_len: 2, + input_total_tokens: 1, + cache_read_tokens: 0, + cache_write_tokens: 0, + output_tokens: 1, + }, + LogEntry::TurnEnd { + ts: 126, + turn_count: 1, + }, + LogEntry::RunCompleted { + entry_id: Some(session_store::LoggedSessionHistoryEntryId( + "fixed-terminal-id".into(), + )), + ts: 126, + interrupted: false, + result: agen::EngineResult::Finished, + active_run_turn_count: None, + }, + ], + ) + .unwrap(); + fs::create_dir_all(worker.join("runs/old-run")).unwrap(); + fs::write(worker.join("runs/old-run/private.log"), "not exported").unwrap(); + drop(opened); + let template = StoppedWorkerTemplate::extract(&worker).unwrap(); + (temp, worker, template) +} + +fn destination(template: &StoppedWorkerTemplate) -> (tempfile::TempDir, FsRuntimeStoreOptions) { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("runtime"); + initialize( + &root, + template.runtime_id(), + template.workspace_id(), + NEW_URL, + ); + ( + temp, + FsRuntimeStoreOptions::new(root).with_runtime_id(template.runtime_id()), + ) +} + +fn assert_fresh(options: &FsRuntimeStoreOptions) { + assert_eq!(fs::read_dir(&options.root).unwrap().count(), 1); + assert!(options.root.join("auth.toml").is_file()); +} + +#[test] +fn extraction_install_roundtrip_preserves_bytes_and_is_inactive() { + let (_source, worker, template) = source(); + let original_worker = fs::read(worker.join(WORKER_FILE)).unwrap(); + let original_metadata = fs::read(worker.join(WORKER_METADATA_FILE)).unwrap(); + let exported = serde_json::to_vec(&template).unwrap(); + let exported_text = String::from_utf8(exported.clone()).unwrap(); + for forbidden in [ + "old-config", + "old-submission-operation", + "restore_guard", + "private_key", + "http://", + "/old/host", + "private.log", + "resolved_manifest_snapshot", + ] { + assert!(!exported_text.contains(forbidden)); + } + let template: StoppedWorkerTemplate = serde_json::from_slice(&exported).unwrap(); + let (_target, options) = destination(&template); + let auth_before = fs::read(options.root.join("auth.toml")).unwrap(); + template.install(&options, request(), &resolved()).unwrap(); + assert_eq!( + fs::read(options.root.join("auth.toml")).unwrap(), + auth_before + ); + assert_eq!(fs::read(worker.join(WORKER_FILE)).unwrap(), original_worker); + assert_eq!( + fs::read(worker.join(WORKER_METADATA_FILE)).unwrap(), + original_metadata + ); + let installed = options + .root + .join(WORKERS_DIR) + .join(template.worker_id().to_string()); + assert_eq!( + fs::read(installed.join("session/session.json")).unwrap(), + template.session_manifest + ); + assert_eq!( + fs::read(installed.join(format!("session/segments/{}.jsonl", template.segment_id()))) + .unwrap(), + template.segment_jsonl + ); + assert!(!installed.join("runs").exists()); + let metadata: WorkerMetadata = + decode(&fs::read(installed.join(WORKER_METADATA_FILE)).unwrap()).unwrap(); + assert_eq!(metadata.active.unwrap().session_id, template.session_id()); + assert!(metadata.workspace_root.is_none() && metadata.resolved_manifest_snapshot.is_some()); + + let opened = FsRuntimeStore::open_or_create(options.root.clone(), RUNTIME).unwrap(); + let state = opened.state.unwrap(); + assert!(state.diagnostics.is_empty()); + let saved = state.workers.get(&worker_id()).unwrap(); + assert_eq!(saved.status, WorkerStatus::Stopped); + assert!(saved.last_finished_submission_request_id.is_none()); + let PersistedWorkerExecutionState::Available(execution) = &saved.execution_state else { + panic!("not available") + }; + assert_eq!(execution.restore_intent, WorkerRestoreIntent::Explicit); + assert!(execution.binding.is_none()); + assert_eq!(execution.request, request()); + // Fresh restore guard, no operation owner/receipt history. + assert_ne!( + serde_json::to_vec(&saved.restore_guard).unwrap(), + serde_json::to_vec( + &decode::(&original_worker) + .unwrap() + .restore_guard + ) + .unwrap() + ); + let guard: GuardState = decode(&serde_json::to_vec(&saved.restore_guard).unwrap()).unwrap(); + assert!(guard.active_request_id.is_none() && guard.owners.is_empty()); + drop(opened.store); + + struct NoExecution; + impl crate::execution::WorkerExecutionBackend for NoExecution { + fn backend_id(&self) -> &str { + "fixture-no-execution" + } + fn spawn_worker( + &self, + _: crate::execution::WorkerExecutionSpawnRequest, + ) -> crate::execution::WorkerExecutionSpawnResult { + panic!("fixture executed") + } + fn dispatch_input( + &self, + _: &WorkerRef, + _: crate::interaction::WorkerInput, + ) -> crate::execution::WorkerExecutionResult { + panic!("fixture submitted") + } + fn restore_worker( + &self, + _: crate::execution::WorkerExecutionRestoreRequest, + ) -> crate::execution::WorkerExecutionSpawnResult { + panic!("fixture auto-restored") + } + } + let runtime = + crate::Runtime::with_fs_store_and_execution_backend(options.clone(), Arc::new(NoExecution)) + .unwrap(); + assert_eq!(runtime.list_workers().unwrap().len(), 1); + assert_eq!(runtime.summary().unwrap().active_worker_count, 0); + assert_eq!( + runtime + .worker_detail(&WorkerRef::new(worker_id())) + .unwrap() + .status, + WorkerStatus::Stopped + ); + drop(runtime); + let runtime = crate::Runtime::with_fs_store(options).unwrap(); + assert_eq!( + runtime.list_workers().unwrap()[0].status, + WorkerStatus::Stopped + ); +} + +#[test] +fn install_rejects_schema_identity_input_and_overwrite() { + let (_source, _, template) = source(); + for change in [ + "template", + "runtime-schema", + "session-schema", + "session-id", + "segment-id", + "runtime-id", + "workspace-id", + "worker-id", + "origin-id", + "duplicate-id", + "truncated", + ] { + let (_target, options) = destination(&template); + let mut invalid = template.clone(); + match change { + "template" => invalid.template_version += 1, + "runtime-schema" => invalid.runtime_schema_version += 1, + "session-schema" => { + invalid.session_manifest = format!( + "{{\"schema_version\":999,\"session_id\":\"{}\"}}", + invalid.session_id + ) + .into_bytes() + } + "session-id" => invalid.session_id = uuid::Uuid::now_v7(), + "segment-id" => invalid.segment_id = uuid::Uuid::nil(), // no external attestation: valid new Segment ID is otherwise portable + "runtime-id" => invalid.runtime_id = "other-runtime".into(), + "workspace-id" => invalid.workspace_id = "other-workspace".into(), + "worker-id" => invalid.worker_id = WorkerId::now_v7(), + "origin-id" => { + invalid.segment_jsonl = String::from_utf8(invalid.segment_jsonl) + .unwrap() + .replace(RUNTIME, "other-runtime") + .into_bytes() + } + "duplicate-id" => { + let last = invalid + .segment_jsonl + .split_inclusive(|b| *b == b'\n') + .last() + .unwrap() + .to_vec(); + invalid.segment_jsonl.extend(last); + } + "truncated" => { + invalid.segment_jsonl.pop(); + } + _ => unreachable!(), + } + assert!( + invalid.install(&options, request(), &resolved()).is_err(), + "accepted {change}" + ); + assert_fresh(&options); + } + let (_target, options) = destination(&template); + let mut launch = request(); + launch.initial_input = Some(crate::interaction::WorkerInput::user("do not execute")); + assert!(template.install(&options, launch, &resolved()).is_err()); + assert_fresh(&options); + template.install(&options, request(), &resolved()).unwrap(); + let worker_path = options + .root + .join(WORKERS_DIR) + .join(worker_id().to_string()) + .join(WORKER_FILE); + let before = fs::read(&worker_path).unwrap(); + assert!(template.install(&options, request(), &resolved()).is_err()); + assert_eq!(fs::read(worker_path).unwrap(), before); +} + +#[test] +fn rejects_uninitialized_untrusted_live_and_partial_destination() { + let (_source, _, template) = source(); + let (_target, options) = destination(&template); + let held = acquire_runtime_store_owner_lock(&options.root).unwrap(); + assert!(template.install(&options, request(), &resolved()).is_err()); + assert_fresh(&options); + drop(held); + initialize(&options.root, "wrong-runtime", WORKSPACE, NEW_URL); + assert!(template.install(&options, request(), &resolved()).is_err()); + initialize(&options.root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456"); + assert!(template.install(&options, request(), &resolved()).is_err()); + fs::remove_file(options.root.join("auth.toml")).unwrap(); + assert!(template.install(&options, request(), &resolved()).is_err()); + initialize(&options.root, RUNTIME, WORKSPACE, NEW_URL); + fs::create_dir(options.root.join(WORKERS_DIR)).unwrap(); + assert!(template.install(&options, request(), &resolved()).is_err()); +} + +#[test] +fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() { + for change in [ + "running", + "pending", + "guard-pending", + "attachment", + "schema", + "worker-id", + "metadata-workspace", + "metadata-session", + "artifact", + "secret", + "url", + "path", + ] { + let (_source, worker, _) = source(); + let mut aggregate: WorkerAggregateRecord = + decode(&fs::read(worker.join(WORKER_FILE)).unwrap()).unwrap(); + match change { + "running" => aggregate.status = WorkerStatus::Running, + "pending" => { + aggregate.execution_state = WorkerExecutionStateRecord::ReconciliationRequired( + PersistedWorkerLifecycleOperation::Restore(PersistedWorkerRestoreOperation { + operation_id: WorkerLifecycleOperationId::new(), + mode: PersistedWorkerRestoreMode::Explicit, + request: request(), + binding: None, + last_settled_status: WorkerStatus::Stopped, + }), + ) + } + "guard-pending" => aggregate.restore_guard = serde_json::from_str(r#"{"observation_token":"old-token","active_request_id":"pending-restore","owners":{}}"#).unwrap(), + "attachment" => aggregate.logical_workdir_attachments.push(LogicalWorkdirAttachment { + alias: workdir::WorkdirAttachmentAlias::new("source").unwrap(), + working_directory_id: "old-workdir".into(), + capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY, + }), + "schema" => aggregate.schema_version += 1, + "worker-id" => aggregate.worker_id = WorkerId::now_v7(), + "metadata-workspace" | "metadata-session" => { + let mut metadata: WorkerMetadata = + decode(&fs::read(worker.join(WORKER_METADATA_FILE)).unwrap()).unwrap(); + if change == "metadata-workspace" { + metadata.workspace_id = Some("wrong-workspace".into()); + } else { + metadata.active.as_mut().unwrap().session_id = uuid::Uuid::now_v7(); + } + atomic_write_json( + &worker.join(WORKER_METADATA_FILE), + &metadata, + "test mismatch", + ) + .unwrap(); + } + "artifact" => { + fs::create_dir_all(worker.join("session/artifacts/paste")).unwrap(); + fs::write( + worker.join("session/artifacts/paste/payload"), + "unsupported", + ) + .unwrap(); + } + "secret" | "url" | "path" => { + let segment = fs::read_dir(worker.join("session/segments")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let replacement = match change { + "secret" => "Bearer secret", + "url" => "https://old-host.invalid", + _ => "/old/host/path", + }; + let text = fs::read_to_string(&segment) + .unwrap() + .replace("artificial fixture reply", replacement); + fs::write(segment, text).unwrap(); + } + _ => unreachable!(), + } + atomic_write_json(&worker.join(WORKER_FILE), &aggregate, "test aggregate").unwrap(); + assert!( + StoppedWorkerTemplate::extract(&worker).is_err(), + "accepted {change}" + ); + } + let (_source, worker, _) = source(); + let held = + acquire_runtime_store_owner_lock(worker.parent().unwrap().parent().unwrap()).unwrap(); + assert!(StoppedWorkerTemplate::extract(&worker).is_err()); + drop(held); +} + +#[test] +fn fresh_manifest_and_memory_settings_are_required() { + let (_source, _, template) = source(); + let (_target, options) = destination(&template); + let mut manifest = resolved(); + manifest.worker.name = "wrong-worker".into(); + assert!(template.install(&options, request(), &manifest).is_err()); + let mut binding = request(); + binding.memory_settings = None; + assert!(template.install(&options, binding, &resolved()).is_err()); + let mut binding = request(); + binding.memory_settings.as_mut().unwrap().workspace_id = "other-workspace".into(); + assert!(template.install(&options, binding, &resolved()).is_err()); + assert_fresh(&options); +} + +#[test] +fn settled_checkpoints_preserve_receipts_but_refuse_pending_work() { + let mut checkpoint = settled_checkpoint(); + assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_ok()); + checkpoint.pending.push(1); + assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err()); + checkpoint.pending.clear(); + checkpoint.activating = Some(1); + assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err()); + checkpoint.activating = None; + checkpoint.invocation_receipts.push(1); + assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err()); + checkpoint.invocation_receipts.clear(); + checkpoint.invocation_recovery_blocked = true; + assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err()); +} + +// Deliberately corrupt only test-owned synthetic source files, never the lab. +fn assert_rejected_session_bytes(worker: &Path, template: &StoppedWorkerTemplate, bytes: Vec) { + let path = worker.join(format!("session/segments/{}.jsonl", template.segment_id())); + fs::write(path, &bytes).unwrap(); + let extraction_error = StoppedWorkerTemplate::extract(worker) + .err() + .expect("invalid source accepted"); + assert!( + !extraction_error + .to_string() + .contains("synthetic-discarded-material") + ); + let (_target, options) = destination(template); + let mut invalid = template.clone(); + invalid.segment_jsonl = bytes; + let installation_error = invalid + .install(&options, request(), &resolved()) + .unwrap_err(); + assert!( + !installation_error + .to_string() + .contains("synthetic-discarded-material") + ); + assert_fresh(&options); +} + +fn synthetic_logs(template: &StoppedWorkerTemplate) -> Vec { + template + .segment_jsonl + .split(|byte| *byte == b'\n') + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_slice(line).unwrap()) + .collect() +} + +fn encode_synthetic_logs(logs: &[LogEntry]) -> Vec { + let mut bytes = Vec::new(); + for log in logs { + serde_json::to_writer(&mut bytes, log).unwrap(); + bytes.push(b'\n'); + } + bytes +} + +#[tokio::test] +async fn missing_system_prompt_matches_worker_restore_rejection() { + let (_source, worker_dir, template) = source(); + let mut logs = synthetic_logs(&template); + assert!(session_store::collect_state(&logs).system_prompt.is_some()); + let LogEntry::AnnotatedSegmentStart { system_prompt, .. } = &mut logs[0] else { + panic!("missing seed") + }; + *system_prompt = None; + assert_rejected_session_bytes(&worker_dir, &template, encode_synthetic_logs(&logs)); + // The owning Worker constructor rejects this before model-client creation, + // allocation, or execution. Do not start a controller or submit any input. + let store = session_store::CombinedStore::new( + WorkerSessionStore::open_read_only(worker_dir.join("session")).unwrap(), + session_store::WorkerAggregateStore::open_read_only(&worker_dir, template.worker_name()) + .unwrap(), + ); + let restored = worker::Worker::restore_from_manifest( + template.session_id(), + template.segment_id(), + resolved(), + store, + worker::PromptCatalogSource::builtins_only(), + ) + .await; + assert!(matches!( + restored, + Err(worker::WorkerError::SegmentSystemPromptMissing { .. }) + )); +} + +#[test] +fn malformed_submission_and_notification_receipts_fail_extraction_and_install() { + let (_source, worker, template) = source(); + for notification in [false, true] { + let mut checkpoint = settled_checkpoint(); + if notification { + checkpoint.notification_receipts = + vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)]; + } else { + checkpoint.receipts = vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)]; + } + let mut logs = synthetic_logs(&template); + for log in &mut logs { + if let LogEntry::AnnotatedUserInput { extensions, .. } = log { + *extensions = vec![checkpoint_extension(&checkpoint)]; + } + } + assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs)); + } +} + +#[test] +fn unknown_fields_at_every_retained_record_depth_are_rejected_without_secret_diagnostics() { + let (_source, worker, template) = source(); + let original = String::from_utf8(template.segment_jsonl.clone()).unwrap(); + // These values are harmless synthetic markers, not lab/private material. + // The compatibility decoder normally discards many of these fields. + let locations = [ + r#""kind":"annotated_segment_start""#, + r#""kind":"invoke""#, + r#""kind":"annotated_user_input""#, + r#""kind":"annotated_assistant_item""#, + r#""kind":"llm_usage""#, + r#""kind":"turn_end""#, + r#""kind":"run_completed""#, + r#""max_tokens":null"#, + r#""kind":"message","role":"assistant""#, + r#""kind":"text","text":"artificial fixture reply""#, + r#""kind":"text","content":"Harmless artificial fixture input.""#, + r#""entry_id":"fixed-entry-id""#, + r#""kind":"model_output""#, + r#""worker_id":"01a1657d-b762-7791-87ae-a49f28cadbce""#, + r#""domain":"worker.pending_activations.v1""#, + r#""next_activation_sequence":2"#, + r#""submission_request_id":"completed-submission-request""#, + r#""notification_request_id":"completed-notification-request""#, + ]; + for location in locations { + assert!(original.contains(location)); + for extra in [ + r#", "reasoning":{"text":"synthetic-discarded-material"}"#, + r#", "encrypted_content":"synthetic-discarded-material""#, + ] { + let altered = original.replacen(location, &format!("{location}{extra}"), 1); + assert_rejected_session_bytes(&worker, &template, altered.into_bytes()); + } + } + let duplicate = original.replacen( + r#""role":"assistant""#, + r#""role":"assistant","role":"assistant""#, + 1, + ); + assert_rejected_session_bytes(&worker, &template, duplicate.into_bytes()); +} + +#[test] +fn opaque_feature_payloads_cannot_bypass_strict_session_validation() { + let (_source, worker, template) = source(); + let mut logs = synthetic_logs(&template); + for log in &mut logs { + if let LogEntry::AnnotatedUserInput { history, .. } = log { + if let session_store::LoggedItem::Message { role, .. } = &mut history[0].item { + *role = session_store::LoggedRole::System; + } + history[0].metadata.input_metadata = Some( + session_store::HistoryInputMetadata::new( + session_store::HistoryInputUsePolicy { + recording_material: session_store::InputEvidenceUse::Exclude, + memory_evidence: session_store::InputEvidenceUse::Exclude, + observation: session_store::InputObservationUse::Hidden, + compaction: session_store::InputCompactionUse::Retain, + }, + Some( + session_store::FeatureInputPayload::encode( + "fixture.test", + 1, + &BTreeMap::from([("reasoning", "harmless synthetic payload")]), + ) + .unwrap(), + ), + None, + ) + .unwrap(), + ); + } + } + assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs)); +} + +#[cfg(unix)] +#[test] +fn refuses_source_and_destination_symlinks() { + let (_source, worker, template) = source(); + let (_target, options) = destination(&template); + std::os::unix::fs::symlink(worker.join(WORKER_FILE), options.root.join("linked")).unwrap(); + assert!(template.install(&options, request(), &resolved()).is_err()); + fs::remove_file(options.root.join("linked")).unwrap(); + std::os::unix::fs::symlink(worker.join(WORKER_FILE), worker.join("runs/link")).unwrap(); + assert!(StoppedWorkerTemplate::extract(&worker).is_err()); +} + +/// Opt-in only: source path supplied explicitly; assertions never display content. +#[test] +#[ignore = "requires the approved artificial lab aggregate via YOI_STOPPED_FIXTURE_SOURCE"] +fn approved_lab_extract_install() { + let path = PathBuf::from( + std::env::var_os("YOI_STOPPED_FIXTURE_SOURCE").expect("explicit source required"), + ); + let template = StoppedWorkerTemplate::extract(&path).unwrap(); + let (_target, options) = destination(&template); + let mut bind = request(); + bind.worker_id = template.worker_id(); + bind.workspace_api.as_mut().unwrap().workspace_id = template.workspace_id().into(); + bind.memory_settings.as_mut().unwrap().workspace_id = template.workspace_id().into(); + let mut manifest = resolved(); + manifest.worker.name = template.worker_name(); + template.install(&options, bind, &manifest).unwrap(); + let installed = options + .root + .join(WORKERS_DIR) + .join(template.worker_id().to_string()); + for relative in [ + "session/session.json".to_string(), + format!("session/segments/{}.jsonl", template.segment_id()), + ] { + assert!( + fs::read(path.join(&relative)).unwrap() == fs::read(installed.join(relative)).unwrap() + ); + } + let runtime = crate::Runtime::with_fs_store(options).unwrap(); + assert_eq!(runtime.list_workers().unwrap().len(), 1); + assert_eq!(runtime.summary().unwrap().active_worker_count, 0); +} diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 695a03379..87c0f676c 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -1682,6 +1682,15 @@ fn bind_workspace_memory_settings( Ok(()) } +/// Read-only validation shared with the offline, feature-gated fixture installer. +#[cfg(feature = "test-support")] +pub(crate) fn validate_fixture_restore_metadata( + metadata: &session_store::WorkerMetadata, + request: &CreateWorkerRequest, +) -> Result<(), String> { + ProfileRuntimeWorkerFactory::manifest_for_restore(metadata, request).map(|_| ()) +} + fn validate_worker_memory_settings( manifest: &manifest::WorkerManifest, request: &CreateWorkerRequest, diff --git a/crates/workspace-server/Cargo.toml b/crates/workspace-server/Cargo.toml index b561ca4eb..0879f2b07 100644 --- a/crates/workspace-server/Cargo.toml +++ b/crates/workspace-server/Cargo.toml @@ -10,6 +10,10 @@ autobins = false name = "yoi-server" path = "src/main.rs" +[features] +# Offline fixture preparation only; build product binaries without this feature. +test-support = [] + [dependencies] api-macros = { workspace = true, features = ["axum"] } async-trait.workspace = true diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index dd117ae7b..c4a95e8c0 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -28,6 +28,8 @@ pub mod runtime_subscription; pub mod server; pub mod skills; pub mod store; +#[cfg(feature = "test-support")] +pub use store::fixture_support as test_support; pub mod subjektiv; #[cfg(test)] mod ticket_browse_performance_tests; diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index d742f0ec6..f12c54334 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -2589,6 +2589,10 @@ pub struct SqliteWorkspaceStore { db_executor: crate::db_executor::DbExecutor, } +#[cfg(feature = "test-support")] +#[path = "test_support.rs"] +pub mod fixture_support; + impl SqliteWorkspaceStore { pub fn open(path: impl AsRef) -> Result { let conn = Connection::open(path)?; @@ -2900,6 +2904,33 @@ impl SqliteWorkspaceStore { request_fingerprint: &str, singleton_key: Option<&str>, current_memory_settings: &WorkspaceMemorySettingsRecord, + ) -> Result { + self.reserve_worker_create_with( + workspace_id, + runtime_id, + allocation_key, + request_fingerprint, + singleton_key, + current_memory_settings, + |_| Ok(None), + |_, _| Ok(()), + ) + } + + // Keep reservation validation, Memory binding and writes in one transaction. The + // nondefault test-support module supplies fixed identity / offline catalog hooks; + // ordinary creation always supplies a newly generated ID and no completion hook. + #[allow(clippy::too_many_arguments)] + fn reserve_worker_create_with( + &self, + workspace_id: &str, + runtime_id: &str, + allocation_key: &str, + request_fingerprint: &str, + singleton_key: Option<&str>, + current_memory_settings: &WorkspaceMemorySettingsRecord, + prepare: impl FnOnce(&rusqlite::Transaction<'_>) -> Result>, + complete: impl FnOnce(&rusqlite::Transaction<'_>, &WorkerCreateReservation) -> Result<()>, ) -> Result { if allocation_key.trim().is_empty() || request_fingerprint.trim().is_empty() { return Err(Error::InvalidInput( @@ -2912,6 +2943,7 @@ impl SqliteWorkspaceStore { validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?; self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let fixed_worker_id = prepare(&tx)?; let existing = tx .query_row( "SELECT worker_id, runtime_id, request_fingerprint, create_fingerprint, \ @@ -3094,7 +3126,7 @@ impl SqliteWorkspaceStore { validate_workspace_memory_settings_snapshot(&snapshot, workspace_id)?; let create_fingerprint = bound_worker_create_fingerprint(request_fingerprint, &snapshot); - let worker_id = WorkerId::now_v7(); + let worker_id = fixed_worker_id.unwrap_or_else(WorkerId::now_v7); let worker_id_text = worker_id.to_string(); let now = chrono::Utc::now().to_rfc3339(); let singleton_generation = match singleton_owner.as_ref() { @@ -3183,13 +3215,15 @@ impl SqliteWorkspaceStore { )); } } - tx.commit()?; - Ok(WorkerCreateReservation { + let reservation = WorkerCreateReservation { worker_id, create_fingerprint, memory_settings: snapshot, singleton, - }) + }; + complete(&tx, &reservation)?; + tx.commit()?; + Ok(reservation) }) } diff --git a/crates/workspace-server/src/test_support.rs b/crates/workspace-server/src/test_support.rs new file mode 100644 index 000000000..fcdf63087 --- /dev/null +++ b/crates/workspace-server/src/test_support.rs @@ -0,0 +1,688 @@ +//! Offline Backend fixture preparation, available only with `test-support`. +//! +//! Use a new isolated database and the existing account, Workspace signing-identity / +//! `ControlPlaneStore::create_workspace_bootstrap`, and Runtime binding APIs first. +//! Generate fresh account credentials and signing keys for each environment; no fixture +//! credentials are accepted here. Install Profile configuration through the existing +//! config-source mechanisms before starting the product processes. +//! +//! This module seeds Backend ownership only. It neither writes Runtime storage nor +//! launches a Worker, fabricates Runtime observations, changes authorization, or sends +//! initial input. The caller must separately install the matching stopped Runtime +//! fixture. Build product binaries in a separate Cargo invocation **without** this +//! feature (Cargo features unify within an invocation). + +use super::*; +use crate::hosts::{WorkerSpawnIntent, WorkerSpawnRequest, worker_spawn_create_fingerprint}; +use worker_runtime::catalog::ProfileSelector; + +/// One ordinary, attachment-free Worker from an externally validated fixture. +/// +/// IDs must match the Runtime fixture. `request` is the unresolved canonical create +/// intent, not a Runtime launch request containing credentials or execution grants. +/// The Profile selector must already be installed by the harness. `initial_submit` +/// must be empty: saved Session contents belong to the Runtime fixture, not new input. +/// Singleton/service, Ticket-assigned and attachment-bearing Workers are deliberately +/// unsupported. +pub struct FixedWorkerSeed<'a> { + pub workspace_id: &'a str, + pub runtime_id: &'a str, + pub worker_id: WorkerId, + pub allocation_key: &'a str, + pub request: &'a WorkerSpawnRequest, +} + +/// Insert a fixed Worker catalog entry and completed creation reservation atomically. +/// +/// Uses the normal reservation validators, authoritative Memory snapshot, fingerprint +/// binding, resource-key allocator and immediate transaction. Even an exact repeat is +/// rejected: this is not an upsert/import/recovery API. Existing allocations, Worker +/// identities and residual lifecycle/attachment records are never adopted or repaired. +/// Returns the creation binding needed to check the corresponding Runtime fixture. +pub fn seed_fixed_worker( + store: &SqliteWorkspaceStore, + seed: FixedWorkerSeed<'_>, +) -> Result { + validate_identifier("workspace_id", seed.workspace_id)?; + validate_identifier("runtime_id", seed.runtime_id)?; + let request = seed.request; + if !request.initial_submit.is_empty() { + return Err(Error::InvalidInput( + "fixture Worker cannot contain initial_submit; offline seeding accepts no new input" + .into(), + )); + } + if !matches!(request.intent, WorkerSpawnIntent::WorkspaceCoding) + || request.singleton_key.is_some() + || request.ticket_assignment.is_some() + || request.resolved_subjektiv_attached + || !request.workdir_attachment_requests.is_empty() + || !request.resolved_workdir_attachment_requests.is_empty() + || !request.resolved_workdir_attachments.is_empty() + || request.resolved_config_bundle.is_some() + || request.resolved_workspace_api.is_some() + || request.resolved_memory_settings.is_some() + || request.resolved_worker_observation_enabled + || !request.resolved_worker_observation_grants.is_empty() + || request.resolved_control_operation.is_some() + || request.resolved_execution_grant.is_some() + { + return Err(Error::InvalidInput( + "fixture Worker must be ordinary unresolved create intent without singleton, assignments, attachments or grants".into(), + )); + } + // Builtin selectors already carry their canonical `builtin:` prefix. + let profile = match &request.profile { + ProfileSelector::Builtin(name) | ProfileSelector::Named(name) => name, + }; + validate_non_empty("fixture Profile selector", profile)?; + let worker_id = seed.worker_id.to_string(); + let display_name = request + .requested_worker_name + .as_deref() + .unwrap_or(&worker_id); + validate_non_empty("fixture Worker display name", display_name)?; + let fingerprint = worker_spawn_create_fingerprint(request).map_err(Error::InvalidInput)?; + let memory = store.get_workspace_memory_settings(seed.workspace_id)?; + store.reserve_worker_create_with( + seed.workspace_id, + seed.runtime_id, + seed.allocation_key, + &fingerprint, + None, + &memory, + |tx| { + require_seed_target(tx, &seed)?; + Ok(Some(seed.worker_id)) + }, + |tx, reservation| { + let now = chrono::Utc::now().to_rfc3339(); + tx.execute( + "INSERT INTO worker_registry ( + workspace_id, runtime_id, worker_id, display_name, profile, + retention_state, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, 'normal', ?6, ?6)", + params![seed.workspace_id, seed.runtime_id, worker_id, display_name, profile, now], + )?; + let changed = tx.execute( + "UPDATE worker_create_reservations SET state = 'created', updated_at = ?4 + WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 AND state = 'reserved'", + params![seed.workspace_id, seed.runtime_id, reservation.worker_id.to_string(), now], + )?; + if changed != 1 { + return Err(Error::Store("fixture creation reservation disappeared".into())); + } + Ok(()) + }, + ) +} + +fn require_seed_target(tx: &rusqlite::Transaction<'_>, seed: &FixedWorkerSeed<'_>) -> Result<()> { + let active: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM workspaces w + JOIN workspace_runtime_bindings r ON r.workspace_id = w.workspace_id + JOIN workspace_signing_identities s ON s.workspace_id = w.workspace_id + WHERE w.workspace_id = ?1 AND w.state = 'active' + AND r.runtime_id = ?2 AND r.revoked_at IS NULL + AND r.authentication_mode = 'workspace_identity' AND s.state = 'active')", + params![seed.workspace_id, seed.runtime_id], + |row| row.get(0), + )?; + if !active { + return Err(Error::InvalidInput( + "fixture requires an active bootstrapped Workspace and authenticated Runtime binding" + .into(), + )); + } + let worker_id = seed.worker_id.to_string(); + let allocation_exists: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM worker_create_reservations + WHERE workspace_id = ?1 AND allocation_key = ?2)", + params![seed.workspace_id, seed.allocation_key], + |row| row.get(0), + )?; + if allocation_exists { + return Err(Error::InvalidInput( + "fixture Worker allocation already exists".into(), + )); + } + // Check by WorkerId, not just the proposed Runtime tuple: orphaned and terminal + // ownership is still a collision. These tables are owned by this store schema. + for table in [ + "worker_create_reservations", + "worker_registry", + "worker_registry_observations", + "worker_registry_projection_removals", + "worker_removal_operations", + "worker_singleton_owners", + "worker_workdir_links", + "worker_restore_intents", + "worker_tombstones", + "worker_orphan_diagnostics", + "worker_registry_projection_diagnostics", + "worker_session_archives", + "worker_diagnostics_archives", + "ticket_assignment_worker_tombstones", + ] { + let exists: bool = tx.query_row( + &format!("SELECT EXISTS(SELECT 1 FROM {table} WHERE worker_id = ?1)"), + [&worker_id], + |row| row.get(0), + )?; + if exists { + return Err(Error::InvalidInput(format!( + "fixture Worker identity already exists in {table}" + ))); + } + } + // A resource key without a catalog/reservation must not be silently reused. + let mut statement = tx.prepare( + "SELECT resource_id FROM workspace_resource_keys WHERE resource_kind = 'worker'", + )?; + let resources = statement.query_map([], |row| row.get::<_, String>(0))?; + for resource in resources { + let resource = resource?; + let existing = resource + .is_ascii() + .then(|| parse_worker_resource_id(&resource)) + .flatten() + .ok_or_else(|| { + Error::InvalidInput("unknown fixture Worker resource identity collision".into()) + })?; + if existing.worker_id == worker_id { + return Err(Error::InvalidInput( + "fixture Worker resource identity already exists".into(), + )); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::hosts::{ + WorkerSpawnAcceptanceRequirement, WorkerSpawnWorkingDirectoryAttachmentRequest, + WorkerSpawnWorkingDirectoryRequest, + }; + use crate::workspace_signing_identity::{ + InMemoryWorkspaceSigningMaterialStore, WorkspaceSigningIdentityService, + }; + + const WORKSPACE: &str = "0197a949-4b6b-7f2a-9d9a-1f87e3a4c501"; + const RUNTIME: &str = "fixture-runtime"; + const WORKER: &str = "0197a949-4b6b-7f2a-9d9a-1f87e3a4c502"; + + // Exercises the same public bootstrap APIs the offline harness uses. All account + // and key identities are newly generated, even across stores with fixed fixture IDs. + fn bootstrap() -> SqliteWorkspaceStore { + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let account = AccountRecord { + account_id: format!("fixture-account-{}", Uuid::now_v7()), + kind: "user".into(), + handle: format!("fixture-{}", Uuid::now_v7()), + display_name: "Fixture owner".into(), + created_at: "2026-10-24T00:00:00Z".into(), + updated_at: "2026-10-24T00:00:00Z".into(), + }; + store.upsert_account(&account).unwrap(); + let identities = WorkspaceSigningIdentityService::new( + Arc::new(store.clone()), + Arc::new(InMemoryWorkspaceSigningMaterialStore::default()), + ); + let (identity, operation) = identities + .prepare_workspace_creation( + "fixture-bootstrap", + "fixture-bootstrap", + WORKSPACE, + &account.account_id, + ) + .unwrap(); + store + .create_workspace_bootstrap( + &WorkspaceBootstrapRecord { + operation_key: "fixture-bootstrap".into(), + request_fingerprint: "fixture-bootstrap".into(), + workspace: WorkspaceRecord { + workspace_id: WORKSPACE.into(), + owner_account_id: account.account_id, + display_name: "Fixture".into(), + state: "active".into(), + created_at: account.created_at, + updated_at: account.updated_at, + }, + repository: None, + }, + &identity, + &operation, + ) + .unwrap(); + let runtime_key = worker_runtime::auth::RuntimeIdentityMaterial::generate(RUNTIME).unwrap(); + store + .upsert_workspace_runtime_binding( + WorkspaceRuntimeBinding { + workspace_id: WORKSPACE.into(), + runtime_id: RUNTIME.into(), + display_name: "Fixture Runtime".into(), + base_url: "http://127.0.0.1:12345".into(), + public_key: runtime_key.public_key, + public_key_fingerprint: String::new(), + binding_id: String::new(), + authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity, + created_at: "2026-10-24T00:00:00Z".into(), + updated_at: "2026-10-24T00:00:00Z".into(), + revoked_at: None, + }, + false, + ) + .unwrap(); + store + } + + fn request() -> WorkerSpawnRequest { + WorkerSpawnRequest { + intent: WorkerSpawnIntent::WorkspaceCoding, + requested_worker_name: Some("Fixture Worker".into()), + singleton_key: None, + acceptance: WorkerSpawnAcceptanceRequirement::SocketReady, + profile: ProfileSelector::Builtin("builtin:coder".into()), + ticket_assignment: None, + initial_submit: Vec::new(), + workdir_attachment_requests: Vec::new(), + resolved_workdir_attachment_requests: Vec::new(), + resolved_workdir_attachments: Vec::new(), + resolved_config_bundle: None, + resolved_workspace_api: None, + resolved_memory_settings: None, + resolved_subjektiv_attached: false, + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: Vec::new(), + resolved_control_operation: None, + resolved_execution_grant: None, + } + } + + fn seed(request: &WorkerSpawnRequest) -> FixedWorkerSeed<'_> { + FixedWorkerSeed { + workspace_id: WORKSPACE, + runtime_id: RUNTIME, + worker_id: WORKER.parse().unwrap(), + allocation_key: "fixture-allocation", + request, + } + } + + fn worker() -> RuntimeWorkerRef { + RuntimeWorkerRef::new(RUNTIME, WORKER) + } + + #[test] + fn fixed_ids_are_isolated_and_seed_has_no_runtime_observation() { + let a = bootstrap(); + let b = bootstrap(); + let request = request(); + let first = seed_fixed_worker(&a, seed(&request)).unwrap(); + let second = seed_fixed_worker(&b, seed(&request)).unwrap(); + assert_eq!(first, second); + assert_eq!(first.worker_id.to_string(), WORKER); + assert_eq!(first.memory_settings.workspace_id, WORKSPACE); + assert_eq!(first.memory_settings.language, "English"); + assert!(first.singleton.is_none()); + assert_ne!( + a.get_workspace_signing_identity(WORKSPACE) + .unwrap() + .unwrap() + .public_key, + b.get_workspace_signing_identity(WORKSPACE) + .unwrap() + .unwrap() + .public_key + ); + assert_ne!( + a.get_workspace_runtime_binding(WORKSPACE, RUNTIME) + .unwrap() + .unwrap() + .public_key, + b.get_workspace_runtime_binding(WORKSPACE, RUNTIME) + .unwrap() + .unwrap() + .public_key + ); + for store in [&a, &b] { + assert_eq!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .as_deref(), + Some("W-1") + ); + assert_eq!( + store + .resolve_worker_resource_reference(WORKSPACE, "W-1") + .unwrap(), + Some(worker()) + ); + assert!( + !store + .has_active_worker_create_reservation(WORKSPACE, &worker()) + .unwrap() + ); + let registry = store + .get_worker_registry(WORKSPACE, &worker()) + .unwrap() + .unwrap(); + assert_eq!(registry.profile.as_deref(), Some("builtin:coder")); + assert_eq!(registry.display_name, "Fixture Worker"); + assert_eq!(registry.retention_state, "normal"); + let projection = store + .with_conn(|conn| read_worker_registry_projection(conn, WORKSPACE, &worker())) + .unwrap() + .unwrap(); + assert!(projection.observation.is_none()); + assert!(projection.job.is_none()); + } + } + + #[test] + fn initial_submit_is_refused_without_registration_or_allocation() { + let store = bootstrap(); + let mut request = request(); + // Even an empty text segment is new input; the vector itself must be empty. + for text in ["new input must not be seeded", ""] { + request.initial_submit = vec![protocol::Segment::text(text)]; + let error = seed_fixed_worker(&store, seed(&request)).unwrap_err(); + assert!( + matches!(error, Error::InvalidInput(ref message) if message.contains("initial_submit")) + ); + assert!( + store + .get_worker_registry(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + assert!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + assert!( + !store + .has_active_worker_create_reservation(WORKSPACE, &worker()) + .unwrap() + ); + } + // The rejected attempts must not consume the allocation or resource counter. + request.initial_submit.clear(); + seed_fixed_worker(&store, seed(&request)).unwrap(); + assert_eq!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .as_deref(), + Some("W-1") + ); + } + + #[test] + fn exact_replay_changed_input_and_cross_runtime_identity_are_refused() { + let store = bootstrap(); + let mut request = request(); + seed_fixed_worker(&store, seed(&request)).unwrap(); + let before = store.get_worker_registry(WORKSPACE, &worker()).unwrap(); + assert!(seed_fixed_worker(&store, seed(&request)).is_err()); + request.requested_worker_name = Some("Must not overwrite".into()); + assert!(seed_fixed_worker(&store, seed(&request)).is_err()); + let mut new_allocation = seed(&request); + new_allocation.allocation_key = "another-allocation"; + assert!(seed_fixed_worker(&store, new_allocation).is_err()); + let mut same_allocation = seed(&request); + same_allocation.worker_id = WorkerId::now_v7(); + assert!(seed_fixed_worker(&store, same_allocation).is_err()); + assert_eq!( + store.get_worker_registry(WORKSPACE, &worker()).unwrap(), + before + ); + + let other = bootstrap(); + let mut record = before.unwrap(); + record.worker.runtime_id = "different-runtime".into(); + other.upsert_worker_registry(&record).unwrap(); + assert!(seed_fixed_worker(&other, seed(&request)).is_err()); + assert!( + other + .get_worker_registry(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + } + + #[test] + fn reserved_and_orphaned_resource_identities_are_not_adopted() { + let store = bootstrap(); + let request = request(); + let memory = store.get_workspace_memory_settings(WORKSPACE).unwrap(); + let ordinary = store + .reserve_worker_create(WORKSPACE, RUNTIME, "ordinary", "fingerprint", None, &memory) + .unwrap(); + let mut collision = seed(&request); + collision.worker_id = ordinary.worker_id; + assert!(seed_fixed_worker(&store, collision).is_err()); + // Store-level fault setup only: the external harness never writes SQL. + store + .with_conn(|conn| { + allocate_resource_key( + conn, + WORKSPACE, + WorkspaceResourceKind::Worker, + &worker_resource_id(&worker()), + "now", + )?; + Ok(()) + }) + .unwrap(); + assert!(seed_fixed_worker(&store, seed(&request)).is_err()); + assert!( + store + .get_worker_registry(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + } + + #[test] + fn unknown_resource_identity_and_retained_removal_fence_are_refused() { + let request = request(); + let malformed = bootstrap(); + malformed + .with_conn(|conn| { + allocate_resource_key( + conn, + WORKSPACE, + WorkspaceResourceKind::Worker, + "unknown-identity", + "now", + )?; + Ok(()) + }) + .unwrap(); + assert!(matches!( + seed_fixed_worker(&malformed, seed(&request)), + Err(Error::InvalidInput(_)) + )); + assert!( + malformed + .get_worker_registry(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + + let removed = bootstrap(); + removed.with_conn(|conn| { + conn.execute( + "INSERT INTO worker_registry_projection_removals(workspace_id, runtime_id, worker_id) VALUES (?1, ?2, ?3)", + params![WORKSPACE, RUNTIME, WORKER], + )?; + Ok(()) + }).unwrap(); + assert!(matches!( + seed_fixed_worker(&removed, seed(&request)), + Err(Error::InvalidInput(_)) + )); + assert!( + removed + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + } + + #[test] + fn unknown_workspace_runtime_and_revoked_binding_are_refused() { + let store = bootstrap(); + let request = request(); + let mut unknown = seed(&request); + unknown.workspace_id = "unknown"; + assert!(seed_fixed_worker(&store, unknown).is_err()); + let mut unknown = seed(&request); + unknown.runtime_id = "unknown"; + assert!(seed_fixed_worker(&store, unknown).is_err()); + store + .revoke_workspace_runtime_binding(WORKSPACE, RUNTIME, "2026-10-24T01:00:00Z") + .unwrap(); + assert!(seed_fixed_worker(&store, seed(&request)).is_err()); + assert!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + } + + #[test] + fn unsupported_intent_is_refused_without_partial_registration() { + let store = bootstrap(); + let base = request(); + let mut requests = Vec::new(); + let mut request = base.clone(); + request.singleton_key = Some("fixture-singleton".into()); + requests.push(request); + let mut request = base.clone(); + request.intent = WorkerSpawnIntent::WorkspaceOrchestrator; + requests.push(request); + let mut request = base.clone(); + request.resolved_subjektiv_attached = true; + requests.push(request); + let mut request = base.clone(); + request + .workdir_attachment_requests + .push(WorkerSpawnWorkingDirectoryAttachmentRequest { + alias: workdir::WorkdirAttachmentAlias::new("repo").unwrap(), + working_directory: WorkerSpawnWorkingDirectoryRequest { + repository_key: "repo".into(), + selector: None, + }, + }); + requests.push(request); + for request in requests { + assert!(seed_fixed_worker(&store, seed(&request)).is_err()); + } + assert!( + store + .get_worker_registry(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + assert!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + assert_eq!( + seed_fixed_worker(&store, seed(&base)) + .unwrap() + .worker_id + .to_string(), + WORKER + ); + } + + #[test] + fn late_catalog_failure_rolls_back_reservation_and_resource_allocation() { + let store = bootstrap(); + let request = request(); + store + .with_conn(|conn| { + conn.execute_batch( + "CREATE TEMP TRIGGER fail_fixture_catalog BEFORE INSERT ON worker_registry + BEGIN SELECT RAISE(ABORT, 'injected catalog failure'); END", + )?; + Ok(()) + }) + .unwrap(); + assert!(seed_fixed_worker(&store, seed(&request)).is_err()); + assert!( + !store + .has_active_worker_create_reservation(WORKSPACE, &worker()) + .unwrap() + ); + assert!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + store + .with_conn(|conn| { + conn.execute_batch("DROP TRIGGER fail_fixture_catalog")?; + Ok(()) + }) + .unwrap(); + seed_fixed_worker(&store, seed(&request)).unwrap(); + assert_eq!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .as_deref(), + Some("W-1") + ); + } + + #[test] + fn ordinary_creation_still_allocates_and_replays_without_catalog_insertion() { + let store = bootstrap(); + let memory = store.get_workspace_memory_settings(WORKSPACE).unwrap(); + let reserve = |key| { + store + .reserve_worker_create(WORKSPACE, RUNTIME, key, "fingerprint", None, &memory) + .unwrap() + }; + let first = reserve("ordinary-one"); + assert_eq!(first, reserve("ordinary-one")); + let second = reserve("ordinary-two"); + assert_ne!(first.worker_id, second.worker_id); + assert_ne!(first.worker_id.to_string(), WORKER); + assert_eq!( + first.worker_id.as_uuid().get_version(), + Some(uuid::Version::SortRand) + ); + let worker = RuntimeWorkerRef::new(RUNTIME, first.worker_id.to_string()); + assert!( + store + .has_active_worker_create_reservation(WORKSPACE, &worker) + .unwrap() + ); + assert!( + store + .get_worker_registry(WORKSPACE, &worker) + .unwrap() + .is_none() + ); + assert!( + store + .reserve_worker_create(WORKSPACE, RUNTIME, "ordinary-one", "changed", None, &memory) + .is_err() + ); + } +} diff --git a/tests/e2e/Cargo.toml b/tests/e2e/Cargo.toml index b308eeb49..53b60d5e3 100644 --- a/tests/e2e/Cargo.toml +++ b/tests/e2e/Cargo.toml @@ -9,6 +9,7 @@ autotests = false [features] default = [] e2e = [] +fixture-support = ["yoi-workspace-server/test-support", "worker-runtime/test-support"] [dependencies] libc.workspace = true @@ -17,11 +18,17 @@ serde = { workspace = true, features = ["derive"] } serde_json.workspace = true sha2.workspace = true tempfile.workspace = true +protocol.workspace = true +tungstenite = { version = "0.29", default-features = false, features = ["handshake"] } [dev-dependencies] chrono = { version = "0.4", default-features = false, features = ["clock"] } yoi-workspace-server.workspace = true server-api.workspace = true +worker-runtime.workspace = true +session-store.workspace = true +config-source.workspace = true +manifest.workspace = true webauthn-rs-proto = "0.5.5" webauthn-authenticator-rs = { version = "0.5.5", default-features = false, features = ["softpasskey"] } @@ -44,3 +51,8 @@ required-features = ["e2e"] name = "worker_environment" path = "tests/worker_environment.rs" required-features = ["e2e"] + +[[test]] +name = "worker_restore" +path = "tests/worker_restore.rs" +required-features = ["e2e", "fixture-support"] diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 678300006..5766c9471 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -7,3 +7,41 @@ - [`PanelHarness`]: PTY経由の入力、観測イベントによる同期、失敗時のartifact保存。 - [`FixtureWorkspace`]: TUI用WorkspaceとHOME／XDG領域の隔離。 - [`WorkerE2eEnvironment`]: Backend／Runtime用プロセスとfixtureの管理。 +- [`FixtureSource`]: 提供資産の全ファイルSHA-256記録、独立copy、原本不変の検証。 +- [`WorkerSessionSubscription`]: 公開Worker WebSocketの初期full Snapshot照合と停止確認。モデル呼出しは行わない。 + +## 固定Workerの段階起動 + +1. 提供fixtureを所有crateの型で検証し、`FixtureSource`で原本hashを記録する。各環境の一時領域へcopyする。fixtureが未提供・不適合なら失敗とし、代替Workerを生成しない。 +2. `initialize_runtime_identity`で固定Runtime IDに新しい鍵を発行する。Backendの型付きstore APIで新しいAccount認証・Workspace署名identity・Runtime binding・固定Worker登録を準備する。 +3. `start_seeded_backend`へ固定Workspace/Runtime IDとその環境の新しいbearerを渡す。Server起動後の公開APIからWorkspaceの公開identityを取得する。 +4. `start_runtime_with_seed`はtrust設定後、Runtime processの起動前にseed closureを呼ぶ。ここで実Server URLを使いRuntime所有crateの型付きinstallを呼ぶ。Runtime store rootは`runtime_data_dir.join("runtime")`。 +5. Worker一覧とdetailから停止状態・観測tokenを確認し、一つの固定request IDで`restore_worker`を一度だけ呼ぶ。結果不明・拒否は成功扱いせず、tokenを自動更新しない。 +6. `WorkerSessionSubscription::connect`で期待`protocol::SessionSnapshot`全体（entry ID、timestamp、順序、本文を含む）を照合する。二環境を同時に保持し、一方のstop/cleanup後も他方の購読・一覧・内容を再確認する。 +7. `stop_worker`、購読の`expect_stopped`、環境cleanup、原本`assert_unchanged`を確認する。 + +所有crateの`test-support`依存はテスト側の非default `fixture-support` featureだけで有効にする。製品binary providerは別Cargo invocationと専用`target/e2e-products`へ`yoi`・`yoi-server`・`yoi-runtime`を追加featureなしでビルドする。support test自身をこのtargetへビルドしてはならない。`YOI_E2E_BIN`等の明示binary overrideを使用する場合も、呼出し側が同じ分離を保証する。既存TUI用`e2e-test` featureは別の`target/e2e-tui-products`へビルドし、restore製品binaryへ混入しない。 + +固定資産の正常restore・並行隔離E2E: + +```sh +cargo test -p yoi-e2e --features e2e,fixture-support --test worker_restore +# 親から提供された候補の検証（読取専用、原本の自動採用や生成はしない） +YOI_E2E_RESTORE_FIXTURE=target/fixture-candidate-v1 \ + cargo test -p yoi-e2e --features e2e,fixture-support --test worker_restore +``` + +標準資産は`tests/e2e/fixtures/worker-codex-v1`の`manifest.json`・`worker-template.json`・`expected.json`・`workspace-config.dcdl`。manifestの各asset hash/size、schemaと固定ID、生成元commit、modelを検証する。Workspace configのimport参照を黙って最新へ置換せず、Gitで生成元commitに対する`resources`と`crates/builtin-source`の一致を確認する（Git履歴がない場合も失敗）。移植したSessionの正規公開projectionを供給された会話行と照合し、その全snapshotを実executionのWebSocketと比較する。欠落fixture・hash不一致・未実行は成功扱いしない。 + +通常のharness回帰試験（製品processは起動しない）: + +```sh +cargo test -p yoi-e2e --lib --no-default-features +``` + +既存の製品process E2Eは別結果として実行・報告する: + +```sh +cargo test -p yoi-e2e --features e2e --test worker_environment +cargo test -p yoi-e2e --features e2e --test panel --test rewind +``` diff --git a/tests/e2e/fixtures/worker-codex-v1/expected.json b/tests/e2e/fixtures/worker-codex-v1/expected.json new file mode 100644 index 000000000..485398932 --- /dev/null +++ b/tests/e2e/fixtures/worker-codex-v1/expected.json @@ -0,0 +1,52 @@ +{ + "source_revision": "1639be98411921308238b3a6a6e75c784c045156", + "model": "codex-oauth/gpt-6.1-sol", + "worker_id": "01a1657d-b762-7791-87ae-a49f28cadbce", + "session_id": "01a1657d-b818-77c1-bd73-175a27562f63", + "rows": [ + { + "entry_id": "01a1657f-18bf-7a03-b8b8-953d27041fe0", + "timestamp": 1792704190655, + "role": "user", + "parts": [ + { + "kind": "text", + "content": "これは保存と復元を確認するための人工的な会話です。合言葉は「青い灯台」です。覚えたことを一文で返してください。Toolは使わないでください。" + } + ] + }, + { + "entry_id": "01a1657f-1f55-71d3-b418-8199b863f59a", + "timestamp": 1792704192341, + "role": "assistant", + "parts": [ + { + "kind": "text", + "text": "合言葉は「青い灯台」と覚えました。" + } + ] + }, + { + "entry_id": "01a1657f-3053-71f3-bf4f-91290d82488b", + "timestamp": 1792704196691, + "role": "user", + "parts": [ + { + "kind": "text", + "content": "先ほどの合言葉は何でしたか？合言葉だけを答えてください。" + } + ] + }, + { + "entry_id": "01a1657f-36ef-78b1-b0c8-b141f13b0c7b", + "timestamp": 1792704198383, + "role": "assistant", + "parts": [ + { + "kind": "text", + "text": "青い灯台" + } + ] + } + ] +} diff --git a/tests/e2e/fixtures/worker-codex-v1/manifest.json b/tests/e2e/fixtures/worker-codex-v1/manifest.json new file mode 100644 index 000000000..6c26e18ef --- /dev/null +++ b/tests/e2e/fixtures/worker-codex-v1/manifest.json @@ -0,0 +1,34 @@ +{ + "format_version": 1, + "source_revision": "1639be98411921308238b3a6a6e75c784c045156", + "runtime_schema_version": 12, + "session_schema_version": 6, + "workspace_id": "01a16553-b901-7191-8ceb-8e2306fe3da9", + "runtime_id": "fixture-951707ee-93ca-4205-9c12-45f08a104a74", + "worker_id": "01a1657d-b762-7791-87ae-a49f28cadbce", + "session_id": "01a1657d-b818-77c1-bd73-175a27562f63", + "segment_id": "01a1657d-b818-77c1-bd73-176172a4bd82", + "model": "codex-oauth/gpt-6.1-sol", + "state": "stopped", + "acquisition": "Real isolated Codex OAuth two-turn artificial conversation; typed offline StoppedWorkerTemplate extraction. Session bytes unchanged; environment credentials and execution receipts excluded.", + "source_file_sha256": { + "worker.json": "912e620013a17f05c1aaf900da48c9ee0ecb1f54121efc3721e03f8755eb5eca", + "metadata.json": "657928a89ec621bd092b86924201ed7b562e9683261789ce18bdb1d248bdc450", + "session/session.json": "22b01daab68a28b9777a1f7abb9ef9f7f6cbae113089446602f5d15421e7c7f3", + "session/segments/01a1657d-b818-77c1-bd73-176172a4bd82.jsonl": "7c5663bb06d9406d5bd0a5086159b31c0d7799458fcb1c6f81e18b63c49f4df5" + }, + "files": { + "worker-template.json": { + "bytes": 48343, + "sha256": "8bf6ce3c71c4f05c82314de93c1e8e51b7502b00915b6adb8b33046944e463b2" + }, + "expected.json": { + "bytes": 1474, + "sha256": "b9846db47ecb8f3f53b2abd9d5f0c766122f86015a7a7df7addf8beab3e9c55a" + }, + "workspace-config.dcdl": { + "bytes": 1141, + "sha256": "9b07507b71733081f077e2e9968a7f1f676c916a243bb355d1813f067a55c415" + } + } +} diff --git a/tests/e2e/fixtures/worker-codex-v1/worker-template.json b/tests/e2e/fixtures/worker-codex-v1/worker-template.json new file mode 100644 index 000000000..dfd032e64 --- /dev/null +++ b/tests/e2e/fixtures/worker-codex-v1/worker-template.json @@ -0,0 +1,5605 @@ +{ + "template_version": 1, + "runtime_schema_version": 12, + "runtime_id": "fixture-951707ee-93ca-4205-9c12-45f08a104a74", + "workspace_id": "01a16553-b901-7191-8ceb-8e2306fe3da9", + "worker_id": "01a1657d-b762-7791-87ae-a49f28cadbce", + "created_at_ms": 1792704100293, + "session_id": "01a1657d-b818-77c1-bd73-175a27562f63", + "segment_id": "01a1657d-b818-77c1-bd73-176172a4bd82", + "session_manifest": [ + 123, + 10, + 32, + 32, + 34, + 115, + 99, + 104, + 101, + 109, + 97, + 95, + 118, + 101, + 114, + 115, + 105, + 111, + 110, + 34, + 58, + 32, + 54, + 44, + 10, + 32, + 32, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 32, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 100, + 45, + 98, + 56, + 49, + 56, + 45, + 55, + 55, + 99, + 49, + 45, + 98, + 100, + 55, + 51, + 45, + 49, + 55, + 53, + 97, + 50, + 55, + 53, + 54, + 50, + 102, + 54, + 51, + 34, + 10, + 125, + 10 + ], + "segment_jsonl": [ + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 115, + 101, + 103, + 109, + 101, + 110, + 116, + 95, + 115, + 116, + 97, + 114, + 116, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 48, + 48, + 52, + 48, + 55, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 100, + 45, + 98, + 56, + 49, + 56, + 45, + 55, + 55, + 99, + 49, + 45, + 98, + 100, + 55, + 51, + 45, + 49, + 55, + 53, + 97, + 50, + 55, + 53, + 54, + 50, + 102, + 54, + 51, + 34, + 44, + 34, + 115, + 121, + 115, + 116, + 101, + 109, + 95, + 112, + 114, + 111, + 109, + 112, + 116, + 34, + 58, + 34, + 195, + 163, + 194, + 129, + 194, + 147, + 195, + 163, + 194, + 130, + 194, + 140, + 195, + 163, + 194, + 129, + 194, + 175, + 195, + 169, + 194, + 154, + 194, + 148, + 195, + 169, + 194, + 155, + 194, + 162, + 195, + 167, + 194, + 146, + 194, + 176, + 195, + 165, + 194, + 162, + 194, + 131, + 195, + 163, + 194, + 129, + 194, + 174, + 195, + 164, + 194, + 186, + 194, + 186, + 195, + 165, + 194, + 183, + 194, + 165, + 195, + 167, + 194, + 154, + 194, + 132, + 195, + 163, + 194, + 129, + 194, + 170, + 195, + 164, + 194, + 188, + 194, + 154, + 195, + 168, + 194, + 169, + 194, + 177, + 195, + 163, + 194, + 131, + 194, + 134, + 195, + 163, + 194, + 130, + 194, + 185, + 195, + 163, + 194, + 131, + 194, + 136, + 195, + 163, + 194, + 129, + 194, + 167, + 195, + 163, + 194, + 129, + 194, + 153, + 195, + 163, + 194, + 128, + 194, + 130, + 195, + 166, + 194, + 151, + 194, + 165, + 195, + 166, + 194, + 156, + 194, + 172, + 195, + 168, + 194, + 170, + 194, + 158, + 195, + 163, + 194, + 129, + 194, + 167, + 195, + 167, + 194, + 176, + 194, + 161, + 195, + 166, + 194, + 189, + 194, + 148, + 195, + 163, + 194, + 129, + 194, + 171, + 195, + 165, + 194, + 191, + 194, + 156, + 195, + 167, + 194, + 173, + 194, + 148, + 195, + 163, + 194, + 129, + 194, + 151, + 195, + 163, + 194, + 129, + 194, + 166, + 195, + 163, + 194, + 129, + 194, + 143, + 195, + 163, + 194, + 129, + 194, + 160, + 195, + 163, + 194, + 129, + 194, + 149, + 195, + 163, + 194, + 129, + 194, + 132, + 195, + 163, + 194, + 128, + 194, + 130, + 84, + 111, + 111, + 108, + 195, + 163, + 194, + 130, + 194, + 132, + 195, + 165, + 194, + 164, + 194, + 150, + 195, + 169, + 194, + 131, + 194, + 168, + 195, + 166, + 194, + 147, + 194, + 141, + 195, + 164, + 194, + 189, + 194, + 156, + 195, + 163, + 194, + 129, + 194, + 175, + 195, + 168, + 194, + 161, + 194, + 140, + 195, + 163, + 194, + 130, + 194, + 143, + 195, + 163, + 194, + 129, + 194, + 154, + 195, + 163, + 194, + 128, + 194, + 129, + 195, + 163, + 194, + 131, + 194, + 166, + 195, + 163, + 194, + 131, + 194, + 188, + 195, + 163, + 194, + 130, + 194, + 182, + 195, + 163, + 194, + 131, + 194, + 188, + 195, + 163, + 194, + 129, + 194, + 140, + 195, + 164, + 194, + 188, + 194, + 154, + 195, + 168, + 194, + 169, + 194, + 177, + 195, + 164, + 194, + 184, + 194, + 173, + 195, + 163, + 194, + 129, + 194, + 171, + 195, + 166, + 194, + 143, + 194, + 144, + 195, + 167, + 194, + 164, + 194, + 186, + 195, + 163, + 194, + 129, + 194, + 151, + 195, + 163, + 194, + 129, + 194, + 159, + 195, + 166, + 194, + 131, + 194, + 133, + 195, + 165, + 194, + 160, + 194, + 177, + 195, + 163, + 194, + 129, + 194, + 160, + 195, + 163, + 194, + 129, + 194, + 145, + 195, + 163, + 194, + 130, + 194, + 146, + 195, + 164, + 194, + 189, + 194, + 191, + 195, + 163, + 194, + 129, + 194, + 163, + 195, + 163, + 194, + 129, + 194, + 166, + 195, + 163, + 194, + 129, + 194, + 143, + 195, + 163, + 194, + 129, + 194, + 160, + 195, + 163, + 194, + 129, + 194, + 149, + 195, + 163, + 194, + 129, + 194, + 132, + 195, + 163, + 194, + 128, + 194, + 130, + 92, + 110, + 92, + 110, + 92, + 110, + 45, + 45, + 45, + 92, + 110, + 35, + 35, + 32, + 87, + 111, + 114, + 107, + 105, + 110, + 103, + 32, + 98, + 111, + 117, + 110, + 100, + 97, + 114, + 105, + 101, + 115, + 34, + 44, + 34, + 99, + 111, + 110, + 102, + 105, + 103, + 34, + 58, + 123, + 34, + 109, + 97, + 120, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 50, + 48, + 52, + 56, + 44, + 34, + 116, + 101, + 109, + 112, + 101, + 114, + 97, + 116, + 117, + 114, + 101, + 34, + 58, + 110, + 117, + 108, + 108, + 44, + 34, + 116, + 111, + 112, + 95, + 112, + 34, + 58, + 110, + 117, + 108, + 108, + 44, + 34, + 116, + 111, + 112, + 95, + 107, + 34, + 58, + 110, + 117, + 108, + 108, + 44, + 34, + 115, + 116, + 111, + 112, + 95, + 115, + 101, + 113, + 117, + 101, + 110, + 99, + 101, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 114, + 101, + 97, + 115, + 111, + 110, + 105, + 110, + 103, + 34, + 58, + 34, + 108, + 111, + 119, + 34, + 125, + 44, + 34, + 104, + 105, + 115, + 116, + 111, + 114, + 121, + 34, + 58, + 91, + 93, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 105, + 110, + 118, + 111, + 107, + 101, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 48, + 54, + 53, + 53, + 44, + 34, + 116, + 114, + 105, + 103, + 103, + 101, + 114, + 34, + 58, + 34, + 117, + 115, + 101, + 114, + 95, + 115, + 101, + 110, + 100, + 34, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 117, + 115, + 101, + 114, + 95, + 105, + 110, + 112, + 117, + 116, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 48, + 54, + 53, + 53, + 44, + 34, + 115, + 101, + 103, + 109, + 101, + 110, + 116, + 115, + 34, + 58, + 91, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 101, + 120, + 116, + 34, + 44, + 34, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 34, + 58, + 34, + 227, + 129, + 147, + 227, + 130, + 140, + 227, + 129, + 175, + 228, + 191, + 157, + 229, + 173, + 152, + 227, + 129, + 168, + 229, + 190, + 169, + 229, + 133, + 131, + 227, + 130, + 146, + 231, + 162, + 186, + 232, + 170, + 141, + 227, + 129, + 153, + 227, + 130, + 139, + 227, + 129, + 159, + 227, + 130, + 129, + 227, + 129, + 174, + 228, + 186, + 186, + 229, + 183, + 165, + 231, + 154, + 132, + 227, + 129, + 170, + 228, + 188, + 154, + 232, + 169, + 177, + 227, + 129, + 167, + 227, + 129, + 153, + 227, + 128, + 130, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 175, + 227, + 128, + 140, + 233, + 157, + 146, + 227, + 129, + 132, + 231, + 129, + 175, + 229, + 143, + 176, + 227, + 128, + 141, + 227, + 129, + 167, + 227, + 129, + 153, + 227, + 128, + 130, + 232, + 166, + 154, + 227, + 129, + 136, + 227, + 129, + 159, + 227, + 129, + 147, + 227, + 129, + 168, + 227, + 130, + 146, + 228, + 184, + 128, + 230, + 150, + 135, + 227, + 129, + 167, + 232, + 191, + 148, + 227, + 129, + 151, + 227, + 129, + 166, + 227, + 129, + 143, + 227, + 129, + 160, + 227, + 129, + 149, + 227, + 129, + 132, + 227, + 128, + 130, + 84, + 111, + 111, + 108, + 227, + 129, + 175, + 228, + 189, + 191, + 227, + 130, + 143, + 227, + 129, + 170, + 227, + 129, + 132, + 227, + 129, + 167, + 227, + 129, + 143, + 227, + 129, + 160, + 227, + 129, + 149, + 227, + 129, + 132, + 227, + 128, + 130, + 34, + 125, + 93, + 44, + 34, + 101, + 120, + 116, + 101, + 110, + 115, + 105, + 111, + 110, + 115, + 34, + 58, + 91, + 123, + 34, + 100, + 111, + 109, + 97, + 105, + 110, + 34, + 58, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 46, + 112, + 101, + 110, + 100, + 105, + 110, + 103, + 95, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 111, + 110, + 115, + 46, + 118, + 49, + 34, + 44, + 34, + 112, + 97, + 121, + 108, + 111, + 97, + 100, + 34, + 58, + 123, + 34, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 110, + 103, + 34, + 58, + 110, + 117, + 108, + 108, + 44, + 34, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 110, + 103, + 95, + 110, + 111, + 116, + 105, + 102, + 105, + 99, + 97, + 116, + 105, + 111, + 110, + 34, + 58, + 91, + 93, + 44, + 34, + 105, + 110, + 118, + 111, + 99, + 97, + 116, + 105, + 111, + 110, + 95, + 114, + 101, + 99, + 101, + 105, + 112, + 116, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 105, + 110, + 118, + 111, + 99, + 97, + 116, + 105, + 111, + 110, + 95, + 114, + 101, + 99, + 111, + 118, + 101, + 114, + 121, + 95, + 98, + 108, + 111, + 99, + 107, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 110, + 101, + 120, + 116, + 95, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 111, + 110, + 95, + 115, + 101, + 113, + 117, + 101, + 110, + 99, + 101, + 34, + 58, + 49, + 44, + 34, + 110, + 111, + 116, + 105, + 102, + 105, + 99, + 97, + 116, + 105, + 111, + 110, + 95, + 114, + 101, + 99, + 101, + 105, + 112, + 116, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 112, + 101, + 110, + 100, + 105, + 110, + 103, + 34, + 58, + 91, + 93, + 44, + 34, + 112, + 101, + 110, + 100, + 105, + 110, + 103, + 95, + 110, + 111, + 116, + 105, + 102, + 105, + 99, + 97, + 116, + 105, + 111, + 110, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 114, + 101, + 99, + 101, + 105, + 112, + 116, + 115, + 34, + 58, + 91, + 123, + 34, + 100, + 105, + 115, + 112, + 111, + 115, + 105, + 116, + 105, + 111, + 110, + 34, + 58, + 34, + 115, + 116, + 97, + 114, + 116, + 101, + 100, + 34, + 44, + 34, + 112, + 97, + 121, + 108, + 111, + 97, + 100, + 95, + 100, + 105, + 103, + 101, + 115, + 116, + 34, + 58, + 34, + 100, + 53, + 51, + 102, + 53, + 53, + 52, + 52, + 52, + 56, + 98, + 99, + 50, + 55, + 55, + 53, + 50, + 97, + 99, + 49, + 49, + 98, + 56, + 54, + 57, + 50, + 48, + 51, + 53, + 48, + 57, + 50, + 49, + 98, + 51, + 54, + 54, + 54, + 97, + 53, + 101, + 54, + 100, + 54, + 53, + 49, + 51, + 97, + 49, + 48, + 98, + 56, + 53, + 102, + 49, + 102, + 56, + 102, + 50, + 52, + 48, + 100, + 97, + 50, + 34, + 44, + 34, + 115, + 111, + 117, + 114, + 99, + 101, + 95, + 110, + 97, + 109, + 101, + 115, + 112, + 97, + 99, + 101, + 34, + 58, + 34, + 98, + 97, + 99, + 107, + 101, + 110, + 100, + 58, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 101, + 45, + 55, + 98, + 98, + 48, + 45, + 56, + 50, + 53, + 101, + 45, + 56, + 50, + 54, + 102, + 97, + 102, + 54, + 51, + 48, + 98, + 51, + 53, + 34, + 44, + 34, + 115, + 117, + 98, + 109, + 105, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 102, + 45, + 55, + 97, + 48, + 51, + 45, + 98, + 56, + 98, + 56, + 45, + 57, + 53, + 50, + 98, + 98, + 98, + 98, + 54, + 100, + 50, + 50, + 57, + 34, + 44, + 34, + 115, + 117, + 98, + 109, + 105, + 115, + 115, + 105, + 111, + 110, + 95, + 114, + 101, + 113, + 117, + 101, + 115, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 101, + 45, + 55, + 98, + 98, + 48, + 45, + 56, + 50, + 53, + 101, + 45, + 56, + 50, + 54, + 102, + 97, + 102, + 54, + 51, + 48, + 98, + 51, + 53, + 34, + 125, + 93, + 125, + 125, + 93, + 44, + 34, + 104, + 105, + 115, + 116, + 111, + 114, + 121, + 34, + 58, + 91, + 123, + 34, + 105, + 116, + 101, + 109, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 101, + 115, + 115, + 97, + 103, + 101, + 34, + 44, + 34, + 114, + 111, + 108, + 101, + 34, + 58, + 34, + 117, + 115, + 101, + 114, + 34, + 44, + 34, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 34, + 58, + 91, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 101, + 120, + 116, + 34, + 44, + 34, + 116, + 101, + 120, + 116, + 34, + 58, + 34, + 227, + 129, + 147, + 227, + 130, + 140, + 227, + 129, + 175, + 228, + 191, + 157, + 229, + 173, + 152, + 227, + 129, + 168, + 229, + 190, + 169, + 229, + 133, + 131, + 227, + 130, + 146, + 231, + 162, + 186, + 232, + 170, + 141, + 227, + 129, + 153, + 227, + 130, + 139, + 227, + 129, + 159, + 227, + 130, + 129, + 227, + 129, + 174, + 228, + 186, + 186, + 229, + 183, + 165, + 231, + 154, + 132, + 227, + 129, + 170, + 228, + 188, + 154, + 232, + 169, + 177, + 227, + 129, + 167, + 227, + 129, + 153, + 227, + 128, + 130, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 175, + 227, + 128, + 140, + 233, + 157, + 146, + 227, + 129, + 132, + 231, + 129, + 175, + 229, + 143, + 176, + 227, + 128, + 141, + 227, + 129, + 167, + 227, + 129, + 153, + 227, + 128, + 130, + 232, + 166, + 154, + 227, + 129, + 136, + 227, + 129, + 159, + 227, + 129, + 147, + 227, + 129, + 168, + 227, + 130, + 146, + 228, + 184, + 128, + 230, + 150, + 135, + 227, + 129, + 167, + 232, + 191, + 148, + 227, + 129, + 151, + 227, + 129, + 166, + 227, + 129, + 143, + 227, + 129, + 160, + 227, + 129, + 149, + 227, + 129, + 132, + 227, + 128, + 130, + 84, + 111, + 111, + 108, + 227, + 129, + 175, + 228, + 189, + 191, + 227, + 130, + 143, + 227, + 129, + 170, + 227, + 129, + 132, + 227, + 129, + 167, + 227, + 129, + 143, + 227, + 129, + 160, + 227, + 129, + 149, + 227, + 129, + 132, + 227, + 128, + 130, + 34, + 125, + 93, + 125, + 44, + 34, + 109, + 101, + 116, + 97, + 100, + 97, + 116, + 97, + 34, + 58, + 123, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 102, + 45, + 55, + 97, + 48, + 51, + 45, + 98, + 56, + 98, + 56, + 45, + 57, + 53, + 51, + 100, + 50, + 55, + 48, + 52, + 49, + 102, + 101, + 48, + 34, + 44, + 34, + 111, + 114, + 105, + 103, + 105, + 110, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 98, + 97, + 99, + 107, + 101, + 110, + 100, + 95, + 105, + 110, + 115, + 116, + 114, + 117, + 99, + 116, + 105, + 111, + 110, + 34, + 44, + 34, + 111, + 112, + 101, + 114, + 97, + 116, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 101, + 45, + 55, + 98, + 98, + 48, + 45, + 56, + 50, + 53, + 101, + 45, + 56, + 50, + 54, + 102, + 97, + 102, + 54, + 51, + 48, + 98, + 51, + 53, + 34, + 125, + 125, + 125, + 93, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 95, + 105, + 116, + 101, + 109, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 48, + 54, + 53, + 54, + 44, + 34, + 101, + 110, + 116, + 114, + 121, + 34, + 58, + 123, + 34, + 105, + 116, + 101, + 109, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 95, + 114, + 101, + 115, + 112, + 111, + 110, + 115, + 101, + 95, + 98, + 111, + 117, + 110, + 100, + 97, + 114, + 121, + 34, + 44, + 34, + 114, + 101, + 115, + 112, + 111, + 110, + 115, + 101, + 95, + 105, + 100, + 34, + 58, + 34, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 45, + 114, + 101, + 115, + 112, + 111, + 110, + 115, + 101, + 45, + 48, + 45, + 48, + 34, + 125, + 44, + 34, + 109, + 101, + 116, + 97, + 100, + 97, + 116, + 97, + 34, + 58, + 123, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 99, + 48, + 45, + 55, + 56, + 56, + 49, + 45, + 97, + 98, + 101, + 101, + 45, + 56, + 97, + 54, + 51, + 102, + 100, + 49, + 56, + 50, + 57, + 55, + 49, + 34, + 44, + 34, + 111, + 114, + 105, + 103, + 105, + 110, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 111, + 100, + 101, + 108, + 95, + 111, + 117, + 116, + 112, + 117, + 116, + 34, + 44, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 34, + 58, + 123, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 100, + 45, + 98, + 56, + 49, + 56, + 45, + 55, + 55, + 99, + 49, + 45, + 98, + 100, + 55, + 51, + 45, + 49, + 55, + 53, + 97, + 50, + 55, + 53, + 54, + 50, + 102, + 54, + 51, + 34, + 125, + 125, + 125, + 125, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 95, + 105, + 116, + 101, + 109, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 50, + 51, + 52, + 49, + 44, + 34, + 101, + 110, + 116, + 114, + 121, + 34, + 58, + 123, + 34, + 105, + 116, + 101, + 109, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 101, + 115, + 115, + 97, + 103, + 101, + 34, + 44, + 34, + 114, + 111, + 108, + 101, + 34, + 58, + 34, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 34, + 44, + 34, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 34, + 58, + 91, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 101, + 120, + 116, + 34, + 44, + 34, + 116, + 101, + 120, + 116, + 34, + 58, + 34, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 175, + 227, + 128, + 140, + 233, + 157, + 146, + 227, + 129, + 132, + 231, + 129, + 175, + 229, + 143, + 176, + 227, + 128, + 141, + 227, + 129, + 168, + 232, + 166, + 154, + 227, + 129, + 136, + 227, + 129, + 190, + 227, + 129, + 151, + 227, + 129, + 159, + 227, + 128, + 130, + 34, + 125, + 93, + 125, + 44, + 34, + 109, + 101, + 116, + 97, + 100, + 97, + 116, + 97, + 34, + 58, + 123, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 102, + 53, + 53, + 45, + 55, + 49, + 100, + 51, + 45, + 98, + 52, + 49, + 56, + 45, + 56, + 49, + 57, + 57, + 98, + 56, + 54, + 51, + 102, + 53, + 57, + 97, + 34, + 44, + 34, + 111, + 114, + 105, + 103, + 105, + 110, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 111, + 100, + 101, + 108, + 95, + 111, + 117, + 116, + 112, + 117, + 116, + 34, + 44, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 34, + 58, + 123, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 100, + 45, + 98, + 56, + 49, + 56, + 45, + 55, + 55, + 99, + 49, + 45, + 98, + 100, + 55, + 51, + 45, + 49, + 55, + 53, + 97, + 50, + 55, + 53, + 54, + 50, + 102, + 54, + 51, + 34, + 125, + 125, + 125, + 125, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 108, + 108, + 109, + 95, + 117, + 115, + 97, + 103, + 101, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 50, + 51, + 52, + 49, + 44, + 34, + 104, + 105, + 115, + 116, + 111, + 114, + 121, + 95, + 108, + 101, + 110, + 34, + 58, + 49, + 44, + 34, + 105, + 110, + 112, + 117, + 116, + 95, + 116, + 111, + 116, + 97, + 108, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 49, + 54, + 53, + 50, + 44, + 34, + 99, + 97, + 99, + 104, + 101, + 95, + 114, + 101, + 97, + 100, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 48, + 44, + 34, + 99, + 97, + 99, + 104, + 101, + 95, + 119, + 114, + 105, + 116, + 101, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 48, + 44, + 34, + 111, + 117, + 116, + 112, + 117, + 116, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 49, + 56, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 117, + 114, + 110, + 95, + 101, + 110, + 100, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 50, + 51, + 52, + 50, + 44, + 34, + 116, + 117, + 114, + 110, + 95, + 99, + 111, + 117, + 110, + 116, + 34, + 58, + 49, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 114, + 117, + 110, + 95, + 99, + 111, + 109, + 112, + 108, + 101, + 116, + 101, + 100, + 34, + 44, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 102, + 53, + 54, + 45, + 55, + 54, + 101, + 51, + 45, + 57, + 53, + 101, + 97, + 45, + 48, + 100, + 97, + 100, + 101, + 49, + 98, + 48, + 52, + 101, + 53, + 102, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 50, + 51, + 52, + 50, + 44, + 34, + 105, + 110, + 116, + 101, + 114, + 114, + 117, + 112, + 116, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 114, + 101, + 115, + 117, + 108, + 116, + 34, + 58, + 34, + 102, + 105, + 110, + 105, + 115, + 104, + 101, + 100, + 34, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 105, + 110, + 118, + 111, + 107, + 101, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 54, + 54, + 57, + 49, + 44, + 34, + 116, + 114, + 105, + 103, + 103, + 101, + 114, + 34, + 58, + 34, + 117, + 115, + 101, + 114, + 95, + 115, + 101, + 110, + 100, + 34, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 117, + 115, + 101, + 114, + 95, + 105, + 110, + 112, + 117, + 116, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 54, + 54, + 57, + 49, + 44, + 34, + 115, + 101, + 103, + 109, + 101, + 110, + 116, + 115, + 34, + 58, + 91, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 101, + 120, + 116, + 34, + 44, + 34, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 34, + 58, + 34, + 229, + 133, + 136, + 227, + 129, + 187, + 227, + 129, + 169, + 227, + 129, + 174, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 175, + 228, + 189, + 149, + 227, + 129, + 167, + 227, + 129, + 151, + 227, + 129, + 159, + 227, + 129, + 139, + 239, + 188, + 159, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 160, + 227, + 129, + 145, + 227, + 130, + 146, + 231, + 173, + 148, + 227, + 129, + 136, + 227, + 129, + 166, + 227, + 129, + 143, + 227, + 129, + 160, + 227, + 129, + 149, + 227, + 129, + 132, + 227, + 128, + 130, + 34, + 125, + 93, + 44, + 34, + 101, + 120, + 116, + 101, + 110, + 115, + 105, + 111, + 110, + 115, + 34, + 58, + 91, + 123, + 34, + 100, + 111, + 109, + 97, + 105, + 110, + 34, + 58, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 46, + 112, + 101, + 110, + 100, + 105, + 110, + 103, + 95, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 111, + 110, + 115, + 46, + 118, + 49, + 34, + 44, + 34, + 112, + 97, + 121, + 108, + 111, + 97, + 100, + 34, + 58, + 123, + 34, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 110, + 103, + 34, + 58, + 110, + 117, + 108, + 108, + 44, + 34, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 110, + 103, + 95, + 110, + 111, + 116, + 105, + 102, + 105, + 99, + 97, + 116, + 105, + 111, + 110, + 34, + 58, + 91, + 93, + 44, + 34, + 105, + 110, + 118, + 111, + 99, + 97, + 116, + 105, + 111, + 110, + 95, + 114, + 101, + 99, + 101, + 105, + 112, + 116, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 105, + 110, + 118, + 111, + 99, + 97, + 116, + 105, + 111, + 110, + 95, + 114, + 101, + 99, + 111, + 118, + 101, + 114, + 121, + 95, + 98, + 108, + 111, + 99, + 107, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 110, + 101, + 120, + 116, + 95, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 111, + 110, + 95, + 115, + 101, + 113, + 117, + 101, + 110, + 99, + 101, + 34, + 58, + 50, + 44, + 34, + 110, + 111, + 116, + 105, + 102, + 105, + 99, + 97, + 116, + 105, + 111, + 110, + 95, + 114, + 101, + 99, + 101, + 105, + 112, + 116, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 112, + 101, + 110, + 100, + 105, + 110, + 103, + 34, + 58, + 91, + 93, + 44, + 34, + 112, + 101, + 110, + 100, + 105, + 110, + 103, + 95, + 110, + 111, + 116, + 105, + 102, + 105, + 99, + 97, + 116, + 105, + 111, + 110, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 114, + 101, + 99, + 101, + 105, + 112, + 116, + 115, + 34, + 58, + 91, + 123, + 34, + 100, + 105, + 115, + 112, + 111, + 115, + 105, + 116, + 105, + 111, + 110, + 34, + 58, + 34, + 115, + 116, + 97, + 114, + 116, + 101, + 100, + 34, + 44, + 34, + 112, + 97, + 121, + 108, + 111, + 97, + 100, + 95, + 100, + 105, + 103, + 101, + 115, + 116, + 34, + 58, + 34, + 100, + 53, + 51, + 102, + 53, + 53, + 52, + 52, + 52, + 56, + 98, + 99, + 50, + 55, + 55, + 53, + 50, + 97, + 99, + 49, + 49, + 98, + 56, + 54, + 57, + 50, + 48, + 51, + 53, + 48, + 57, + 50, + 49, + 98, + 51, + 54, + 54, + 54, + 97, + 53, + 101, + 54, + 100, + 54, + 53, + 49, + 51, + 97, + 49, + 48, + 98, + 56, + 53, + 102, + 49, + 102, + 56, + 102, + 50, + 52, + 48, + 100, + 97, + 50, + 34, + 44, + 34, + 115, + 111, + 117, + 114, + 99, + 101, + 95, + 110, + 97, + 109, + 101, + 115, + 112, + 97, + 99, + 101, + 34, + 58, + 34, + 98, + 97, + 99, + 107, + 101, + 110, + 100, + 58, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 101, + 45, + 55, + 98, + 98, + 48, + 45, + 56, + 50, + 53, + 101, + 45, + 56, + 50, + 54, + 102, + 97, + 102, + 54, + 51, + 48, + 98, + 51, + 53, + 34, + 44, + 34, + 115, + 117, + 98, + 109, + 105, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 102, + 45, + 55, + 97, + 48, + 51, + 45, + 98, + 56, + 98, + 56, + 45, + 57, + 53, + 50, + 98, + 98, + 98, + 98, + 54, + 100, + 50, + 50, + 57, + 34, + 44, + 34, + 115, + 117, + 98, + 109, + 105, + 115, + 115, + 105, + 111, + 110, + 95, + 114, + 101, + 113, + 117, + 101, + 115, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 101, + 45, + 55, + 98, + 98, + 48, + 45, + 56, + 50, + 53, + 101, + 45, + 56, + 50, + 54, + 102, + 97, + 102, + 54, + 51, + 48, + 98, + 51, + 53, + 34, + 125, + 44, + 123, + 34, + 100, + 105, + 115, + 112, + 111, + 115, + 105, + 116, + 105, + 111, + 110, + 34, + 58, + 34, + 115, + 116, + 97, + 114, + 116, + 101, + 100, + 34, + 44, + 34, + 112, + 97, + 121, + 108, + 111, + 97, + 100, + 95, + 100, + 105, + 103, + 101, + 115, + 116, + 34, + 58, + 34, + 98, + 100, + 101, + 101, + 49, + 53, + 56, + 98, + 99, + 57, + 50, + 53, + 102, + 101, + 98, + 56, + 49, + 100, + 54, + 56, + 52, + 48, + 102, + 100, + 49, + 54, + 100, + 50, + 100, + 101, + 56, + 98, + 50, + 49, + 97, + 51, + 48, + 50, + 52, + 98, + 49, + 53, + 102, + 57, + 50, + 99, + 102, + 53, + 48, + 54, + 50, + 53, + 97, + 97, + 56, + 50, + 101, + 97, + 48, + 52, + 49, + 97, + 101, + 98, + 34, + 44, + 34, + 115, + 111, + 117, + 114, + 99, + 101, + 95, + 110, + 97, + 109, + 101, + 115, + 112, + 97, + 99, + 101, + 34, + 58, + 34, + 98, + 97, + 99, + 107, + 101, + 110, + 100, + 58, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 48, + 53, + 50, + 45, + 55, + 54, + 100, + 49, + 45, + 97, + 52, + 99, + 49, + 45, + 49, + 51, + 56, + 100, + 101, + 55, + 56, + 101, + 52, + 51, + 56, + 100, + 34, + 44, + 34, + 115, + 117, + 98, + 109, + 105, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 48, + 53, + 51, + 45, + 55, + 49, + 102, + 51, + 45, + 98, + 102, + 52, + 102, + 45, + 57, + 49, + 49, + 55, + 48, + 55, + 57, + 102, + 49, + 100, + 102, + 97, + 34, + 44, + 34, + 115, + 117, + 98, + 109, + 105, + 115, + 115, + 105, + 111, + 110, + 95, + 114, + 101, + 113, + 117, + 101, + 115, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 48, + 53, + 50, + 45, + 55, + 54, + 100, + 49, + 45, + 97, + 52, + 99, + 49, + 45, + 49, + 51, + 56, + 100, + 101, + 55, + 56, + 101, + 52, + 51, + 56, + 100, + 34, + 125, + 93, + 125, + 125, + 93, + 44, + 34, + 104, + 105, + 115, + 116, + 111, + 114, + 121, + 34, + 58, + 91, + 123, + 34, + 105, + 116, + 101, + 109, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 101, + 115, + 115, + 97, + 103, + 101, + 34, + 44, + 34, + 114, + 111, + 108, + 101, + 34, + 58, + 34, + 117, + 115, + 101, + 114, + 34, + 44, + 34, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 34, + 58, + 91, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 101, + 120, + 116, + 34, + 44, + 34, + 116, + 101, + 120, + 116, + 34, + 58, + 34, + 229, + 133, + 136, + 227, + 129, + 187, + 227, + 129, + 169, + 227, + 129, + 174, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 175, + 228, + 189, + 149, + 227, + 129, + 167, + 227, + 129, + 151, + 227, + 129, + 159, + 227, + 129, + 139, + 239, + 188, + 159, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 160, + 227, + 129, + 145, + 227, + 130, + 146, + 231, + 173, + 148, + 227, + 129, + 136, + 227, + 129, + 166, + 227, + 129, + 143, + 227, + 129, + 160, + 227, + 129, + 149, + 227, + 129, + 132, + 227, + 128, + 130, + 34, + 125, + 93, + 125, + 44, + 34, + 109, + 101, + 116, + 97, + 100, + 97, + 116, + 97, + 34, + 58, + 123, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 48, + 53, + 51, + 45, + 55, + 49, + 102, + 51, + 45, + 98, + 102, + 52, + 102, + 45, + 57, + 49, + 50, + 57, + 48, + 100, + 56, + 50, + 52, + 56, + 56, + 98, + 34, + 44, + 34, + 111, + 114, + 105, + 103, + 105, + 110, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 98, + 97, + 99, + 107, + 101, + 110, + 100, + 95, + 105, + 110, + 115, + 116, + 114, + 117, + 99, + 116, + 105, + 111, + 110, + 34, + 44, + 34, + 111, + 112, + 101, + 114, + 97, + 116, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 48, + 53, + 50, + 45, + 55, + 54, + 100, + 49, + 45, + 97, + 52, + 99, + 49, + 45, + 49, + 51, + 56, + 100, + 101, + 55, + 56, + 101, + 52, + 51, + 56, + 100, + 34, + 125, + 125, + 125, + 93, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 95, + 105, + 116, + 101, + 109, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 54, + 54, + 57, + 50, + 44, + 34, + 101, + 110, + 116, + 114, + 121, + 34, + 58, + 123, + 34, + 105, + 116, + 101, + 109, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 95, + 114, + 101, + 115, + 112, + 111, + 110, + 115, + 101, + 95, + 98, + 111, + 117, + 110, + 100, + 97, + 114, + 121, + 34, + 44, + 34, + 114, + 101, + 115, + 112, + 111, + 110, + 115, + 101, + 95, + 105, + 100, + 34, + 58, + 34, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 45, + 114, + 101, + 115, + 112, + 111, + 110, + 115, + 101, + 45, + 49, + 45, + 49, + 34, + 125, + 44, + 34, + 109, + 101, + 116, + 97, + 100, + 97, + 116, + 97, + 34, + 58, + 123, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 48, + 53, + 52, + 45, + 55, + 48, + 54, + 49, + 45, + 97, + 49, + 102, + 54, + 45, + 52, + 100, + 55, + 101, + 101, + 100, + 51, + 98, + 52, + 102, + 53, + 53, + 34, + 44, + 34, + 111, + 114, + 105, + 103, + 105, + 110, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 111, + 100, + 101, + 108, + 95, + 111, + 117, + 116, + 112, + 117, + 116, + 34, + 44, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 34, + 58, + 123, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 100, + 45, + 98, + 56, + 49, + 56, + 45, + 55, + 55, + 99, + 49, + 45, + 98, + 100, + 55, + 51, + 45, + 49, + 55, + 53, + 97, + 50, + 55, + 53, + 54, + 50, + 102, + 54, + 51, + 34, + 125, + 125, + 125, + 125, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 95, + 105, + 116, + 101, + 109, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 56, + 51, + 56, + 51, + 44, + 34, + 101, + 110, + 116, + 114, + 121, + 34, + 58, + 123, + 34, + 105, + 116, + 101, + 109, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 101, + 115, + 115, + 97, + 103, + 101, + 34, + 44, + 34, + 114, + 111, + 108, + 101, + 34, + 58, + 34, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 34, + 44, + 34, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 34, + 58, + 91, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 101, + 120, + 116, + 34, + 44, + 34, + 116, + 101, + 120, + 116, + 34, + 58, + 34, + 233, + 157, + 146, + 227, + 129, + 132, + 231, + 129, + 175, + 229, + 143, + 176, + 34, + 125, + 93, + 125, + 44, + 34, + 109, + 101, + 116, + 97, + 100, + 97, + 116, + 97, + 34, + 58, + 123, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 54, + 101, + 102, + 45, + 55, + 56, + 98, + 49, + 45, + 98, + 48, + 99, + 56, + 45, + 98, + 49, + 52, + 49, + 102, + 49, + 51, + 98, + 48, + 99, + 55, + 98, + 34, + 44, + 34, + 111, + 114, + 105, + 103, + 105, + 110, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 111, + 100, + 101, + 108, + 95, + 111, + 117, + 116, + 112, + 117, + 116, + 34, + 44, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 34, + 58, + 123, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 100, + 45, + 98, + 56, + 49, + 56, + 45, + 55, + 55, + 99, + 49, + 45, + 98, + 100, + 55, + 51, + 45, + 49, + 55, + 53, + 97, + 50, + 55, + 53, + 54, + 50, + 102, + 54, + 51, + 34, + 125, + 125, + 125, + 125, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 108, + 108, + 109, + 95, + 117, + 115, + 97, + 103, + 101, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 56, + 51, + 56, + 52, + 44, + 34, + 104, + 105, + 115, + 116, + 111, + 114, + 121, + 95, + 108, + 101, + 110, + 34, + 58, + 52, + 44, + 34, + 105, + 110, + 112, + 117, + 116, + 95, + 116, + 111, + 116, + 97, + 108, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 49, + 54, + 57, + 54, + 44, + 34, + 99, + 97, + 99, + 104, + 101, + 95, + 114, + 101, + 97, + 100, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 49, + 53, + 51, + 54, + 44, + 34, + 99, + 97, + 99, + 104, + 101, + 95, + 119, + 114, + 105, + 116, + 101, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 48, + 44, + 34, + 111, + 117, + 116, + 112, + 117, + 116, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 56, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 117, + 114, + 110, + 95, + 101, + 110, + 100, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 56, + 51, + 56, + 52, + 44, + 34, + 116, + 117, + 114, + 110, + 95, + 99, + 111, + 117, + 110, + 116, + 34, + 58, + 50, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 114, + 117, + 110, + 95, + 99, + 111, + 109, + 112, + 108, + 101, + 116, + 101, + 100, + 34, + 44, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 54, + 102, + 48, + 45, + 55, + 102, + 53, + 48, + 45, + 97, + 52, + 56, + 102, + 45, + 56, + 55, + 55, + 99, + 98, + 54, + 97, + 100, + 100, + 53, + 56, + 54, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 56, + 51, + 56, + 52, + 44, + 34, + 105, + 110, + 116, + 101, + 114, + 114, + 117, + 112, + 116, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 114, + 101, + 115, + 117, + 108, + 116, + 34, + 58, + 34, + 102, + 105, + 110, + 105, + 115, + 104, + 101, + 100, + 34, + 125, + 10 + ] +} \ No newline at end of file diff --git a/tests/e2e/fixtures/worker-codex-v1/workspace-config.dcdl b/tests/e2e/fixtures/worker-codex-v1/workspace-config.dcdl new file mode 100644 index 000000000..60afe9221 --- /dev/null +++ b/tests/e2e/fixtures/worker-codex-v1/workspace-config.dcdl @@ -0,0 +1,22 @@ +{ + profile = { + default_profile = "project:fixture-chat"; + entries = [{ + selector = "project:fixture-chat"; + label = "Isolated Codex fixture conversation"; + description = "Artificial conversation only; no Workdir, Tools, Subject or background work."; + profile = (import "$builtin/profiles/job.dcdl") // { + slug = "fixture-chat"; + worker = { mode = "tools"; }; + model = { ref = "codex-oauth/gpt-6.1-sol"; }; + scope = "workspace_read"; + delegation_scope = "workspace_read"; + permissions = { default_action = "deny"; rule = []; }; + engine = { instruction = "default"; reasoning = "low"; max_tokens = 2048; max_turns = 4; language = "Japanese"; }; + session = { record_event_trace = false; }; + feature = { workdir_catalog = { enabled = false; }; }; + }; + }]; + }; + prompts = { default_prompt = "これは隔離環境の人工的な会話テストです。日本語で簡潔に応答してください。Toolや外部操作は行わず、ユーザーが会話中に提示した情報だけを使ってください。"; }; +} as WorkspaceConfigSchema diff --git a/tests/e2e/src/fixture_source.rs b/tests/e2e/src/fixture_source.rs new file mode 100644 index 000000000..de5100275 --- /dev/null +++ b/tests/e2e/src/fixture_source.rs @@ -0,0 +1,336 @@ +//! Bounded byte-preserving transport, independent of the owning store's schema validation. +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use sha2::{Digest, Sha256}; + +use crate::{HarnessError, Result}; + +#[derive(Debug, Clone, Copy)] +struct Limits { + file_bytes: u64, + total_bytes: u64, + entries: usize, + depth: usize, +} + +const LIMITS: Limits = Limits { + file_bytes: 8 * 1024 * 1024, + total_bytes: 32 * 1024 * 1024, + entries: 256, + depth: 8, +}; + +#[derive(Debug, Clone)] +pub struct FixtureSource { + root: PathBuf, + hashes: BTreeMap, + flat_files: Option>, +} + +fn reject(message: &'static str) -> HarnessError { + HarnessError::Protocol(message.into()) +} + +impl FixtureSource { + /// Capture a bounded tree without generating or repairing store data. + pub fn capture(root: &Path) -> Result { + Self::capture_with(root, None) + } + + /// Validate the complete flat file set before reading any content. The same + /// allowlist is retained for copy and later source-integrity checks. + pub fn capture_flat(root: &Path, names: &[&str]) -> Result { + let files: BTreeSet<_> = names.iter().map(PathBuf::from).collect(); + if files.len() != names.len() + || files.iter().any(|name| { + name.components().count() != 1 + || !matches!( + name.components().next(), + Some(std::path::Component::Normal(_)) + ) + }) + { + return Err(reject("invalid fixture file allowlist")); + } + Self::capture_with(root, Some(files)) + } + + fn capture_with(root: &Path, flat_files: Option>) -> Result { + let hashes = inventory(root, flat_files.as_ref(), LIMITS)?; + if hashes.is_empty() { + return Err(reject("provided fixture is empty")); + } + Ok(Self { + root: root.to_owned(), + hashes, + flat_files, + }) + } + + /// Read a known asset through a descriptor with a real byte limit, including + /// when its size grows after metadata inspection. Errors never include paths. + pub fn read_bounded(path: &Path) -> Result> { + read_regular(path, LIMITS.file_bytes) + } + + pub fn hashes(&self) -> &BTreeMap { + &self.hashes + } + + pub fn assert_unchanged(&self) -> Result<()> { + if inventory(&self.root, self.flat_files.as_ref(), LIMITS)? != self.hashes { + return Err(reject("provided fixture changed during execution")); + } + Ok(()) + } + + /// The destination must not exist. A copy is never a link to the source. + pub fn copy_to(&self, destination: &Path) -> Result<()> { + self.assert_unchanged()?; + let parent = destination + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + if parent + .canonicalize()? + .starts_with(self.root.canonicalize()?) + { + return Err(reject("fixture copy destination is inside the original")); + } + fs::create_dir(destination)?; + let mut remaining = LIMITS.total_bytes; + for (relative, expected) in &self.hashes { + let bytes = read_regular(&self.root.join(relative), LIMITS.file_bytes.min(remaining))?; + remaining -= bytes.len() as u64; + if hash(&bytes) != *expected { + return Err(reject("fixture changed while copying")); + } + let target = destination.join(relative); + fs::create_dir_all(target.parent().unwrap())?; + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(target)?; + file.write_all(&bytes)?; + } + if inventory(destination, self.flat_files.as_ref(), LIMITS)? != self.hashes { + return Err(reject("fixture copy failed hash verification")); + } + self.assert_unchanged() + } +} + +fn hash(bytes: &[u8]) -> String { + format!( + "sha256:{}", + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + ) +} + +fn read_limited(reader: impl Read, limit: u64) -> Result> { + let mut bytes = Vec::new(); + reader + .take(limit + 1) + .read_to_end(&mut bytes) + .map_err(|_| reject("fixture content read failed"))?; + if bytes.len() as u64 > limit { + return Err(reject("fixture byte budget exceeded")); + } + Ok(bytes) +} + +fn read_regular(path: &Path, limit: u64) -> Result> { + let mut options = fs::OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + // Do not follow a replaced final symlink or block on a replaced FIFO. + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + let file = options + .open(path) + .map_err(|_| reject("fixture asset open failed"))?; + let metadata = file + .metadata() + .map_err(|_| reject("fixture asset metadata failed"))?; + if !metadata.is_file() || metadata.len() > limit { + return Err(reject("fixture asset is not a bounded regular file")); + } + read_limited(file, limit) +} + +fn inventory( + root: &Path, + flat: Option<&BTreeSet>, + limits: Limits, +) -> Result> { + struct Budget { + entries: usize, + bytes: u64, + } + fn visit( + root: &Path, + path: &Path, + depth: usize, + limits: Limits, + flat: Option<&BTreeSet>, + budget: &mut Budget, + files: &mut BTreeSet, + ) -> Result<()> { + if depth > limits.depth || budget.entries == 0 { + return Err(reject("fixture tree budget exceeded")); + } + budget.entries -= 1; + let metadata = fs::symlink_metadata(path).map_err(|_| reject("fixture metadata failed"))?; + let relative = path.strip_prefix(root).unwrap(); + if depth != 0 && flat.is_some_and(|names| !names.contains(relative)) { + return Err(reject("unexpected fixture file set")); + } + if metadata.is_dir() && (depth == 0 || flat.is_none()) { + for entry in fs::read_dir(path).map_err(|_| reject("fixture directory read failed"))? { + let entry = entry.map_err(|_| reject("fixture directory entry failed"))?; + visit(root, &entry.path(), depth + 1, limits, flat, budget, files)?; + } + } else if metadata.is_file() && depth != 0 { + if metadata.len() > limits.file_bytes || metadata.len() > budget.bytes { + return Err(reject("fixture byte budget exceeded")); + } + budget.bytes -= metadata.len(); + files.insert(relative.to_owned()); + } else { + return Err(reject("fixture contains a symlink or special file")); + } + Ok(()) + } + // Preflight the entire shape and metadata budget before reading any body. + let mut files = BTreeSet::new(); + visit( + root, + root, + 0, + limits, + flat, + &mut Budget { + entries: limits.entries, + bytes: limits.total_bytes, + }, + &mut files, + )?; + if flat.is_some_and(|expected| expected != &files) { + return Err(reject("unexpected fixture file set")); + } + let mut remaining = limits.total_bytes; + let mut hashes = BTreeMap::new(); + for relative in files { + let bytes = read_regular(&root.join(&relative), limits.file_bytes.min(remaining))?; + remaining -= bytes.len() as u64; + hashes.insert(relative, hash(&bytes)); + } + Ok(hashes) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Transport-only synthetic bytes, never a substitute saved Worker fixture. + #[test] + fn copies_are_independent_and_source_mutation_is_detected() -> Result<()> { + let temp = tempfile::tempdir()?; + let source = temp.path().join("source"); + fs::create_dir(&source)?; + fs::write(source.join("bytes"), b"original")?; + let fixture = FixtureSource::capture_flat(&source, &["bytes"])?; + assert!(fixture.copy_to(&source.join("nested-copy")).is_err()); + assert!(!source.join("nested-copy").exists()); + fixture.assert_unchanged()?; + let first = temp.path().join("first"); + let second = temp.path().join("second"); + fixture.copy_to(&first)?; + fixture.copy_to(&second)?; + fs::write(first.join("bytes"), b"changed")?; + fs::remove_dir_all(first)?; + assert_eq!(fs::read(second.join("bytes"))?, b"original"); + fixture.assert_unchanged()?; + fs::write(source.join("bytes"), b"changed")?; + assert!(fixture.assert_unchanged().is_err()); + Ok(()) + } + + #[test] + fn rejects_missing_empty_existing_destination_and_extra_file() -> Result<()> { + let temp = tempfile::tempdir()?; + assert!(FixtureSource::capture(&temp.path().join("missing")).is_err()); + assert!(FixtureSource::capture(temp.path()).is_err()); + fs::write(temp.path().join("bytes"), b"original")?; + let source = FixtureSource::capture(temp.path())?; + assert!(source.copy_to(temp.path()).is_err()); + fs::write(temp.path().join("extra"), b"new")?; + assert!(source.assert_unchanged().is_err()); + Ok(()) + } + + #[test] + fn flat_allowlist_rejects_extra_entries_before_reading_content() -> Result<()> { + let temp = tempfile::tempdir()?; + fs::write(temp.path().join("asset"), b"ok")?; + // Sparse oversized input is never read. Even an allowed file that cannot + // be decoded does not take precedence over an unexpected file set. + fs::File::create(temp.path().join("extra"))?.set_len(LIMITS.file_bytes + 1)?; + let error = FixtureSource::capture_flat(temp.path(), &["asset"]).unwrap_err(); + assert!(error.to_string().contains("unexpected fixture file set")); + fs::remove_file(temp.path().join("extra"))?; + assert!(FixtureSource::capture_flat(temp.path(), &["missing"]).is_err()); + fs::create_dir(temp.path().join("dir"))?; + assert!(FixtureSource::capture_flat(temp.path(), &["asset", "dir"]).is_err()); + Ok(()) + } + + #[test] + fn tree_count_depth_total_and_actual_read_budgets_are_enforced() -> Result<()> { + let temp = tempfile::tempdir()?; + fs::write(temp.path().join("a"), b"1234")?; + fs::write(temp.path().join("b"), b"1234")?; + for limits in [ + Limits { + entries: 2, + ..LIMITS + }, + Limits { + total_bytes: 7, + ..LIMITS + }, + Limits { + file_bytes: 3, + ..LIMITS + }, + ] { + assert!(inventory(temp.path(), None, limits).is_err()); + } + fs::create_dir_all(temp.path().join("nested/deep"))?; + assert!(inventory(temp.path(), None, Limits { depth: 1, ..LIMITS }).is_err()); + // A reader can return more bytes than earlier metadata reported. Bound + // actual IO rather than trusting that observation or an EOF at stat size. + assert!(read_limited(std::io::Cursor::new(b"12345"), 4).is_err()); + assert_eq!(read_limited(std::io::Cursor::new(b"1234"), 4)?, b"1234"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn rejects_symbolic_links() -> Result<()> { + let temp = tempfile::tempdir()?; + fs::write(temp.path().join("bytes"), b"original")?; + std::os::unix::fs::symlink("bytes", temp.path().join("link"))?; + assert!(FixtureSource::capture(temp.path()).is_err()); + assert!(FixtureSource::read_bounded(&temp.path().join("link")).is_err()); + Ok(()) + } +} diff --git a/tests/e2e/src/lib.rs b/tests/e2e/src/lib.rs index 176c7c2ff..9fcdb4ff0 100644 --- a/tests/e2e/src/lib.rs +++ b/tests/e2e/src/lib.rs @@ -137,7 +137,9 @@ fn tested_yoi_env_policy_overview() -> TestedYoiEnvPolicy { #[derive(Debug)] pub enum HarnessError { Io(io::Error), - Json(serde_json::Error), + // Never retain decode errors: serde messages can contain rejected field names + // or values from private fixture/protocol payloads, including through Debug. + Json, CommandFailed { program: PathBuf, args: Vec, @@ -163,7 +165,7 @@ impl std::fmt::Display for HarnessError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Io(err) => write!(f, "io error: {err}"), - Self::Json(err) => write!(f, "json error: {err}"), + Self::Json => write!(f, "invalid JSON document"), Self::CommandFailed { program, args, @@ -209,8 +211,8 @@ impl From for HarnessError { } impl From for HarnessError { - fn from(value: serde_json::Error) -> Self { - Self::Json(value) + fn from(_: serde_json::Error) -> Self { + Self::Json } } @@ -1268,7 +1270,11 @@ impl Drop for PanelHarness { } } +mod fixture_source; +pub use fixture_source::FixtureSource; mod worker_environment; +mod worker_session; +pub use worker_session::{WorkerSessionSubscription, validate_full_snapshot}; pub use worker_environment::{ WorkerBinaryProviderInfo, WorkerE2eEnvironment, WorkerEnvironmentCleanupReport, @@ -1627,6 +1633,8 @@ fn resolve_yoi_binary() -> Result { "e2e-test".to_owned(), "--bin".to_owned(), "yoi".to_owned(), + "--target-dir".to_owned(), + tui_product_target_dir()?.to_string_lossy().into_owned(), ]; if test_profile() == "release" { args.push("--release".to_owned()); @@ -1648,7 +1656,9 @@ fn resolve_yoi_binary() -> Result { }); } - let binary = current_target_profile_dir()?.join(binary_name()); + let binary = tui_product_target_dir()? + .join(test_profile()) + .join(binary_name()); let info = BinaryProviderInfo { provider: "cargo-build".to_owned(), binary, @@ -1675,6 +1685,26 @@ fn workspace_root() -> Result { .ok_or_else(|| HarnessError::Protocol("could not resolve workspace root".to_owned())) } +/// Product processes must never reuse the target used by test-support builds. +fn product_target_dir() -> Result { + dedicated_product_target("e2e-products") +} + +// Existing PTY tests need their observer feature, never on restore binaries. +fn tui_product_target_dir() -> Result { + dedicated_product_target("e2e-tui-products") +} + +fn dedicated_product_target(name: &str) -> Result { + let target = workspace_root()?.join("target").join(name); + if current_target_profile_dir()?.parent() == Some(target.as_path()) { + return Err(HarnessError::Protocol( + "run support tests outside reserved product binary targets".to_owned(), + )); + } + Ok(target) +} + fn current_target_profile_dir() -> Result { let mut path = std::env::current_exe()?; while let Some(name) = path.file_name().and_then(|name| name.to_str()) { diff --git a/tests/e2e/src/worker_environment.rs b/tests/e2e/src/worker_environment.rs index 2183560cd..1fbdc4b0b 100644 --- a/tests/e2e/src/worker_environment.rs +++ b/tests/e2e/src/worker_environment.rs @@ -13,12 +13,13 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use super::{ - FixtureAllocation, HarnessError, Result, allocate_fixture, command_display, - current_target_profile_dir, now_ms, test_profile, workspace_root, yoi_binary_info, + FixtureAllocation, HarnessError, Result, allocate_fixture, command_display, now_ms, + product_target_dir, test_profile, workspace_root, }; const STARTUP_WAIT: Duration = Duration::from_secs(15); const EXIT_WAIT: Duration = Duration::from_secs(5); +const SETUP_WAIT: Duration = Duration::from_secs(15); const HTTP_WAIT_STEP: Duration = Duration::from_millis(50); const MAX_PROCESS_LOG_BYTES: usize = 256 * 1024; const MAX_COMMAND_OUTPUT_BYTES: usize = 64 * 1024; @@ -61,22 +62,20 @@ pub fn worker_binary_provider_info() -> Result { fn resolve_worker_binaries() -> Result { let workspace_root = workspace_root()?; let profile = test_profile(); - let yoi = yoi_binary_info()?; - let mut records = vec![binary_record( - "yoi", - yoi.binary, - yoi.provider, - yoi.build_command, - )?]; - + let mut records = Vec::new(); + let yoi_override = resolve_override("YOI_E2E_BIN", &workspace_root)?; let server_override = resolve_override("YOI_E2E_SERVER_BIN", &workspace_root)?; let runtime_override = resolve_override("YOI_E2E_RUNTIME_BIN", &workspace_root)?; let mut build_command = None; - if server_override.is_none() || runtime_override.is_none() { + if yoi_override.is_none() || server_override.is_none() || runtime_override.is_none() { let cargo = PathBuf::from(std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into())); let mut args = vec![ "build".to_string(), "-p".to_string(), + "yoi".to_string(), + "--bin".to_string(), + "yoi".to_string(), + "-p".to_string(), "yoi-workspace-server".to_string(), "--bin".to_string(), "yoi-server".to_string(), @@ -84,6 +83,8 @@ fn resolve_worker_binaries() -> Result { "worker-runtime".to_string(), "--bin".to_string(), "yoi-runtime".to_string(), + "--target-dir".to_string(), + product_target_dir()?.to_string_lossy().into_owned(), ]; if profile == "release" { args.push("--release".to_string()); @@ -106,7 +107,16 @@ fn resolve_worker_binaries() -> Result { build_command = Some(rendered); } - let profile_dir = current_target_profile_dir()?; + let profile_dir = product_target_dir()?.join(&profile); + let (yoi_path, yoi_provider, yoi_build) = match yoi_override { + Some(path) => (path, "YOI_E2E_BIN".to_string(), None), + None => ( + profile_dir.join(executable_name("yoi")), + "cargo-build".to_string(), + build_command.clone(), + ), + }; + records.push(binary_record("yoi", yoi_path, yoi_provider, yoi_build)?); let (server_path, server_provider, server_build) = match server_override { Some(path) => (path, "YOI_E2E_SERVER_BIN".to_string(), None), None => ( @@ -196,6 +206,17 @@ fn executable_name(name: &str) -> String { format!("{name}{}", std::env::consts::EXE_SUFFIX) } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum WorkerProcessShutdownOutcome { + #[default] + NotStarted, + Succeeded, + Failed { + error: String, + }, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct WorkerEnvironmentCleanupReport { pub fixture_root: PathBuf, @@ -204,6 +225,10 @@ pub struct WorkerEnvironmentCleanupReport { pub cleanup_success: bool, pub fixture_root_exists_after: bool, pub cleanup_error: Option, + pub runtime_shutdown: WorkerProcessShutdownOutcome, + pub server_shutdown: WorkerProcessShutdownOutcome, + pub root_cleanup_success: bool, + pub root_cleanup_error: Option, pub report_path: PathBuf, } @@ -231,6 +256,10 @@ pub struct WorkerE2eEnvironment { pub runtime_url: Option, server: Option, runtime: Option, + runtime_shutdown: WorkerProcessShutdownOutcome, + server_shutdown: WorkerProcessShutdownOutcome, + shutdown_error: Option, + runtime_public_identity: Option, access_token: Option, client: Client, binaries: WorkerBinaryProviderInfo, @@ -292,6 +321,10 @@ impl WorkerE2eEnvironment { runtime_url: None, server: None, runtime: None, + runtime_shutdown: WorkerProcessShutdownOutcome::NotStarted, + server_shutdown: WorkerProcessShutdownOutcome::NotStarted, + shutdown_error: None, + runtime_public_identity: None, access_token: None, client, binaries, @@ -319,8 +352,9 @@ impl WorkerE2eEnvironment { "Backend-only restart requires a running Server and no Runtime".to_string(), )); } - if let Some(mut server) = self.server.take() { - server.shutdown(EXIT_WAIT)?; + shutdown_process(&mut self.server, &mut self.server_shutdown); + if let WorkerProcessShutdownOutcome::Failed { error } = &self.server_shutdown { + return Err(HarnessError::Protocol(error.clone())); } self.server_url = None; self.start_server_process() @@ -374,6 +408,47 @@ impl WorkerE2eEnvironment { Ok(()) } + /// Start a store populated by the Backend's typed test-support API. The seed + /// supplies fixed catalog IDs, but the bearer must be freshly issued for this + /// environment. No database mutation or fixture credential import occurs here. + pub fn start_seeded_backend( + &mut self, + workspace_id: String, + runtime_id: String, + access_token: String, + ) -> Result<()> { + if self.server.is_some() || self.runtime.is_some() || self.workspace_id.is_some() { + return Err(HarnessError::Protocol( + "fixture seed requires an unstarted environment".into(), + )); + } + if workspace_id.is_empty() || runtime_id.is_empty() || access_token.is_empty() { + return Err(HarnessError::Protocol( + "fixture seed omitted an identity or fresh bearer".into(), + )); + } + if !self.server_data_dir.join("server.db").is_file() { + return Err(HarnessError::Protocol( + "typed Backend seed has not created server.db".into(), + )); + } + if self.runtime_public_identity.is_some() { + if self.runtime_id != runtime_id { + return Err(HarnessError::Protocol( + "seeded Runtime ID differs from initialized identity".into(), + )); + } + } else { + assert_absent_or_empty(&self.runtime_data_dir, "Runtime data directory")?; + } + self.workspace_id = Some(workspace_id); + self.runtime_id = runtime_id; + self.access_token = Some(access_token); + self.start_server_process()?; + self.export_workspace_public_identity()?; + self.write_metadata("seeded-backend-running") + } + pub fn start_backend(&mut self) -> Result<()> { self.start_empty_backend()?; let server_binary = self.binaries.binary("yoi-server")?; @@ -417,7 +492,15 @@ impl WorkerE2eEnvironment { "create Workspace", )?; let workspace_id = json_string(&created, "/workspace/workspace_id")?; - self.workspace_id = Some(workspace_id.clone()); + self.workspace_id = Some(workspace_id); + self.export_workspace_public_identity()?; + self.write_metadata("backend-running")?; + Ok(()) + } + + fn export_workspace_public_identity(&self) -> Result<()> { + let server_url = self.required_server_url()?; + let workspace_id = self.required_workspace_id()?; let identity = self.request_json( self.client.get(format!( "{server_url}/api/w/{workspace_id}/settings/signing-identity" @@ -429,17 +512,27 @@ impl WorkerE2eEnvironment { })?; let bundle_path = self.root.join("workspace-public-identity.json"); fs::write(&bundle_path, serde_json::to_vec_pretty(&public_bundle)?)?; - self.write_metadata("backend-running")?; Ok(()) } fn start_runtime(&mut self) -> Result<()> { - let bundle_path = self.root.join("workspace-public-identity.json"); - let server_url = self.required_server_url()?.to_string(); - let workspace_id = self.required_workspace_id()?.to_string(); - let runtime_binary = self.binaries.binary("yoi-runtime")?; + self.start_runtime_with_seed(|_| Ok(()))?; + self.assert_connected_and_empty() + } + + /// Generate fresh keys before offline Backend registration when fixed IDs are + /// required. Returns only the public bundle; no fixture keys are accepted. + pub fn initialize_runtime_identity(&mut self, runtime_id: String) -> Result { + if self.runtime_public_identity.is_some() || self.runtime.is_some() { + return Err(HarnessError::Protocol( + "Runtime identity is already initialized".into(), + )); + } + assert_absent_or_empty(&self.runtime_data_dir, "Runtime data directory")?; + self.runtime_id = runtime_id; + let binary = self.binaries.binary("yoi-runtime")?; self.run_command( - &runtime_binary, + &binary, &[ "identity", "init", @@ -450,8 +543,8 @@ impl WorkerE2eEnvironment { ], false, )?; - let runtime_identity = self.run_command( - &runtime_binary, + let identity = self.run_command( + &binary, &[ "identity", "show", @@ -461,7 +554,31 @@ impl WorkerE2eEnvironment { ], false, )?; - let runtime_identity: Value = serde_json::from_slice(&runtime_identity.stdout)?; + let identity: Value = serde_json::from_slice(&identity.stdout)?; + self.runtime_public_identity = Some(identity.clone()); + Ok(identity) + } + + /// Initialize fresh Runtime keys/trust, then invoke the typed Runtime seed + /// before starting its process. The actual Backend URL is now available. + pub fn start_runtime_with_seed( + &mut self, + seed: impl FnOnce(&Self) -> Result<()>, + ) -> Result<()> { + if self.runtime.is_some() { + return Err(HarnessError::Protocol("Runtime is already running".into())); + } + if self.runtime_public_identity.is_none() { + self.initialize_runtime_identity(self.runtime_id.clone())?; + } + let runtime_identity = self + .runtime_public_identity + .clone() + .expect("initialized identity"); + let bundle_path = self.root.join("workspace-public-identity.json"); + let server_url = self.required_server_url()?.to_string(); + let workspace_id = self.required_workspace_id()?.to_string(); + let runtime_binary = self.binaries.binary("yoi-runtime")?; self.run_command( &runtime_binary, &[ @@ -497,6 +614,8 @@ impl WorkerE2eEnvironment { )); } + seed(self)?; + self.write_metadata("runtime-seeded")?; let mut command = self.isolated_command(&runtime_binary); command.args([ "--bind", @@ -519,22 +638,38 @@ impl WorkerE2eEnvironment { self.runtime = Some(runtime); self.runtime_url = Some(runtime_url.clone()); + let registered = self.request_json( + self.client + .get(format!("{server_url}/api/w/{workspace_id}/runtimes")), + "observe Runtime registration before endpoint update", + )?; + let expected_binding_id = registered + .get("items") + .and_then(Value::as_array) + .and_then(|items| { + items.iter().find(|item| { + item.get("runtime_id").and_then(Value::as_str) == Some(self.runtime_id.as_str()) + }) + }) + .and_then(|item| item.pointer("/management/binding/binding_id")) + .cloned(); self.request_json( self.client .post(format!("{server_url}/api/w/{workspace_id}/runtimes")) .json(&serde_json::json!({ "public_bundle": runtime_identity, "display_name": format!("E2E {}", self.runtime_id), - "endpoint": runtime_url + "endpoint": runtime_url, + "expected_binding_id": expected_binding_id, })), "register Runtime", )?; - self.assert_connected_and_empty()?; + self.assert_connected()?; self.write_metadata("running")?; Ok(()) } - pub fn assert_connected_and_empty(&self) -> Result<()> { + pub fn assert_connected(&self) -> Result<()> { let server_url = self.required_server_url()?; let workspace_id = self.required_workspace_id()?; // connection testは実署名要求。保存済みの接続flagを認証成功の代わりにしない。 @@ -545,47 +680,125 @@ impl WorkerE2eEnvironment { )), "verify Runtime connection", )?; - if connection.get("status").and_then(Value::as_str) != Some("compatible") { - return Err(HarnessError::Protocol(format!( - "Runtime connection was not compatible: {connection}" - ))); - } + require_compatible_connection(&connection)?; let binding_id = json_string(&connection, "/binding_id")?; let runtimes = self.request_json( self.client .get(format!("{server_url}/api/w/{workspace_id}/runtimes")), "list Runtimes", )?; - let connected = runtimes - .get("items") - .and_then(Value::as_array) - .is_some_and(|items| { - items.iter().any(|item| { - item.get("runtime_id").and_then(Value::as_str) == Some(self.runtime_id.as_str()) - && item - .pointer("/management/binding/binding_id") - .and_then(Value::as_str) - == Some(binding_id.as_str()) - }) - }); - if !connected { - return Err(HarnessError::Protocol(format!( - "Runtime {} is not verified in {runtimes}", - self.runtime_id - ))); - } - let workers = self.request_json( + require_registered_runtime(&runtimes, &self.runtime_id, &binding_id) + } + + pub fn list_workers(&self) -> Result { + let server_url = self.required_server_url()?; + let workspace_id = self.required_workspace_id()?; + self.request_json( self.client .get(format!("{server_url}/api/w/{workspace_id}/workers")), "list Workers", - )?; - if workers - .get("items") - .and_then(Value::as_array) - .is_none_or(|items| !items.is_empty()) - { + ) + } + + pub fn assert_connected_and_empty(&self) -> Result<()> { + self.assert_connected()?; + let workers = self.list_workers()?; + require_empty_worker_list(&workers) + } + + fn worker_url(&self, worker_id: &str, suffix: &str) -> Result { + let mut url = reqwest::Url::parse(self.required_server_url()?) + .map_err(|error| HarnessError::Protocol(format!("invalid Server URL: {error}")))?; + url.path_segments_mut() + .map_err(|_| HarnessError::Protocol("Server URL has no path segments".into()))? + .extend([ + "api", + "w", + self.required_workspace_id()?, + "runtimes", + &self.runtime_id, + "workers", + worker_id, + ]); + if !suffix.is_empty() { + url.path_segments_mut().unwrap().extend(suffix.split('/')); + } + Ok(url.into()) + } + + pub fn worker_detail(&self, worker_id: &str) -> Result { + self.request_json( + self.client.get(self.worker_url(worker_id, "")?), + "read Worker detail", + ) + } + + /// Exactly one restore submission. No retries, replacement token or new intent. + pub fn restore_worker( + &self, + worker_id: &str, + request_id: &str, + expected_observation_token: &str, + ) -> Result { + self.request_json( + self.client + .post(self.worker_url(worker_id, "restore")?) + .timeout(STARTUP_WAIT) + .json(&serde_json::json!({ + "request_id": request_id, + "expected_observation_token": expected_observation_token, + })), + "restore Worker", + ) + } + + pub fn stop_worker(&self, worker_id: &str) -> Result { + self.request_json( + self.client + .post(self.worker_url(worker_id, "stop")?) + .json(&serde_json::json!({})), + "stop Worker", + ) + } + + /// Authorization stays in memory; callers must not persist or debug this request. + pub fn worker_protocol_request(&self, worker_id: &str) -> Result { + let token = self.access_token.as_deref().ok_or_else(|| { + HarnessError::Protocol("Backend access token is not initialized".into()) + })?; + self.client + .get(self.worker_url(worker_id, "protocol/ws")?) + .bearer_auth(token) + .build() + .map_err(|_| HarnessError::Protocol("build Worker protocol request".into())) + } + + /// The other environment must reject this environment's fresh bearer even + /// when both contain identical Workspace/Runtime/Worker IDs. Never logs tokens. + pub fn assert_authentication_isolated_from(&self, other: &Self) -> Result<()> { + let token = self + .access_token + .as_deref() + .ok_or_else(|| HarnessError::Protocol("missing source bearer".into()))?; + if other.access_token.as_deref() == Some(token) { + return Err(HarnessError::Protocol( + "environments share authentication material".into(), + )); + } + let (server, workspace) = other.backend_target()?; + let response = self + .client + .get(format!("{server}/api/w/{workspace}/workers")) + .bearer_auth(token) + .send() + .map_err(|_| { + HarnessError::Protocol("cross-environment authorization request failed".into()) + })?; + let status = response.status().as_u16(); + other.append_operation("reject other environment bearer", status)?; + if status != 401 { return Err(HarnessError::Protocol(format!( - "fresh Worker list is not empty: {workers}" + "cross-environment bearer was not rejected with 401: {status}" ))); } Ok(()) @@ -636,7 +849,7 @@ impl WorkerE2eEnvironment { "assignee": null, "readiness": null, "risk_flags": [], - "workflow_state": "planning", + "workflow_state": "open", "queued_by": null, "queued_at": null, "repository_key": null, @@ -697,30 +910,53 @@ impl WorkerE2eEnvironment { } pub fn shutdown(&mut self) -> Result<()> { - if let Some(mut runtime) = self.runtime.take() { - runtime.shutdown(EXIT_WAIT)?; + // 一方の異常でも他方を回収し、再呼出しやroot削除で失敗を消さない。 + shutdown_process(&mut self.runtime, &mut self.runtime_shutdown); + shutdown_process(&mut self.server, &mut self.server_shutdown); + let mut errors = Vec::new(); + for (name, outcome) in [ + ("Runtime", &self.runtime_shutdown), + ("Server", &self.server_shutdown), + ] { + if let WorkerProcessShutdownOutcome::Failed { error } = outcome { + errors.push(format!("{name}: {error}")); + } } - if let Some(mut server) = self.server.take() { - server.shutdown(EXIT_WAIT)?; + if self.write_metadata("stopped").is_err() { + errors.push("write stopped metadata failed".to_string()); + } + if self.shutdown_error.is_none() && !errors.is_empty() { + self.shutdown_error = Some(errors.join("; ")); + } + match &self.shutdown_error { + Some(error) => Err(HarnessError::Protocol(error.clone())), + None => Ok(()), } - self.write_metadata("stopped")?; - Ok(()) } pub fn cleanup(mut self) -> Result { - self.shutdown()?; self.cleanup_inner(true) } fn cleanup_inner(&mut self, strict: bool) -> Result { - let mut cleanup_error = None; + let _ = self.shutdown(); + let mut root_cleanup_error = None; if let Some(temp_root) = self.temp_root.take() - && let Err(error) = temp_root.close() + && temp_root.close().is_err() { - cleanup_error = Some(error.to_string()); + root_cleanup_error = Some("remove fixture root failed".to_string()); } let fixture_root_exists_after = self.root.exists(); - let cleanup_success = cleanup_error.is_none() && !fixture_root_exists_after; + if fixture_root_exists_after && root_cleanup_error.is_none() { + root_cleanup_error = Some("fixture root still exists".to_string()); + } + let root_cleanup_success = root_cleanup_error.is_none() && !fixture_root_exists_after; + let cleanup_error = match (&self.shutdown_error, &root_cleanup_error) { + (Some(shutdown), Some(root)) => Some(format!("{shutdown}; {root}")), + (Some(error), _) | (_, Some(error)) => Some(error.clone()), + (None, None) => None, + }; + let cleanup_success = self.shutdown_error.is_none() && root_cleanup_success; let report = WorkerEnvironmentCleanupReport { fixture_root: self.root.clone(), artifacts_dir: self.artifacts_dir.clone(), @@ -728,6 +964,10 @@ impl WorkerE2eEnvironment { cleanup_success, fixture_root_exists_after, cleanup_error, + runtime_shutdown: self.runtime_shutdown.clone(), + server_shutdown: self.server_shutdown.clone(), + root_cleanup_success, + root_cleanup_error, report_path: self.artifacts_dir.join("cleanup.json"), }; fs::write(&report.report_path, serde_json::to_vec_pretty(&report)?)?; @@ -773,18 +1013,8 @@ impl WorkerE2eEnvironment { let response = request .send() .map_err(|error| HarnessError::Protocol(format!("{operation}: {error}")))?; - let status = response.status(); - let body = response - .bytes() - .map_err(|error| HarnessError::Protocol(format!("{operation}: {error}")))?; - self.append_operation(operation, status.as_u16())?; - if !status.is_success() { - return Err(HarnessError::Protocol(format!( - "{operation} returned {status}: {}", - bounded_text(&body, MAX_COMMAND_OUTPUT_BYTES) - ))); - } - serde_json::from_slice(&body).map_err(HarnessError::from) + self.append_operation(operation, response.status().as_u16())?; + decode_api_response(response, operation) } fn append_operation(&self, operation: &str, status: u16) -> Result<()> { @@ -809,13 +1039,33 @@ impl WorkerE2eEnvironment { binary: &Path, args: &[&str], redact_stdout: bool, + ) -> Result { + self.run_command_with_timeout(binary, args, redact_stdout, SETUP_WAIT) + } + + fn run_command_with_timeout( + &self, + binary: &Path, + args: &[&str], + redact_stdout: bool, + timeout: Duration, ) -> Result { let args_owned = args .iter() .map(|arg| (*arg).to_string()) .collect::>(); let mut command = self.isolated_command(binary); - let output = command.args(args).output()?; + command.args(args).stdin(Stdio::null()); + // credentialを含むstdoutは呼出元へだけ返す。raw fileは作らない。 + let mut process = ManagedProcess::spawn_captured( + "setup command", + command, + None, + None, + MAX_COMMAND_OUTPUT_BYTES + 1, + )?; + let result = process.collect_output(timeout); + let output = result.as_ref().ok(); let mut log = OpenOptions::new() .create(true) .append(true) @@ -826,17 +1076,21 @@ impl WorkerE2eEnvironment { "ts_ms": now_ms(), "binary": binary, "args": args, - "status": output.status.code(), + "status": output.and_then(|output| output.status.code()), "stdout": if redact_stdout { "".to_string() } else { - bounded_text(&output.stdout, MAX_COMMAND_OUTPUT_BYTES) + output.map(|output| bounded_text(&output.stdout, MAX_COMMAND_OUTPUT_BYTES)) + .unwrap_or_default() }, - "stderr": bounded_text(&output.stderr, MAX_COMMAND_OUTPUT_BYTES), + "stderr": output.map(|output| bounded_text(&output.stderr, MAX_COMMAND_OUTPUT_BYTES)) + .unwrap_or_default(), + "error": result.as_ref().err().map(ToString::to_string), "env_clear": true, }), )?; writeln!(log)?; + let output = result?; if !output.status.success() { return Err(HarnessError::CommandFailed { program: binary.to_path_buf(), @@ -908,16 +1162,52 @@ impl WorkerE2eEnvironment { } } +// The operation is a caller-owned static label, never a response field. Refused +// response bodies may contain secrets; do not even collect them for diagnostics. +fn decode_api_response(response: reqwest::blocking::Response, operation: &str) -> Result { + let status = response.status(); + if !status.is_success() { + return Err(HarnessError::Protocol(format!( + "{operation} returned HTTP {}", + status.as_u16() + ))); + } + let mut body = std::io::Read::take(response, MAX_COMMAND_OUTPUT_BYTES as u64 + 1); + let mut bytes = Vec::new(); + std::io::Read::read_to_end(&mut body, &mut bytes) + .map_err(|_| HarnessError::Protocol(format!("{operation}: response read failed")))?; + if bytes.len() > MAX_COMMAND_OUTPUT_BYTES { + return Err(HarnessError::Protocol(format!( + "{operation}: response byte budget exceeded" + ))); + } + serde_json::from_slice(&bytes).map_err(HarnessError::from) +} + impl Drop for WorkerE2eEnvironment { fn drop(&mut self) { - if let Some(mut runtime) = self.runtime.take() { - let _ = runtime.shutdown(EXIT_WAIT); - } - if let Some(mut server) = self.server.take() { - let _ = server.shutdown(EXIT_WAIT); - } if self.temp_root.is_some() { let _ = self.cleanup_inner(false); + } else if self.runtime.is_some() || self.server.is_some() { + let _ = self.shutdown(); + } + } +} + +fn shutdown_process( + process: &mut Option, + outcome: &mut WorkerProcessShutdownOutcome, +) { + if let Some(mut process) = process.take() { + let next = match process.shutdown(EXIT_WAIT) { + Ok(_) => WorkerProcessShutdownOutcome::Succeeded, + // shutdownはstatus／OSエラーだけを返し、captureしたstdout/stderrは含めない。 + Err(error) => WorkerProcessShutdownOutcome::Failed { + error: error.to_string(), + }, + }; + if !matches!(outcome, WorkerProcessShutdownOutcome::Failed { .. }) { + *outcome = next; } } } @@ -928,15 +1218,26 @@ struct ManagedProcess { child: Child, stdout: Arc>>, stderr: Arc>>, - readers: Vec>, + readers: Vec>>, } impl ManagedProcess { - fn spawn( + fn spawn(name: &str, command: Command, stdout_path: &Path, stderr_path: &Path) -> Result { + Self::spawn_captured( + name, + command, + Some(stdout_path.to_path_buf()), + Some(stderr_path.to_path_buf()), + MAX_PROCESS_LOG_BYTES, + ) + } + + fn spawn_captured( name: &str, mut command: Command, - stdout_path: &Path, - stderr_path: &Path, + stdout_path: Option, + stderr_path: Option, + capture_limit: usize, ) -> Result { command.stdout(Stdio::piped()).stderr(Stdio::piped()); let mut child = command.spawn()?; @@ -951,8 +1252,8 @@ impl ManagedProcess { let stdout = Arc::new(Mutex::new(Vec::new())); let stderr = Arc::new(Mutex::new(Vec::new())); let readers = vec![ - capture_stream(stdout_pipe, stdout_path.to_path_buf(), Arc::clone(&stdout)), - capture_stream(stderr_pipe, stderr_path.to_path_buf(), Arc::clone(&stderr)), + capture_stream(stdout_pipe, stdout_path, Arc::clone(&stdout), capture_limit), + capture_stream(stderr_pipe, stderr_path, Arc::clone(&stderr), capture_limit), ]; Ok(Self { name: name.to_string(), @@ -1023,64 +1324,161 @@ impl ManagedProcess { ) } - fn shutdown(&mut self, timeout: Duration) -> Result { - if let Some(status) = self.child.try_wait()? { - self.join_readers(); - return Ok(status); + fn collect_output(&mut self, timeout: Duration) -> Result { + let deadline = Instant::now() + timeout; + let result = self.wait_until(deadline); + let status = match result { + Ok(Some(status)) => status, + other => { + // timeoutでも所有childをkillしてreapする。出力は診断へコピーしない。 + let _ = self.kill_and_reap(EXIT_WAIT); + let _ = self.join_readers_until(Instant::now() + EXIT_WAIT); + return match other { + Err(error) => Err(error), + _ => Err(HarnessError::Protocol( + "setup command deadline exceeded".into(), + )), + }; + } + }; + // child終了後もpipeが保持される場合を、同じsetup deadlineで失敗にする。 + self.join_readers_until(deadline)?; + let stdout = self + .stdout + .lock() + .map_err(|_| HarnessError::Protocol("setup stdout capture poisoned".into()))? + .clone(); + let stderr = self + .stderr + .lock() + .map_err(|_| HarnessError::Protocol("setup stderr capture poisoned".into()))? + .clone(); + if stdout.len() > MAX_COMMAND_OUTPUT_BYTES || stderr.len() > MAX_COMMAND_OUTPUT_BYTES { + return Err(HarnessError::Protocol( + "setup command output byte budget exceeded".into(), + )); } - let signal_result = unsafe { libc::kill(self.child.id() as i32, libc::SIGTERM) }; - if signal_result != 0 { - return Err(io_error("send SIGTERM", std::io::Error::last_os_error())); - } - let start = Instant::now(); + Ok(std::process::Output { + status, + stdout, + stderr, + }) + } + + fn wait_until(&mut self, deadline: Instant) -> Result> { loop { if let Some(status) = self.child.try_wait()? { - self.join_readers(); - if status.success() { - return Ok(status); - } - return Err(HarnessError::Protocol(format!( - "{} did not complete graceful shutdown: {status}", - self.name - ))); + return Ok(Some(status)); } - if start.elapsed() >= timeout { - let _ = self.child.kill(); - let _ = self.child.wait(); - self.join_readers(); - return Err(HarnessError::Protocol(format!( - "{} did not exit within {timeout:?}", - self.name - ))); + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Ok(None); } - thread::sleep(Duration::from_millis(20)); + thread::sleep(remaining.min(Duration::from_millis(20))); } } - fn join_readers(&mut self) { - for reader in self.readers.drain(..) { - let _ = reader.join(); + fn kill_and_reap(&mut self, timeout: Duration) -> Result { + if let Ok(Some(status)) = self.child.try_wait() { + return Ok(status); } + // killと自然終了の競合ではtry_waitで回収を確認する。 + let _ = self.child.kill(); + self.wait_until(Instant::now() + timeout)?.ok_or_else(|| { + HarnessError::Protocol(format!( + "{} could not be reaped within {timeout:?}", + self.name + )) + }) + } + + fn shutdown(&mut self, timeout: Duration) -> Result { + let result = (|| { + if let Some(status) = self.child.try_wait()? { + return Ok(status); + } + let signal_result = unsafe { libc::kill(self.child.id() as i32, libc::SIGTERM) }; + if signal_result != 0 { + return Err(io_error("send SIGTERM", std::io::Error::last_os_error())); + } + self.wait_until(Instant::now() + timeout)?.ok_or_else(|| { + HarnessError::Protocol(format!("{} did not exit within {timeout:?}", self.name)) + }) + })(); + if result.is_err() { + let _ = self.kill_and_reap(EXIT_WAIT); + } + let readers = self.join_readers_until(Instant::now() + EXIT_WAIT); + let status = result?; + readers?; + self.require_successful_exit(status) + } + + fn require_successful_exit(&self, status: ExitStatus) -> Result { + if status.success() { + Ok(status) + } else { + Err(HarnessError::Protocol(format!( + "{} did not complete graceful shutdown: {status}", + self.name + ))) + } + } + + fn join_readers_until(&mut self, deadline: Instant) -> Result<()> { + let mut failed = false; + while !self.readers.is_empty() { + let mut index = 0; + while index < self.readers.len() { + if self.readers[index].is_finished() { + let reader = self.readers.swap_remove(index); + if !matches!(reader.join(), Ok(Ok(()))) { + failed = true; + } + } else { + index += 1; + } + } + if self.readers.is_empty() { + break; + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + // stdのblocking Readは取消せない。異常なpipe保持時はthreadをdetachし、 + // 呼出元の待機だけをboundedにする。子孫processの回収までは保証しない。 + self.readers.clear(); + return Err(HarnessError::Protocol(format!( + "{} output readers did not finish before deadline", + self.name + ))); + } + thread::sleep(remaining.min(Duration::from_millis(20))); + } + if failed { + return Err(HarnessError::Protocol(format!( + "{} output reader failed", + self.name + ))); + } + Ok(()) } } impl Drop for ManagedProcess { fn drop(&mut self) { - if self.child.try_wait().ok().flatten().is_none() { - let _ = self.child.kill(); - let _ = self.child.wait(); - } - self.join_readers(); + let _ = self.kill_and_reap(EXIT_WAIT); + let _ = self.join_readers_until(Instant::now() + EXIT_WAIT); } } fn capture_stream( mut stream: impl Read + Send + 'static, - path: PathBuf, + path: Option, shared: Arc>>, -) -> JoinHandle<()> { + capture_limit: usize, +) -> JoinHandle> { thread::spawn(move || { - let mut file = File::create(path).ok(); + let mut file = path.and_then(|path| File::create(path).ok()); let mut written = 0usize; let mut truncated = false; let mut buffer = [0u8; 4096]; @@ -1089,12 +1487,12 @@ fn capture_stream( Ok(0) => break, Ok(count) => count, Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, - Err(_) => break, + Err(error) => return Err(error), }; if let Ok(mut captured) = shared.lock() - && captured.len() < MAX_PROCESS_LOG_BYTES + && captured.len() < capture_limit { - let remaining = MAX_PROCESS_LOG_BYTES - captured.len(); + let remaining = capture_limit - captured.len(); captured.extend_from_slice(&buffer[..count.min(remaining)]); } if let Some(file) = file.as_mut() { @@ -1110,6 +1508,7 @@ fn capture_stream( } } } + Ok(()) }) } @@ -1145,12 +1544,55 @@ fn assert_absent_or_empty(path: &Path, label: &str) -> Result<()> { ))) } -fn json_string(value: &Value, pointer: &str) -> Result { +fn require_compatible_connection(connection: &Value) -> Result<()> { + if connection.get("status").and_then(Value::as_str) != Some("compatible") { + return Err(HarnessError::Protocol( + "Runtime connection was not compatible".to_owned(), + )); + } + Ok(()) +} + +fn require_registered_runtime(runtimes: &Value, runtime_id: &str, binding_id: &str) -> Result<()> { + let connected = runtimes + .get("items") + .and_then(Value::as_array) + .is_some_and(|items| { + items.iter().any(|item| { + item.get("runtime_id").and_then(Value::as_str) == Some(runtime_id) + && item + .pointer("/management/binding/binding_id") + .and_then(Value::as_str) + == Some(binding_id) + }) + }); + if !connected { + return Err(HarnessError::Protocol( + "Runtime registration did not match the verified binding".to_owned(), + )); + } + Ok(()) +} + +fn require_empty_worker_list(workers: &Value) -> Result<()> { + if workers + .get("items") + .and_then(Value::as_array) + .is_none_or(|items| !items.is_empty()) + { + return Err(HarnessError::Protocol( + "fresh Worker list is not empty or has invalid shape".to_owned(), + )); + } + Ok(()) +} + +fn json_string(value: &Value, pointer: &'static str) -> Result { value .pointer(pointer) .and_then(Value::as_str) .map(str::to_owned) - .ok_or_else(|| HarnessError::Protocol(format!("JSON response omitted {pointer}: {value}"))) + .ok_or_else(|| HarnessError::Protocol(format!("JSON response omitted string at {pointer}"))) } fn url_port(url: &str) -> Result { @@ -1175,7 +1617,357 @@ fn io_error(operation: &str, error: std::io::Error) -> HarnessError { #[cfg(test)] mod startup_address_tests { - use super::parse_address_after; + use super::*; + + // 製品binary providerを呼ばず、同じ所有・収集・cleanup経路へ合成childだけを接続する。 + fn synthetic_environment(artifacts_dir: &Path) -> Result { + let temp_root = tempfile::tempdir()?; + let root = temp_root.path().to_path_buf(); + fs::create_dir_all(artifacts_dir)?; + Ok(WorkerE2eEnvironment { + temp_root: Some(temp_root), + home: root.clone(), + xdg_data_home: root.clone(), + xdg_state_home: root.clone(), + xdg_config_home: root.clone(), + xdg_runtime_dir: root.clone(), + server_data_dir: root.join("server"), + runtime_data_dir: root.join("runtime"), + server_config: root.join("server.toml"), + artifacts_dir: artifacts_dir.to_path_buf(), + workspace_id: None, + runtime_id: "synthetic-runtime".into(), + account_handle: "synthetic-account".into(), + server_url: None, + runtime_url: None, + server: None, + runtime: None, + runtime_shutdown: WorkerProcessShutdownOutcome::NotStarted, + server_shutdown: WorkerProcessShutdownOutcome::NotStarted, + shutdown_error: None, + runtime_public_identity: None, + access_token: None, + client: Client::builder().no_proxy().build().unwrap(), + binaries: WorkerBinaryProviderInfo { + workspace_root: root.clone(), + profile: "synthetic".into(), + binaries: Vec::new(), + }, + root, + }) + } + + fn synthetic_process(root: &Path, name: &str, script: &str) -> Result { + let mut command = Command::new("/bin/sh"); + command + .env_clear() + .current_dir(root) + .stdin(Stdio::null()) + .args(["-c", script]); + ManagedProcess::spawn_captured(name, command, None, None, MAX_COMMAND_OUTPUT_BYTES + 1) + } + + fn assert_secret_free(diagnostic: impl std::fmt::Debug) { + assert!(!format!("{diagnostic:?}").contains("private-test-marker")); + } + + #[test] + fn setup_stdout_is_returned_but_redacted_in_logs_and_nonzero_errors() -> Result<()> { + let artifacts = tempfile::tempdir()?; + let environment = synthetic_environment(artifacts.path())?; + // script自体はsecretの完全値を含めず、stdoutの漏洩だけを検出する。 + let script = "printf 'private-%s' test-marker"; + let output = environment.run_command(Path::new("/bin/sh"), &["-c", script], true)?; + assert_eq!(output.stdout, b"private-test-marker"); + let error = environment + .run_command( + Path::new("/bin/sh"), + &["-c", "printf 'private-%s' test-marker; exit 7"], + true, + ) + .unwrap_err(); + assert!( + matches!(&error, HarnessError::CommandFailed { status, stdout, .. } + if status.code() == Some(7) && stdout == "") + ); + assert_secret_free(&error); + assert_secret_free(error.to_string()); + let log = fs::read_to_string(artifacts.path().join("setup-commands.jsonl"))?; + assert_secret_free(&log); + for line in log.lines() { + let record: Value = serde_json::from_str(line)?; + assert_eq!(record["stdout"], ""); + } + // setup経路はraw stdout/stderr fileを作らない。 + assert_eq!(fs::read_dir(artifacts.path())?.count(), 1); + let report = environment.cleanup()?; + assert!(report.cleanup_success && report.root_cleanup_success); + Ok(()) + } + + #[test] + fn setup_deadline_kills_and_reaps_owned_child_without_logging_stdout() -> Result<()> { + let artifacts = tempfile::tempdir()?; + let environment = synthetic_environment(artifacts.path())?; + let script = "trap '' TERM; printf 'private-%s' test-marker; while :; do :; done"; + let timeout = Duration::from_millis(100); + let mut process = synthetic_process(&environment.root, "setup command", script)?; + let started = Instant::now(); + let error = process.collect_output(timeout).unwrap_err(); + assert!(error.to_string().contains("deadline exceeded")); + assert_secret_free(&error); + assert!(started.elapsed() < timeout + EXIT_WAIT + EXIT_WAIT); + assert!(process.child.try_wait()?.is_some()); + assert!(process.readers.is_empty()); + let error = environment + .run_command_with_timeout(Path::new("/bin/sh"), &["-c", script], true, timeout) + .unwrap_err(); + assert_secret_free(&error); + let log = fs::read_to_string(artifacts.path().join("setup-commands.jsonl"))?; + assert_secret_free(&log); + let record: Value = serde_json::from_str(log.trim())?; + assert_eq!(record["stdout"], ""); + assert!( + record["error"] + .as_str() + .unwrap() + .contains("deadline exceeded") + ); + assert_eq!(fs::read_dir(artifacts.path())?.count(), 1); + Ok(()) + } + + #[test] + fn setup_output_overflow_is_bounded_drained_and_rejected() -> Result<()> { + let root = tempfile::tempdir()?; + let mut process = synthetic_process( + root.path(), + "setup command", + "i=0; while [ \"$i\" -lt 20000 ]; do printf 0123456789abcdef; printf fedcba9876543210 >&2; i=$((i + 1)); done", + )?; + let error = process.collect_output(SETUP_WAIT).unwrap_err(); + assert!(error.to_string().contains("output byte budget exceeded")); + // 両pipeを上限後もdrainしなければこの成功exitまで到達しない。 + assert!(process.child.try_wait()?.unwrap().success()); + assert!(process.readers.is_empty()); + assert_eq!( + process.stdout.lock().unwrap().len(), + MAX_COMMAND_OUTPUT_BYTES + 1 + ); + assert_eq!( + process.stderr.lock().unwrap().len(), + MAX_COMMAND_OUTPUT_BYTES + 1 + ); + assert_eq!(fs::read_dir(root.path())?.count(), 0); + Ok(()) + } + + #[test] + fn setup_output_at_byte_limit_is_returned_without_truncation() -> Result<()> { + let root = tempfile::tempdir()?; + let mut process = synthetic_process( + root.path(), + "setup command", + "i=0; while [ \"$i\" -lt 4096 ]; do printf 0123456789abcdef; i=$((i + 1)); done", + )?; + let output = process.collect_output(SETUP_WAIT)?; + assert!(output.status.success()); + assert_eq!(output.stdout.len(), MAX_COMMAND_OUTPUT_BYTES); + assert!( + output + .stdout + .chunks_exact(16) + .all(|chunk| chunk == b"0123456789abcdef") + ); + assert!(output.stderr.is_empty()); + Ok(()) + } + + #[test] + fn failed_output_reader_rejects_successful_child_without_raw_error() -> Result<()> { + struct FailedReader; + impl Read for FailedReader { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + Err(std::io::Error::other("private-test-marker")) + } + } + let root = tempfile::tempdir()?; + let mut process = synthetic_process(root.path(), "setup command", "exit 0")?; + process.readers.push(capture_stream( + FailedReader, + None, + Arc::new(Mutex::new(Vec::new())), + MAX_COMMAND_OUTPUT_BYTES + 1, + )); + let error = process.collect_output(SETUP_WAIT).unwrap_err(); + assert!(error.to_string().contains("output reader failed")); + assert_secret_free(&error); + assert_secret_free(error.to_string()); + assert!(process.child.try_wait()?.unwrap().success()); + Ok(()) + } + + #[test] + fn unfinished_output_reader_has_bounded_join_and_is_not_success() -> Result<()> { + let root = tempfile::tempdir()?; + let mut process = synthetic_process(root.path(), "setup command", "exit 0")?; + assert!( + process + .wait_until(Instant::now() + EXIT_WAIT)? + .unwrap() + .success() + ); + // 子孫を残さず、pipe保持と同じ未完readerをchannelで決定的に再現する。 + let (release, pending) = std::sync::mpsc::channel::<()>(); + let (finished, completion) = std::sync::mpsc::channel(); + process.readers.push(thread::spawn(move || { + let _ = pending.recv(); + let _ = finished.send(()); + Ok(()) + })); + let timeout = Duration::from_millis(50); + let started = Instant::now(); + let error = process.collect_output(timeout).unwrap_err(); + drop(release); + completion.recv_timeout(EXIT_WAIT).unwrap(); + assert!( + error + .to_string() + .contains("readers did not finish before deadline") + ); + assert!(started.elapsed() < timeout + EXIT_WAIT); + assert!(process.readers.is_empty()); + Ok(()) + } + + #[test] + fn runtime_failure_still_reaps_live_server_and_survives_repeated_shutdown() -> Result<()> { + let artifacts = tempfile::tempdir()?; + let mut environment = synthetic_environment(artifacts.path())?; + let mut runtime = synthetic_process(&environment.root, "runtime", "exit 7")?; + assert_eq!( + runtime + .wait_until(Instant::now() + EXIT_WAIT)? + .unwrap() + .code(), + Some(7) + ); + environment.runtime = Some(runtime); + let server = synthetic_process( + &environment.root, + "server", + "trap 'exit 0' TERM; printf 'ready 127.0.0.1:12345\\n'; while :; do :; done", + )?; + server.wait_for_address("ready ", EXIT_WAIT)?; + environment.server = Some(server); + for _ in 0..2 { + assert!(environment.shutdown().is_err()); + assert!(environment.runtime.is_none() && environment.server.is_none()); + assert!(matches!( + environment.runtime_shutdown, + WorkerProcessShutdownOutcome::Failed { .. } + )); + assert_eq!( + environment.server_shutdown, + WorkerProcessShutdownOutcome::Succeeded + ); + } + let root = environment.root.clone(); + assert!(environment.cleanup().is_err()); + assert!(!root.exists()); + let report: WorkerEnvironmentCleanupReport = + serde_json::from_slice(&fs::read(artifacts.path().join("cleanup.json"))?)?; + assert!(!report.cleanup_success); + assert!(report.root_cleanup_success); + assert!(report.cleanup_error.is_some()); + Ok(()) + } + + #[test] + fn cleanup_and_drop_keep_process_failures_separate_from_root_cleanup() -> Result<()> { + for explicit_cleanup in [false, true] { + for (runtime_code, server_code) in [(0, 0), (7, 0), (0, 9), (7, 9)] { + let artifacts = tempfile::tempdir()?; + let mut environment = synthetic_environment(artifacts.path())?; + for (name, code) in [("runtime", runtime_code), ("server", server_code)] { + let mut process = synthetic_process( + &environment.root, + name, + &format!("printf 'private-%s' test-marker; exit {code}"), + )?; + assert_eq!( + process + .wait_until(Instant::now() + EXIT_WAIT)? + .unwrap() + .code(), + Some(code) + ); + if name == "runtime" { + environment.runtime = Some(process); + } else { + environment.server = Some(process); + } + } + let root = environment.root.clone(); + let expected_success = runtime_code == 0 && server_code == 0; + if explicit_cleanup { + assert_eq!(environment.cleanup().is_ok(), expected_success); + } else { + drop(environment); + } + let raw_report = fs::read_to_string(artifacts.path().join("cleanup.json"))?; + assert_secret_free(&raw_report); + let report: WorkerEnvironmentCleanupReport = serde_json::from_str(&raw_report)?; + assert_eq!(report.cleanup_success, expected_success); + assert_eq!(report.cleanup_error.is_none(), expected_success); + assert!(report.root_cleanup_success); + assert!(report.root_cleanup_error.is_none()); + assert!(!report.fixture_root_exists_after && !root.exists()); + for (outcome, code) in [ + (report.runtime_shutdown, runtime_code), + (report.server_shutdown, server_code), + ] { + if code == 0 { + assert_eq!(outcome, WorkerProcessShutdownOutcome::Succeeded); + } else { + assert!(matches!( + outcome, + WorkerProcessShutdownOutcome::Failed { .. } + )); + } + } + } + } + Ok(()) + } + + #[test] + fn shutdown_rejects_nonzero_exit_even_when_process_already_exited() -> super::Result<()> { + let root = tempfile::tempdir()?; + for code in [0, 7] { + let mut command = std::process::Command::new("/bin/sh"); + command + .env_clear() + .current_dir(root.path()) + .args(["-c", &format!("exit {code}")]); + let mut process = super::ManagedProcess::spawn( + "already-exited", + command, + &root.path().join(format!("{code}.stdout")), + &root.path().join(format!("{code}.stderr")), + )?; + // Reap first to deterministically exercise the initial try_wait branch. + assert_eq!(process.child.wait()?.code(), Some(code)); + let result = process.shutdown(std::time::Duration::from_secs(1)); + if code == 0 { + assert!(result?.success()); + } else { + assert!(matches!(result, Err(super::HarnessError::Protocol(message)) + if message.contains("did not complete graceful shutdown"))); + } + } + Ok(()) + } #[test] fn startup_address_waits_for_complete_valid_socket_record() { @@ -1205,3 +1997,107 @@ mod startup_address_tests { } } } + +#[cfg(test)] +mod response_diagnostic_tests { + use super::*; + use serde_json::json; + use std::io::{Read, Write}; + use std::net::TcpListener; + + fn assert_secret_free(error: &HarnessError) { + for diagnostic in [error.to_string(), format!("{error:?}")] { + assert!(!diagnostic.contains("private-test-marker"), "{diagnostic}"); + } + assert!(std::error::Error::source(error).is_none()); + } + + #[test] + fn json_decode_errors_do_not_retain_rejected_fields_values_or_enum_names() { + #[derive(Debug, Deserialize)] + #[serde(deny_unknown_fields)] + struct Document { + #[allow(dead_code)] + count: u64, + } + for raw in [ + r#"{"count":"private-test-marker"}"#, + r#"{"count":1,"private-test-marker":true}"#, + ] { + let error: HarnessError = serde_json::from_str::(raw).unwrap_err().into(); + assert_secret_free(&error); + } + let raw = serde_json::json!({"kind":"private-test-marker"}); + let error: HarnessError = serde_json::from_value::(raw) + .unwrap_err() + .into(); + assert_secret_free(&error); + } + + #[test] + fn valid_json_contract_failures_do_not_retain_response_fields_or_values() -> Result<()> { + let response: Value = serde_json::from_str( + r#"{"status":"private-test-marker","items":[{"private-test-marker":"private-test-marker"}],"binding_id":{ "private-test-marker":true }}"#, + )?; + for error in [ + require_compatible_connection(&response).unwrap_err(), + require_registered_runtime(&response, "runtime-test", "binding-test").unwrap_err(), + require_empty_worker_list(&response).unwrap_err(), + json_string(&response, "/binding_id").unwrap_err(), + json_string(&response, "/workspace/workspace_id").unwrap_err(), + ] { + assert_secret_free(&error); + } + require_compatible_connection(&json!({"status":"compatible"}))?; + require_registered_runtime( + &json!({"items":[{"runtime_id":"runtime-test","management":{"binding":{"binding_id":"binding-test"}}}]}), + "runtime-test", + "binding-test", + )?; + require_empty_worker_list(&json!({"items":[]}))?; + assert_eq!( + json_string(&json!({"id":"worker-test"}), "/id")?, + "worker-test" + ); + Ok(()) + } + + #[test] + fn http_failure_bodies_and_malformed_success_bodies_are_not_diagnostics() -> Result<()> { + for (status, body) in [ + ("400 Bad Request", "private-test-marker"), + ("200 OK", "private-test-marker"), + ] { + let listener = TcpListener::bind("127.0.0.1:0")?; + let address = listener.local_addr()?; + let server = thread::spawn(move || -> std::io::Result<()> { + let (mut connection, _) = listener.accept()?; + connection.set_read_timeout(Some(Duration::from_secs(3)))?; + let mut request = [0; 4096]; + connection.read(&mut request)?; + write!( + connection, + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + )?; + Ok(()) + }); + let response = Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(3)) + .build() + .unwrap() + .get(format!("http://{address}")) + .send() + .unwrap(); + let error = decode_api_response(response, "test response").unwrap_err(); + server.join().unwrap()?; + assert_secret_free(&error); + if status.starts_with("400") { + assert!(error.to_string().contains("HTTP 400")); + } + } + Ok(()) + } +} diff --git a/tests/e2e/src/worker_session.rs b/tests/e2e/src/worker_session.rs new file mode 100644 index 000000000..fb2a454e4 --- /dev/null +++ b/tests/e2e/src/worker_session.rs @@ -0,0 +1,284 @@ +//! Live public Worker protocol assertions. No Submit/Resume/model request is sent. +use std::net::TcpStream; +use std::time::{Duration, Instant}; + +use protocol::{Event, SessionSnapshot, WorkerState}; +use tungstenite::{Message, WebSocket, client::IntoClientRequest}; + +use crate::{HarnessError, Result, WorkerE2eEnvironment}; + +const WS_WAIT: Duration = Duration::from_secs(10); + +pub struct WorkerSessionSubscription { + socket: WebSocket, +} + +impl WorkerSessionSubscription { + pub fn connect( + environment: &WorkerE2eEnvironment, + worker_id: &str, + expected: &SessionSnapshot, + ) -> Result { + let authorized = environment.worker_protocol_request(worker_id)?; + let mut url = authorized.url().clone(); + url.set_scheme("ws") + .map_err(|_| error("invalid protocol URL"))?; + let mut request = url + .as_str() + .into_client_request() + .map_err(|_| error("build protocol handshake"))?; + request.headers_mut().extend(authorized.headers().clone()); + // Only this environment's loopback endpoint is permitted, never a proxy. + let address = url + .socket_addrs(|| None) + .map_err(|_| error("resolve protocol address"))? + .into_iter() + .find(|address| address.ip().is_loopback()) + .ok_or_else(|| error("protocol address is not loopback"))?; + let stream = TcpStream::connect_timeout(&address, WS_WAIT)?; + stream.set_read_timeout(Some(WS_WAIT))?; + stream.set_write_timeout(Some(WS_WAIT))?; + let (socket, _) = + tungstenite::client(request, stream).map_err(|_| error("protocol handshake failed"))?; + let mut subscription = Self { socket }; + let deadline = Instant::now() + WS_WAIT; + loop { + match subscription.read(deadline)? { + Message::Text(text) => { + let raw: serde_json::Value = serde_json::from_str(&text)?; + validate_full_snapshot(&raw, expected)?; + // Only validated public content is retained, never request headers. + std::fs::write( + environment.artifacts_dir.join("restored-session.json"), + serde_json::to_vec_pretty(&raw)?, + )?; + return Ok(subscription); + } + Message::Ping(_) => subscription + .socket + .flush() + .map_err(|_| error("flush protocol Pong"))?, + Message::Pong(_) => {} + _ => return Err(error("protocol ended before its initial full Snapshot")), + } + } + } + + /// Prove an already-open subscription remains usable after the other + /// environment is stopped and cleaned up. This does not invoke the model. + pub fn assert_alive(&mut self) -> Result<()> { + let payload = b"restore-e2e-isolation".to_vec(); + self.socket + .send(Message::Ping(payload.clone().into())) + .map_err(|_| error("send protocol Ping"))?; + let deadline = Instant::now() + WS_WAIT; + loop { + match self.read(deadline)? { + Message::Pong(bytes) if bytes.as_ref() == payload.as_slice() => return Ok(()), + Message::Ping(_) => self + .socket + .flush() + .map_err(|_| error("flush protocol Pong"))?, + Message::Pong(_) => {} + _ => { + return Err(error( + "idle subscription changed or ended during isolation check", + )); + } + } + } + } + + /// A public stop must terminate the connected execution, not merely update + /// a retained catalog row. Require the shutdown event or websocket close. + pub fn expect_stopped(mut self) -> Result<()> { + let deadline = Instant::now() + WS_WAIT; + loop { + match self.read(deadline)? { + Message::Close(_) => return Ok(()), + Message::Text(text) => match serde_json::from_str::(&text)? { + Event::Shutdown => return Ok(()), + Event::WorkerState { .. } => {} + _ => return Err(error("unexpected protocol event after Worker stop")), + }, + Message::Ping(_) => self + .socket + .flush() + .map_err(|_| error("flush protocol Pong"))?, + Message::Pong(_) => {} + _ => return Err(error("unexpected protocol frame after Worker stop")), + } + } + } + + fn read(&mut self, deadline: Instant) -> Result { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|duration| !duration.is_zero()) + .ok_or_else(|| error("protocol deadline exceeded"))?; + self.socket.get_mut().set_read_timeout(Some(remaining))?; + self.socket + .read() + .map_err(|_| error("protocol read failed or timed out")) + } +} + +fn error(message: &str) -> HarnessError { + HarnessError::Protocol(message.into()) +} + +pub fn validate_full_snapshot(raw: &serde_json::Value, expected: &SessionSnapshot) -> Result<()> { + let event: Event = serde_json::from_value(raw.clone())?; + // Product DTOs intentionally tolerate unknown fields for compatibility. This + // fixture boundary does not: anything discarded by typed decoding must never + // be persisted to an artifact, including fields outside the Session payload. + if serde_json::to_value(&event)? != *raw { + return Err(error( + "noncanonical or unknown fields in full protocol Snapshot envelope", + )); + } + let Event::Snapshot { + session, + state, + in_flight, + internal_workers, + .. + } = event + else { + return Err(error("first protocol event is not a full Snapshot")); + }; + if state.state != WorkerState::Idle || !in_flight.is_empty() || !internal_workers.is_empty() { + return Err(error( + "restored Worker is not an idle, completed standalone Session", + )); + } + // Typed equality covers entry order/ID/timestamp/provenance/content; raw + // equality also refuses unknown fields that could leak private data. + if session != *expected + || raw.pointer("/data/session") != Some(&serde_json::to_value(expected)?) + { + return Err(error( + "full Session snapshot differs from supplied expected rows", + )); + } + if !session.pending_submissions.submissions.is_empty() { + return Err(error("fixture has pending submissions")); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use protocol::{ + Greeting, InFlightSnapshot, Segment, SessionEntryProvenance, SessionSnapshotEntry, + SessionSnapshotEntryData, WorkerStateSnapshot, WorkerStatus, + }; + + // Protocol-validation samples only, not a substitute saved Worker fixture. + fn sample() -> (serde_json::Value, SessionSnapshot) { + let session = SessionSnapshot { + pending_submissions: Default::default(), + entries: vec![SessionSnapshotEntry { + entry_id: "E-unit-1".into(), + timestamp: 123, + provenance: SessionEntryProvenance::HumanInput, + derived_from: vec![], + use_policy: None, + data: SessionSnapshotEntryData::UserInput { + segments: vec![Segment::text("unit input")], + }, + }], + }; + let event = Event::Snapshot { + session: session.clone(), + state: WorkerStateSnapshot::from(WorkerStatus::Idle), + in_flight: InFlightSnapshot::default(), + internal_workers: vec![], + greeting: Greeting { + worker_name: "unit".into(), + cwd: ".".into(), + provider: "unit".into(), + model: "unit".into(), + reasoning: None, + scope_summary: "none".into(), + tools: vec![], + context_window: 0, + context_tokens: 0, + context_usage: None, + }, + }; + (serde_json::to_value(event).unwrap(), session) + } + + #[test] + fn full_snapshot_requires_exact_ids_timestamps_contents_and_no_unknown_fields() { + let (raw, expected) = sample(); + validate_full_snapshot(&raw, &expected).unwrap(); + for (key, value) in [ + ("entry_id", serde_json::json!("different")), + ("timestamp", serde_json::json!(124)), + ( + "hidden_reasoning", + serde_json::json!("must never be exposed"), + ), + ("segments", serde_json::json!([])), + ] { + let mut changed = raw.clone(); + changed["data"]["session"]["entries"][0][key] = value; + assert!( + validate_full_snapshot(&changed, &expected).is_err(), + "{key}" + ); + } + let mut missing = raw.clone(); + missing["data"]["session"]["entries"] = serde_json::json!([]); + assert!(validate_full_snapshot(&missing, &expected).is_err()); + let mut extra = raw; + let entry = extra["data"]["session"]["entries"][0].clone(); + extra["data"]["session"]["entries"] + .as_array_mut() + .unwrap() + .push(entry); + assert!(validate_full_snapshot(&extra, &expected).is_err()); + } + + #[test] + fn full_snapshot_rejects_unknown_fields_outside_session_before_artifact_write() { + let (raw, expected) = sample(); + for pointer in [ + "", + "/data", + "/data/greeting", + "/data/state", + "/data/state/state", + ] { + let mut changed = raw.clone(); + changed + .pointer_mut(pointer) + .unwrap() + .as_object_mut() + .unwrap() + .insert( + "hidden_reasoning".into(), + serde_json::json!("must not reach artifact"), + ); + assert!( + validate_full_snapshot(&changed, &expected).is_err(), + "{pointer}" + ); + } + let mut changed = raw; + changed["data"]["greeting"]["private_key"] = serde_json::json!("must not reach artifact"); + assert!(validate_full_snapshot(&changed, &expected).is_err()); + } + + #[test] + fn non_snapshot_event_cannot_stand_in_for_live_snapshot() { + let (_, expected) = sample(); + assert!( + validate_full_snapshot(&serde_json::to_value(Event::Shutdown).unwrap(), &expected) + .is_err() + ); + } +} diff --git a/tests/e2e/tests/panel.rs b/tests/e2e/tests/panel.rs index 058e27827..87636bb62 100644 --- a/tests/e2e/tests/panel.rs +++ b/tests/e2e/tests/panel.rs @@ -41,13 +41,13 @@ fn backend_panel_loads_backend_owned_tickets_and_supports_reload_navigation_and_ &event.data, &ready_ticket_id, "Planning E2E Ticket A", - "planning", + "open", ) && has_backend_ticket( &event.data, &planning_ticket_id, "Planning E2E Ticket B", - "planning", + "open", ) }, )?; @@ -82,13 +82,13 @@ fn backend_panel_loads_backend_owned_tickets_and_supports_reload_navigation_and_ &event.data, &ready_ticket_id, "Planning E2E Ticket A", - "planning", + "open", ) && has_backend_ticket( &event.data, &planning_ticket_id, "Planning E2E Ticket B", - "planning", + "open", ) }, )?; @@ -137,7 +137,7 @@ fn backend_panel_shell_enter_path_uses_the_same_backend_authority() -> yoi_e2e:: &event.data, &ready_ticket_id, "Planning E2E Ticket A", - "planning", + "open", ) }, )?; diff --git a/tests/e2e/tests/worker_restore.rs b/tests/e2e/tests/worker_restore.rs new file mode 100644 index 000000000..3d1619847 --- /dev/null +++ b/tests/e2e/tests/worker_restore.rs @@ -0,0 +1,274 @@ +//! Fixed, externally supplied Worker restore. Never generates a fixture or sends model input. +#[path = "worker_restore/fixture.rs"] +mod fixture; +#[path = "worker_restore/seed.rs"] +mod seed; + +use std::fs; +use std::time::{Duration, Instant}; + +use fixture::{Fixture, failure}; +use seed::Installed; +use server_api::{ + RuntimeWorkerLifecycleResult, RuntimeWorkerShowResponse, WorkerListResponse, + WorkerOperationState, WorkerRestoreResponse, WorkerRestoreState, WorkerSummary, +}; +use yoi_e2e::{FixtureSource, Result, WorkerE2eEnvironment, WorkerSessionSubscription}; + +#[test] +fn supplied_stopped_worker_restores_exact_live_session_and_cleans_up() -> Result<()> { + let fixture = Fixture::load()?; + let result = (|| { + let installed = seed::start(&fixture)?; + let subscription = restore(&installed, &fixture, "fixed-fixture-restore-single")?; + stop_and_cleanup(installed, vec![subscription], &fixture) + })(); + fixture.source.assert_unchanged()?; + result +} + +#[test] +fn concurrent_fixed_id_restores_keep_sessions_storage_and_cleanup_isolated() -> Result<()> { + let fixture = Fixture::load()?; + let result = (|| { + let (first, second) = std::thread::scope(|scope| { + let first = scope.spawn(|| seed::start(&fixture)); + let second = seed::start(&fixture); + ( + first + .join() + .map_err(|_| failure("first environment startup panicked")) + .and_then(|result| result), + second, + ) + }); + let first = first?; + let second = second?; + assert_eq!( + first.environment.workspace_id, + second.environment.workspace_id + ); + assert_eq!(first.environment.runtime_id, second.environment.runtime_id); + assert_ne!(first.environment.root, second.environment.root); + assert_ne!( + first.environment.server_data_dir, + second.environment.server_data_dir + ); + assert_ne!( + first.environment.runtime_data_dir, + second.environment.runtime_data_dir + ); + assert_ne!( + first.environment.artifacts_dir, + second.environment.artifacts_dir + ); + assert_ne!( + first.environment.account_handle, + second.environment.account_handle + ); + assert_ne!(first.runtime_public_key, second.runtime_public_key); + assert_ne!(first.workspace_public_key, second.workspace_public_key); + assert_ne!( + first.environment.server_port()?, + second.environment.server_port()? + ); + assert_ne!( + first.environment.runtime_port()?, + second.environment.runtime_port()? + ); + assert_ne!( + first.environment.server_pid(), + second.environment.server_pid() + ); + assert_ne!( + first.environment.runtime_pid(), + second.environment.runtime_pid() + ); + first + .environment + .assert_authentication_isolated_from(&second.environment)?; + second + .environment + .assert_authentication_isolated_from(&first.environment)?; + let (first_subscription, second_subscription) = std::thread::scope(|scope| { + // Deliberately reuse request identity in isolated stores, just like fixed IDs. + let a = scope.spawn(|| restore(&first, &fixture, "fixed-fixture-restore-concurrent")); + let b = restore(&second, &fixture, "fixed-fixture-restore-concurrent"); + ( + a.join() + .map_err(|_| failure("first restore panicked")) + .and_then(|result| result), + b, + ) + }); + let first_subscription = first_subscription?; + let mut second_subscription = second_subscription?; + let second_session_root = second + .environment + .runtime_data_dir + .join("runtime/workers") + .join(&fixture.manifest.worker_id) + .join("session"); + let second_session = FixtureSource::capture(&second_session_root)?; + let first_root = first.environment.root.clone(); + let first_artifacts = first.environment.artifacts_dir.clone(); + stop_and_cleanup(first, vec![first_subscription], &fixture)?; + assert!(!first_root.exists()); + assert!(first_artifacts.join("cleanup.json").is_file()); + assert!(second.environment.root.is_dir()); + assert!(second.environment.server_data_dir.is_dir()); + assert!(second.environment.runtime_data_dir.is_dir()); + assert!( + second + .environment + .artifacts_dir + .join("restored-session.json") + .is_file() + ); + second.environment.assert_connected()?; + assert_single_state(&second.environment, &fixture, "idle")?; + second_subscription.assert_alive()?; + // A second live connection independently verifies content after peer cleanup. + let mut reconnected = WorkerSessionSubscription::connect( + &second.environment, + &fixture.manifest.worker_id, + &second.snapshot, + )?; + reconnected.assert_alive()?; + second_session.assert_unchanged()?; + stop_and_cleanup(second, vec![second_subscription, reconnected], &fixture)?; + Ok(()) + })(); + fixture.source.assert_unchanged()?; + result +} + +fn restore( + installed: &Installed, + fixture: &Fixture, + request_id: &str, +) -> Result { + phase(&installed.environment, "observe-stopped")?; + let stopped = assert_single_state(&installed.environment, fixture, "stopped")?; + let token = stopped + .restore_observation_token + .as_deref() + .filter(|token| !token.is_empty()) + .ok_or_else(|| failure("stopped Worker did not expose a restore observation token"))?; + phase(&installed.environment, "restore")?; + let result: WorkerRestoreResponse = serde_json::from_value( + installed + .environment + .restore_worker(&fixture.manifest.worker_id, request_id, token)?, + )?; + if result.workspace_id != fixture.manifest.workspace_id + || result.runtime_id != fixture.manifest.runtime_id + || result.worker_id != fixture.manifest.worker_id + || result.result.state != WorkerRestoreState::Accepted + { + return Err(failure(format!( + "restore did not commit accepted: {:?}", + result.result.state + ))); + } + assert_single_state(&installed.environment, fixture, "idle")?; + let detail: RuntimeWorkerShowResponse = serde_json::from_value( + installed + .environment + .worker_detail(&fixture.manifest.worker_id)?, + )?; + if detail.worker.worker_id != fixture.manifest.worker_id + || detail.worker.runtime_id != fixture.manifest.runtime_id + || detail.worker.state != "idle" + || detail + .worker + .worker_state + .as_ref() + .is_none_or(|state| state.state != protocol::WorkerState::Idle) + { + return Err(failure( + "restored Worker detail is not the single live idle execution", + )); + } + phase(&installed.environment, "subscribe-live-session")?; + let mut subscription = WorkerSessionSubscription::connect( + &installed.environment, + &fixture.manifest.worker_id, + &installed.snapshot, + )?; + subscription.assert_alive()?; + phase(&installed.environment, "live-session-verified")?; + Ok(subscription) +} + +fn assert_single_state( + environment: &WorkerE2eEnvironment, + fixture: &Fixture, + state: &str, +) -> Result { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let list: WorkerListResponse = serde_json::from_value(environment.list_workers()?)?; + if list.workspace_id != fixture.manifest.workspace_id || list.items.len() > 1 { + return Err(failure( + "Workspace Worker list has wrong identity or duplicate Workers", + )); + } + if let Some(worker) = list.items.into_iter().next() { + if worker.worker_id != fixture.manifest.worker_id + || worker.runtime_id != fixture.manifest.runtime_id + { + return Err(failure("Workspace listed an unexpected Worker identity")); + } + if worker.state == state { + return Ok(worker); + } + } + if Instant::now() >= deadline { + return Err(failure(format!( + "Worker list did not become exactly one {state} Worker" + ))); + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn stop_and_cleanup( + installed: Installed, + subscriptions: Vec, + fixture: &Fixture, +) -> Result<()> { + phase(&installed.environment, "stop")?; + let result: RuntimeWorkerLifecycleResult = serde_json::from_value( + installed + .environment + .stop_worker(&fixture.manifest.worker_id)?, + )?; + if result.state != WorkerOperationState::Accepted { + return Err(failure("Worker stop was not accepted")); + } + installed.environment.assert_connected()?; + for subscription in subscriptions { + subscription.expect_stopped()?; + // A transport close caused by Server/Runtime shutdown is not stop evidence. + installed.environment.assert_connected()?; + } + assert_single_state(&installed.environment, fixture, "stopped")?; + phase(&installed.environment, "shutdown-cleanup")?; + let report = installed.environment.cleanup()?; + if !report.cleanup_success || report.fixture_root_exists_after { + return Err(failure("environment cleanup failed")); + } + if !report.report_path.is_file() { + return Err(failure("cleanup evidence is missing")); + } + Ok(()) +} + +fn phase(environment: &WorkerE2eEnvironment, phase: &str) -> Result<()> { + fs::write( + environment.artifacts_dir.join("restore-phase.json"), + serde_json::to_vec_pretty(&serde_json::json!({"phase":phase}))?, + )?; + Ok(()) +} diff --git a/tests/e2e/tests/worker_restore/fixture.rs b/tests/e2e/tests/worker_restore/fixture.rs new file mode 100644 index 000000000..707a159fe --- /dev/null +++ b/tests/e2e/tests/worker_restore/fixture.rs @@ -0,0 +1,257 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use protocol::{SessionMessageRole, SessionSnapshot, SessionSnapshotEntryData}; +use serde::Deserialize; +use serde_json::{Value, json}; +use worker_runtime::test_support::StoppedWorkerTemplate; +use yoi_e2e::{FixtureSource, HarnessError, Result}; + +pub const PROFILE: &str = "project:fixture-chat"; +const ASSETS: [&str; 3] = [ + "expected.json", + "worker-template.json", + "workspace-config.dcdl", +]; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Manifest { + pub format_version: u32, + #[serde(rename = "source_revision")] + pub source_commit: String, + pub runtime_schema_version: u32, + pub session_schema_version: u32, + pub workspace_id: String, + pub runtime_id: String, + pub worker_id: String, + pub session_id: String, + pub segment_id: String, + pub model: String, + pub state: String, + pub acquisition: String, + pub source_file_sha256: BTreeMap, + pub files: BTreeMap, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Asset { + pub bytes: u64, + pub sha256: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Expected { + #[serde(rename = "source_revision")] + source_commit: String, + model: String, + worker_id: String, + session_id: String, + rows: Vec, +} + +pub struct Fixture { + pub root: PathBuf, + pub source: FixtureSource, + pub manifest: Manifest, + expected: Expected, +} + +pub fn failure(message: impl Into) -> HarnessError { + HarnessError::Protocol(message.into()) +} +pub fn support(result: std::result::Result) -> Result { + result.map_err(|error| failure(error.to_string())) +} + +impl Fixture { + pub fn load() -> Result { + // Candidate override is explicit and read-only. Missing assets are failures, + // never skips, and no test generates or repairs a fixture. + let root = std::env::var_os("YOI_E2E_RESTORE_FIXTURE") + .map(PathBuf::from) + .map(|path| { + if path.is_absolute() { + path + } else { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join(path) + } + }) + .unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/worker-codex-v1") + }); + let file_names: Vec<_> = std::iter::once("manifest.json").chain(ASSETS).collect(); + let source = FixtureSource::capture_flat(&root, &file_names)?; + let manifest: Manifest = + serde_json::from_slice(&bounded_read(&root.join("manifest.json"))?)?; + if manifest.format_version != 1 + || manifest.state != "stopped" + || manifest.runtime_schema_version == 0 + || manifest.session_schema_version == 0 + || manifest.acquisition.is_empty() + || manifest.source_file_sha256.is_empty() + || manifest.source_commit.len() != 40 + || !manifest + .source_commit + .bytes() + .all(|c| c.is_ascii_hexdigit()) + { + return Err(failure("unsupported/incomplete fixture manifest")); + } + let names: BTreeSet<_> = manifest.files.keys().map(String::as_str).collect(); + if names != BTreeSet::from(ASSETS) { + return Err(failure("unexpected fixture asset set")); + } + for (name, asset) in &manifest.files { + if asset.bytes != fs::metadata(root.join(name))?.len() + || source.hashes().get(Path::new(name)) != Some(&format!("sha256:{}", asset.sha256)) + { + return Err(failure(format!("fixture asset hash/size mismatch: {name}"))); + } + } + let expected: Expected = + serde_json::from_slice(&bounded_read(&root.join("expected.json"))?)?; + if expected.source_commit != manifest.source_commit + || expected.model != manifest.model + || expected.worker_id != manifest.worker_id + || expected.session_id != manifest.session_id + || expected.rows.is_empty() + { + return Err(failure( + "expected conversation identity differs from manifest", + )); + } + verify_builtin_commit(&manifest.source_commit)?; + let fixture = Self { + root, + source, + manifest, + expected, + }; + fixture.template(&fixture.root)?; + fixture.source.assert_unchanged()?; + Ok(fixture) + } + + pub fn template(&self, copy: &Path) -> Result { + let bytes = bounded_read(©.join("worker-template.json"))?; + let template: StoppedWorkerTemplate = serde_json::from_slice(&bytes)?; + let shape: Value = serde_json::from_slice(&bytes)?; + let session_manifest_bytes: Vec = + serde_json::from_value(shape["session_manifest"].clone())?; + let segment_bytes: Vec = serde_json::from_value(shape["segment_jsonl"].clone())?; + for (name, bytes) in [ + ( + "session/session.json".to_owned(), + session_manifest_bytes.as_slice(), + ), + ( + format!("session/segments/{}.jsonl", self.manifest.segment_id), + segment_bytes.as_slice(), + ), + ] { + use sha2::{Digest, Sha256}; + let digest = Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + if self.manifest.source_file_sha256.get(&name) != Some(&digest) { + return Err(failure( + "exported Session bytes differ from acquisition hash evidence", + )); + } + } + let session_manifest: Value = serde_json::from_slice(&session_manifest_bytes)?; + let m = &self.manifest; + if template.workspace_id() != m.workspace_id + || template.runtime_id() != m.runtime_id + || template.worker_id().to_string() != m.worker_id + || template.session_id().to_string() != m.session_id + || template.segment_id().to_string() != m.segment_id + || shape["runtime_schema_version"] != m.runtime_schema_version + || session_manifest["schema_version"] != m.session_schema_version + { + return Err(failure("template schema/identity differs from manifest")); + } + Ok(template) + } + + pub fn verify_conversation(&self, snapshot: &SessionSnapshot) -> Result<()> { + let mut rows = Vec::new(); + for entry in &snapshot.entries { + let (role, parts) = match &entry.data { + SessionSnapshotEntryData::UserInput { segments } => { + ("user", serde_json::to_value(segments)?) + } + SessionSnapshotEntryData::Message { + role: SessionMessageRole::Assistant, + content, + } => ("assistant", serde_json::to_value(content)?), + // Display-only accounting is not a conversation row. Its exact + // IDs, timestamp and values are still checked in the full WS snapshot. + SessionSnapshotEntryData::RunStats { .. } => continue, + _ => { + return Err(failure( + "unexpected non-conversation entry in full Session snapshot", + )); + } + }; + rows.push(json!({"entry_id":entry.entry_id,"timestamp":entry.timestamp,"role":role,"parts":parts})); + } + if rows != self.expected.rows { + return Err(failure( + "full public conversation rows differ from expected IDs/time/roles/text/order", + )); + } + Ok(()) + } +} + +fn bounded_read(path: &Path) -> Result> { + FixtureSource::read_bounded(path) +} + +fn verify_builtin_commit(source_commit: &str) -> Result<()> { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap(); + // Resolve the immutable commit before diff; unavailable commits must not be treated + // as an empty diff. All resources are checked, including transitive imports. + let commit = Command::new("git") + .args(["cat-file", "-t", source_commit]) + .current_dir(workspace) + .output()?; + if !commit.status.success() || commit.stdout != b"commit\n" { + return Err(failure( + "fixture source commit is unavailable for builtin verification", + )); + } + let status = Command::new("git") + .args([ + "diff", + "--quiet", + source_commit, + "--", + "resources", + "crates/builtin-source", + ]) + .current_dir(workspace) + .status()?; + if !status.success() { + return Err(failure( + "builtin/config resources differ from fixture source commit; provide a new version or frozen references", + )); + } + Ok(()) +} diff --git a/tests/e2e/tests/worker_restore/seed.rs b/tests/e2e/tests/worker_restore/seed.rs new file mode 100644 index 000000000..afc7804e4 --- /dev/null +++ b/tests/e2e/tests/worker_restore/seed.rs @@ -0,0 +1,482 @@ +use std::fs; +use std::net::TcpListener; +use std::path::Path; +use std::sync::Arc; + +use config_source::{ConfigTreeChange, VirtualPath}; +use protocol::SessionSnapshot; +use serde_json::Value; +use worker_runtime::FsRuntimeStoreOptions; +use worker_runtime::catalog::{ + ConfigBundleRef, CreateWorkerRequest, ProfileSelector, ProfileSourceArchiveSource, + WorkspaceApiRef, +}; +use yoi_e2e::{Result, WorkerE2eEnvironment}; +use yoi_workspace_server::auth::{mint_secret, new_id, now_rfc3339, token_hash}; +use yoi_workspace_server::config_source::{ConfigCommitRequest, WorkspaceConfigSchemaRegistry}; +use yoi_workspace_server::hosts::{ + WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest, +}; +use yoi_workspace_server::profile_settings::{ + ProfileConfigSchemaProvider, build_virtual_profile_config_bundle, + project_profiles_from_workspace_config, +}; +use yoi_workspace_server::store::{ + AccountRecord, ApiTokenRecord, UserRecord, WorkspaceBootstrapRecord, + WorkspaceRuntimeAuthenticationMode, WorkspaceRuntimeBinding, +}; +use yoi_workspace_server::test_support::{FixedWorkerSeed, seed_fixed_worker}; +use yoi_workspace_server::workspace_signing_identity::{ + FsWorkspaceSigningMaterialStore, WorkspaceSigningIdentityService, + workspace_signing_material_root, +}; +use yoi_workspace_server::{ControlPlaneStore, SqliteWorkspaceStore, WorkspaceRecord}; + +use super::fixture::{Fixture, PROFILE, failure, support}; + +pub struct Installed { + pub environment: WorkerE2eEnvironment, + pub snapshot: SessionSnapshot, + pub runtime_public_key: String, + pub workspace_public_key: String, +} + +pub fn start(fixture: &Fixture) -> Result { + let mut environment = WorkerE2eEnvironment::new()?; + super::phase(&environment, "fixture-copy-verify")?; + let copy = environment.root.join("provided-fixture"); + fixture.source.copy_to(©)?; + fs::write( + environment.artifacts_dir.join("fixture-manifest.json"), + fs::read(copy.join("manifest.json"))?, + )?; + let template = fixture.template(©)?; + let runtime_identity = + environment.initialize_runtime_identity(fixture.manifest.runtime_id.clone())?; + let runtime_public_key = runtime_identity["public_key"] + .as_str() + .ok_or_else(|| failure("Runtime public bundle omitted key"))? + .to_owned(); + + // Reserve an environment-local placeholder endpoint only while the Runtime is + // offline. Its real :0 endpoint replaces this binding through public API CAS. + let reserved_endpoint = TcpListener::bind("127.0.0.1:0")?; + super::phase( + &environment, + "offline-backend-bootstrap-profile-worker-seed", + )?; + let bootstrap = bootstrap( + &environment, + fixture, + ©, + &runtime_identity, + &format!("http://{}", reserved_endpoint.local_addr()?), + &template.worker_name(), + )?; + // Close the placeholder before Backend probes it: an accepting TCP listener + // with no HTTP service would stall discovery until its transport timeout. + drop(reserved_endpoint); + super::phase(&environment, "start-seeded-backend")?; + environment.start_seeded_backend( + fixture.manifest.workspace_id.clone(), + fixture.manifest.runtime_id.clone(), + bootstrap.token, + )?; + let mut snapshot = None; + super::phase(&environment, "initialize-runtime-trust")?; + environment.start_runtime_with_seed(|environment| { + super::phase(environment, "install-runtime-fixture-and-validate-session")?; + let options = FsRuntimeStoreOptions::new(environment.runtime_data_dir.join("runtime")) + .with_runtime_id(&fixture.manifest.runtime_id); + let request = CreateWorkerRequest { + worker_id: template.worker_id(), + create_fingerprint: bootstrap.create_fingerprint.clone(), + profile: ProfileSelector::Named(PROFILE.into()), + display_name: Some("Restore E2E fixture".into()), + profile_source: ProfileSourceArchiveSource::WorkspaceConfig { + archive: bootstrap.archive.clone(), + }, + config_bundle: Some(bootstrap.config_bundle.clone()), + workspace_api: Some(WorkspaceApiRef { + workspace_id: fixture.manifest.workspace_id.clone(), + base_url: environment.backend_target()?.0.into(), + }), + initial_input: None, + workdir_attachment_requests: vec![], + workdir_attachments: vec![], + worker_observation_enabled: false, + worker_observation_grants: vec![], + memory_settings: Some(bootstrap.memory_settings.clone()), + subjektiv_attached: false, + backend_job: None, + }; + support(template.install(&options, request, &bootstrap.manifest))?; + let sessions = support(session_store::WorkerSessionStore::open_read_only( + options + .root + .join("workers") + .join(template.worker_id().to_string()) + .join("session"), + ))?; + let (logs, _) = support(sessions.read_all_read_only_bounded( + template.session_id(), + template.segment_id(), + 8 * 1024 * 1024, + ))?; + let projected = session_store::project_session_snapshot(template.session_id(), &logs); + fixture.verify_conversation(&projected)?; + snapshot = Some(full_restored_snapshot(&sessions, &logs)); + Ok(()) + })?; + fixture.source.assert_unchanged()?; + Ok(Installed { + environment, + snapshot: snapshot.ok_or_else(|| failure("Runtime seed was not invoked"))?, + runtime_public_key, + workspace_public_key: bootstrap.workspace_public_key, + }) +} + +fn full_restored_snapshot( + store: &dyn session_store::Store, + logs: &[session_store::LogEntry], +) -> SessionSnapshot { + session_store::public_snapshot::project_current_session_snapshot_with_accounting(store, logs) +} + +#[cfg(test)] +mod tests { + use super::*; + use protocol::{ + Event, Greeting, InFlightSnapshot, InvokeKind, SessionSnapshotEntryData, + WorkerStateSnapshot, WorkerStatus, + }; + use session_store::{ + LogEntry, LoggedContentPart, LoggedHistoryEntry, LoggedItem, LoggedRole, + LoggedSessionHistoryEntryId, LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, + }; + + #[test] + fn completed_run_full_snapshot_retains_invoke_and_usage_accounting() -> Result<()> { + // Projection regression only: no saved Worker or substitute E2E fixture. + let temp = tempfile::tempdir()?; + let store = support(session_store::WorkerSessionStore::new(temp.path()))?; + let session_id = session_store::new_session_id(); + let logs = vec![ + LogEntry::AnnotatedSegmentStart { + ts: 1, + session_id, + system_prompt: None, + config: Default::default(), + history: vec![], + forked_from: None, + compacted_from: None, + }, + LogEntry::Invoke { + ts: 10, + trigger: InvokeKind::UserSend, + }, + LogEntry::AnnotatedAssistantItem { + ts: 20, + entry: LoggedHistoryEntry { + item: LoggedItem::Message { + role: LoggedRole::Assistant, + content: vec![LoggedContentPart::Text { + text: "unit response".into(), + }], + }, + metadata: LoggedSessionHistoryMetadata { + entry_id: LoggedSessionHistoryEntryId("unit-response".into()), + origin: LoggedSessionHistoryOrigin::LegacyUnknown, + derivation: None, + input_metadata: None, + }, + }, + }, + LogEntry::LlmUsage { + ts: 30, + history_len: 1, + input_total_tokens: 100, + cache_read_tokens: 20, + cache_write_tokens: 0, + output_tokens: 5, + }, + serde_json::from_value(serde_json::json!({"kind":"run_completed", "ts":40, + "interrupted":false,"result":"finished"}))?, + ]; + let conversation = session_store::project_session_snapshot(session_id, &logs); + let expected = full_restored_snapshot(&store, &logs); + assert_eq!(expected.entries.len(), conversation.entries.len() + 2); + assert!(matches!( + expected.entries[expected.entries.len() - 2].data, + SessionSnapshotEntryData::Invoke { + trigger: InvokeKind::UserSend + } + )); + assert_eq!(expected.entries[expected.entries.len() - 2].timestamp, 10); + assert!(matches!( + expected.entries.last().unwrap().data, + SessionSnapshotEntryData::Usage { + input_tokens: 100, + cache_read_input_tokens: 20, + output_tokens: 5 + } + )); + assert_eq!(expected.entries.last().unwrap().timestamp, 30); + let live = session_store::public_snapshot::project_current_session_snapshot_with_accounting( + &store, &logs, + ); + let event = Event::Snapshot { + session: live, + state: WorkerStateSnapshot::from(WorkerStatus::Idle), + in_flight: InFlightSnapshot::default(), + internal_workers: vec![], + greeting: Greeting { + worker_name: "unit".into(), + cwd: ".".into(), + provider: "unit".into(), + model: "unit".into(), + reasoning: None, + scope_summary: "none".into(), + tools: vec![], + context_window: 0, + context_tokens: 0, + context_usage: None, + }, + }; + let raw = serde_json::to_value(event)?; + yoi_e2e::validate_full_snapshot(&raw, &expected)?; + assert!(yoi_e2e::validate_full_snapshot(&raw, &conversation).is_err()); + Ok(()) + } +} + +struct Bootstrap { + token: String, + workspace_public_key: String, + create_fingerprint: String, + archive: worker_runtime::profile_archive::ProfileSourceArchiveRef, + config_bundle: ConfigBundleRef, + manifest: manifest::WorkerManifest, + memory_settings: manifest::WorkspaceMemorySettingsSnapshot, +} + +fn bootstrap( + environment: &WorkerE2eEnvironment, + fixture: &Fixture, + copy: &Path, + runtime_identity: &Value, + endpoint: &str, + worker_name: &str, +) -> Result { + fs::create_dir_all(&environment.server_data_dir)?; + let database = environment.server_data_dir.join("server.db"); + if database.exists() { + return Err(failure("Backend seed refuses an existing database")); + } + let store = support(SqliteWorkspaceStore::open(&database))?; + let now = now_rfc3339(); + let account_id = new_id("acct-user"); + let user_id = new_id("user"); + support(store.upsert_account(&AccountRecord { + account_id: account_id.clone(), + kind: "user".into(), + handle: environment.account_handle.clone(), + display_name: "Restore E2E owner".into(), + created_at: now.clone(), + updated_at: now.clone(), + }))?; + support(store.upsert_user(&UserRecord { + user_id: user_id.clone(), + account_id: account_id.clone(), + handle: environment.account_handle.clone(), + display_name: "Restore E2E owner".into(), + created_at: now.clone(), + updated_at: now.clone(), + }))?; + let token = mint_secret("yoi_api"); + support(store.create_api_token(&ApiTokenRecord { + token_id: new_id("api-token"), + token_hash: token_hash(&token), + user_id, + label: "restore-e2e".into(), + created_at: now.clone(), + expires_at: None, + revoked_at: None, + last_used_at: None, + }))?; + let identities = WorkspaceSigningIdentityService::new( + Arc::new(store.clone()), + Arc::new(FsWorkspaceSigningMaterialStore::new( + workspace_signing_material_root(&database), + )), + ); + let (identity, operation) = support(identities.prepare_workspace_creation( + "restore-e2e-bootstrap", + "restore-e2e-bootstrap", + &fixture.manifest.workspace_id, + &account_id, + ))?; + let workspace_public_key = identity.public_key.clone(); + support(store.create_workspace_bootstrap( + &WorkspaceBootstrapRecord { + operation_key: "restore-e2e-bootstrap".into(), + request_fingerprint: "restore-e2e-bootstrap".into(), + workspace: WorkspaceRecord { + workspace_id: fixture.manifest.workspace_id.clone(), + owner_account_id: account_id, + display_name: "Restore E2E".into(), + state: "active".into(), + created_at: now.clone(), + updated_at: now.clone(), + }, + repository: None, + }, + &identity, + &operation, + ))?; + support( + store.upsert_workspace_runtime_binding( + WorkspaceRuntimeBinding { + workspace_id: fixture.manifest.workspace_id.clone(), + runtime_id: fixture.manifest.runtime_id.clone(), + display_name: format!("E2E {}", fixture.manifest.runtime_id), + base_url: endpoint.into(), + public_key: runtime_identity["public_key"] + .as_str() + .ok_or_else(|| failure("missing Runtime public key"))? + .into(), + public_key_fingerprint: String::new(), + binding_id: String::new(), + authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity, + created_at: now.clone(), + updated_at: now.clone(), + revoked_at: None, + }, + false, + ), + )?; + + let schema = support( + WorkspaceConfigSchemaRegistry::default() + .with_provider(Arc::new(ProfileConfigSchemaProvider)) + .with_provider(Arc::new( + yoi_workspace_server::prompt_settings::PromptConfigSchemaProvider, + )) + .with_provider(Arc::new( + yoi_workspace_server::runtime_settings::RuntimeConfigSchemaProvider, + )) + .with_provider(Arc::new( + yoi_workspace_server::repository_access::RepositoryAccessConfigSchemaProvider, + )) + .with_provider(Arc::new( + yoi_workspace_server::skills::SkillConfigSchemaProvider, + )) + .compose(), + )?; + let state = support(store.ensure_workspace_config_materialized_with_schema( + &fixture.manifest.workspace_id, + &now, + schema.clone(), + ))?; + let main = support(VirtualPath::parse("main.dcdl"))?; + let current = state + .snapshot + .get(&main) + .ok_or_else(|| failure("Workspace config omitted main.dcdl"))?; + let candidate = support(store.evaluate_workspace_config_candidate_with_schema( + &fixture.manifest.workspace_id, + &ConfigCommitRequest { + base_digest: state.snapshot.digest.clone(), + changes: vec![ConfigTreeChange::Update { + path: main.clone(), + expected_digest: current.content_digest.clone(), + content: fs::read_to_string(copy.join("workspace-config.dcdl"))?, + }], + entrypoints: vec![main], + }, + schema, + ))?; + let state = support( + store.commit_evaluated_workspace_config(&fixture.manifest.workspace_id, &candidate), + )?; + let projection = support(project_profiles_from_workspace_config( + &fixture.manifest.workspace_id, + &state, + ))?; + let bundle = support(build_virtual_profile_config_bundle( + &projection, + &state, + &fixture.manifest.workspace_id, + &now, + PROFILE, + ))? + .ok_or_else(|| failure("fixture Profile is not registered"))?; + let archive = bundle + .profile_source_archive + .as_ref() + .ok_or_else(|| failure("fixture Profile archive is absent"))?; + let mut resolved = support( + support(archive.verify())?.resolve_profile( + PROFILE, + &environment + .runtime_data_dir + .join("runtime/workers") + .join(&fixture.manifest.worker_id), + worker_name, + ), + )?; + if resolved.model.ref_.as_deref() != Some(fixture.manifest.model.as_str()) { + return Err(failure("resolved Profile changed the supplied model")); + } + let request = WorkerSpawnRequest { + intent: WorkerSpawnIntent::WorkspaceCoding, + requested_worker_name: Some("Restore E2E fixture".into()), + singleton_key: None, + acceptance: WorkerSpawnAcceptanceRequirement::SocketReady, + profile: ProfileSelector::Named(PROFILE.into()), + ticket_assignment: None, + initial_submit: vec![], + workdir_attachment_requests: vec![], + resolved_workdir_attachment_requests: vec![], + resolved_workdir_attachments: vec![], + resolved_config_bundle: None, + resolved_workspace_api: None, + resolved_memory_settings: None, + resolved_subjektiv_attached: false, + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: vec![], + resolved_control_operation: None, + resolved_execution_grant: None, + }; + let reservation = support(seed_fixed_worker( + &store, + FixedWorkerSeed { + workspace_id: &fixture.manifest.workspace_id, + runtime_id: &fixture.manifest.runtime_id, + worker_id: support(fixture.manifest.worker_id.parse())?, + allocation_key: "restore-e2e-worker", + request: &request, + }, + ))?; + if resolved.feature.memory.profile.enabled { + support( + resolved + .feature + .memory + .bind_workspace_settings(reservation.memory_settings.clone()), + )?; + } + Ok(Bootstrap { + manifest: resolved, + memory_settings: reservation.memory_settings, + token, + workspace_public_key, + create_fingerprint: reservation.create_fingerprint, + archive: archive.reference.clone(), + config_bundle: ConfigBundleRef { + id: bundle.metadata.id, + digest: bundle.metadata.digest, + }, + }) +} diff --git a/tools/state-contract-name-exceptions.json b/tools/state-contract-name-exceptions.json index 933d7a696..bebae656f 100644 --- a/tools/state-contract-name-exceptions.json +++ b/tools/state-contract-name-exceptions.json @@ -1,4 +1,37 @@ [ + { + "path": "tests/e2e/fixtures/worker-codex-v1/manifest.json", + "kind": "saved_data", + "reason": "ユーザー提供済v1固定資産の生成元Git commitの保存key。原本の取得証拠とSHA-256を保持するため変更しない。Worker実行状態や競合制御の番号ではなく、生成元resources照合のみに使う。", + "lines": [ + { + "sha256": "fbf8f92576eb6afbb03502d2f95144651373ce8cd7b4401bc06f801fdb4db28d", + "count": 1 + } + ] + }, + { + "path": "tests/e2e/fixtures/worker-codex-v1/expected.json", + "kind": "saved_data", + "reason": "ユーザー提供済v1固定資産の生成元Git commitの保存key。原本の取得証拠とSHA-256を保持するため変更しない。Worker実行状態や競合制御の番号ではなく、生成元resources照合のみに使う。", + "lines": [ + { + "sha256": "fbf8f92576eb6afbb03502d2f95144651373ce8cd7b4401bc06f801fdb4db28d", + "count": 1 + } + ] + }, + { + "path": "tests/e2e/tests/worker_restore/fixture.rs", + "kind": "saved_data", + "reason": "提供済v1原本の生成元Git commitを読むための固定保存key対応。原本bytesとhashを変更しない。実行状態の観測tokenや更新番号に利用せず、現行Rust内ではsource_commitとして扱う。", + "lines": [ + { + "sha256": "4dd4810d1a31b812596b9d6cc8a8b27f398476174dfa6f0c3c1881aeadb5ade4", + "count": 2 + } + ] + }, { "path": "crates/subjektiv/src/journal/memory_links/tests.rs", "kind": "saved_data", From a77453338cf3d23c3ff886cf537c3cdf092edd15 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 09:16:36 +0900 Subject: [PATCH 06/28] feat: mcp retain stdio process ownership before initialization Allow startup cleanup to own and reap the exact spawned client across handshake failure or cancellation. --- crates/mcp/src/stdio.rs | 267 +++++++++++++++++++++++++++------------- 1 file changed, 184 insertions(+), 83 deletions(-) diff --git a/crates/mcp/src/stdio.rs b/crates/mcp/src/stdio.rs index a3eb426b3..174afd401 100644 --- a/crates/mcp/src/stdio.rs +++ b/crates/mcp/src/stdio.rs @@ -617,7 +617,8 @@ impl fmt::Display for McpRequestStage { } } -/// A running initialized stdio MCP client. +/// An owned stdio MCP process, initialized by `connect` or explicitly after +/// `spawn_uninitialized` when startup cleanup must retain it before any await. pub struct McpStdioClient { server_name: String, limits: McpStdioLimits, @@ -636,6 +637,9 @@ pub struct McpStdioClient { impl McpStdioClient { /// Spawn, initialize, negotiate capabilities, and send notifications/initialized. + /// Initialization errors attempt shutdown, but this convenience API cannot + /// return an unreaped client on failure/cancellation. Its error is not proof + /// of cleanup; use `spawn_uninitialized` with retained ownership when needed. pub async fn connect( spec: McpStdioServerSpec, limits: McpStdioLimits, @@ -667,6 +671,21 @@ impl McpStdioClient { async fn spawn( spec: McpStdioServerSpec, limits: McpStdioLimits, + ) -> Result { + Self::spawn_uninitialized(spec, limits) + } + + /// Spawn without awaiting the handshake, so a cleanup owner can retain the + /// client before initialization can fail or be cancelled. Call + /// `initialize_with_timeout` before discovery; on error/cancellation, shut + /// down this same client rather than spawning a replacement. + /// + /// Once the process spawns, this returns its owner even if an expected pipe + /// is missing. Initialization then fails, but cleanup can still reap the + /// actual child rather than relying on kill-on-drop for a post-spawn error. + pub fn spawn_uninitialized( + spec: McpStdioServerSpec, + limits: McpStdioLimits, ) -> Result { let redactor = spec.redactor(); let mut command = Command::new(&spec.command); @@ -688,29 +707,23 @@ impl McpStdioClient { McpErrorKind::Io(redactor.redact(&err.to_string())), ) })?; - let stdin = child.stdin.take().ok_or_else(|| { - McpClientError::new( - &spec.name, - McpPhase::Spawn, - McpErrorKind::Protocol("child stdin was not piped".into()), - ) - })?; - let stdout = child.stdout.take().ok_or_else(|| { - McpClientError::new( - &spec.name, - McpPhase::Spawn, - McpErrorKind::Protocol("child stdout was not piped".into()), - ) - })?; - let stderr = child.stderr.take().ok_or_else(|| { - McpClientError::new( - &spec.name, - McpPhase::Spawn, - McpErrorKind::Protocol("child stderr was not piped".into()), - ) - })?; + let stdin = child.stdin.take(); + let stdout = child.stdout.take(); + let stderr = child.stderr.take(); + let missing_pipe = if stdin.is_none() { + Some("stdin") + } else if stdout.is_none() { + Some("stdout") + } else if stderr.is_none() { + Some("stderr") + } else { + None + }; - let stdin: SharedStdin = Arc::new(Mutex::new(Some(Box::new(stdin)))); + let stdin: SharedStdin = + Arc::new(Mutex::new(stdin.map(|stdin| { + Box::new(stdin) as Box + }))); let diagnostics = Arc::new(Mutex::new(BoundedDiagnostics::new( spec.name.clone(), limits.max_diagnostic_lines, @@ -718,16 +731,32 @@ impl McpStdioClient { ))); let list_changes = Arc::new(Mutex::new(BoundedListChanged::new(spec.name.clone()))); let (tx, rx) = mpsc::channel(16); - let reader_task = spawn_stdout_reader( - spec.name.clone(), - stdout, - stdin.clone(), - tx, - limits.clone(), - redactor.clone(), - list_changes.clone(), - ); - let stderr_task = spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone()); + // Do not return an error that discards an already spawned child. Queue + // setup failure before reader events so initialization cannot succeed. + if let Some(pipe) = missing_pipe { + tx.try_send(ReaderEvent::Error(McpClientError::new( + &spec.name, + McpPhase::Spawn, + McpErrorKind::Protocol(format!("child {pipe} was not piped")), + ))) + .expect("new response channel has capacity"); + } + let reader_task = match stdout { + Some(stdout) => spawn_stdout_reader( + spec.name.clone(), + stdout, + stdin.clone(), + tx, + limits.clone(), + redactor.clone(), + list_changes.clone(), + ), + None => tokio::spawn(async {}), + }; + let stderr_task = match stderr { + Some(stderr) => spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone()), + None => tokio::spawn(async {}), + }; Ok(Self { server_name: spec.name, @@ -746,6 +775,25 @@ impl McpStdioClient { }) } + /// Perform the bounded handshake on an already retained client. Failure or + /// cancellation does not drop or shut down the client: its owner must do so. + /// Do not retry initialization on a partially initialized connection. + pub async fn initialize_with_timeout(&mut self) -> Result<(), McpClientError> { + let started = Instant::now(); + match timeout(self.limits.startup_timeout, self.initialize()).await { + Ok(result) => result, + Err(_) => Err(McpClientError::new( + &self.server_name, + McpPhase::Initialize, + McpErrorKind::Timeout { + operation: "startup".to_string(), + elapsed: started.elapsed(), + }, + ) + .with_diagnostics(self.snapshot_diagnostics().await)), + } + } + async fn initialize(&mut self) -> Result<(), McpClientError> { let result: InitializeResult = self .request( @@ -1080,6 +1128,8 @@ impl McpStdioClient { } /// Close stdin and wait for process exit, falling back to terminate and kill. + /// Errors and cancellation retain the child for another attempt on this + /// client. Only confirmed reaping releases the process handle. pub async fn shutdown(&mut self) -> Result { self.shutdown_started = true; let deadline = Instant::now() + self.limits.shutdown_timeout; @@ -1091,64 +1141,55 @@ impl McpStdioClient { .await .is_ok(); - let mut child = match self.child.take() { - Some(child) => child, - None => return Ok(ShutdownReport::already_finished()), - }; + // Keep the actual process handle in the client across every await and + // error. Taking it before wait loses retry/reaping authority on cancel. + if self.child.is_none() { + return Ok(ShutdownReport::already_finished()); + } if !closed { self.reader_task.abort(); - return self.terminate_then_kill(child).await; + return self.terminate_then_kill().await; } + let child = self.child.as_mut().expect("checked child"); if let Ok(Some(status)) = child.try_wait() { - self.reader_task.abort(); - self.stderr_task.abort(); - return Ok(ShutdownReport { - exit_status: Some(status), - terminated: false, - killed: false, - }); + return Ok(self.finish_shutdown(status, false, false)); } match timeout_at(deadline, child.wait()).await { - Ok(Ok(status)) => { - self.reader_task.abort(); - self.stderr_task.abort(); - Ok(ShutdownReport { - exit_status: Some(status), - terminated: false, - killed: false, - }) - } + Ok(Ok(status)) => Ok(self.finish_shutdown(status, false, false)), Ok(Err(err)) => Err(McpClientError::new( &self.server_name, McpPhase::Shutdown, McpErrorKind::Io(self.redactor.redact(&err.to_string())), ) .with_diagnostics(self.snapshot_diagnostics().await)), - Err(_) => self.terminate_then_kill(child).await, + Err(_) => self.terminate_then_kill().await, } } - async fn terminate_then_kill( + fn finish_shutdown( &mut self, - mut child: Child, - ) -> Result { - let mut terminated = false; - let mut killed = false; - if send_terminate(&mut child).is_ok() { - terminated = true; + status: ExitStatus, + terminated: bool, + killed: bool, + ) -> ShutdownReport { + // No await separates confirmed reaping from releasing the handle. + drop(self.child.take()); + self.reader_task.abort(); + self.stderr_task.abort(); + ShutdownReport { + exit_status: Some(status), + terminated, + killed, } + } + + async fn terminate_then_kill(&mut self) -> Result { + let child = self.child.as_mut().expect("shutdown retains child"); + let terminated = send_terminate(child).is_ok(); match timeout(self.limits.kill_timeout, child.wait()).await { - Ok(Ok(status)) => { - self.reader_task.abort(); - self.stderr_task.abort(); - Ok(ShutdownReport { - exit_status: Some(status), - terminated, - killed, - }) - } + Ok(Ok(status)) => Ok(self.finish_shutdown(status, terminated, false)), Ok(Err(err)) => Err(McpClientError::new( &self.server_name, McpPhase::Shutdown, @@ -1163,7 +1204,6 @@ impl McpStdioClient { McpErrorKind::Io(self.redactor.redact(&err.to_string())), ) })?; - killed = true; let status = timeout(self.limits.kill_timeout, child.wait()) .await .map_err(|_| { @@ -1183,13 +1223,7 @@ impl McpStdioClient { McpErrorKind::Io(self.redactor.redact(&err.to_string())), ) })?; - self.reader_task.abort(); - self.stderr_task.abort(); - Ok(ShutdownReport { - exit_status: Some(status), - terminated, - killed, - }) + Ok(self.finish_shutdown(status, terminated, true)) } } } @@ -1197,10 +1231,10 @@ impl McpStdioClient { impl Drop for McpStdioClient { fn drop(&mut self) { - if !self.shutdown_started { - if let Some(child) = &mut self.child { - let _ = child.start_kill(); - } + // An attempted shutdown is not proof of exit. Errors and cancellation + // leave the child here so Drop must still make a best-effort kill. + if let Some(child) = &mut self.child { + let _ = child.start_kill(); } self.reader_task.abort(); self.stderr_task.abort(); @@ -1784,3 +1818,70 @@ fn send_terminate(child: &mut Child) -> Result<(), ()> { #[cfg(test)] mod deadline_tests; + +#[cfg(all(test, unix))] +mod shutdown_retention_tests { + //! The real client and OS process are used; only the external server is a + //! local shell that acknowledges initialization and ignores TERM/EOF. + use super::*; + use std::future::{Future, poll_fn}; + use std::task::Poll; + + #[tokio::test] + async fn cancelled_graceful_or_terminate_wait_retries_reaping_the_same_child() { + for cancel_in_fallback in [false, true] { + let script = r#"trap '' TERM +read init || exit 1 +printf '%s\n' '{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-11-25","capabilities":{},"serverInfo":{"name":"retained","version":"1"}}}' +while :; do :; done +"#; + let mut client = McpStdioClient::connect( + McpStdioServerSpec::new("retained", "/bin/sh").args(["-c", script]), + McpStdioLimits { + shutdown_timeout: if cancel_in_fallback { + Duration::ZERO + } else { + Duration::from_secs(30) + }, + kill_timeout: Duration::from_secs(30), + ..Default::default() + }, + ) + .await + .expect("initialize after installing TERM handler"); + let pid = client.child.as_ref().unwrap().id(); + let mut shutdown = Box::pin(client.shutdown()); + poll_fn(|cx| { + assert!(shutdown.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + drop(shutdown); + assert_eq!(client.child.as_ref().and_then(Child::id), pid); + assert!(client.shutdown_started); + + client.limits.shutdown_timeout = Duration::from_millis(20); + client.limits.kill_timeout = Duration::from_millis(50); + let report = timeout(Duration::from_secs(2), client.shutdown()) + .await + .expect("retry bounded") + .expect("retry reaps original child"); + assert!( + report.exit_status.is_some(), + "not an already-finished receipt" + ); + assert!(report.killed, "TERM was ignored"); + assert!(client.child.is_none()); + let mut status = 0; + assert_eq!( + unsafe { libc::waitpid(pid.unwrap() as libc::pid_t, &mut status, libc::WNOHANG) }, + -1 + ); + assert_eq!( + std::io::Error::last_os_error().raw_os_error(), + Some(libc::ECHILD), + "shutdown, not a later observer, reaped the original process" + ); + } + } +} From 848a1e9a5b5c8d366617e0dda93d7f828e59941f Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 09:16:37 +0900 Subject: [PATCH 07/28] feat: worker separate restored controller preparation from publication Prepare the real receiver, full snapshot and paired streams privately; retain activation effects and startup cleanup until the host decision. --- crates/worker/src/bootstrap.rs | 220 +++++++++++++ crates/worker/src/controller.rs | 473 ++++++++++++++++++++++++---- crates/worker/src/feature/mcp.rs | 308 ++++++++++++++++-- crates/worker/src/spawn/registry.rs | 426 ++++++++++++++++++++++--- crates/worker/src/worker.rs | 163 ++++++++-- 5 files changed, 1412 insertions(+), 178 deletions(-) diff --git a/crates/worker/src/bootstrap.rs b/crates/worker/src/bootstrap.rs index 00e6c5ddf..2a591cb3f 100644 --- a/crates/worker/src/bootstrap.rs +++ b/crates/worker/src/bootstrap.rs @@ -323,3 +323,223 @@ where } } } + +/// Host bindings for one unpublished restored Controller. These do not expose +/// Worker execution or permit arbitrary mutation during preparation. +pub struct RestoredControllerBindings { + pub backend_job: Option, + pub workdir_sessions: std::sync::Arc, + pub observation_provider: + Option>, + pub subjektiv_attached: bool, +} + +/// The same real receiver, snapshot sources and paired subscription retained +/// privately across the Runtime's durable restore decision. +pub struct UnpublishedRestoredController { + initialized: Option, St>>, + activation: Option, + streams: Option, + subjektiv_attached: bool, + effects_applied: bool, + activation_failed: bool, + closing: bool, +} + +pub struct ActivatedRestoredController { + pub controller: BootstrappedWorker, + pub streams: crate::ipc::protocol_session::WorkerProtocolSessionStreams, +} + +impl PreparedRestoredWorker +where + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, +{ + pub fn workspace_client_handle( + &mut self, + ) -> Result, WorkerError> { + Ok(self.worker_mut()?.workspace_client_handle()) + } + + pub fn retain_startup_feature_cleanup( + &mut self, + ) -> Result { + Ok(self.worker_mut()?.retain_startup_feature_cleanup()) + } + + /// Initialize the real Controller without allocation, canonical restore + /// writes, attribution changes, registry reclamation or input consumption. + /// MCP discovery and Feature tasks belong to the retained startup cleanup. + pub async fn prepare_runtime_controller( + mut self, + run_dir: &std::path::Path, + transport: WorkerControllerTransport, + bindings: RestoredControllerBindings, + ) -> Result, WorkerBootstrapError> { + let worker = self.worker_mut().map_err(WorkerBootstrapError::Worker)?; + if let Some(binding) = bindings.backend_job { + worker.bind_backend_job(binding).map_err(|error| { + WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.into())) + })?; + } + worker + .workdir_sessions() + .close_all() + .await + .map_err(|error| { + WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.to_string())) + })?; + worker.bind_workdir_sessions(bindings.workdir_sessions); + worker.bind_worker_observation_provider(bindings.observation_provider); + let flow_enabled = worker.manifest().feature.flow.enabled; + if flow_enabled { + let report = worker + .install_runtime_flow_transition_feature() + .map_err(|error| { + WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error)) + })?; + if report.reports.iter().any(|report| !report.installed) { + return Err(WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle( + "Flow transition installation refused".into(), + ))); + } + } + let (worker, activation) = self + .into_controller_parts() + .map_err(WorkerBootstrapError::Worker)?; + let initialized = WorkerController::initialize_restored(worker, run_dir, transport) + .await + .map_err(|source| WorkerBootstrapError::Controller { + source, + cleanup_failed: true, + })?; + if flow_enabled { + initialized.handle.shared_state.enable_flow_transition(); + } + let streams = { + let _guard = initialized.handle.protocol_snapshot_publish_guard(); + crate::ipc::protocol_session::subscribe_worker_protocol_session(&initialized.handle) + }; + Ok(UnpublishedRestoredController { + initialized: Some(initialized), + activation: Some(activation), + streams: Some(streams), + subjektiv_attached: bindings.subjektiv_attached, + effects_applied: false, + activation_failed: false, + closing: false, + }) + } +} + +impl UnpublishedRestoredController +where + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, +{ + pub fn snapshot(&self) -> &protocol::Event { + &self + .streams + .as_ref() + .expect("unpublished restore streams") + .snapshot_event + } + + /// Only publication failures after confirmed effects can retry on this + /// exact owner. Partial activation must be reconciled, never blindly rerun. + pub fn can_retry_publication(&self) -> bool { + self.initialized.is_some() + && self.effects_applied + && !self.activation_failed + && !self.closing + } + + /// Apply canonical restore effects after the host's durable decision, while + /// retaining the same private Controller for subsequent publication. + pub async fn apply_effects(&mut self) -> Result<(), WorkerBootstrapError> { + let initialized = self + .initialized + .as_mut() + .ok_or_else(|| WorkerBootstrapError::Worker(WorkerError::RestoreCandidateClosing))?; + if self.activation_failed || self.closing { + return Err(WorkerBootstrapError::Worker( + WorkerError::RestoreCandidateClosing, + )); + } + if !self.effects_applied { + self.activation_failed = true; + self.activation + .as_ref() + .expect("restore activation") + .activate(&mut initialized.worker) + .await + .map_err(WorkerBootstrapError::Worker)?; + initialized + .worker + .finalize_subjektiv_session_attribution( + self.subjektiv_attached, + crate::SubjektivSessionAttributionLifecycle::RestoredSession, + ) + .map_err(WorkerBootstrapError::Worker)?; + self.effects_applied = true; + self.activation_failed = false; + self.activation = None; + } + Ok(()) + } + + pub fn allocation_cleanup( + &self, + ) -> Option { + self.initialized + .as_ref() + .and_then(|initialized| initialized.worker.restore_allocation_cleanup_handle()) + } + + pub async fn activate(&mut self) -> Result { + self.apply_effects().await?; + let initialized = self + .initialized + .as_mut() + .expect("same initialized Controller"); + let publication = initialized.prepare_publication().await.map_err(|source| { + WorkerBootstrapError::Controller { + source, + cleanup_failed: true, + } + })?; + let initialized = self + .initialized + .take() + .expect("same initialized Controller"); + let (handle, shutdown, controller_task) = initialized.start(publication); + Ok(ActivatedRestoredController { + controller: BootstrappedWorker { + handle, + shutdown, + controller_task, + }, + streams: self.streams.take().expect("same paired subscription"), + }) + } + + pub async fn discard(&mut self) -> Result<(), WorkerError> { + self.closing = true; + if let Some(initialized) = self.initialized.as_mut() { + initialized + .worker + .retain_startup_feature_cleanup() + .shutdown() + .await + .map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?; + initialized + .worker + .workdir_sessions() + .close_all() + .await + .map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?; + } + self.streams = None; + self.initialized = None; + Ok(()) + } +} diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index 6721753d6..c20d5d833 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -23,7 +23,7 @@ use crate::shutdown_after_idle::{ ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role, take_shutdown_request_after_status, }; -use crate::spawn::registry::SpawnedWorkerRegistry; +use crate::spawn::registry::{PreparedSpawnedWorkerRegistryLoad, SpawnedWorkerRegistry}; use crate::spawn::tool::sub_worker_spawn_tool; use crate::worker::{ PendingSubmissionHandle, SystemItemCommitter, Worker, WorkerError, WorkerRunResult, @@ -688,6 +688,28 @@ pub enum WorkerControllerTransport { pub struct WorkerController; +// Own the real command receiver and all snapshot sources before starting any +// protocol listener or controller loop. Fresh startup may materialize the head; +// Runtime restore requires an already materialized source and defers canonical +// restore effects until its durable decision. Tool discovery can own external +// resources, so private initialization is not synonymous with side-effect-free. +pub(crate) struct InitializedWorkerController { + pub(crate) worker: Worker, + pub(crate) handle: WorkerHandle, + method_rx: mpsc::Receiver, + shutdown_tx: oneshot::Sender<()>, + shutdown_rx: ShutdownReceiver, + registry_reclaim: PreparedSpawnedWorkerRegistryLoad, + shutdown_after_idle: ShutdownAfterIdleRequest, + transport: WorkerControllerTransport, + command_events: Option, +} + +pub(crate) struct PreparedControllerPublication { + socket_server: Option, + command_observer: Option>, +} + impl WorkerController { /// Explicit recovery of an unmaterialized Session head. This installs the /// saved Feature composition needed to produce the seed, but creates no @@ -877,9 +899,8 @@ impl WorkerController { } async fn spawn_initialized( - mut worker: Worker, + worker: Worker, runtime_base: &Path, - runtime_managed: bool, runtime_run: Option<&Path>, transport: WorkerControllerTransport, @@ -888,6 +909,71 @@ impl WorkerController { C: LlmClient + Clone + 'static, St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { + Self::initialize( + worker, + runtime_base, + runtime_managed, + runtime_run, + transport, + ) + .await? + .publish() + .await + } + + async fn initialize( + worker: Worker, + runtime_base: &Path, + runtime_managed: bool, + runtime_run: Option<&Path>, + transport: WorkerControllerTransport, + ) -> Result, std::io::Error> + where + C: LlmClient + Clone + 'static, + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + Self::initialize_inner( + worker, + runtime_base, + runtime_managed, + runtime_run, + transport, + true, + ) + .await + } + + pub(crate) async fn initialize_restored( + worker: Worker, St>, + run_dir: &Path, + transport: WorkerControllerTransport, + ) -> std::io::Result, St>> + where + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + let parent = run_dir + .parent() + .ok_or_else(|| std::io::Error::other("run path has no parent"))?; + Self::initialize_inner(worker, parent, true, Some(run_dir), transport, false).await + } + + async fn initialize_inner( + mut worker: Worker, + runtime_base: &Path, + runtime_managed: bool, + runtime_run: Option<&Path>, + transport: WorkerControllerTransport, + canonical_startup: bool, + ) -> Result, std::io::Error> + where + C: LlmClient + Clone + 'static, + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + if !canonical_startup && worker.needs_initial_session_head_materialization() { + return Err(std::io::Error::other( + "restore source requires operation-owned head recovery", + )); + } worker .validate_subjektiv_body_publication() .map_err(std::io::Error::other)?; @@ -913,22 +999,13 @@ impl WorkerController { }); let spawner_name = worker.manifest().worker.name.clone(); - let self_parent_socket = worker.callback_socket().cloned(); - let loaded_registry = SpawnedWorkerRegistry::load_from_worker_state_with_reclaim( + let registry_reclaim = SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim( runtime_dir.clone(), worker.store().clone(), spawner_name.clone(), Some(worker.scope().clone()), - ) - .await?; - let reclaimed_unreachable = loaded_registry.reclaimed_unreachable; - let spawned_registry = loaded_registry.registry; - if reclaimed_unreachable { - worker.push_notify( - "Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume." - .to_string(), - ); - } + )?; + let spawned_registry = registry_reclaim.registry.clone(); // Hand the alerter to the Worker so internal operations (compaction, // AGENTS.md ingestion during the first turn) can emit user-facing @@ -1020,9 +1097,6 @@ impl WorkerController { if let Some(fs_for_view) = fs_for_view.as_ref() { shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view.clone())); } - runtime_dir.write_manifest(&manifest_toml).await?; - runtime_dir.write_status(&shared_state).await?; - let artifact_store: Arc = Arc::new(worker.store().clone()); let session_id = worker.session_id(); let pending_activations = worker.pending_activation_state(); @@ -1047,25 +1121,88 @@ impl WorkerController { pending_activations, }; - // Do not start an observer before fallible head/recovery/runtime-file - // initialization. The only remaining fallible step is socket startup, - // whose error path must abort AND join this task before returning. - let command_observer = fs_for_view + let command_events = fs_for_view .as_ref() - .and_then(|session| wire_workdir_command_events(session, &in_flight)); - let socket_server = match transport { - WorkerControllerTransport::UnixSocket => match SocketServer::start(&handle).await { - Ok(server) => Some(server), - Err(error) => { - if let Some(observer) = command_observer { - observer.abort(); - let _ = observer.await; - } - return Err(error); - } - }, + .map(|session| PreparedWorkdirCommandEvents::new(session, &handle.in_flight)); + Ok(InitializedWorkerController { + worker, + handle, + method_rx, + shutdown_tx, + shutdown_rx, + registry_reclaim, + shutdown_after_idle, + transport, + command_events, + }) + } +} + +impl InitializedWorkerController +where + C: LlmClient + Clone + 'static, + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, +{ + async fn publish( + mut self, + ) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error> { + let publication = self.prepare_publication().await?; + Ok(self.start(publication)) + } + + pub(crate) async fn prepare_publication( + &mut self, + ) -> std::io::Result { + let runtime_dir = &self.handle.runtime_dir; + runtime_dir + .write_manifest(&self.handle.shared_state.manifest_toml) + .await?; + runtime_dir.write_status(&self.handle.shared_state).await?; + let loaded_registry = self.registry_reclaim.apply().await?; + debug_assert!(Arc::ptr_eq( + &loaded_registry.registry, + &self.handle.spawned_registry + )); + if loaded_registry.reclaimed_unreachable { + self.worker.push_notify("Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume.".to_string()); + } + let socket_server = match self.transport { + WorkerControllerTransport::UnixSocket => Some(SocketServer::start(&self.handle).await?), WorkerControllerTransport::InProcess => None, }; + let command_observer = self + .command_events + .take() + .and_then(PreparedWorkdirCommandEvents::start); + Ok(PreparedControllerPublication { + socket_server, + command_observer, + }) + } + + pub(crate) fn start( + self, + publication: PreparedControllerPublication, + ) -> (WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>) { + let Self { + mut worker, + handle, + method_rx, + shutdown_tx, + shutdown_rx, + shutdown_after_idle, + .. + } = self; + let PreparedControllerPublication { + socket_server, + command_observer, + } = publication; + let runtime_dir = handle.runtime_dir.clone(); + let shared_state = handle.shared_state.clone(); + let working_event_tx = handle.working_event_tx.clone(); + let spawned_registry = handle.spawned_registry.clone(); + let self_parent_socket = worker.callback_socket().cloned(); + let spawner_name = worker.manifest().worker.name.clone(); // === 5. controller_loop === // Clone cancel sender and notification buffer before moving worker @@ -1094,7 +1231,7 @@ impl WorkerController { command_observer, )); - Ok((handle, shutdown_rx, controller_task)) + (handle, shutdown_rx, controller_task) } } @@ -1102,30 +1239,51 @@ pub(crate) fn wire_workdir_command_events( session: &Arc, in_flight: &InFlightEvents, ) -> Option> { - in_flight.replace_command_snapshot(protocol_command_snapshots(session.as_ref())); - let Some(mut events) = session.subscribe_command_events() else { - return None; - }; - // Keep only a weak reference in the observer task. Holding the session - // strongly here would keep its broadcast sender alive forever and prevent - // the receiver from observing closure during Worker teardown. - let session = Arc::downgrade(session); - let in_flight = in_flight.clone(); - Some(tokio::spawn(async move { - loop { - match events.recv().await { - Ok(event) => in_flight.publish_command_event(protocol_command_event(event)), - Err(broadcast::error::RecvError::Lagged(_)) => { - let Some(session) = session.upgrade() else { - break; - }; - in_flight - .replace_command_snapshot(protocol_command_snapshots(session.as_ref())); - } - Err(broadcast::error::RecvError::Closed) => break, - } + PreparedWorkdirCommandEvents::new(session, in_flight).start() +} + +struct PreparedWorkdirCommandEvents { + session: std::sync::Weak, + in_flight: InFlightEvents, + events: Option>, +} + +impl PreparedWorkdirCommandEvents { + fn new(session: &Arc, in_flight: &InFlightEvents) -> Self { + // Subscribe before reading command state; hold the receiver privately + // across the durable decision instead of creating a publication gap. + let events = session.subscribe_command_events(); + in_flight.replace_command_snapshot(protocol_command_snapshots(session.as_ref())); + Self { + session: Arc::downgrade(session), + in_flight: in_flight.clone(), + events, } - })) + } + + fn start(self) -> Option> { + let Self { + session, + in_flight, + events, + } = self; + let mut events = events?; + Some(tokio::spawn(async move { + loop { + match events.recv().await { + Ok(event) => in_flight.publish_command_event(protocol_command_event(event)), + Err(broadcast::error::RecvError::Lagged(_)) => { + let Some(session) = session.upgrade() else { + break; + }; + in_flight + .replace_command_snapshot(protocol_command_snapshots(session.as_ref())); + } + Err(broadcast::error::RecvError::Closed) => break, + } + } + })) + } } fn protocol_command_snapshots(session: &dyn WorkdirSession) -> Vec { @@ -1851,9 +2009,17 @@ where .add_module(crate::feature::builtin::orchestration::orchestration_feature()); } if let Some(workspace_root) = local_workspace_root.as_ref() { - if let Some(module) = + let module = if let Some(cleanup) = worker.startup_feature_cleanup_handle() { + crate::feature::mcp::discover_stdio_tool_feature_with_cleanup( + &mcp_config, + workspace_root, + &cleanup.mcp_discovery(), + ) + .await? + } else { crate::feature::mcp::discover_stdio_tool_feature(&mcp_config, workspace_root).await - { + }; + if let Some(module) = module { feature_registry = feature_registry.with_module(module); } } @@ -4123,6 +4289,171 @@ mod tests { } } + // Component connection: real Worker/Store, initialized endpoint, paired + // protocol subscription and Controller loop. The model boundary is an + // empty non-network script; this test must not request any generation. + #[tokio::test] + async fn initialized_controller_publishes_the_same_endpoint_and_subscription() { + let root = TempDir::new().unwrap(); + let manifest = manifest::WorkerManifest::from_toml( + "[worker]\nname = 'p'\n[model]\nscheme = 'anthropic'\nmodel_id = 'test'\n[engine]\n[scope]\n", + ) + .unwrap(); + let client = ScriptedClient::new(vec![]); + let requests = client.requests.clone(); + let engine = + Engine::<_, agen::state::Mutable, crate::SessionHistoryMetadata>::new_annotated( + client, + ); + let store = session_store::CombinedStore::new( + session_store::FsStore::new(root.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(root.path().join("workers")).unwrap(), + ); + let mut worker = Worker::new( + manifest, + engine, + store, + crate::worker::WorkerWorkspaceContext::local_filesystem(None), + crate::worker::WorkerFilesystemAuthority::None, + manifest::Scope::empty(), + ) + .await + .unwrap(); + worker.set_system_prompt_template( + crate::SystemPromptTemplate::parse( + "default", + crate::PromptCatalogSource::builtins_only(), + ) + .unwrap(), + ); + let prepared = WorkerController::initialize( + worker, + root.path(), + false, + None, + WorkerControllerTransport::UnixSocket, + ) + .await + .unwrap(); + let endpoint = prepared.handle.clone(); + let mut streams = + crate::ipc::protocol_session::subscribe_worker_protocol_session(&endpoint); + let entries = endpoint.committed_entries(); + assert!(matches!( + &streams.snapshot_event, + Event::Snapshot { greeting, state, session, .. } + if !greeting.tools.is_empty() + && state.state == WorkerState::Idle + && session == &session_store::public_snapshot::project_current_session_snapshot(&entries) + )); + assert!(!endpoint.protocol_is_closed()); + assert!(!endpoint.runtime_dir.socket_path().exists()); + assert!(!endpoint.runtime_dir.path().join("status.json").exists()); + assert!(!endpoint.runtime_dir.path().join("manifest.toml").exists()); + assert!(matches!( + streams.events.try_recv(), + Err(broadcast::error::TryRecvError::Empty) + )); + assert!(requests.lock().unwrap().is_empty()); + + let (handle, shutdown, task) = prepared.publish().await.unwrap(); + assert!(handle.same_controller(&endpoint)); + assert!(handle.runtime_dir.socket_path().exists()); + assert!(handle.runtime_dir.path().join("status.json").exists()); + handle.send(Method::ListPendingSubmissions).await.unwrap(); + let pending = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Event::PendingSubmissionsChanged { pending } = + streams.events.recv().await.unwrap() + { + break pending; + } + } + }) + .await + .unwrap(); + assert_eq!(pending, protocol::PendingSubmissionsSnapshot::default()); + assert_eq!( + serde_json::to_value(handle.committed_entries()).unwrap(), + serde_json::to_value(entries).unwrap(), + ); + handle + .send(Method::Shutdown { + command: WorkerCommandEnvelope { command_id: 1 }, + }) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), shutdown) + .await + .unwrap() + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap(); + assert!(endpoint.protocol_is_closed()); + assert!(!endpoint.runtime_dir.socket_path().exists()); + assert!(requests.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn tool_initialization_failure_does_not_reclaim_saved_children() { + let root = TempDir::new().unwrap(); + let mut manifest = manifest::WorkerManifest::from_toml( + "[worker]\nname = 'p'\n[model]\nscheme = 'anthropic'\nmodel_id = 'test'\n[engine]\n[scope]\n", + ).unwrap(); + manifest.feature.ticket.enabled = true; + let client = ScriptedClient::new(vec![]); + let requests = client.requests.clone(); + let store = session_store::CombinedStore::new( + session_store::FsStore::new(root.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(root.path().join("workers")).unwrap(), + ); + // New feature activation is refused by the real unavailable host; + // legacy child metadata must not be consumed before that refusal. + let worker = Worker::new( + manifest, + Engine::<_, agen::state::Mutable, crate::SessionHistoryMetadata>::new_annotated( + client, + ), + store.clone(), + crate::worker::WorkerWorkspaceContext::no_workspace(), + crate::worker::WorkerFilesystemAuthority::None, + manifest::Scope::empty(), + ) + .await + .unwrap(); + let mut metadata = session_store::WorkerMetadata::new( + "p", + Some(session_store::WorkerActiveSegmentRef { + session_id: worker.session_id(), + segment_id: Some(worker.segment_id()), + }), + ); + metadata + .spawned_children + .push(session_store::WorkerSpawnedChild { + worker_name: "legacy".into(), + socket_path: root.path().join("legacy.sock"), + callback_address: root.path().join("p/sock"), + scope_delegated: Vec::new(), + }); + store.write(&metadata).unwrap(); + let before = store.read_by_name("p").unwrap(); + let result = WorkerController::initialize( + worker, + root.path(), + false, + None, + WorkerControllerTransport::InProcess, + ) + .await; + assert!(result.is_err()); + assert_eq!(store.read_by_name("p").unwrap(), before); + assert!(!root.path().join("p/spawned_workers.json").exists()); + assert!(requests.lock().unwrap().is_empty()); + } + #[derive(Debug)] struct DriveWorkspaceHost { available: bool, @@ -4450,20 +4781,24 @@ mod tests { assert!(feature_install < materialize_gate); assert!(materialize_gate < materialize); assert!(materialize < exposure); - let observer = startup.find("let command_observer = fs_for_view").unwrap(); + let observer_preparation = startup + .find("PreparedWorkdirCommandEvents::new(session") + .unwrap(); let recover = startup.find(".recover_unfinished_compaction()").unwrap(); let files = startup - .find("runtime_dir.write_status(&shared_state).await?") + .find(".write_status(&self.handle.shared_state)") + .unwrap(); + let listener = startup + .find("Some(SocketServer::start(&self.handle).await?)") + .unwrap(); + let observer_start = startup + .find(".and_then(PreparedWorkdirCommandEvents::start)") .unwrap(); let loop_start = startup.find("tokio::spawn(controller_loop(").unwrap(); - assert!(recover < observer && files < observer && observer < loop_start); - let socket_startup = &startup[observer..loop_start]; - let abort = socket_startup.find("observer.abort()").unwrap(); - let join = socket_startup.find("observer.await").unwrap(); - let failure = socket_startup.find("return Err(error)").unwrap(); + assert!(recover < observer_preparation && observer_preparation < files); assert!( - abort < join && join < failure, - "startup cannot detach its observer on error" + files < listener && listener < observer_start && observer_start < loop_start, + "observer must not start before the last fallible listener preparation" ); assert!( startup[materialize_gate..materialize] diff --git a/crates/worker/src/feature/mcp.rs b/crates/worker/src/feature/mcp.rs index 259af6db3..a617acb72 100644 --- a/crates/worker/src/feature/mcp.rs +++ b/crates/worker/src/feature/mcp.rs @@ -8,6 +8,7 @@ //! the regular registry decides Tool-name collisions. use std::collections::BTreeMap; +use std::io; use std::path::Path; use std::sync::Arc; @@ -25,6 +26,7 @@ use mcp::stdio::{ }; use serde::Serialize; use serde_json::{Map, Value}; +use tokio::sync::Mutex; use super::{ FeatureDescriptor, FeatureDiagnostic, FeatureInstallContext, FeatureInstallError, @@ -52,8 +54,88 @@ const MAX_LIST_ITEMS: usize = 128; const MAX_RESOURCE_CONTENTS: usize = 16; const MAX_PROMPT_MESSAGES: usize = 32; +/// Owns discovery clients until process reaping is confirmed. Retain a clone in +/// startup cleanup *before* awaiting discovery. Stop/cancel discovery before +/// calling `shutdown`; the shared lock serializes discovery and cleanup, not +/// cancellation of an in-flight discovery task. +/// +/// Error or cancellation leaves the actual clients here for another shutdown +/// attempt. Dropping the final owner only provides best-effort process killing. +#[derive(Clone, Default)] +pub struct McpDiscoveryCleanup { + clients: Arc>>, +} + +impl McpDiscoveryCleanup { + pub fn new() -> Self { + Self::default() + } + + /// Try every retained client once, removing only confirmed shutdowns. An + /// error is not a cleanup receipt; retry this same owner, not discovery. + pub async fn shutdown(&self) -> io::Result<()> { + let mut clients = self.clients.lock().await; + let mut first_error = None; + let mut index = 0; + while index < clients.len() { + match clients[index].shutdown().await { + Ok(_) => { + clients.remove(index); + } + Err(err) => { + first_error.get_or_insert_with(|| discovery_shutdown_error(&err)); + index += 1; + } + } + } + match first_error { + Some(err) => Err(err), + None => Ok(()), + } + } +} + +fn discovery_shutdown_error(err: &McpClientError) -> io::Error { + io::Error::other(bounded_diagnostic(format!( + "MCP discovery shutdown unconfirmed; client retained for cleanup retry: {err}" + ))) +} + +/// Restore preparation must fail if discovery cannot confirm process shutdown. +/// The caller retains `cleanup` before this future is polled, including during +/// initialization. Ordinary discovery failures remain isolated contributions +/// only after their client has been reaped. +pub async fn discover_stdio_tool_feature_with_cleanup( + config: &McpConfig, + workspace_root: &Path, + cleanup: &McpDiscoveryCleanup, +) -> Result, io::Error> { + if config.stdio_servers.is_empty() { + return Ok(None); + } + let mut feature = McpStdioToolFeature::new(); + for server in &config.stdio_servers { + match resolve_stdio_server(server, workspace_root, None) { + Ok(spec) => { + feature.add_contribution( + discover_server_tools_with_cleanup(spec, cleanup, McpStdioLimits::default()) + .await?, + ); + } + Err(err) => { + feature.add_diagnostic(FeatureDiagnostic::error(bounded_diagnostic(format!( + "failed to resolve MCP stdio server `{}`: {err}", + server.name + )))) + } + } + } + Ok(Some(feature)) +} + /// Discover enabled MCP stdio server tools and return a single feature module /// containing startup contributions for normal ToolRegistry installation. +/// Callers needing retryable startup cleanup must use the cleanup-aware API. pub async fn discover_stdio_tool_feature( config: &McpConfig, workspace_root: &Path, @@ -81,21 +163,66 @@ pub async fn discover_stdio_tool_feature( } async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderContribution { + let declaration = provider_declaration(&spec.name, None); + let cleanup = McpDiscoveryCleanup::new(); + match discover_server_tools_with_cleanup(spec, &cleanup, McpStdioLimits::default()).await { + Ok(contribution) => contribution, + Err(err) => { + ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string())) + } + } +} + +async fn discover_server_tools_with_cleanup( + spec: McpStdioServerSpec, + cleanup: &McpDiscoveryCleanup, + limits: McpStdioLimits, +) -> io::Result { + let declaration = provider_declaration(&spec.name, None); + let mut clients = cleanup.clients.lock().await; + if !clients.is_empty() { + return Err(io::Error::other( + "MCP discovery cleanup is pending; shut down the retained owner before retrying discovery", + )); + } + let client = match McpStdioClient::spawn_uninitialized(spec.clone(), limits) { + Ok(client) => client, + Err(err) => { + // The synchronous spawn API only errors before a process exists; + // pipe/handshake failures retain a client and use the shutdown path. + return Ok(ProtocolProviderContribution::failed( + declaration, + bounded_diagnostic(err.to_string()), + )); + } + }; + // Spawn is synchronous: the owner holds the client before the first await + // that can leave a running process on initialization failure/cancellation. + clients.push(client); + let client = clients.last_mut().expect("retained discovery client"); + let contribution = match client.initialize_with_timeout().await { + Ok(()) => discover_connected_server_tools(spec, client).await, + Err(err) => { + ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string())) + } + }; + client + .shutdown() + .await + .map_err(|err| discovery_shutdown_error(&err))?; + drop(clients.pop()); + Ok(contribution) +} + +async fn discover_connected_server_tools( + spec: McpStdioServerSpec, + client: &mut McpStdioClient, +) -> ProtocolProviderContribution { let declaration = provider_declaration(&spec.name, None); let mut contribution = ProtocolProviderContribution::ready(declaration.clone()); let server_namespace = sanitize_segment(&spec.name); let execution_spec = spec.clone(); - let mut client = match McpStdioClient::connect(spec, McpStdioLimits::default()).await { - Ok(client) => client, - Err(err) => { - return ProtocolProviderContribution::failed( - declaration, - bounded_diagnostic(err.to_string()), - ); - } - }; - let (server_version, has_tools, has_resources, has_prompts) = client .initialize_result() .map(|result| { @@ -133,18 +260,10 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont { Ok(list) => list, Err(err) => { - let mut failed = ProtocolProviderContribution::failed( + return ProtocolProviderContribution::failed( declaration, bounded_diagnostic(err.to_string()), ); - if let Err(shutdown_err) = client.shutdown().await { - failed = failed.with_diagnostic(FeatureDiagnostic::warning( - bounded_diagnostic(format!( - "MCP server shutdown after discovery failure failed: {shutdown_err}" - )), - )); - } - return failed; } }; let list_changes = client.snapshot_list_changes().await; @@ -166,20 +285,12 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont { Ok(list) => list, Err(err) => { - let mut failed = ProtocolProviderContribution::failed( + return ProtocolProviderContribution::failed( declaration, bounded_diagnostic(format!( "MCP server sent notifications/tools/list_changed during tool discovery and refresh failed: {err}" )), ); - if let Err(shutdown_err) = client.shutdown().await { - failed = failed.with_diagnostic(FeatureDiagnostic::warning( - bounded_diagnostic(format!( - "MCP server shutdown after discovery refresh failure failed: {shutdown_err}" - )), - )); - } - return failed; } }; let refresh_changes = client.snapshot_list_changes().await; @@ -197,13 +308,6 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont } else { None }; - let shutdown_result = client.shutdown().await; - if let Err(err) = shutdown_result { - contribution = - contribution.with_diagnostic(FeatureDiagnostic::warning(bounded_diagnostic(format!( - "MCP server shutdown after capability discovery failed: {err}" - )))); - } if let Some(list) = list { contribution = normalize_listed_tools( @@ -1819,6 +1923,140 @@ read exit_notification || true McpStdioServerSpec::new("shell-mock", "/bin/sh").args(["-c".to_string(), script]) } + #[cfg(unix)] + fn discovery_config(spec: McpStdioServerSpec) -> McpConfig { + McpConfig { + stdio_servers: vec![manifest::McpStdioServerConfig { + name: spec.name, + command: spec.command, + args: spec.args, + cwd: None, + env: Default::default(), + }], + } + } + + #[cfg(unix)] + #[tokio::test] + async fn cancelled_discovery_retains_initializing_client_in_cleanup_clone() { + use std::future::{Future, poll_fn}; + use std::task::Poll; + use std::time::Duration; + + let config = discovery_config( + McpStdioServerSpec::new("initializing", "/bin/sh") + .args(["-c", "while read line; do :; done"]), + ); + let cleanup = McpDiscoveryCleanup::new(); + let retained = cleanup.clone(); + let mut discovery = Box::pin(discover_stdio_tool_feature_with_cleanup( + &config, + Path::new("."), + &cleanup, + )); + poll_fn(|cx| { + assert!(discovery.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + drop(discovery); + drop(cleanup); + assert_eq!(retained.clients.lock().await.len(), 1); + let retry_error = + discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &retained) + .await + .err() + .expect("discovery must not replace an unreaped client"); + assert!(retry_error.to_string().contains("cleanup is pending")); + assert_eq!(retained.clients.lock().await.len(), 1); + tokio::time::timeout(Duration::from_secs(3), retained.shutdown()) + .await + .expect("cleanup bounded") + .expect("initializing process reaped"); + assert!(retained.clients.lock().await.is_empty()); + retained + .shutdown() + .await + .expect("confirmed cleanup repeatable"); + } + + #[cfg(unix)] + #[tokio::test] + async fn cancelled_owner_shutdown_keeps_client_for_retry() { + use std::future::{Future, poll_fn}; + use std::task::Poll; + use std::time::Duration; + + let cleanup = McpDiscoveryCleanup::new(); + cleanup.clients.lock().await.push( + McpStdioClient::spawn_uninitialized( + McpStdioServerSpec::new("shutdown-pending", "/bin/sh") + .args(["-c", "while :; do :; done"]), + McpStdioLimits { + shutdown_timeout: Duration::from_millis(50), + kill_timeout: Duration::from_millis(100), + ..Default::default() + }, + ) + .expect("local process spawned"), + ); + let mut shutdown = Box::pin(cleanup.shutdown()); + poll_fn(|cx| { + assert!(shutdown.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + drop(shutdown); + assert_eq!(cleanup.clients.lock().await.len(), 1); + tokio::time::timeout(Duration::from_secs(2), cleanup.shutdown()) + .await + .expect("retry bounded") + .expect("retained client shutdown"); + assert!(cleanup.clients.lock().await.is_empty()); + } + + #[cfg(unix)] + #[tokio::test] + async fn cleanup_aware_discovery_reaps_success_and_initialize_failure() { + let cleanup = McpDiscoveryCleanup::new(); + let config = discovery_config(shell_capability_server(r#"{"resources":{}}"#)); + let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup) + .await + .expect("discovery cleanup confirmed") + .expect("configured feature"); + assert!( + feature.contributions[0] + .tools + .iter() + .any(|tool| { tool.name() == "Mcp_shell_mock_resources_read" }) + ); + assert!(cleanup.clients.lock().await.is_empty()); + + let spec = McpStdioServerSpec::new("init-rejected", "/bin/sh").args([ + "-c", + r#"read init || exit 1 +printf '%s\n' '{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"initialization rejected"}}' +while read line; do :; done +"#, + ]); + let config = discovery_config(spec); + let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup) + .await + .expect("failure isolated only after cleanup") + .expect("failed contribution preserved"); + assert!(feature.contributions[0].tools.is_empty()); + assert!( + feature.contributions[0] + .diagnostics + .iter() + .any(|diagnostic| { + diagnostic.severity == FeatureDiagnosticSeverity::Error + && diagnostic.message.contains("initialization rejected") + }) + ); + assert!(cleanup.clients.lock().await.is_empty()); + } + #[test] fn mcp_resource_prompt_operation_tools_are_namespaced_and_untrusted() { let declaration = provider_declaration("demo", Some("1.0.0")); diff --git a/crates/worker/src/spawn/registry.rs b/crates/worker/src/spawn/registry.rs index dd45b6f47..f2528b4e0 100644 --- a/crates/worker/src/spawn/registry.rs +++ b/crates/worker/src/spawn/registry.rs @@ -340,6 +340,84 @@ pub struct SpawnedWorkerRegistryLoad { pub reclaimed_unreachable: bool, } +/// Read-only restore preparation using the registry that will be published by the Host. +/// The Host must retain its lifecycle exclusion from preparation through application; +/// the metadata reread detects source drift, but is not an atomic store transaction. +/// Dropping an unapplied plan does not reclaim legacy child state. +pub(crate) struct PreparedSpawnedWorkerRegistryLoad { + pub(crate) registry: Arc, + /// Projected outcome, not evidence that reclamation has already happened. + pub(crate) reclaimed_unreachable: bool, + runtime_dir: Arc, + store: St, + worker_name: String, + parent_scope: Option, + persisted_children: Vec, + valid_records: Vec, + legacy_projection_exists: bool, +} + +impl PreparedSpawnedWorkerRegistryLoad +where + St: WorkerMetadataStore + Clone + Send + Sync + 'static, +{ + /// Apply only after all other fallible tool initialization, before socket publication. + /// Effects retain their original order: projection, canonical reclaim history, then + /// allocation/scope. An error may follow committed effects; no rollback is attempted. + pub(crate) async fn apply(&mut self) -> io::Result { + let metadata = self + .store + .read_by_name(&self.worker_name) + .map_err(store_error_to_io)?; + let reclaimed: Vec<_> = self + .persisted_children + .iter() + .map(reclaimed_child_from_metadata) + .collect(); + // A prior apply may have committed the canonical reclaim and then failed + // in allocation/scope cleanup. Under the host's same lifecycle lock, + // confirm its exact retained facts rather than appending them again. + let already_reclaimed = !reclaimed.is_empty() + && metadata.as_ref().is_some_and(|metadata| { + metadata.spawned_children.is_empty() + && metadata.reclaimed_children.ends_with(&reclaimed) + }); + if !already_reclaimed + && metadata + .as_ref() + .map(|metadata| metadata.spawned_children.as_slice()) + .unwrap_or_default() + != self.persisted_children.as_slice() + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "spawned-child metadata changed after registry preparation", + )); + } + + // Runtime projection is migration input only; the normal Internal registry is never + // materialized into spawned_workers.json. Only existence was captured, not its contents. + if !self.persisted_children.is_empty() || self.legacy_projection_exists { + self.runtime_dir.write_spawned_workers(&[]).await?; + } + if !reclaimed.is_empty() && !already_reclaimed { + self.store + .reclaim_spawned_children(&self.worker_name, reclaimed) + .map_err(store_error_to_io)?; + } + for record in &self.valid_records { + reclaim_record(&self.worker_name, self.parent_scope.as_ref(), record)?; + } + + let reclaimed_unreachable = self.reclaimed_unreachable; + self.reclaimed_unreachable = false; + Ok(SpawnedWorkerRegistryLoad { + registry: self.registry.clone(), + reclaimed_unreachable, + }) + } +} + impl SpawnedWorkerRegistry { pub(crate) fn bash_output(&self) -> tools::bash_output::BashOutputView { self.bash_output.lock().unwrap().clone() @@ -426,6 +504,23 @@ impl SpawnedWorkerRegistry { worker_name: String, parent_scope: Option, ) -> io::Result + where + St: WorkerMetadataStore + Clone + Send + Sync + 'static, + { + Self::prepare_from_worker_state_with_reclaim(runtime_dir, store, worker_name, parent_scope)? + .apply() + .await + } + + /// Capture legacy child facts without starting children or changing metadata, + /// projection, allocation, or scope. Tool initialization must clone this registry, + /// not construct a replacement, and defer reclamation to the owned plan's `apply`. + pub(crate) fn prepare_from_worker_state_with_reclaim( + runtime_dir: Arc, + store: St, + worker_name: String, + parent_scope: Option, + ) -> io::Result> where St: WorkerMetadataStore + Clone + Send + Sync + 'static, { @@ -442,54 +537,33 @@ impl SpawnedWorkerRegistry { Ok(record) => { warn!( worker = %record.worker_name, - "reclaiming legacy persisted process SubWorker during Internal session restore" + "captured legacy persisted process SubWorker for reclamation during Internal session restore" ); valid_records.push(record); } Err(error) => warn!( error = %error, worker = %child.worker_name, - "clearing corrupt legacy persisted process SubWorker record" + "captured corrupt legacy persisted process SubWorker record for clearing" ), } } - // Runtime projection is migration input only; the normal Internal registry is never - // materialized into spawned_workers.json. let legacy_projection_exists = runtime_dir.path().join("spawned_workers.json").exists(); - if !persisted_children.is_empty() || legacy_projection_exists { - runtime_dir.write_spawned_workers(&[]).await?; - } - if !persisted_children.is_empty() { - let reclaimed = persisted_children - .iter() - .map(reclaimed_child_from_metadata) - .collect(); - store - .reclaim_spawned_children(&worker_name, reclaimed) - .map_err(store_error_to_io)?; - } - for record in &valid_records { - reclaim_record(&worker_name, parent_scope.as_ref(), record)?; - } - - Ok(SpawnedWorkerRegistryLoad { - registry: Arc::new(Self { - internal_records: std::sync::Mutex::new(Vec::new()), - service_records: std::sync::Mutex::new(Vec::new()), - #[cfg(test)] - fail_service_stops: std::sync::Mutex::new(HashMap::new()), - internal_names: std::sync::Mutex::new(HashSet::new()), - internal_shutting_down: AtomicBool::new(false), - pending_internal_spawns: AtomicUsize::new(0), - pending_internal_notify: Notify::new(), - internal_spawn_cleanup_failed: AtomicBool::new(false), - parent_scope, - bash_output: Mutex::new(tools::bash_output::BashOutputView::new()), - parent_protocol: Mutex::new(None), - protocol_publish_lock: Arc::new(Mutex::new(())), - }), + let registry = match &parent_scope { + Some(scope) => Self::new_internal(worker_name.clone(), scope.clone()), + None => Self::new(runtime_dir.clone()), + }; + Ok(PreparedSpawnedWorkerRegistryLoad { + registry, reclaimed_unreachable: !persisted_children.is_empty(), + runtime_dir, + store, + worker_name, + parent_scope, + persisted_children, + valid_records, + legacy_projection_exists, }) } @@ -1140,6 +1214,10 @@ impl Drop for SpawnedWorkerRegistry { .filter(|record| !record.scope_reclaimed.load(Ordering::Acquire)) .flat_map(delegated_write_rules) .collect::>(); + // Empty prepared registries own no delegations, including when abandoned or rejected. + if write_rules.is_empty() { + return; + } let _ = parent_scope.update(|current| current.with_removed_deny_rules(write_rules)); } } @@ -1165,19 +1243,18 @@ fn reclaim_record( parent_scope: Option<&SharedScope>, record: &SpawnedWorkerRecord, ) -> io::Result<()> { - if let Ok(path) = worker_allocation::default_allocation_path() { - if let Ok(mut guard) = worker_allocation::LockFileGuard::open(&path) { - match worker_allocation::reclaim_delegated_scope( - &mut guard, - parent_name, - &record.worker_name, - &record.scope_delegated, - ) { - Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {} - Err(error) => return Err(io::Error::other(error)), - } - } + let path = worker_allocation::default_allocation_path().map_err(io::Error::other)?; + let mut guard = worker_allocation::LockFileGuard::open(&path).map_err(io::Error::other)?; + match worker_allocation::reclaim_delegated_scope( + &mut guard, + parent_name, + &record.worker_name, + &record.scope_delegated, + ) { + Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {} + Err(error) => return Err(io::Error::other(error)), } + drop(guard); if let Some(parent_scope) = parent_scope { let write_rules = record .scope_delegated @@ -1240,10 +1317,18 @@ fn store_error_to_io(error: WorkerStoreError) -> io::Error { #[cfg(test)] mod tests { + //! Legacy restore cases connect the registry to canonical aggregate storage, + //! runtime projections, and temporary SharedScope rules. Successful apply uses + //! a corrupt legacy record to exercise durable clearing without consulting the + //! process-global legacy allocation file; valid-record scope reclaim is also + //! covered by the existing external restore test. + + use std::path::PathBuf; use std::time::Duration; use manifest::{Scope, ScopeConfig}; - use session_store::LogEntry; + use session_store::{LogEntry, WorkerAggregateStore, WorkerMetadata, WorkerSpawnedScopeRule}; + use tempfile::TempDir; use super::*; use crate::internal_worker::{ @@ -1251,6 +1336,249 @@ mod tests { test_internal_worker_session_with_greeting, }; + struct LegacyRestoreFixture { + runtime_dir: Arc, + store: WorkerAggregateStore, + parent_scope: SharedScope, + scope_root: PathBuf, + metadata_path: PathBuf, + // Keep all storage and scope paths alive until the other fixture fields are dropped. + _root: TempDir, + } + + impl LegacyRestoreFixture { + async fn new(permission: &str) -> Self { + let root = TempDir::new().unwrap(); + let scope_root = root.path().join("scope"); + std::fs::create_dir(&scope_root).unwrap(); + let aggregate = root.path().join("aggregate"); + let store = WorkerAggregateStore::new(&aggregate, "parent").unwrap(); + let runtime_dir = Arc::new( + RuntimeDir::create(&root.path().join("runtime"), "parent") + .await + .unwrap(), + ); + let mut metadata = + WorkerMetadata::new("parent", None).with_workspace_id("restore-test-workspace"); + metadata.spawned_children.push(WorkerSpawnedChild { + worker_name: "legacy-child".into(), + socket_path: root.path().join("legacy.sock"), + callback_address: runtime_dir.socket_path(), + scope_delegated: vec![WorkerSpawnedScopeRule { + target: scope_root.clone(), + permission: permission.into(), + recursive: true, + symlink_policy: Default::default(), + }], + }); + store.write(&metadata).unwrap(); + let write_rule = ScopeRule { + target: scope_root.clone(), + permission: Permission::Write, + recursive: true, + symlink_policy: Default::default(), + }; + let parent_scope = SharedScope::new( + Scope::from_config(&ScopeConfig { + allow: vec![write_rule.clone()], + deny: vec![write_rule], + }) + .unwrap(), + ); + Self { + runtime_dir, + store, + parent_scope, + scope_root, + metadata_path: aggregate.join("metadata.json"), + _root: root, + } + } + + fn prepare(&self) -> PreparedSpawnedWorkerRegistryLoad { + SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim( + self.runtime_dir.clone(), + self.store.clone(), + "parent".into(), + Some(self.parent_scope.clone()), + ) + .unwrap() + } + + fn projection_path(&self) -> PathBuf { + self.runtime_dir.path().join("spawned_workers.json") + } + } + + #[tokio::test] + async fn registry_preparation_and_abandonment_leave_legacy_state_unchanged() { + for projection_exists in [false, true] { + let fixture = LegacyRestoreFixture::new("write").await; + if projection_exists { + // Projection contents are deliberately not decoded during preparation. + std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap(); + } + let metadata_before = std::fs::read(&fixture.metadata_path).unwrap(); + let projection_before = std::fs::read(fixture.projection_path()).ok(); + let scope_before = fixture.parent_scope.snapshot(); + assert!(!scope_before.is_writable(&fixture.scope_root)); + + let prepared = fixture.prepare(); + + assert!(prepared.reclaimed_unreachable); + assert!(prepared.registry.list_internal().is_empty()); + assert!(prepared.registry.internal_worker_snapshots().is_empty()); + assert_eq!( + std::fs::read(&fixture.metadata_path).unwrap(), + metadata_before + ); + assert_eq!( + std::fs::read(fixture.projection_path()).ok(), + projection_before + ); + assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot())); + assert!(!fixture.runtime_dir.socket_path().exists()); + drop(prepared); + assert_eq!( + std::fs::read(&fixture.metadata_path).unwrap(), + metadata_before + ); + assert_eq!( + std::fs::read(fixture.projection_path()).ok(), + projection_before + ); + assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot())); + } + } + + #[tokio::test] + async fn prepared_apply_clears_canonical_children_and_projection_on_same_registry() { + let fixture = LegacyRestoreFixture::new("unsupported-legacy-permission").await; + std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap(); + let scope_before = fixture.parent_scope.snapshot(); + let mut prepared = fixture.prepare(); + let shared_registry = prepared.registry.clone(); + // Unrelated metadata may advance while tools are initialized. + fixture.store.add_peer("parent", "new-peer").unwrap(); + let mut expected = fixture.store.read_by_name("parent").unwrap().unwrap(); + expected.reclaimed_children.extend( + expected + .spawned_children + .iter() + .map(reclaimed_child_from_metadata), + ); + expected.spawned_children.clear(); + + let loaded = prepared.apply().await.unwrap(); + + assert!(loaded.reclaimed_unreachable); + assert!(Arc::ptr_eq(&shared_registry, &loaded.registry)); + assert!(shared_registry.list_internal().is_empty()); + assert_eq!( + fixture.store.read_by_name("parent").unwrap(), + Some(expected) + ); + let projection: serde_json::Value = + serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap(); + assert_eq!(projection, serde_json::json!([])); + // Corrupt records are audited and cleared, not interpreted as scope grants. + assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot())); + assert!( + !fixture + .parent_scope + .snapshot() + .is_writable(&fixture.scope_root) + ); + assert!(!fixture.runtime_dir.socket_path().exists()); + } + + #[tokio::test] + async fn prepared_apply_rejects_child_source_drift_before_any_effects() { + for drift in ["scope", "socket", "removal", "metadata-deletion"] { + let fixture = LegacyRestoreFixture::new("write").await; + std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap(); + let scope_before = fixture.parent_scope.snapshot(); + let mut prepared = fixture.prepare(); + if drift == "metadata-deletion" { + fixture.store.delete_by_name("parent").unwrap(); + } else { + fixture + .store + .update_by_name("parent", |metadata| match drift { + "scope" => { + metadata.spawned_children[0].scope_delegated[0].recursive = false + } + "socket" => { + metadata.spawned_children[0].socket_path = + fixture.runtime_dir.path().join("replacement.sock"); + } + "removal" => metadata.spawned_children.clear(), + _ => unreachable!(), + }) + .unwrap(); + } + let metadata_before = std::fs::read(&fixture.metadata_path).ok(); + let projection_before = std::fs::read(fixture.projection_path()).unwrap(); + + let error = prepared + .apply() + .await + .err() + .expect("source drift must fail"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(std::fs::read(&fixture.metadata_path).ok(), metadata_before); + assert_eq!( + std::fs::read(fixture.projection_path()).unwrap(), + projection_before + ); + assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot())); + assert!(!fixture.runtime_dir.socket_path().exists()); + } + } + + #[tokio::test] + async fn prepared_apply_clears_projection_only_without_reporting_child_reclaim() { + let fixture = LegacyRestoreFixture::new("write").await; + fixture + .store + .set_spawned_children("parent", Vec::new()) + .unwrap(); + std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap(); + let metadata_before = std::fs::read(&fixture.metadata_path).unwrap(); + let mut prepared = fixture.prepare(); + assert!(!prepared.reclaimed_unreachable); + + let loaded = prepared.apply().await.unwrap(); + + assert!(!loaded.reclaimed_unreachable); + assert_eq!( + std::fs::read(&fixture.metadata_path).unwrap(), + metadata_before + ); + let projection: serde_json::Value = + serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap(); + assert_eq!(projection, serde_json::json!([])); + } + + #[tokio::test] + async fn prepared_apply_projection_failure_keeps_canonical_children_and_scope() { + let fixture = LegacyRestoreFixture::new("write").await; + std::fs::create_dir(fixture.projection_path()).unwrap(); + let metadata_before = std::fs::read(&fixture.metadata_path).unwrap(); + let scope_before = fixture.parent_scope.snapshot(); + let mut prepared = fixture.prepare(); + + assert!(prepared.apply().await.is_err()); + + assert_eq!( + std::fs::read(&fixture.metadata_path).unwrap(), + metadata_before + ); + assert!(fixture.projection_path().is_dir()); + assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot())); + } + fn registry() -> Arc { let scope = Scope::from_config(&ScopeConfig { allow: vec![ScopeRule { diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 59eb439a7..14be7b385 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -3335,6 +3335,7 @@ pub struct WorkerStartupFeatureCleanup { } struct StartupFeatureCleanupInner { + mcp_discovery: crate::feature::mcp::McpDiscoveryCleanup, registries: Mutex>, settlement: tokio::sync::Mutex< Option>>, @@ -3345,12 +3346,17 @@ impl WorkerStartupFeatureCleanup { fn new(registry: FeatureBackgroundTaskRegistry) -> Self { Self { inner: Arc::new(StartupFeatureCleanupInner { + mcp_discovery: crate::feature::mcp::McpDiscoveryCleanup::new(), registries: Mutex::new(vec![registry]), settlement: tokio::sync::Mutex::new(None), }), } } + pub(crate) fn mcp_discovery(&self) -> crate::feature::mcp::McpDiscoveryCleanup { + self.inner.mcp_discovery.clone() + } + fn retain(&self, registry: FeatureBackgroundTaskRegistry) { self.inner .registries @@ -3370,6 +3376,7 @@ impl WorkerStartupFeatureCleanup { .lock() .expect("startup Feature cleanup poisoned") .clone(); + let mcp_discovery = self.inner.mcp_discovery.clone(); Box::pin(async move { let mut failure = None; for registry in registries { @@ -3377,6 +3384,14 @@ impl WorkerStartupFeatureCleanup { failure.get_or_insert(error); } } + if let Err(error) = mcp_discovery.shutdown().await { + failure.get_or_insert_with(|| { + crate::hook::HookError::new( + crate::hook::HookErrorCategory::Dependency, + error.to_string(), + ) + }); + } failure.map_or(Ok(()), Err) }) }); @@ -3414,11 +3429,84 @@ impl RestoreCandidateWorkdirCleanup { pub struct PreparedRestoredWorker { worker: Option, St>>, closing: bool, + activation: Option, +} + +/// Exact existing restore source retained while a Controller is initialized. +/// +/// This carries no new Worker identity or execution authority. The host must keep +/// the per-Worker operation lock and invoke activation only after durable commit. +/// Controller initialization must not change the captured source before then. +pub(crate) struct RestoredWorkerActivation { metadata: Option, entries: serde_json::Value, manifest: serde_json::Value, } +impl RestoredWorkerActivation { + fn revalidate(&self, worker: &Worker, St>) -> Result<(), WorkerError> + where + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + let changed = |component| WorkerError::RestoreCandidateChanged { + worker_name: worker.manifest.worker.name.clone(), + component, + }; + if worker.store.read_by_name(&worker.manifest.worker.name)? != self.metadata { + return Err(changed("metadata")); + } + if restore_snapshot_value(&worker.manifest)? != self.manifest { + return Err(changed("manifest")); + } + let entries = worker + .store + .read_all(worker.session_id(), worker.segment_id())?; + if restore_snapshot_value(&entries)? != self.entries { + return Err(changed("segment")); + } + Ok(()) + } + + /// Acquire the existing top-level writer allocation after commit, checking + /// exact source both before and after acquisition, then apply restore effects. + /// On failure the Controller still owns the Worker; cleanup handles do not + /// establish rollback and allocation release must wait until it is dropped. + pub(crate) async fn activate( + &self, + worker: &mut Worker, St>, + ) -> Result<(), WorkerError> + where + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + self.revalidate(worker) + .map_err(|error| worker.construction_failure(error))?; + let allocate = || -> Result { + let socket_path = dir::default_base() + .map_err(ScopeLockError::from)? + .join(&worker.manifest.worker.name) + .join("sock"); + Ok(worker_allocation::install_top_level_with_deny( + worker.manifest.worker.name.clone(), + std::process::id(), + socket_path, + worker.scope.snapshot().allow_rules(), + worker.scope.snapshot().deny_rules(), + worker.segment_id(), + )?) + }; + worker.scope_allocation = + Some(allocate().map_err(|error| worker.construction_failure(error))?); + self.revalidate(worker) + .map_err(|error| worker.construction_failure(error))?; + worker + .finish_restored_construction_in_place(false) + .await + .map_err(|source| WorkerError::RestoreActivationEffects { + source: Box::new(source), + }) + } +} + impl PreparedRestoredWorker where St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, @@ -3460,23 +3548,32 @@ where } fn revalidate(&self, worker: &Worker, St>) -> Result<(), WorkerError> { - let changed = |component| WorkerError::RestoreCandidateChanged { - worker_name: worker.manifest.worker.name.clone(), - component, - }; - if worker.store.read_by_name(&worker.manifest.worker.name)? != self.metadata { - return Err(changed("metadata")); + self.activation + .as_ref() + .expect("restore candidate activation already transferred") + .revalidate(worker) + } + + /// Transfer the read-only reconstruction for Controller initialization, not + /// execution. Validate once without allocation or canonical restore effects; + /// the Controller must retain activation until its durable commit succeeds. + pub(crate) fn into_controller_parts( + mut self, + ) -> Result<(Worker, St>, RestoredWorkerActivation), WorkerError> { + let worker = self + .worker + .take() + .ok_or(WorkerError::RestoreCandidateClosing)?; + if self.closing { + return Err(worker.construction_failure(WorkerError::RestoreCandidateClosing)); } - if restore_snapshot_value(&worker.manifest)? != self.manifest { - return Err(changed("manifest")); - } - let entries = worker - .store - .read_all(worker.session_id(), worker.segment_id())?; - if restore_snapshot_value(&entries)? != self.entries { - return Err(changed("segment")); - } - Ok(()) + self.revalidate(&worker) + .map_err(|error| worker.construction_failure(error))?; + let activation = self + .activation + .take() + .expect("restore candidate activation already transferred"); + Ok((worker, activation)) } /// Acquire writer allocation only after exact source validation, validate @@ -9730,9 +9827,11 @@ where let mut candidate = PreparedRestoredWorker { worker: Some(worker), closing: false, - metadata, - entries: entries_snapshot, - manifest: manifest_snapshot, + activation: Some(RestoredWorkerActivation { + metadata, + entries: entries_snapshot, + manifest: manifest_snapshot, + }), }; if let Err(error) = candidate.revalidate(candidate.worker()) { let worker = candidate.worker.take().expect("candidate just constructed"); @@ -9742,6 +9841,16 @@ where } async fn finish_restored_construction(mut self) -> Result { + self.finish_restored_construction_in_place(true).await?; + Ok(self) + } + + /// Apply the same restore effects while an initialized Controller retains + /// ownership. Errors retain cleanup handles, but do not drop this Worker. + async fn finish_restored_construction_in_place( + &mut self, + reclaim_delegations: bool, + ) -> Result<(), WorkerError> { self.pending_submission_handle() .reconcile_uploaded_file_pins() .map_err(|error| self.construction_failure(error.into()))?; @@ -9752,10 +9861,12 @@ where segment_id: self.segment_id(), }) .map_err(|error| self.construction_failure(error))?; - self.reconcile_restored_delegations() - .await - .map_err(|error| self.construction_failure(error))?; - Ok(self) + if reclaim_delegations { + self.reconcile_restored_delegations() + .await + .map_err(|error| self.construction_failure(error))?; + } + Ok(()) } async fn reconcile_restored_delegations(&mut self) -> Result<(), WorkerError> { @@ -10596,8 +10707,10 @@ fn compact_failure_category(error: &WorkerError) -> CompactFailureCategory { #[derive(Debug, thiserror::Error)] pub enum WorkerError { - /// A constructor failed after acquiring resources. Retain these handles until - /// explicit cleanup succeeds; the Worker has dropped before this is returned. + /// A construction or activation step failed after acquiring resources. Retain + /// these handles until explicit cleanup succeeds. Consuming constructors drop + /// the Worker before returning; in-place activation leaves its owner responsible + /// for dropping it before allocation release. #[error("{source}")] ConstructionCleanup { #[source] From 1b13563e196f7a199cd1422dbe3472b0e90d61bb Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 09:16:37 +0900 Subject: [PATCH 08/28] feat: runtime commit restore decisions before execution publication Persist explicit restore commit evidence, reconcile canonical write uncertainty, and publish the same prepared controller before Accepted. Keep owned cleanup and publication failures distinct. Preserve optional transport/store feature boundaries and adapt external controller constructors. --- crates/worker-runtime/Cargo.toml | 4 +- crates/worker-runtime/src/execution.rs | 27 + crates/worker-runtime/src/fs_store.rs | 587 ++++++++++- crates/worker-runtime/src/http_server.rs | 84 ++ .../src/http_server/runtime_management_api.rs | 55 +- crates/worker-runtime/src/resource.rs | 2 + .../src/retention/runtime_v5_tests.rs | 38 +- crates/worker-runtime/src/runtime.rs | 674 +++++++++--- crates/worker-runtime/src/test_support.rs | 11 +- .../worker-runtime/src/test_support_tests.rs | 33 + crates/worker-runtime/src/worker_backend.rs | 969 +++++++++++++----- .../src/worker_backend/restore_recovery.rs | 10 +- .../worker_backend_restore_candidate_tests.rs | 781 ++++++++++++-- .../worker_backend_restore_failure_tests.rs | 79 +- ...orker_backend_restore_publication_tests.rs | 535 ++++++++++ crates/worker-runtime/src/worker_source.rs | 55 +- .../src/server_assignment_attention_tests.rs | 1 + .../src/server_assignment_delivery_tests.rs | 1 + 18 files changed, 3348 insertions(+), 598 deletions(-) create mode 100644 crates/worker-runtime/src/worker_backend_restore_publication_tests.rs diff --git a/crates/worker-runtime/Cargo.toml b/crates/worker-runtime/Cargo.toml index 32f6ea537..e61b4fe4f 100644 --- a/crates/worker-runtime/Cargo.toml +++ b/crates/worker-runtime/Cargo.toml @@ -22,14 +22,14 @@ fs-store = ["dep:fs4"] # Offline fixture tooling only; never enabled by the product defaults. test-support = ["fs-store", "ws-server"] http-server = ["dep:axum", "dep:tower", "dep:reqwest"] -ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"] +ws-server = ["http-server", "axum/ws", "tokio/sync"] [dependencies] agen.workspace = true async-trait.workspace = true base64.workspace = true axum = { workspace = true, optional = true } -futures = { workspace = true, optional = true } +futures.workspace = true decodal.workspace = true flow = { path = "../flow" } fs4 = { workspace = true, optional = true } diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index 719811ad0..b95347102 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -650,6 +650,24 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { self.restore_worker(request) } + /// Discard only the exact unpublished operation after Runtime has proved + /// its durable decision uncommitted. This is not Stop and must not affect a + /// connected execution. RolledBack requires confirmed cleanup and no repair. + fn discard_uncommitted_restore( + &self, + _operation_id: WorkerLifecycleOperationId, + _worker_ref: &WorkerRef, + ) -> WorkerExecutionSpawnResult { + WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::unsupported( + WorkerExecutionOperation::Restore, + "backend cannot confirm unpublished restore cleanup", + ), + worker_state: None, + workdir_attachments: Vec::new(), + } + } + fn activate_restored_worker( &self, _operation_id: WorkerLifecycleOperationId, @@ -927,6 +945,15 @@ impl WorkerExecutionBackendRef { self.backend.reconcile_restore(request) } + pub(crate) fn discard_uncommitted_restore( + &self, + operation_id: WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> WorkerExecutionSpawnResult { + self.backend + .discard_uncommitted_restore(operation_id, worker_ref) + } + pub(crate) fn activate_restored_worker( &self, operation_id: WorkerLifecycleOperationId, diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index dcfb1b42c..5418550d1 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -43,7 +43,8 @@ use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; #[path = "test_support.rs"] pub mod test_support; -const SCHEMA_VERSION: u32 = 12; +pub(crate) const SCHEMA_VERSION: u32 = 13; +const RESTORE_ADMISSION_SCHEMA_VERSION: u32 = 12; // Frozen source schemas: do not move their meaning when adding a migration. const SOURCE_SCHEMA_VERSION: u32 = 11; const EXECUTION_SCHEMA_VERSION: u32 = 10; @@ -124,6 +125,8 @@ pub struct FsRuntimeStore { root: PathBuf, _owner_lock: Option>, worker_write_fault: Arc, + #[cfg(test)] + worker_reconcile_sync_fault: Arc, } impl PartialEq for FsRuntimeStore { @@ -164,6 +167,12 @@ impl FsRuntimeStore { .store(fault as u8, Ordering::Release); } + #[cfg(test)] + pub(crate) fn fail_next_worker_reconcile_sync(&self) { + self.worker_reconcile_sync_fault + .store(true, Ordering::Release); + } + pub(crate) fn open_or_create( root: PathBuf, runtime_id: &str, @@ -223,6 +232,8 @@ impl FsRuntimeStore { root, _owner_lock: Some(owner_lock), worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), + #[cfg(test)] + worker_reconcile_sync_fault: Default::default(), }; let state = if snapshot_exists { Some(store.load_runtime_state()?) @@ -261,6 +272,124 @@ impl FsRuntimeStore { Ok(()) } + /// Read one canonical aggregate without migration, cleanup, or tolerant + /// startup enumeration. Reconciliation must not interpret an unreadable + /// record as absent or replace it with an older in-memory projection. + #[cfg(test)] + pub(crate) fn read_worker_record( + &self, + worker_ref: &WorkerRef, + ) -> Result { + self.read_worker_record_file(worker_ref) + .map(|(record, _)| record) + } + + /// Reobserve and sync canonical authority after an uncertain aggregate write. + /// + /// The caller must hold the Worker operation lock; this store retains the + /// Runtime owner lock. Sync the same file that was validated, then its directory + /// and the directories owning newly created Worker paths. Read equality alone + /// cannot discharge a failed post-rename sync. Any error leaves the caller's + /// write outcome unresolved; this never writes an older memory projection. + /// A returned record proves neither a matching operation nor live activation: + /// the lifecycle owner must still compare request/operation, Stop and receipt. + pub(crate) fn read_durable_worker_record( + &self, + worker_ref: &WorkerRef, + ) -> Result { + self.read_durable_worker_record_with_sync(worker_ref, |file| { + #[cfg(test)] + if self + .worker_reconcile_sync_fault + .swap(false, Ordering::AcqRel) + { + return Err(std::io::Error::other( + "injected reconciliation sync failure", + )); + } + file.sync_all() + }) + } + + /// Compare the entire typed aggregate, including request owner, receipt, + /// attachments and Stop, after durability is confirmed under owner locks. + pub(crate) fn reconcile_worker_record( + &self, + expected: &PersistedWorkerRecord, + ) -> Result<(PersistedWorkerRecord, bool), RuntimeError> { + let observed = self.read_durable_worker_record(&expected.worker_ref)?; + let document = |record: &PersistedWorkerRecord| { + serde_json::to_value(WorkerAggregateRecord::from_persisted(record)) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string())) + }; + let matches = document(&observed)? == document(expected)?; + Ok((observed, matches)) + } + + fn read_durable_worker_record_with_sync( + &self, + worker_ref: &WorkerRef, + mut sync: impl FnMut(&File) -> std::io::Result<()>, + ) -> Result { + if self._owner_lock.is_none() { + return Err(RuntimeError::InvalidRequest( + "Worker aggregate reconciliation requires the Runtime store owner lock".to_string(), + )); + } + let (record, file) = self.read_worker_record_file(worker_ref)?; + let worker_dir = self.worker_dir(&worker_ref.worker_id); + let path = worker_dir.join(WORKER_FILE); + let unknown = |source| RuntimeError::StoreCommitOutcomeUnknown { + operation: "sync reconciled Worker aggregate", + path: path.clone(), + source, + }; + sync(&file).map_err(&unknown)?; + for directory in [worker_dir, self.root.join(WORKERS_DIR), self.root.clone()] { + let file = File::open(directory).map_err(&unknown)?; + sync(&file).map_err(&unknown)?; + } + Ok(record) + } + + fn read_worker_record_file( + &self, + worker_ref: &WorkerRef, + ) -> Result<(PersistedWorkerRecord, File), RuntimeError> { + let worker_dir = self.worker_dir(&worker_ref.worker_id); + for directory in [&self.root, &self.root.join(WORKERS_DIR), &worker_dir] { + let metadata = fs::symlink_metadata(directory).map_err(|source| { + if source.kind() == std::io::ErrorKind::NotFound { + RuntimeError::StoreMissing { + operation: "reconcile Worker aggregate", + path: directory.to_path_buf(), + } + } else { + runtime_io_error("reconcile Worker aggregate", directory, source) + } + })?; + if !metadata.file_type().is_dir() { + return Err(RuntimeError::StoreCorrupt { + operation: "reconcile Worker aggregate", + path: directory.to_path_buf(), + message: "Worker aggregate directory is not a regular directory".to_string(), + }); + } + } + let path = worker_dir.join(WORKER_FILE); + let (aggregate, file): (WorkerAggregateRecord, File) = + read_bounded_json_file(&path, "reconcile Worker aggregate")?; + let record = aggregate.validate(&path)?; + if record.worker_ref != *worker_ref || record.worker_id != worker_ref.worker_id { + return Err(RuntimeError::StoreCorrupt { + operation: "reconcile Worker aggregate", + path, + message: "Worker aggregate does not match the requested identity".to_string(), + }); + } + Ok((record, file)) + } + pub(crate) fn delete_worker_record(&self, worker_id: &WorkerId) -> Result<(), RuntimeError> { self.delete_worker_record_with_intent( worker_id, @@ -523,6 +652,7 @@ pub(crate) enum PersistedWorkerRestoreMode { pub(crate) struct PersistedWorkerRestoreOperation { pub(crate) operation_id: WorkerLifecycleOperationId, pub(crate) mode: PersistedWorkerRestoreMode, + pub(crate) commit: crate::runtime::RestoreCommitFact, pub(crate) request: CreateWorkerRequest, pub(crate) binding: Option, pub(crate) last_settled_status: WorkerStatus, @@ -648,8 +778,8 @@ fn plan_runtime_store_migration( format!("Runtime store schema version {schema_version} is out of range"), ) })?; - let staging = migration_sibling(root, "schema-v12-staging")?; - let backup = migration_sibling(root, "pre-schema-v12-backup")?; + let staging = migration_sibling(root, "schema-v13-staging")?; + let backup = migration_sibling(root, "pre-schema-v13-backup")?; if staging.exists() || backup.exists() { return Err(runtime_store_corrupt( root, @@ -698,11 +828,12 @@ fn plan_runtime_store_migration( | AGGREGATE_SCHEMA_VERSION | EXECUTION_SCHEMA_VERSION | SOURCE_SCHEMA_VERSION + | RESTORE_ADMISSION_SCHEMA_VERSION ) { return Err(runtime_store_corrupt( &runtime_path, format!( - "unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, or {SCHEMA_VERSION}" + "unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, or {SCHEMA_VERSION}" ), )); } @@ -1034,10 +1165,10 @@ fn migrate_worker_record( ) -> Result { if matches!( source_schema_version, - EXECUTION_SCHEMA_VERSION | SOURCE_SCHEMA_VERSION + EXECUTION_SCHEMA_VERSION | SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION ) { - // Schemas 10 and 11 already use the current Worker aggregate layout. - // Only advance its version; never regenerate Restore admission identity. + // Admission identity survives migration. Older pending records did not + // distinguish a committed decision from preparation or failed cleanup. if identity_document .get("schema_version") .and_then(serde_json::Value::as_u64) @@ -1048,6 +1179,7 @@ fn migrate_worker_record( "Worker schema does not match Runtime migration source".into(), )); } + migrate_restore_commit_fact(&mut identity_document, identity_path)?; identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION); let aggregate: WorkerAggregateRecord = serde_json::from_value(identity_document) .map_err(|error| runtime_store_corrupt(identity_path, error.to_string()))?; @@ -1059,6 +1191,7 @@ fn migrate_worker_record( remove_legacy_job_input_counter(&mut identity_document); migrate_legacy_request_evidence(&mut identity_document) .map_err(|message| runtime_store_corrupt(identity_path, message.into()))?; + migrate_restore_commit_fact(&mut identity_document, identity_path)?; identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION); let aggregate: WorkerAggregateRecord = serde_json::from_value(identity_document).map_err(|error| { @@ -1125,6 +1258,42 @@ fn migrate_worker_record( Ok(WorkerAggregateRecord::from_v8(identity, execution)) } +// Missing commit evidence is not proof of rollback. This conversion applies +// only to versioned legacy aggregates, including Restore nested under Stop. +fn migrate_restore_commit_fact( + document: &mut serde_json::Value, + path: &Path, +) -> Result<(), RuntimeError> { + let execution = &mut document["execution_state"]; + if execution["state"] != "reconciliation_required" { + return Ok(()); + } + let operation = &mut execution["execution"]; + let restore = match operation["operation"].as_str() { + Some("restore") => Some(&mut operation["intent"]), + Some("stop") => operation["intent"] + .as_object_mut() + .and_then(|stop| stop.get_mut("pending_restore")) + .filter(|restore| !restore.is_null()), + _ => None, + }; + if let Some(restore) = restore { + let restore = restore.as_object_mut().ok_or_else(|| { + runtime_store_corrupt(path, "legacy pending Restore must be an object".into()) + })?; + if restore + .insert("commit".into(), serde_json::json!("legacy_unknown")) + .is_some() + { + return Err(runtime_store_corrupt( + path, + "legacy Restore contains unexpected commit evidence".into(), + )); + } + } + Ok(()) +} + // Frozen schema-v9 conversion. Preserve the exact token and all Restore owners // and receipts; generating a fresh token here would lose retry/admission authority. fn migrate_v9_restore_guard( @@ -1437,7 +1606,10 @@ fn migrate_runtime_document( object .entry("subject_create_recovery_fences") .or_insert_with(|| serde_json::json!({})); - if source_schema_version != SOURCE_SCHEMA_VERSION { + if !matches!( + source_schema_version, + SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION + ) { object.remove("workers"); // This legacy cache is not Runtime lifecycle or restore authority. Bundles // are reacquired from Workspace content authority, never renumbered/rehash-pinned here. @@ -1505,6 +1677,8 @@ fn validate_current_runtime_store(root: &Path) -> Result<(), RuntimeError> { root: root.to_path_buf(), _owner_lock: None, worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), + #[cfg(test)] + worker_reconcile_sync_fault: Default::default(), }; store.load_runtime_state().map(|_| ()) } @@ -1514,6 +1688,8 @@ fn validate_completed_migration_staging(root: &Path) -> Result<(), RuntimeError> root: root.to_path_buf(), _owner_lock: None, worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), + #[cfg(test)] + worker_reconcile_sync_fault: Default::default(), }; let state = store.load_runtime_state()?; if let Some(worker) = state.workers.values().find(|worker| { @@ -1559,14 +1735,16 @@ fn recover_runtime_store_migration(root: &Path) -> Result<(), RuntimeError> { "pre-schema-v10-backup", "schema-v11-staging", "pre-schema-v11-backup", + "schema-v12-staging", + "pre-schema-v12-backup", ] { let earlier = migration_sibling(root, suffix)?; if earlier.exists() { - return Err(runtime_store_corrupt(root, "unfinished earlier Runtime migration requires explicit recovery before schema-v12 startup".into())); + return Err(runtime_store_corrupt(root, "unfinished earlier Runtime migration requires explicit recovery before schema-v13 startup".into())); } } - let staging = migration_sibling(root, "schema-v12-staging")?; - let backup = migration_sibling(root, "pre-schema-v12-backup")?; + let staging = migration_sibling(root, "schema-v13-staging")?; + let backup = migration_sibling(root, "pre-schema-v13-backup")?; let root_exists = migration_directory_exists(root)?; let staging_exists = migration_directory_exists(&staging)?; let backup_exists = migration_directory_exists(&backup)?; @@ -1768,8 +1946,8 @@ fn migrate_runtime_store( if !plan.migration_required { return Ok(plan); } - let staging = migration_sibling(root, "schema-v12-staging")?; - let backup = migration_sibling(root, "pre-schema-v12-backup")?; + let staging = migration_sibling(root, "schema-v13-staging")?; + let backup = migration_sibling(root, "pre-schema-v13-backup")?; if staging.exists() || backup.exists() { return Err(runtime_store_corrupt( root, @@ -1966,9 +2144,12 @@ fn migrate_runtime_store_in_place( )); } } - // Schema 11 metadata was validated above but needs no conversion. + // Schema 11/12 metadata was validated above but needs no conversion. if let Some(metadata) = metadata - && plan.current_schema_version != SOURCE_SCHEMA_VERSION + && !matches!( + plan.current_schema_version, + SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION + ) { atomic_write_json( &migrated_dir.join(WORKER_METADATA_FILE), @@ -2597,6 +2778,19 @@ impl WorkerExecutionRecord { } fn read_bounded_json(path: &Path, operation: &'static str) -> Result +where + T: for<'de> Deserialize<'de>, +{ + read_bounded_json_file(path, operation).map(|(value, _)| value) +} + +// Keep the validated file open so reconciliation syncs the observed authority, +// not a separately reopened projection. The Runtime owner and Worker locks +// exclude cooperating writers throughout the read and sync. +fn read_bounded_json_file( + path: &Path, + operation: &'static str, +) -> Result<(T, File), RuntimeError> where T: for<'de> Deserialize<'de>, { @@ -2628,13 +2822,14 @@ where ), }); } - let file = File::open(path).map_err(|source| RuntimeError::StoreIo { + let mut file = File::open(path).map_err(|source| RuntimeError::StoreIo { operation, path: path.to_path_buf(), source, })?; let mut bytes = Vec::with_capacity(metadata.len() as usize); - file.take(MAX_WORKER_RECORD_BYTES + 1) + (&mut file) + .take(MAX_WORKER_RECORD_BYTES + 1) .read_to_end(&mut bytes) .map_err(|source| RuntimeError::StoreIo { operation, @@ -2648,11 +2843,12 @@ where message: format!("record exceeds limit {MAX_WORKER_RECORD_BYTES} while reading"), }); } - serde_json::from_slice(&bytes).map_err(|source| RuntimeError::StoreCorrupt { + let value = serde_json::from_slice(&bytes).map_err(|source| RuntimeError::StoreCorrupt { operation, path: path.to_path_buf(), message: source.to_string(), - }) + })?; + Ok((value, file)) } fn read_json(path: &Path, operation: &'static str) -> Result @@ -2913,7 +3109,7 @@ mod tests { let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-test"); let plan = FsRuntimeStore::migration_plan(&options).unwrap(); assert_eq!(plan.current_schema_version, 11); - assert_eq!(plan.target_schema_version, 12); + assert_eq!(plan.target_schema_version, SCHEMA_VERSION); assert!(plan.migration_required); assert_eq!(plan.worker_count, 1); assert_eq!(plan.migrated_worker_aggregate_count, 0); @@ -2928,13 +3124,13 @@ mod tests { let opened = FsRuntimeStore::open_or_create(root.clone(), "runtime-test").unwrap(); let state = opened.state.as_ref().unwrap(); let persisted_worker = state.workers.get(&worker_id).unwrap(); - expected_worker["schema_version"] = serde_json::json!(12); + expected_worker["schema_version"] = serde_json::json!(SCHEMA_VERSION); assert_eq!( serde_json::to_value(WorkerAggregateRecord::from_persisted(persisted_worker)) .unwrap(), expected_worker ); - runtime["schema_version"] = serde_json::json!(12); + runtime["schema_version"] = serde_json::json!(SCHEMA_VERSION); if fence_state == "absent" { runtime["subject_create_recovery_fences"] = serde_json::json!({}); } @@ -2970,8 +3166,8 @@ mod tests { drop(reopened); let plan = FsRuntimeStore::migrate(&options).unwrap(); assert!(!plan.migration_required); - assert_eq!(plan.current_schema_version, 12); - assert_eq!(plan.target_schema_version, 12); + assert_eq!(plan.current_schema_version, SCHEMA_VERSION); + assert_eq!(plan.target_schema_version, SCHEMA_VERSION); assert!( !FsRuntimeStore::migration_plan(&options) .unwrap() @@ -2982,12 +3178,12 @@ mod tests { assert_eq!(fs::read(&metadata_path).unwrap(), metadata); assert_eq!(fs::read(&history_path).unwrap(), history); assert!( - !migration_sibling(&root, "schema-v12-staging") + !migration_sibling(&root, "schema-v13-staging") .unwrap() .exists() ); assert!( - !migration_sibling(&root, "pre-schema-v12-backup") + !migration_sibling(&root, "pre-schema-v13-backup") .unwrap() .exists() ); @@ -3456,7 +3652,7 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime-store"); write_empty_schema_v8_store(&root); - let staging = migration_sibling(&root, "schema-v12-staging").unwrap(); + let staging = migration_sibling(&root, "schema-v13-staging").unwrap(); fs::create_dir_all(&staging).unwrap(); fs::write(staging.join("partial"), b"partial").unwrap(); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); @@ -3471,7 +3667,7 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime-store"); write_empty_schema_v8_store(&root); - let backup = migration_sibling(&root, "pre-schema-v12-backup").unwrap(); + let backup = migration_sibling(&root, "pre-schema-v13-backup").unwrap(); fs::rename(&root, &backup).unwrap(); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); @@ -3485,10 +3681,10 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime-store"); write_empty_schema_v8_store(&root); - let staging = migration_sibling(&root, "schema-v12-staging").unwrap(); + let staging = migration_sibling(&root, "schema-v13-staging").unwrap(); copy_runtime_tree(&root, &root, &staging).unwrap(); migrate_runtime_store_in_place(&staging, "test-runtime").unwrap(); - let backup = migration_sibling(&root, "pre-schema-v12-backup").unwrap(); + let backup = migration_sibling(&root, "pre-schema-v13-backup").unwrap(); fs::rename(&root, &backup).unwrap(); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); @@ -3503,10 +3699,10 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime-store"); write_empty_schema_v8_store(&root); - let staging = migration_sibling(&root, "schema-v12-staging").unwrap(); + let staging = migration_sibling(&root, "schema-v13-staging").unwrap(); fs::create_dir_all(&staging).unwrap(); fs::write(staging.join(RUNTIME_FILE), b"{\"schema_version\":8}").unwrap(); - let backup = migration_sibling(&root, "pre-schema-v12-backup").unwrap(); + let backup = migration_sibling(&root, "pre-schema-v13-backup").unwrap(); fs::rename(&root, &backup).unwrap(); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); @@ -3523,7 +3719,7 @@ mod tests { let root = temp.path().join("runtime-store"); write_empty_schema_v8_store(&root); migrate_runtime_store_in_place(&root, "test-runtime").unwrap(); - let backup = migration_sibling(&root, "pre-schema-v12-backup").unwrap(); + let backup = migration_sibling(&root, "pre-schema-v13-backup").unwrap(); write_empty_schema_v8_store(&backup); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); @@ -3538,6 +3734,8 @@ mod tests { "pre-schema-v10-backup", "schema-v11-staging", "pre-schema-v11-backup", + "schema-v12-staging", + "pre-schema-v12-backup", ] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime-store"); @@ -3692,6 +3890,7 @@ mod tests { let error = opened.store.write_worker_record(&worker).unwrap_err(); assert!(matches!(error, RuntimeError::StoreIo { .. })); assert!(!aggregate_path.exists()); + assert!(opened.store.read_worker_record(&worker.worker_ref).is_err()); worker.created_at_ms = Some(1); opened.store.write_worker_record(&worker).unwrap(); @@ -3707,6 +3906,17 @@ mod tests { let committed: WorkerAggregateRecord = read_bounded_json(&aggregate_path, "read test Worker aggregate").unwrap(); assert_eq!(committed.created_at_ms, Some(2)); + let before = fs::read(&aggregate_path).unwrap(); + assert_eq!( + opened + .store + .read_worker_record(&worker.worker_ref) + .unwrap() + .created_at_ms, + Some(2), + "unknown write must be reconciled from the canonical record" + ); + assert_eq!(fs::read(&aggregate_path).unwrap(), before); assert!( !aggregate_path .with_file_name(WORKER_EXECUTION_FILE) @@ -3714,6 +3924,317 @@ mod tests { ); } + #[test] + fn canonical_worker_read_rejects_unreadable_or_wrong_identity_without_repair() { + let temp = tempfile::tempdir().unwrap(); + let opened = + FsRuntimeStore::open_or_create(temp.path().join("runtime-store"), "test-runtime") + .unwrap(); + let requested = WorkerRef::new(WorkerId::now_v7()); + let path = opened + .store + .worker_dir(&requested.worker_id) + .join(WORKER_FILE); + assert!(opened.store.read_worker_record(&requested).is_err()); + assert!(!path.exists()); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let foreign = WorkerId::now_v7(); + let foreign_record = migrate_worker_record( + schema_v7_worker_document(foreign), + LEGACY_SCHEMA_VERSION, + None, + &path + .parent() + .unwrap() + .parent() + .unwrap() + .join(foreign.to_string()) + .join(WORKER_FILE), + ) + .unwrap(); + let documents = [ + b"not-json".to_vec(), + serde_json::to_vec(&foreign_record).unwrap(), + ]; + for document in documents { + fs::write(&path, &document).unwrap(); + assert!(opened.store.read_worker_record(&requested).is_err()); + assert_eq!(fs::read(&path).unwrap(), document); + } + File::create(&path) + .unwrap() + .set_len(MAX_WORKER_RECORD_BYTES + 1) + .unwrap(); + assert!(opened.store.read_worker_record(&requested).is_err()); + assert_eq!( + fs::metadata(&path).unwrap().len(), + MAX_WORKER_RECORD_BYTES + 1 + ); + } + + fn canonical_worker_store() -> ( + tempfile::TempDir, + FsRuntimeStore, + PersistedWorkerRecord, + PathBuf, + ) { + let temp = tempfile::tempdir().unwrap(); + let opened = + FsRuntimeStore::open_or_create(temp.path().join("runtime-store"), "test-runtime") + .unwrap(); + let worker_id = WorkerId::now_v7(); + let path = opened.store.worker_dir(&worker_id).join(WORKER_FILE); + let mut worker = migrate_worker_record( + schema_v7_worker_document(worker_id), + LEGACY_SCHEMA_VERSION, + None, + &path, + ) + .unwrap() + .validate(&path) + .unwrap(); + worker.created_at_ms = Some(1); + opened.store.write_worker_record(&worker).unwrap(); + (temp, opened.store, worker, path) + } + + #[test] + fn legacy_pending_restore_migrates_as_unknown_and_preserves_stop_ownership() { + use crate::runtime::RestoreCommitFact; + for under_stop in [false, true] { + let (_temp, store, mut worker, path) = canonical_worker_store(); + let root = store.root.clone(); + let PersistedWorkerExecutionState::Available(execution) = + worker.execution_state.clone() + else { + panic!("fixture execution unavailable") + }; + let restore = PersistedWorkerRestoreOperation { + operation_id: WorkerLifecycleOperationId::new(), + mode: PersistedWorkerRestoreMode::Explicit, + commit: RestoreCommitFact::Committed, + request: execution.request.clone(), + binding: execution.binding.clone(), + last_settled_status: worker.status, + }; + let operation = if under_stop { + PersistedWorkerLifecycleOperation::Stop(PersistedWorkerStopOperation { + operation_id: WorkerLifecycleOperationId::new(), + pending_restore: Some(restore), + request: Some(execution.request), + binding: execution.binding, + restore_intent: execution.restore_intent, + last_settled_status: worker.status, + }) + } else { + PersistedWorkerLifecycleOperation::Restore(restore) + }; + worker.execution_state = + PersistedWorkerExecutionState::ReconciliationRequired(operation); + let mut legacy = + serde_json::to_value(WorkerAggregateRecord::from_persisted(&worker)).unwrap(); + legacy["schema_version"] = serde_json::json!(RESTORE_ADMISSION_SCHEMA_VERSION); + let intent = &mut legacy["execution_state"]["execution"]["intent"]; + let restore = if under_stop { + &mut intent["pending_restore"] + } else { + intent + }; + restore.as_object_mut().unwrap().remove("commit"); + let expected_request = restore["request"].clone(); + let before_guard = legacy["restore_guard"].clone(); + fs::write(&path, serde_json::to_vec(&legacy).unwrap()).unwrap(); + write_empty_schema_v8_store(&root); + let runtime_path = root.join(RUNTIME_FILE); + let mut runtime: serde_json::Value = read_json(&runtime_path, "read fixture").unwrap(); + runtime["schema_version"] = serde_json::json!(RESTORE_ADMISSION_SCHEMA_VERSION); + fs::write(&runtime_path, serde_json::to_vec(&runtime).unwrap()).unwrap(); + drop(store); + + let opened = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); + let saved = opened.store.read_worker_record(&worker.worker_ref).unwrap(); + let PersistedWorkerExecutionState::ReconciliationRequired(operation) = + saved.execution_state + else { + panic!("migration discarded pending ownership") + }; + let restored = match operation { + PersistedWorkerLifecycleOperation::Restore(restore) => restore, + PersistedWorkerLifecycleOperation::Stop(stop) => stop.pending_restore.unwrap(), + }; + assert_eq!(restored.commit, RestoreCommitFact::LegacyUnknown); + assert_eq!( + serde_json::to_value(restored.request).unwrap(), + expected_request + ); + let disk: serde_json::Value = read_json(&path, "read migrated Worker").unwrap(); + assert_eq!(disk["restore_guard"], before_guard); + assert_eq!(disk["worker_ref"], legacy["worker_ref"]); + let settled = disk["execution_state"]["execution"]["intent"].clone(); + let legacy_intent = &legacy["execution_state"]["execution"]["intent"]; + if under_stop { + assert_eq!(settled["operation_id"], legacy_intent["operation_id"]); + assert_eq!( + settled["pending_restore"]["operation_id"], + legacy_intent["pending_restore"]["operation_id"] + ); + } else { + assert_eq!(settled["operation_id"], legacy_intent["operation_id"]); + } + drop(opened); + let reopened = FsRuntimeStore::open_or_create(root, "test-runtime").unwrap(); + assert!( + reopened + .store + .read_worker_record(&worker.worker_ref) + .is_ok() + ); + } + } + + #[test] + fn uncertain_worker_write_sync_failure_stays_unknown_until_sync_recovers() { + // Real filesystem aggregate/rename/read/validation; substitute only a + // failing file/directory sync syscall, never the reconciliation owner. + let (_temp, store, mut worker, path) = canonical_worker_store(); + worker.created_at_ms = Some(2); + store.fail_next_worker_write(AtomicWriteFault::AfterRename); + assert!(matches!( + store.write_worker_record(&worker), + Err(RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); + let bytes = fs::read(&path).unwrap(); + assert_eq!( + store + .read_worker_record(&worker.worker_ref) + .unwrap() + .created_at_ms, + Some(2) + ); + // The readable replacement is not a durability confirmation. Inject + // failure at the file and at each directory owning its canonical path. + for fail_at in 0..4 { + let mut sync_count = 0; + let result = store.read_durable_worker_record_with_sync(&worker.worker_ref, |file| { + let index = sync_count; + sync_count += 1; + if index == fail_at { + Err(std::io::Error::other( + "injected reconciliation sync failure", + )) + } else { + file.sync_all() + } + }); + assert!(matches!( + result, + Err(RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); + assert_eq!(fs::read(&path).unwrap(), bytes); + } + let reconciled = store + .read_durable_worker_record(&worker.worker_ref) + .unwrap(); + assert_eq!(reconciled.created_at_ms, Some(2)); + assert_eq!(reconciled.execution_state, worker.execution_state); + assert_eq!(fs::read(&path).unwrap(), bytes); + } + + #[test] + fn pre_rename_failure_durable_reread_preserves_old_canonical_record() { + let (_temp, store, mut worker, path) = canonical_worker_store(); + let bytes = fs::read(&path).unwrap(); + worker.created_at_ms = Some(2); + store.fail_next_worker_write(AtomicWriteFault::BeforeRename); + assert!(matches!( + store.write_worker_record(&worker), + Err(RuntimeError::StoreIo { .. }) + )); + let reconciled = store + .read_durable_worker_record(&worker.worker_ref) + .unwrap(); + assert_eq!(reconciled.created_at_ms, Some(1)); + assert_eq!(fs::read(&path).unwrap(), bytes); + } + + #[test] + fn durable_worker_reread_rejects_invalid_authority_before_sync_without_repair() { + let (_temp, store, worker, path) = canonical_worker_store(); + let bytes = fs::read(&path).unwrap(); + let foreign = WorkerId::now_v7(); + let foreign_record = migrate_worker_record( + schema_v7_worker_document(foreign), + LEGACY_SCHEMA_VERSION, + None, + &path + .parent() + .unwrap() + .parent() + .unwrap() + .join(foreign.to_string()) + .join(WORKER_FILE), + ) + .unwrap(); + for document in [ + b"not-json".to_vec(), + serde_json::to_vec(&foreign_record).unwrap(), + ] { + fs::write(&path, &document).unwrap(); + assert!(matches!( + store.read_durable_worker_record_with_sync(&worker.worker_ref, |_| { + panic!("invalid authority must not reach durability confirmation") + }), + Err(RuntimeError::StoreCorrupt { .. }) + )); + assert_eq!(fs::read(&path).unwrap(), document); + } + File::create(&path) + .unwrap() + .set_len(MAX_WORKER_RECORD_BYTES + 1) + .unwrap(); + assert!(matches!( + store.read_durable_worker_record_with_sync(&worker.worker_ref, |_| { + panic!("oversized authority must not reach durability confirmation") + }), + Err(RuntimeError::StoreCorrupt { .. }) + )); + assert_eq!( + fs::metadata(&path).unwrap().len(), + MAX_WORKER_RECORD_BYTES + 1 + ); + fs::remove_file(&path).unwrap(); + assert!(matches!( + store.read_durable_worker_record_with_sync(&worker.worker_ref, |_| { + panic!("missing authority must not reach durability confirmation") + }), + Err(RuntimeError::StoreMissing { .. }) + )); + assert!(!path.exists()); + fs::write(&path, &bytes).unwrap(); + assert!(store.read_durable_worker_record(&worker.worker_ref).is_ok()); + assert_eq!(fs::read(&path).unwrap(), bytes); + } + + #[cfg(unix)] + #[test] + fn canonical_worker_reread_rejects_symlinked_worker_directory() { + let (_temp, store, worker, path) = canonical_worker_store(); + let worker_dir = path.parent().unwrap(); + let retained_dir = worker_dir.with_extension("retained"); + fs::rename(worker_dir, &retained_dir).unwrap(); + std::os::unix::fs::symlink(&retained_dir, worker_dir).unwrap(); + let bytes = fs::read(retained_dir.join(WORKER_FILE)).unwrap(); + assert!(matches!( + store.read_worker_record(&worker.worker_ref), + Err(RuntimeError::StoreCorrupt { .. }) + )); + assert!(matches!( + store.read_durable_worker_record(&worker.worker_ref), + Err(RuntimeError::StoreCorrupt { .. }) + )); + assert_eq!(fs::read(retained_dir.join(WORKER_FILE)).unwrap(), bytes); + } + #[test] fn schema_v8_pair_migrates_to_one_worker_aggregate() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 57b7d0eb6..0fd7d7e91 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -25,6 +25,7 @@ use crate::interaction::{WorkerInput, WorkerInteractionAck}; use crate::management::{RuntimeSummary, WorkerDeleteResult}; #[cfg(feature = "ws-server")] use crate::observation::WorkerObservationCursor; +#[cfg(feature = "fs-store")] use crate::retention::{ WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory, }; @@ -1567,6 +1568,7 @@ fn protocol_error_event(message: impl Into) -> protocol::Event { } } +#[cfg(feature = "fs-store")] #[allow(dead_code)] async fn worker_retention_inventory( State(state): State, @@ -1588,6 +1590,7 @@ async fn worker_retention_inventory( .map_err(RuntimeHttpRestError::runtime) } +#[cfg(feature = "fs-store")] #[allow(dead_code)] async fn execute_worker_retention( State(state): State, @@ -2620,6 +2623,87 @@ mod tests { .unwrap() } + // Generated management routes and real Workspace capability middleware stay + // connected even when this process has no filesystem retention provider. + #[cfg(not(feature = "fs-store"))] + #[tokio::test] + async fn memory_retention_routes_preserve_auth_and_return_typed_unavailable() { + let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap(); + let trust = workspace_trust(&identity); + let runtime = Runtime::new_memory(); + let app = runtime_http_router_with_workspace_auth( + runtime, + None, + workspace_test_auth(vec![trust.clone()]), + ); + let worker_id = WorkerId::now_v7(); + let body = runtime_api::WorkerRetentionExecutionRequest { + operation_id: "memory-retention".into(), + input_fingerprint: "input".into(), + archive_id: None, + workspace_id: "workspace-a".into(), + source_runtime_id: "runtime-test".into(), + worker_id, + source_created_at: "2026-01-01T00:00:00Z".into(), + removed_at: "2026-01-02T00:00:00Z".into(), + effective_profile: None, + retention_class: None, + policy_id: "policy".into(), + session_disposition: runtime_api::SessionDisposition::Purge, + diagnostics_disposition: runtime_api::DiagnosticsDisposition::Purge, + }; + for (method, path, body) in [ + ( + "GET", + format!("/v1/workers/{worker_id}/retention/inventory"), + vec![], + ), + ( + "POST", + format!("/v1/workers/{worker_id}/retention/execute"), + serde_json::to_vec(&body).unwrap(), + ), + ] { + let unsigned = app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(&path) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(body.clone())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED); + let claims = signed_claims(&trust, method, &path, &body); + let response = + signed_request(app.clone(), &identity, &claims, body, "workspace-a").await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + let error: RuntimeHttpErrorResponse = read_json(response).await; + assert_eq!(error.error.code, "worker_retention_unavailable"); + } + for (mut value, expected_status, expected_code) in [ + (body.clone(), StatusCode::NOT_FOUND, "worker_not_found"), + (body.clone(), StatusCode::BAD_REQUEST, "worker_id_mismatch"), + ] { + if expected_status == StatusCode::NOT_FOUND { + value.workspace_id = "workspace-b".into(); + } else { + value.worker_id = WorkerId::now_v7(); + } + let path = format!("/v1/workers/{worker_id}/retention/execute"); + let bytes = serde_json::to_vec(&value).unwrap(); + let claims = signed_claims(&trust, "POST", &path, &bytes); + let response = + signed_request(app.clone(), &identity, &claims, bytes, "workspace-a").await; + assert_eq!(response.status(), expected_status); + let error: RuntimeHttpErrorResponse = read_json(response).await; + assert_eq!(error.error.code, expected_code); + } + } + #[tokio::test] async fn ordinary_signed_ping_succeeds_without_handshake_and_rejects_replay() { let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap(); diff --git a/crates/worker-runtime/src/http_server/runtime_management_api.rs b/crates/worker-runtime/src/http_server/runtime_management_api.rs index e7daf4906..8bcdb123c 100644 --- a/crates/worker-runtime/src/http_server/runtime_management_api.rs +++ b/crates/worker-runtime/src/http_server/runtime_management_api.rs @@ -856,13 +856,25 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { ) })?; let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?; - let inventory = self - .state - .runtime - .worker_retention_inventory(&scope.workspace_id, &worker_ref) - .map_err(RuntimeHttpRestError::runtime) - .map_err(api_error)?; - response(inventory) + #[cfg(feature = "fs-store")] + { + let inventory = self + .state + .runtime + .worker_retention_inventory(&scope.workspace_id, &worker_ref) + .map_err(RuntimeHttpRestError::runtime) + .map_err(api_error)?; + response(inventory) + } + #[cfg(not(feature = "fs-store"))] + { + let _ = (scope, worker_ref); + Err(runtime_api::RuntimeApiError::new( + 503, + "worker_retention_unavailable", + "Worker retention requires filesystem storage", + )) + } } async fn execute_retention( @@ -880,27 +892,36 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { ) })?; let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?; - let request: WorkerRetentionExecutionRequest = request(value)?; - if request.workspace_id != scope.workspace_id { + if value.workspace_id != scope.workspace_id { return Err(runtime_api::RuntimeApiError::new( StatusCode::NOT_FOUND.as_u16(), "worker_not_found", "worker does not belong to the authenticated Workspace", )); } - if request.worker_id != worker_ref.worker_id { + if value.worker_id != worker_ref.worker_id { return Err(runtime_api::RuntimeApiError::new( StatusCode::BAD_REQUEST.as_u16(), "worker_id_mismatch", "Worker retention request worker_id does not match route worker_id", )); } - let result = self - .state - .runtime - .execute_worker_retention(&request) - .map_err(RuntimeHttpRestError::runtime) - .map_err(api_error)?; - response(result) + #[cfg(feature = "fs-store")] + { + let request: WorkerRetentionExecutionRequest = request(value)?; + let result = self + .state + .runtime + .execute_worker_retention(&request) + .map_err(RuntimeHttpRestError::runtime) + .map_err(api_error)?; + response(result) + } + #[cfg(not(feature = "fs-store"))] + Err(runtime_api::RuntimeApiError::new( + 503, + "worker_retention_unavailable", + "Worker retention requires filesystem storage", + )) } } diff --git a/crates/worker-runtime/src/resource.rs b/crates/worker-runtime/src/resource.rs index c9a4dc0db..04eb2ca91 100644 --- a/crates/worker-runtime/src/resource.rs +++ b/crates/worker-runtime/src/resource.rs @@ -1,6 +1,8 @@ +#[cfg(feature = "http-server")] use crate::auth::BACKEND_RESOURCE_FETCH_PERMISSION; use crate::identity::WorkerId; use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef, sha256_hex}; +#[cfg(feature = "http-server")] use crate::workspace_request::RuntimeWorkspaceRequestClient; use async_trait::async_trait; use serde::{Deserialize, Serialize}; diff --git a/crates/worker-runtime/src/retention/runtime_v5_tests.rs b/crates/worker-runtime/src/retention/runtime_v5_tests.rs index 47d75c940..b14a77e80 100644 --- a/crates/worker-runtime/src/retention/runtime_v5_tests.rs +++ b/crates/worker-runtime/src/retention/runtime_v5_tests.rs @@ -3,7 +3,7 @@ //! only the empty segment envelope is built with current APIs. use super::tests::{request, write_json}; use super::*; -use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions}; +use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions, SCHEMA_VERSION}; use serde_json::{Value, json}; use session_store::{LogEntry, SegmentId, SessionId, Store, WorkerSessionStore}; use std::collections::BTreeMap; @@ -226,7 +226,7 @@ fn completed_v4_archive( #[test] fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() { - for runtime_schema in [11, 12] { + for runtime_schema in [11, 12, SCHEMA_VERSION] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); let (worker, session, segment) = runtime_fixture(&root, runtime_schema); @@ -329,7 +329,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() { (storage.checksum_sha256, storage.bytes, storage.file_count), (checksum, bytes, count) ); - if runtime_schema == 12 { + if runtime_schema == SCHEMA_VERSION { for name in [ PathBuf::from("runtime.json"), PathBuf::from(format!("workers/{worker}/worker.json")), @@ -346,8 +346,8 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() { ); drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap()); assert_eq!(tree_bytes(&root), once, "retry rewrote migrated evidence"); - assert!(!temp.path().join(".runtime.schema-v12-staging").exists()); - assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists()); + assert!(!temp.path().join(".runtime.schema-v13-staging").exists()); + assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists()); // session-storage, not the immutable receipt's old checksum, attests // current bytes. Tampering with those bytes must remain detectable. let path = archive_session @@ -360,7 +360,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() { #[test] fn corrupt_archive_does_not_block_startup_or_healthy_session_migration_and_repair_converges() { - for runtime_schema in [11, 12] { + for runtime_schema in [11, 12, SCHEMA_VERSION] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); let (worker, session, segment) = runtime_fixture(&root, runtime_schema); @@ -440,8 +440,8 @@ fn unmigratable_archive_preserves_evidence_across_startup_and_reopen() { ] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); - let (live_worker, live_session, live_segment) = runtime_fixture(&root, 12); - let (worker, _, segment) = stopped_worker(&root, 12); + let (live_worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION); + let (worker, _, segment) = stopped_worker(&root, SCHEMA_VERSION); let provider = FsWorkerRetentionProvider::new(&root); let path = provider .worker_dir(worker) @@ -542,8 +542,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() { ] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); - let (worker, _, _) = runtime_fixture(&root, 12); - completed_v4_archive(&root, 12); + let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION); + completed_v4_archive(&root, SCHEMA_VERSION); let provider = FsWorkerRetentionProvider::new(&root); fs::remove_dir_all(provider.worker_dir(worker)).unwrap(); let archive = provider.archive_dir("archive-a").unwrap(); @@ -575,8 +575,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() { fn archive_isolation_does_not_bypass_corrupt_erasure_authority() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); - runtime_fixture(&root, 12); - let (session, _, _, _) = completed_v4_archive(&root, 12); + runtime_fixture(&root, SCHEMA_VERSION); + let (session, _, _, _) = completed_v4_archive(&root, SCHEMA_VERSION); let ledger = root.join("retention/session-sources"); fs::create_dir_all(&ledger).unwrap(); fs::write(ledger.join(format!("{session}.json")), b"{broken\n").unwrap(); @@ -591,7 +591,7 @@ fn archive_isolation_does_not_bypass_corrupt_erasure_authority() { fn runtime_v5_probe_and_staging_skip_erased_originals() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); - let (worker, live_session, live_segment) = runtime_fixture(&root, 12); + let (worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION); let erased = root .join("orphaned-workers") .join(WorkerId::now_v7().to_string()); @@ -643,7 +643,7 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() { for corruption in ["broken-json", "future-schema"] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); - let (worker, session, segment) = runtime_fixture(&root, 12); + let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION); let session_root = root .join("workers") .join(worker.to_string()) @@ -674,8 +674,8 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() { assert_eq!(tree_bytes(&root), before); assert!(FsRuntimeStore::migrate(&options).is_err()); assert_eq!(tree_bytes(&root), before); - assert!(!temp.path().join(".runtime.schema-v12-staging").exists()); - assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists()); + assert!(!temp.path().join(".runtime.schema-v13-staging").exists()); + assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists()); } } @@ -686,7 +686,7 @@ fn current_runtime_session_migration_still_rejects_live_worker_sockets() { // or alias is needed, and the real live-socket migration refusal stays tested. let temp = tempfile::tempdir_in(".").unwrap(); let root = Path::new(temp.path().file_name().unwrap()).join("runtime"); - let (worker, _, _) = runtime_fixture(&root, 12); + let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION); let before = tree_bytes(&root); let runs = root.join("workers").join(worker.to_string()).join("runs"); fs::create_dir_all(runs.join("1")).unwrap(); @@ -697,7 +697,7 @@ fn current_runtime_session_migration_still_rejects_live_worker_sockets() { drop(listener); fs::remove_dir_all(runs).unwrap(); assert_eq!(tree_bytes(&root), before); - assert!(!temp.path().join(".runtime.schema-v12-staging").exists()); + assert!(!temp.path().join(".runtime.schema-v13-staging").exists()); } #[test] @@ -705,7 +705,7 @@ fn current_runtime_upgrades_v5_role_metadata_through_staging_without_reserializi { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); - let (worker, session, segment) = runtime_fixture(&root, 12); + let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION); let aggregate = root.join("workers").join(worker.to_string()); let path = aggregate .join("session/segments") diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index fc53df360..251e423c2 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -2105,6 +2105,16 @@ impl Runtime { // Authorization is read-only until token admission succeeds. state.ensure_workspace_owner(scope, false)?; } + if let Err(error) = self.lock()?.refresh_uncertain_worker_write(worker_ref) { + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_canonical_write_unknown", + sanitize_worker_create_failure_message(&error.to_string()), + )), + None, + )); + } if request.request_id.trim().is_empty() || request.request_id.len() > 512 || request.expected_observation_token.is_empty() @@ -2219,6 +2229,7 @@ impl Runtime { .unwrap_or_else(|| PendingWorkerRestore { operation_id: WorkerLifecycleOperationId::new(), mode: WorkerRestoreMode::Explicit, + commit: RestoreCommitFact::NotCommitted, last_settled_status: candidate.status, }); let operation_id = pending.operation_id; @@ -2242,10 +2253,9 @@ impl Runtime { receipt: None, }, ); - match state.persist_worker_record(&candidate) { + match state.persist_restore_worker_record(&candidate) { Ok(()) => {} Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { - state.workers.insert(worker_ref.worker_id, candidate); return Ok(( Some(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, @@ -2389,10 +2399,13 @@ impl Runtime { if candidate.restore_guard.active_request_id.is_some() || candidate.pending_restore.is_some() { + if result.state != WorkerRestoreState::Accepted { + candidate.ensure_restore_rollback_allowed()?; + } candidate.restore_guard.finish(result.clone()); candidate.pending_restore = None; candidate.restore_candidate_context = None; - state.persist_worker_record(&candidate)?; + state.persist_restore_worker_record(&candidate)?; state.workers.insert(worker_ref.worker_id, candidate); if result.state == WorkerRestoreState::Accepted { state.publish_worker_upsert(worker_ref.worker_id)?; @@ -2431,6 +2444,15 @@ impl Runtime { )); } let pending_restore = worker.pending_restore; + if pending_restore + .is_some_and(|pending| pending.commit == RestoreCommitFact::LegacyUnknown) + { + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_legacy_decision_unknown", + "The saved Restore lacks commit evidence; do not replay execution or infer rollback", + )); + } let reconcile = pending_restore.is_some() && worker .restore_guard @@ -2603,7 +2625,7 @@ impl Runtime { workdir_attachments.clone(), ); match commit { - Ok(worker) => { + Ok(_) => { if let Err(error) = backend.activate_restored_worker(operation_id, worker_ref) { @@ -2631,6 +2653,7 @@ impl Runtime { sanitize_worker_create_failure_message(&error.to_string()), )); } + let worker = self.lock()?.worker(worker_ref)?.detail(); let result = RuntimeWorkerRestoreResult::accepted(worker); if let Err(error) = self.finish_restore_receipt(worker_ref, &result) { return Ok(RuntimeWorkerRestoreResult::failed( @@ -2642,10 +2665,38 @@ impl Runtime { Ok(result) } Err(error) => { + let known_uncommitted = + !matches!(error, RuntimeError::StoreCommitOutcomeUnknown { .. }) + && self + .lock()? + .worker(worker_ref)? + .pending_restore + .is_some_and(|pending| { + pending.operation_id == operation_id + && pending.commit == RestoreCommitFact::NotCommitted + }); + if known_uncommitted { + if let WorkerExecutionSpawnResult::RolledBack(_) = + backend.discard_uncommitted_restore(operation_id, worker_ref) + { + self.discard_restore_candidate_context( + worker_ref, + &candidate_context, + )?; + let receipt = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "worker_restore_decision_not_committed", + sanitize_worker_create_failure_message(&error.to_string()), + ); + if self.settle_restore_rollback(worker_ref, &receipt).is_ok() { + return Ok(receipt); + } + } + } tracing::error!( worker_id = %worker_ref.worker_id, error = %error, - "Worker restore activated but commit did not confirm success; preserving operation and execution" + "Worker restore decision did not confirm success; preserving operation ownership" ); if let Err(retain_error) = self.retain_restore_execution_evidence( worker_ref, @@ -3601,6 +3652,7 @@ impl Runtime { self.ensure_worker_in_workspace(scope, worker_ref)?; } + self.lock()?.refresh_uncertain_worker_write(worker_ref)?; self.lock()?.ensure_subject_not_reserved(worker_ref, true)?; let (backend, pending_stop, pending_restore, status) = { let state = self.lock()?; @@ -4355,18 +4407,18 @@ impl Runtime { candidate.pending_restore = Some(PendingWorkerRestore { operation_id, mode, + commit: candidate + .pending_restore + .map_or(RestoreCommitFact::NotCommitted, |pending| pending.commit), last_settled_status: candidate.status, }); candidate.restore_candidate_context = Some(candidate_context); - match state.persist_worker_record(&candidate) { + match state.persist_restore_worker_record(&candidate) { Ok(()) => { state.workers.insert(worker_ref.worker_id, candidate); Ok(()) } - Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { - state.workers.insert(worker_ref.worker_id, candidate); - Err(error) - } + Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => Err(error), Err(error) => Err(error), } } @@ -4378,11 +4430,12 @@ impl Runtime { ) -> Result<(), RuntimeError> { let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); + candidate.ensure_restore_rollback_allowed()?; candidate.restore_guard.rotate(); candidate.restore_guard.finish(receipt.clone()); candidate.pending_restore = None; candidate.restore_candidate_context = None; - state.persist_worker_record(&candidate)?; + state.persist_restore_worker_record(&candidate)?; state.workers.insert(worker_ref.worker_id, candidate); Ok(()) } @@ -4398,7 +4451,11 @@ impl Runtime { let mut state = self.lock()?; state.ensure_worker_ref(worker_ref)?; let previous = state.worker(worker_ref)?; - if previous.pending_stop.is_some() { + if previous.pending_stop.is_some() + || previous + .pending_restore + .is_none_or(|pending| pending.commit == RestoreCommitFact::LegacyUnknown) + { return Err(RuntimeError::RestoreObservationConflict { worker_id: worker_ref.worker_id, }); @@ -4446,6 +4503,7 @@ impl Runtime { } let _ = candidate.apply_worker_state(&worker_state); if let Some(pending) = candidate.pending_restore.as_mut() { + pending.commit = RestoreCommitFact::Committed; pending.last_settled_status = candidate.status; } candidate.restore_intent = restore_intent_for_status(candidate.status); @@ -4459,7 +4517,7 @@ impl Runtime { // Persist the execution projection before publishing observations. Terminal // receipt settlement follows successful observation activation separately. - state.persist_worker_record(&candidate)?; + state.persist_restore_worker_record(&candidate)?; let detail = candidate.detail(); state.workers.insert(worker_ref.worker_id, candidate); @@ -4499,28 +4557,15 @@ impl Runtime { fn retain_restore_execution_evidence( &self, worker_ref: &WorkerRef, - _worker_state: protocol::WorkerStateSnapshot, - workdir_attachments: Vec, + _workdir_attachments: Vec, candidate_context: crate::execution::WorkerExecutionContext, ) -> Result<(), RuntimeError> { let mut state = self.lock()?; - let mut candidate = state.worker(worker_ref)?.clone(); - if candidate.pending_restore.is_none() { - return Err(RuntimeError::InvalidRequest(format!( - "worker {} has no pending restore operation", - worker_ref.worker_id - ))); - } - - candidate.restore_guard.rotate(); - candidate.execution_metadata_available = true; - candidate.execution_bound = true; - candidate.restore_candidate_context = Some(candidate_context); - candidate.workdir_attachments = workdir_attachments; - state.persist_worker_record(&candidate)?; - state.workers.insert(worker_ref.worker_id, candidate); - Ok(()) + // Keep the actual resource owner even if canonical authority is unreadable. + // Execution evidence cannot authorize overwriting a post-rename decision. + state.worker_mut(worker_ref)?.restore_candidate_context = Some(candidate_context); + state.refresh_uncertain_worker_write(worker_ref) } fn retain_create_reconciliation_pending( @@ -5080,6 +5125,14 @@ impl Runtime { ) })?; let provider = FsWorkerRetentionProvider::new(store.runtime_dir()); + if state + .unreconciled_worker_writes + .contains(&request.worker_id) + { + return Err(RuntimeError::InvalidRequest( + "Worker canonical write outcome must be reconciled before retention effects".into(), + )); + } if let Some(completed) = provider.completed_for(request)? { state.workers.remove(&request.worker_id); state.persist_runtime_snapshot()?; @@ -5178,6 +5231,9 @@ struct RuntimeState { #[cfg(feature = "fs-store")] next_diagnostic_id: u64, workers: BTreeMap, + // Ephemeral write barrier, not authority: only canonical reread + sync can + // release it. Prevents another mutation from persisting an older projection. + unreconciled_worker_writes: std::collections::BTreeSet, subject_create_recovery_fences: BTreeMap, subject_release_operations: BTreeMap< (WorkerId, String), @@ -5216,6 +5272,7 @@ impl RuntimeState { #[cfg(feature = "fs-store")] next_diagnostic_id: 1, workers: BTreeMap::new(), + unreconciled_worker_writes: Default::default(), subject_create_recovery_fences: BTreeMap::new(), subject_release_operations: BTreeMap::new(), workdir_operations: BTreeMap::new(), @@ -5249,6 +5306,7 @@ impl RuntimeState { #[cfg(feature = "fs-store")] next_diagnostic_id: 1, workers: BTreeMap::new(), + unreconciled_worker_writes: Default::default(), subject_create_recovery_fences: BTreeMap::new(), subject_release_operations: BTreeMap::new(), workdir_operations: BTreeMap::new(), @@ -5277,112 +5335,13 @@ impl RuntimeState { let diagnostics = persisted.diagnostics; let next_diagnostic_id = persisted.next_diagnostic_id; for (worker_id, worker) in persisted.workers { - let ( - request, - execution_metadata_available, - execution_bound, - restore_intent, - pending_restore, - pending_stop, - ) = match worker.execution_state { - PersistedWorkerExecutionState::Available(execution) => ( - Some(execution.request), - true, - execution.binding.is_some(), - execution.restore_intent, - None, - None, - ), - PersistedWorkerExecutionState::ReconciliationRequired(operation) => match operation - { - PersistedWorkerLifecycleOperation::Restore(operation) => { - let restore_intent = - restore_intent_for_status(operation.last_settled_status); - ( - Some(operation.request), - true, - operation.binding.is_some(), - restore_intent, - Some(PendingWorkerRestore { - operation_id: operation.operation_id, - mode: operation.mode.into(), - last_settled_status: operation.last_settled_status, - }), - None, - ) - } - PersistedWorkerLifecycleOperation::Stop(operation) => ( - operation.request.clone(), - operation.request.is_some(), - operation.binding.is_some(), - operation.restore_intent, - operation - .pending_restore - .map(|restore| PendingWorkerRestore { - operation_id: restore.operation_id, - mode: restore.mode.into(), - last_settled_status: restore.last_settled_status, - }), - Some(PendingWorkerStop { - operation_id: operation.operation_id, - last_settled_status: operation.last_settled_status, - }), - ), - }, - PersistedWorkerExecutionState::Unavailable => ( - None, - false, - false, - restore_intent_for_status(worker.status), - None, - None, - ), - }; - let logical_workdir_attachments = if worker.logical_workdir_attachments.is_empty() { - let claims = request - .as_ref() - .map(|request| request.workdir_attachments.as_slice()) - .unwrap_or_default(); - logical_workdir_attachments_from_statuses(&worker.workdir_attachments, claims) - } else { - worker.logical_workdir_attachments - }; - workers.insert( - worker_id, - WorkerRecord { - restore_guard: { - let mut guard = worker.restore_guard; - // A prior active controller is not live in this process. - // Stopped observations remain stable across restart. - if worker.status.is_active() && execution_bound { - guard.rotate(); - } - guard - }, - worker_ref: worker.worker_ref, - worker_id: worker.worker_id, - status: worker.status, - worker_state: None, - last_finished_submission_request_id: worker.last_finished_submission_request_id, - execution_reconstructed: true, - workspace_id: worker.workspace_id, - profile: worker.profile, - display_name: worker.display_name, - profile_source: worker.profile_source, - config_bundle: worker.config_bundle, - request, - created_at_ms: worker.created_at_ms, - execution_metadata_available, - execution_bound, - restore_intent, - pending_restore, - pending_stop, - restore_candidate_context: None, - workdir_attachments: worker.workdir_attachments, - logical_workdir_attachments, - internal_workers: InternalWorkerActivityProjection::default(), - }, - ); + let mut record = WorkerRecord::from_persisted(worker); + // A prior active Controller is not live in this process. Keep + // stopped observations stable; canonical rereads never rotate them. + if record.status.is_active() && record.execution_bound { + record.restore_guard.rotate(); + } + workers.insert(worker_id, record); } Ok(Self { @@ -5396,6 +5355,7 @@ impl RuntimeState { workspace_backend_resource_clients: BTreeMap::new(), next_diagnostic_id, workers, + unreconciled_worker_writes: Default::default(), subject_create_recovery_fences: persisted.subject_create_recovery_fences, subject_release_operations: persisted .subject_releases @@ -5458,8 +5418,80 @@ impl RuntimeState { Ok(()) } + /// An uncertain Restore write is resolved only from the locked canonical + /// aggregate and a successful durability barrier, never a second blind write. + fn persist_restore_worker_record(&mut self, worker: &WorkerRecord) -> Result<(), RuntimeError> { + match self.persist_worker_record(worker) { + Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + self.unreconciled_worker_writes.insert(worker.worker_id); + #[cfg(feature = "fs-store")] + if let RuntimePersistence::Fs(store) = &self.persistence { + match store.reconcile_worker_record(&worker.persisted_record()) { + Ok((observed, matches)) => { + self.apply_canonical_worker_record(observed)?; + if matches { + return Ok(()); + } + } + Err(reconcile_error) => { + tracing::warn!(worker_id = %worker.worker_id, error = %reconcile_error, + "Restore aggregate remains uncertain after canonical reconciliation"); + } + } + } + Err(error) + } + outcome => outcome, + } + } + + #[cfg(feature = "fs-store")] + fn apply_canonical_worker_record( + &mut self, + observed: PersistedWorkerRecord, + ) -> Result<(), RuntimeError> { + let previous = self.worker(&observed.worker_ref)?; + let mut current = WorkerRecord::from_persisted(observed); + // Canonical authority replaces only the saved projection. A read is + // neither process reconstruction nor evidence that a Controller closed. + current.execution_reconstructed = previous.execution_reconstructed; + current.restore_candidate_context = previous.restore_candidate_context.clone(); + current.internal_workers = previous.internal_workers.clone(); + if current.status == previous.status { + current.worker_state = previous.worker_state.clone(); + } + self.unreconciled_worker_writes.remove(¤t.worker_id); + self.workers.insert(current.worker_id, current); + Ok(()) + } + + fn refresh_uncertain_worker_write( + &mut self, + worker_ref: &WorkerRef, + ) -> Result<(), RuntimeError> { + if !self + .unreconciled_worker_writes + .contains(&worker_ref.worker_id) + { + return Ok(()); + } + #[cfg(feature = "fs-store")] + if let RuntimePersistence::Fs(store) = &self.persistence { + let observed = store.read_durable_worker_record(worker_ref)?; + return self.apply_canonical_worker_record(observed); + } + Err(RuntimeError::InvalidRequest( + "Worker canonical write outcome remains unresolved".into(), + )) + } + #[cfg(feature = "fs-store")] fn persist_worker_record(&self, worker: &WorkerRecord) -> Result<(), RuntimeError> { + if self.unreconciled_worker_writes.contains(&worker.worker_id) { + return Err(RuntimeError::InvalidRequest( + "Worker canonical write outcome must be reconciled before another mutation".into(), + )); + } if let Some(store) = self.fs_store() { store.write_worker_record(&worker.persisted_record())?; } @@ -5468,20 +5500,25 @@ impl RuntimeState { #[cfg(feature = "fs-store")] fn persist_worker(&self, worker_id: &WorkerId) -> Result<(), RuntimeError> { - if let Some(store) = self.fs_store() { + if self.fs_store().is_some() { let worker = self.workers .get(worker_id) .ok_or_else(|| RuntimeError::WorkerNotFound { worker_id: *worker_id, })?; - store.write_worker_record(&worker.persisted_record())?; + self.persist_worker_record(worker)?; } Ok(()) } #[cfg(feature = "fs-store")] fn delete_worker_record(&self, worker_id: &WorkerId) -> Result<(), RuntimeError> { + if self.unreconciled_worker_writes.contains(worker_id) { + return Err(RuntimeError::InvalidRequest( + "Worker canonical write outcome must be reconciled before removal".into(), + )); + } if let Some(store) = self.fs_store() { store.delete_worker_record(worker_id)?; } @@ -5950,6 +5987,7 @@ impl RuntimeState { .and_then(|worker| worker.workspace_id.clone()) .unwrap_or_else(|| "".to_string()); let mut candidate = self.worker(worker_ref)?.clone(); + candidate.ensure_restore_rollback_allowed()?; candidate.execution_bound = false; candidate.restore_guard.finish(receipt.clone()); @@ -6274,10 +6312,21 @@ impl InternalWorkerActivityProjection { } } +/// A durable decision is separate from both admission and a live Controller. +/// Legacy pending operations lack sufficient evidence to choose either branch. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum RestoreCommitFact { + NotCommitted, + Committed, + LegacyUnknown, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct PendingWorkerRestore { operation_id: WorkerLifecycleOperationId, mode: WorkerRestoreMode, + commit: RestoreCommitFact, last_settled_status: WorkerStatus, } @@ -6410,6 +6459,17 @@ struct WorkerRecord { } impl WorkerRecord { + fn ensure_restore_rollback_allowed(&self) -> Result<(), RuntimeError> { + if self + .pending_restore + .is_some_and(|pending| pending.commit != RestoreCommitFact::NotCommitted) + { + return Err(RuntimeError::InvalidRequest( + "A committed or legacy-unknown Restore cannot be settled as a rollback".into(), + )); + } + Ok(()) + } fn apply_logical_workdir_attachments( &mut self, mut attachments: Vec, @@ -6592,6 +6652,105 @@ impl WorkerRecord { } } + #[cfg(feature = "fs-store")] + fn from_persisted(worker: PersistedWorkerRecord) -> Self { + let ( + request, + execution_metadata_available, + execution_bound, + restore_intent, + pending_restore, + pending_stop, + ) = match worker.execution_state { + PersistedWorkerExecutionState::Available(execution) => ( + Some(execution.request), + true, + execution.binding.is_some(), + execution.restore_intent, + None, + None, + ), + PersistedWorkerExecutionState::ReconciliationRequired(operation) => match operation { + PersistedWorkerLifecycleOperation::Restore(operation) => { + let restore_intent = restore_intent_for_status(operation.last_settled_status); + ( + Some(operation.request), + true, + operation.binding.is_some(), + restore_intent, + Some(PendingWorkerRestore { + operation_id: operation.operation_id, + mode: operation.mode.into(), + commit: operation.commit, + last_settled_status: operation.last_settled_status, + }), + None, + ) + } + PersistedWorkerLifecycleOperation::Stop(operation) => ( + operation.request.clone(), + operation.request.is_some(), + operation.binding.is_some(), + operation.restore_intent, + operation + .pending_restore + .map(|restore| PendingWorkerRestore { + operation_id: restore.operation_id, + mode: restore.mode.into(), + commit: restore.commit, + last_settled_status: restore.last_settled_status, + }), + Some(PendingWorkerStop { + operation_id: operation.operation_id, + last_settled_status: operation.last_settled_status, + }), + ), + }, + PersistedWorkerExecutionState::Unavailable => ( + None, + false, + false, + restore_intent_for_status(worker.status), + None, + None, + ), + }; + let logical_workdir_attachments = if worker.logical_workdir_attachments.is_empty() { + let claims = request + .as_ref() + .map(|request| request.workdir_attachments.as_slice()) + .unwrap_or_default(); + logical_workdir_attachments_from_statuses(&worker.workdir_attachments, claims) + } else { + worker.logical_workdir_attachments + }; + Self { + restore_guard: worker.restore_guard, + worker_ref: worker.worker_ref, + worker_id: worker.worker_id, + status: worker.status, + worker_state: None, + last_finished_submission_request_id: worker.last_finished_submission_request_id, + execution_reconstructed: true, + workspace_id: worker.workspace_id, + profile: worker.profile, + display_name: worker.display_name, + profile_source: worker.profile_source, + config_bundle: worker.config_bundle, + request, + created_at_ms: worker.created_at_ms, + execution_metadata_available, + execution_bound, + restore_intent, + pending_restore, + pending_stop, + restore_candidate_context: None, + workdir_attachments: worker.workdir_attachments, + logical_workdir_attachments, + internal_workers: InternalWorkerActivityProjection::default(), + } + } + #[cfg(feature = "fs-store")] fn persisted_record(&self) -> PersistedWorkerRecord { let execution_state = if let Some(pending) = self.pending_stop { @@ -6602,6 +6761,7 @@ impl WorkerRecord { PersistedWorkerRestoreOperation { operation_id: restore.operation_id, mode: restore.mode.into(), + commit: restore.commit, request: self .request .clone() @@ -6625,6 +6785,7 @@ impl WorkerRecord { PersistedWorkerLifecycleOperation::Restore(PersistedWorkerRestoreOperation { operation_id: pending.operation_id, mode: pending.mode.into(), + commit: pending.commit, request, binding: self .execution_bound @@ -8994,6 +9155,7 @@ mod tests { .pending_restore = Some(PendingWorkerRestore { operation_id: WorkerLifecycleOperationId::new(), mode: WorkerRestoreMode::Explicit, + commit: RestoreCommitFact::NotCommitted, last_settled_status: WorkerStatus::Stopped, }); let id = "workdir-irrelevant-to-recovery"; @@ -10933,6 +11095,7 @@ mod tests { })).unwrap()); runtime_store(&runtime) .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + runtime_store(&runtime).fail_next_worker_reconcile_sync(); assert_eq!( runtime .restore_worker_operation(&worker.worker_ref, request.clone()) @@ -11031,6 +11194,7 @@ mod tests { .operation_id; runtime_store(&runtime) .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + runtime_store(&runtime).fail_next_worker_reconcile_sync(); request.preparation = None; assert_eq!( runtime @@ -11221,6 +11385,7 @@ mod tests { ); runtime_store(&runtime) .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + runtime_store(&runtime).fail_next_worker_reconcile_sync(); assert_eq!( runtime .restore_worker_operation(&worker.worker_ref, request.clone()) @@ -11228,9 +11393,21 @@ mod tests { .state, WorkerRestoreState::ReconciliationRequired ); + // Resolve admission authority before injecting a different write fault. + // Otherwise the next failed sync would only exercise admission reread. + { + let lease = runtime.worker_operation_lock(worker.worker_id).unwrap(); + let _guard = lease.lock().unwrap(); + runtime + .lock() + .unwrap() + .refresh_uncertain_worker_write(&worker.worker_ref) + .unwrap(); + } // This write interrupts idempotent attachment completion after admission. runtime_store(&runtime) .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + runtime_store(&runtime).fail_next_worker_reconcile_sync(); assert_eq!( runtime .restore_worker_operation(&worker.worker_ref, request.clone()) @@ -11873,6 +12050,9 @@ mod tests { .unwrap() .operation_id; runtime_store(&runtime).fail_next_worker_write(fault); + if matches!(fault, crate::fs_store::AtomicWriteFault::AfterRename) { + runtime_store(&runtime).fail_next_worker_reconcile_sync(); + } gate.release(); assert_eq!( restoring.join().unwrap().unwrap().state, @@ -13137,6 +13317,7 @@ mod tests { assert_eq!(summary.worker_count, 0); } + #[cfg(feature = "fs-store")] #[test] fn delete_worker_persistence_failure_retains_identity_and_bounded_diagnostic() { let root = tempfile::tempdir().unwrap(); @@ -13236,6 +13417,7 @@ mod tests { )); } + #[cfg(feature = "fs-store")] #[test] fn delete_worker_waits_for_execution_shutdown_before_removing_snapshot() { let root = tempfile::tempdir().unwrap(); @@ -13406,7 +13588,7 @@ mod tests { }) .unwrap_err(); assert!(error.to_string().contains( - "unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, or 12" + "unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, or 13" )); let _ = std::fs::remove_dir_all(root); @@ -13448,7 +13630,10 @@ mod tests { let worker_aggregate: serde_json::Value = serde_json::from_slice(&std::fs::read(worker_store_dir.join("worker.json")).unwrap()) .unwrap(); - assert_eq!(worker_aggregate["schema_version"], serde_json::json!(12)); + assert_eq!( + worker_aggregate["schema_version"], + serde_json::json!(crate::fs_store::SCHEMA_VERSION) + ); assert_eq!(worker_aggregate["status"], serde_json::json!("stopped")); assert_eq!( worker_aggregate["execution_state"]["state"], @@ -14802,7 +14987,10 @@ mod tests { ); let migrated_aggregate: serde_json::Value = serde_json::from_slice(&std::fs::read(&worker_path).unwrap()).unwrap(); - assert_eq!(migrated_aggregate["schema_version"], serde_json::json!(12)); + assert_eq!( + migrated_aggregate["schema_version"], + serde_json::json!(crate::fs_store::SCHEMA_VERSION) + ); assert!(migrated_aggregate.get("request").is_none()); assert!(migrated_aggregate.get("execution").is_none()); assert_eq!( @@ -14826,6 +15014,184 @@ mod tests { let _ = std::fs::remove_dir_all(root); } + #[cfg(feature = "fs-store")] + #[test] + fn restore_decision_reread_preserves_commit_across_uncertainty_stop_and_restart() { + // Component scope: real Runtime aggregate owner/store, substituted + // execution backend. This does not attest Controller readiness or input. + for sync_failure in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("decision-reconciliation"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("decision fact")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let lease = runtime.worker_operation_lock(worker.worker_id).unwrap(); + let guard = lease.lock().unwrap(); + let operation_id = WorkerLifecycleOperationId::new(); + runtime + .begin_restore_operation( + &worker.worker_ref, + operation_id, + WorkerRestoreMode::Explicit, + runtime.restore_candidate_execution_context(worker.worker_ref.clone()), + ) + .unwrap(); + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + if sync_failure { + runtime_store(&runtime).fail_next_worker_reconcile_sync(); + } + let result = runtime.commit_restored_worker_execution( + &worker.worker_ref, + protocol::WorkerStatus::Idle.into(), + WorkerStatus::Idle, + Vec::new(), + ); + assert_eq!(result.is_ok(), !sync_failure); + let path = options + .root + .join("workers") + .join(worker.worker_id.to_string()) + .join("worker.json"); + let bytes = std::fs::read(&path).unwrap(); + let observed = runtime_store(&runtime) + .read_worker_record(&worker.worker_ref) + .unwrap(); + let PersistedWorkerExecutionState::ReconciliationRequired( + PersistedWorkerLifecycleOperation::Restore(pending), + ) = observed.execution_state + else { + panic!("committed decision lost its operation") + }; + assert_eq!(pending.operation_id, operation_id); + assert_eq!(pending.commit, RestoreCommitFact::Committed); + assert_eq!( + pending.request, + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .request + .clone() + .unwrap() + ); + let rollback = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "must-not-revert", + "committed decision cannot become rollback", + ); + assert!( + runtime + .settle_restore_rollback(&worker.worker_ref, &rollback) + .is_err() + ); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + // Cleanup confirmation from a backend is not evidence that the + // Runtime's durable decision was uncommitted. + assert!( + runtime + .lock() + .unwrap() + .record_restore_failure( + &worker.worker_ref, + &WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "backend confirmed cleanup after reconstruction failure", + ), + &rollback, + ) + .is_err() + ); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + if sync_failure { + assert!( + runtime + .lock() + .unwrap() + .unreconciled_worker_writes + .contains(&worker.worker_id) + ); + // Substitute unreadable storage only. No production recovery + // path repairs it from an old memory record. + std::fs::write(&path, b"not-json").unwrap(); + assert!( + runtime + .lock() + .unwrap() + .refresh_uncertain_worker_write(&worker.worker_ref) + .is_err() + ); + assert!( + runtime + .lock() + .unwrap() + .persist_worker(&worker.worker_id) + .is_err() + ); + assert!( + runtime + .begin_stop_operation(&worker.worker_ref, WorkerLifecycleOperationId::new()) + .is_err() + ); + assert_eq!(std::fs::read(&path).unwrap(), b"not-json"); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + // End the storage fault with the exact post-rename bytes, not + // an inferred projection; recovery still performs its real sync. + std::fs::write(&path, &bytes).unwrap(); + runtime + .lock() + .unwrap() + .refresh_uncertain_worker_write(&worker.worker_ref) + .unwrap(); + } + let current = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap(); + assert_eq!(current.commit, RestoreCommitFact::Committed); + assert_eq!(current.operation_id, operation_id); + runtime + .begin_stop_operation(&worker.worker_ref, WorkerLifecycleOperationId::new()) + .unwrap(); + let saved = runtime_store(&runtime) + .read_worker_record(&worker.worker_ref) + .unwrap(); + let PersistedWorkerExecutionState::ReconciliationRequired( + PersistedWorkerLifecycleOperation::Stop(stop), + ) = saved.execution_state + else { + panic!("Stop ownership was not saved") + }; + assert_eq!( + stop.pending_restore.unwrap().commit, + RestoreCommitFact::Committed + ); + drop(guard); + drop(lease); + drop(runtime); + let reopened = Runtime::with_fs_store(options).unwrap(); + let state = reopened.lock().unwrap(); + let restored = state.worker(&worker.worker_ref).unwrap(); + assert!(restored.pending_stop.is_some()); + assert_eq!( + restored.pending_restore.unwrap().commit, + RestoreCommitFact::Committed + ); + assert_eq!(restored.pending_restore.unwrap().operation_id, operation_id); + } + } + #[cfg(feature = "fs-store")] #[test] fn fs_store_preserves_an_unknown_restore_operation_commit() { @@ -14849,9 +15215,11 @@ mod tests { runtime_store(&runtime) .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + runtime_store(&runtime).fail_next_worker_reconcile_sync(); + let operation_id = WorkerLifecycleOperationId::new(); let result = runtime.begin_restore_operation( &worker.worker_ref, - WorkerLifecycleOperationId::new(), + operation_id, WorkerRestoreMode::Explicit, runtime.restore_candidate_execution_context(worker.worker_ref.clone()), ); @@ -14866,14 +15234,18 @@ mod tests { runtime.worker_detail(&worker.worker_ref).unwrap().status, WorkerStatus::Stopped ); - let operation_id = runtime - .lock() - .unwrap() - .worker(&worker.worker_ref) - .unwrap() - .pending_restore - .unwrap() - .operation_id; + { + let state = runtime.lock().unwrap(); + assert!(state.unreconciled_worker_writes.contains(&worker.worker_id)); + assert!( + state + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .is_none(), + "failed sync must not promote an unconfirmed in-memory candidate" + ); + } let aggregate: serde_json::Value = serde_json::from_slice( &std::fs::read( root.join("workers") diff --git a/crates/worker-runtime/src/test_support.rs b/crates/worker-runtime/src/test_support.rs index fb0f32693..ca9192f8f 100644 --- a/crates/worker-runtime/src/test_support.rs +++ b/crates/worker-runtime/src/test_support.rs @@ -376,6 +376,8 @@ impl StoppedWorkerTemplate { root: stage.clone(), _owner_lock: Some(lock), worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), + #[cfg(test)] + worker_reconcile_sync_fault: Default::default(), }; let result = (|| { let worker = PersistedWorkerRecord { @@ -463,8 +465,15 @@ impl StoppedWorkerTemplate { } fn validate_shape(&self) -> Result<(), RuntimeError> { + // Version 1 templates carry only settled Session material, never a + // lifecycle operation. Schema 12 -> 13 changes pending decision evidence, + // not that payload. Accept the fixed schema-12 artifact read-only and + // install through the current aggregate owner; never rewrite its bytes. if self.template_version != TEMPLATE_VERSION - || self.runtime_schema_version != SCHEMA_VERSION + || !matches!( + self.runtime_schema_version, + RESTORE_ADMISSION_SCHEMA_VERSION | SCHEMA_VERSION + ) { return Err(reject("unsupported template schema")); } diff --git a/crates/worker-runtime/src/test_support_tests.rs b/crates/worker-runtime/src/test_support_tests.rs index 203434258..a4931def8 100644 --- a/crates/worker-runtime/src/test_support_tests.rs +++ b/crates/worker-runtime/src/test_support_tests.rs @@ -563,6 +563,7 @@ fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() { PersistedWorkerLifecycleOperation::Restore(PersistedWorkerRestoreOperation { operation_id: WorkerLifecycleOperationId::new(), mode: PersistedWorkerRestoreMode::Explicit, + commit: crate::runtime::RestoreCommitFact::NotCommitted, request: request(), binding: None, last_settled_status: WorkerStatus::Stopped, @@ -632,6 +633,38 @@ fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() { drop(held); } +#[test] +fn schema_v12_settled_template_installs_current_aggregate_without_rewriting_session() { + let (_source, _, mut template) = source(); + template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION; + let bytes = serde_json::to_vec(&template).unwrap(); + let (_target, options) = destination(&template); + template.install(&options, request(), &resolved()).unwrap(); + assert_eq!(serde_json::to_vec(&template).unwrap(), bytes); + let worker = options.root.join(WORKERS_DIR).join(worker_id().to_string()); + assert_eq!( + fs::read(worker.join("session/session.json")).unwrap(), + template.session_manifest + ); + assert_eq!( + fs::read(worker.join(format!("session/segments/{}.jsonl", template.segment_id))).unwrap(), + template.segment_jsonl + ); + let aggregate: WorkerAggregateRecord = + read_bounded_json(&worker.join(WORKER_FILE), "read installed fixture").unwrap(); + let saved = aggregate.validate(&worker.join(WORKER_FILE)).unwrap(); + assert_eq!(saved.status, WorkerStatus::Stopped); + assert!(matches!( + saved.execution_state, + PersistedWorkerExecutionState::Available(_) + )); + // Unsupported template schemas do not enter the versioned reader. + template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION - 1; + assert!(template.validate_shape().is_err()); + template.runtime_schema_version = SCHEMA_VERSION + 1; + assert!(template.validate_shape().is_err()); +} + #[test] fn fresh_manifest_and_memory_settings_are_required() { let (_source, _, template) = source(); diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 87c0f676c..cf6bfe5d5 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -25,13 +25,17 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, RwLock, mpsc}; use std::time::Duration; -use crate::auth::{BACKEND_RESOURCE_FETCH_PERMISSION, RuntimeIdentityMaterial}; +#[cfg(feature = "http-server")] +use crate::auth::BACKEND_RESOURCE_FETCH_PERMISSION; +use crate::auth::RuntimeIdentityMaterial; use crate::catalog::{ CreateWorkerRequest, ProfileSourceArchiveSource, RepositoryRefObservation, RepositoryRefObservationRequest, WorkingDirectoryAttachmentStatus, WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest, WorkingDirectoryStatus, }; -use crate::config_bundle::{ConfigBundle, workspace_config_etag}; +use crate::config_bundle::ConfigBundle; +#[cfg(feature = "http-server")] +use crate::config_bundle::workspace_config_etag; #[cfg(feature = "ws-server")] use crate::execution::WorkerProtocolTransport; use crate::execution::{ @@ -43,12 +47,13 @@ use crate::execution::{ use crate::identity::WorkerRef; use crate::interaction::{WorkerInput, WorkerInputKind}; use crate::resource::BackendResourceClient; -use crate::worker_source::{ - EmbeddedWorkerMutationDispatcher, RuntimeOwnedWorkspaceClient, RuntimeWorkerMutationForwarder, -}; +use crate::worker_source::EmbeddedWorkerMutationDispatcher; +#[cfg(feature = "http-server")] +use crate::worker_source::{RuntimeOwnedWorkspaceClient, RuntimeWorkerMutationForwarder}; use crate::working_directory::{ WorkingDirectoryBinding, WorkingDirectoryDiagnostic, WorkingDirectoryMaterializer, }; +#[cfg(feature = "http-server")] use crate::workspace_request::{RuntimeWorkspaceRequest, RuntimeWorkspaceRequestClient}; use async_trait::async_trait; #[cfg(test)] @@ -137,6 +142,8 @@ use workdir::{ #[cfg(test)] use worker::WorkerController; +#[cfg(any(feature = "http-server", test))] +use worker::WorkspaceId; use worker::feature::builtin::manage_workdir::WorkspaceAttachedWorkdirSession; use worker::feature::builtin::{ CompositeWorkerObservationProvider, WorkerObservationError, WorkerObservationProvider, @@ -146,10 +153,10 @@ use worker::feature::builtin::{ #[cfg(feature = "ws-server")] use worker::ipc::protocol_session::{WorkerProtocolSessionStreams, live_log_entry_event}; use worker::{ - PreparedWorker, PromptCatalogSource, SegmentLogSink, SubjektivSessionAttributionLifecycle, - Worker, WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout, - WorkerControllerTransport, WorkerError, WorkerFilesystemAuthority, WorkerHandle, - WorkerSharedState, WorkerWorkspaceContext, WorkspaceClient, WorkspaceId, + PromptCatalogSource, SegmentLogSink, SubjektivSessionAttributionLifecycle, Worker, + WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout, WorkerControllerTransport, + WorkerError, WorkerFilesystemAuthority, WorkerHandle, WorkerSharedState, + WorkerWorkspaceContext, WorkspaceClient, }; #[cfg(feature = "ws-server")] @@ -165,8 +172,13 @@ const DEFAULT_BACKEND_ID: &str = "worker-crate"; const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10); const SPAWN_RESTORE_TASK_TIMEOUT: Duration = Duration::from_secs(60); const USER_INPUT_TASK_TIMEOUT: Duration = Duration::from_secs(10); +#[cfg(feature = "http-server")] const WORKSPACE_CONFIG_HTTP_TIMEOUT: Duration = Duration::from_secs(8); +#[cfg(feature = "http-server")] const MAX_WORKSPACE_CONFIG_RESPONSE_BYTES: usize = 72 * 1024 * 1024; +#[cfg(not(feature = "http-server"))] +const WORKSPACE_HTTP_TRANSPORT_UNAVAILABLE: &str = + "Workspace API HTTP transport is unavailable: worker-runtime requires the http-server feature"; // Leave adapter cancellation margin after the durable submission deadline. const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(9); @@ -175,6 +187,7 @@ pub struct RuntimeWorkerController { pub shutdown: Arc>>, pub controller_task: tokio::task::JoinHandle<()>, pub workspace_client: Arc, + pub restore_streams: Option, } /// An unpublished restore resource owner. No protocol endpoint is available @@ -184,6 +197,14 @@ pub struct RuntimeWorkerController { pub trait RuntimeRestoreCandidate: Send { fn initial_state(&self) -> protocol::WorkerStateSnapshot; + fn has_durable_effects(&self) -> bool { + false + } + + fn can_retry_publication(&self) -> bool { + false + } + async fn activate(&mut self) -> Result; async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure>; @@ -198,6 +219,7 @@ pub enum RuntimeRestoreStage { Recovery, Binding, Feature, + ActivationEffects, Controller, } @@ -379,6 +401,14 @@ impl RuntimeRestoreCandidate for EffectRetainingRestoreCandidate { self.candidate.initial_state() } + fn has_durable_effects(&self) -> bool { + true + } + + fn can_retry_publication(&self) -> bool { + self.candidate.can_retry_publication() + } + async fn activate(&mut self) -> Result { self.candidate.activate().await.map_err(|mut failure| { failure.durable_effects_possible = true; @@ -493,7 +523,8 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { _worker_ref: &WorkerRef, _workspace_id: Option<&str>, _handle: &WorkerHandle, - ) { + ) -> Result<(), String> { + Ok(()) } } @@ -512,17 +543,25 @@ struct RuntimeObservedWorker { } impl RuntimeWorkerObservationHub { - fn register(&self, worker_ref: WorkerRef, workspace_id: Option, handle: &WorkerHandle) { - if let Ok(mut workers) = self.workers.lock() { - workers.insert( - worker_ref, - RuntimeObservedWorker { - workspace_id, - shared_state: Arc::downgrade(&handle.shared_state), - sink: handle.sink.clone(), - }, - ); - } + fn register( + &self, + worker_ref: WorkerRef, + workspace_id: Option, + handle: &WorkerHandle, + ) -> Result<(), String> { + let mut workers = self + .workers + .lock() + .map_err(|_| "Worker observation registry lock is poisoned".to_string())?; + workers.insert( + worker_ref, + RuntimeObservedWorker { + workspace_id, + shared_state: Arc::downgrade(&handle.shared_state), + sink: handle.sink.clone(), + }, + ); + Ok(()) } fn get( @@ -739,6 +778,7 @@ pub struct ProfileRuntimeWorkerFactory { prompt_projection_cache: Arc, runtime_id: Option, worker_mutation_identity: Option, + #[cfg(feature = "http-server")] workspace_request_clients: Arc>, embedded_worker_mutation_dispatcher: Option>, controller_transport: WorkerControllerTransport, @@ -763,8 +803,11 @@ struct RestoreFaults { executions: std::sync::atomic::AtomicUsize, cleanup_fail: AtomicBool, socket_failure: AtomicBool, + publication_failure: AtomicBool, factory_gate: Mutex>>, factory_entered: Mutex>>, + prepared_gate: Mutex>>, + prepared_entered: Mutex>>, cleanup_gate: Mutex>>, cleanup_entered: Mutex>>, } @@ -825,6 +868,7 @@ impl ProfileRuntimeWorkerFactory { prompt_projection_cache: Arc::new(WorkspacePromptProjectionCache::default()), runtime_id: None, worker_mutation_identity: None, + #[cfg(feature = "http-server")] workspace_request_clients: Arc::new(HashMap::new()), embedded_worker_mutation_dispatcher: None, controller_transport: WorkerControllerTransport::InProcess, @@ -860,6 +904,7 @@ impl ProfileRuntimeWorkerFactory { self } + #[cfg(feature = "http-server")] pub fn with_workspace_request_client(mut self, client: RuntimeWorkspaceRequestClient) -> Self { self.runtime_id = Some(client.runtime_id().to_string()); Arc::make_mut(&mut self.workspace_request_clients) @@ -1168,6 +1213,7 @@ impl ProfileRuntimeWorkerFactory { #[derive(Debug, Clone)] enum RuntimeWorkspaceBackendRef { None, + #[cfg(feature = "http-server")] Http { workspace_id: String, base_url: String, @@ -1176,28 +1222,45 @@ enum RuntimeWorkspaceBackendRef { } impl RuntimeWorkspaceBackendRef { - fn from_worker_request(request: &CreateWorkerRequest, runtime_id: Option<&str>) -> Self { - if let (Some(api), Some(runtime_id)) = (request.workspace_api.as_ref(), runtime_id) { - return Self::Http { + fn from_worker_request( + request: &CreateWorkerRequest, + runtime_id: Option<&str>, + ) -> Result { + #[cfg(not(feature = "http-server"))] + if request.workspace_api.is_some() { + return Err(WORKSPACE_HTTP_TRANSPORT_UNAVAILABLE); + } + match (request.workspace_api.as_ref(), runtime_id) { + #[cfg(feature = "http-server")] + (Some(api), Some(runtime_id)) => Ok(Self::Http { workspace_id: api.workspace_id.clone(), base_url: api.base_url.clone(), runtime_id: runtime_id.to_string(), - }; + }), + _ => Ok(Self::None), } - Self::None } fn worker_context( &self, - worker_ref: &WorkerRef, - workspace_scope: Option<&crate::runtime::RuntimeWorkspaceScope>, - mutation_identity: Option<&RuntimeIdentityMaterial>, - workspace_request_client: Option<&RuntimeWorkspaceRequestClient>, - embedded_dispatcher: Option<&Arc>, - prompt_projection_cache: Option>, + #[cfg(feature = "http-server")] worker_ref: &WorkerRef, + #[cfg(feature = "http-server")] workspace_scope: Option< + &crate::runtime::RuntimeWorkspaceScope, + >, + #[cfg(feature = "http-server")] mutation_identity: Option<&RuntimeIdentityMaterial>, + #[cfg(feature = "http-server")] workspace_request_client: Option< + &RuntimeWorkspaceRequestClient, + >, + #[cfg(feature = "http-server")] embedded_dispatcher: Option< + &Arc, + >, + #[cfg(feature = "http-server")] prompt_projection_cache: Option< + Arc, + >, ) -> WorkerWorkspaceContext { match self { Self::None => WorkerWorkspaceContext::no_workspace(), + #[cfg(feature = "http-server")] Self::Http { workspace_id, base_url, @@ -1327,14 +1390,6 @@ async fn fetch_workspace_config_http( Ok(WorkspaceConfigFetchResult::Modified(bundle)) } -#[cfg(not(feature = "http-server"))] -async fn fetch_workspace_config_http( - _request: &WorkspaceConfigFetchRequest, - _client: &T, -) -> Result { - Err("Workspace Config fetch requires the worker-runtime http-server feature".to_string()) -} - fn initial_workdir_session_capabilities( alias: &WorkdirAttachmentAlias, claims: &[crate::catalog::WorkingDirectoryAttachmentClaim], @@ -1767,16 +1822,24 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { &self, request: WorkspaceConfigFetchRequest, ) -> Result { - let client = self - .workspace_request_clients - .get(&request.workspace_api.workspace_id) - .ok_or_else(|| { - format!( - "Workspace request client is unavailable for workspace {}", - request.workspace_api.workspace_id - ) - })?; - fetch_workspace_config_http(&request, client).await + #[cfg(feature = "http-server")] + { + let client = self + .workspace_request_clients + .get(&request.workspace_api.workspace_id) + .ok_or_else(|| { + format!( + "Workspace request client is unavailable for workspace {}", + request.workspace_api.workspace_id + ) + })?; + fetch_workspace_config_http(&request, client).await + } + #[cfg(not(feature = "http-server"))] + { + let _ = request; + Err("Workspace Config fetch is unsupported: worker-runtime requires the http-server feature".to_string()) + } } fn retained_session_snapshot( @@ -1866,7 +1929,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { let workspace_backend_ref = RuntimeWorkspaceBackendRef::from_worker_request( &request.request, self.runtime_id.as_deref(), - ); + )?; let observation_runtime_id = self.runtime_id.clone(); let observation_workspace_id = request .request @@ -1875,17 +1938,24 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .map(|api| api.workspace_id.clone()); let observation_grants = request.request.worker_observation_grants.clone(); let observation_enabled = request.request.worker_observation_enabled; + #[cfg(feature = "http-server")] let workspace_request_client = request .request .workspace_api .as_ref() .and_then(|api| self.workspace_request_clients.get(&api.workspace_id)); let workspace_context = workspace_backend_ref.worker_context( + #[cfg(feature = "http-server")] &request.worker_ref, + #[cfg(feature = "http-server")] request.workspace_scope.as_ref(), + #[cfg(feature = "http-server")] self.worker_mutation_identity.as_ref(), + #[cfg(feature = "http-server")] workspace_request_client, + #[cfg(feature = "http-server")] self.embedded_worker_mutation_dispatcher.as_ref(), + #[cfg(feature = "http-server")] Some(self.prompt_projection_cache.clone()), ); let selector = profile.as_ref(); @@ -2044,16 +2114,19 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { if flow_transition_enabled { handle.shared_state.enable_flow_transition(); } - self.observation_hub.register( + if let Err(error) = self.observation_hub.register( request.worker_ref.clone(), observation_workspace_id, &handle, - ); + ) { + tracing::warn!(%error, "Worker observation registration unavailable"); + } Ok(RuntimeWorkerController { handle, shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), controller_task, workspace_client, + restore_streams: None, }) } @@ -2061,6 +2134,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { &self, request: &WorkerExecutionRestoreRequest, ) -> Result<(), String> { + #[cfg(not(feature = "http-server"))] + RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + self.runtime_id.as_deref(), + )?; let worker_name = Self::runtime_worker_name_for_ref(&request.worker_ref); let worker_aggregate_dir = self.worker_aggregate_dir(&request.worker_ref)?; if !worker_aggregate_dir.is_dir() { @@ -2134,6 +2212,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { &self, request: &WorkerExecutionRestoreRequest, ) -> Result { + #[cfg(not(feature = "http-server"))] + RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + self.runtime_id.as_deref(), + ) + .map_err(|cause| RuntimeRestoreFailure::refused(RuntimeRestoreStage::Preflight, cause))?; self.recover_saved_restore_source(request) .await .map_err(|mut failure| { @@ -2146,6 +2230,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { &self, request: WorkerExecutionRestoreRequest, ) -> Result, RuntimeRestoreFailure> { + #[cfg(not(feature = "http-server"))] + RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + self.runtime_id.as_deref(), + ) + .map_err(|cause| RuntimeRestoreFailure::refused(RuntimeRestoreStage::Preflight, cause))?; self.preflight_restore(&request).await.map_err(|_| { RuntimeRestoreFailure::refused( RuntimeRestoreStage::Preflight, @@ -2157,6 +2247,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .executions .fetch_add(1, Ordering::SeqCst); let mut stage = RuntimeRestoreStage::Metadata; + let mut preparation_failure = None; // Reconstruction owns read-only resources; cleanup confirmation must // come from that owner even when it cannot return a candidate. let result: Result, &'static str> = async { @@ -2178,18 +2269,25 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { let workspace_backend_ref = RuntimeWorkspaceBackendRef::from_worker_request( &request.request, self.runtime_id.as_deref(), - ); + )?; + #[cfg(feature = "http-server")] let workspace_request_client = request .request .workspace_api .as_ref() .and_then(|api| self.workspace_request_clients.get(&api.workspace_id)); let workspace_context = workspace_backend_ref.worker_context( + #[cfg(feature = "http-server")] &request.worker_ref, + #[cfg(feature = "http-server")] request.workspace_scope.as_ref(), + #[cfg(feature = "http-server")] self.worker_mutation_identity.as_ref(), + #[cfg(feature = "http-server")] workspace_request_client, + #[cfg(feature = "http-server")] self.embedded_worker_mutation_dispatcher.as_ref(), + #[cfg(feature = "http-server")] Some(self.prompt_projection_cache.clone()), ); @@ -2272,14 +2370,19 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .map_err(|_| "injected restore boundary failure")?; validate_worker_memory_settings(candidate.manifest(), &request.request) .map_err(|_| "saved memory settings mismatch")?; - Ok(Box::new(ProfileRestoreCandidate { - prepared: Some(candidate), - factory: self.clone(), - request: request.clone(), - }) as Box) + self.prepare_unpublished_restore(candidate, &request) + .await + .map(|candidate| Box::new(candidate) as Box) + .map_err(|failure| { + let cause = failure.cause; + preparation_failure = Some(failure); + cause + }) } .await; - result.map_err(|cause| RuntimeRestoreFailure::started(stage, cause)) + result.map_err(|cause| { + preparation_failure.unwrap_or_else(|| RuntimeRestoreFailure::started(stage, cause)) + }) } async fn cleanup_failed_restore( @@ -2429,51 +2532,128 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { worker_ref: &WorkerRef, workspace_id: Option<&str>, handle: &WorkerHandle, - ) { + ) -> Result<(), String> { self.observation_hub - .register(worker_ref.clone(), workspace_id.map(str::to_string), handle); + .register(worker_ref.clone(), workspace_id.map(str::to_string), handle) } } type ProfileRestoreStore = CombinedStore; struct ProfileRestoreCandidate { - prepared: Option>, + prepared: Option>, + workspace_client: Arc, factory: ProfileRuntimeWorkerFactory, request: WorkerExecutionRestoreRequest, + #[cfg(test)] + effects_boundary_failed: bool, } #[async_trait] impl RuntimeRestoreCandidate for ProfileRestoreCandidate { fn initial_state(&self) -> protocol::WorkerStateSnapshot { - protocol::WorkerStateSnapshot::initial() + match self + .prepared + .as_ref() + .expect("same unpublished Controller") + .snapshot() + { + protocol::Event::Snapshot { state, .. } => state.clone(), + _ => unreachable!("prepared Controller full snapshot"), + } + } + + fn can_retry_publication(&self) -> bool { + #[cfg(test)] + if self.effects_boundary_failed { + return false; + } + self.prepared + .as_ref() + .is_some_and(|prepared| prepared.can_retry_publication()) } async fn activate(&mut self) -> Result { - let prepared = self.prepared.take().ok_or_else(|| { + let prepared = self.prepared.as_mut().ok_or_else(|| { RuntimeRestoreFailure::unproven( RuntimeRestoreStage::Controller, "restore candidate already consumed", ) })?; - let worker = prepared.activate().await.map_err(|error| { - self.factory - .retain_construction_failure(&self.request, &error); - let mut failure = RuntimeRestoreFailure::started( - RuntimeRestoreStage::Reconstruction, - restore_worker_cause(&error), - ); - failure.durable_effects_possible = - matches!(error, WorkerError::RestoreActivationEffects { .. }); + let map_error = |error: WorkerBootstrapError| { + let (stage, cause, io_kind) = match &error { + WorkerBootstrapError::Worker(error) => { + self.factory + .retain_construction_failure(&self.request, error); + ( + RuntimeRestoreStage::Reconstruction, + restore_worker_cause(error), + None, + ) + } + WorkerBootstrapError::Controller { source, .. } => ( + RuntimeRestoreStage::Controller, + "restored Worker publication failed", + Some(source.kind()), + ), + }; + let mut failure = RuntimeRestoreFailure::started(stage, cause); + failure.durable_effects_possible = true; + failure.io_kind = io_kind; failure - })?; + }; + prepared.apply_effects().await.map_err(&map_error)?; self.factory - .activate_prepared_restore(worker, &self.request) - .await - .map_err(|mut failure| { - failure.durable_effects_possible = true; - failure - }) + .failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get_mut(&self.request.worker_ref) + .expect("retained restore owner") + .allocation = prepared.allocation_cleanup(); + #[cfg(test)] + if self + .factory + .restore_checkpoint(RuntimeRestoreStage::ActivationEffects) + .is_err() + { + self.effects_boundary_failed = true; + let mut failure = RuntimeRestoreFailure::started( + RuntimeRestoreStage::ActivationEffects, + "injected restore boundary failure", + ); + failure.durable_effects_possible = true; + return Err(failure); + } + let started = async { + #[cfg(test)] + if self + .factory + .restore_faults + .publication_failure + .swap(false, Ordering::SeqCst) + { + return Err(WorkerBootstrapError::Controller { + source: std::io::Error::other("injected publication boundary failure"), + cleanup_failed: true, + }); + } + prepared.activate().await + } + .await + .map_err(map_error)?; + self.prepared = None; + self.factory + .failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&self.request.worker_ref); + Ok(RuntimeWorkerController { + handle: started.controller.handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.controller.shutdown))), + controller_task: started.controller.controller_task, + workspace_client: self.workspace_client.clone(), + restore_streams: Some(started.streams), + }) } async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure> { @@ -2489,52 +2669,32 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate { } impl ProfileRuntimeWorkerFactory { - async fn activate_prepared_restore( + async fn prepare_unpublished_restore( &self, - mut worker: Worker< - Box, - CombinedStore, - >, + mut candidate: worker::bootstrap::PreparedRestoredWorker, request: &WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result { let mut stage = RuntimeRestoreStage::Binding; let mut io_kind = None; - let result: Result = async { - let feature_tasks = worker.retain_startup_feature_cleanup(); + let result: Result = async { + let feature_tasks = candidate + .retain_startup_feature_cleanup() + .map_err(|_| "restore cleanup ownership unavailable")?; self.failed_restore_resources .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert( - request.worker_ref.clone(), - FailedRestoreResources { - operation_id: request.operation_id, - sessions: vec![worker.workdir_sessions()], - candidate_workdir_cleanup: None, - allocation: worker.restore_allocation_cleanup_handle(), - feature_tasks: Some(feature_tasks), - }, - ); - let observation_runtime_id = self.runtime_id.clone(); - let observation_workspace_id = request - .request - .workspace_api - .as_ref() - .map(|api| api.workspace_id.clone()); - let observation_grants = request.request.worker_observation_grants.clone(); - let observation_enabled = request.request.worker_observation_enabled; - if let Some(binding) = request.request.backend_job.as_ref() { - worker - .bind_backend_job(binding.clone()) - .map_err(|_| "restored Backend Job binding failed")?; - } - validate_worker_memory_settings(worker.manifest(), &request.request) - .map_err(|_| "saved memory settings mismatch")?; + .get_mut(&request.worker_ref) + .expect("retained restore owner") + .feature_tasks = Some(feature_tasks); + let workspace_client = candidate + .workspace_client_handle() + .map_err(|_| "restore Workspace binding unavailable")?; let workdir_sessions = restored_workdir_router( &request.workdir_attachments, &request.logical_workdir_attachments, &request.deferred_workdir_sessions, &request.request.workdir_attachments, - worker.workspace_client_handle(), + workspace_client.clone(), ) .map_err(|_| "restored Workdir binding failed")?; self.failed_restore_resources @@ -2544,56 +2704,43 @@ impl ProfileRuntimeWorkerFactory { .expect("retained restore owner") .sessions .push(workdir_sessions.clone()); - worker - .workdir_sessions() - .close_all() - .await - .map_err(|_| "initial Workdir close failed")?; - worker.bind_workdir_sessions(workdir_sessions); - if let (Some(runtime_id), Some(workspace_id)) = - (observation_runtime_id, observation_workspace_id.clone()) - && observation_enabled + let mut providers: Vec> = vec![Arc::new( + WorkspaceClientWorkerObservationProvider::new(workspace_client.clone()), + )]; + let observation_provider = if let (Some(runtime_id), Some(api)) = ( + self.runtime_id.clone(), + request.request.workspace_api.as_ref(), + ) && request.request.worker_observation_enabled { - let mut providers: Vec> = vec![Arc::new( - WorkspaceClientWorkerObservationProvider::new(worker.workspace_client_handle()), - )]; - if !observation_grants.is_empty() { + if !request.request.worker_observation_grants.is_empty() { providers.push(Arc::new(RuntimeGrantedWorkerObservationProvider { runtime_id, - workspace_id, - grants: observation_grants.into_iter().take(100).collect(), + workspace_id: api.workspace_id.clone(), + grants: request + .request + .worker_observation_grants + .iter() + .take(100) + .cloned() + .collect(), hub: self.observation_hub.clone(), })); } - worker.bind_worker_observation_provider(Some(Arc::new( - CompositeWorkerObservationProvider::new(providers), - ))); - } - worker - .finalize_subjektiv_session_attribution( - request.request.subjektiv_attached, - SubjektivSessionAttributionLifecycle::RestoredSession, - ) - .map_err(|error| restore_worker_cause(&error))?; - let flow_transition_enabled = worker.manifest().feature.flow.enabled; - stage = RuntimeRestoreStage::Feature; + Some(Arc::new(CompositeWorkerObservationProvider::new(providers)) + as Arc) + } else { + None + }; #[cfg(test)] - self.restore_checkpoint(stage) - .map_err(|_| "injected restore boundary failure")?; - if flow_transition_enabled { - let report = worker - .install_runtime_flow_transition_feature() - .map_err(|_| "Flow transition installation failed")?; - if report.reports.iter().any(|report| !report.installed) { - return Err("Flow transition installation refused"); - } + { + stage = RuntimeRestoreStage::Feature; + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; } - stage = RuntimeRestoreStage::Controller; #[cfg(test)] self.restore_checkpoint(stage) .map_err(|_| "injected restore boundary failure")?; - let workspace_client = worker.workspace_client_handle(); let run_dir = self .worker_restore_run_dir(&request.worker_ref, request.operation_id) .map_err(|_| "restore run directory unavailable")?; @@ -2604,36 +2751,48 @@ impl ProfileRuntimeWorkerFactory { } else { transport }; - let started = PreparedWorker::new( - worker, - WorkerBootstrapLayout::RuntimeManagedRun { - run_dir: run_dir.clone(), - }, - transport, - ) - .start() - .await - .map_err(|error| match error { - WorkerBootstrapError::Worker(error) => restore_worker_cause(&error), - WorkerBootstrapError::Controller { source, .. } => { - io_kind = Some(source.kind()); - "restored Worker controller startup failed" + let prepared = candidate + .prepare_runtime_controller( + &run_dir, + transport, + worker::bootstrap::RestoredControllerBindings { + backend_job: request.request.backend_job.clone(), + workdir_sessions, + observation_provider, + subjektiv_attached: request.request.subjektiv_attached, + }, + ) + .await + .map_err(|error| match error { + WorkerBootstrapError::Worker(error) => { + stage = RuntimeRestoreStage::Reconstruction; + self.retain_construction_failure(request, &error); + restore_worker_cause(&error) + } + WorkerBootstrapError::Controller { source, .. } => { + io_kind = Some(source.kind()); + "restored Worker preparation failed" + } + })?; + #[cfg(test)] + { + let gate = self.restore_faults.prepared_gate.lock().unwrap().take(); + if let Some(gate) = gate { + if let Some(entered) = + self.restore_faults.prepared_entered.lock().unwrap().take() + { + let _ = entered.send(prepared.snapshot().clone()); + } + gate.notified().await; } - })?; - self.failed_restore_resources - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(&request.worker_ref); - let (handle, shutdown_rx, controller_task) = - (started.handle, started.shutdown, started.controller_task); - if flow_transition_enabled { - handle.shared_state.enable_flow_transition(); } - Ok(RuntimeWorkerController { - handle, - shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), - controller_task, + Ok(ProfileRestoreCandidate { + prepared: Some(prepared), workspace_client, + factory: self.clone(), + request: request.clone(), + #[cfg(test)] + effects_boundary_failed: false, }) } .await; @@ -3402,7 +3561,6 @@ where }) } - #[cfg(feature = "ws-server")] fn validate_protocol_execution_under_lock( &self, worker_ref: &WorkerRef, @@ -3750,6 +3908,7 @@ where restore_operation_id: Option, workdir_attachment_statuses: Vec, workspace_client: Option>, + restore_streams: Option, ) -> WorkerExecutionSpawnResult { self.connect_controller_scope( operation, @@ -3762,6 +3921,7 @@ where restore_operation_id, workdir_attachment_statuses, workspace_client, + restore_streams, ) } @@ -3780,9 +3940,10 @@ where restore_operation_id: Option, workdir_attachment_statuses: Vec, workspace_client: Option>, + restore_streams: Option, ) -> WorkerExecutionSpawnResult { #[cfg(feature = "ws-server")] - let streams = subscribe_worker_protocol_session(&handle); + let streams = restore_streams.unwrap_or_else(|| subscribe_worker_protocol_session(&handle)); #[cfg(feature = "ws-server")] let worker_state_snapshot = match &streams.snapshot_event { Event::Snapshot { state, .. } => state.clone(), @@ -3841,7 +4002,15 @@ where let mut entry_events = streams.log_entries; let bridge_worker_state = worker_state.clone(); let bridge_handle = handle.clone(); - let bridge_task = match self.spawn_on_adapter_runtime(async move { + let initial_publication = if restore_operation_id.is_some() { + bridge_context + .publish_protocol_event(streams.snapshot_event.clone()) + .map(|_| ()) + .map_err(|error| error.to_string()) + } else { + Ok(()) + }; + let bridge_task = match initial_publication.and_then(|()| self.spawn_on_adapter_runtime(async move { loop { tokio::select! { event = events.recv() => { @@ -3898,7 +4067,7 @@ where } } } - }) { + })) { Ok(task) => task, Err(message) => { drop(workers); @@ -4442,14 +4611,27 @@ where Ok(guard) => guard, Err(_) => return unavailable("worker operation lock is poisoned".to_string()), }; - let workers = match self.workers.lock() { - Ok(workers) => workers, + let execution = match self.workers.lock() { + Ok(workers) => workers.get(&request.worker_ref).cloned(), Err(_) => return unavailable("worker adapter registry lock is poisoned".to_string()), }; - if workers.contains_key(&request.worker_ref) { - return runtime_api::WorkerSessionAvailability::LiveProtocol; + if let Some(execution) = execution { + // Match protocol attach against the same actual Controller, without + // holding the registry lock across validation or retained reads. + match self.validate_protocol_execution_under_lock(&request.worker_ref, &execution) { + Ok(()) => return runtime_api::WorkerSessionAvailability::LiveProtocol, + Err(error) + if error.outcome == crate::execution::WorkerExecutionOutcome::Errored => + { + return unavailable(error.message.unwrap_or_else(|| { + "Worker protocol availability could not be checked".to_string() + })); + } + // Closed or pending executions can still expose canonical + // read-only history, but must not advertise a live endpoint. + Err(_) => {} + } } - drop(workers); match self.factory.retained_session_snapshot(&request.worker_ref) { Ok(retained) => runtime_api::WorkerSessionAvailability::RetainedSnapshot { @@ -4766,6 +4948,7 @@ where None, Vec::new(), Some(controller.workspace_client), + controller.restore_streams, ); } return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( @@ -4894,6 +5077,7 @@ where None, workdir_attachment_statuses(&workdir_attachments), Some(controller.workspace_client), + controller.restore_streams, ) } @@ -5247,6 +5431,21 @@ where }; } } + if let Some(Err(failure)) = pending + .discard_result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + failure.to_string(), + ), + worker_state: None, + workdir_attachments: pending.request.previous_workdir_attachments.clone(), + }; + } if pending.activation_started.load(Ordering::Acquire) && !pending.owns_prepared_candidate() { let activated = pending @@ -5352,6 +5551,139 @@ where } } + fn discard_uncommitted_restore( + &self, + operation_id: crate::execution::WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> WorkerExecutionSpawnResult { + let discarded = (|| -> Result { + let operation_lock = self.worker_lock(worker_ref)?; + let _guard = operation_lock + .lock() + .map_err(|_| "worker operation lock is poisoned")?; + let pending = self + .pending_restore(worker_ref)? + .ok_or("unpublished restore owner is unavailable")?; + if pending.request.operation_id != operation_id + || pending.activation_started.load(Ordering::Acquire) + { + return Err( + "uncommitted discard cannot target another or activated operation".into(), + ); + } + if self + .workers + .lock() + .map_err(|_| "worker registry lock is poisoned")? + .contains_key(worker_ref) + { + return Err("uncommitted discard cannot target a connected execution".into()); + } + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + if !pending.discard_started.swap(true, Ordering::AcqRel) { + let owner = pending.clone(); + let task = + self.spawn_on_adapter_runtime(async move { + let retained = owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let cleanup = + match retained { + Some(Ok(mut candidate)) => { + let effects = candidate.has_durable_effects(); + match candidate.discard().await { + Ok(()) => { + let mut failure = RuntimeRestoreFailure::started( + RuntimeRestoreStage::Controller, + "restore decision was not committed", + ); + failure.cleanup = RuntimeRestoreCleanup::Confirmed; + failure.durable_effects_possible = effects; + *owner.result.lock().unwrap_or_else( + std::sync::PoisonError::into_inner, + ) = Some(Err(failure)); + Ok(()) + } + Err(error) => { + *owner.result.lock().unwrap_or_else( + std::sync::PoisonError::into_inner, + ) = Some(Ok(candidate)); + Err(error) + } + } + } + Some(Err(failure)) => { + let result = if failure.needs_cleanup() { + Err(RuntimeRestoreCleanupFailure::Ownership) + } else { + Ok(()) + }; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure)); + result + } + None => Err(RuntimeRestoreCleanupFailure::Ownership), + }; + *owner + .discard_result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(cleanup); + })?; + *pending + .task + .try_lock() + .map_err(|_| "restore discard task is already owned")? = Some(task); + } + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + let cleanup = *pending + .discard_result + .lock() + .map_err(|_| "restore discard result lock is poisoned")?; + match cleanup { + Some(Ok(())) => {} + Some(Err(error)) => { + pending.discard_started.store(false, Ordering::Release); + return Err(error.to_string()); + } + None => return Err("restore discard is still operation-owned".into()), + } + let failure = pending + .result + .lock() + .map_err(|_| "restore result lock is poisoned")? + .as_ref() + .and_then(|result| result.as_ref().err()) + .cloned() + .ok_or("restore discard has no confirmed result")?; + if failure.durable_effects_possible { + return Err("uncommitted candidate cleanup confirmed, but prior canonical repair effects require reconciliation".into()); + } + self.completed_restore_failures + .lock() + .map_err(|_| "restore failure receipts lock is poisoned")? + .insert(worker_ref.clone(), (operation_id, failure.clone())); + self.remove_pending_restore(worker_ref)?; + Ok(WorkerExecutionSpawnResult::RolledBack( + WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + failure.to_string(), + ), + )) + })(); + discarded.unwrap_or_else( + |message| WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message), + worker_state: None, + workdir_attachments: Vec::new(), + }, + ) + } + fn activate_restored_worker( &self, operation_id: crate::execution::WorkerLifecycleOperationId, @@ -5377,58 +5709,67 @@ where pending.activation_started.store(true, Ordering::Release); let owner = pending.clone(); let factory = self.factory.clone(); - let task = self.spawn_on_adapter_runtime(async move { - let candidate = owner - .result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .take(); - let result = match candidate { - Some(Ok(mut candidate)) => { - let result = candidate.activate().await; - match result { - Ok(controller) => Ok(controller), - Err(mut failure) => { - *owner - .result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(Err(failure.clone())); - match candidate.discard().await { - Ok(()) => { - failure.cleanup = RuntimeRestoreCleanup::Confirmed + let task = + self.spawn_on_adapter_runtime(async move { + let candidate = owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let result = + match candidate { + Some(Ok(mut candidate)) => { + let result = candidate.activate().await; + match result { + Ok(controller) => Ok(controller), + Err(failure) if candidate.can_retry_publication() => { + // The durable decision owns this exact receiver and + // paired streams. Do not discard it or rerun restore + // effects merely because publication was unavailable. + *owner.result.lock().unwrap_or_else( + std::sync::PoisonError::into_inner, + ) = Some(Ok(candidate)); + Err(failure) } - Err(error) => { - failure.cleanup_failure.get_or_insert(error); - failure.cleanup = RuntimeRestoreCleanup::Failed(error); + Err(mut failure) => { + *owner.result.lock().unwrap_or_else( + std::sync::PoisonError::into_inner, + ) = Some(Err(failure.clone())); + match candidate.discard().await { + Ok(()) => { + failure.cleanup = + RuntimeRestoreCleanup::Confirmed + } + Err(error) => { + failure.cleanup_failure.get_or_insert(error); + failure.cleanup = + RuntimeRestoreCleanup::Failed(error); + } + } + let failure = finish_restore_failure( + factory.as_ref(), + &owner.request, + failure, + ) + .await; + *owner.result.lock().unwrap_or_else( + std::sync::PoisonError::into_inner, + ) = Some(Err(failure.clone())); + Err(failure) } } - let failure = finish_restore_failure( - factory.as_ref(), - &owner.request, - failure, - ) - .await; - *owner - .result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(Err(failure.clone())); - Err(failure) } - } - } - Some(Err(failure)) => Err(failure), - None => Err(RuntimeRestoreFailure::unproven( - RuntimeRestoreStage::Controller, - "restore candidate ownership unavailable", - )), - }; - *owner - .activated - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(result); - })?; + Some(Err(failure)) => Err(failure), + None => Err(RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "restore candidate ownership unavailable", + )), + }; + *owner + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(result); + })?; *pending .task .try_lock() @@ -5456,6 +5797,7 @@ where Some(operation_id), pending.request.previous_workdir_attachments.clone(), Some(controller.workspace_client), + controller.restore_streams, ) { WorkerExecutionSpawnResult::Connected { .. } => { self.remove_pending_restore(worker_ref)? @@ -5473,11 +5815,16 @@ where if execution.restore_operation_id != Some(operation_id) { return Err("restored Worker operation identity does not match".into()); } + if execution.shutdown_requested.load(Ordering::Acquire) + || execution.handle.protocol_is_closed() + { + return Err("restored Worker protocol endpoint is not attachable".into()); + } self.factory.activate_restored_controller( worker_ref, execution.workspace_id.as_deref(), &execution.handle, - ); + )?; Ok(()) } @@ -5956,6 +6303,7 @@ mod tests { use crate::management::RuntimeOptions; #[cfg(feature = "ws-server")] use crate::observation::WorkerObservationCursor; + use crate::restore_test_environment::require_restore_test_environment; use crate::working_directory::RuntimeGitMaterializer; use agen::Engine; use agen::llm_client::event::{Event as LlmEvent, ResponseStatus, StatusEvent}; @@ -5968,6 +6316,25 @@ mod tests { LoggedSessionHistoryEntryId, LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, Store, WorkerActiveSegmentRef, WorkerMetadata, WorkerMetadataStore, }; + use worker::PreparedWorker; + + // Both candidate and failure suites need the same guarded, checkout-owned + // temporary root, independently of Runtime's optional filesystem catalog. + fn restore_failure_tempdir() -> tempfile::TempDir { + require_restore_test_environment(); + let target = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("target"); + fs::create_dir_all(&target).unwrap(); + tempfile::Builder::new() + .prefix("r") + .rand_bytes(3) + .tempdir_in(target) + .unwrap() + } #[test] fn production_source_has_no_repository_derived_runtime_store() { @@ -6165,6 +6532,10 @@ mod tests { (saved, request) } + // This Subject-backed fixture verifies attribution through an explicit + // loopback Workspace transport. Its Profile lifecycle consumers require + // HTTP; no-HTTP tests instead check the explicit transport refusal. + #[cfg(feature = "http-server")] async fn saved_subjektiv_restore_fixture( root: &Path, materialize_head: bool, @@ -6351,6 +6722,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn profile_backend_existing_restore_attestation_is_read_only_under_concurrency() { let root = tempfile::tempdir().unwrap(); let runtime_store = root.path().join("runtime"); @@ -6692,6 +7064,7 @@ mod tests { assert!(backend.stop_worker(&request.worker_ref).is_accepted()); } + #[cfg(feature = "http-server")] fn profile_stop_fixture( root: &Path, ) -> (WorkerRuntimeExecutionBackend, WorkerExecutionRestoreRequest) { @@ -6767,6 +7140,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn stop_rejected_shutdown_resets_request_and_retries_from_actual_snapshot() { for stale in [false, true] { let root = tempfile::tempdir().unwrap(); @@ -6810,6 +7184,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn stop_late_shutdown_rejection_after_timeout_retains_waiter_without_double_enqueue() { let root = tempfile::tempdir().unwrap(); let (backend, request) = profile_stop_fixture(root.path()); @@ -6943,6 +7318,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn unconnected_cleanup_rejected_shutdown_retains_scope_for_retry() { let root = tempfile::tempdir().unwrap(); let (backend, request) = profile_stop_fixture(root.path()); @@ -6979,6 +7355,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn duplicate_connect_rejected_shutdown_retains_actual_candidate_until_stop_retry() { for operation in [ WorkerExecutionOperation::Spawn, @@ -7024,6 +7401,7 @@ mod tests { Some(request.operation_id), Vec::new(), candidate.workspace_client.clone(), + None, ) }; match result { @@ -7111,6 +7489,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn duplicate_connect_timeout_retains_candidate_waiter_and_no_false_stop() { let root = tempfile::tempdir().unwrap(); let candidate_root = tempfile::tempdir().unwrap(); @@ -7155,6 +7534,7 @@ mod tests { Some(request.operation_id), Vec::new(), candidate.workspace_client.clone(), + None, ) }; assert!(matches!( @@ -7317,6 +7697,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn stop_joins_healthy_self_terminated_controller_with_closed_endpoint() { let root = tempfile::tempdir().unwrap(); let (backend, request) = profile_stop_fixture(root.path()); @@ -7388,6 +7769,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn stop_and_unconnected_cleanup_retain_abnormal_controller_completion_across_retries() { let root = tempfile::tempdir().unwrap(); let (backend, request) = profile_stop_fixture(root.path()); @@ -7465,6 +7847,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] async fn profile_factory_saved_subjektiv_manifest_survives_restart_and_explicit_restore() { let root = tempfile::tempdir().unwrap(); let runtime_store = root.path().join("runtime"); @@ -7535,6 +7918,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn profile_backend_restores_saved_workdir_reference_without_a_materializer() { let root = tempfile::tempdir().unwrap(); let store = root.path().join("runtime"); @@ -7599,6 +7983,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] async fn profile_factory_pending_subjektiv_restore_requires_operation_owned_launch_material() { let root = tempfile::tempdir().unwrap(); let runtime_store = root.path().join("runtime"); @@ -7796,6 +8181,7 @@ mod tests { // The local input matrix belongs to saved_manifest_policy_* above. #[tokio::test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] async fn profile_factory_saved_manifest_mismatch_rejected_before_live_work() { let root = tempfile::tempdir().unwrap(); let runtime_store = root.path().join("runtime"); @@ -8154,6 +8540,98 @@ mod tests { .unwrap(); } + #[cfg(not(feature = "http-server"))] + #[tokio::test] + async fn workspace_api_without_http_rejects_spawn_and_restore_before_resource_creation() { + let root = tempfile::tempdir().unwrap(); + let profile_dir = root.path().join("profiles"); + let store_dir = root.path().join("runtime"); + for runtime_id in [None, Some("runtime-a")] { + let mut factory = + ProfileRuntimeWorkerFactory::new(&profile_dir).with_runtime_store_dir(&store_dir); + if let Some(runtime_id) = runtime_id { + factory = factory.with_runtime_id(runtime_id); + } + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let mut restore = direct_restore_request(worker_ref.clone()); + restore.request.worker_id = worker_ref.worker_id; + restore.request.workspace_api = Some(WorkspaceApiRef { + workspace_id: "workspace-a".to_string(), + base_url: "https://workspace.invalid".to_string(), + }); + let spawn = WorkerExecutionSpawnRequest { + worker_ref, + request: restore.request.clone(), + workspace_scope: None, + context: restore.context.clone(), + workdir_attachments: BTreeMap::new(), + config_bundle: None, + }; + + let error = factory + .spawn_controller(spawn) + .await + .err() + .expect("HTTP is unavailable"); + assert!(error.contains("HTTP transport is unavailable"), "{error}"); + let error = factory.preflight_restore(&restore).await.unwrap_err(); + assert!(error.contains("HTTP transport is unavailable"), "{error}"); + let failure = factory.recover_restore_source(&restore).await.unwrap_err(); + assert_eq!(failure.stage, RuntimeRestoreStage::Preflight); + assert_eq!(failure.resources, RuntimeRestoreResources::None); + assert!(!failure.needs_cleanup()); + assert!(!failure.durable_effects_possible); + assert!( + failure + .to_string() + .contains("HTTP transport is unavailable") + ); + let failure = factory + .prepare_restore(restore) + .await + .err() + .expect("HTTP is unavailable"); + assert_eq!(failure.stage, RuntimeRestoreStage::Preflight); + assert_eq!(failure.resources, RuntimeRestoreResources::None); + assert!(!failure.needs_cleanup()); + assert!(!failure.durable_effects_possible); + assert!( + failure + .to_string() + .contains("HTTP transport is unavailable") + ); + } + assert!(!profile_dir.exists()); + assert!(!store_dir.exists()); + } + + #[cfg(not(feature = "http-server"))] + #[tokio::test] + async fn workspace_config_without_http_returns_unsupported_instead_of_cached_success() { + let root = tempfile::tempdir().unwrap(); + let factory = ProfileRuntimeWorkerFactory::new(root.path()); + let bundle = test_bundle(); + let bundle_ref = ConfigBundleRef { + id: bundle.metadata.id, + digest: bundle.metadata.digest, + }; + let result = factory + .fetch_workspace_config(WorkspaceConfigFetchRequest { + workspace_api: WorkspaceApiRef { + workspace_id: "workspace-a".to_string(), + base_url: "https://workspace.invalid".to_string(), + }, + profile: ProfileSelector::Builtin("builtin:companion".to_string()), + expected: bundle_ref.clone(), + cached: Some(bundle_ref), + }) + .await; + let error = result.unwrap_err(); + assert!(error.contains("unsupported"), "{error}"); + assert!(error.contains("http-server"), "{error}"); + } + + #[cfg(feature = "http-server")] #[test] fn profile_factory_routes_workspace_requests_by_workspace_id() { let profiles = tempfile::tempdir().unwrap(); @@ -8192,6 +8670,7 @@ mod tests { ); } + #[cfg(feature = "http-server")] #[tokio::test] async fn workspace_config_refresh_uses_workspace_scoped_request_client() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -8514,6 +8993,7 @@ mod tests { ); } + #[cfg(feature = "http-server")] #[test] fn restart_restore_reconstructs_runtime_owned_worker_mutation_client() { let identity = RuntimeIdentityMaterial::generate("runtime-source").unwrap(); @@ -8695,6 +9175,7 @@ mod tests { } } + #[cfg(feature = "fs-store")] #[test] fn create_timeout_cancels_factory_and_removes_persisted_worker() { let root = tempfile::tempdir().unwrap(); @@ -8901,13 +9382,19 @@ mod tests { let workspace_backend_ref = RuntimeWorkspaceBackendRef::from_worker_request( &request.request, Some("runtime-test"), - ); + )?; let workspace_context = workspace_backend_ref.worker_context( + #[cfg(feature = "http-server")] &request.worker_ref, + #[cfg(feature = "http-server")] request.workspace_scope.as_ref(), + #[cfg(feature = "http-server")] None, + #[cfg(feature = "http-server")] None, + #[cfg(feature = "http-server")] None, + #[cfg(feature = "http-server")] None, ); let workspace_context = if workspace_context.client().is_available() @@ -8954,6 +9441,7 @@ mod tests { shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), controller_task: tokio::spawn(async {}), workspace_client, + restore_streams: None, }) } async fn restore_controller( @@ -10385,6 +10873,7 @@ mod tests { assert!(!socket_path.exists()); } + #[cfg(feature = "fs-store")] #[test] #[serial_test::serial(worker_allocation)] fn shared_bootstrap_preserves_in_process_transport_for_fresh_and_restored_runtime_workers() { @@ -10649,6 +11138,7 @@ mod tests { } #[test] + #[cfg(feature = "ws-server")] fn running_worker_waits_for_durable_notify_receipt() { let client = MockClient::sequential(vec![MockResponse::Hang(vec![])]); let runtime_base = tempfile::tempdir().unwrap(); @@ -11336,6 +11826,7 @@ mod tests { } #[test] + #[cfg(feature = "http-server")] fn subject_release_reserves_real_controller_and_excludes_runtime_lifecycle() { use runtime_api::{SubjectReleaseAction as Action, SubjectReleaseState as State}; let runtime_base = tempfile::tempdir().unwrap(); diff --git a/crates/worker-runtime/src/worker_backend/restore_recovery.rs b/crates/worker-runtime/src/worker_backend/restore_recovery.rs index 8dd47ee15..54feb63bd 100644 --- a/crates/worker-runtime/src/worker_backend/restore_recovery.rs +++ b/crates/worker-runtime/src/worker_backend/restore_recovery.rs @@ -47,18 +47,26 @@ impl ProfileRuntimeWorkerFactory { let workspace = RuntimeWorkspaceBackendRef::from_worker_request( &request.request, self.runtime_id.as_deref(), - ); + ) + .map_err(cause)?; + #[cfg(feature = "http-server")] let request_client = request .request .workspace_api .as_ref() .and_then(|api| self.workspace_request_clients.get(&api.workspace_id)); let context = workspace.worker_context( + #[cfg(feature = "http-server")] &request.worker_ref, + #[cfg(feature = "http-server")] request.workspace_scope.as_ref(), + #[cfg(feature = "http-server")] self.worker_mutation_identity.as_ref(), + #[cfg(feature = "http-server")] request_client, + #[cfg(feature = "http-server")] self.embedded_worker_mutation_dispatcher.as_ref(), + #[cfg(feature = "http-server")] Some(self.prompt_projection_cache.clone()), ); if pending && context.workspace_id().is_some() { diff --git a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs index 443fbac6f..690a4025f 100644 --- a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs @@ -165,23 +165,35 @@ mod restore_candidate { } fn with_transport(transport: WorkerControllerTransport) -> Self { + Self::with_seed(transport, false) + } + + fn no_model() -> Self { + Self::with_seed(WorkerControllerTransport::InProcess, true) + } + + fn with_seed(transport: WorkerControllerTransport, no_model: bool) -> Self { let root = restore_failure_tempdir(); - let model = MockClient::sequential(vec![ - MockResponse::Complete(vec![ - LlmEvent::tool_use_start(0, "candidate-task-probe", "TaskCreate"), - LlmEvent::tool_input_delta( - 0, - r#"{"subject":"Activated candidate","description":"Execution probe"}"#, - ), - LlmEvent::tool_use_stop(0), - LlmEvent::Status(StatusEvent { - status: ResponseStatus::Completed, - }), - ]), - MockResponse::Complete(simple_text_events()), - MockResponse::Complete(simple_text_events()), - MockResponse::Complete(simple_text_events()), - ]); + let model = if no_model { + MockClient::new(Vec::new()) + } else { + MockClient::sequential(vec![ + MockResponse::Complete(vec![ + LlmEvent::tool_use_start(0, "candidate-task-probe", "TaskCreate"), + LlmEvent::tool_input_delta( + 0, + r#"{"subject":"Activated candidate","description":"Execution probe"}"#, + ), + LlmEvent::tool_use_stop(0), + LlmEvent::Status(StatusEvent { + status: ResponseStatus::Completed, + }), + ]), + MockResponse::Complete(simple_text_events()), + MockResponse::Complete(simple_text_events()), + MockResponse::Complete(simple_text_events()), + ]) + }; let injected = model.clone(); let mut factory = ProfileRuntimeWorkerFactory::new(root.path()) .with_runtime_store_dir(root.path()) @@ -264,15 +276,19 @@ mod restore_candidate { let aggregate = fixture_root.join("workers").join(worker_ref.worker_id.to_string()); let session = WorkerSessionStore::new(aggregate.join("session")).unwrap(); let metadata = WorkerAggregateStore::new(&aggregate, &manifest.worker.name).unwrap(); - let mut engine = Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated( - MockClient::sequential(vec![MockResponse::Hang(simple_text_events().into_iter().take(2).collect())]), - ); + let seed_model = if no_model { + MockClient::new(Vec::new()) + } else { + MockClient::sequential(vec![MockResponse::Hang(simple_text_events().into_iter().take(2).collect())]) + }; + let mut engine = Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(seed_model); engine.set_system_prompt("Committed restore candidate fixture prompt"); let mut worker = Worker::new( manifest.clone(), engine, CombinedStore::new(session.clone(), metadata.clone()), RuntimeWorkspaceBackendRef::from_worker_request(&create, Some("candidate-runtime")) + .unwrap() .worker_context(&worker_ref, Some(&crate::runtime::RuntimeWorkspaceScope::new( "workspace-saved-subjektiv", "server-main", )), None, None, None, None), @@ -282,51 +298,60 @@ mod restore_candidate { worker.enable_worker_metadata_write_through().unwrap(); worker.materialize_durable_session_head().await.unwrap(); let (session_id, segment_id) = (worker.session_id(), worker.segment_id()); - let workspace_client = worker.workspace_client_handle(); - let started = PreparedWorker::new( - worker, - WorkerBootstrapLayout::RuntimeManagedRun { run_dir: fixture_root.join("seed") }, - WorkerControllerTransport::InProcess, - ).start().await.unwrap(); - let mut seed = SeedController(Some(RuntimeWorkerController { - handle: started.handle, - shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), - controller_task: started.controller_task, - workspace_client, - })); - let handle = seed.0.as_ref().unwrap().handle.clone(); - let mut started_events = handle.subscribe(); - submit(&handle, "candidate-setup-turn").await; - // Running and streaming observations can arrive in either - // order. Do not discard TextDelta while awaiting WorkerState. - let mut running = handle.shared_state.catalog_status() == WorkerStatus::Running; - let mut streaming = false; - receive_matching(&mut started_events, "seed running and streaming", |event| { - if let Event::WorkerState { snapshot } = event { - running |= snapshot.catalog_status() == protocol::WorkerStatus::Running; - } - streaming |= matches!(event, Event::TextDelta { .. }); - running && streaming - }).await; - let state = Arc::new(RwLock::new(handle.shared_state.snapshot())); - let mut events = handle.subscribe(); - handle.send(Method::Pause { command: next_internal_command(&state).unwrap() }) - .await.unwrap(); - receive_matching(&mut events, "seed pause acknowledgement", |event| { - matches!(event, Event::WorkerState { snapshot } - if snapshot.catalog_status() == protocol::WorkerStatus::Paused) - }).await; - let mut receipts = Vec::new(); - for id in [FIRST, SECOND] { - let receipt = submit(&handle, id).await; - assert!(matches!(receipt, Event::SubmissionAccepted { - disposition: protocol::SubmissionDisposition::Queued, .. + let receipts = if no_model { + // Persist a real empty Session without starting a seed + // Controller or submitting any model input. + drop(worker); + Vec::new() + } else { + let workspace_client = worker.workspace_client_handle(); + let started = PreparedWorker::new( + worker, + WorkerBootstrapLayout::RuntimeManagedRun { run_dir: fixture_root.join("seed") }, + WorkerControllerTransport::InProcess, + ).start().await.unwrap(); + let mut seed = SeedController(Some(RuntimeWorkerController { + handle: started.handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), + controller_task: started.controller_task, + workspace_client, + restore_streams: None, })); - receipts.push(serde_json::to_value(receipt).unwrap()); - if id == FIRST { notify(&handle).await; } - } - assert_eq!(pending(&handle).await.submissions.len(), 2); - shutdown_profile_controller(seed.0.take().unwrap()).await; + let handle = seed.0.as_ref().unwrap().handle.clone(); + let mut started_events = handle.subscribe(); + submit(&handle, "candidate-setup-turn").await; + // Running and streaming observations can arrive in either + // order. Do not discard TextDelta while awaiting WorkerState. + let mut running = handle.shared_state.catalog_status() == WorkerStatus::Running; + let mut streaming = false; + receive_matching(&mut started_events, "seed running and streaming", |event| { + if let Event::WorkerState { snapshot } = event { + running |= snapshot.catalog_status() == protocol::WorkerStatus::Running; + } + streaming |= matches!(event, Event::TextDelta { .. }); + running && streaming + }).await; + let state = Arc::new(RwLock::new(handle.shared_state.snapshot())); + let mut events = handle.subscribe(); + handle.send(Method::Pause { command: next_internal_command(&state).unwrap() }) + .await.unwrap(); + receive_matching(&mut events, "seed pause acknowledgement", |event| { + matches!(event, Event::WorkerState { snapshot } + if snapshot.catalog_status() == protocol::WorkerStatus::Paused) + }).await; + let mut receipts = Vec::new(); + for id in [FIRST, SECOND] { + let receipt = submit(&handle, id).await; + assert!(matches!(receipt, Event::SubmissionAccepted { + disposition: protocol::SubmissionDisposition::Queued, .. + })); + receipts.push(serde_json::to_value(receipt).unwrap()); + if id == FIRST { notify(&handle).await; } + } + assert_eq!(pending(&handle).await.submissions.len(), 2); + shutdown_profile_controller(seed.0.take().unwrap()).await; + receipts + }; // This orphan pin would be removed by the normal activation // reconciliation. Preparation and discard must leave it alone. let orphan = session.write_uploaded_file( @@ -358,15 +383,17 @@ mod restore_candidate { host, _root: root, }; - let checkpoint = queue_checkpoint(&fixture.entries()); - assert_eq!(checkpoint["pending"].as_array().unwrap().len(), 2); - assert_eq!( - checkpoint["pending_notifications"] - .as_array() - .unwrap() - .len(), - 1 - ); + if !no_model { + let checkpoint = queue_checkpoint(&fixture.entries()); + assert_eq!(checkpoint["pending"].as_array().unwrap().len(), 2); + assert_eq!( + checkpoint["pending_notifications"] + .as_array() + .unwrap() + .len(), + 1 + ); + } fixture.assert_quiet(); fixture } @@ -443,14 +470,23 @@ mod restore_candidate { .get(&self.request.worker_ref) .is_none() ); - assert!( - !self - .backend - .factory - .worker_restore_run_dir(&self.request.worker_ref, self.request.operation_id,) - .unwrap() - .exists() - ); + let run_dir = self.backend.factory.worker_restore_run_dir( + &self.request.worker_ref, self.request.operation_id, + ).unwrap(); + // Private initialization owns only disposable operation-scoped run + // artifacts. It has not published a listener, liveness files or a + // delegated-child projection, and canonical files are checked by + // each caller independently. + let mut artifacts = std::fs::read_dir(&run_dir).unwrap() + .map(|entry| entry.unwrap().file_name().into_string().unwrap()).collect::>(); + artifacts.sort(); + assert_eq!(artifacts, ["artifacts", "spawned", "worker.err.log", "worker.out.log"]); + for log in ["worker.err.log", "worker.out.log"] { + assert_eq!(std::fs::metadata(run_dir.join(log)).unwrap().len(), 0); + } + for directory in ["artifacts", "spawned"] { + assert_eq!(std::fs::read_dir(run_dir.join(directory)).unwrap().count(), 0); + } let table = worker::runtime::worker_allocation::LockFileGuard::open( &worker::runtime::worker_allocation::default_allocation_path().unwrap(), ) @@ -521,6 +557,588 @@ mod restore_candidate { } } + #[test] + #[serial_test::serial(worker_allocation)] + fn session_availability_requires_attachable_current_controller() { + let fixture = Fixture::no_model(); + let worker_ref = &fixture.request.worker_ref; + let observe = || { + fixture + .backend + .worker_session(WorkerSessionObservationRequest { + worker_ref: worker_ref.clone(), + }) + }; + let assert_retained = |availability| { + assert!(matches!( + availability, + runtime_api::WorkerSessionAvailability::RetainedSnapshot { identity, .. } + if identity.session_id == fixture.session_id.to_string() + && identity.segment_id == fixture.segment_id.to_string() + )); + }; + fixture.prepare(); + assert_retained(observe()); + let pending = fixture + .backend + .pending_restore(worker_ref) + .unwrap() + .unwrap(); + fixture + .backend + .activate_restored_worker(fixture.request.operation_id, worker_ref) + .unwrap(); + let execution = fixture.backend.workers.lock().unwrap()[worker_ref].clone(); + assert!(!execution.handle.protocol_is_closed()); + assert!(!execution.shutdown_requested.load(Ordering::Acquire)); + assert!(matches!( + observe(), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + #[cfg(feature = "ws-server")] + let transport = fixture + .backend + .clone() + .attach_worker_protocol(worker_ref) + .unwrap(); + #[cfg(feature = "ws-server")] + transport.validate().unwrap(); + + // Retain the same real operation owner alongside its registered actual + // Controller to exercise the pending-resource visibility guard alone. + let operation_lock = fixture.backend.worker_lock(worker_ref).unwrap(); + { + let _guard = operation_lock.lock().unwrap(); + fixture + .backend + .pending_restores + .lock() + .unwrap() + .insert(worker_ref.clone(), pending); + } + let pending_session = observe(); + #[cfg(feature = "ws-server")] + let pending_protocol = ( + transport.validate(), + fixture.backend.clone().attach_worker_protocol(worker_ref), + ); + // Reset injected visibility before assertions so Fixture can also stop + // and release its allocation when an assertion unwinds. + { + let _guard = operation_lock.lock().unwrap(); + fixture.backend.remove_pending_restore(worker_ref).unwrap(); + } + assert_retained(pending_session); + #[cfg(feature = "ws-server")] + { + assert!(pending_protocol.0.is_err()); + assert!(matches!( + pending_protocol.1, + Err(error) if error.outcome == crate::execution::WorkerExecutionOutcome::Busy + )); + } + assert!(matches!( + observe(), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + { + let _guard = operation_lock.lock().unwrap(); + execution.shutdown_requested.store(true, Ordering::Release); + } + let stopping_session = observe(); + #[cfg(feature = "ws-server")] + let stopping_protocol = ( + transport.validate(), + fixture.backend.clone().attach_worker_protocol(worker_ref), + ); + { + let _guard = operation_lock.lock().unwrap(); + execution.shutdown_requested.store(false, Ordering::Release); + } + assert!(!execution.handle.protocol_is_closed()); + assert_retained(stopping_session); + #[cfg(feature = "ws-server")] + { + assert!(stopping_protocol.0.is_err()); + assert!(stopping_protocol.1.is_err()); + transport.validate().unwrap(); + } + assert!(matches!( + observe(), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + let handle = execution.handle.clone(); + let command = next_internal_command(&execution.worker_state).unwrap(); + fixture + .backend + .run_on_adapter_runtime(async move { + handle + .send(Method::Shutdown { command }) + .await + .map_err(|error| error.to_string())?; + tokio::time::timeout(WAIT, handle.protocol_closed()) + .await + .map_err(|_| "Controller did not close".to_string())?; + Ok(()) + }) + .unwrap(); + assert!(execution.handle.protocol_is_closed()); + assert!(!execution.shutdown_requested.load(Ordering::Acquire)); + assert!( + fixture + .backend + .workers + .lock() + .unwrap() + .contains_key(worker_ref) + ); + assert_retained(observe()); + #[cfg(feature = "ws-server")] + { + assert!(transport.validate().is_err()); + assert!( + fixture + .backend + .clone() + .attach_worker_protocol(worker_ref) + .is_err() + ); + } + assert!(fixture.backend.stop_worker(worker_ref).is_accepted()); + #[cfg(feature = "ws-server")] + drop(transport); + drop(execution); + fixture.assert_released(); + fixture.assert_quiet(); + } + + // Connects Profile restore activation to ordinary protocol input and the + // durable Paused boundary; Runtime admission/Accepted ordering is outside it. + #[test] + #[cfg(feature = "ws-server")] + #[serial_test::serial(worker_allocation)] + fn activated_empty_candidate_accepts_body_input_and_queues_while_paused() { + use crate::execution::WorkerProtocolTransport; + + const S1: &str = "candidate-body-s1"; + const S2: &str = "candidate-body-s2"; + const IDLE_ONLY: &str = "candidate-body-idle-only"; + const OUTPUT: &str = "committed candidate output before pause"; + + // Keep one real connection (paired full snapshot + subscription) for + // every method and observation. Never reconnect to hide missing events. + fn observe( + backend: &WorkerRuntimeExecutionBackend, + mut transport: WorkerProtocolTransport, + phase: &'static str, + no_new_run: bool, + mut matches: impl FnMut(&Event) -> bool + Send + 'static, + ) -> (WorkerProtocolTransport, Event) { + backend + .run_on_adapter_runtime(async move { + let mut seen = Vec::new(); + let matched = tokio::time::timeout(WAIT, async { + loop { + let event = transport + .events + .recv() + .await + .expect("attached protocol stream"); + seen.push(format!("{event:?}")); + if no_new_run { + assert!( + !matches!( + event, + Event::InvokeStart { .. } + | Event::TurnStart { .. } + | Event::UserMessage { .. } + | Event::TextDelta { .. } + | Event::TextDone { .. } + ), + "paused input started another run during {phase}: {event:?}" + ); + } + if matches(&event) { + break event; + } + assert!( + !matches!( + event, + Event::Error { .. } + | Event::SubmissionRejected { .. } + | Event::NotificationRejected { .. } + | Event::Shutdown + ), + "unexpected protocol failure during {phase}: {event:?}" + ); + } + }) + .await + .unwrap_or_else(|_| { + panic!("protocol deadline during {phase}; observed {seen:?}") + }); + Ok((transport, matched)) + }) + .unwrap() + } + + fn user_inputs(entries: &[LogEntry]) -> Vec> { + entries + .iter() + .filter_map(|entry| match entry { + LogEntry::AnnotatedUserInput { segments, .. } => Some(segments.clone()), + _ => None, + }) + .collect() + } + + // Declaration order keeps the bootstrap runtime (which owns the + // existing loopback Host task) alive through Fixture::drop, even on panic. + let bootstrap_backend; + let mut fixture = Fixture::no_model(); + let script = MockClient::sequential(vec![MockResponse::Hang(vec![ + LlmEvent::text_block_start(0), + LlmEvent::text_delta(0, OUTPUT), + LlmEvent::text_block_stop(0, None), + ])]); + // Hang explicitly chains stream::pending() after a complete text block. + // Hang([]) would exercise pre-token rollback instead of durable Paused. + let injected = script.clone(); + let factory = fixture + .backend + .factory + .as_ref() + .clone() + .with_spawn_model_client_factory(move || Box::new(injected.clone())); + bootstrap_backend = std::mem::replace( + &mut fixture.backend, + Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap()), + ); + let worker_ref = &fixture.request.worker_ref; + assert!(user_inputs(&fixture.entries()).is_empty()); + assert!( + !fixture + .entries() + .iter() + .any(|entry| matches!(entry, LogEntry::Invoke { .. })) + ); + let before = fixture.canonical_files(); + fixture.prepare(); // actual Profile prepare_restore; not MockFactory::spawn + fixture.assert_quiet(); // independent empty bootstrap client remains unused + assert_eq!(script.call_count.load(Ordering::SeqCst), 0); + assert!(script.captured.lock().unwrap().is_empty()); + assert_eq!(fixture.canonical_files(), before); + assert!( + fixture + .backend + .clone() + .attach_worker_protocol(worker_ref) + .is_err() + ); + fixture + .backend + .activate_restored_worker(fixture.request.operation_id, worker_ref) + .unwrap(); + let execution = fixture.backend.workers.lock().unwrap()[worker_ref].clone(); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(worker_ref) + .unwrap(); + transport.validate().unwrap(); + assert!( + matches!( + &transport.snapshot, + Event::Snapshot { session, state, in_flight, .. } + if state.state == protocol::WorkerState::Idle + && session.pending_submissions == protocol::PendingSubmissionsSnapshot::default() + && session == &session_store::public_snapshot::project_current_session_snapshot(&fixture.entries()) + && in_flight.is_empty() + ), + "expected full input-ready snapshot: {:?}", + transport.snapshot + ); + assert!(user_inputs(&fixture.entries()).is_empty()); + assert_eq!(script.call_count.load(Ordering::SeqCst), 0); + + assert!( + transport + .dispatch(Method::submit_text(S1, S1)) + .unwrap() + .is_empty() + ); + let (transport, _) = observe( + &fixture.backend, + transport, + "S1 committed and streaming", + false, + { + let (mut accepted, mut user_message, mut running, mut text_done) = + (false, false, false, false); + move |event| { + match event { + Event::SubmissionAccepted { + submission_request_id, + disposition, + .. + } if submission_request_id == S1 => { + assert_eq!(*disposition, protocol::SubmissionDisposition::Started); + accepted = true; + } + Event::UserMessage { segments, .. } => { + assert_eq!(segments, &vec![protocol::Segment::text(S1)]); + user_message = true; + } + Event::WorkerState { snapshot } => { + running |= snapshot.state + == protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )); + } + Event::TextDone { text } => { + assert_eq!(text, OUTPUT); + text_done = true; + } + _ => {} + } + accepted && user_message && running && text_done + } + }, + ); + // Started is only an admission receipt. Require the actual canonical + // AnnotatedUserInput plus its model-visible history before sending Pause. + assert_eq!( + user_inputs(&fixture.entries()), + vec![vec![protocol::Segment::text(S1)]] + ); + let history = session_store::collect_state(&fixture.entries()).annotated_history; + assert!(history.iter().any(|entry| matches!( + &entry.item, LoggedItem::Message { content, .. } + if content.iter().any(|part| matches!(part, LoggedContentPart::Text { text } if text == S1)) + ))); + assert_eq!(script.call_count.load(Ordering::SeqCst), 1); + let pause = next_internal_command(&execution.worker_state).unwrap(); + transport + .dispatch(Method::Pause { command: pause }) + .unwrap(); + let (transport, _) = observe(&fixture.backend, transport, "Pause completion", false, { + let (mut acknowledged, mut paused, mut ended) = (false, false, false); + move |event| { + match event { + Event::CommandAcknowledged { acknowledgement } + if acknowledgement.command_id == pause.command_id => + { + assert_eq!(acknowledgement.command, protocol::WorkerCommandKind::Pause); + assert_eq!( + acknowledgement.disposition, + protocol::WorkerCommandDisposition::Accepted + ); + acknowledged = true; + } + Event::WorkerState { snapshot } => { + paused |= snapshot.state + == protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Paused, + )); + } + Event::RunEnd { result } => { + assert_eq!(*result, protocol::RunResult::Paused); + ended = true; + } + _ => {} + } + acknowledged && paused && ended + } + }); + assert_eq!( + execution.handle.shared_state.snapshot().state, + protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Paused + )) + ); + let paused_entries = fixture.entries(); + assert_eq!( + user_inputs(&paused_entries), + vec![vec![protocol::Segment::text(S1)]] + ); + let invokes = paused_entries + .iter() + .filter(|entry| matches!(entry, LogEntry::Invoke { .. })) + .count(); + assert_eq!(invokes, 1); + // The hanging model response has not appended a final assistant item. + // Prove the actual durable Paused boundary, not a streaming TextDone's + // assumed equivalence to a completed assistant history message. + assert!(paused_entries.iter().any(|entry| matches!( + entry, + LogEntry::RunCompleted { + interrupted: true, + result: agen::EngineResult::Paused, + .. + } + ))); + + transport.dispatch(Method::submit_text(S2, S2)).unwrap(); + let (transport, queued) = observe( + &fixture.backend, + transport, + "S2 Queued receipt", + true, + |event| matches!(event, Event::SubmissionAccepted { submission_request_id, .. } if submission_request_id == S2), + ); + let submission_id = match &queued { + Event::SubmissionAccepted { + submission_id, + disposition, + .. + } => { + assert_eq!(*disposition, protocol::SubmissionDisposition::Queued); + submission_id.clone() + } + _ => unreachable!(), + }; + let checkpoint = queue_checkpoint(&fixture.entries()); + assert_eq!(checkpoint["pending"].as_array().unwrap().len(), 1); + assert_eq!(checkpoint["pending"][0]["submission_request_id"], S2); + assert_eq!(checkpoint["pending"][0]["submission_id"], submission_id); + assert_eq!( + checkpoint["pending"][0]["input"], + serde_json::to_value(vec![protocol::Segment::text(S2)]).unwrap() + ); + let receipts = checkpoint["receipts"].as_array().unwrap(); + let s2_receipts: Vec<_> = receipts + .iter() + .filter(|receipt| receipt["submission_request_id"] == S2) + .collect(); + assert_eq!(s2_receipts.len(), 1); + assert_eq!(s2_receipts[0]["submission_id"], submission_id); + assert_eq!(s2_receipts[0]["disposition"], "queued"); + transport.dispatch(Method::ListPendingSubmissions).unwrap(); + let (transport, listed) = observe( + &fixture.backend, + transport, + "S2 pending list", + true, + |event| matches!(event, Event::PendingSubmissionsChanged { .. }), + ); + let pending = match listed { + Event::PendingSubmissionsChanged { pending } => pending, + _ => unreachable!(), + }; + assert_eq!(pending.submissions.len(), 1); + assert_eq!(pending.head_id.as_deref(), Some(submission_id.as_str())); + assert_eq!(pending.submissions[0].submission_id, submission_id); + + let before_retry = fixture.canonical_files(); + transport.dispatch(Method::submit_text(S2, S2)).unwrap(); + let (transport, replay) = observe( + &fixture.backend, + transport, + "S2 exact retry", + true, + |event| matches!(event, Event::SubmissionAccepted { submission_request_id, .. } if submission_request_id == S2), + ); + assert_eq!( + serde_json::to_value(replay).unwrap(), + serde_json::to_value(queued).unwrap() + ); + assert_eq!(fixture.canonical_files(), before_retry); + transport.dispatch(Method::ListPendingSubmissions).unwrap(); + let (transport, listed) = observe( + &fixture.backend, + transport, + "retry retains one pending", + true, + |event| matches!(event, Event::PendingSubmissionsChanged { .. }), + ); + assert!( + matches!(listed, Event::PendingSubmissionsChanged { pending: actual } if actual == pending) + ); + + // Idle-only admission is a different contract from ordinary Submit: + // observe the real rejection on this connection, not a Queued receipt. + let mut idle_only = WorkerInput::user(IDLE_ONLY); + idle_only.kind = WorkerInputKind::UserIfIdle; + idle_only.submission_request_id = Some(IDLE_ONLY.into()); + let rejected = fixture.backend.dispatch_input(worker_ref, idle_only); + assert!(!rejected.is_accepted(), "{rejected:?}"); + assert!( + rejected.message_or_default().contains("WorkerNotify"), + "{rejected:?}" + ); + let (transport, _) = observe( + &fixture.backend, + transport, + "Paused idle-only rejection", + true, + |event| { + matches!(event, Event::SubmissionRejected { submission_request_id, message } + if submission_request_id == IDLE_ONLY && message.contains("WorkerNotify")) + }, + ); + assert_eq!(fixture.canonical_files(), before_retry); + assert_eq!(queue_checkpoint(&fixture.entries()), checkpoint); + + transport + .dispatch(Method::CancelPendingSubmission { submission_id }) + .unwrap(); + let (transport, _) = observe(&fixture.backend, transport, "cancel S2", true, |event| { + matches!(event, Event::PendingSubmissionsChanged { pending } + if pending.submissions.is_empty() && pending.head_id.is_none()) + }); + transport.dispatch(Method::ListPendingSubmissions).unwrap(); + let (transport, _) = observe( + &fixture.backend, + transport, + "empty pending list after cancel", + true, + |event| { + matches!(event, Event::PendingSubmissionsChanged { pending } + if pending == &protocol::PendingSubmissionsSnapshot::default()) + }, + ); + let entries = fixture.entries(); + assert!( + queue_checkpoint(&entries)["pending"] + .as_array() + .unwrap() + .is_empty() + ); + assert_eq!( + user_inputs(&entries), + vec![vec![protocol::Segment::text(S1)]] + ); + assert_eq!( + entries + .iter() + .filter(|entry| matches!(entry, LogEntry::Invoke { .. })) + .count(), + invokes + ); + assert_eq!( + execution.handle.shared_state.snapshot().state, + protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Paused + )) + ); + assert_eq!(script.call_count.load(Ordering::SeqCst), 1); + assert_eq!(script.captured.lock().unwrap().len(), 1); + fixture.assert_quiet(); + transport.validate().unwrap(); + // Normal Stop owns shutdown and the Controller/transport abort+join + // barrier; do not simulate closure or inject a terminal state. + assert!(fixture.backend.stop_worker(worker_ref).is_accepted()); + assert!(execution.handle.protocol_is_closed()); + assert!(execution.tasks.tasks.lock().unwrap().is_empty()); + assert!(transport.validate().is_err()); + drop(transport); + drop(execution); + fixture.assert_released(); + assert_eq!(script.call_count.load(Ordering::SeqCst), 1); + drop(fixture); + drop(bootstrap_backend); + } + #[test] #[serial_test::serial(worker_allocation)] fn prepare_saved_inputs_rejects_new_input_and_discard_preserves_authority() { @@ -896,6 +1514,7 @@ mod restore_candidate { } } + #[cfg(feature = "fs-store")] #[test] #[serial_test::serial(worker_allocation)] fn attachment_preparation_replay_cannot_hide_logical_authority_revocation() { @@ -1101,4 +1720,6 @@ mod restore_candidate { server_api::WorkerRestoreState::Accepted ); } + + include!("worker_backend_restore_publication_tests.rs"); } diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs index 91530ab28..620fd140d 100644 --- a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -2,26 +2,6 @@ // Worker/bootstrap, local Git Workdir provider, and allocation table. External // model I/O is replaced for saved-input coverage; this is not product E2E. -use crate::restore_test_environment::require_restore_test_environment; - -fn restore_failure_tempdir() -> tempfile::TempDir { - require_restore_test_environment(); - // Explicitly stay inside this checkout, even when TMPDIR points elsewhere. - // Keep fixture components short; no symlink/alias to evade socket limits. - let target = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap() - .join("target"); - fs::create_dir_all(&target).unwrap(); - tempfile::Builder::new() - .prefix("r") - .rand_bytes(3) - .tempdir_in(target) - .unwrap() -} - struct RestoreOpenCountingMaterializer { inner: RuntimeGitMaterializer, opens: Arc, @@ -705,13 +685,8 @@ impl ProfileFailureFixture { #[test] #[serial_test::serial(worker_allocation)] fn profile_restore_prior_effects_survive_read_only_reprepare_failure() { - for (activation_stage, read_stage) in [ - (RuntimeRestoreStage::Feature, RuntimeRestoreStage::Metadata), - ( - RuntimeRestoreStage::Controller, - RuntimeRestoreStage::Reconstruction, - ), - ] { + for read_stage in [RuntimeRestoreStage::Metadata, RuntimeRestoreStage::Reconstruction] { + let activation_stage = RuntimeRestoreStage::ActivationEffects; let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload(); fixture.fail(activation_stage); @@ -908,7 +883,7 @@ fn profile_restore_prior_effects_from_pending_head_repair_survive_read_only_repr fn profile_restore_prior_effects_survive_rebuilt_candidate_pre_effect_activation_failure() { let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload(); - fixture.fail(RuntimeRestoreStage::Feature); + fixture.fail(RuntimeRestoreStage::ActivationEffects); let request = fixture .runtime .test_restore_request(&fixture.worker.worker_ref); @@ -926,7 +901,7 @@ fn profile_restore_prior_effects_survive_rebuilt_candidate_pre_effect_activation .pending_restore(&fixture.worker.worker_ref) .unwrap() .unwrap(); - fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Feature); + fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::ActivationEffects); let effected = persisted_files(&artifacts); assert_ne!(effected, pinned); @@ -1215,7 +1190,7 @@ fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() { #[serial_test::serial(worker_allocation)] fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_deferred_provider() { let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); - fixture.fail(RuntimeRestoreStage::Feature); + fixture.fail(RuntimeRestoreStage::ActivationEffects); fixture .backend .factory @@ -1256,7 +1231,7 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer { let retained = pending.result.lock().unwrap(); let failure = retained.as_ref().unwrap().as_ref().err().unwrap(); - assert_eq!(failure.stage, RuntimeRestoreStage::Feature); + assert_eq!(failure.stage, RuntimeRestoreStage::ActivationEffects); assert_eq!(failure.resources, RuntimeRestoreResources::WorkerStartup); assert_eq!( failure.cleanup, @@ -1281,12 +1256,17 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer assert!(!pending.request.deferred_workdir_sessions.is_empty()); assert_eq!(owned.operation_id, pending.request.operation_id); assert!(owned.allocation.is_some()); + // The private Controller has already closed its router before the later + // factory cleanup fault. Keep the remaining cleanup owner, not live aliases. + assert!(!owned.sessions.is_empty()); assert!( owned .sessions .iter() - .any(|sessions| !sessions.aliases().is_empty()) + .all(|sessions| sessions.aliases().is_empty()) ); + // Raw deferred handles are closed by factory cleanup after this injected + // fault is removed; do not open them while cleanup is still pending. let repeat = fixture .runtime .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) @@ -1484,7 +1464,7 @@ async fn failed_restore_raw_cleanup_is_operation_bound_and_does_not_follow_reuse fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconciles_without_execution() { let fixture = ProfileFailureFixture::new(Duration::from_secs(2)); - fixture.fail(RuntimeRestoreStage::Feature); + fixture.fail(RuntimeRestoreStage::ActivationEffects); let release = Arc::new(tokio::sync::Notify::new()); let (entered_tx, entered) = mpsc::channel(); *fixture @@ -1579,7 +1559,7 @@ fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconcile #[cfg(unix)] #[test] #[serial_test::serial(worker_allocation)] -fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation() { +fn profile_restore_real_socket_failure_retains_committed_candidate_until_cleanup() { let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); fixture .backend @@ -1601,7 +1581,7 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( ); let diagnostic = serde_json::to_string(&result).unwrap(); assert!( - diagnostic.contains("controller startup failed"), + diagnostic.contains("publication failed"), "{diagnostic}" ); assert!(diagnostic.contains("InvalidInput"), "{diagnostic}"); @@ -1624,15 +1604,14 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( assert!(!diagnostic.contains(&fixture.root.path().to_string_lossy().to_string())); assert!(!socket.exists()); assert!(fixture.backend.workers.lock().unwrap().is_empty()); - assert!( - fixture - .backend - .factory - .failed_restore_resources - .lock() - .unwrap() - .is_empty() - ); + assert!(pending.owns_prepared_candidate()); + assert!(pending.result.lock().unwrap().as_ref().unwrap().as_ref().unwrap().can_retry_publication()); + assert!(fixture.backend.factory.failed_restore_resources.lock().unwrap()[&fixture.worker.worker_ref].allocation.is_some()); + let durable: serde_json::Value = serde_json::from_slice(&fs::read( + fixture.root.path().join("s/workers").join(fixture.worker.worker_ref.worker_id.to_string()).join("worker.json") + ).unwrap()).unwrap(); + assert_eq!(durable["execution_state"]["execution"]["intent"]["commit"], "committed"); + assert!(durable["restore_guard"]["owners"][&request.request_id]["receipt"].is_null()); assert!( fixture .backend @@ -1647,10 +1626,14 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( .restore_faults .socket_failure .store(false, Ordering::SeqCst); - eprintln!( - "restore-fault stage=socket-start real-bootstrap=true partial-run=removed allocation=released" - ); - fixture.resume_same_operation(request, operation_id); + // A real permanent path-limit error cannot be repaired by silently + // constructing another endpoint/transport. Explicit cleanup settles the + // retained owner. This case does not assert autonomous terminal handling. + drop(pending); + fixture.runtime.stop_worker(&fixture.worker.worker_ref, None).unwrap(); + fixture.assert_released(); + assert!(!socket.parent().unwrap().exists()); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 1); } #[test] diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs new file mode 100644 index 000000000..394fc64d9 --- /dev/null +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -0,0 +1,535 @@ +// Included in restore_candidate: real Runtime catalog/store, Profile candidate, +// Controller and protocol; only the non-network model client and loopback +// Workspace transport are substituted. These are ordinary integration tests, +// not product-process E2E or live model generation. +#[cfg(all(feature = "fs-store", feature = "ws-server"))] +mod publication { + use super::*; + use crate::fs_store::{ + AtomicWriteFault, FsRuntimeStoreOptions, PersistedWorkerExecution, + PersistedWorkerExecutionState, PersistedWorkerRecord, + }; + + fn stopped_runtime(fixture: &Fixture) -> EmbeddedRuntime { + let options = FsRuntimeStoreOptions { + root: fixture._root.path().join("catalog"), + runtime_id: "candidate-runtime".into(), + display_name: None, + }; + let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( + options.clone(), + fixture.backend.clone(), + ) + .unwrap(); + let create = fixture.request.request.clone(); + let worker_ref = fixture.request.worker_ref.clone(); + runtime + .test_runtime_store() + .write_worker_record(&PersistedWorkerRecord { + restore_guard: crate::runtime::RestoreGuard::default(), + worker_ref: worker_ref.clone(), + worker_id: worker_ref.worker_id, + profile: create.profile.clone(), + display_name: None, + profile_source: create.profile_source.reference(), + config_bundle: create.config_bundle.clone(), + created_at_ms: None, + status: crate::catalog::WorkerStatus::Stopped, + last_finished_submission_request_id: None, + execution_state: PersistedWorkerExecutionState::Available( + PersistedWorkerExecution { + request: create, + binding: None, + restore_intent: crate::catalog::WorkerRestoreIntent::Explicit, + }, + ), + workspace_id: Some( + fixture + .request + .workspace_scope + .as_ref() + .unwrap() + .workspace_id + .clone(), + ), + workdir_attachments: Vec::new(), + logical_workdir_attachments: Vec::new(), + }) + .unwrap(); + drop(runtime); + EmbeddedRuntime::with_fs_store_and_execution_backend(options, fixture.backend.clone()) + .unwrap() + } + + fn durable_operation( + runtime: &EmbeddedRuntime, + worker_ref: &WorkerRef, + ) -> Option { + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + match &loaded.workers[&worker_ref.worker_id].execution_state { + PersistedWorkerExecutionState::ReconciliationRequired( + crate::fs_store::PersistedWorkerLifecycleOperation::Restore(operation), + ) => Some(operation.clone()), + _ => None, + } + } + + fn prepared_gate(fixture: &Fixture) -> (Arc, mpsc::Receiver) { + let release = Arc::new(tokio::sync::Notify::new()); + let (entered, received) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .prepared_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .prepared_entered + .lock() + .unwrap() = Some(entered); + (release, received) + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn runtime_accepted_follows_real_prepared_snapshot_commit_attach_and_input() { + let bootstrap_backend; + let mut fixture = Fixture::no_model(); + let script = MockClient::sequential(vec![MockResponse::Complete(simple_text_events())]); + let injected = script.clone(); + let factory = fixture + .backend + .factory + .as_ref() + .clone() + .with_spawn_model_client_factory(move || Box::new(injected.clone())); + bootstrap_backend = std::mem::replace( + &mut fixture.backend, + Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap()), + ); + let runtime = Arc::new(stopped_runtime(&fixture)); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let before = fixture.canonical_files(); + let (release, prepared) = prepared_gate(&fixture); + let restoring = { + let runtime = runtime.clone(); + let worker_ref = worker_ref.clone(); + let scope = scope.clone(); + let request = request.clone(); + std::thread::spawn(move || { + runtime.restore_worker_operation_scoped(&scope, &worker_ref, request) + }) + }; + let snapshot = prepared.recv_timeout(WAIT).unwrap(); + assert!( + matches!(&snapshot, Event::Snapshot { session, greeting, state, in_flight, .. } + if session == &session_store::public_snapshot::project_current_session_snapshot(&fixture.entries()) + && !greeting.tools.is_empty() && state.state == protocol::WorkerState::Idle && in_flight.is_empty()) + ); + let pending = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!( + pending.commit, + crate::runtime::RestoreCommitFact::NotCommitted + ); + assert_eq!(fixture.canonical_files(), before); + fixture.assert_quiet(); + fixture.assert_no_allocation(); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!( + fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .is_none() + ); + // External attach shares the in-flight backend operation lock; do not + // synchronously acquire that lock while this test holds its prepare gate. + // The empty execution registry and absent hub/allocation prove private + // ownership here; attach is exercised immediately after Accepted below. + assert_eq!(script.call_count.load(Ordering::SeqCst), 0); + release.notify_one(); + let accepted = restoring.join().unwrap().unwrap(); + assert_eq!( + accepted.state, + server_api::WorkerRestoreState::Accepted, + "{accepted:?}" + ); + assert!(durable_operation(&runtime, &worker_ref).is_none()); + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + let guard = + serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap(); + assert!(!guard["owners"][&request.request_id]["receipt"].is_null()); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + assert!( + fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .is_some() + ); + let mut transport = fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + transport.validate().unwrap(); + assert_eq!( + serde_json::to_value(&transport.snapshot).unwrap(), + serde_json::to_value(snapshot).unwrap() + ); + assert!(matches!( + fixture + .backend + .worker_session(crate::execution::WorkerSessionObservationRequest { + worker_ref: worker_ref.clone(), + }), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + transport + .dispatch(Method::submit_text( + "vertical-body-input", + "vertical-body-input", + )) + .unwrap(); + fixture + .backend + .run_on_adapter_runtime(async move { + let (mut admitted, mut committed, mut text, mut ended) = + (false, false, false, false); + tokio::time::timeout(WAIT, async { + while !(admitted && committed && text && ended) { + match transport.events.recv().await.unwrap() { + Event::SubmissionAccepted { + submission_request_id, + .. + } if submission_request_id == "vertical-body-input" => admitted = true, + Event::UserMessage { segments, .. } => { + assert_eq!( + segments, + vec![protocol::Segment::text("vertical-body-input")] + ); + committed = true; + } + Event::TextDone { text: output } => { + assert_eq!(output, "hello from worker"); + text = true; + } + Event::RunEnd { .. } => ended = true, + Event::SubmissionRejected { .. } | Event::Error { .. } => { + panic!("normal input rejected after Accepted") + } + _ => {} + } + } + }) + .await + .map_err(|_| "normal input did not settle".to_string())?; + // Capture and release the whole transport, not only its events field; + // its dispatch/validation closures retain the actual execution. + drop(transport); + Ok(()) + }) + .unwrap(); + assert!(fixture.entries().iter().any( + |entry| matches!(entry, LogEntry::AnnotatedUserInput { segments, .. } + if segments == &vec![protocol::Segment::text("vertical-body-input")]) + )); + assert_eq!(script.call_count.load(Ordering::SeqCst), 1); + let replay = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!(replay.state, server_api::WorkerRestoreState::Accepted); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + let mut stale = request; + stale.request_id = "vertical-new-stale-intent".into(); + assert!( + matches!(runtime.restore_worker_operation_scoped(&scope, &worker_ref, stale), Err(crate::error::RuntimeError::RestoreObservationConflict { worker_id }) if worker_id == worker_ref.worker_id) + ); + runtime + .stop_worker_scoped(&scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + assert_eq!(script.call_count.load(Ordering::SeqCst), 1); + drop(runtime); + drop(bootstrap_backend); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn decision_write_faults_preserve_prepared_owner_or_confirm_uncommitted_cleanup() { + for (fault, sync_unknown) in [ + (AtomicWriteFault::BeforeRename, false), + (AtomicWriteFault::AfterRename, false), + (AtomicWriteFault::AfterRename, true), + ] { + let fixture = Fixture::no_model(); + let runtime = Arc::new(stopped_runtime(&fixture)); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let before = fixture.canonical_files(); + let (release, prepared) = prepared_gate(&fixture); + let restoring = { + let runtime = runtime.clone(); + let worker_ref = worker_ref.clone(); + let scope = scope.clone(); + let request = request.clone(); + std::thread::spawn(move || { + runtime.restore_worker_operation_scoped(&scope, &worker_ref, request) + }) + }; + prepared.recv_timeout(WAIT).unwrap(); + let operation = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!( + operation.commit, + crate::runtime::RestoreCommitFact::NotCommitted + ); + runtime.test_runtime_store().fail_next_worker_write(fault); + if sync_unknown { + runtime + .test_runtime_store() + .fail_next_worker_reconcile_sync(); + } + release.notify_one(); + let result = restoring.join().unwrap().unwrap(); + match fault { + AtomicWriteFault::BeforeRename => { + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{result:?}" + ); + assert_eq!(fixture.canonical_files(), before); + fixture.assert_released(); + } + AtomicWriteFault::AfterRename if !sync_unknown => { + assert_eq!( + result.state, + server_api::WorkerRestoreState::Accepted, + "{result:?}" + ); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + transport.validate().unwrap(); + drop(transport); + runtime + .stop_worker_scoped(&scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + } + AtomicWriteFault::AfterRename => { + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{result:?}" + ); + assert_eq!( + durable_operation(&runtime, &worker_ref) + .unwrap() + .operation_id, + operation.operation_id + ); + assert_eq!( + durable_operation(&runtime, &worker_ref).unwrap().commit, + crate::runtime::RestoreCommitFact::Committed + ); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + fixture.assert_no_allocation(); + assert_eq!(fixture.canonical_files(), before); + let replay = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request) + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::Accepted, + "{replay:?}" + ); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + runtime + .stop_worker_scoped(&scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + } + _ => unreachable!(), + } + fixture.assert_quiet(); + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn uncommitted_cleanup_failure_fences_discarding_candidate_from_activation() { + let fixture = Fixture::no_model(); + let runtime = Arc::new(stopped_runtime(&fixture)); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let before = fixture.canonical_files(); + let (release, prepared) = prepared_gate(&fixture); + let restoring = { + let runtime = runtime.clone(); + let worker_ref = worker_ref.clone(); + let scope = scope.clone(); + let request = request.clone(); + std::thread::spawn(move || { + runtime.restore_worker_operation_scoped(&scope, &worker_ref, request) + }) + }; + prepared.recv_timeout(WAIT).unwrap(); + runtime.test_runtime_store().fail_next_worker_write(AtomicWriteFault::BeforeRename); + fixture.backend.factory.restore_faults.cleanup_fail.store(true, Ordering::SeqCst); + release.notify_one(); + let failed = restoring.join().unwrap().unwrap(); + assert_eq!(failed.state, server_api::WorkerRestoreState::ReconciliationRequired, "{failed:?}"); + let pending = fixture.backend.pending_restore(&worker_ref).unwrap().unwrap(); + assert!(matches!(pending.discard_result.lock().unwrap().as_ref(), Some(Err(_)))); + assert_eq!(durable_operation(&runtime, &worker_ref).unwrap().commit, crate::runtime::RestoreCommitFact::NotCommitted); + for cleanup_fails in [true, false] { + fixture.backend.factory.restore_faults.cleanup_fail.store(cleanup_fails, Ordering::SeqCst); + let replay = runtime.restore_worker_operation_scoped(&scope, &worker_ref, request.clone()).unwrap(); + assert_eq!(replay.state, server_api::WorkerRestoreState::ReconciliationRequired, "discarding candidate must not activate: {replay:?}"); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!(fixture.backend.factory.observation_hub.get(&worker_ref).is_none()); + assert!(fixture.backend.clone().attach_worker_protocol(&worker_ref).is_err()); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 1); + assert_eq!(fixture.canonical_files(), before); + fixture.assert_no_allocation(); + fixture.assert_quiet(); + } + // Explicit teardown verifies this retained owner's cleanup. Autonomous + // cleanup convergence is a separate operation-driver contract. + drop(pending); + runtime.stop_worker_scoped(&scope, &worker_ref, None).unwrap(); + fixture.assert_released(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn publication_failure_retains_same_committed_candidate_without_reconstruction() { + let fixture = Fixture::no_model(); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + fixture + .backend + .factory + .restore_faults + .publication_failure + .store(true, Ordering::SeqCst); + let failed = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + failed.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{failed:?}" + ); + let pending = fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .unwrap(); + assert!(pending.owns_prepared_candidate()); + assert!(pending.activation_started.load(Ordering::SeqCst)); + assert!( + pending + .result + .lock() + .unwrap() + .as_ref() + .unwrap() + .as_ref() + .unwrap() + .can_retry_publication() + ); + let durable = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!(durable.operation_id, pending.request.operation_id); + assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!( + fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .is_none() + ); + assert!( + fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .is_err() + ); + fixture.assert_quiet(); + let result = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::Accepted, + "{result:?}" + ); + let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert_eq!(execution.restore_operation_id, Some(durable.operation_id)); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + transport.validate().unwrap(); + drop(transport); + drop(execution); + drop(pending); + runtime + .stop_worker_scoped(&scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + fixture.assert_quiet(); + } +} diff --git a/crates/worker-runtime/src/worker_source.rs b/crates/worker-runtime/src/worker_source.rs index a50fafc98..57f2e1f00 100644 --- a/crates/worker-runtime/src/worker_source.rs +++ b/crates/worker-runtime/src/worker_source.rs @@ -1,23 +1,32 @@ use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +#[cfg(feature = "http-server")] +use std::time::Duration; +use std::time::{SystemTime, UNIX_EPOCH}; +#[cfg(feature = "http-server")] use crate::workspace_request::{ RuntimeWorkspaceRequest, RuntimeWorkspaceRequestClient, RuntimeWorkspaceRequestError, }; +use worker::WorkspaceResponse; +#[cfg(feature = "http-server")] use worker::{ WorkspaceBinaryRequest, WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspacePromptCatalogResolution, WorkspacePromptProjection, WorkspaceRequest, - WorkspaceRequestMethod, WorkspaceResponse, WorkspaceServerOperation, + WorkspaceRequestMethod, WorkspaceServerOperation, }; use crate::auth::{ RuntimeAuthError, RuntimeIdentityMaterial, RuntimeWorkerMutationSourceSigner, - WORKER_REMOVE_PERMISSION, WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION, - WorkerMutationActorKind, WorkerMutationOperation, WorkerMutationSourceClaims, new_token_id, + WORKER_REMOVE_PERMISSION, WorkerMutationActorKind, WorkerMutationOperation, + WorkerMutationSourceClaims, new_token_id, }; +#[cfg(feature = "http-server")] +use crate::auth::{WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION}; use crate::runtime::RuntimeWorkspaceScope; +#[cfg(feature = "http-server")] use crate::worker_backend::WorkspacePromptProjectionCache; +#[cfg(feature = "http-server")] fn ticket_assignment_role_segment(role: server_api::TicketAssignmentRole) -> &'static str { match role { server_api::TicketAssignmentRole::Orchestrator => "orchestrator", @@ -144,6 +153,7 @@ pub trait EmbeddedWorkerMutationDispatcher: Send + Sync { #[derive(Clone)] enum RuntimeWorkerMutationTransport { + #[cfg(feature = "http-server")] Remote { request_client: RuntimeWorkspaceRequestClient, }, @@ -161,6 +171,7 @@ pub struct RuntimeWorkerMutationForwarder { } impl RuntimeWorkerMutationForwarder { + #[cfg(feature = "http-server")] pub fn remote( identity: &RuntimeIdentityMaterial, scope: RuntimeWorkspaceScope, @@ -206,6 +217,7 @@ impl RuntimeWorkerMutationForwarder { target_worker_id, )?; match (&self.transport, proof) { + #[cfg(feature = "http-server")] ( RuntimeWorkerMutationTransport::Remote { request_client }, RuntimeOwnedWorkerMutationProof::Remote(token), @@ -231,6 +243,7 @@ impl RuntimeWorkerMutationForwarder { } } +#[cfg(feature = "http-server")] struct RemoteWorkerRemoveHttpRequest { request_client: RuntimeWorkspaceRequestClient, source_worker_id: String, @@ -240,6 +253,7 @@ struct RemoteWorkerRemoveHttpRequest { reason: String, } +#[cfg(feature = "http-server")] fn execute_remote_worker_remove_http( request: RemoteWorkerRemoveHttpRequest, ) -> Result { @@ -263,6 +277,7 @@ fn execute_remote_worker_remove_http( execute_remote_worker_remove_http_blocking(request) } +#[cfg(feature = "http-server")] fn execute_remote_worker_remove_http_blocking( request: RemoteWorkerRemoveHttpRequest, ) -> Result { @@ -315,6 +330,7 @@ fn execute_remote_worker_remove_http_blocking( } } +#[cfg(feature = "http-server")] #[derive(Clone)] pub struct RuntimeOwnedWorkspaceClient { workspace_id: String, @@ -325,6 +341,7 @@ pub struct RuntimeOwnedWorkspaceClient { prompt_projection_cache: Option>, } +#[cfg(feature = "http-server")] impl RuntimeOwnedWorkspaceClient { pub fn new( workspace_id: impl Into, @@ -495,6 +512,7 @@ impl RuntimeOwnedWorkspaceClient { } } +#[cfg(feature = "http-server")] impl std::fmt::Debug for RuntimeOwnedWorkspaceClient { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter @@ -510,6 +528,7 @@ impl std::fmt::Debug for RuntimeOwnedWorkspaceClient { } } +#[cfg(feature = "http-server")] fn generated_workspace_response( response: Result>, ) -> Result { @@ -530,6 +549,7 @@ fn generated_workspace_response( } } +#[cfg(feature = "http-server")] impl WorkspaceClient for RuntimeOwnedWorkspaceClient { fn workspace_id(&self) -> Option<&str> { Some(self.request_client.workspace_id()) @@ -929,12 +949,17 @@ fn unix_now_seconds() -> u64 { #[cfg(test)] mod tests { use super::*; + #[cfg(feature = "http-server")] use crate::auth::{ - RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WorkerMutationSourceExpectation, - decode_runtime_request_source_claims, decode_worker_mutation_source_claims, - request_body_digest, verify_worker_mutation_source_proof, + RUNTIME_REQUEST_SOURCE_PROOF_HEADER, decode_runtime_request_source_claims, + request_body_digest, + }; + use crate::auth::{ + WorkerMutationSourceExpectation, decode_worker_mutation_source_claims, + verify_worker_mutation_source_proof, }; + #[cfg(feature = "http-server")] fn binary_loopback(response: Vec) -> (String, std::thread::JoinHandle<(String, Vec)>) { use std::io::{Read, Write}; let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); @@ -972,6 +997,7 @@ mod tests { (base_url, server) } + #[cfg(feature = "http-server")] #[test] fn generated_ticket_assignment_transport_signs_exact_authoring_and_bounded_read_subjects() { use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source}; @@ -1104,6 +1130,7 @@ mod tests { } } + #[cfg(feature = "http-server")] #[test] fn binary_roundtrip_signs_exact_query_method_worker_and_raw_body() { use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source}; @@ -1194,6 +1221,7 @@ mod tests { } } + #[cfg(feature = "http-server")] #[test] fn binary_responses_enforce_declared_and_streamed_limits_with_fixed_errors() { for response in [ @@ -1213,6 +1241,7 @@ mod tests { } } + #[cfg(feature = "http-server")] #[test] fn binary_redirect_returns_status_without_forwarding_authentication() { let (base_url, server) = binary_loopback(b"HTTP/1.1 307 Temporary Redirect\r\nLocation: http://127.0.0.1:1/private\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec()); @@ -1231,6 +1260,7 @@ mod tests { assert!(result.body.is_empty()); } + #[cfg(feature = "http-server")] #[test] fn binary_invalid_or_normalized_targets_fail_with_fixed_errors_before_http() { let client = RuntimeOwnedWorkspaceClient::new( @@ -1277,6 +1307,7 @@ mod tests { ); } + #[cfg(feature = "http-server")] #[test] fn current_prompt_projection_uses_the_shared_runtime_cache_without_http() { let cache = Arc::new(WorkspacePromptProjectionCache::default()); @@ -1312,6 +1343,7 @@ mod tests { assert!(Arc::ptr_eq(&projection.catalog, &second.catalog)); } + #[cfg(feature = "http-server")] #[test] fn prompt_projection_cache_miss_fetches_content_identity() { use std::io::{Read, Write}; @@ -1356,6 +1388,7 @@ mod tests { assert_eq!(fetched.projection.source_digest, "source-3"); } + #[cfg(feature = "http-server")] #[test] fn concurrent_prompt_projection_miss_fetches_once_and_shares_catalog() { use std::io::{Read, Write}; @@ -1427,6 +1460,7 @@ mod tests { ); } + #[cfg(feature = "http-server")] #[test] fn current_prompt_projection_rejects_cross_workspace_response() { use std::io::{Read, Write}; @@ -1473,6 +1507,7 @@ mod tests { assert!(error.to_string().contains("scope mismatch")); } + #[cfg(feature = "http-server")] #[test] fn ordinary_workspace_forwarding_stamps_runtime_identity_and_signs_path_and_query() { use std::io::{Read, Write}; @@ -1531,6 +1566,7 @@ mod tests { ); } + #[cfg(feature = "http-server")] #[test] fn workspace_worker_discovery_signs_dedicated_permission_and_encoded_query() { use std::io::{Read, Write}; @@ -1652,6 +1688,7 @@ mod tests { ); } + #[cfg(feature = "http-server")] #[tokio::test(flavor = "multi_thread")] async fn remote_forwarder_is_safe_in_async_runtime_and_stamps_signed_proof() { use std::io::{Read, Write}; @@ -1815,6 +1852,7 @@ mod tests { assert_eq!(reason, "retire obsolete Worker"); } + #[cfg(feature = "http-server")] #[test] fn runtime_owned_workspace_client_returns_successful_response_without_request_timeout() { // Effective deadline clearing is exercised with a short inherited deadline in @@ -1838,6 +1876,7 @@ mod tests { server.join().unwrap(); } + #[cfg(feature = "http-server")] #[test] fn runtime_owned_workspace_client_honors_per_request_timeouts() { let (base_url, server) = delayed_workspace_response(Duration::from_millis(75)); @@ -1857,6 +1896,7 @@ mod tests { server.join().unwrap(); } + #[cfg(feature = "http-server")] #[test] fn runtime_owned_workspace_client_reports_request_timeouts() { let (base_url, server) = delayed_workspace_response(Duration::from_millis(75)); @@ -1878,6 +1918,7 @@ mod tests { server.join().unwrap(); } + #[cfg(feature = "http-server")] fn delayed_workspace_response(delay: Duration) -> (String, std::thread::JoinHandle<()>) { use std::io::{Read, Write}; use std::net::TcpListener; diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs index cb88a2636..71d77f16f 100644 --- a/crates/workspace-server/src/server_assignment_attention_tests.rs +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -477,6 +477,7 @@ impl AttentionWorkerFactory { shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), controller_task: started.controller_task, workspace_client, + restore_streams: None, }) } } diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs index 25031c8f6..c713af5a7 100644 --- a/crates/workspace-server/src/server_assignment_delivery_tests.rs +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -835,6 +835,7 @@ impl DeliveryWorkerFactory { shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), controller_task: started.controller_task, workspace_client, + restore_streams: None, }) } } From ceb01f6eaa4a467a74274b1df2564ac092b38816 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 10:03:56 +0900 Subject: [PATCH 09/28] feat: runtime own restore continuation beyond caller timeout --- crates/worker-runtime/src/execution.rs | 84 ++- crates/worker-runtime/src/fs_store.rs | 93 +++- crates/worker-runtime/src/runtime.rs | 138 ++++- .../src/runtime/restore_continuation.rs | 498 ++++++++++++++++++ .../worker-runtime/src/test_support_tests.rs | 1 + crates/worker-runtime/src/worker_backend.rs | 94 +++- .../src/worker_backend/restore_recovery.rs | 4 + .../worker_backend_restore_failure_tests.rs | 2 + ...orker_backend_restore_publication_tests.rs | 138 +++++ 9 files changed, 1030 insertions(+), 22 deletions(-) create mode 100644 crates/worker-runtime/src/runtime/restore_continuation.rs diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index b95347102..39faadab5 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -242,6 +242,9 @@ enum WorkerExecutionPublicationState { #[derive(Clone)] pub struct WorkerExecutionContext { worker_ref: WorkerRef, + restore_progress: Option>, + restore_repair_authorizer: + Option Result<(), RuntimeError> + Send + Sync>>, #[cfg(feature = "ws-server")] observation_publisher: WorkerObservationPublisher, #[cfg(feature = "ws-server")] @@ -256,6 +259,8 @@ impl WorkerExecutionContext { ) -> Self { Self { worker_ref, + restore_progress: None, + restore_repair_authorizer: None, observation_publisher, publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Active)), } @@ -268,6 +273,8 @@ impl WorkerExecutionContext { ) -> Self { Self { worker_ref, + restore_progress: None, + restore_repair_authorizer: None, observation_publisher, publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Pending( VecDeque::new(), @@ -277,12 +284,56 @@ impl WorkerExecutionContext { #[cfg(not(feature = "ws-server"))] pub(crate) fn new(worker_ref: WorkerRef) -> Self { - Self { worker_ref } + Self { + worker_ref, + restore_progress: None, + restore_repair_authorizer: None, + } } #[cfg(not(feature = "ws-server"))] pub(crate) fn candidate(worker_ref: WorkerRef) -> Self { - Self { worker_ref } + Self { + worker_ref, + restore_progress: None, + restore_repair_authorizer: None, + } + } + + pub(crate) fn with_restore_repair_authorizer( + mut self, + authorizer: Arc< + dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync, + >, + ) -> Self { + self.restore_repair_authorizer = Some(authorizer); + self + } + + pub(crate) fn authorize_restore_source_repair( + &self, + operation: WorkerLifecycleOperationId, + ) -> Result<(), RuntimeError> { + match &self.restore_repair_authorizer { + Some(authorizer) => authorizer(operation), + // Direct factory/component users own their source repair separately; + // only Runtime contexts carry a Runtime aggregate authority callback. + None => Ok(()), + } + } + + pub(crate) fn with_restore_progress( + mut self, + progress: Arc, + ) -> Self { + self.restore_progress = Some(progress); + self + } + + pub(crate) fn restore_progressed(&self, operation_id: WorkerLifecycleOperationId) { + if let Some(progress) = &self.restore_progress { + progress(operation_id); + } } pub fn worker_ref(&self) -> &WorkerRef { @@ -424,6 +475,7 @@ pub struct WorkerExecutionSpawnRequest { /// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker. #[derive(Clone, Debug)] pub struct WorkerExecutionRestoreRequest { + pub source_repair_possible: bool, pub operation_id: WorkerLifecycleOperationId, pub worker_ref: WorkerRef, pub request: crate::catalog::CreateWorkerRequest, @@ -565,6 +617,15 @@ impl fmt::Debug for WorkerProtocolTransport { } } +/// Derived resource disposition for an existing restore owner. Not durable +/// authority: Runtime must reread its decision before choosing any action. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RestoreRecoveryDisposition { + InFlight, + Continue, + Discard, +} + /// The connection selects this backend; WorkerRef identifies an execution, /// never a capability or authorization token. pub trait WorkerExecutionBackend: Send + Sync + 'static { @@ -643,6 +704,16 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { None } + /// Opt in to Runtime-owned continuation. Waiting is signaled through the + /// original execution context; no callback may wait for Runtime locks. + fn restore_recovery_disposition( + &self, + _operation_id: WorkerLifecycleOperationId, + _worker_ref: &WorkerRef, + ) -> Option { + None + } + fn reconcile_restore( &self, request: WorkerExecutionRestoreRequest, @@ -938,6 +1009,15 @@ impl WorkerExecutionBackendRef { .completed_restore_failure(operation_id, worker_ref) } + pub(crate) fn restore_recovery_disposition( + &self, + operation_id: WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Option { + self.backend + .restore_recovery_disposition(operation_id, worker_ref) + } + pub(crate) fn reconcile_restore( &self, request: WorkerExecutionRestoreRequest, diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index 5418550d1..d2b9c86de 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -43,7 +43,8 @@ use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; #[path = "test_support.rs"] pub mod test_support; -pub(crate) const SCHEMA_VERSION: u32 = 13; +pub(crate) const SCHEMA_VERSION: u32 = 14; +const RESTORE_DECISION_SCHEMA_VERSION: u32 = 13; const RESTORE_ADMISSION_SCHEMA_VERSION: u32 = 12; // Frozen source schemas: do not move their meaning when adding a migration. const SOURCE_SCHEMA_VERSION: u32 = 11; @@ -653,6 +654,7 @@ pub(crate) struct PersistedWorkerRestoreOperation { pub(crate) operation_id: WorkerLifecycleOperationId, pub(crate) mode: PersistedWorkerRestoreMode, pub(crate) commit: crate::runtime::RestoreCommitFact, + pub(crate) source_repair_possible: bool, pub(crate) request: CreateWorkerRequest, pub(crate) binding: Option, pub(crate) last_settled_status: WorkerStatus, @@ -829,11 +831,12 @@ fn plan_runtime_store_migration( | EXECUTION_SCHEMA_VERSION | SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION + | RESTORE_DECISION_SCHEMA_VERSION ) { return Err(runtime_store_corrupt( &runtime_path, format!( - "unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, or {SCHEMA_VERSION}" + "unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, {RESTORE_DECISION_SCHEMA_VERSION}, or {SCHEMA_VERSION}" ), )); } @@ -1165,7 +1168,10 @@ fn migrate_worker_record( ) -> Result { if matches!( source_schema_version, - EXECUTION_SCHEMA_VERSION | SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION + EXECUTION_SCHEMA_VERSION + | SOURCE_SCHEMA_VERSION + | RESTORE_ADMISSION_SCHEMA_VERSION + | RESTORE_DECISION_SCHEMA_VERSION ) { // Admission identity survives migration. Older pending records did not // distinguish a committed decision from preparation or failed cleanup. @@ -1179,7 +1185,10 @@ fn migrate_worker_record( "Worker schema does not match Runtime migration source".into(), )); } - migrate_restore_commit_fact(&mut identity_document, identity_path)?; + if source_schema_version != RESTORE_DECISION_SCHEMA_VERSION { + migrate_restore_commit_fact(&mut identity_document, identity_path)?; + } + migrate_restore_repair_fact(&mut identity_document, identity_path)?; identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION); let aggregate: WorkerAggregateRecord = serde_json::from_value(identity_document) .map_err(|error| runtime_store_corrupt(identity_path, error.to_string()))?; @@ -1294,6 +1303,42 @@ fn migrate_restore_commit_fact( Ok(()) } +// Earlier writers could repair another canonical store before commit without +// retaining that authority in this aggregate. Missing evidence is never false. +fn migrate_restore_repair_fact( + document: &mut serde_json::Value, + path: &Path, +) -> Result<(), RuntimeError> { + let execution = &mut document["execution_state"]; + if execution["state"] != "reconciliation_required" { + return Ok(()); + } + let operation = &mut execution["execution"]; + let restore = match operation["operation"].as_str() { + Some("restore") => Some(&mut operation["intent"]), + Some("stop") => operation["intent"] + .as_object_mut() + .and_then(|stop| stop.get_mut("pending_restore")) + .filter(|restore| !restore.is_null()), + _ => None, + }; + if let Some(restore) = restore { + let restore = restore.as_object_mut().ok_or_else(|| { + runtime_store_corrupt(path, "legacy pending Restore must be an object".into()) + })?; + if restore + .insert("source_repair_possible".into(), serde_json::json!(true)) + .is_some() + { + return Err(runtime_store_corrupt( + path, + "legacy Restore contains unexpected source repair evidence".into(), + )); + } + } + Ok(()) +} + // Frozen schema-v9 conversion. Preserve the exact token and all Restore owners // and receipts; generating a fresh token here would lose retry/admission authority. fn migrate_v9_restore_guard( @@ -1608,7 +1653,7 @@ fn migrate_runtime_document( .or_insert_with(|| serde_json::json!({})); if !matches!( source_schema_version, - SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION + SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION | RESTORE_DECISION_SCHEMA_VERSION ) { object.remove("workers"); // This legacy cache is not Runtime lifecycle or restore authority. Bundles @@ -2148,7 +2193,9 @@ fn migrate_runtime_store_in_place( if let Some(metadata) = metadata && !matches!( plan.current_schema_version, - SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION + SOURCE_SCHEMA_VERSION + | RESTORE_ADMISSION_SCHEMA_VERSION + | RESTORE_DECISION_SCHEMA_VERSION ) { atomic_write_json( @@ -4001,7 +4048,13 @@ mod tests { #[test] fn legacy_pending_restore_migrates_as_unknown_and_preserves_stop_ownership() { use crate::runtime::RestoreCommitFact; - for under_stop in [false, true] { + for (source_schema, under_stop) in [ + RESTORE_ADMISSION_SCHEMA_VERSION, + RESTORE_DECISION_SCHEMA_VERSION, + ] + .into_iter() + .flat_map(|schema| [false, true].map(move |stop| (schema, stop))) + { let (_temp, store, mut worker, path) = canonical_worker_store(); let root = store.root.clone(); let PersistedWorkerExecutionState::Available(execution) = @@ -4013,6 +4066,7 @@ mod tests { operation_id: WorkerLifecycleOperationId::new(), mode: PersistedWorkerRestoreMode::Explicit, commit: RestoreCommitFact::Committed, + source_repair_possible: false, request: execution.request.clone(), binding: execution.binding.clone(), last_settled_status: worker.status, @@ -4033,21 +4087,27 @@ mod tests { PersistedWorkerExecutionState::ReconciliationRequired(operation); let mut legacy = serde_json::to_value(WorkerAggregateRecord::from_persisted(&worker)).unwrap(); - legacy["schema_version"] = serde_json::json!(RESTORE_ADMISSION_SCHEMA_VERSION); + legacy["schema_version"] = serde_json::json!(source_schema); let intent = &mut legacy["execution_state"]["execution"]["intent"]; let restore = if under_stop { &mut intent["pending_restore"] } else { intent }; - restore.as_object_mut().unwrap().remove("commit"); + if source_schema == RESTORE_ADMISSION_SCHEMA_VERSION { + restore.as_object_mut().unwrap().remove("commit"); + } + restore + .as_object_mut() + .unwrap() + .remove("source_repair_possible"); let expected_request = restore["request"].clone(); let before_guard = legacy["restore_guard"].clone(); fs::write(&path, serde_json::to_vec(&legacy).unwrap()).unwrap(); write_empty_schema_v8_store(&root); let runtime_path = root.join(RUNTIME_FILE); let mut runtime: serde_json::Value = read_json(&runtime_path, "read fixture").unwrap(); - runtime["schema_version"] = serde_json::json!(RESTORE_ADMISSION_SCHEMA_VERSION); + runtime["schema_version"] = serde_json::json!(source_schema); fs::write(&runtime_path, serde_json::to_vec(&runtime).unwrap()).unwrap(); drop(store); @@ -4062,7 +4122,18 @@ mod tests { PersistedWorkerLifecycleOperation::Restore(restore) => restore, PersistedWorkerLifecycleOperation::Stop(stop) => stop.pending_restore.unwrap(), }; - assert_eq!(restored.commit, RestoreCommitFact::LegacyUnknown); + assert_eq!( + restored.commit, + if source_schema == RESTORE_DECISION_SCHEMA_VERSION { + RestoreCommitFact::Committed + } else { + RestoreCommitFact::LegacyUnknown + } + ); + assert!( + restored.source_repair_possible, + "older source-repair effects must remain possible" + ); assert_eq!( serde_json::to_value(restored.request).unwrap(), expected_request diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 251e423c2..6c396e6b4 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -1,3 +1,4 @@ +mod restore_continuation; use crate::catalog::{ ConfigBundleRef, CreateWorkerRequest, LogicalWorkdirAttachment, ProfileSelector, ProfileSourceArchiveSource, RepositoryRefObservation, RepositoryRefObservationRequest, @@ -323,6 +324,7 @@ impl Drop for RuntimeEventSelectorSubscription { pub struct Runtime { inner: Arc>, worker_operations: Arc>>>>, + restore_continuations: Arc, } struct WorkerOperationLease { @@ -586,6 +588,7 @@ impl Runtime { Self { inner: Arc::new(Mutex::new(state)), worker_operations: Arc::new(Mutex::new(BTreeMap::new())), + restore_continuations: Arc::default(), } } @@ -662,6 +665,7 @@ impl Runtime { let runtime = Self { inner: Arc::new(Mutex::new(state)), worker_operations: Arc::new(Mutex::new(BTreeMap::new())), + restore_continuations: Arc::default(), }; runtime.restore_persisted_worker_executions()?; Ok(runtime) @@ -2230,6 +2234,7 @@ impl Runtime { operation_id: WorkerLifecycleOperationId::new(), mode: WorkerRestoreMode::Explicit, commit: RestoreCommitFact::NotCommitted, + source_repair_possible: false, last_settled_status: candidate.status, }); let operation_id = pending.operation_id; @@ -2256,6 +2261,10 @@ impl Runtime { match state.persist_restore_worker_record(&candidate) { Ok(()) => {} Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + drop(state); + if !coordinate { + self.start_restore_continuation(worker_ref.clone(), operation_id)?; + } return Ok(( Some(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, @@ -2287,8 +2296,10 @@ impl Runtime { return Ok((None, owner.request.preparation.clone())); } } - self.restore_admitted_worker_under_lock(worker_ref, WorkerRestoreMode::Explicit) - .map(|result| (Some(result), None)) + let result = + self.restore_admitted_worker_under_lock(worker_ref, WorkerRestoreMode::Explicit); + self.ensure_restore_continuation(worker_ref)?; + result.map(|result| (Some(result), None)) } #[cfg(all(test, feature = "fs-store"))] @@ -2529,6 +2540,8 @@ impl Runtime { .cloned() }); let request = WorkerExecutionRestoreRequest { + source_repair_possible: pending_restore + .is_some_and(|pending| pending.source_repair_possible), operation_id, worker_ref: worker_ref.clone(), request: worker_request, @@ -4075,12 +4088,26 @@ impl Runtime { crate::execution::WorkerExecutionContext::new(worker_ref) } + fn restore_progress_notifier( + &self, + worker_ref: WorkerRef, + ) -> Arc { + let continuations = Arc::downgrade(&self.restore_continuations); + Arc::new(move |operation| { + if let Some(continuations) = continuations.upgrade() { + continuations.signal(&worker_ref, operation); + } + }) + } + #[cfg(feature = "ws-server")] fn restore_candidate_execution_context( &self, worker_ref: WorkerRef, ) -> crate::execution::WorkerExecutionContext { let runtime = Arc::downgrade(&self.inner); + let progress = self.restore_progress_notifier(worker_ref.clone()); + let repair = self.restore_repair_authorizer(worker_ref.clone()); crate::execution::WorkerExecutionContext::candidate( worker_ref, Arc::new(move |worker_ref, payload| { @@ -4101,6 +4128,8 @@ impl Runtime { Ok(state.push_worker_observation_event(worker_ref, payload)) }), ) + .with_restore_progress(progress) + .with_restore_repair_authorizer(repair) } #[cfg(not(feature = "ws-server"))] @@ -4108,7 +4137,11 @@ impl Runtime { &self, worker_ref: WorkerRef, ) -> crate::execution::WorkerExecutionContext { + let progress = self.restore_progress_notifier(worker_ref.clone()); + let repair = self.restore_repair_authorizer(worker_ref.clone()); crate::execution::WorkerExecutionContext::candidate(worker_ref) + .with_restore_progress(progress) + .with_restore_repair_authorizer(repair) } #[cfg(feature = "fs-store")] @@ -4410,6 +4443,9 @@ impl Runtime { commit: candidate .pending_restore .map_or(RestoreCommitFact::NotCommitted, |pending| pending.commit), + source_repair_possible: candidate + .pending_restore + .is_some_and(|pending| pending.source_repair_possible), last_settled_status: candidate.status, }); candidate.restore_candidate_context = Some(candidate_context); @@ -6327,6 +6363,9 @@ struct PendingWorkerRestore { operation_id: WorkerLifecycleOperationId, mode: WorkerRestoreMode, commit: RestoreCommitFact, + // False proves Runtime never granted this operation authority to repair + // canonical source before its decision. True includes legacy uncertainty. + source_repair_possible: bool, last_settled_status: WorkerStatus, } @@ -6460,10 +6499,9 @@ struct WorkerRecord { impl WorkerRecord { fn ensure_restore_rollback_allowed(&self) -> Result<(), RuntimeError> { - if self - .pending_restore - .is_some_and(|pending| pending.commit != RestoreCommitFact::NotCommitted) - { + if self.pending_restore.is_some_and(|pending| { + pending.commit != RestoreCommitFact::NotCommitted || pending.source_repair_possible + }) { return Err(RuntimeError::InvalidRequest( "A committed or legacy-unknown Restore cannot be settled as a rollback".into(), )); @@ -6682,6 +6720,7 @@ impl WorkerRecord { operation_id: operation.operation_id, mode: operation.mode.into(), commit: operation.commit, + source_repair_possible: operation.source_repair_possible, last_settled_status: operation.last_settled_status, }), None, @@ -6698,6 +6737,7 @@ impl WorkerRecord { operation_id: restore.operation_id, mode: restore.mode.into(), commit: restore.commit, + source_repair_possible: restore.source_repair_possible, last_settled_status: restore.last_settled_status, }), Some(PendingWorkerStop { @@ -6762,6 +6802,7 @@ impl WorkerRecord { operation_id: restore.operation_id, mode: restore.mode.into(), commit: restore.commit, + source_repair_possible: restore.source_repair_possible, request: self .request .clone() @@ -6786,6 +6827,7 @@ impl WorkerRecord { operation_id: pending.operation_id, mode: pending.mode.into(), commit: pending.commit, + source_repair_possible: pending.source_repair_possible, request, binding: self .execution_bound @@ -9156,6 +9198,7 @@ mod tests { operation_id: WorkerLifecycleOperationId::new(), mode: WorkerRestoreMode::Explicit, commit: RestoreCommitFact::NotCommitted, + source_repair_possible: false, last_settled_status: WorkerStatus::Stopped, }); let id = "workdir-irrelevant-to-recovery"; @@ -13588,7 +13631,7 @@ mod tests { }) .unwrap_err(); assert!(error.to_string().contains( - "unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, or 13" + "unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, 13, or 14" )); let _ = std::fs::remove_dir_all(root); @@ -15014,6 +15057,87 @@ mod tests { let _ = std::fs::remove_dir_all(root); } + #[cfg(feature = "fs-store")] + #[test] + fn restore_source_repair_authority_is_durable_before_effects_without_reentering_lifecycle_lock() + { + for fault in [ + None, + Some(crate::fs_store::AtomicWriteFault::BeforeRename), + Some(crate::fs_store::AtomicWriteFault::AfterRename), + ] { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("repair-authority"); + let runtime = Runtime::with_fs_store_and_execution_backend( + options, + Arc::new(TestExecutionBackend::default()), + ) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("repair authority")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let lease = runtime.worker_operation_lock(worker.worker_id).unwrap(); + let _guard = lease.lock().unwrap(); + let operation = WorkerLifecycleOperationId::new(); + runtime + .begin_restore_operation( + &worker.worker_ref, + operation, + WorkerRestoreMode::Explicit, + runtime.restore_candidate_execution_context(worker.worker_ref.clone()), + ) + .unwrap(); + if let Some(fault) = fault { + runtime_store(&runtime).fail_next_worker_write(fault); + if matches!(fault, crate::fs_store::AtomicWriteFault::AfterRename) { + runtime_store(&runtime).fail_next_worker_reconcile_sync(); + } + } + let authorizer = runtime.restore_repair_authorizer(worker.worker_ref.clone()); + let callback = authorizer.clone(); + let (sent, received) = std::sync::mpsc::channel(); + let task = std::thread::spawn(move || sent.send(callback(operation)).unwrap()); + // The lifecycle lock above remains held while this exact factory + // callback persists its grant. Waiting on that lock would deadlock. + let granted = received + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("repair callback reentered lifecycle lock"); + task.join().unwrap(); + assert_eq!(granted.is_ok(), fault.is_none()); + let saved = runtime_store(&runtime) + .read_worker_record(&worker.worker_ref) + .unwrap(); + let PersistedWorkerExecutionState::ReconciliationRequired( + PersistedWorkerLifecycleOperation::Restore(saved), + ) = saved.execution_state + else { + panic!("repair grant lost operation") + }; + assert_eq!(saved.operation_id, operation); + assert_eq!(saved.commit, RestoreCommitFact::NotCommitted); + assert_eq!( + saved.source_repair_possible, + !matches!(fault, Some(crate::fs_store::AtomicWriteFault::BeforeRename)) + ); + // A denied/unknown grant must not authorize external repair. Once + // storage is healthy the same operation reconciles, never a new ID. + authorizer(operation).unwrap(); + let rollback = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "unsafe", + "repair effects may exist", + ); + assert!( + runtime + .settle_restore_rollback(&worker.worker_ref, &rollback) + .is_err() + ); + } + } + #[cfg(feature = "fs-store")] #[test] fn restore_decision_reread_preserves_commit_across_uncertainty_stop_and_restart() { diff --git a/crates/worker-runtime/src/runtime/restore_continuation.rs b/crates/worker-runtime/src/runtime/restore_continuation.rs new file mode 100644 index 000000000..78cda5fdc --- /dev/null +++ b/crates/worker-runtime/src/runtime/restore_continuation.rs @@ -0,0 +1,498 @@ +//! Runtime-owned continuation of an admitted restore, independent of its caller. +//! +//! Completion callbacks only signal a condition variable. They never acquire a +//! Runtime/lifecycle lock and are never joined by this driver while holding one. +//! Each step revalidates the canonical owner under the existing Worker lock; +//! waiting/backoff and driver teardown happen outside it. A repeated obstruction +//! parks this owner with its reason, rather than spinning or inventing rollback. + +use super::*; +use crate::execution::RestoreRecoveryDisposition; +use std::collections::HashMap; +use std::sync::{Condvar, Weak}; +use std::thread::JoinHandle; +use std::time::Duration; + +type Key = (WorkerRef, WorkerLifecycleOperationId); + +#[derive(Debug, Default)] +pub(super) struct RestoreContinuations { + owners: Mutex>>, + #[cfg(test)] + paused: std::sync::atomic::AtomicBool, +} + +#[derive(Debug, Default)] +struct Schedule { + signaled: bool, + cancelled: bool, + finished: bool, + blocked_reason: Option, + parked: bool, +} + +#[derive(Debug, Default)] +struct Continuation { + schedule: Mutex, + wake: Condvar, + task: Mutex>>, +} + +impl RestoreContinuations { + pub(super) fn signal(&self, worker_ref: &WorkerRef, operation: WorkerLifecycleOperationId) { + if let Some(owner) = self + .owners + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&(worker_ref.clone(), operation)) + { + let mut schedule = owner + .schedule + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + schedule.signaled = true; + owner.wake.notify_all(); + } + } +} + +impl Drop for RestoreContinuations { + fn drop(&mut self) { + let owners = self + .owners + .get_mut() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for owner in owners.values() { + let mut schedule = owner + .schedule + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + schedule.cancelled = true; + owner.wake.notify_all(); + } + for owner in owners.values() { + if let Some(task) = owner + .task + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + if task.thread().id() != std::thread::current().id() { + let _ = task.join(); + } + } + } + } +} + +#[cfg(test)] +pub(crate) struct ContinuationPause(Arc); + +#[cfg(test)] +impl Drop for ContinuationPause { + fn drop(&mut self) { + self.0 + .paused + .store(false, std::sync::atomic::Ordering::Release); + for owner in self.0.owners.lock().unwrap().values() { + owner.schedule.lock().unwrap().signaled = true; + owner.wake.notify_all(); + } + } +} + +enum Step { + Finished, + Waiting, + Contended, + Retry(String), +} + +impl Runtime { + pub(super) fn restore_repair_authorizer( + &self, + worker_ref: WorkerRef, + ) -> Arc Result<(), RuntimeError> + Send + Sync> { + let inner = Arc::downgrade(&self.inner); + Arc::new(move |operation| { + let inner = inner.upgrade().ok_or(RuntimeError::RuntimeStopped)?; + // The factory caller owns the lifecycle lock. Never reacquire it + // here: it may be awaiting this exact factory task. + let mut state = inner.lock().map_err(|_| RuntimeError::StatePoisoned)?; + state.ensure_running()?; + state.refresh_uncertain_worker_write(&worker_ref)?; + let worker = state.worker(&worker_ref)?; + state.ensure_subject_not_reserved(&worker_ref, false)?; + if worker.pending_stop.is_some() + || worker.pending_restore.is_none_or(|pending| { + pending.operation_id != operation + || pending.commit == RestoreCommitFact::LegacyUnknown + }) + { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + if let Some(owner) = worker + .restore_guard + .active_request_id + .as_ref() + .and_then(|id| worker.restore_guard.owners.get(id)) + { + owner.validate_workdir_preparation_authority(worker)?; + } + if worker + .pending_restore + .is_some_and(|pending| pending.source_repair_possible) + { + return Ok(()); + } + let mut candidate = worker.clone(); + candidate + .pending_restore + .as_mut() + .expect("validated restore owner") + .source_repair_possible = true; + // Grant before touching another canonical store. Unknown durability + // denies repair until authoritative reconciliation; no old-memory write. + state.persist_restore_worker_record(&candidate)?; + state.workers.insert(worker_ref.worker_id, candidate); + Ok(()) + }) + } + + #[cfg(test)] + pub(crate) fn test_wait_restore_continuation_parked( + &self, + worker_ref: &WorkerRef, + operation: WorkerLifecycleOperationId, + timeout: Duration, + ) -> String { + let owner = self + .restore_continuations + .owners + .lock() + .unwrap() + .get(&(worker_ref.clone(), operation)) + .cloned() + .unwrap(); + let (schedule, waited) = owner + .wake + .wait_timeout_while(owner.schedule.lock().unwrap(), timeout, |s| { + !s.parked && !s.finished + }) + .unwrap(); + assert!( + !waited.timed_out() && schedule.parked, + "continuation did not park: {:?}", + schedule.blocked_reason + ); + schedule.blocked_reason.clone().expect("parked reason") + } + + /// Gate autonomous steps when a component test observes a single fault + /// boundary. Dedicated continuation tests run without this gate. + #[cfg(test)] + pub(crate) fn test_pause_restore_continuation(&self) -> ContinuationPause { + assert!( + !self + .restore_continuations + .paused + .swap(true, std::sync::atomic::Ordering::AcqRel) + ); + ContinuationPause(self.restore_continuations.clone()) + } + + #[cfg(test)] + pub(crate) fn test_wait_restore_continuation( + &self, + worker_ref: &WorkerRef, + operation: WorkerLifecycleOperationId, + timeout: Duration, + ) { + let owner = self + .restore_continuations + .owners + .lock() + .unwrap() + .get(&(worker_ref.clone(), operation)) + .cloned() + .expect("continuation owner"); + let schedule = owner.schedule.lock().unwrap(); + let (schedule, waited) = owner + .wake + .wait_timeout_while(schedule, timeout, |s| !s.finished && !s.cancelled) + .unwrap(); + assert!( + !waited.timed_out() && schedule.finished, + "restore continuation did not settle: {:?}", + schedule.blocked_reason + ); + } + + pub(super) fn ensure_restore_continuation( + &self, + worker_ref: &WorkerRef, + ) -> Result<(), RuntimeError> { + let operation = { + let state = self.lock()?; + let worker = state.worker(worker_ref)?; + worker + .pending_restore + .map(|pending| pending.operation_id) + .or_else(|| { + worker + .restore_guard + .active_request_id + .as_ref() + .and_then(|id| worker.restore_guard.owners.get(id)) + .filter(|owner| owner.receipt.is_none()) + .map(|owner| owner.operation_id) + }) + }; + if let Some(operation) = operation { + self.start_restore_continuation(worker_ref.clone(), operation)?; + } + Ok(()) + } + + pub(super) fn start_restore_continuation( + &self, + worker_ref: WorkerRef, + operation: WorkerLifecycleOperationId, + ) -> Result<(), RuntimeError> { + { + let state = self.lock()?; + let Some(backend) = &state.execution_backend else { + return Ok(()); + }; + if backend + .restore_recovery_disposition(operation, &worker_ref) + .is_none() + { + return Ok(()); + } + } + let key = (worker_ref.clone(), operation); + let mut owners = self + .restore_continuations + .owners + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + if owners.contains_key(&key) { + return Ok(()); + } + let owner = Arc::new(Continuation::default()); + let retained = owner.clone(); + let state = Arc::downgrade(&self.inner); + let operations = Arc::downgrade(&self.worker_operations); + let continuations = Arc::downgrade(&self.restore_continuations); + let task = std::thread::Builder::new() + .name("restore-continuation".into()) + .spawn(move || { + run( + state, + operations, + continuations, + retained, + worker_ref, + operation, + ); + }) + .map_err(|error| { + RuntimeError::InvalidRequest(format!( + "Restore continuation could not start: {error}" + )) + })?; + *owner.task.lock().map_err(|_| RuntimeError::StatePoisoned)? = Some(task); + owners.insert(key, owner); + Ok(()) + } + + fn continue_restore_operation( + &self, + worker_ref: &WorkerRef, + operation: WorkerLifecycleOperationId, + ) -> Result { + #[cfg(test)] + if self + .restore_continuations + .paused + .load(std::sync::atomic::Ordering::Acquire) + { + return Ok(Step::Contended); + } + let lease = self.worker_operation_lock(worker_ref.worker_id)?; + // Another caller may be waiting for a backend task. Do not queue a join + // of that caller/driver while retaining the lock it needs to complete. + let _guard = match lease.try_lock() { + Ok(guard) => guard, + Err(std::sync::TryLockError::WouldBlock) => return Ok(Step::Contended), + Err(_) => return Err(RuntimeError::StatePoisoned), + }; + let (backend, pending) = { + let mut state = self.lock()?; + state.ensure_running()?; + state.refresh_uncertain_worker_write(worker_ref)?; + let worker = state.worker(worker_ref)?; + if worker.pending_stop.is_some() { + return Ok(Step::Finished); + } + let owner = worker + .restore_guard + .active_request_id + .as_ref() + .and_then(|id| worker.restore_guard.owners.get(id)); + if owner.is_some_and(|owner| owner.operation_id != operation || owner.receipt.is_some()) + { + return Ok(Step::Finished); + } + let Some(pending) = worker.pending_restore else { + return Ok(Step::Finished); + }; + if pending.operation_id != operation { + return Ok(Step::Finished); + } + if pending.commit == RestoreCommitFact::LegacyUnknown { + return Ok(Step::Retry( + "Saved Restore has no authoritative commit evidence".into(), + )); + } + state.ensure_subject_not_reserved(worker_ref, false)?; + if let Some(owner) = owner { + owner.validate_workdir_preparation_authority(worker)?; + } + let Some(backend) = state.execution_backend.clone() else { + return Ok(Step::Finished); + }; + (backend, pending) + }; + match backend.restore_recovery_disposition(operation, worker_ref) { + None => return Ok(Step::Finished), + Some(RestoreRecoveryDisposition::InFlight) => return Ok(Step::Waiting), + Some(RestoreRecoveryDisposition::Discard) + if pending.commit == RestoreCommitFact::NotCommitted => + { + match backend.discard_uncommitted_restore(operation, worker_ref) { + WorkerExecutionSpawnResult::RolledBack(result) => { + let receipt = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "worker_restore_rolled_back", + result.message_or_default(), + ); + self.settle_restore_rollback(worker_ref, &receipt)?; + return Ok(Step::Finished); + } + result => { + return Ok(Step::Retry(format!( + "Restore cleanup remains unresolved: {result:?}" + ))); + } + } + } + Some(_) => {} + } + let result = self.restore_admitted_worker_under_lock(worker_ref, pending.mode)?; + if backend.restore_recovery_disposition(operation, worker_ref) + == Some(RestoreRecoveryDisposition::InFlight) + { + return Ok(Step::Waiting); + } + if result.state == WorkerRestoreState::ReconciliationRequired { + Ok(Step::Retry(result.message.unwrap_or_else(|| { + "Restore reconciliation remains pending".into() + }))) + } else { + Ok(Step::Finished) + } + } +} + +fn run( + inner: Weak>, + operations: Weak>>>>, + continuations: Weak, + owner: Arc, + worker_ref: WorkerRef, + operation: WorkerLifecycleOperationId, +) { + let mut repeated = 0u32; + let mut last_reason = None; + loop { + let (Some(inner), Some(worker_operations), Some(restore_continuations)) = ( + inner.upgrade(), + operations.upgrade(), + continuations.upgrade(), + ) else { + break; + }; + owner + .schedule + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .parked = false; + let runtime = Runtime { + inner, + worker_operations, + restore_continuations, + }; + let step = runtime.continue_restore_operation(&worker_ref, operation); + // No lifecycle lock or strong Runtime/registry reference survives waiting. + drop(runtime); + let mut schedule = owner + .schedule + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if schedule.cancelled { + break; + } + let contended = matches!(&step, Ok(Step::Contended)); + let reason = match step { + Ok(Step::Finished) => { + schedule.finished = true; + owner.wake.notify_all(); + break; + } + Ok(Step::Waiting) | Ok(Step::Contended) => None, + Ok(Step::Retry(reason)) => Some(reason), + Err(error) => Some(sanitize_worker_create_failure_message(&error.to_string())), + }; + if let Some(reason) = &reason { + repeated = repeated.saturating_add(1); + last_reason = Some(reason.clone()); + schedule.blocked_reason = Some(reason.clone()); + } + // Completed failed attempts can have signaled while this step joined + // them. Do not let our own unchanged failure reset the retry budget. + if reason.is_some() { + schedule.signaled = false; + } + if (!contended && reason.is_none()) || repeated >= 4 { + if repeated >= 4 { + schedule.parked = true; + owner.wake.notify_all(); + tracing::warn!(worker_id = %worker_ref.worker_id, operation_id = %operation, reason = ?last_reason, + "Restore continuation parked after bounded attempts; operation remains unresolved"); + } + schedule = owner + .wake + .wait_while(schedule, |s| !s.signaled && !s.cancelled) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } else { + let delay = Duration::from_millis(100 * (1u64 << repeated.min(4))); + schedule = owner + .wake + .wait_timeout_while(schedule, delay, |s| !s.signaled && !s.cancelled) + .unwrap_or_else(std::sync::PoisonError::into_inner) + .0; + } + if schedule.cancelled { + break; + } + if schedule.signaled { + schedule.signaled = false; + // A completion wakes this same owner, not a new request. Its next + // observed result, not the signal itself, determines progress. + } + } +} diff --git a/crates/worker-runtime/src/test_support_tests.rs b/crates/worker-runtime/src/test_support_tests.rs index a4931def8..562ec2e8e 100644 --- a/crates/worker-runtime/src/test_support_tests.rs +++ b/crates/worker-runtime/src/test_support_tests.rs @@ -564,6 +564,7 @@ fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() { operation_id: WorkerLifecycleOperationId::new(), mode: PersistedWorkerRestoreMode::Explicit, commit: crate::runtime::RestoreCommitFact::NotCommitted, + source_repair_possible: false, request: request(), binding: None, last_settled_status: WorkerStatus::Stopped, diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index cf6bfe5d5..3d95f99ff 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -370,10 +370,23 @@ async fn prepare_restore_operation( factory: &F, request: WorkerExecutionRestoreRequest, ) -> Result, RuntimeRestoreFailure> { + let prior_effects = request.source_repair_possible; factory.preflight_restore(&request).await.map_err(|_| { - RuntimeRestoreFailure::refused(RuntimeRestoreStage::Preflight, "restore preflight refused") + let mut failure = RuntimeRestoreFailure::refused( + RuntimeRestoreStage::Preflight, + "restore preflight refused", + ); + failure.durable_effects_possible = prior_effects; + failure })?; - let repaired = factory.recover_restore_source(&request).await?; + let repaired = factory + .recover_restore_source(&request) + .await + .map_err(|mut failure| { + failure.durable_effects_possible |= prior_effects; + failure + })? + || prior_effects; let candidate = factory .prepare_restore(request) .await @@ -804,6 +817,7 @@ struct RestoreFaults { cleanup_fail: AtomicBool, socket_failure: AtomicBool, publication_failure: AtomicBool, + publication_attempts: std::sync::atomic::AtomicUsize, factory_gate: Mutex>>, factory_entered: Mutex>>, prepared_gate: Mutex>>, @@ -2626,6 +2640,11 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate { } let started = async { #[cfg(test)] + self.factory + .restore_faults + .publication_attempts + .fetch_add(1, Ordering::SeqCst); + #[cfg(test)] if self .factory .restore_faults @@ -3120,6 +3139,7 @@ struct RuntimeWorkerExecution { struct PendingRuntimeRestore { request: WorkerExecutionRestoreRequest, task: tokio::sync::Mutex>>, + task_in_flight: AtomicBool, result: Mutex, RuntimeRestoreFailure>>>, activated: Mutex>>, // Monotonic for this operation, not a claim that its current candidate has @@ -3131,7 +3151,24 @@ struct PendingRuntimeRestore { failure: Mutex>, } +struct RestoreTaskCompletion(Arc); + +impl Drop for RestoreTaskCompletion { + fn drop(&mut self) { + self.0.task_in_flight.store(false, Ordering::Release); + self.0 + .request + .context + .restore_progressed(self.0.request.operation_id); + } +} + impl PendingRuntimeRestore { + fn task_completion(self: &Arc) -> RestoreTaskCompletion { + self.task_in_flight.store(true, Ordering::Release); + RestoreTaskCompletion(self.clone()) + } + fn owns_prepared_candidate(&self) -> bool { matches!( self.result @@ -3455,7 +3492,9 @@ where if task.is_some() { return Err("restore task has not been joined".into()); } + let completion = pending.task_completion(); *task = Some(self.spawn_on_adapter_runtime(async move { + let _completion = completion; if resume_activation && !failure.needs_cleanup() && (failure.durable_effects_possible @@ -5340,6 +5379,7 @@ where let pending = Arc::new(PendingRuntimeRestore { request: request.clone(), task: tokio::sync::Mutex::new(None), + task_in_flight: AtomicBool::new(false), result: Mutex::new(None), activated: Mutex::new(None), activation_started: AtomicBool::new(false), @@ -5361,7 +5401,9 @@ where resources.insert(worker_ref.clone(), Arc::clone(&pending)); let factory = Arc::clone(&self.factory); let result_owner = Arc::clone(&pending); + let completion = pending.task_completion(); let task = self.spawn_on_adapter_runtime(async move { + let _completion = completion; let result = match prepare_restore_operation(factory.as_ref(), request.clone()) .await { @@ -5551,6 +5593,42 @@ where } } + fn restore_recovery_disposition( + &self, + operation_id: crate::execution::WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Option { + use crate::execution::RestoreRecoveryDisposition as D; + let pending = self.pending_restore(worker_ref).ok().flatten(); + Some(match pending { + Some(pending) if pending.request.operation_id == operation_id => { + if pending.task_in_flight.load(Ordering::Acquire) { + D::InFlight + } else if pending + .discard_result + .lock() + .ok() + .is_some_and(|r| matches!(*r, Some(Ok(())))) + { + // Cleanup is proven; prior repair effects may now require + // reconstruction, not repeated discard of a closed candidate. + D::Continue + } else if pending.discard_started.load(Ordering::Acquire) + || pending + .discard_result + .lock() + .ok() + .is_some_and(|r| r.is_some()) + { + D::Discard + } else { + D::Continue + } + } + _ => D::Continue, + }) + } + fn discard_uncommitted_restore( &self, operation_id: crate::execution::WorkerLifecycleOperationId, @@ -5582,8 +5660,10 @@ where self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; if !pending.discard_started.swap(true, Ordering::AcqRel) { let owner = pending.clone(); + let completion = pending.task_completion(); let task = self.spawn_on_adapter_runtime(async move { + let _completion = completion; let retained = owner .result .lock() @@ -5709,8 +5789,10 @@ where pending.activation_started.store(true, Ordering::Release); let owner = pending.clone(); let factory = self.factory.clone(); + let completion = pending.task_completion(); let task = self.spawn_on_adapter_runtime(async move { + let _completion = completion; let candidate = owner .result .lock() @@ -6683,6 +6765,7 @@ mod tests { }); store.write(&metadata).unwrap(); WorkerExecutionRestoreRequest { + source_repair_possible: false, operation_id: crate::execution::WorkerLifecycleOperationId::new(), worker_ref: worker_ref.clone(), request, @@ -7614,6 +7697,7 @@ mod tests { Arc::new(PendingRuntimeRestore { request: direct_restore_request(worker_ref.clone()), task: tokio::sync::Mutex::new(None), + task_in_flight: AtomicBool::new(false), result: Mutex::new(None), activated: Mutex::new(None), activation_started: AtomicBool::new(false), @@ -7657,6 +7741,7 @@ mod tests { let pending = Arc::new(PendingRuntimeRestore { request: request.clone(), task: tokio::sync::Mutex::new(Some(failed)), + task_in_flight: AtomicBool::new(false), result: Mutex::new(None), activated: Mutex::new(None), activation_started: AtomicBool::new(false), @@ -10156,6 +10241,7 @@ mod tests { fn direct_restore_request(worker_ref: WorkerRef) -> WorkerExecutionRestoreRequest { WorkerExecutionRestoreRequest { + source_repair_possible: false, operation_id: crate::execution::WorkerLifecycleOperationId::new(), request: create_request("pending operation"), workspace_scope: None, @@ -10729,6 +10815,7 @@ mod tests { .with_runtime_store_dir(&runtime_store_dir) .with_remote_worker_mutation_identity(identity) .restore_controller(WorkerExecutionRestoreRequest { + source_repair_possible: false, operation_id: crate::execution::WorkerLifecycleOperationId::new(), worker_ref: worker_ref.clone(), request, @@ -10826,6 +10913,7 @@ mod tests { .with_runtime_store_dir(&runtime_store_dir) .with_controller_transport(WorkerControllerTransport::InProcess) .restore_controller(WorkerExecutionRestoreRequest { + source_repair_possible: false, operation_id: crate::execution::WorkerLifecycleOperationId::new(), worker_ref: worker_ref.clone(), request: create_request("embedded restore"), @@ -10990,6 +11078,7 @@ mod tests { runtime.stop_worker(&worker.worker_ref, None).unwrap(); let operation_id = crate::execution::WorkerLifecycleOperationId::new(); let restore_request = WorkerExecutionRestoreRequest { + source_repair_possible: false, operation_id, worker_ref: worker.worker_ref.clone(), request, @@ -11383,6 +11472,7 @@ mod tests { let detail = runtime.create_worker(request.clone()).unwrap(); runtime.stop_worker(&detail.worker_ref, None).unwrap(); let restore = WorkerExecutionRestoreRequest { + source_repair_possible: false, operation_id: crate::execution::WorkerLifecycleOperationId::new(), worker_ref: detail.worker_ref.clone(), request, diff --git a/crates/worker-runtime/src/worker_backend/restore_recovery.rs b/crates/worker-runtime/src/worker_backend/restore_recovery.rs index 54feb63bd..4c751e872 100644 --- a/crates/worker-runtime/src/worker_backend/restore_recovery.rs +++ b/crates/worker-runtime/src/worker_backend/restore_recovery.rs @@ -33,6 +33,10 @@ impl ProfileRuntimeWorkerFactory { if !pending && !incomplete { return Ok(false); } + request + .context + .authorize_restore_source_repair(request.operation_id) + .map_err(|_| cause("canonical source repair authority is not durably confirmed"))?; // The accepted Runtime operation already owns this Session. Its stable // Session identity and sole committed seed, not a new restore request, // are used to reconcile an interrupted first-head publication. diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs index 620fd140d..d622a7462 100644 --- a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -688,6 +688,7 @@ fn profile_restore_prior_effects_survive_read_only_reprepare_failure() { for read_stage in [RuntimeRestoreStage::Metadata, RuntimeRestoreStage::Reconstruction] { let activation_stage = RuntimeRestoreStage::ActivationEffects; let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + let _pause = fixture.runtime.test_pause_restore_continuation(); let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload(); fixture.fail(activation_stage); let request = fixture @@ -882,6 +883,7 @@ fn profile_restore_prior_effects_from_pending_head_repair_survive_read_only_repr #[serial_test::serial(worker_allocation)] fn profile_restore_prior_effects_survive_rebuilt_candidate_pre_effect_activation_failure() { let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + let _pause = fixture.runtime.test_pause_restore_continuation(); let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload(); fixture.fail(RuntimeRestoreStage::ActivationEffects); let request = fixture diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs index 394fc64d9..3cc482aab 100644 --- a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -94,6 +94,141 @@ mod publication { (release, received) } + #[test] + #[serial_test::serial(worker_allocation)] + fn autonomous_late_prepare_and_publication_settle_without_another_request() { + for publication_failure in [false, true] { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend).unwrap().spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let request = runtime.test_restore_request(&worker_ref); + let (release, prepared) = prepared_gate(&fixture); + fixture.backend.factory.restore_faults.publication_failure.store(publication_failure, Ordering::SeqCst); + let first = runtime.restore_worker_operation_scoped(fixture.request.workspace_scope.as_ref().unwrap(), &worker_ref, request.clone()).unwrap(); + prepared.recv_timeout(WAIT).unwrap(); + assert_eq!(first.state, server_api::WorkerRestoreState::ReconciliationRequired); + let operation = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!(operation.commit, crate::runtime::RestoreCommitFact::NotCommitted); + fixture.assert_no_allocation(); + release.notify_one(); + runtime.test_wait_restore_continuation(&worker_ref, operation.operation_id, WAIT); + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + let receipt = serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap(); + assert_eq!(receipt["owners"][&request.request_id]["receipt"]["state"], "accepted"); + assert!(durable_operation(&runtime, &worker_ref).is_none()); + let transport = fixture.backend.clone().attach_worker_protocol(&worker_ref).unwrap(); + transport.validate().unwrap(); + drop(transport); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 1); + fixture.assert_quiet(); + runtime.stop_worker_scoped(fixture.request.workspace_scope.as_ref().unwrap(), &worker_ref, None).unwrap(); + fixture.assert_released(); + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn autonomous_cleanup_dispatch_settles_uncommitted_candidate_without_repost_or_stop() { + let fixture = Fixture::no_model(); + let runtime = Arc::new(stopped_runtime(&fixture)); + let pause = runtime.test_pause_restore_continuation(); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let before = fixture.canonical_files(); + let (release, prepared) = prepared_gate(&fixture); + let restoring = { + let runtime = runtime.clone(); + let worker_ref = worker_ref.clone(); + let scope = scope.clone(); + let request = request.clone(); + std::thread::spawn(move || runtime.restore_worker_operation_scoped(&scope, &worker_ref, request)) + }; + prepared.recv_timeout(WAIT).unwrap(); + runtime.test_runtime_store().fail_next_worker_write(AtomicWriteFault::BeforeRename); + fixture.backend.factory.restore_faults.cleanup_fail.store(true, Ordering::SeqCst); + release.notify_one(); + let first = restoring.join().unwrap().unwrap(); + assert_eq!(first.state, server_api::WorkerRestoreState::ReconciliationRequired); + let operation = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!(operation.commit, crate::runtime::RestoreCommitFact::NotCommitted); + let pending = fixture.backend.pending_restore(&worker_ref).unwrap().unwrap(); + assert!(matches!(pending.discard_result.lock().unwrap().as_ref(), Some(Err(_)))); + fixture.backend.factory.restore_faults.cleanup_fail.store(false, Ordering::SeqCst); + drop(pending); + drop(pause); + runtime.test_wait_restore_continuation(&worker_ref, operation.operation_id, WAIT); + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + let receipt = serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap(); + assert_eq!(receipt["owners"][&request.request_id]["receipt"]["state"], "rolled_back"); + assert_eq!(fixture.canonical_files(), before); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 1); + fixture.assert_released(); + fixture.assert_quiet(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn autonomous_unchanged_publication_obstruction_parks_without_false_terminal_outcome() { + let fixture = Fixture::no_model(); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + fixture.backend.factory.restore_faults.socket_failure.store(true, Ordering::SeqCst); + let request = runtime.test_restore_request(&worker_ref); + let first = runtime.restore_worker_operation_scoped(fixture.request.workspace_scope.as_ref().unwrap(), &worker_ref, request.clone()).unwrap(); + assert_eq!(first.state, server_api::WorkerRestoreState::ReconciliationRequired); + let pending = fixture.backend.pending_restore(&worker_ref).unwrap().unwrap(); + let reason = runtime.test_wait_restore_continuation_parked(&worker_ref, pending.request.operation_id, WAIT); + assert!(reason.contains("publication failed"), "{reason}"); + let durable = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed); + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + let guard = serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap(); + assert!(guard["owners"][&request.request_id]["receipt"].is_null()); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 1); + // Initial call plus at most four backed-off continuation attempts. The + // parked condition is observed directly, not guessed using a sleep. + assert!(fixture.backend.factory.restore_faults.publication_attempts.load(Ordering::SeqCst) <= 5); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!(fixture.backend.factory.observation_hub.get(&worker_ref).is_none()); + drop(pending); + // Permanent path-length obstruction: explicit fixture teardown only, + // not the successful completion path asserted by autonomous tests. + runtime.stop_worker_scoped(fixture.request.workspace_scope.as_ref().unwrap(), &worker_ref, None).unwrap(); + fixture.assert_released(); + fixture.assert_quiet(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn autonomous_late_failure_settles_without_another_request() { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend).unwrap().spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let request = runtime.test_restore_request(&worker_ref); + let release = Arc::new(tokio::sync::Notify::new()); + let (entered, prepared) = mpsc::channel(); + *fixture.backend.factory.restore_faults.factory_gate.lock().unwrap() = Some(release.clone()); + *fixture.backend.factory.restore_faults.factory_entered.lock().unwrap() = Some(entered); + *fixture.backend.factory.restore_faults.fail_at.lock().unwrap() = Some(RuntimeRestoreStage::Feature); + let before = fixture.canonical_files(); + let first = runtime.restore_worker_operation_scoped(fixture.request.workspace_scope.as_ref().unwrap(), &worker_ref, request.clone()).unwrap(); + prepared.recv_timeout(WAIT).unwrap(); + assert_eq!(first.state, server_api::WorkerRestoreState::ReconciliationRequired); + let operation = durable_operation(&runtime, &worker_ref).unwrap(); + release.notify_one(); + runtime.test_wait_restore_continuation(&worker_ref, operation.operation_id, WAIT); + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + let receipt = serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap(); + assert_eq!(receipt["owners"][&request.request_id]["receipt"]["state"], "rolled_back"); + assert_eq!(fixture.canonical_files(), before); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 1); + fixture.assert_released(); + fixture.assert_quiet(); + } + #[test] #[serial_test::serial(worker_allocation)] fn runtime_accepted_follows_real_prepared_snapshot_commit_attach_and_input() { @@ -288,6 +423,7 @@ mod publication { ] { let fixture = Fixture::no_model(); let runtime = Arc::new(stopped_runtime(&fixture)); + let _pause = runtime.test_pause_restore_continuation(); let worker_ref = fixture.request.worker_ref.clone(); let scope = fixture.request.workspace_scope.clone().unwrap(); let request = runtime.test_restore_request(&worker_ref); @@ -396,6 +532,7 @@ mod publication { fn uncommitted_cleanup_failure_fences_discarding_candidate_from_activation() { let fixture = Fixture::no_model(); let runtime = Arc::new(stopped_runtime(&fixture)); + let _pause = runtime.test_pause_restore_continuation(); let worker_ref = fixture.request.worker_ref.clone(); let scope = fixture.request.workspace_scope.clone().unwrap(); let request = runtime.test_restore_request(&worker_ref); @@ -443,6 +580,7 @@ mod publication { fn publication_failure_retains_same_committed_candidate_without_reconstruction() { let fixture = Fixture::no_model(); let runtime = stopped_runtime(&fixture); + let _pause = runtime.test_pause_restore_continuation(); let worker_ref = fixture.request.worker_ref.clone(); let scope = fixture.request.workspace_scope.clone().unwrap(); let request = runtime.test_restore_request(&worker_ref); From 9e3f87ac5237e48e61396c16a0ec9ae7896f6e3c Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 11:11:59 +0900 Subject: [PATCH 10/28] fix: runtime retain restore facts when Stop supersedes publication --- crates/runtime-api/src/lib.rs | 25 +++- crates/server-api/src/lib.rs | 12 +- crates/tui/src/backend_worker_picker.rs | 69 +++++---- .../src/http_server/runtime_management_api.rs | 6 + crates/worker-runtime/src/runtime.rs | 137 ++++++++++++++++-- .../src/runtime/restore_continuation.rs | 16 ++ ...orker_backend_restore_publication_tests.rs | 37 +++++ crates/workspace-server/src/hosts.rs | 3 + openapi/server-api.json | 27 ++-- .../src/lib/generated/companion-api.ts | 2 +- .../src/lib/generated/repository-api.ts | 2 +- .../src/lib/generated/runtime-api.ts | 4 +- web/workspace/src/lib/generated/ticket-api.ts | 2 +- .../src/lib/generated/worker-launch-api.ts | 2 +- .../src/lib/workspace/api/runtime-workers.ts | 9 +- .../workers/workers-page.browser.spec.ts | 34 ++++- .../test/sidebar/worker-actions.test.ts | 1 + web/workspace/tests/runtime-workers.test.ts | 3 +- 18 files changed, 314 insertions(+), 77 deletions(-) diff --git a/crates/runtime-api/src/lib.rs b/crates/runtime-api/src/lib.rs index 729243ba6..e2804d18c 100644 --- a/crates/runtime-api/src/lib.rs +++ b/crates/runtime-api/src/lib.rs @@ -86,6 +86,8 @@ pub enum WorkerRestoreState { Accepted, Rejected, RolledBack, + /// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted. + Superseded, ReconciliationRequired, } @@ -1540,13 +1542,24 @@ mod tests { ) .unwrap(); assert!(request.preparation.is_none()); - for state in [ - "accepted", - "rejected", - "rolled_back", - "reconciliation_required", + for (state, wire) in [ + (WorkerRestoreState::Accepted, "accepted"), + (WorkerRestoreState::Rejected, "rejected"), + (WorkerRestoreState::RolledBack, "rolled_back"), + (WorkerRestoreState::Superseded, "superseded"), + ( + WorkerRestoreState::ReconciliationRequired, + "reconciliation_required", + ), ] { - assert!(serde_json::from_value::(serde_json::json!(state)).is_ok()); + assert_eq!( + serde_json::from_value::(serde_json::json!(wire)).unwrap(), + state + ); + assert_eq!( + serde_json::to_value(state).unwrap(), + serde_json::json!(wire) + ); } assert!( serde_json::from_value::(serde_json::json!( diff --git a/crates/server-api/src/lib.rs b/crates/server-api/src/lib.rs index 1de9be8e8..91502a293 100644 --- a/crates/server-api/src/lib.rs +++ b/crates/server-api/src/lib.rs @@ -544,8 +544,9 @@ pub struct WorkspaceWorkerSessionHistoryResponse { /// comparisons do not replace admission checks under the Runtime operation lock; conflicts do /// not trigger automatic token replacement. /// [`WorkerRestoreState`] Accepted means Restore committed, not that the subsequent run succeeded. -/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects, and -/// ReconciliationRequired needing settled-state checks. Response loss does not imply rollback. +/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects, +/// Superseded by a later Stop, and ReconciliationRequired needing settled-state checks. +/// Response loss does not imply rollback. #[api(reqwest, axum, openapi)] pub trait ServerApi { #[get("/health", status = 200, error_status = 400)] @@ -10566,8 +10567,8 @@ pub enum WorkerOperationState { /// Cross-layer outcome of an explicit Worker restore operation. /// /// `Rejected` is reserved for read-only preflight failures. Once live restore -/// work starts, failure is either a confirmed `RolledBack` operation or a -/// `ReconciliationRequired` result whose commit state must be reread/retried. +/// work starts, distinguish confirmed `RolledBack`, unresolved +/// `ReconciliationRequired`, and a later Stop's `Superseded` outcome. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(rename_all = "snake_case")] @@ -10575,6 +10576,8 @@ pub enum WorkerRestoreState { Accepted, Rejected, RolledBack, + /// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted. + Superseded, ReconciliationRequired, } @@ -12668,6 +12671,7 @@ mod tests { (WorkerRestoreState::Accepted, "accepted"), (WorkerRestoreState::Rejected, "rejected"), (WorkerRestoreState::RolledBack, "rolled_back"), + (WorkerRestoreState::Superseded, "superseded"), ( WorkerRestoreState::ReconciliationRequired, "reconciliation_required", diff --git a/crates/tui/src/backend_worker_picker.rs b/crates/tui/src/backend_worker_picker.rs index 13c77901f..7812f85e3 100644 --- a/crates/tui/src/backend_worker_picker.rs +++ b/crates/tui/src/backend_worker_picker.rs @@ -1119,39 +1119,45 @@ mod tests { } #[tokio::test] - async fn rejected_resume_does_not_observe_or_open_retained_state() { - let lifecycle = FakeBackendWorkerLifecycle { - restore_response: restore_response(BackendWorkerRestoreState::Rejected), - observation: session_observation(serde_json::json!({ - "availability": "unavailable", - "reason": "retention_missing", - "message": "must not be opened" - })), - calls: std::sync::Mutex::new(Vec::new()), - }; - let target = client::BackendRuntimeTarget::new( - "http://127.0.0.1:3000", - "workspace-a", - "runtime-a", - "worker-a", - ); + async fn settled_nonaccepted_resume_does_not_retry_observe_or_open_retained_state() { + for state in [ + BackendWorkerRestoreState::Rejected, + BackendWorkerRestoreState::RolledBack, + BackendWorkerRestoreState::Superseded, + ] { + let lifecycle = FakeBackendWorkerLifecycle { + restore_response: restore_response(state), + observation: session_observation(serde_json::json!({ + "availability": "unavailable", + "reason": "retention_missing", + "message": "must not be opened" + })), + calls: std::sync::Mutex::new(Vec::new()), + }; + let target = client::BackendRuntimeTarget::new( + "http://127.0.0.1:3000", + "workspace-a", + "runtime-a", + "worker-a", + ); - let error = prepare_selected_worker( - &lifecycle, - target, - BackendWorkerPickerIntent::Resume, - Some(client::BackendWorkerRestoreRequest { - expected_observation_token: "observed-worker-state".into(), - request_id: "resume-operation".into(), - }), - |_| Ok(false), - ) - .await - .unwrap_err() - .to_string(); + let error = prepare_selected_worker( + &lifecycle, + target, + BackendWorkerPickerIntent::Resume, + Some(client::BackendWorkerRestoreRequest { + expected_observation_token: "observed-worker-state".into(), + request_id: "resume-operation".into(), + }), + |_| panic!("settled nonaccepted restore must not offer recovery retry"), + ) + .await + .unwrap_err() + .to_string(); - assert!(error.contains("Rejected")); - assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]); + assert!(error.contains(&format!("{state:?}"))); + assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]); + } } #[test] @@ -1176,6 +1182,7 @@ mod tests { for state in [ BackendWorkerRestoreState::Rejected, BackendWorkerRestoreState::RolledBack, + BackendWorkerRestoreState::Superseded, BackendWorkerRestoreState::ReconciliationRequired, ] { let mut response = restore_response(state); diff --git a/crates/worker-runtime/src/http_server/runtime_management_api.rs b/crates/worker-runtime/src/http_server/runtime_management_api.rs index 8bcdb123c..7db666b82 100644 --- a/crates/worker-runtime/src/http_server/runtime_management_api.rs +++ b/crates/worker-runtime/src/http_server/runtime_management_api.rs @@ -694,6 +694,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { server_api::WorkerRestoreState::RolledBack => { runtime_api::WorkerRestoreState::RolledBack } + server_api::WorkerRestoreState::Superseded => { + runtime_api::WorkerRestoreState::Superseded + } server_api::WorkerRestoreState::ReconciliationRequired => { runtime_api::WorkerRestoreState::ReconciliationRequired } @@ -744,6 +747,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { server_api::WorkerRestoreState::RolledBack => { runtime_api::WorkerRestoreState::RolledBack } + server_api::WorkerRestoreState::Superseded => { + runtime_api::WorkerRestoreState::Superseded + } server_api::WorkerRestoreState::ReconciliationRequired => { runtime_api::WorkerRestoreState::ReconciliationRequired } diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 6c396e6b4..ee9d0468e 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -225,6 +225,28 @@ impl RuntimeWorkerRestoreResult { } } + fn superseded_by_stop(pending: Option) -> Self { + let (code, message) = match pending { + Some(pending) if pending.commit == RestoreCommitFact::Committed => ( + "worker_restore_committed_then_stopped", + "Restore committed, then its execution was stopped; prior durable effects are not rolled back", + ), + Some(pending) if pending.commit == RestoreCommitFact::LegacyUnknown => ( + "worker_restore_unknown_decision_then_stopped", + "Restore execution was stopped; its earlier commit decision remains unknown", + ), + Some(pending) if pending.source_repair_possible => ( + "worker_restore_source_repair_then_stopped", + "Uncommitted Restore was stopped; prior source repair effects may remain", + ), + _ => ( + "worker_restore_superseded_by_stop", + "Worker Restore was superseded by confirmed Stop", + ), + }; + Self::failed(WorkerRestoreState::Superseded, code, message) + } + fn into_legacy_result(self) -> Result { match (self.state, self.worker) { (WorkerRestoreState::Accepted, Some(worker)) => Ok(worker), @@ -3684,11 +3706,7 @@ impl Runtime { worker.restore_guard.rotate(); worker .restore_guard - .finish(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::RolledBack, - "worker_restore_superseded_by_stop", - "Worker Restore was superseded by Stop", - )); + .finish(RuntimeWorkerRestoreResult::superseded_by_stop(None)); state.persist_worker_record(&worker)?; let ack = WorkerLifecycleAck { worker_ref: worker_ref.clone(), @@ -4250,6 +4268,7 @@ impl Runtime { WorkerRestoreState::Accepted => return Ok(()), WorkerRestoreState::Rejected | WorkerRestoreState::RolledBack + | WorkerRestoreState::Superseded | WorkerRestoreState::ReconciliationRequired => {} } let mut state = self.lock()?; @@ -4363,6 +4382,8 @@ impl Runtime { worker_ref.worker_id ))); } + let restore_receipt = + RuntimeWorkerRestoreResult::superseded_by_stop(candidate.pending_restore); candidate.pending_stop = None; candidate.pending_restore = None; if let Some(context) = candidate.restore_candidate_context.take() { @@ -4373,13 +4394,7 @@ impl Runtime { candidate.worker_state = None; candidate.restore_intent = WorkerRestoreIntent::Explicit; - candidate - .restore_guard - .finish(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::RolledBack, - "worker_restore_superseded_by_stop", - "Worker Restore was superseded by Stop", - )); + candidate.restore_guard.finish(restore_receipt); candidate.internal_workers.clear(); state.persist_worker_record(&candidate)?; let ack = WorkerLifecycleAck { @@ -15138,6 +15153,104 @@ mod tests { } } + #[cfg(feature = "fs-store")] + #[test] + fn stop_retains_restore_decision_evidence_in_superseded_receipt_across_restart() { + // Component scope: real Runtime admission and durable store; execution + // is substituted. Controller shutdown races are covered separately. + for (commit, repaired, code) in [ + ( + RestoreCommitFact::NotCommitted, + false, + "worker_restore_superseded_by_stop", + ), + ( + RestoreCommitFact::NotCommitted, + true, + "worker_restore_source_repair_then_stopped", + ), + ( + RestoreCommitFact::Committed, + false, + "worker_restore_committed_then_stopped", + ), + ( + RestoreCommitFact::LegacyUnknown, + true, + "worker_restore_unknown_decision_then_stopped", + ), + ] { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("stop-restore-evidence"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("stop restore evidence")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + *backend.restore_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::Errored( + WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "publication not confirmed", + ), + )); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + { + // Fixture represents each canonical checkpoint. Stop must keep + // its classification, rather than infer rollback from cleanup. + let mut state = runtime.lock().unwrap(); + let record = state.worker_mut(&worker.worker_ref).unwrap(); + let pending = record.pending_restore.as_mut().unwrap(); + pending.commit = commit; + pending.source_repair_possible = repaired; + state.persist_worker(&worker.worker_id).unwrap(); + } + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let receipt = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Superseded); + assert_eq!(receipt.reason_code.as_deref(), Some(code)); + assert!(receipt.worker.is_none()); + drop(runtime); + let restarted_backend = Arc::new(TestExecutionBackend::default()); + let restarted = + Runtime::with_fs_store_and_execution_backend(options, restarted_backend.clone()) + .unwrap(); + assert_eq!( + restarted + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(), + receipt + ); + assert_eq!(*restarted_backend.restore_count.lock().unwrap(), 0); + assert_eq!( + restarted.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + let mut input = WorkerInput::user("must not revive"); + input.kind = WorkerInputKind::UserIfIdle; + assert!(restarted.send_input(&worker.worker_ref, input).is_err()); + let mut stale = request; + stale.request_id = "new-intent-with-old-token".into(); + assert!(matches!( + restarted.restore_worker_operation(&worker.worker_ref, stale), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + } + } + #[cfg(feature = "fs-store")] #[test] fn restore_decision_reread_preserves_commit_across_uncertainty_stop_and_restart() { diff --git a/crates/worker-runtime/src/runtime/restore_continuation.rs b/crates/worker-runtime/src/runtime/restore_continuation.rs index 78cda5fdc..3de663219 100644 --- a/crates/worker-runtime/src/runtime/restore_continuation.rs +++ b/crates/worker-runtime/src/runtime/restore_continuation.rs @@ -330,6 +330,22 @@ impl Runtime { Err(std::sync::TryLockError::WouldBlock) => return Ok(Step::Contended), Err(_) => return Err(RuntimeError::StatePoisoned), }; + let stop_is_owned = { + let mut state = self.lock()?; + state.ensure_running()?; + state.refresh_uncertain_worker_write(worker_ref)?; + let worker = state.worker(worker_ref)?; + worker.pending_stop.is_some() + && worker + .pending_restore + .is_some_and(|pending| pending.operation_id == operation) + }; + if stop_is_owned { + // Stop supersedes publication, not ownership of late resources. + // Retain this same continuation until its cleanup is confirmed. + self.reconcile_pending_stop_under_lock(worker_ref)?; + return Ok(Step::Finished); + } let (backend, pending) = { let mut state = self.lock()?; state.ensure_running()?; diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs index 3cc482aab..28323570f 100644 --- a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -94,6 +94,43 @@ mod publication { (release, received) } + #[test] + #[serial_test::serial(worker_allocation)] + fn stop_timeout_retains_autonomous_late_cleanup_and_blocks_remove_until_confirmed() { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend).unwrap().spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let pause = runtime.test_pause_restore_continuation(); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let before = fixture.canonical_files(); + let (release, prepared) = prepared_gate(&fixture); + let result = runtime.restore_worker_operation_scoped(scope, &worker_ref, request.clone()).unwrap(); + prepared.recv_timeout(WAIT).unwrap(); + assert_eq!(result.state, server_api::WorkerRestoreState::ReconciliationRequired); + let operation = durable_operation(&runtime, &worker_ref).unwrap(); + assert!(runtime.stop_worker_scoped(scope, &worker_ref, None).is_err()); + assert!(runtime.delete_worker_scoped(scope, &worker_ref).is_err()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + release.notify_one(); + drop(pause); + runtime.test_wait_restore_continuation(&worker_ref, operation.operation_id, WAIT); + assert_eq!(runtime.worker_detail(&worker_ref).unwrap().status, crate::catalog::WorkerStatus::Stopped); + assert!(durable_operation(&runtime, &worker_ref).is_none()); + assert!(fixture.backend.pending_restore(&worker_ref).unwrap().is_none()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!(fixture.backend.factory.observation_hub.get(&worker_ref).is_none()); + assert!(fixture.backend.clone().attach_worker_protocol(&worker_ref).is_err()); + let receipt = runtime.restore_worker_operation_scoped(scope, &worker_ref, request).unwrap(); + assert_eq!(receipt.state, server_api::WorkerRestoreState::Superseded); + assert_eq!(receipt.reason_code.as_deref(), Some("worker_restore_superseded_by_stop")); + assert_eq!(fixture.canonical_files(), before); + fixture.assert_released(); + fixture.assert_quiet(); + assert!(runtime.delete_worker_scoped(scope, &worker_ref).unwrap().deleted); + } + #[test] #[serial_test::serial(worker_allocation)] fn autonomous_late_prepare_and_publication_settle_without_another_request() { diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 4027b221a..f687c4bf8 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -5063,6 +5063,9 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { runtime_api::WorkerRestoreState::RolledBack => { server_api::WorkerRestoreState::RolledBack } + runtime_api::WorkerRestoreState::Superseded => { + server_api::WorkerRestoreState::Superseded + } runtime_api::WorkerRestoreState::ReconciliationRequired => { server_api::WorkerRestoreState::ReconciliationRequired } diff --git a/openapi/server-api.json b/openapi/server-api.json index 720c13a72..7966ef8e5 100644 --- a/openapi/server-api.json +++ b/openapi/server-api.json @@ -15697,14 +15697,23 @@ "type": "object" }, "WorkerRestoreState": { - "description": "Cross-layer outcome of an explicit Worker restore operation.\n\n`Rejected` is reserved for read-only preflight failures. Once live restore\nwork starts, failure is either a confirmed `RolledBack` operation or a\n`ReconciliationRequired` result whose commit state must be reread/retried.", - "enum": [ - "accepted", - "rejected", - "rolled_back", - "reconciliation_required" - ], - "type": "string" + "description": "Cross-layer outcome of an explicit Worker restore operation.\n\n`Rejected` is reserved for read-only preflight failures. Once live restore\nwork starts, distinguish confirmed `RolledBack`, unresolved\n`ReconciliationRequired`, and a later Stop's `Superseded` outcome.", + "oneOf": [ + { + "enum": [ + "accepted", + "rejected", + "rolled_back", + "reconciliation_required" + ], + "type": "string" + }, + { + "const": "superseded", + "description": "A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.", + "type": "string" + } + ] }, "WorkerRetentionResponse": { "additionalProperties": false, @@ -18155,7 +18164,7 @@ "info": { "title": "Yoi Server API", "version": "0.1.0", - "x-yoi-source-digest": "sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7" + "x-yoi-source-digest": "sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e" }, "openapi": "3.1.0", "paths": { diff --git a/web/workspace/src/lib/generated/companion-api.ts b/web/workspace/src/lib/generated/companion-api.ts index ecd2b3f5d..ec620e61a 100644 --- a/web/workspace/src/lib/generated/companion-api.ts +++ b/web/workspace/src/lib/generated/companion-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-companion-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7) +// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e) // Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/companion-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check diff --git a/web/workspace/src/lib/generated/repository-api.ts b/web/workspace/src/lib/generated/repository-api.ts index bc61338f5..455ae30e0 100644 --- a/web/workspace/src/lib/generated/repository-api.ts +++ b/web/workspace/src/lib/generated/repository-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-repository-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7) +// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e) // Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output // Output: web/workspace/src/lib/generated/repository-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check diff --git a/web/workspace/src/lib/generated/runtime-api.ts b/web/workspace/src/lib/generated/runtime-api.ts index 72647613a..4ca1126ec 100644 --- a/web/workspace/src/lib/generated/runtime-api.ts +++ b/web/workspace/src/lib/generated/runtime-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-runtime-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7) +// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e) // Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/runtime-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check @@ -181,7 +181,7 @@ export type WorkerRestoreResponse = { result: WorkerRestoreResult; runtime_id: s export type WorkerRestoreResult = { diagnostics?: Array; state: WorkerRestoreState; worker?: WorkerSummary | null; }; -export type WorkerRestoreState = "accepted" | "rejected" | "rolled_back" | "reconciliation_required"; +export type WorkerRestoreState = "accepted" | "rejected" | "rolled_back" | "reconciliation_required" | "superseded"; export type WorkerRetentionResponse = { pinned: boolean; retention_state: string; runtime_id: string; worker_id: string; workspace_id: string; }; diff --git a/web/workspace/src/lib/generated/ticket-api.ts b/web/workspace/src/lib/generated/ticket-api.ts index 18b0a6804..fa858f1d9 100644 --- a/web/workspace/src/lib/generated/ticket-api.ts +++ b/web/workspace/src/lib/generated/ticket-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7) +// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e) // Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/ticket-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check diff --git a/web/workspace/src/lib/generated/worker-launch-api.ts b/web/workspace/src/lib/generated/worker-launch-api.ts index 8dd4a964b..d25719a4b 100644 --- a/web/workspace/src/lib/generated/worker-launch-api.ts +++ b/web/workspace/src/lib/generated/worker-launch-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-worker-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7) +// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e) // Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/worker-launch-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check diff --git a/web/workspace/src/lib/workspace/api/runtime-workers.ts b/web/workspace/src/lib/workspace/api/runtime-workers.ts index e00b83485..fcd88470c 100644 --- a/web/workspace/src/lib/workspace/api/runtime-workers.ts +++ b/web/workspace/src/lib/workspace/api/runtime-workers.ts @@ -363,8 +363,13 @@ export function parseWorkerRestoreResponse( exactKeys(result, ["state"], ["diagnostics", "worker"], `${path}.result`); const state = boundedString(result.state, `${path}.result.state`); if ( - !["accepted", "rejected", "rolled_back", "reconciliation_required"] - .includes(state) + ![ + "accepted", + "rejected", + "rolled_back", + "superseded", + "reconciliation_required", + ].includes(state) ) { fail(`${path}.result.state`, "has an unknown value"); } diff --git a/web/workspace/src/routes/w/[workspaceId]/workers/workers-page.browser.spec.ts b/web/workspace/src/routes/w/[workspaceId]/workers/workers-page.browser.spec.ts index a72afa7cb..255879fe9 100644 --- a/web/workspace/src/routes/w/[workspaceId]/workers/workers-page.browser.spec.ts +++ b/web/workspace/src/routes/w/[workspaceId]/workers/workers-page.browser.spec.ts @@ -427,7 +427,12 @@ test.each(["switch", "dispose"] as const)( ); function restoreResponse( - state: "accepted" | "rejected" | "rolled_back" | "reconciliation_required", + state: + | "accepted" + | "rejected" + | "rolled_back" + | "superseded" + | "reconciliation_required", extra = {}, ) { return Response.json({ @@ -461,7 +466,13 @@ test.each(["stopped", "idle", "running", "paused", "unavailable"] as const)( ); test.each( - ["accepted", "rejected", "rolled_back", "reconciliation_required"] as const, + [ + "accepted", + "rejected", + "rolled_back", + "superseded", + "reconciliation_required", + ] as const, )( "Restore classifies %s through the shared helper, without treating HTTP 200 as success", async (state) => { @@ -470,10 +481,11 @@ test.each( const unsubscribe = workspaceAlerts.subscribe((value) => { alerts = value; }); - const request = vi.fn(async (input: RequestInfo | URL) => - String(input).endsWith("/restore") - ? restoreResponse(state) - : Response.json(plan()) + const request = vi.fn( + async (input: RequestInfo | URL, _init?: RequestInit) => + String(input).endsWith("/restore") + ? restoreResponse(state) + : Response.json(plan()), ); vi.stubGlobal("fetch", request); render(WorkersPage, { data: data() }); @@ -498,6 +510,16 @@ test.each( screen.queryByRole("button", { name: "Retry Restore Catalog worker" }), ), ).toBe(state === "reconciliation_required"); + expect( + request.mock.calls.filter(([url]) => String(url).endsWith("/restore")), + ).toHaveLength(1); + expect( + request.mock.calls + .filter(([, init]) => init?.method && init.method !== "GET") + .map(([url]) => String(url)), + ).toEqual([ + "/api/w/page-test/runtimes/runtime-a/workers/worker-a/restore", + ]); // The response is never projected into the catalog by the action. expect(screen.getByRole("cell", { name: "stopped" })).toBeTruthy(); unsubscribe(); diff --git a/web/workspace/test/sidebar/worker-actions.test.ts b/web/workspace/test/sidebar/worker-actions.test.ts index 730618f12..a4d8d8792 100644 --- a/web/workspace/test/sidebar/worker-actions.test.ts +++ b/web/workspace/test/sidebar/worker-actions.test.ts @@ -305,6 +305,7 @@ Deno.test("Restore classification never promotes nonaccepted results and preserv "accepted", "rejected", "rolled_back", + "superseded", "reconciliation_required", ] as const ) { diff --git a/web/workspace/tests/runtime-workers.test.ts b/web/workspace/tests/runtime-workers.test.ts index 2de3059f3..a868cf508 100644 --- a/web/workspace/tests/runtime-workers.test.ts +++ b/web/workspace/tests/runtime-workers.test.ts @@ -173,12 +173,13 @@ function restoredWorker() { restore_observation_token: "generation-b", }; } -Deno.test("Restore parser preserves all four states and optional Workspace summary", () => { +Deno.test("Restore parser preserves typed outcomes and optional Workspace summary", () => { for ( const state of [ "accepted", "rejected", "rolled_back", + "superseded", "reconciliation_required", ] ) { From 6be4e8388bb8448a6477c03fa72b72f7f5fe7a6e Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 11:49:15 +0900 Subject: [PATCH 11/28] feat: recover saved restore operations by their durable decision --- crates/worker-runtime/src/execution.rs | 44 +- crates/worker-runtime/src/runtime.rs | 274 ++++++++- .../src/runtime/restore_continuation.rs | 11 +- crates/worker-runtime/src/worker_backend.rs | 533 +++++++++++++++++- .../worker_backend_restore_failure_tests.rs | 12 + ...orker_backend_restore_publication_tests.rs | 127 +++++ 6 files changed, 945 insertions(+), 56 deletions(-) diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index 39faadab5..ffa6e807f 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -621,9 +621,13 @@ impl fmt::Debug for WorkerProtocolTransport { /// authority: Runtime must reread its decision before choosing any action. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum RestoreRecoveryDisposition { + /// No live or pending owner for this exact operation. Not cleanup evidence. + Absent, InFlight, Continue, Discard, + /// A retained cleanup-only owner; never reconstruct or activate it. + Reclaim, } /// The connection selects this backend; WorkerRef identifies an execution, @@ -704,8 +708,15 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { None } - /// Opt in to Runtime-owned continuation. Waiting is signaled through the - /// original execution context; no callback may wait for Runtime locks. + /// Opt in to Runtime-owned autonomous restore continuation. Pending tasks + /// signal completion through the original execution context; callbacks must + /// not wait for Runtime locks. Independent of resource disposition support. + fn supports_restore_continuation(&self) -> bool { + false + } + + /// Describe exact operation resource ownership for warm/cold recovery + /// dispatch. This does not opt in to callbacks or autonomous continuation. fn restore_recovery_disposition( &self, _operation_id: WorkerLifecycleOperationId, @@ -721,6 +732,24 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { self.restore_worker(request) } + /// Reclaim an uncommitted operation after restart without preparing, + /// repairing, or activating a replacement candidate. Runtime must establish + /// the durable uncommitted decision before calling this seam. Unsupported + /// cleanup is uncertainty, never a successful rollback. + fn reclaim_uncommitted_restore( + &self, + _request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::unsupported( + WorkerExecutionOperation::Restore, + "backend cannot reclaim an uncommitted restore after restart", + ), + worker_state: None, + workdir_attachments: Vec::new(), + } + } + /// Discard only the exact unpublished operation after Runtime has proved /// its durable decision uncommitted. This is not Stop and must not affect a /// connected execution. RolledBack requires confirmed cleanup and no repair. @@ -1009,6 +1038,10 @@ impl WorkerExecutionBackendRef { .completed_restore_failure(operation_id, worker_ref) } + pub(crate) fn supports_restore_continuation(&self) -> bool { + self.backend.supports_restore_continuation() + } + pub(crate) fn restore_recovery_disposition( &self, operation_id: WorkerLifecycleOperationId, @@ -1025,6 +1058,13 @@ impl WorkerExecutionBackendRef { self.backend.reconcile_restore(request) } + pub(crate) fn reclaim_uncommitted_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + self.backend.reclaim_uncommitted_restore(request) + } + pub(crate) fn discard_uncommitted_restore( &self, operation_id: WorkerLifecycleOperationId, diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index ee9d0468e..f55d359f8 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -171,6 +171,12 @@ enum WorkerRestoreMode { Automatic, } +#[derive(Clone, Copy, Eq, PartialEq)] +enum RestoreRecoveryAction { + Continue, + ReclaimUncommitted, +} + #[cfg(feature = "fs-store")] impl From for PersistedWorkerRestoreMode { fn from(mode: WorkerRestoreMode) -> Self { @@ -2332,6 +2338,56 @@ impl Runtime { } } + // Seed durable crash checkpoints without launching a second Controller. + // Tests then reopen the real store with empty backend resource maps. + #[cfg(all(test, feature = "fs-store"))] + pub(crate) fn test_stage_restore_checkpoint( + &self, + worker_ref: &WorkerRef, + commit: RestoreCommitFact, + source_repair_possible: bool, + ) -> WorkerLifecycleOperationId { + let lease = self.worker_operation_lock(worker_ref.worker_id).unwrap(); + let _guard = lease.lock().unwrap(); + let operation = self + .lock() + .unwrap() + .worker(worker_ref) + .unwrap() + .pending_restore + .unwrap(); + self.begin_restore_operation( + worker_ref, + operation.operation_id, + operation.mode, + self.restore_candidate_execution_context(worker_ref.clone()), + ) + .unwrap(); + if source_repair_possible { + self.restore_repair_authorizer(worker_ref.clone())(operation.operation_id).unwrap(); + } + match commit { + RestoreCommitFact::Committed => { + self.commit_restored_worker_execution( + worker_ref, + protocol::WorkerStatus::Idle.into(), + WorkerStatus::Idle, + Vec::new(), + ) + .unwrap(); + } + RestoreCommitFact::LegacyUnknown => { + let mut state = self.lock().unwrap(); + let mut worker = state.worker(worker_ref).unwrap().clone(); + worker.pending_restore.as_mut().unwrap().commit = commit; + state.persist_restore_worker_record(&worker).unwrap(); + state.workers.insert(worker_ref.worker_id, worker); + } + RestoreCommitFact::NotCommitted => {} + } + operation.operation_id + } + #[cfg(test)] pub(crate) fn test_restore_request( &self, @@ -2357,6 +2413,19 @@ impl Runtime { &self, worker_ref: &WorkerRef, mode: WorkerRestoreMode, + ) -> Result { + self.restore_admitted_with_recovery_under_lock( + worker_ref, + mode, + RestoreRecoveryAction::Continue, + ) + } + + fn restore_admitted_with_recovery_under_lock( + &self, + worker_ref: &WorkerRef, + mode: WorkerRestoreMode, + recovery: RestoreRecoveryAction, ) -> Result { self.lock()? .ensure_subject_not_reserved(worker_ref, false)?; @@ -2407,14 +2476,28 @@ impl Runtime { } }; if let Some(detail) = committed_detail { - // Recover a receipt gap without launching execution again. + // A historical binding is not current readiness. The exact + // execution must still attest its operation and endpoint. + let backend = self.lock()?.execution_backend.clone(); + let readiness = backend + .ok_or_else(|| "Runtime has no Worker execution backend".to_owned()) + .and_then(|backend| { + backend.activate_restored_worker(owner.operation_id, worker_ref) + }); + if let Err(message) = readiness { + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_saved_binding_not_ready", + sanitize_worker_create_failure_message(&message), + )); + } let result = RuntimeWorkerRestoreResult::accepted(detail); self.finish_restore_receipt(worker_ref, &result)?; return Ok(result); } } } - let result = self.restore_worker_under_lock(worker_ref, mode)?; + let result = self.restore_worker_with_recovery_under_lock(worker_ref, mode, recovery)?; self.finish_restore_receipt(worker_ref, &result)?; Ok(result) } @@ -2451,6 +2534,19 @@ impl Runtime { &self, worker_ref: &WorkerRef, mode: WorkerRestoreMode, + ) -> Result { + self.restore_worker_with_recovery_under_lock( + worker_ref, + mode, + RestoreRecoveryAction::Continue, + ) + } + + fn restore_worker_with_recovery_under_lock( + &self, + worker_ref: &WorkerRef, + mode: WorkerRestoreMode, + recovery: RestoreRecoveryAction, ) -> Result { self.lock()? .ensure_subject_not_reserved(worker_ref, false)?; @@ -2562,8 +2658,12 @@ impl Runtime { .cloned() }); let request = WorkerExecutionRestoreRequest { - source_repair_possible: pending_restore - .is_some_and(|pending| pending.source_repair_possible), + // A committed decision also permits canonical activation effects. + // Cold reconstruction must retain that possibility through even a + // read-only failure, not cache it as an uncommitted rollback. + source_repair_possible: pending_restore.is_some_and(|pending| { + pending.source_repair_possible || pending.commit == RestoreCommitFact::Committed + }), operation_id, worker_ref: worker_ref.clone(), request: worker_request, @@ -2585,6 +2685,27 @@ impl Runtime { ) }; + let uncommitted = self + .lock()? + .worker(worker_ref)? + .pending_restore + .is_some_and(|pending| { + pending.operation_id == operation_id + && pending.commit == RestoreCommitFact::NotCommitted + }); + let reclaim = uncommitted + && (recovery == RestoreRecoveryAction::ReclaimUncommitted + || (reconcile + && match backend.restore_recovery_disposition(operation_id, worker_ref) { + Some(crate::execution::RestoreRecoveryDisposition::Reclaim) | None => true, + Some(crate::execution::RestoreRecoveryDisposition::Absent) => self + .lock()? + .worker(worker_ref)? + .restore_candidate_context + .is_none(), + Some(_) => false, + })); + let reconcile = reconcile || reclaim; if !reconcile && let Err(result) = backend.preflight_restore(&request) { // A preflight may attest an already-connected Controller snapshot. // Repeated restore is read-only; do not replace its operation owner. @@ -2643,6 +2764,8 @@ impl Runtime { let has_completed_failure = completed_failure.is_some(); let restore_result = if let Some(result) = completed_failure { result + } else if reclaim { + backend.reclaim_uncommitted_restore(request) } else if reconcile { backend.reconcile_restore(request) } else { @@ -4205,6 +4328,7 @@ impl Runtime { .lock() .map_err(|_| RuntimeError::StatePoisoned)?; if let Err(error) = self.reconcile_pending_stop_under_lock(&worker_ref) { + self.ensure_restore_continuation(&worker_ref)?; tracing::warn!( worker_id = %worker_ref.worker_id, error = %error, @@ -4242,8 +4366,34 @@ impl Runtime { let _operation_guard = operation_lock .lock() .map_err(|_| RuntimeError::StatePoisoned)?; - let result = self.restore_admitted_worker_under_lock(&worker_ref, mode)?; - self.record_startup_restore_result(&worker_ref, &result)?; + let recovery = { + let mut state = self.lock()?; + state.refresh_uncertain_worker_write(&worker_ref)?; + let pending = state.worker(&worker_ref)?.pending_restore; + let retained = pending.is_some_and(|pending| { + state.execution_backend.as_ref().is_some_and(|backend| { + matches!( + backend.restore_recovery_disposition(pending.operation_id, &worker_ref), + Some( + crate::execution::RestoreRecoveryDisposition::Continue + | crate::execution::RestoreRecoveryDisposition::InFlight + | crate::execution::RestoreRecoveryDisposition::Discard + ) + ) + }) + }); + if pending.is_some_and(|pending| pending.commit == RestoreCommitFact::NotCommitted) + && !retained + { + RestoreRecoveryAction::ReclaimUncommitted + } else { + RestoreRecoveryAction::Continue + } + }; + let result = + self.restore_admitted_with_recovery_under_lock(&worker_ref, mode, recovery); + self.ensure_restore_continuation(&worker_ref)?; + self.record_startup_restore_result(&worker_ref, &result?)?; } for worker_ref in candidates { @@ -4252,8 +4402,9 @@ impl Runtime { .lock() .map_err(|_| RuntimeError::StatePoisoned)?; let result = - self.restore_admitted_worker_under_lock(&worker_ref, WorkerRestoreMode::Automatic)?; - self.record_startup_restore_result(&worker_ref, &result)?; + self.restore_admitted_worker_under_lock(&worker_ref, WorkerRestoreMode::Automatic); + self.ensure_restore_continuation(&worker_ref)?; + self.record_startup_restore_result(&worker_ref, &result?)?; } Ok(()) } @@ -8411,6 +8562,7 @@ mod tests { restore_gate: Mutex>>, restore_count: Mutex, restore_operation_ids: Mutex>, + reclaim_operation_ids: Mutex>, activation_result: Mutex>>, activation_gate: Mutex>>, activation_operation_ids: Mutex>, @@ -8708,6 +8860,57 @@ mod tests { } } + fn restore_recovery_disposition( + &self, + operation_id: WorkerLifecycleOperationId, + _worker_ref: &WorkerRef, + ) -> Option { + Some( + if self + .restore_operation_ids + .lock() + .unwrap() + .contains(&operation_id) + { + crate::execution::RestoreRecoveryDisposition::Continue + } else { + crate::execution::RestoreRecoveryDisposition::Absent + }, + ) + } + + fn reclaim_uncommitted_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + self.reclaim_operation_ids + .lock() + .unwrap() + .push(request.operation_id); + assert!( + !self + .contexts + .lock() + .unwrap() + .contains_key(&request.worker_ref.worker_id) + ); + if request.source_repair_possible { + WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "prior repair may remain", + ), + worker_state: None, + workdir_attachments: Vec::new(), + } + } else { + WorkerExecutionSpawnResult::RolledBack(WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "cold request resources confirmed absent", + )) + } + } + fn reconcile_restore( &self, request: WorkerExecutionRestoreRequest, @@ -9242,7 +9445,9 @@ mod tests { Some(&owner), ) .unwrap(); - assert_eq!(result.unwrap().state, WorkerRestoreState::Accepted); + assert_eq!(result.unwrap().state, WorkerRestoreState::RolledBack); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + assert_eq!(backend.reclaim_operation_ids.lock().unwrap().len(), 1); assert!(preparation.is_none()); assert!( runtime @@ -11212,7 +11417,7 @@ mod tests { #[cfg(feature = "fs-store")] #[test] - fn restore_guard_interrupted_preparation_replays_original_pending_operation() { + fn restore_guard_interrupted_preparation_reclaims_original_pending_operation() { let dir = tempfile::tempdir().unwrap(); let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) .with_runtime_id("guard-partial-preparation"); @@ -11280,10 +11485,10 @@ mod tests { let receipt = resumed .restore_worker_operation(&worker.worker_ref, request) .unwrap(); - assert_eq!(receipt.state, WorkerRestoreState::Accepted); - assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!(receipt.state, WorkerRestoreState::RolledBack); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); assert_eq!( - backend.restore_operation_ids.lock().unwrap().as_slice(), + backend.reclaim_operation_ids.lock().unwrap().as_slice(), &[operation_id] ); assert!( @@ -11422,7 +11627,7 @@ mod tests { #[cfg(feature = "fs-store")] #[test] - fn restore_guard_unknown_preparation_write_retries_same_pending_operation() { + fn restore_guard_unknown_preparation_write_reclaims_same_pending_operation() { let dir = tempfile::tempdir().unwrap(); let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) .with_runtime_id("guard-unknown-preparation"); @@ -11484,12 +11689,16 @@ mod tests { .restore_worker_operation(&worker.worker_ref, request) .unwrap() .state, - WorkerRestoreState::Accepted + WorkerRestoreState::RolledBack + ); + assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 0); + assert_eq!( + resumed_backend.reclaim_operation_ids.lock().unwrap().len(), + 1 ); - assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 1); assert_eq!( resumed.worker_detail(&worker.worker_ref).unwrap().status, - WorkerStatus::Idle + WorkerStatus::Stopped ); } @@ -11667,7 +11876,7 @@ mod tests { Runtime::with_fs_store_and_execution_backend(options, resumed_backend.clone()).unwrap(); assert_eq!( resumed_backend - .restore_operation_ids + .reclaim_operation_ids .lock() .unwrap() .as_slice(), @@ -11676,8 +11885,8 @@ mod tests { let receipt = resumed .restore_worker_operation(&worker.worker_ref, request) .unwrap(); - assert_eq!(receipt.state, WorkerRestoreState::Accepted); - assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 1); + assert_eq!(receipt.state, WorkerRestoreState::RolledBack); + assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 0); } #[test] @@ -15521,6 +15730,21 @@ mod tests { restarted_backend.clone(), ) .unwrap(); + assert_eq!( + restarted_backend + .reclaim_operation_ids + .lock() + .unwrap() + .as_slice(), + &[operation_id] + ); + assert_eq!(*restarted_backend.restore_count.lock().unwrap(), 0); + assert_eq!( + restarted.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + // A deliberate fresh intent may restore after the old uncommitted + // request was reclaimed. It must allocate a different operation. let retry = restarted .restore_worker_operation( &worker.worker_ref, @@ -15529,13 +15753,9 @@ mod tests { .unwrap(); assert_eq!(retry.state, WorkerRestoreState::Accepted); assert_eq!(*restarted_backend.restore_count.lock().unwrap(), 1); - assert_eq!( - restarted_backend - .restore_operation_ids - .lock() - .unwrap() - .as_slice(), - &[operation_id] + assert_ne!( + restarted_backend.restore_operation_ids.lock().unwrap()[0], + operation_id ); assert!( restarted diff --git a/crates/worker-runtime/src/runtime/restore_continuation.rs b/crates/worker-runtime/src/runtime/restore_continuation.rs index 3de663219..ad50b4d2d 100644 --- a/crates/worker-runtime/src/runtime/restore_continuation.rs +++ b/crates/worker-runtime/src/runtime/restore_continuation.rs @@ -266,9 +266,10 @@ impl Runtime { let Some(backend) = &state.execution_backend else { return Ok(()); }; - if backend - .restore_recovery_disposition(operation, &worker_ref) - .is_none() + if !backend.supports_restore_continuation() + || backend + .restore_recovery_disposition(operation, &worker_ref) + .is_none() { return Ok(()); } @@ -396,6 +397,10 @@ impl Runtime { "worker_restore_rolled_back", result.message_or_default(), ); + #[cfg(feature = "fs-store")] + self.lock()? + .record_restore_failure(worker_ref, &result, &receipt)?; + #[cfg(not(feature = "fs-store"))] self.settle_restore_rollback(worker_ref, &receipt)?; return Ok(Step::Finished); } diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 3d95f99ff..7e1288701 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -2455,25 +2455,27 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .await .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; } - if let Some(resources) = retained { - if let Some(allocation) = resources.allocation { - allocation - .release() - .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; - } else { - // Reconstruction may fail before returning its allocation guard. - // Absence is checked at the allocation authority, not inferred from Err. - let path = worker::runtime::worker_allocation::default_allocation_path() - .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; - let table = worker::runtime::worker_allocation::LockFileGuard::open(&path) - .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; - if table - .data() - .find(&Self::runtime_worker_name_for_ref(&request.worker_ref)) - .is_some() - { - return Err(RuntimeRestoreCleanupFailure::Allocation); - } + if let Some(allocation) = retained + .as_ref() + .and_then(|resources| resources.allocation.as_ref()) + { + allocation + .release() + .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; + } else { + // A cold restart loses the retained map as well as allocation guards. + // Prove absence at the allocation authority even with an empty map; + // never release a possibly different owner by Worker name. + let path = worker::runtime::worker_allocation::default_allocation_path() + .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; + let table = worker::runtime::worker_allocation::LockFileGuard::open(&path) + .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; + if table + .data() + .find(&Self::runtime_worker_name_for_ref(&request.worker_ref)) + .is_some() + { + return Err(RuntimeRestoreCleanupFailure::Allocation); } } // restore_controller starts the Controller only as its final fallible @@ -2486,10 +2488,18 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .map_err(|_| RuntimeRestoreCleanupFailure::Artifacts)?; Self::remove_restore_run_artifacts(&run_dir) .map_err(|_| RuntimeRestoreCleanupFailure::Artifacts)?; - self.failed_restore_resources + let mut resources = self + .failed_restore_resources .lock() - .map_err(|_| RuntimeRestoreCleanupFailure::Ownership)? - .remove(&request.worker_ref); + .map_err(|_| RuntimeRestoreCleanupFailure::Ownership)?; + if resources + .get(&request.worker_ref) + .is_some_and(|owner| owner.operation_id != request.operation_id) + { + return Err(RuntimeRestoreCleanupFailure::Ownership); + } + resources.remove(&request.worker_ref); + drop(resources); self.observation_hub.get(&request.worker_ref); Ok(()) } @@ -2763,6 +2773,29 @@ impl ProfileRuntimeWorkerFactory { let run_dir = self .worker_restore_run_dir(&request.worker_ref, request.operation_id) .map_err(|_| "restore run directory unavailable")?; + // A process restart loses the candidate, not its exact operation's + // disposable directory. Never infer writer absence from empty maps. + match std::fs::symlink_metadata(&run_dir) { + Ok(_) => { + self.stopped_restore_run_dirs(&request.worker_ref) + .map_err(|_| "restore run artifacts cannot be validated")?; + let path = worker::runtime::worker_allocation::default_allocation_path() + .map_err(|_| "restore allocation authority unavailable")?; + let table = worker::runtime::worker_allocation::LockFileGuard::open(&path) + .map_err(|_| "restore allocation authority unavailable")?; + if table + .data() + .find(&Self::runtime_worker_name_for_ref(&request.worker_ref)) + .is_some() + { + return Err("restore run directory still has an allocation owner"); + } + Self::remove_restore_run_artifacts(&run_dir) + .map_err(|_| "restore run artifacts cannot be reclaimed")?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Err("restore run artifacts cannot be inspected"), + } let transport = self.controller_transport; #[cfg(test)] let transport = if self.restore_faults.socket_failure.load(Ordering::SeqCst) { @@ -3138,6 +3171,9 @@ struct RuntimeWorkerExecution { // marker. Timeout never drops the factory future or a late Controller result. struct PendingRuntimeRestore { request: WorkerExecutionRestoreRequest, + // Transient ownership mode, not a durable phase. Cold restart reclamation + // must never enter preparation, canonical repair, or activation. + cleanup_only: bool, task: tokio::sync::Mutex>>, task_in_flight: AtomicBool, result: Mutex, RuntimeRestoreFailure>>>, @@ -3469,6 +3505,7 @@ where pending: &Arc, resume_activation: bool, ) -> Result<(), String> { + let resume_activation = resume_activation && !pending.cleanup_only; let failure = pending .result .lock() @@ -5317,6 +5354,16 @@ where } }; if let Some(pending) = admitted_pending.as_ref() { + if pending.cleanup_only { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "cleanup-only restore cannot prepare or reconstruct execution", + ), + worker_state: None, + workdir_attachments: pending.request.previous_workdir_attachments.clone(), + }; + } if request.request != pending.request.request || request.workspace_scope != pending.request.workspace_scope || request.logical_workdir_attachments @@ -5378,6 +5425,7 @@ where .remove(&worker_ref); let pending = Arc::new(PendingRuntimeRestore { request: request.clone(), + cleanup_only: false, task: tokio::sync::Mutex::new(None), task_in_flight: AtomicBool::new(false), result: Mutex::new(None), @@ -5593,17 +5641,23 @@ where } } + fn supports_restore_continuation(&self) -> bool { + true + } + fn restore_recovery_disposition( &self, operation_id: crate::execution::WorkerLifecycleOperationId, worker_ref: &WorkerRef, ) -> Option { use crate::execution::RestoreRecoveryDisposition as D; - let pending = self.pending_restore(worker_ref).ok().flatten(); + let pending = self.pending_restore(worker_ref).ok()?; Some(match pending { Some(pending) if pending.request.operation_id == operation_id => { if pending.task_in_flight.load(Ordering::Acquire) { D::InFlight + } else if pending.cleanup_only { + D::Reclaim } else if pending .discard_result .lock() @@ -5625,10 +5679,132 @@ where D::Continue } } - _ => D::Continue, + _ => { + let workers = self.workers.lock().ok()?; + if workers + .get(worker_ref) + .is_some_and(|execution| execution.restore_operation_id == Some(operation_id)) + { + D::Continue + } else { + D::Absent + } + } }) } + fn reclaim_uncommitted_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + let reclaimed = (|| -> Result { + let worker_ref = &request.worker_ref; + let operation_id = request.operation_id; + let operation_lock = self.worker_lock(worker_ref)?; + let _guard = operation_lock + .lock() + .map_err(|_| "worker operation lock is poisoned")?; + if request.context.worker_ref() != worker_ref { + return Err("restore cleanup context does not match Worker authority".into()); + } + if self + .workers + .lock() + .map_err(|_| "worker registry lock is poisoned")? + .contains_key(worker_ref) + { + return Err("uncommitted reclaim cannot target a connected execution".into()); + } + let pending = match self.pending_restore(worker_ref)? { + Some(pending) => { + if pending.request.operation_id != operation_id || !pending.cleanup_only { + return Err( + "uncommitted reclaim cannot replace an in-memory restore owner".into(), + ); + } + if request.request != pending.request.request + || request.workspace_scope != pending.request.workspace_scope + || request.logical_workdir_attachments + != pending.request.logical_workdir_attachments + || request.previous_workdir_attachments + != pending.request.previous_workdir_attachments + || request.source_repair_possible != pending.request.source_repair_possible + { + return Err( + "current Worker authority differs from the owned restore cleanup" + .into(), + ); + } + pending + } + None => { + if let Some(result) = self.completed_restore_failure(operation_id, worker_ref) { + return Ok(result); + } + let mut failure = RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "restore decision was not committed before restart", + ); + failure.durable_effects_possible = request.source_repair_possible; + let pending = Arc::new(PendingRuntimeRestore { + request: request.clone(), + cleanup_only: true, + task: tokio::sync::Mutex::new(None), + task_in_flight: AtomicBool::new(false), + result: Mutex::new(Some(Err(failure))), + activated: Mutex::new(None), + activation_started: AtomicBool::new(false), + discard_started: AtomicBool::new(false), + discard_result: Mutex::new(None), + failure: Mutex::new(None), + }); + self.pending_restores + .lock() + .map_err(|_| "pending restore resources lock is poisoned")? + .insert(worker_ref.clone(), pending.clone()); + pending + } + }; + // Join the prior task before retrying. A timeout retains this owner, + // its original context notifier, and its exact task/JoinHandle. + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + self.retry_pending_restore_cleanup(&pending, false)?; + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + let failure = pending + .result + .lock() + .map_err(|_| "restore result lock is poisoned")? + .as_ref() + .and_then(|result| result.as_ref().err()) + .cloned() + .ok_or("restore cleanup has no retained failure result")?; + if failure.needs_cleanup() { + return Err(failure.to_string()); + } + if failure.durable_effects_possible { + return Err("Restore resource cleanup is confirmed; possible earlier canonical effects still require reconciliation".into()); + } + self.completed_restore_failures + .lock() + .map_err(|_| "restore failure receipts lock is poisoned")? + .insert(worker_ref.clone(), (operation_id, failure.clone())); + self.remove_pending_restore(worker_ref)?; + Ok(WorkerExecutionSpawnResult::RolledBack( + WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + failure.to_string(), + ), + )) + })(); + reclaimed.unwrap_or_else( + |message| WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message), + worker_state: None, + workdir_attachments: request.previous_workdir_attachments.clone(), + }, + ) + } + fn discard_uncommitted_restore( &self, operation_id: crate::execution::WorkerLifecycleOperationId, @@ -5643,6 +5819,7 @@ where .pending_restore(worker_ref)? .ok_or("unpublished restore owner is unavailable")?; if pending.request.operation_id != operation_id + || pending.cleanup_only || pending.activation_started.load(Ordering::Acquire) { return Err( @@ -5784,6 +5961,9 @@ where if pending.request.operation_id != operation_id { return Err("restored Worker operation identity does not match".into()); } + if pending.cleanup_only { + return Err("cleanup-only restore cannot activate execution".into()); + } self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; if pending.owns_prepared_candidate() { pending.activation_started.store(true, Ordering::Release); @@ -7696,6 +7876,7 @@ mod tests { worker_ref.clone(), Arc::new(PendingRuntimeRestore { request: direct_restore_request(worker_ref.clone()), + cleanup_only: false, task: tokio::sync::Mutex::new(None), task_in_flight: AtomicBool::new(false), result: Mutex::new(None), @@ -7740,6 +7921,7 @@ mod tests { .unwrap(); let pending = Arc::new(PendingRuntimeRestore { request: request.clone(), + cleanup_only: false, task: tokio::sync::Mutex::new(Some(failed)), task_in_flight: AtomicBool::new(false), result: Mutex::new(None), @@ -10255,6 +10437,309 @@ mod tests { } } + // Backend component seam only: no saved Worker, model, or Controller is + // constructed. Any accidental preparation/recovery is an immediate failure. + struct CleanupOnlyFactory { + cleanup_calls: Arc, + cleanup_allowed: Arc, + release: Option>, + } + + #[async_trait] + impl RuntimeWorkerFactory for CleanupOnlyFactory { + async fn spawn_controller( + &self, + _: WorkerExecutionSpawnRequest, + ) -> Result { + panic!("reclaim must not spawn") + } + + async fn preflight_restore(&self, _: &WorkerExecutionRestoreRequest) -> Result<(), String> { + panic!("reclaim must not preflight preparation") + } + + async fn recover_restore_source( + &self, + _: &WorkerExecutionRestoreRequest, + ) -> Result { + panic!("reclaim must not repair canonical source") + } + + async fn prepare_restore( + &self, + _: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + panic!("reclaim must not prepare") + } + + async fn cleanup_failed_restore( + &self, + _: &WorkerExecutionRestoreRequest, + ) -> Result<(), RuntimeRestoreCleanupFailure> { + if self.cleanup_calls.fetch_add(1, Ordering::SeqCst) == 0 { + if let Some(release) = &self.release { + release.notified().await; + } + } + if self.cleanup_allowed.load(Ordering::SeqCst) { + Ok(()) + } else { + Err(RuntimeRestoreCleanupFailure::Workdir) + } + } + + async fn reconcile_stopped_worker(&self, _: &WorkerRef) -> Result<(), String> { + panic!("reclaim must not clean all Worker operations through Stop") + } + } + + #[test] + fn cold_reclaim_retries_cleanup_only_and_keeps_cleaned_receipt_idempotent() { + use crate::execution::RestoreRecoveryDisposition as D; + let cleanup_calls = Arc::new(AtomicUsize::new(0)); + let cleanup_allowed = Arc::new(AtomicBool::new(false)); + let backend = WorkerRuntimeExecutionBackend::new(CleanupOnlyFactory { + cleanup_calls: cleanup_calls.clone(), + cleanup_allowed: cleanup_allowed.clone(), + release: None, + }) + .unwrap(); + let request = direct_restore_request(WorkerRef::new(WorkerId::now_v7())); + let worker_ref = &request.worker_ref; + assert_eq!( + backend.restore_recovery_disposition(request.operation_id, worker_ref), + Some(D::Absent) + ); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + let owner = backend.pending_restore(worker_ref).unwrap().unwrap(); + assert!(owner.cleanup_only); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 1); + assert_eq!( + backend.restore_recovery_disposition(request.operation_id, worker_ref), + Some(D::Reclaim) + ); + // Wrong operation and changed authority must not replace or retry it. + let mut different = request.clone(); + different.operation_id = crate::execution::WorkerLifecycleOperationId::new(); + assert!(matches!( + backend.reclaim_uncommitted_restore(different), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + let mut mismatched = request.clone(); + mismatched.request = create_request("changed cleanup authority"); + assert!(matches!( + backend.reclaim_uncommitted_restore(mismatched), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert!(Arc::ptr_eq( + &owner, + &backend.pending_restore(worker_ref).unwrap().unwrap() + )); + // Erroneous generic callers cannot reconstruct, activate, or substitute + // in-process candidate discard for cold cleanup. + assert!(matches!( + backend.restore_worker(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert!( + backend + .activate_restored_worker(request.operation_id, worker_ref) + .is_err() + ); + assert!(matches!( + backend.discard_uncommitted_restore(request.operation_id, worker_ref), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 1); + cleanup_allowed.store(true, Ordering::SeqCst); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::RolledBack(_) + )); + assert!(backend.pending_restore(worker_ref).unwrap().is_none()); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::RolledBack(_) + )); + assert!(matches!( + backend.restore_worker(request), + WorkerExecutionSpawnResult::RolledBack(_) + )); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 2); + assert!(backend.workers.lock().unwrap().is_empty()); + } + + #[test] + fn cold_reclaim_preserves_possible_effects_without_repreparing() { + use crate::execution::RestoreRecoveryDisposition as D; + let cleanup_calls = Arc::new(AtomicUsize::new(0)); + let backend = WorkerRuntimeExecutionBackend::new(CleanupOnlyFactory { + cleanup_calls: cleanup_calls.clone(), + cleanup_allowed: Arc::new(AtomicBool::new(true)), + release: None, + }) + .unwrap(); + let mut request = direct_restore_request(WorkerRef::new(WorkerId::now_v7())); + request.source_repair_possible = true; + for _ in 0..2 { + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + } + let owner = backend + .pending_restore(&request.worker_ref) + .unwrap() + .unwrap(); + assert!(owner.cleanup_only); + let retained = owner.result.lock().unwrap(); + let failure = retained.as_ref().unwrap().as_ref().err().unwrap(); + assert!(failure.durable_effects_possible); + assert_eq!(failure.cleanup, RuntimeRestoreCleanup::Confirmed); + drop(retained); + // Even an internal retry with the wrong resume flag cannot prepare. + backend.retry_pending_restore_cleanup(&owner, true).unwrap(); + assert!(owner.task.try_lock().unwrap().is_none()); + assert!( + backend + .completed_restore_failure(request.operation_id, &request.worker_ref) + .is_none() + ); + assert_eq!( + backend.restore_recovery_disposition(request.operation_id, &request.worker_ref), + Some(D::Reclaim) + ); + assert!(matches!( + backend.reconcile_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert!( + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .is_err() + ); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 1); + } + + #[test] + fn cold_reclaim_timeout_retains_same_task_and_notifies_before_cleanup_retry() { + use crate::execution::RestoreRecoveryDisposition as D; + let cleanup_calls = Arc::new(AtomicUsize::new(0)); + let cleanup_allowed = Arc::new(AtomicBool::new(false)); + let release = Arc::new(tokio::sync::Notify::new()); + let mut backend = WorkerRuntimeExecutionBackend::new(CleanupOnlyFactory { + cleanup_calls: cleanup_calls.clone(), + cleanup_allowed: cleanup_allowed.clone(), + release: Some(release.clone()), + }) + .unwrap() + .with_spawn_restore_timeout(Duration::from_millis(1)); + let mut request = direct_restore_request(WorkerRef::new(WorkerId::now_v7())); + let (progress, progressed) = mpsc::channel(); + request.context = request + .context + .with_restore_progress(Arc::new(move |operation| { + let _ = progress.send(operation); + })); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + let owner = backend + .pending_restore(&request.worker_ref) + .unwrap() + .unwrap(); + let task_id = owner.task.try_lock().unwrap().as_ref().unwrap().id(); + assert_eq!( + backend.restore_recovery_disposition(request.operation_id, &request.worker_ref), + Some(D::InFlight) + ); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert!(Arc::ptr_eq( + &owner, + &backend + .pending_restore(&request.worker_ref) + .unwrap() + .unwrap() + )); + assert_eq!( + owner.task.try_lock().unwrap().as_ref().unwrap().id(), + task_id + ); + release.notify_one(); + assert_eq!( + progressed.recv_timeout(Duration::from_secs(5)).unwrap(), + request.operation_id + ); + backend + .wait_pending_restore(&owner, Duration::from_secs(5)) + .unwrap(); + assert!(owner.task.try_lock().unwrap().is_none()); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 1); + assert_eq!( + backend.restore_recovery_disposition(request.operation_id, &request.worker_ref), + Some(D::Reclaim) + ); + backend.spawn_restore_timeout = Duration::from_secs(5); + cleanup_allowed.store(true, Ordering::SeqCst); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::RolledBack(_) + )); + assert_eq!( + progressed.recv_timeout(Duration::from_secs(5)).unwrap(), + request.operation_id + ); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 2); + assert!(backend.workers.lock().unwrap().is_empty()); + } + + #[test] + fn cold_reclaim_refuses_existing_candidate_owner_even_for_same_operation() { + let backend = WorkerRuntimeExecutionBackend::new(CleanupOnlyFactory { + cleanup_calls: Arc::new(AtomicUsize::new(0)), + cleanup_allowed: Arc::new(AtomicBool::new(true)), + release: None, + }) + .unwrap(); + let request = direct_restore_request(WorkerRef::new(WorkerId::now_v7())); + let owner = Arc::new(PendingRuntimeRestore { + request: request.clone(), + cleanup_only: false, + task: tokio::sync::Mutex::new(None), + task_in_flight: AtomicBool::new(false), + result: Mutex::new(None), + activated: Mutex::new(None), + activation_started: AtomicBool::new(false), + discard_started: AtomicBool::new(false), + discard_result: Mutex::new(None), + failure: Mutex::new(None), + }); + backend + .pending_restores + .lock() + .unwrap() + .insert(request.worker_ref.clone(), owner.clone()); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert!(Arc::ptr_eq( + &owner, + &backend + .pending_restore(&request.worker_ref) + .unwrap() + .unwrap() + )); + assert_eq!(backend.factory.cleanup_calls.load(Ordering::SeqCst), 0); + } + #[test] fn absent_execution_stop_reconciles_pending_restore_cleanup_without_launching() { let restore_calls = Arc::new(AtomicUsize::new(0)); diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs index d622a7462..99823169c 100644 --- a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -193,6 +193,18 @@ max_tokens = 100 self.inner.restore_worker(request) } + fn restore_recovery_disposition( + &self, + operation_id: crate::execution::WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Option { + self.inner.restore_recovery_disposition(operation_id, worker_ref) + } + + fn reclaim_uncommitted_restore(&self, request: WorkerExecutionRestoreRequest) -> WorkerExecutionSpawnResult { + self.inner.reclaim_uncommitted_restore(request) + } + fn reconcile_restore( &self, request: WorkerExecutionRestoreRequest, diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs index 28323570f..d7518319b 100644 --- a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -94,6 +94,133 @@ mod publication { (release, received) } + #[test] + #[serial_test::serial(worker_allocation)] + fn startup_fault_recovery_retains_cleanup_or_committed_reconstruction_without_repost() { + use crate::runtime::RestoreCommitFact as F; + for committed in [false, true] { + let fixture = Fixture::no_model(); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let request = runtime.test_restore_request(&worker_ref); + runtime.coordinate_worker_restore_operation(&worker_ref, runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), preparation: request.preparation.clone(), + }, fixture.request.workspace_scope.as_ref()).unwrap(); + let operation = runtime.test_stage_restore_checkpoint(&worker_ref, if committed { F::Committed } else { F::NotCommitted }, false); + let before = fixture.canonical_files(); + drop(runtime); + if committed { + *fixture.backend.factory.restore_faults.fail_at.lock().unwrap() = Some(RuntimeRestoreStage::Metadata); + } else { + fixture.backend.factory.restore_faults.cleanup_fail.store(true, Ordering::SeqCst); + } + let options = FsRuntimeStoreOptions::new(fixture._root.path().join("catalog")).with_runtime_id("candidate-runtime"); + let restarted = EmbeddedRuntime::with_fs_store_and_execution_backend(options, fixture.backend.clone()).unwrap(); + let pause = restarted.test_pause_restore_continuation(); + assert_eq!(durable_operation(&restarted, &worker_ref).unwrap().operation_id, operation); + assert!(fixture.backend.completed_restore_failure(operation, &worker_ref).is_none()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + *fixture.backend.factory.restore_faults.fail_at.lock().unwrap() = None; + fixture.backend.factory.restore_faults.cleanup_fail.store(false, Ordering::SeqCst); + drop(pause); + restarted.test_wait_restore_continuation(&worker_ref, operation, WAIT); + let receipt = restarted.restore_worker_operation(&worker_ref, request).unwrap(); + assert_eq!(receipt.state, if committed { server_api::WorkerRestoreState::Accepted } else { server_api::WorkerRestoreState::RolledBack }); + if committed { + let transport = fixture.backend.clone().attach_worker_protocol(&worker_ref).unwrap(); + transport.validate().unwrap(); + drop(transport); + restarted.stop_worker(&worker_ref, None).unwrap(); + } else { + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 0); + assert_eq!(fixture.canonical_files(), before); + } + fixture.assert_released(); + fixture.assert_quiet(); + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn startup_dispatches_saved_decisions_without_replaying_uncommitted_execution() { + use crate::runtime::RestoreCommitFact as F; + for (commit, repaired, expected) in [ + (F::NotCommitted, false, server_api::WorkerRestoreState::RolledBack), + (F::NotCommitted, true, server_api::WorkerRestoreState::ReconciliationRequired), + (F::Committed, false, server_api::WorkerRestoreState::Accepted), + (F::LegacyUnknown, true, server_api::WorkerRestoreState::ReconciliationRequired), + ] { + let fixture = Fixture::no_model(); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + assert!(runtime.coordinate_worker_restore_operation(&worker_ref, runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: request.preparation.clone(), + }, Some(scope)).unwrap().0.is_none()); + let operation = runtime.test_stage_restore_checkpoint(&worker_ref, commit, repaired); + let before = fixture.canonical_files(); + let run = fixture.backend.factory.worker_restore_run_dir(&worker_ref, operation).unwrap(); + std::fs::create_dir_all(&run).unwrap(); + std::fs::write(run.join("before-restart.marker"), "disposable").unwrap(); + if commit == F::NotCommitted && !repaired { + // Fixture for an interrupted automatic restore. A rollback must + // not leave an active/Automatic projection that restores again. + let store = runtime.test_runtime_store(); + let mut record = store.read_worker_record(&worker_ref).unwrap(); + record.status = crate::catalog::WorkerStatus::Idle; + let PersistedWorkerExecutionState::ReconciliationRequired( + crate::fs_store::PersistedWorkerLifecycleOperation::Restore(saved), + ) = &mut record.execution_state else { panic!("pending fixture") }; + saved.mode = crate::fs_store::PersistedWorkerRestoreMode::Automatic; + saved.last_settled_status = crate::catalog::WorkerStatus::Idle; + saved.binding = Some(crate::fs_store::PersistedWorkerExecutionBinding {}); + store.write_worker_record(&record).unwrap(); + } + drop(runtime); + assert!(fixture.backend.pending_restore(&worker_ref).unwrap().is_none()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + let options = FsRuntimeStoreOptions::new(fixture._root.path().join("catalog")).with_runtime_id("candidate-runtime"); + let restarted = EmbeddedRuntime::with_fs_store_and_execution_backend(options.clone(), fixture.backend.clone()).unwrap(); + if expected == server_api::WorkerRestoreState::ReconciliationRequired { + let reason = restarted.test_wait_restore_continuation_parked(&worker_ref, operation, WAIT); + assert!(!reason.is_empty()); + let pending = durable_operation(&restarted, &worker_ref).unwrap(); + assert_eq!(pending.operation_id, operation); + assert_eq!(pending.commit, commit); + assert_eq!(pending.source_repair_possible, repaired); + } + assert!(restarted.worker_detail(&worker_ref).is_ok(), "{commit:?}: {:?}", restarted.diagnostics()); + let receipt = restarted.restore_worker_operation(&worker_ref, request.clone()).unwrap(); + assert_eq!(receipt.state, expected, "{commit:?}: {receipt:?}; backend failure: {:?}", fixture.backend.completed_restore_failure(operation, &worker_ref)); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), usize::from(commit == F::Committed)); + if commit == F::Committed { + let transport = fixture.backend.clone().attach_worker_protocol(&worker_ref).unwrap(); + transport.validate().unwrap(); + drop(transport); + } else { + assert_eq!(fixture.canonical_files(), before); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!(fixture.backend.factory.observation_hub.get(&worker_ref).is_none()); + assert_eq!(run.exists(), commit == F::LegacyUnknown); + } + if expected == server_api::WorkerRestoreState::RolledBack { + assert_eq!(restarted.worker_detail(&worker_ref).unwrap().status, crate::catalog::WorkerStatus::Stopped); + drop(restarted); + let second = EmbeddedRuntime::with_fs_store_and_execution_backend(options, fixture.backend.clone()).unwrap(); + assert_eq!(second.restore_worker_operation_scoped(scope, &worker_ref, request).unwrap(), receipt); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 0); + } else { + restarted.stop_worker_scoped(scope, &worker_ref, None).unwrap(); + } + fixture.assert_released(); + fixture.assert_quiet(); + } + } + #[test] #[serial_test::serial(worker_allocation)] fn stop_timeout_retains_autonomous_late_cleanup_and_blocks_remove_until_confirmed() { From f5ebae1134d7ce4f3f174a4dc68aa397a994b81f Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 12:27:33 +0900 Subject: [PATCH 12/28] test: verify restore response loss and owned runtime restart --- tests/e2e/README.md | 8 +- tests/e2e/src/worker_environment.rs | 84 +++++++- .../src/worker_environment/response_loss.rs | 198 ++++++++++++++++++ tests/e2e/tests/worker_restore.rs | 180 ++++++++++++++++ 4 files changed, 464 insertions(+), 6 deletions(-) create mode 100644 tests/e2e/src/worker_environment/response_loss.rs diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 5766c9471..a54eb8517 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -22,7 +22,7 @@ 所有crateの`test-support`依存はテスト側の非default `fixture-support` featureだけで有効にする。製品binary providerは別Cargo invocationと専用`target/e2e-products`へ`yoi`・`yoi-server`・`yoi-runtime`を追加featureなしでビルドする。support test自身をこのtargetへビルドしてはならない。`YOI_E2E_BIN`等の明示binary overrideを使用する場合も、呼出し側が同じ分離を保証する。既存TUI用`e2e-test` featureは別の`target/e2e-tui-products`へビルドし、restore製品binaryへ混入しない。 -固定資産の正常restore・並行隔離E2E: +固定資産の正常restore・並行隔離・応答喪失回復E2E: ```sh cargo test -p yoi-e2e --features e2e,fixture-support --test worker_restore @@ -33,6 +33,12 @@ YOI_E2E_RESTORE_FIXTURE=target/fixture-candidate-v1 \ 標準資産は`tests/e2e/fixtures/worker-codex-v1`の`manifest.json`・`worker-template.json`・`expected.json`・`workspace-config.dcdl`。manifestの各asset hash/size、schemaと固定ID、生成元commit、modelを検証する。Workspace configのimport参照を黙って最新へ置換せず、Gitで生成元commitに対する`resources`と`crates/builtin-source`の一致を確認する（Git履歴がない場合も失敗）。移植したSessionの正規公開projectionを供給された会話行と照合し、その全snapshotを実executionのWebSocketと比較する。欠落fixture・hash不一致・未実行は成功扱いしない。 +応答喪失ケースは同じ`WorkerE2eEnvironment`内の単発loopback relayで公開restore要求をそのまま転送し、完全なupstream応答を受け取った後に呼出元へ0byteで切断する。呼出元のtransport失敗とrelay側のAccepted観測は別の事実として記録する。回収は元のrequest ID・観測tokenをそのまま使い、fresh intentへの古いtokenはHTTP409を要求する。 + +process終了ケースはそのAccepted receipt確認後に環境が所有するRuntime childだけをkill/reapし、同じroot・Runtime identity・trustから再起動する。seedの再実行・tokenの差替えはしない。再起動前後のcanonical owner全体のhash・operation ID・owner数が一致し、公開APIの保存結果とfull Session snapshotを回収できることを確認する。公開Worker投影の観測tokenは再起動で変わるため、そのfieldだけはcurrent detailとの一致を別に確認する。canonical診断はboundedな読取りのみで、要求・receiptの内容や認証情報はartifactへ保存しない。 + +これは**commit済みかつterminal receipt保存後**のprocess終了であり、未commit候補やcommit直後・receipt前の終了証明ではない。固定Codex資産へSubmit/Notifyを送らないため、通常入力・Paused admissionの製品E2Eの証明でもない。検証jobはこの明示targetを選択する必要があり、通常suiteだけでは代替しない。 + 通常のharness回帰試験（製品processは起動しない）: ```sh diff --git a/tests/e2e/src/worker_environment.rs b/tests/e2e/src/worker_environment.rs index 1fbdc4b0b..749ef4c83 100644 --- a/tests/e2e/src/worker_environment.rs +++ b/tests/e2e/src/worker_environment.rs @@ -1,3 +1,6 @@ +#[path = "worker_environment/response_loss.rs"] +mod response_loss; + use std::fs::{self, File, OpenOptions}; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; @@ -571,12 +574,7 @@ impl WorkerE2eEnvironment { if self.runtime_public_identity.is_none() { self.initialize_runtime_identity(self.runtime_id.clone())?; } - let runtime_identity = self - .runtime_public_identity - .clone() - .expect("initialized identity"); let bundle_path = self.root.join("workspace-public-identity.json"); - let server_url = self.required_server_url()?.to_string(); let workspace_id = self.required_workspace_id()?.to_string(); let runtime_binary = self.binaries.binary("yoi-runtime")?; self.run_command( @@ -616,6 +614,20 @@ impl WorkerE2eEnvironment { seed(self)?; self.write_metadata("runtime-seeded")?; + self.start_runtime_process() + } + + fn start_runtime_process(&mut self) -> Result<()> { + if self.runtime.is_some() { + return Err(HarnessError::Protocol("Runtime is already running".into())); + } + let runtime_binary = self.binaries.binary("yoi-runtime")?; + let runtime_identity = self + .runtime_public_identity + .clone() + .ok_or_else(|| HarnessError::Protocol("Runtime identity is not initialized".into()))?; + let server_url = self.required_server_url()?.to_string(); + let workspace_id = self.required_workspace_id()?.to_string(); let mut command = self.isolated_command(&runtime_binary); command.args([ "--bind", @@ -669,6 +681,36 @@ impl WorkerE2eEnvironment { Ok(()) } + /// Kill/reap only the Runtime child owned by this environment, retaining roots + /// and identities. Restart neither seeds storage nor rotates restore requests. + pub fn crash_and_restart_runtime(&mut self) -> Result<()> { + let runtime = self + .runtime + .as_mut() + .ok_or_else(|| HarnessError::Protocol("Runtime is not running".into()))?; + let pid = runtime.id(); + let status = runtime.kill_and_reap(EXIT_WAIT)?; + runtime.join_readers_until(Instant::now() + EXIT_WAIT)?; + fs::write( + self.artifacts_dir.join("runtime-crash.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "owned_pid": pid, "reaped": true, "exit_status": status.to_string(), + "same_runtime_id": self.runtime_id, + }))?, + )?; + self.runtime.take(); + for stream in ["stdout", "stderr"] { + fs::rename( + self.artifacts_dir.join(format!("runtime.{stream}.log")), + self.artifacts_dir + .join(format!("runtime-before-crash.{stream}.log")), + )?; + } + self.runtime_url = None; + self.runtime_shutdown = WorkerProcessShutdownOutcome::NotStarted; + self.start_runtime_process() + } + pub fn assert_connected(&self) -> Result<()> { let server_url = self.required_server_url()?; let workspace_id = self.required_workspace_id()?; @@ -752,6 +794,38 @@ impl WorkerE2eEnvironment { ) } + /// Forward one unchanged public restore request through an owned loopback + /// relay, consume its complete upstream response, and close without delivering + /// any response bytes. The returned JSON is relay evidence, not caller success. + pub fn restore_worker_losing_response( + &self, + worker_id: &str, + request_id: &str, + expected_observation_token: &str, + ) -> Result { + let token = self.access_token.as_deref().ok_or_else(|| { + HarnessError::Protocol("Backend access token is not initialized".into()) + })?; + let request = self + .client + .post(self.worker_url(worker_id, "restore")?) + .bearer_auth(token) + .header(reqwest::header::CONNECTION, "close") + .timeout(STARTUP_WAIT + EXIT_WAIT) + .json(&serde_json::json!({ + "request_id": request_id, + "expected_observation_token": expected_observation_token, + })) + .build() + .map_err(|_| HarnessError::Protocol("build response-loss request".into()))?; + let evidence = response_loss::drop_response(&self.client, request)?; + self.append_operation( + "restore response consumed by relay, caller transport failed", + 0, + )?; + Ok(evidence) + } + pub fn stop_worker(&self, worker_id: &str) -> Result { self.request_json( self.client diff --git a/tests/e2e/src/worker_environment/response_loss.rs b/tests/e2e/src/worker_environment/response_loss.rs new file mode 100644 index 000000000..ee6654c85 --- /dev/null +++ b/tests/e2e/src/worker_environment/response_loss.rs @@ -0,0 +1,198 @@ +//! A single bounded response-loss injection in the shared product harness. +use std::io::{Read, Write}; +use std::net::{IpAddr, TcpListener, TcpStream}; +use std::time::Instant; + +use reqwest::blocking::{Client, Request}; +use serde_json::Value; + +use super::{EXIT_WAIT, HTTP_WAIT_STEP, STARTUP_WAIT}; +use crate::{HarnessError, Result}; + +const MAX_HEADER: usize = 16 * 1024; +const MAX_MESSAGE: usize = 2 * 1024 * 1024; + +fn failure(message: &str) -> HarnessError { + HarnessError::Protocol(message.into()) +} + +pub(super) fn drop_response(client: &Client, mut request: Request) -> Result { + let host: IpAddr = request + .url() + .host_str() + .ok_or_else(|| failure("missing relay host"))? + .parse() + .map_err(|_| failure("relay target must be a literal loopback address"))?; + if !host.is_loopback() || request.url().scheme() != "http" { + return Err(failure( + "response-loss relay only permits owned loopback HTTP", + )); + } + let port = request + .url() + .port() + .ok_or_else(|| failure("missing relay port"))?; + let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))?; + listener.set_nonblocking(true)?; + let relay_address = listener.local_addr()?; + request + .url_mut() + .set_port(Some(relay_address.port())) + .map_err(|_| failure("set relay port"))?; + request + .url_mut() + .set_host(Some("127.0.0.1")) + .map_err(|_| failure("set relay host"))?; + std::thread::scope(|scope| { + let relay = scope.spawn(move || -> Result { + let deadline = Instant::now() + STARTUP_WAIT; + let mut downstream = loop { + match listener.accept() { + Ok((stream, peer)) if peer.ip().is_loopback() => break stream, + Ok(_) => return Err(failure("non-loopback relay peer")), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if Instant::now() >= deadline { + return Err(failure("response-loss relay accept deadline exceeded")); + } + std::thread::sleep(HTTP_WAIT_STEP); + } + Err(error) => return Err(error.into()), + } + }; + let request_bytes = read_message(&mut DeadlineStream { + stream: &mut downstream, + deadline, + })?; + let mut upstream = TcpStream::connect_timeout(&(host, port).into(), EXIT_WAIT)?; + upstream.set_write_timeout(Some(EXIT_WAIT))?; + // The body, request identity and authorization bytes are forwarded + // unchanged. Neither request nor response headers enter diagnostics. + upstream.write_all(&request_bytes)?; + let response = read_message(&mut DeadlineStream { + stream: &mut upstream, + deadline, + })?; + let boundary = + header_end(&response).ok_or_else(|| failure("missing upstream header"))?; + let status = std::str::from_utf8(&response[..boundary]) + .ok() + .and_then(|header| header.lines().next()) + .and_then(|line| line.split_whitespace().nth(1)); + if status != Some("200") { + return Err(failure("restore relay upstream did not return HTTP 200")); + } + let value = serde_json::from_slice(&response[boundary..])?; + // Intentionally send zero bytes back; the caller must observe loss. + drop(downstream); + Ok(value) + }); + let caller_failed = client.execute(request).is_err(); + let evidence = relay + .join() + .map_err(|_| failure("response-loss relay panicked"))??; + if !caller_failed { + return Err(failure("caller unexpectedly received a response")); + } + Ok(evidence) + }) +} + +struct DeadlineStream<'a> { + stream: &'a mut TcpStream, + deadline: Instant, +} + +impl Read for DeadlineStream<'_> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let remaining = self.deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "relay deadline exceeded", + )); + } + self.stream.set_read_timeout(Some(remaining))?; + self.stream.read(buffer) + } +} + +fn header_end(bytes: &[u8]) -> Option { + bytes + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|i| i + 4) +} + +fn read_message(stream: &mut impl Read) -> Result> { + let mut bytes = Vec::new(); + let mut buffer = [0; 4096]; + let mut expected = None; + loop { + let count = stream.read(&mut buffer)?; + if count == 0 { + return Err(failure("relay peer closed before complete HTTP message")); + } + bytes.extend_from_slice(&buffer[..count]); + if bytes.len() > MAX_MESSAGE { + return Err(failure("relay HTTP byte budget exceeded")); + } + if expected.is_none() { + if let Some(boundary) = header_end(&bytes) { + if boundary > MAX_HEADER { + return Err(failure("relay HTTP header budget exceeded")); + } + let header = std::str::from_utf8(&bytes[..boundary]) + .map_err(|_| failure("invalid relay HTTP header"))?; + let mut lengths = header + .lines() + .filter_map(|line| line.split_once(':')) + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .map(|(_, value)| value.trim().parse::()); + let length = lengths + .next() + .ok_or_else(|| failure("relay requires Content-Length"))? + .map_err(|_| failure("invalid relay Content-Length"))?; + if lengths.next().is_some() + || length > MAX_MESSAGE - boundary + || header + .lines() + .filter_map(|line| line.split_once(':')) + .any(|(name, _)| name.eq_ignore_ascii_case("transfer-encoding")) + { + return Err(failure("unsupported relay HTTP framing")); + } + expected = Some(boundary + length); + } else if bytes.len() > MAX_HEADER { + return Err(failure("relay HTTP header budget exceeded")); + } + } + if let Some(expected) = expected { + if bytes.len() == expected { + return Ok(bytes); + } + if bytes.len() > expected { + return Err(failure("unexpected relay trailing bytes")); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn framing_rejects_incomplete_ambiguous_or_unbounded_messages() { + for bytes in [ + b"HTTP/1.1 200 OK\r\n\r\n".as_slice(), + b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nshort", + b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nContent-Length: 0\r\n\r\n", + b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nTransfer-Encoding: chunked\r\n\r\n", + b"HTTP/1.1 200 OK\r\nContent-Length: 999999999\r\n\r\n", + ] { + assert!(read_message(&mut &bytes[..]).is_err()); + } + let valid = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\n{}"; + assert_eq!(read_message(&mut &valid[..]).unwrap(), valid); + } +} diff --git a/tests/e2e/tests/worker_restore.rs b/tests/e2e/tests/worker_restore.rs index 3d1619847..75ef29cd2 100644 --- a/tests/e2e/tests/worker_restore.rs +++ b/tests/e2e/tests/worker_restore.rs @@ -143,6 +143,186 @@ fn concurrent_fixed_id_restores_keep_sessions_storage_and_cleanup_isolated() -> result } +#[test] +fn lost_http_response_recovers_exact_request_and_rejects_fresh_stale_intent() -> Result<()> { + response_loss_recovery(false) +} + +#[test] +fn committed_response_loss_survives_owned_runtime_process_kill_and_restart() -> Result<()> { + response_loss_recovery(true) +} + +fn response_loss_recovery(crash_runtime: bool) -> Result<()> { + let fixture = Fixture::load()?; + let result = (|| { + let mut installed = seed::start(&fixture)?; + let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?; + let token = stopped + .restore_observation_token + .as_deref() + .filter(|token| !token.is_empty()) + .ok_or_else(|| failure("missing stopped observation token"))?; + let request_id = "fixed-fixture-response-loss"; + phase( + &installed.environment, + "drop-http-response-after-complete-upstream", + )?; + let observed = installed.environment.restore_worker_losing_response( + &fixture.manifest.worker_id, + request_id, + token, + )?; + let receipt: WorkerRestoreResponse = serde_json::from_value(observed.clone())?; + if receipt.result.state != WorkerRestoreState::Accepted { + return Err(failure( + "upstream did not reach Accepted before response loss", + )); + } + fs::write( + installed + .environment + .artifacts_dir + .join("response-loss.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "caller_received_response": false, + "relay_upstream_state": receipt.result.state, + "request_id": request_id, + "crash_after_committed_terminal_receipt": crash_runtime, + }))?, + )?; + let owner_before = restore_owner_evidence(&installed.environment, &fixture, request_id)?; + if crash_runtime { + phase( + &installed.environment, + "kill-owned-runtime-after-committed-receipt", + )?; + installed.environment.crash_and_restart_runtime()?; + } + phase( + &installed.environment, + "same-request-original-token-recovery", + )?; + let recovered = + installed + .environment + .restore_worker(&fixture.manifest.worker_id, request_id, token)?; + if crash_runtime { + // The public response projects the current observation fence. Runtime + // startup invalidates prior active observations; this is not a change + // to the original request token or its saved outcome. + let current = installed + .environment + .worker_detail(&fixture.manifest.worker_id)?; + assert_eq!( + recovered.pointer("/result/worker/restore_observation_token"), + current.get("restore_observation_token") + ); + let mut expected = observed.clone(); + expected["result"]["worker"]["restore_observation_token"] = + recovered["result"]["worker"]["restore_observation_token"].clone(); + assert_eq!( + recovered, expected, + "restart may change only the public observation fence, not the saved outcome" + ); + } else { + assert_eq!( + recovered, observed, + "exact request must recover the saved receipt" + ); + } + assert_single_state(&installed.environment, &fixture, "idle")?; + let mut subscription = WorkerSessionSubscription::connect( + &installed.environment, + &fixture.manifest.worker_id, + &installed.snapshot, + )?; + subscription.assert_alive()?; + let stale = installed.environment.restore_worker( + &fixture.manifest.worker_id, + "fixed-fixture-fresh-stale-intent", + token, + ); + match stale { + Err(error) if error.to_string().contains("HTTP 409") => {} + _ => { + return Err(failure( + "fresh intent with old observation was not HTTP 409", + )); + } + } + let owner_after = restore_owner_evidence(&installed.environment, &fixture, request_id)?; + assert_eq!( + owner_after, owner_before, + "saved owner/operation/receipt must not change" + ); + fs::write( + installed + .environment + .artifacts_dir + .join("restore-owner-evidence.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "before": owner_before, "after": owner_after, + "request_token_replaced": false, + }))?, + )?; + installed.environment.assert_connected()?; + subscription.assert_alive()?; + stop_and_cleanup(installed, vec![subscription], &fixture) + })(); + fixture.source.assert_unchanged()?; + result +} + +// Read-only diagnostic of the product-written aggregate; never seeds or edits it. +// Persist only a digest, operation ID and owner count, not prompts or credentials. +fn restore_owner_evidence( + environment: &WorkerE2eEnvironment, + fixture: &Fixture, + request_id: &str, +) -> Result { + use sha2::{Digest, Sha256}; + use std::io::Read; + let path = environment + .runtime_data_dir + .join("runtime/workers") + .join(&fixture.manifest.worker_id) + .join("worker.json"); + let mut bytes = Vec::new(); + fs::File::open(path)? + .take(8 * 1024 * 1024 + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > 8 * 1024 * 1024 { + return Err(failure("Worker aggregate byte budget exceeded")); + } + let aggregate: serde_json::Value = serde_json::from_slice(&bytes)?; + let owners = aggregate + .pointer("/restore_guard/owners") + .and_then(serde_json::Value::as_object) + .ok_or_else(|| failure("canonical Worker omitted restore owners"))?; + let owner = owners + .get(request_id) + .ok_or_else(|| failure("canonical owner missing"))?; + if owner + .pointer("/receipt/state") + .and_then(serde_json::Value::as_str) + != Some("accepted") + { + return Err(failure("canonical owner has no Accepted receipt")); + } + let operation = owner + .get("operation_id") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| failure("canonical owner omitted operation ID"))?; + let digest: String = Sha256::digest(serde_json::to_vec(owner)?) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + Ok( + serde_json::json!({"operation_id": operation, "owner_sha256": digest, "owner_count": owners.len()}), + ) +} + fn restore( installed: &Installed, fixture: &Fixture, From f35dcef4d076d57451a17d62d95cc1d30deddfd0 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 13:10:54 +0900 Subject: [PATCH 13/28] test: synchronize owned runtime crashes at restore decision checkpoints --- crates/worker-runtime/Cargo.toml | 2 + crates/worker-runtime/README.md | 27 + crates/worker-runtime/src/lib.rs | 2 + crates/worker-runtime/src/main.rs | 72 +++ .../worker-runtime/src/restore_checkpoints.rs | 143 +++++ .../src/restore_checkpoints/tests.rs | 139 +++++ crates/worker-runtime/src/runtime.rs | 26 + .../src/runtime/restore_checkpoint.rs | 111 ++++ .../src/runtime/restore_checkpoint_tests.rs | 296 +++++++++++ tests/e2e/Cargo.toml | 1 + tests/e2e/README.md | 18 + tests/e2e/src/worker_environment.rs | 93 ++++ tests/e2e/tests/worker_restore.rs | 3 + tests/e2e/tests/worker_restore/checkpoint.rs | 496 ++++++++++++++++++ tests/e2e/tests/worker_restore/seed.rs | 5 +- 15 files changed, 1433 insertions(+), 1 deletion(-) create mode 100644 crates/worker-runtime/src/restore_checkpoints.rs create mode 100644 crates/worker-runtime/src/restore_checkpoints/tests.rs create mode 100644 crates/worker-runtime/src/runtime/restore_checkpoint.rs create mode 100644 crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs create mode 100644 tests/e2e/tests/worker_restore/checkpoint.rs diff --git a/crates/worker-runtime/Cargo.toml b/crates/worker-runtime/Cargo.toml index e61b4fe4f..c57cfa6a8 100644 --- a/crates/worker-runtime/Cargo.toml +++ b/crates/worker-runtime/Cargo.toml @@ -21,6 +21,8 @@ default = ["ws-server", "fs-store"] fs-store = ["dep:fs4"] # Offline fixture tooling only; never enabled by the product defaults. test-support = ["fs-store", "ws-server"] +# Explicit process-test synchronization; independent of offline fixture tooling. +restore-checkpoints = ["fs-store"] http-server = ["dep:axum", "dep:tower", "dep:reqwest"] ws-server = ["http-server", "axum/ws", "tokio/sync"] diff --git a/crates/worker-runtime/README.md b/crates/worker-runtime/README.md index e4f041745..37c4db1f0 100644 --- a/crates/worker-runtime/README.md +++ b/crates/worker-runtime/README.md @@ -39,3 +39,30 @@ not stripped or rewritten. Portability/secret-marker checks do not replace review of the supplied artificial conversation or its independently verified hashes. The installer refuses any destination content other than initialized `auth.toml`; interrupted publication is fail-closed, not an overwrite/recovery facility. + +## Owned-process Restore checkpoints + +The independent, nondefault `restore-checkpoints` feature exists only for explicit +process tests in the shared `tests/e2e` harness. Offline `test-support` does not enable +it. Normal binaries reject `--test-restore-checkpoint`; no environment variable or +HTTP endpoint activates this facility. Do not deploy a checkpoint-enabled binary +as a replacement for the default product. + +A test-enabled Runtime accepts `--test-restore-checkpoint ` with a +bounded regular JSON file selecting a loopback socket, Worker ID, one stage and a +diagnostic nonce. The nonce is only a handshake label, never execution identity, +request authority or routing state. The one-shot stages are real prepared candidate +with `NotCommitted`, and durable `Committed` before activation/terminal receipt. +Before acknowledging, the existing bounded canonical reader validates the exact +Worker/operation/decision, syncs its file/directories and requires no saved receipt. +No pending JSON is fabricated and no lifecycle work is skipped. + +The Runtime sends bounded newline-delimited JSON evidence, then waits at most eight +seconds for byte `R`. EOF, invalid reply, unavailable peer or deadline releases the +wait into the original operation. The Runtime-state mutex is not held across that +wait; existing per-Worker ownership still covers the actual candidate and cleanup. +The harness verifies the ack, kills/reaps only its owned child and inspects the +canonical aggregate offline before restarting the same roots/identity without the +checkpoint flag or another fixture install. These tests are a separate cfg/binary +lane from featureless product regression; they do not certify arbitrary OS failure, +external power loss or a real model call. diff --git a/crates/worker-runtime/src/lib.rs b/crates/worker-runtime/src/lib.rs index bb72c71e3..c00c5fe77 100644 --- a/crates/worker-runtime/src/lib.rs +++ b/crates/worker-runtime/src/lib.rs @@ -16,6 +16,8 @@ pub mod management; pub mod observation; pub mod profile_archive; pub mod resource; +#[cfg(feature = "restore-checkpoints")] +pub mod restore_checkpoints; #[cfg(test)] #[path = "../../../tests/support/restore_environment.rs"] mod restore_test_environment; diff --git a/crates/worker-runtime/src/main.rs b/crates/worker-runtime/src/main.rs index d8b985f43..e40577695 100644 --- a/crates/worker-runtime/src/main.rs +++ b/crates/worker-runtime/src/main.rs @@ -117,6 +117,12 @@ fn run() -> Result<(), ProcessError> { let listener = tokio::net::TcpListener::bind(config.http.bind_addr).await?; let local_addr = listener.local_addr()?; let worker_runtime = build_runtime(&config)?; + #[cfg(feature = "restore-checkpoints")] + if let Some(checkpoint) = &config.test_restore_checkpoint { + worker_runtime + .install_test_restore_checkpoint(checkpoint.clone()) + .map_err(ProcessError::Runtime)?; + } eprintln!( "yoi-runtime listening on {local_addr}; intended client is a trusted backend/proxy, not a browser" ); @@ -391,6 +397,21 @@ where let (flag, inline_value) = split_flag_value(arg)?; match flag.as_str() { "--help" | "-h" => return Ok(None), + #[cfg(feature = "restore-checkpoints")] + "--test-restore-checkpoint" => { + let path = take_value(&flag, inline_value, &mut args)?; + if config.test_restore_checkpoint.is_some() { + return Err(ProcessError::usage( + "--test-restore-checkpoint may only be specified once".into(), + )); + } + config.test_restore_checkpoint = Some( + worker_runtime::restore_checkpoints::RestoreCheckpointConfig::from_path( + Path::new(&path), + ) + .map_err(ProcessError::Runtime)?, + ); + } "--bind" => { let value = take_value(&flag, inline_value, &mut args)?; config.http.bind_addr = value.parse::().map_err(|error| { @@ -465,6 +486,12 @@ where } apply_store_selection(&mut config); + #[cfg(feature = "restore-checkpoints")] + if config.test_restore_checkpoint.is_some() && config.no_store { + return Err(ProcessError::usage( + "--test-restore-checkpoint requires an fs Runtime store".into(), + )); + } Ok(Some(config)) } @@ -521,6 +548,8 @@ struct ProcessConfig { no_store: bool, backend_resource_endpoint: Option, backend_resource_token: Option, + #[cfg(feature = "restore-checkpoints")] + test_restore_checkpoint: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -542,6 +571,8 @@ impl ProcessConfig { no_store: false, backend_resource_endpoint: None, backend_resource_token: None, + #[cfg(feature = "restore-checkpoints")] + test_restore_checkpoint: None, }) } @@ -1227,6 +1258,47 @@ Auth commands: mod tests { use super::*; + #[test] + fn checkpoint_cli_is_absent_by_default() { + let config = parse_args([] as [&str; 0]).unwrap().unwrap(); + #[cfg(feature = "restore-checkpoints")] + assert!(config.test_restore_checkpoint.is_none()); + #[cfg(not(feature = "restore-checkpoints"))] + { + let _ = config; + let error = parse_args(["--test-restore-checkpoint", "never-opened.json"]).unwrap_err(); + assert!(error.to_string().contains("unknown argument")); + } + } + + #[cfg(feature = "restore-checkpoints")] + #[test] + fn checkpoint_cli_requires_valid_explicit_file_and_fs_store() { + use worker_runtime::restore_checkpoints::{ + RestoreCheckpointConfig, RestoreCheckpointStage, + }; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("checkpoint.json"); + let checkpoint = RestoreCheckpointConfig { + endpoint: "127.0.0.1:1234".parse().unwrap(), + worker_id: protocol::WorkerId::now_v7(), + stage: RestoreCheckpointStage::CommittedPreterminal, + nonce: "cli-test".into(), + }; + std::fs::write(&path, serde_json::to_vec(&checkpoint).unwrap()).unwrap(); + let args = ["--test-restore-checkpoint", path.to_str().unwrap()]; + let config = parse_args(args).unwrap().unwrap(); + assert_eq!(config.test_restore_checkpoint, Some(checkpoint)); + assert!(parse_args([args[0], args[1], "--no-store"]).is_err()); + assert!(parse_args([args[0], args[1], args[0], args[1]]).is_err()); + assert!(parse_args([args[0]]).is_err()); + std::fs::write(&path, b"private-invalid-body").unwrap(); + let error = parse_args(args).unwrap_err().to_string(); + assert!(!error.contains("private-invalid-body")); + std::fs::remove_file(&path).unwrap(); + assert!(parse_args(args).is_err()); + } + #[test] fn help_does_not_start_server() { assert!(parse_args(["--help"]).unwrap().is_none()); diff --git a/crates/worker-runtime/src/restore_checkpoints.rs b/crates/worker-runtime/src/restore_checkpoints.rs new file mode 100644 index 000000000..91321afa6 --- /dev/null +++ b/crates/worker-runtime/src/restore_checkpoints.rs @@ -0,0 +1,143 @@ +//! Explicit, ephemeral synchronization for owned Runtime process tests only. +//! +//! The nonce authenticates no Runtime action and is never a routing identity. +//! Socket failure, EOF and the fixed deadline only release the gate; the lifecycle +//! owner continues its original operation and remains responsible for cleanup. + +use std::fs::OpenOptions; +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpStream}; +use std::path::Path; +use std::time::{Duration, Instant}; + +use protocol::WorkerId; +use serde::{Deserialize, Serialize}; + +use crate::error::RuntimeError; +use crate::execution::WorkerLifecycleOperationId; + +pub const RESTORE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(8); +pub const MAX_RESTORE_CHECKPOINT_CONFIG_BYTES: usize = 4096; +pub const MAX_RESTORE_CHECKPOINT_ACK_BYTES: usize = 4096; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RestoreCheckpointStage { + PreparedNotCommitted, + CommittedPreterminal, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RestoreCheckpointCommit { + NotCommitted, + Committed, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RestoreCheckpointConfig { + pub endpoint: SocketAddr, + pub worker_id: WorkerId, + pub stage: RestoreCheckpointStage, + pub nonce: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RestoreCheckpointAck { + pub nonce: String, + pub stage: RestoreCheckpointStage, + pub runtime_id: String, + pub worker_id: WorkerId, + pub operation_id: WorkerLifecycleOperationId, + pub commit: RestoreCheckpointCommit, + pub receipt_absent: bool, + pub pid: u32, +} + +fn invalid_config() -> RuntimeError { + // Do not include serde errors, paths or input contents in process diagnostics. + RuntimeError::InvalidRequest("invalid test restore checkpoint configuration".into()) +} + +impl RestoreCheckpointConfig { + pub fn validate(&self) -> Result<(), RuntimeError> { + if !self.endpoint.ip().is_loopback() + || self.endpoint.port() == 0 + || self.nonce.is_empty() + || self.nonce.len() > 256 + || self.nonce.chars().any(char::is_control) + { + return Err(invalid_config()); + } + Ok(()) + } + + pub fn from_path(path: &Path) -> Result { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + // A FIFO must not stall startup before its regular-file check. + options.custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW); + } + let file = options.open(path).map_err(|_| invalid_config())?; + let metadata = file.metadata().map_err(|_| invalid_config())?; + if !metadata.is_file() || metadata.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 { + return Err(invalid_config()); + } + let mut bytes = Vec::new(); + file.take(MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 + 1) + .read_to_end(&mut bytes) + .map_err(|_| invalid_config())?; + if bytes.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES { + return Err(invalid_config()); + } + let config: Self = serde_json::from_slice(&bytes).map_err(|_| invalid_config())?; + config.validate()?; + Ok(config) + } +} + +pub(crate) fn wait_for_release(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) { + let _ = exchange(config, ack); +} + +fn remaining(deadline: Instant) -> std::io::Result { + deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or_else(|| std::io::Error::from(std::io::ErrorKind::TimedOut)) +} + +fn exchange(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) -> std::io::Result<()> { + let deadline = Instant::now() + RESTORE_CHECKPOINT_TIMEOUT; + let mut bytes = serde_json::to_vec(ack)?; + bytes.push(b'\n'); + if bytes.len() > MAX_RESTORE_CHECKPOINT_ACK_BYTES { + return Err(std::io::Error::from(std::io::ErrorKind::InvalidData)); + } + let mut stream = TcpStream::connect_timeout(&config.endpoint, remaining(deadline)?)?; + // Each syscall shares one budget; partial writes cannot reset the deadline. + let mut pending = bytes.as_slice(); + while !pending.is_empty() { + stream.set_write_timeout(Some(remaining(deadline)?))?; + let written = stream.write(pending)?; + if written == 0 { + return Err(std::io::Error::from(std::io::ErrorKind::WriteZero)); + } + pending = &pending[written..]; + } + stream.set_read_timeout(Some(remaining(deadline)?))?; + let mut release = [0]; + stream.read_exact(&mut release)?; + if release != [b'R'] { + return Err(std::io::Error::from(std::io::ErrorKind::InvalidData)); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/worker-runtime/src/restore_checkpoints/tests.rs b/crates/worker-runtime/src/restore_checkpoints/tests.rs new file mode 100644 index 000000000..16735224d --- /dev/null +++ b/crates/worker-runtime/src/restore_checkpoints/tests.rs @@ -0,0 +1,139 @@ +use super::*; +use std::io::{BufRead, BufReader}; +use std::net::TcpListener; +use std::thread; + +fn config(endpoint: SocketAddr) -> RestoreCheckpointConfig { + RestoreCheckpointConfig { + endpoint, + worker_id: WorkerId::now_v7(), + stage: RestoreCheckpointStage::PreparedNotCommitted, + nonce: "checkpoint-test-handshake".into(), + } +} + +fn ack(config: &RestoreCheckpointConfig) -> RestoreCheckpointAck { + RestoreCheckpointAck { + nonce: config.nonce.clone(), + stage: config.stage, + runtime_id: "checkpoint-runtime".into(), + worker_id: config.worker_id, + operation_id: WorkerLifecycleOperationId::new(), + commit: RestoreCheckpointCommit::NotCommitted, + receipt_absent: true, + pid: std::process::id(), + } +} + +#[test] +fn config_rejects_nonliteral_nonloopback_bad_stage_unknown_fields_and_invalid_worker() { + let valid = serde_json::to_value(config("127.0.0.1:1234".parse().unwrap())).unwrap(); + for (field, value) in [ + ("endpoint", "localhost:1234"), + ("endpoint", "192.0.2.1:1234"), + ("endpoint", "0.0.0.0:1234"), + ("endpoint", "127.0.0.1:0"), + ("endpoint", "[::ffff:127.0.0.1]:1234"), + ("worker_id", "not-a-worker-id"), + ("stage", "after_terminal"), + ("nonce", ""), + ("nonce", "secret\nbody"), + ("timeout", "unbounded"), + ] { + let mut invalid = valid.clone(); + invalid[field] = value.into(); + let parsed = serde_json::from_value::(invalid); + assert!( + parsed.is_err() || parsed.unwrap().validate().is_err(), + "{field}" + ); + } + for endpoint in ["127.0.0.1:1234", "[::1]:1234"] { + config(endpoint.parse().unwrap()).validate().unwrap(); + } +} + +#[test] +fn config_file_bounds_and_errors_do_not_disclose_contents() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.json"); + assert!(RestoreCheckpointConfig::from_path(&path).is_err()); + std::fs::write(&path, vec![b'x'; MAX_RESTORE_CHECKPOINT_CONFIG_BYTES + 1]).unwrap(); + assert!(RestoreCheckpointConfig::from_path(&path).is_err()); + std::fs::write( + &path, + br#"{"nonce":"private-config-body","stage":"secret"}"#, + ) + .unwrap(); + let error = RestoreCheckpointConfig::from_path(&path) + .unwrap_err() + .to_string(); + assert!(!error.contains("private-config-body")); + assert!(!error.contains("secret")); + let mut valid = config("127.0.0.1:1234".parse().unwrap()); + valid.nonce = "x".repeat(257); + assert!(valid.validate().is_err()); + valid.nonce = "valid".into(); + std::fs::write(&path, serde_json::to_vec(&valid).unwrap()).unwrap(); + assert_eq!(RestoreCheckpointConfig::from_path(&path).unwrap(), valid); + assert!(RestoreCheckpointConfig::from_path(dir.path()).is_err()); +} + +#[test] +fn explicit_release_eof_and_invalid_release_close_the_owned_stream() { + for release in [Some(b'R'), None, Some(b'X')] { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let config = config(listener.local_addr().unwrap()); + let expected = ack(&config); + let sent = expected.clone(); + let task = thread::spawn(move || exchange(&config, &sent)); + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut line = String::new(); + BufReader::new(&mut stream).read_line(&mut line).unwrap(); + assert!(line.len() <= MAX_RESTORE_CHECKPOINT_ACK_BYTES); + assert_eq!( + serde_json::from_str::(&line).unwrap(), + expected + ); + if let Some(byte) = release { + stream.write_all(&[byte]).unwrap(); + } else { + stream.shutdown(std::net::Shutdown::Write).unwrap(); + } + assert_eq!(task.join().unwrap().is_ok(), release == Some(b'R')); + let mut byte = [0]; + assert_eq!(stream.read(&mut byte).unwrap(), 0); + } +} + +#[test] +fn absent_listener_resumes_without_waiting_or_retrying() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let config = config(listener.local_addr().unwrap()); + drop(listener); + let start = Instant::now(); + wait_for_release(&config, &ack(&config)); + assert!(start.elapsed() < Duration::from_secs(2)); +} + +#[test] +fn silent_peer_hits_fixed_deadline_and_stream_is_dropped() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let config = config(listener.local_addr().unwrap()); + let start = Instant::now(); + let task = thread::spawn(move || exchange(&config, &ack(&config))); + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(12))) + .unwrap(); + let mut reader = BufReader::new(&mut stream); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + assert!(task.join().unwrap().is_err()); + assert!(start.elapsed() >= RESTORE_CHECKPOINT_TIMEOUT - Duration::from_millis(100)); + assert!(start.elapsed() < RESTORE_CHECKPOINT_TIMEOUT + Duration::from_secs(3)); + assert_eq!(reader.read(&mut [0]).unwrap(), 0); +} diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index f55d359f8..4f9141cc1 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "restore-checkpoints")] +mod restore_checkpoint; mod restore_continuation; use crate::catalog::{ ConfigBundleRef, CreateWorkerRequest, LogicalWorkdirAttachment, ProfileSelector, @@ -2776,6 +2778,12 @@ impl Runtime { worker_state, workdir_attachments, } => { + #[cfg(feature = "restore-checkpoints")] + self.test_restore_checkpoint( + worker_ref, + operation_id, + crate::restore_checkpoints::RestoreCheckpointStage::PreparedNotCommitted, + ); let commit = self.commit_restored_worker_execution( worker_ref, worker_state.clone(), @@ -2784,6 +2792,12 @@ impl Runtime { ); match commit { Ok(_) => { + #[cfg(feature = "restore-checkpoints")] + self.test_restore_checkpoint( + worker_ref, + operation_id, + crate::restore_checkpoints::RestoreCheckpointStage::CommittedPreterminal, + ); if let Err(error) = backend.activate_restored_worker(operation_id, worker_ref) { @@ -5419,6 +5433,8 @@ impl std::fmt::Debug for BackendResourceClientRef { #[derive(Debug)] struct RuntimeState { + #[cfg(feature = "restore-checkpoints")] + restore_checkpoint: Option, display_name: Option, backend: RuntimeBackendKind, /// Backend-bound stable identity used for cross-boundary retention evidence. @@ -5466,6 +5482,8 @@ impl RuntimeState { display_name, backend: RuntimeBackendKind::Memory, runtime_identity: None, + #[cfg(feature = "restore-checkpoints")] + restore_checkpoint: None, persistence: RuntimePersistence::Memory, status: RuntimeStatus::Running, execution_backend: None, @@ -5500,6 +5518,8 @@ impl RuntimeState { display_name, backend: RuntimeBackendKind::FsStore, runtime_identity: None, + #[cfg(feature = "restore-checkpoints")] + restore_checkpoint: None, persistence: RuntimePersistence::Fs(store), status: RuntimeStatus::Running, execution_backend: None, @@ -5550,6 +5570,8 @@ impl RuntimeState { display_name: persisted.display_name, backend: RuntimeBackendKind::FsStore, runtime_identity: None, + #[cfg(feature = "restore-checkpoints")] + restore_checkpoint: None, persistence: RuntimePersistence::Fs(store), status: persisted.status, execution_backend: None, @@ -7553,6 +7575,10 @@ fn subscription_worker_state(status: WorkerStatus) -> SubscriptionWorkerState { #[cfg(test)] mod tests { + #[cfg(feature = "restore-checkpoints")] + mod restore_checkpoint_tests { + include!("runtime/restore_checkpoint_tests.rs"); + } use super::*; use crate::catalog::{ ConfigBundleRef, MaterializerKind, ProfileSelector, RepositoryMaterializationContext, diff --git a/crates/worker-runtime/src/runtime/restore_checkpoint.rs b/crates/worker-runtime/src/runtime/restore_checkpoint.rs new file mode 100644 index 000000000..96da2eeb9 --- /dev/null +++ b/crates/worker-runtime/src/runtime/restore_checkpoint.rs @@ -0,0 +1,111 @@ +use super::*; +use crate::restore_checkpoints::{ + RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage, + wait_for_release, +}; + +impl Runtime { + /// Install an explicitly selected, one-shot process-test gate after normal startup. + /// It is not persisted or consulted by admission, routing or recovery authority. + pub fn install_test_restore_checkpoint( + &self, + config: RestoreCheckpointConfig, + ) -> Result<(), RuntimeError> { + config.validate()?; + let mut state = self.lock()?; + if !matches!(state.persistence, RuntimePersistence::Fs(_)) { + return Err(RuntimeError::InvalidRequest( + "test restore checkpoint requires an fs Runtime store".into(), + )); + } + state.restore_checkpoint = Some(config); + Ok(()) + } + + // The lifecycle caller holds the per-Worker operation lock. No Runtime-state + // mutex is retained while waiting; the real backend candidate remains owned by + // the same operation throughout this synchronous call. + pub(super) fn test_restore_checkpoint( + &self, + worker_ref: &WorkerRef, + operation_id: WorkerLifecycleOperationId, + stage: RestoreCheckpointStage, + ) { + let config = { + let Ok(mut state) = self.lock() else { + return; + }; + let slot = &mut state.restore_checkpoint; + if !slot.as_ref().is_some_and(|config| { + config.worker_id == worker_ref.worker_id && config.stage == stage + }) { + return; + } + // Consume even if canonical verification or the handshake fails. + slot.take().expect("matched checkpoint") + }; + let Ok(ack) = self.restore_checkpoint_ack(&config, worker_ref, operation_id) else { + return; + }; + wait_for_release(&config, &ack); + } + + pub(super) fn restore_checkpoint_ack( + &self, + config: &RestoreCheckpointConfig, + worker_ref: &WorkerRef, + operation_id: WorkerLifecycleOperationId, + ) -> Result { + let invalid = || { + RuntimeError::InvalidRequest("test restore checkpoint authority did not match".into()) + }; + let state = self.lock()?; + let RuntimePersistence::Fs(store) = &state.persistence else { + return Err(invalid()); + }; + // The existing helper validates the exact bounded canonical aggregate and + // syncs that same file plus its owning directories. Never manufacture a + // journal marker or infer commit from an in-memory projection. + let record = store.read_durable_worker_record(worker_ref)?; + let PersistedWorkerExecutionState::ReconciliationRequired( + PersistedWorkerLifecycleOperation::Restore(pending), + ) = &record.execution_state + else { + return Err(invalid()); + }; + let expected = match config.stage { + RestoreCheckpointStage::PreparedNotCommitted => RestoreCommitFact::NotCommitted, + RestoreCheckpointStage::CommittedPreterminal => RestoreCommitFact::Committed, + }; + if config.worker_id != worker_ref.worker_id + || pending.operation_id != operation_id + || pending.commit != expected + { + return Err(invalid()); + } + let guard = &record.restore_guard; + let id = guard.active_request_id.as_ref().ok_or_else(invalid)?; + let owner = guard.owners.get(id).ok_or_else(invalid)?; + if owner.operation_id != operation_id + || owner.request.request_id != *id + || owner.receipt.is_some() + { + return Err(invalid()); + } + let runtime_id = state.runtime_identity.clone().ok_or_else(invalid)?; + Ok(RestoreCheckpointAck { + nonce: config.nonce.clone(), + stage: config.stage, + runtime_id, + worker_id: record.worker_id, + operation_id, + commit: match expected { + RestoreCommitFact::NotCommitted => RestoreCheckpointCommit::NotCommitted, + RestoreCommitFact::Committed => RestoreCheckpointCommit::Committed, + RestoreCommitFact::LegacyUnknown => unreachable!(), + }, + receipt_absent: true, + pid: std::process::id(), + }) + } +} diff --git a/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs b/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs new file mode 100644 index 000000000..7369d6c6b --- /dev/null +++ b/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs @@ -0,0 +1,296 @@ +use super::*; +use crate::restore_checkpoints::{ + RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage, +}; +use std::io::{BufRead, BufReader, Write}; +use std::net::TcpListener; +use std::time::Duration; + +#[test] +fn checkpoint_stages_ack_canonical_decision_and_eof_resumes_same_operation_once() { + // Component scope: durable Runtime with a substituted execution backend. + // Real prepared Controllers and process death are owned by the E2E suite. + for (stage, release) in [ + (RestoreCheckpointStage::PreparedNotCommitted, "release"), + (RestoreCheckpointStage::CommittedPreterminal, "eof"), + (RestoreCheckpointStage::CommittedPreterminal, "timeout"), + ] { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("checkpoint-runtime"), + backend.clone(), + ) + .unwrap(); + runtime.bind_runtime_identity("checkpoint-runtime").unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("checkpoint original operation")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + runtime + .install_test_restore_checkpoint(RestoreCheckpointConfig { + endpoint: listener.local_addr().unwrap(), + worker_id: worker.worker_id, + stage, + nonce: "runtime-stage-test".into(), + }) + .unwrap(); + let cloned = runtime.clone(); + let worker_ref = worker.worker_ref.clone(); + let sent_request = request.clone(); + let (sent, received) = std::sync::mpsc::channel(); + let task = std::thread::spawn(move || { + sent.send(cloned.restore_worker_operation(&worker_ref, sent_request)) + .unwrap(); + }); + listener.set_nonblocking(true).unwrap(); + let stream = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + let listener = tokio::net::TcpListener::from_std(listener).unwrap(); + let (stream, _) = tokio::time::timeout(Duration::from_secs(5), listener.accept()) + .await + .expect("checkpoint did not connect") + .unwrap(); + stream.into_std().unwrap() + }); + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(3))) + .unwrap(); + let mut reader = BufReader::new(stream); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + let ack: RestoreCheckpointAck = serde_json::from_str(&line).unwrap(); + assert_eq!(ack.stage, stage); + assert_eq!(ack.nonce, "runtime-stage-test"); + assert_eq!(ack.runtime_id, "checkpoint-runtime"); + assert_eq!(ack.worker_id, worker.worker_id); + assert_eq!(ack.pid, std::process::id()); + assert!(ack.receipt_absent); + assert_eq!( + ack.commit, + match stage { + RestoreCheckpointStage::PreparedNotCommitted => + RestoreCheckpointCommit::NotCommitted, + RestoreCheckpointStage::CommittedPreterminal => RestoreCheckpointCommit::Committed, + } + ); + assert_eq!( + backend.restore_operation_ids.lock().unwrap().as_slice(), + &[ack.operation_id] + ); + assert!(backend.activation_operation_ids.lock().unwrap().is_empty()); + assert!(received.try_recv().is_err()); + let canonical = runtime_store(&runtime) + .read_durable_worker_record(&worker.worker_ref) + .unwrap(); + let id = canonical.restore_guard.active_request_id.as_ref().unwrap(); + assert_eq!(id, &request.request_id); + assert!(canonical.restore_guard.owners[id].receipt.is_none()); + // Release, EOF and timeout all resume the original lifecycle owner. + let start = std::time::Instant::now(); + if release == "release" { + reader.get_mut().write_all(b"R").unwrap(); + } else if release == "eof" { + reader + .get_mut() + .shutdown(std::net::Shutdown::Write) + .unwrap(); + } + let result = received + .recv_timeout(Duration::from_secs(12)) + .unwrap() + .unwrap(); + assert!(start.elapsed() < Duration::from_secs(11)); + task.join().unwrap(); + assert_eq!(result.state, WorkerRestoreState::Accepted); + assert_eq!( + backend.activation_operation_ids.lock().unwrap().as_slice(), + &[ack.operation_id] + ); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + result + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert!(runtime.lock().unwrap().restore_checkpoint.is_none()); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let next = runtime.test_restore_request(&worker.worker_ref); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, next) + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 2); + } +} + +#[test] +fn checkpoint_disabled_mismatched_or_unreachable_does_not_change_restore_result() { + for mode in ["disabled", "mismatch", "unreachable"] { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("checkpoint-runtime"), + backend.clone(), + ) + .unwrap(); + runtime.bind_runtime_identity("checkpoint-runtime").unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("checkpoint absent")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + if mode != "disabled" { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + drop(listener); + runtime + .install_test_restore_checkpoint(RestoreCheckpointConfig { + endpoint, + worker_id: if mode == "mismatch" { + WorkerId::now_v7() + } else { + worker.worker_id + }, + stage: RestoreCheckpointStage::PreparedNotCommitted, + nonce: "absent-test".into(), + }) + .unwrap(); + } + let result = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(result.state, WorkerRestoreState::Accepted, "{mode}"); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + result + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!( + runtime.lock().unwrap().restore_checkpoint.is_some(), + mode == "mismatch" + ); + } +} + +#[test] +fn checkpoint_verification_requires_canonical_worker_operation_commit_and_absent_receipt() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("runtime"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(&root).with_runtime_id("checkpoint-runtime"), + backend.clone(), + ) + .unwrap(); + runtime.bind_runtime_identity("checkpoint-runtime").unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("checkpoint validation")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + *backend.restore_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::Errored( + WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, "not yet connected"), + )); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + let operation = backend.restore_operation_ids.lock().unwrap()[0]; + let mut config = RestoreCheckpointConfig { + endpoint: "127.0.0.1:1234".parse().unwrap(), + worker_id: worker.worker_id, + stage: RestoreCheckpointStage::PreparedNotCommitted, + nonce: "verification-test".into(), + }; + let path = root + .join("workers") + .join(worker.worker_id.to_string()) + .join("worker.json"); + let before = std::fs::read(&path).unwrap(); + assert!( + runtime + .restore_checkpoint_ack(&config, &worker.worker_ref, operation) + .is_ok() + ); + assert!( + runtime + .restore_checkpoint_ack( + &config, + &worker.worker_ref, + WorkerLifecycleOperationId::new() + ) + .is_err() + ); + config.worker_id = WorkerId::now_v7(); + assert!( + runtime + .restore_checkpoint_ack(&config, &worker.worker_ref, operation) + .is_err() + ); + config.worker_id = worker.worker_id; + config.stage = RestoreCheckpointStage::CommittedPreterminal; + assert!( + runtime + .restore_checkpoint_ack(&config, &worker.worker_ref, operation) + .is_err() + ); + config.stage = RestoreCheckpointStage::PreparedNotCommitted; + runtime_store(&runtime).fail_next_worker_reconcile_sync(); + assert!( + runtime + .restore_checkpoint_ack(&config, &worker.worker_ref, operation) + .is_err() + ); + assert_eq!( + std::fs::read(&path).unwrap(), + before, + "verification must not write authority" + ); + // In-memory equality must not hide an unreadable canonical aggregate. + std::fs::write(&path, b"unreadable canonical aggregate").unwrap(); + assert!( + runtime + .restore_checkpoint_ack(&config, &worker.worker_ref, operation) + .is_err() + ); + assert_eq!( + std::fs::read(&path).unwrap(), + b"unreadable canonical aggregate" + ); + std::fs::write(&path, before).unwrap(); + backend.restore_result.lock().unwrap().take(); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + assert!( + runtime + .restore_checkpoint_ack(&config, &worker.worker_ref, operation) + .is_err(), + "terminal operation cannot ack a preterminal checkpoint" + ); +} diff --git a/tests/e2e/Cargo.toml b/tests/e2e/Cargo.toml index 53b60d5e3..000155fe2 100644 --- a/tests/e2e/Cargo.toml +++ b/tests/e2e/Cargo.toml @@ -10,6 +10,7 @@ autotests = false default = [] e2e = [] fixture-support = ["yoi-workspace-server/test-support", "worker-runtime/test-support"] +restore-checkpoints = ["fixture-support", "worker-runtime/restore-checkpoints"] [dependencies] libc.workspace = true diff --git a/tests/e2e/README.md b/tests/e2e/README.md index a54eb8517..4622758ab 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -39,6 +39,24 @@ process終了ケースはそのAccepted receipt確認後に環境が所有する これは**commit済みかつterminal receipt保存後**のprocess終了であり、未commit候補やcommit直後・receipt前の終了証明ではない。固定Codex資産へSubmit/Notifyを送らないため、通常入力・Paused admissionの製品E2Eの証明でもない。検証jobはこの明示targetを選択する必要があり、通常suiteだけでは代替しない。 +## Opt-in Runtime checkpoint process試験 + +上のfeatureless製品4case（target内の既存seed projection試験1件とは別）に加え、`restore-checkpoints`を明示した実Runtimeで2つのlive windowを試験する。このfeatureはoffline fixture-supportと独立しており、通常RuntimeではCLI入口も環境変数による入口も無い。 + +```sh +# 同期を有効にしたRuntimeだけを専用targetへ。通常製品binaryを上書きしない。 +cargo build --locked -p worker-runtime --bin yoi-runtime \ + --features restore-checkpoints --target-dir target/e2e-checkpoint-products +cargo test --locked -p yoi-e2e --features e2e,restore-checkpoints \ + --test worker_restore checkpoint:: -- --test-threads=1 +``` + +`WorkerE2eEnvironment::use_checkpoint_runtime`はその環境のRuntime binary/hashと、隔離root内の明示configだけを選択する。global provider・環境変数・通常起動への暗黙設定は変更しない。configは一度だけCLIへ渡し、再起動時は渡さない。test側のlink featureをfeatureless製品binaryへ混入させない。 + +共有harnessはloopback ackのstage/Worker/operation/PID/nonceを確認し、直ちに所有childをkill/reapする。**offline正本を確認してから**同roots/identity/trustを再起動するため、ack後のdeadlineとの競合を成功扱いしない。NotCommittedの実私有候補は同operationのcleanup-only→RolledBack/Stopped、Committed/preterminalは同じdecision→実protocol/full Session snapshot→Acceptedへ回収する。元のrequest/token、owner数1、同operation、stale fresh intent拒否、保存source hash、restore run path数、最終cleanupを検証する。JSONでpending状態を作らず、seedは初回のみ。 + +明示解除・peer EOF・observer scopeの意図的panicも試験する。gateの8秒deadlineとscoped HTTP callの15秒deadlineが待機を制限し、通常operationの継続後に環境のRAIIがprocess/rootを回収する。これはtest scope失敗の回収証明であり、OSによるharness強制終了や電源断の一般的な保証ではない。原fixtureへのSubmit/Notifyや実モデル呼出しは行わない。checkpoint ack/at-death/recovery、kill/reap、binary/hashとcleanup artifactはfeaturelessログと分けて報告する。 + 通常のharness回帰試験（製品processは起動しない）: ```sh diff --git a/tests/e2e/src/worker_environment.rs b/tests/e2e/src/worker_environment.rs index 749ef4c83..6ff2c40cb 100644 --- a/tests/e2e/src/worker_environment.rs +++ b/tests/e2e/src/worker_environment.rs @@ -266,6 +266,8 @@ pub struct WorkerE2eEnvironment { access_token: Option, client: Client, binaries: WorkerBinaryProviderInfo, + #[cfg(feature = "restore-checkpoints")] + restore_checkpoint_config: Option, } impl WorkerE2eEnvironment { @@ -331,11 +333,49 @@ impl WorkerE2eEnvironment { access_token: None, client, binaries, + #[cfg(feature = "restore-checkpoints")] + restore_checkpoint_config: None, }; environment.write_metadata("created")?; Ok(environment) } + /// Select the separately built test-enabled Runtime for this environment + /// only. The checkpoint file must already belong to its isolated root. + #[cfg(feature = "restore-checkpoints")] + pub fn use_checkpoint_runtime(&mut self, binary: PathBuf, config: &Path) -> Result<()> { + if self.runtime.is_some() || self.server.is_some() { + return Err(HarnessError::Protocol( + "checkpoint selection requires an unstarted environment".into(), + )); + } + let config = config.canonicalize()?; + if !config.starts_with(self.root.canonicalize()?) || !config.is_file() { + return Err(HarnessError::Protocol( + "checkpoint config is outside the owned environment".into(), + )); + } + let record = binary_record( + "yoi-runtime", + binary.canonicalize()?, + "restore-checkpoints-opt-in".into(), + None, + )?; + let runtime = self + .binaries + .binaries + .iter_mut() + .find(|record| record.name == "yoi-runtime") + .ok_or_else(|| HarnessError::Protocol("missing Runtime binary record".into()))?; + *runtime = record; + self.restore_checkpoint_config = Some(config); + fs::write( + self.artifacts_dir.join("binary-provider.json"), + serde_json::to_vec_pretty(&self.binaries)?, + )?; + Ok(()) + } + pub fn start(&mut self) -> Result<()> { self.start_backend()?; self.start_runtime() @@ -639,6 +679,10 @@ impl WorkerE2eEnvironment { "--fs-root", self.runtime_data_dir.to_string_lossy().as_ref(), ]); + #[cfg(feature = "restore-checkpoints")] + if let Some(config) = self.restore_checkpoint_config.take() { + command.arg("--test-restore-checkpoint").arg(config); + } let runtime = ManagedProcess::spawn( "runtime", command, @@ -684,6 +728,13 @@ impl WorkerE2eEnvironment { /// Kill/reap only the Runtime child owned by this environment, retaining roots /// and identities. Restart neither seeds storage nor rotates restore requests. pub fn crash_and_restart_runtime(&mut self) -> Result<()> { + self.crash_runtime()?; + self.start_runtime_process() + } + + /// Leave the owned Runtime reaped and offline for read-only canonical crash + /// inspection before restart. No root, identity or request is recreated. + pub fn crash_runtime(&mut self) -> Result<()> { let runtime = self .runtime .as_mut() @@ -708,6 +759,17 @@ impl WorkerE2eEnvironment { } self.runtime_url = None; self.runtime_shutdown = WorkerProcessShutdownOutcome::NotStarted; + Ok(()) + } + + /// Restart the previously initialized identity without seed or checkpoint flags. + #[cfg(feature = "restore-checkpoints")] + pub fn restart_runtime_after_crash(&mut self) -> Result<()> { + if self.runtime.is_some() || self.runtime_public_identity.is_none() { + return Err(HarnessError::Protocol( + "Runtime is not an offline initialized child".into(), + )); + } self.start_runtime_process() } @@ -794,6 +856,37 @@ impl WorkerE2eEnvironment { ) } + /// Build one authenticated call that may run in a scoped test thread while + /// the owner observes a checkpoint. The HTTP request deadline bounds joining + /// that thread even on test failure; credentials never enter artifacts. + #[cfg(feature = "restore-checkpoints")] + pub fn pending_restore_call( + &self, + worker_id: &str, + request_id: &str, + expected_observation_token: &str, + ) -> Result Result + Send + 'static + use<>> { + let token = self.access_token.as_deref().ok_or_else(|| { + HarnessError::Protocol("Backend access token is not initialized".into()) + })?; + let request = self + .client + .post(self.worker_url(worker_id, "restore")?) + .bearer_auth(token) + .timeout(STARTUP_WAIT) + .json(&serde_json::json!({"request_id": request_id, + "expected_observation_token": expected_observation_token})) + .build() + .map_err(|_| HarnessError::Protocol("build pending restore request".into()))?; + let client = self.client.clone(); + Ok(move || { + let response = client + .execute(request) + .map_err(|_| HarnessError::Protocol("pending restore transport failed".into()))?; + decode_api_response(response, "pending restore") + }) + } + /// Forward one unchanged public restore request through an owned loopback /// relay, consume its complete upstream response, and close without delivering /// any response bytes. The returned JSON is relay evidence, not caller success. diff --git a/tests/e2e/tests/worker_restore.rs b/tests/e2e/tests/worker_restore.rs index 75ef29cd2..accb62ed8 100644 --- a/tests/e2e/tests/worker_restore.rs +++ b/tests/e2e/tests/worker_restore.rs @@ -1,4 +1,7 @@ //! Fixed, externally supplied Worker restore. Never generates a fixture or sends model input. +#[cfg(feature = "restore-checkpoints")] +#[path = "worker_restore/checkpoint.rs"] +mod checkpoint; #[path = "worker_restore/fixture.rs"] mod fixture; #[path = "worker_restore/seed.rs"] diff --git a/tests/e2e/tests/worker_restore/checkpoint.rs b/tests/e2e/tests/worker_restore/checkpoint.rs new file mode 100644 index 000000000..e55e815be --- /dev/null +++ b/tests/e2e/tests/worker_restore/checkpoint.rs @@ -0,0 +1,496 @@ +//! Opt-in real-process checkpoint cases; never substitute the featureless lane. +use super::*; +use std::collections::BTreeMap; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::path::{Path, PathBuf}; +use worker_runtime::restore_checkpoints::{ + MAX_RESTORE_CHECKPOINT_ACK_BYTES, RestoreCheckpointAck, RestoreCheckpointCommit, + RestoreCheckpointConfig, RestoreCheckpointStage, +}; + +struct Peer { + listener: TcpListener, + config: RestoreCheckpointConfig, + connection: Option, +} + +impl Peer { + fn configure( + environment: &mut WorkerE2eEnvironment, + fixture: &Fixture, + stage: RestoreCheckpointStage, + ) -> Result { + let listener = TcpListener::bind("127.0.0.1:0")?; + listener.set_nonblocking(true)?; + let config = RestoreCheckpointConfig { + endpoint: listener.local_addr()?, + worker_id: fixture::support(fixture.manifest.worker_id.parse())?, + stage, + nonce: format!( + "owned-checkpoint-{}-{}", + std::process::id(), + listener.local_addr()?.port() + ), + }; + let path = environment.root.join("restore-checkpoint.json"); + fs::write(&path, serde_json::to_vec(&config)?)?; + let provider = yoi_e2e::worker_binary_provider_info()?; + let binary = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../target/e2e-checkpoint-products") + .join(provider.profile) + .join("yoi-runtime"); + environment.use_checkpoint_runtime(binary, &path)?; + Ok(Self { + listener, + config, + connection: None, + }) + } + + fn acknowledge(&mut self, environment: &WorkerE2eEnvironment) -> Result { + let deadline = Instant::now() + Duration::from_secs(15); + let (mut stream, address) = loop { + match self.listener.accept() { + Ok(value) => break value, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if Instant::now() >= deadline { + return Err(failure("checkpoint ack deadline exceeded")); + } + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => return Err(error.into()), + } + }; + if !address.ip().is_loopback() { + return Err(failure("non-loopback checkpoint peer")); + } + let mut bytes = Vec::new(); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(failure("checkpoint ack read deadline exceeded")); + } + stream.set_read_timeout(Some(remaining))?; + let mut byte = [0]; + stream.read_exact(&mut byte)?; + if byte == [b'\n'] { + break; + } + bytes.push(byte[0]); + if bytes.len() > MAX_RESTORE_CHECKPOINT_ACK_BYTES { + return Err(failure("checkpoint ack oversized")); + } + } + let ack: RestoreCheckpointAck = serde_json::from_slice(&bytes)?; + if ack.nonce != self.config.nonce + || ack.stage != self.config.stage + || ack.worker_id != self.config.worker_id + || ack.runtime_id != environment.runtime_id + || Some(ack.pid) != environment.runtime_pid() + || !ack.receipt_absent + { + return Err(failure( + "checkpoint ack does not match owned process and stage", + )); + } + fs::write( + environment.artifacts_dir.join("checkpoint-ack.json"), + serde_json::to_vec_pretty(&ack)?, + )?; + self.connection = Some(stream); + Ok(ack) + } + + fn release(&mut self) -> Result<()> { + let mut stream = self + .connection + .take() + .ok_or_else(|| failure("checkpoint not acknowledged"))?; + stream.set_write_timeout(Some(Duration::from_secs(1)))?; + stream.write_all(b"R")?; + Ok(()) + } +} + +fn aggregate_path(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> PathBuf { + environment + .runtime_data_dir + .join("runtime/workers") + .join(&fixture.manifest.worker_id) +} + +fn canonical(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> Result { + let mut bytes = Vec::new(); + fs::File::open(aggregate_path(environment, fixture).join("worker.json"))? + .take(8 * 1024 * 1024 + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > 8 * 1024 * 1024 { + return Err(failure("canonical aggregate oversized")); + } + Ok(serde_json::from_slice(&bytes)?) +} + +fn saved_source_hashes( + environment: &WorkerE2eEnvironment, + fixture: &Fixture, +) -> Result> { + fn visit(root: &Path, path: &Path, files: &mut BTreeMap) -> Result<()> { + if files.len() > 128 { + return Err(failure("saved source file budget exceeded")); + } + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() { + return Err(failure("unexpected saved source symlink")); + } + if metadata.is_dir() { + for entry in fs::read_dir(path)? { + visit(root, &entry?.path(), files)?; + } + } else if metadata.is_file() { + if metadata.len() > 8 * 1024 * 1024 { + return Err(failure("saved source byte budget exceeded")); + } + use sha2::{Digest, Sha256}; + let digest: String = Sha256::digest(fs::read(path)?) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + files.insert( + path.strip_prefix(root) + .unwrap() + .to_string_lossy() + .into_owned(), + digest, + ); + } else { + return Err(failure("unexpected saved source entry type")); + } + Ok(()) + } + let root = aggregate_path(environment, fixture); + let mut files = BTreeMap::new(); + for name in ["metadata.json", "session"] { + visit(&root, &root.join(name), &mut files)?; + } + if files.len() < 2 { + return Err(failure("saved source inventory is incomplete")); + } + Ok(files) +} + +fn restore_runs(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> Result> { + let runs = aggregate_path(environment, fixture).join("runs"); + if !runs.exists() { + return Ok(Vec::new()); + } + let mut names = Vec::new(); + for entry in fs::read_dir(runs)? { + let entry = entry?; + if entry.file_name().to_string_lossy().starts_with("restore-") { + if !entry.file_type()?.is_dir() { + return Err(failure("restore run is not a directory")); + } + names.push(entry.file_name().to_string_lossy().into_owned()); + } + } + names.sort(); + Ok(names) +} + +#[test] +fn acknowledged_not_committed_candidate_process_death_recovers_same_request_as_rollback() +-> Result<()> { + crash_window(RestoreCheckpointStage::PreparedNotCommitted) +} + +#[test] +fn acknowledged_committed_preterminal_process_death_reconstructs_same_decision() -> Result<()> { + crash_window(RestoreCheckpointStage::CommittedPreterminal) +} + +fn crash_window(stage: RestoreCheckpointStage) -> Result<()> { + let fixture = Fixture::load()?; + let result = (|| { + let mut environment = WorkerE2eEnvironment::new()?; + let mut peer = Peer::configure(&mut environment, &fixture, stage)?; + let mut installed = seed::start_in(&fixture, environment)?; + let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?; + let token = stopped + .restore_observation_token + .ok_or_else(|| failure("missing original observation"))?; + let original_sources = saved_source_hashes(&installed.environment, &fixture)?; + let request_id = "owned-process-checkpoint-request"; + let call = installed.environment.pending_restore_call( + &fixture.manifest.worker_id, + request_id, + &token, + )?; + let ack = std::thread::scope(|scope| -> Result { + let request = scope.spawn(call); + let ack = peer.acknowledge(&installed.environment)?; + installed.environment.crash_runtime()?; + // With the actual child reaped there can be no deadline race while + // inspecting the exact decision and nonterminal owner at process death. + let saved = canonical(&installed.environment, &fixture)?; + let op = saved + .pointer("/execution_state/execution/intent") + .ok_or_else(|| failure("checkpoint lacks canonical pending operation"))?; + let owner = &saved["restore_guard"]["owners"][request_id]; + assert_eq!(op["operation_id"], serde_json::to_value(ack.operation_id)?); + assert_eq!(owner["operation_id"], op["operation_id"]); + assert!(owner["receipt"].is_null()); + assert_eq!(owner["request"]["expected_observation_token"], token); + assert_eq!(op["source_repair_possible"], false); + let committed = stage == RestoreCheckpointStage::CommittedPreterminal; + assert_eq!( + op["commit"], + if committed { + "committed" + } else { + "not_committed" + } + ); + assert_eq!( + ack.commit, + if committed { + RestoreCheckpointCommit::Committed + } else { + RestoreCheckpointCommit::NotCommitted + } + ); + assert_eq!( + saved_source_hashes(&installed.environment, &fixture)?, + original_sources + ); + assert_eq!( + restore_runs(&installed.environment, &fixture)?, + vec![format!("restore-{}", ack.operation_id)] + ); + fs::write( + installed + .environment + .artifacts_dir + .join("checkpoint-at-death.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "runtime_reaped_before_inspection": installed.environment.runtime_pid().is_none(), + "operation_id": op["operation_id"], "commit": op["commit"], + "receipt_absent": owner["receipt"].is_null(), + "source_repair_possible": op["source_repair_possible"], + "same_original_request_and_token": true, + "restore_runs": restore_runs(&installed.environment, &fixture)?, + "saved_source_hashes": saved_source_hashes(&installed.environment, &fixture)?, + }))?, + )?; + let outcome = request + .join() + .map_err(|_| failure("public restore caller panicked"))?; + if let Ok(value) = outcome { + let receipt: WorkerRestoreResponse = serde_json::from_value(value)?; + assert_ne!( + receipt.result.state, + WorkerRestoreState::Accepted, + "no caller Accepted may precede either checkpoint kill" + ); + } + installed.environment.restart_runtime_after_crash()?; + assert_ne!(installed.environment.runtime_pid(), Some(ack.pid)); + Ok(ack) + })?; + drop(peer); + let recovered = installed.environment.restore_worker( + &fixture.manifest.worker_id, + request_id, + &token, + )?; + let receipt: WorkerRestoreResponse = serde_json::from_value(recovered.clone())?; + let committed = stage == RestoreCheckpointStage::CommittedPreterminal; + assert_eq!( + receipt.result.state, + if committed { + WorkerRestoreState::Accepted + } else { + WorkerRestoreState::RolledBack + } + ); + assert_eq!( + installed.environment.restore_worker( + &fixture.manifest.worker_id, + request_id, + &token + )?, + recovered + ); + let final_saved = canonical(&installed.environment, &fixture)?; + let owners = final_saved["restore_guard"]["owners"] + .as_object() + .ok_or_else(|| failure("missing canonical owners"))?; + assert_eq!(owners.len(), 1); + assert_eq!( + owners[request_id]["operation_id"], + serde_json::to_value(ack.operation_id)? + ); + assert_eq!( + owners[request_id]["request"]["expected_observation_token"], + token + ); + assert_eq!( + owners[request_id]["receipt"]["state"], + if committed { "accepted" } else { "rolled_back" } + ); + let stale = installed.environment.restore_worker( + &fixture.manifest.worker_id, + "new-stale-checkpoint-intent", + &token, + ); + if !matches!(stale, Err(error) if error.to_string().contains("HTTP 409")) { + return Err(failure("fresh stale intent was not rejected")); + } + fs::write( + installed + .environment + .artifacts_dir + .join("checkpoint-recovery.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "ack": ack, "recovered_state": receipt.result.state, + "canonical_operation_id": owners[request_id]["operation_id"], + "owner_count": owners.len(), "same_original_request_and_token": true, + "saved_source_hashes_before": original_sources, + "saved_source_hashes_after": saved_source_hashes(&installed.environment, &fixture)?, + "restore_runs_after": restore_runs(&installed.environment, &fixture)?, + }))?, + )?; + if committed { + assert_single_state(&installed.environment, &fixture, "idle")?; + assert_eq!( + restore_runs(&installed.environment, &fixture)?, + vec![format!("restore-{}", ack.operation_id)] + ); + let mut subscription = WorkerSessionSubscription::connect( + &installed.environment, + &fixture.manifest.worker_id, + &installed.snapshot, + )?; + subscription.assert_alive()?; + stop_and_cleanup(installed, vec![subscription], &fixture) + } else { + assert_single_state(&installed.environment, &fixture, "stopped")?; + assert!(restore_runs(&installed.environment, &fixture)?.is_empty()); + assert_eq!( + saved_source_hashes(&installed.environment, &fixture)?, + original_sources + ); + let report = installed.environment.cleanup()?; + assert!(report.cleanup_success && !report.fixture_root_exists_after); + Ok(()) + } + })(); + fixture.source.assert_unchanged()?; + result +} + +#[test] +fn acknowledged_checkpoint_peer_disconnect_resumes_original_operation_and_tears_down() -> Result<()> +{ + observer_completion(false) +} + +#[test] +fn acknowledged_checkpoint_explicit_release_resumes_original_operation_and_tears_down() -> Result<()> +{ + observer_completion(true) +} + +fn observer_completion(release: bool) -> Result<()> { + let fixture = Fixture::load()?; + let result = (|| { + let mut environment = WorkerE2eEnvironment::new()?; + let mut peer = Peer::configure( + &mut environment, + &fixture, + RestoreCheckpointStage::PreparedNotCommitted, + )?; + let installed = seed::start_in(&fixture, environment)?; + let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?; + let token = stopped + .restore_observation_token + .ok_or_else(|| failure("missing observation"))?; + let call = installed.environment.pending_restore_call( + &fixture.manifest.worker_id, + "peer-disconnect-request", + &token, + )?; + std::thread::scope(|scope| -> Result<()> { + let request = scope.spawn(call); + peer.acknowledge(&installed.environment)?; + // EOF models an abandoned observer, not a successful restore result. + if release { + peer.release()?; + } else { + drop(peer); + } + let value = request + .join() + .map_err(|_| failure("public caller panicked"))??; + let response: WorkerRestoreResponse = serde_json::from_value(value)?; + assert_eq!(response.result.state, WorkerRestoreState::Accepted); + Ok(()) + })?; + let mut subscription = WorkerSessionSubscription::connect( + &installed.environment, + &fixture.manifest.worker_id, + &installed.snapshot, + )?; + subscription.assert_alive()?; + stop_and_cleanup(installed, vec![subscription], &fixture) + })(); + fixture.source.assert_unchanged()?; + result +} + +#[test] +fn acknowledged_checkpoint_failed_observer_scope_has_bounded_owned_cleanup() -> Result<()> { + let fixture = Fixture::load()?; + let environment = WorkerE2eEnvironment::new()?; + let root = environment.root.clone(); + let artifacts = environment.artifacts_dir.clone(); + let started = Instant::now(); + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<()> { + let mut environment = environment; + let mut peer = Peer::configure( + &mut environment, + &fixture, + RestoreCheckpointStage::PreparedNotCommitted, + )?; + let installed = seed::start_in(&fixture, environment)?; + let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?; + let token = stopped + .restore_observation_token + .ok_or_else(|| failure("missing observation"))?; + let call = installed.environment.pending_restore_call( + &fixture.manifest.worker_id, + "failed-observer-request", + &token, + )?; + std::thread::scope(|scope| -> Result<()> { + scope.spawn(call); + peer.acknowledge(&installed.environment)?; + // Leave the peer connected while unwinding the scoped HTTP caller. + // The Runtime's own deadline must unblock it, then RAII must reap + // both owned processes and remove the environment, not wait forever. + panic!("intentional checkpoint observer failure"); + }) + })); + assert!(outcome.is_err(), "the observer failure was not exercised"); + assert!( + started.elapsed() < Duration::from_secs(45), + "owned cleanup exceeded scenario budget" + ); + assert!(!root.exists(), "failed observer left its isolated root"); + let cleanup: serde_json::Value = + serde_json::from_slice(&fs::read(artifacts.join("cleanup.json"))?)?; + assert_eq!(cleanup["cleanup_success"], true); + assert_eq!(cleanup["fixture_root_exists_after"], false); + fixture.source.assert_unchanged()?; + Ok(()) +} diff --git a/tests/e2e/tests/worker_restore/seed.rs b/tests/e2e/tests/worker_restore/seed.rs index afc7804e4..f95085b06 100644 --- a/tests/e2e/tests/worker_restore/seed.rs +++ b/tests/e2e/tests/worker_restore/seed.rs @@ -42,7 +42,10 @@ pub struct Installed { } pub fn start(fixture: &Fixture) -> Result { - let mut environment = WorkerE2eEnvironment::new()?; + start_in(fixture, WorkerE2eEnvironment::new()?) +} + +pub fn start_in(fixture: &Fixture, mut environment: WorkerE2eEnvironment) -> Result { super::phase(&environment, "fixture-copy-verify")?; let copy = environment.root.join("provided-fixture"); fixture.source.copy_to(©)?; From a10875d68e8cf554e8a7916511adc78d12303d9f Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 13:36:37 +0900 Subject: [PATCH 14/28] fix: retire the exact stopped Controller from runtime observation --- crates/worker-runtime/src/worker_backend.rs | 41 +++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 7e1288701..4d879bb71 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -539,6 +539,16 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { ) -> Result<(), String> { Ok(()) } + + /// Retire observation of this exact Controller after confirmed cleanup. + /// Called under the same per-Worker lock as publication and replacement. + fn deactivate_stopped_controller( + &self, + _worker_ref: &WorkerRef, + _handle: &WorkerHandle, + ) -> Result<(), String> { + Ok(()) + } } /// Production factory that resolves a normal Worker profile and spawns it under @@ -577,6 +587,21 @@ impl RuntimeWorkerObservationHub { Ok(()) } + fn unregister(&self, worker_ref: &WorkerRef, handle: &WorkerHandle) -> Result<(), String> { + let mut workers = self + .workers + .lock() + .map_err(|_| "Worker observation registry lock is poisoned".to_string())?; + if workers.get(worker_ref).is_some_and(|entry| { + entry + .shared_state + .ptr_eq(&Arc::downgrade(&handle.shared_state)) + }) { + workers.remove(worker_ref); + } + Ok(()) + } + fn get( &self, worker_ref: &WorkerRef, @@ -2560,6 +2585,14 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { self.observation_hub .register(worker_ref.clone(), workspace_id.map(str::to_string), handle) } + + fn deactivate_stopped_controller( + &self, + worker_ref: &WorkerRef, + handle: &WorkerHandle, + ) -> Result<(), String> { + self.observation_hub.unregister(worker_ref, handle) + } } type ProfileRestoreStore = CombinedStore; @@ -4463,6 +4496,14 @@ where if let Err(message) = self.cleanup_execution_resources(&execution) { return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } + // External consumers may retain a closed handle indefinitely. Weak + // shared-state liveness is not evidence of a published execution. + if let Err(message) = self + .factory + .deactivate_stopped_controller(worker_ref, &execution.handle) + { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } match self.workers.lock() { Ok(mut workers) => { From e1945d977c96c7e179dfbadedd5d3fe457276e19 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 13:40:51 +0900 Subject: [PATCH 15/28] test: prove real postcommit activation timeout ownership and Stop fencing --- crates/worker-runtime/src/worker_backend.rs | 83 +++ ...orker_backend_restore_publication_tests.rs | 641 ++++++++++++++++++ 2 files changed, 724 insertions(+) diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 4d879bb71..d8eb3cd15 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -847,10 +847,43 @@ struct RestoreFaults { factory_entered: Mutex>>, prepared_gate: Mutex>>, prepared_entered: Mutex>>, + activation_effects_gate: Mutex>, + activation_started_gate: Mutex>, cleanup_gate: Mutex>>, cleanup_entered: Mutex>>, } +// Test-only synchronization inside the real postcommit candidate, not around +// factory preparation. The receiver is moved into the operation-owned task; +// dropping the one-shot sender releases it even during test unwinding. +#[cfg(test)] +struct RestoreActivationGate { + entered: mpsc::Sender, + release: tokio::sync::oneshot::Receiver<()>, +} + +#[cfg(test)] +struct RestoreActivationEvidence { + operation_id: crate::execution::WorkerLifecycleOperationId, + candidate: usize, + snapshot: Event, + handle: Option, + allocation_retained: bool, +} + +#[cfg(test)] +impl RestoreActivationGate { + async fn wait(self, evidence: RestoreActivationEvidence) { + let _ = self.entered.send(evidence); + assert!( + tokio::time::timeout(Duration::from_secs(30), self.release) + .await + .is_ok(), + "test did not release the internal restore activation gate" + ); + } +} + impl ProfileRuntimeWorkerFactory { fn retain_construction_failure( &self, @@ -2631,6 +2664,8 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate { } async fn activate(&mut self) -> Result { + #[cfg(test)] + let candidate_identity = self as *const Self as usize; let prepared = self.prepared.as_mut().ok_or_else(|| { RuntimeRestoreFailure::unproven( RuntimeRestoreStage::Controller, @@ -2668,6 +2703,30 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate { .expect("retained restore owner") .allocation = prepared.allocation_cleanup(); #[cfg(test)] + { + let gate = self + .factory + .restore_faults + .activation_effects_gate + .lock() + .unwrap() + .take(); + if let Some(gate) = gate { + let allocation_retained = self.factory.failed_restore_resources.lock().unwrap() + [&self.request.worker_ref] + .allocation + .is_some(); + gate.wait(RestoreActivationEvidence { + operation_id: self.request.operation_id, + candidate: candidate_identity, + snapshot: prepared.snapshot().clone(), + handle: None, + allocation_retained, + }) + .await; + } + } + #[cfg(test)] if self .factory .restore_checkpoint(RuntimeRestoreStage::ActivationEffects) @@ -2703,6 +2762,30 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate { } .await .map_err(map_error)?; + #[cfg(test)] + { + let gate = self + .factory + .restore_faults + .activation_started_gate + .lock() + .unwrap() + .take(); + if let Some(gate) = gate { + let allocation_retained = self.factory.failed_restore_resources.lock().unwrap() + [&self.request.worker_ref] + .allocation + .is_some(); + gate.wait(RestoreActivationEvidence { + operation_id: self.request.operation_id, + candidate: candidate_identity, + snapshot: started.streams.snapshot_event.clone(), + handle: Some(started.controller.handle.clone()), + allocation_retained, + }) + .await; + } + } self.prepared = None; self.factory .failed_restore_resources diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs index d7518319b..c27401d6c 100644 --- a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -94,6 +94,647 @@ mod publication { (release, received) } + // One-shot sender Drop is the RAII release, including a panic before the + // explicit release. The task also has a 30s fail-safe independent of Drop. + struct ActivationRelease(tokio::sync::oneshot::Sender<()>); + + impl ActivationRelease { + fn release(self) { + let _ = self.0.send(()); + } + } + + fn activation_gate( + fixture: &Fixture, + started: bool, + ) -> (ActivationRelease, mpsc::Receiver) { + let (release, receiver) = tokio::sync::oneshot::channel(); + let (entered, evidence) = mpsc::channel(); + let faults = &fixture.backend.factory.restore_faults; + let slot = if started { + &faults.activation_started_gate + } else { + &faults.activation_effects_gate + }; + assert!( + slot.lock() + .unwrap() + .replace(RestoreActivationGate { + entered, + release: receiver + }) + .is_none() + ); + (ActivationRelease(release), evidence) + } + + fn restore_owner( + runtime: &EmbeddedRuntime, + worker_ref: &WorkerRef, + request_id: &str, + ) -> serde_json::Value { + let record = runtime + .test_runtime_store() + .read_worker_record(worker_ref) + .unwrap(); + let guard = serde_json::to_value(&record.restore_guard).unwrap(); + assert_eq!(guard["owners"].as_object().unwrap().len(), 1); + assert!( + guard["owners"] + .as_object() + .unwrap() + .contains_key(request_id) + ); + guard["owners"][request_id].clone() + } + + fn assert_activation_owned( + fixture: &Fixture, + runtime: &EmbeddedRuntime, + request_id: &str, + evidence: &RestoreActivationEvidence, + ) -> Arc { + let worker_ref = &fixture.request.worker_ref; + let durable = durable_operation(runtime, worker_ref).unwrap(); + assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed); + assert_eq!(durable.operation_id, evidence.operation_id); + let owner = restore_owner(runtime, worker_ref, request_id); + assert_eq!( + owner["operation_id"], + serde_json::to_value(evidence.operation_id).unwrap() + ); + assert!( + owner["receipt"].is_null(), + "committed activation cannot have a terminal receipt yet" + ); + let pending = fixture + .backend + .pending_restore(worker_ref) + .unwrap() + .unwrap(); + assert_eq!(pending.request.operation_id, evidence.operation_id); + assert!(pending.activation_started.load(Ordering::Acquire)); + assert!(pending.task_in_flight.load(Ordering::Acquire)); + assert!(pending.task.try_lock().unwrap().is_some()); + assert!( + pending.result.lock().unwrap().is_none(), + "the task, not another caller, owns the candidate" + ); + assert!(pending.activated.lock().unwrap().is_none()); + assert!(evidence.allocation_retained); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!( + fixture + .backend + .factory + .observation_hub + .get(worker_ref) + .is_none() + ); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + fixture.assert_quiet(); + pending + } + + fn alert_marker(handle: &WorkerHandle, marker: &str) { + handle.alerter.alert( + protocol::AlertLevel::Warn, + protocol::AlertSource::Worker, + marker.into(), + ); + } + + fn is_marker(event: &Event, marker: &str) -> bool { + matches!(event, Event::Alert(alert) if alert.message == marker) + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn postcommit_activation_timeout_autonomously_publishes_same_controller_and_paired_streams() { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let (effects_release, effects_entered) = activation_gate(&fixture, false); + let (started_release, started_entered) = activation_gate(&fixture, true); + let observations = runtime.subscribe_worker_observation().unwrap(); + let first = runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let effects = effects_entered.recv_timeout(WAIT).unwrap(); + assert!(effects.handle.is_none()); + let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &effects); + let canonical = fixture.canonical_files(); + let owner = restore_owner(&runtime, &worker_ref, &request.request_id); + effects_release.release(); + let started = started_entered.recv_timeout(WAIT).unwrap(); + let same_pending = + assert_activation_owned(&fixture, &runtime, &request.request_id, &started); + assert!(Arc::ptr_eq(&pending, &same_pending)); + assert_eq!(effects.candidate, started.candidate); + assert_eq!( + serde_json::to_value(&effects.snapshot).unwrap(), + serde_json::to_value(&started.snapshot).unwrap() + ); + assert_eq!( + restore_owner(&runtime, &worker_ref, &request.request_id), + owner + ); + let old = started.handle.unwrap(); + assert!(!old.protocol_is_closed()); + // This live event is queued on the precommit paired receiver. A newly + // subscribed bridge would miss it; it is not in its original snapshot. + alert_marker(&old, "activation-paired-before-publication"); + started_release.release(); + runtime.test_wait_restore_continuation(&worker_ref, started.operation_id, WAIT); + assert!(durable_operation(&runtime, &worker_ref).is_none()); + let settled = restore_owner(&runtime, &worker_ref, &request.request_id); + assert_eq!(settled["operation_id"], owner["operation_id"]); + assert_eq!(settled["receipt"]["state"], "accepted"); + let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(old.same_controller(&execution.handle)); + assert_eq!(execution.restore_operation_id, Some(started.operation_id)); + assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .publication_attempts + .load(Ordering::SeqCst), + 1 + ); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + transport.validate().unwrap(); + assert_eq!( + serde_json::to_value(&transport.snapshot).unwrap(), + serde_json::to_value(&effects.snapshot).unwrap() + ); + fixture + .backend + .run_on_adapter_runtime(async move { + let mut observations = observations; + tokio::time::timeout(WAIT, async { + loop { + let observed = observations.recv().await.unwrap(); + if is_marker(&observed.payload, "activation-paired-before-publication") { + break; + } + } + }) + .await + .map_err(|_| "paired receiver lost its prepublication event".to_string())?; + Ok(()) + }) + .unwrap(); + assert!( + fixture.canonical_files() == canonical, + "canonical files changed unexpectedly" + ); + drop(transport); + drop(execution); + drop(pending); + drop(same_pending); + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + assert!(old.protocol_is_closed()); + fixture.assert_released(); + fixture.assert_quiet(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn postcommit_activation_timeout_late_nonretryable_failure_never_fakes_rollback() { + for cleanup_fails in [false, true] { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let (release, entered) = activation_gate(&fixture, false); + let faults = &fixture.backend.factory.restore_faults; + *faults.fail_at.lock().unwrap() = Some(RuntimeRestoreStage::ActivationEffects); + faults.cleanup_fail.store(cleanup_fails, Ordering::SeqCst); + let first = runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let effects = entered.recv_timeout(WAIT).unwrap(); + let pending = + assert_activation_owned(&fixture, &runtime, &request.request_id, &effects); + let canonical = fixture.canonical_files(); + release.release(); + if cleanup_fails { + let reason = runtime.test_wait_restore_continuation_parked( + &worker_ref, + effects.operation_id, + WAIT, + ); + assert!(!reason.is_empty()); + let failure = pending + .activated + .lock() + .unwrap() + .as_ref() + .unwrap() + .as_ref() + .err() + .unwrap() + .clone(); + assert_eq!(failure.stage, RuntimeRestoreStage::ActivationEffects); + assert!(failure.durable_effects_possible); + assert!(failure.needs_cleanup()); + assert!(matches!(failure.cleanup, RuntimeRestoreCleanup::Failed(_))); + let durable = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!(durable.operation_id, effects.operation_id); + assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed); + assert!( + restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null() + ); + assert!(Arc::ptr_eq( + &pending, + &fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .unwrap() + )); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!( + fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .is_none() + ); + assert!( + fixture + .backend + .completed_restore_failure(effects.operation_id, &worker_ref) + .is_none() + ); + assert_eq!(faults.executions.load(Ordering::SeqCst), 1); + // Obstruction proof ends here. Stop below is fixture teardown, + // not an autonomous terminal-failure/success claim. + faults.cleanup_fail.store(false, Ordering::SeqCst); + } else { + // A nonretryable candidate must be discarded, not reactivated. + // Reconstruction belongs to the same committed operation. + runtime.test_wait_restore_continuation(&worker_ref, effects.operation_id, WAIT); + let settled = restore_owner(&runtime, &worker_ref, &request.request_id); + assert_eq!(settled["receipt"]["state"], "accepted"); + assert_eq!( + fixture.backend.workers.lock().unwrap()[&worker_ref].restore_operation_id, + Some(effects.operation_id) + ); + assert_eq!(faults.executions.load(Ordering::SeqCst), 2); + assert_eq!(faults.publication_attempts.load(Ordering::SeqCst), 1); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + transport.validate().unwrap(); + drop(transport); + } + assert!( + fixture.canonical_files() == canonical, + "canonical files changed unexpectedly" + ); + drop(pending); + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + fixture.assert_quiet(); + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn postcommit_activation_timeout_stop_remove_fence_late_cleanup_then_fresh_replacement() { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let pause = runtime.test_pause_restore_continuation(); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let (release, entered) = activation_gate(&fixture, true); + let first = runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let started = entered.recv_timeout(WAIT).unwrap(); + let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started); + let old = started.handle.unwrap(); + let canonical = fixture.canonical_files(); + alert_marker(&old, "superseded-old-buffered"); + let stop_error = runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap_err(); + assert!( + stop_error + .to_string() + .contains("restore factory is still pending"), + "Stop must time out joining the activation task" + ); + let remove_error = runtime + .delete_worker_scoped(scope, &worker_ref) + .unwrap_err(); + assert!( + remove_error + .to_string() + .contains("pending stop reconciliation"), + "Remove must be fenced by the unresolved Stop" + ); + assert!(Arc::ptr_eq( + &pending, + &fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .unwrap() + )); + assert_ne!( + runtime.worker_detail(&worker_ref).unwrap().status, + crate::catalog::WorkerStatus::Stopped + ); + assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null()); + assert!(pending.task_in_flight.load(Ordering::Acquire)); + assert!(!old.protocol_is_closed()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!( + fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .is_none() + ); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + release.release(); + drop(pause); + runtime.test_wait_restore_continuation( + &worker_ref, + started.operation_id, + Duration::from_secs(10), + ); + assert_eq!( + runtime.worker_detail(&worker_ref).unwrap().status, + crate::catalog::WorkerStatus::Stopped + ); + assert!(old.protocol_is_closed()); + assert!(!pending.task_in_flight.load(Ordering::Acquire)); + assert!( + pending.task.try_lock().unwrap().is_none(), + "late task must have been joined" + ); + fixture.assert_released(); + // Confirmed Stop deletes the fixture orphan upload, not its Session, + // metadata, or committed entries. Keep the comparison content-private. + let mut stopped_canonical = canonical.clone(); + let orphans = canonical + .keys() + .filter(|path| { + path.parent() == Some(std::path::Path::new("session/artifacts/paste")) + && path.to_string_lossy().ends_with(".file.json") + }) + .cloned() + .collect::>(); + assert_eq!(orphans.len(), 1); + stopped_canonical.remove(&orphans[0]); + assert!( + fixture.canonical_files() == stopped_canonical, + "Stop must remove only the disposable orphan upload" + ); + let superseded = restore_owner(&runtime, &worker_ref, &request.request_id); + assert_eq!(superseded["receipt"]["state"], "superseded"); + assert!( + runtime + .read_worker_observation_events( + &worker_ref, + crate::observation::WorkerObservationCursor::zero() + ) + .unwrap() + .is_empty() + ); + // Only a fresh request from a fresh observation may create a replacement. + let replacement_request = runtime.test_restore_request(&worker_ref); + assert_ne!(replacement_request.request_id, request.request_id); + let accepted = runtime + .restore_worker_operation_scoped(scope, &worker_ref, replacement_request) + .unwrap(); + assert_eq!(accepted.state, server_api::WorkerRestoreState::Accepted); + let replacement = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(!old.same_controller(&replacement.handle)); + { + // Even a late retirement callback identifies the actual old + // Controller, never whichever execution now has this WorkerRef. + let lock = fixture.backend.worker_lock(&worker_ref).unwrap(); + let _guard = lock.lock().unwrap(); + fixture + .backend + .factory + .deactivate_stopped_controller(&worker_ref, &old) + .unwrap(); + let (_, observed_state, _) = fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .unwrap(); + assert!(Arc::ptr_eq( + &observed_state, + &replacement.handle.shared_state + )); + } + assert_ne!(replacement.restore_operation_id, Some(started.operation_id)); + assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 2 + ); + // A diagnostic event on the retained old endpoint must never enter the + // new bridge, and a command on that closed endpoint must not be rerouted. + alert_marker(&old, "superseded-old-after-replacement"); + let old_command = old.clone(); + assert!( + fixture + .backend + .run_on_adapter_runtime(async move { + old_command + .send(Method::Pause { + command: protocol::WorkerCommandEnvelope::new(77111), + }) + .await + .map_err(|error| error.to_string()) + }) + .is_err() + ); + assert_eq!( + replacement.handle.shared_state.snapshot().state, + protocol::WorkerState::Idle + ); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + transport.validate().unwrap(); + let mut observations = runtime.subscribe_worker_observation().unwrap(); + alert_marker(&replacement.handle, "replacement-live-barrier"); + fixture + .backend + .run_on_adapter_runtime(async move { + tokio::time::timeout(WAIT, async { + loop { + let observed = observations.recv().await.unwrap(); + assert!(!is_marker(&observed.payload, "superseded-old-buffered")); + assert!(!is_marker( + &observed.payload, + "superseded-old-after-replacement" + )); + if is_marker(&observed.payload, "replacement-live-barrier") { + break; + } + } + }) + .await + .map_err(|_| "replacement bridge did not progress".to_string())?; + Ok(()) + }) + .unwrap(); + let events = runtime + .read_worker_observation_events( + &worker_ref, + crate::observation::WorkerObservationCursor::zero(), + ) + .unwrap(); + assert!( + !events + .iter() + .any(|event| is_marker(&event.payload, "superseded-old-buffered") + || is_marker(&event.payload, "superseded-old-after-replacement")) + ); + assert!( + fixture.canonical_files() == stopped_canonical, + "replacement changed canonical Session content" + ); + drop(transport); + drop(replacement); + drop(pending); + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + fixture.assert_quiet(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn postcommit_activation_timeout_gate_release_on_panic_keeps_owned_task_bounded() { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let (release, entered) = activation_gate(&fixture, true); + let first = runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let started = entered.recv_timeout(WAIT).unwrap(); + let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started); + let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || { + let _release_on_unwind = release; + panic!("intentional test-owner unwind at activation gate"); + })); + assert!(panic.is_err()); + runtime.test_wait_restore_continuation(&worker_ref, started.operation_id, WAIT); + assert_eq!( + restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"], + "accepted" + ); + assert!(!pending.task_in_flight.load(Ordering::Acquire)); + assert!(pending.task.try_lock().unwrap().is_none()); + assert!( + started + .handle + .as_ref() + .unwrap() + .same_controller(&fixture.backend.workers.lock().unwrap()[&worker_ref].handle) + ); + drop(pending); + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + assert!(started.handle.unwrap().protocol_is_closed()); + fixture.assert_released(); + fixture.assert_quiet(); + } + #[test] #[serial_test::serial(worker_allocation)] fn startup_fault_recovery_retains_cleanup_or_committed_reconstruction_without_repost() { From 2b24db5a15b50d886d321402cdd953bfe4a7b18a Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 13:49:48 +0900 Subject: [PATCH 16/28] test: fence superseded restore replay and old Shutdown after replacement --- ...orker_backend_restore_publication_tests.rs | 52 ++++++++++++++----- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs index c27401d6c..9a905c4bd 100644 --- a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -600,6 +600,23 @@ mod publication { )); } assert_ne!(replacement.restore_operation_id, Some(started.operation_id)); + // Recovering the superseded intent must not settle or mutate the new + // execution, even though both requests address the same WorkerRef. + let replacement_owner = replacement.restore_operation_id; + let replay = runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!(replay.state, server_api::WorkerRestoreState::Superseded); + let saved = runtime + .test_runtime_store() + .read_worker_record(&worker_ref) + .unwrap(); + let guard = serde_json::to_value(&saved.restore_guard).unwrap(); + assert_eq!(guard["owners"][&request.request_id], superseded); + assert_eq!( + fixture.backend.workers.lock().unwrap()[&worker_ref].restore_operation_id, + replacement_owner + ); assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); assert_eq!( fixture @@ -613,20 +630,27 @@ mod publication { // A diagnostic event on the retained old endpoint must never enter the // new bridge, and a command on that closed endpoint must not be rerouted. alert_marker(&old, "superseded-old-after-replacement"); - let old_command = old.clone(); - assert!( - fixture - .backend - .run_on_adapter_runtime(async move { - old_command - .send(Method::Pause { - command: protocol::WorkerCommandEnvelope::new(77111), - }) - .await - .map_err(|error| error.to_string()) - }) - .is_err() - ); + for method in [ + Method::Pause { + command: protocol::WorkerCommandEnvelope::new(77111), + }, + Method::Shutdown { + command: protocol::WorkerCommandEnvelope::new(77112), + }, + ] { + let old_command = old.clone(); + assert!( + fixture + .backend + .run_on_adapter_runtime(async move { + old_command + .send(method) + .await + .map_err(|error| error.to_string()) + }) + .is_err() + ); + } assert_eq!( replacement.handle.shared_state.snapshot().state, protocol::WorkerState::Idle From f85d8311042653b85df03c8d3cf750d8127c0ce4 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 14:10:02 +0900 Subject: [PATCH 17/28] fix: retain committed controller paired streams across bridge admission failure --- crates/worker-runtime/src/worker_backend.rs | 302 ++++++++++++-------- 1 file changed, 190 insertions(+), 112 deletions(-) diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index d8eb3cd15..80d8118d6 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -2974,6 +2974,8 @@ impl ProfileRuntimeWorkerFactory { #[derive(Clone)] struct RuntimeExecutionTaskScope { + #[cfg(feature = "ws-server")] + protocol_bridge: Arc>>, tasks: Arc>>>, shutdown_admission: Arc>>, // Actual unconnected Controllers whose cleanup failed, owned by this @@ -2981,6 +2983,15 @@ struct RuntimeExecutionTaskScope { cleanup_candidates: Arc>>>, } +// The structured scope owns the original paired subscription until task admission +// succeeds. A failed spawn drops a future, not the committed Controller's streams. +#[cfg(feature = "ws-server")] +struct RuntimeProtocolBridge { + context: crate::execution::WorkerExecutionContext, + streams: Arc>>, + snapshot_pending: bool, +} + // Evidence for one exact command on this actual Controller. Keep the send task // and receiver across bounded waits: unknown admission never authorizes resend. struct RuntimeShutdownAdmission { @@ -3062,6 +3073,8 @@ struct RuntimeExecutionTaskCompletion { impl RuntimeExecutionTaskScope { fn new(controller_task: tokio::task::JoinHandle<()>) -> Self { let scope = Self { + #[cfg(feature = "ws-server")] + protocol_bridge: Arc::default(), tasks: Arc::new(Mutex::new(Vec::new())), shutdown_admission: Arc::new(tokio::sync::Mutex::new(None)), cleanup_candidates: Arc::new(Mutex::new(Vec::new())), @@ -4117,6 +4130,108 @@ where ) } + #[cfg(feature = "ws-server")] + fn start_protocol_bridge( + &self, + handle: &WorkerHandle, + worker_state: &Arc>, + tasks: &RuntimeExecutionTaskScope, + ) -> Result<(), String> { + let mut retained = tasks + .protocol_bridge + .lock() + .map_err(|_| "protocol bridge owner lock is poisoned".to_string())?; + let Some(bridge) = retained.as_mut() else { + return Ok(()); + }; + if bridge.snapshot_pending { + let streams = bridge + .streams + .lock() + .map_err(|_| "protocol streams lock is poisoned")?; + bridge + .context + .publish_protocol_event( + streams + .as_ref() + .expect("unadmitted bridge owns streams") + .snapshot_event + .clone(), + ) + .map_err(|error| error.to_string())?; + bridge.snapshot_pending = false; + } + let streams = bridge.streams.clone(); + let bridge_context = bridge.context.clone(); + let bridge_worker_state = worker_state.clone(); + let bridge_handle = handle.clone(); + let task = self.spawn_on_adapter_runtime(async move { + let streams = streams.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + .take().expect("one bridge consumes the paired subscription"); + let mut events = streams.events; + let mut entry_events = streams.log_entries; + loop { + tokio::select! { + event = events.recv() => { + match event { + Ok(event) => { + let mut event = match bridge_handle.alerter.freshen_event(event) { + Ok(event) => event, + Err(_) => { + // This shared bridge also carries Worker state: do + // not terminate it when a bounded Alert was evicted. + // Replace both receivers with the paired snapshot. + let replacement = subscribe_worker_protocol_session(&bridge_handle); + events = replacement.events; + entry_events = replacement.log_entries; + let mut snapshot = replacement.snapshot_event; + if apply_protocol_worker_state(&bridge_worker_state, &mut snapshot).is_err() { + break; + } + let _ = bridge_context.publish_protocol_event(snapshot); + for alert in replacement.alert_snapshot { + let _ = bridge_context.publish_protocol_event(Event::Alert(alert)); + } + continue; + } + }; + match apply_protocol_worker_state(&bridge_worker_state, &mut event) { + Ok(true) => { + let _ = bridge_context.publish_protocol_event(event); + } + Ok(false) => {} + Err(message) => { + let _ = bridge_context.publish_protocol_event(Event::Error { + code: protocol::ErrorCode::Internal, + message: format!("worker state stream rejected: {message}"), + }); + break; + } + } + } + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(broadcast::error::RecvError::Closed) => break, + } + } + entry = entry_events.recv() => { + match entry { + Ok(entry) => { + if let Some(event) = live_log_entry_event(entry) { + let _ = bridge_context.publish_protocol_event(event); + } + } + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(broadcast::error::RecvError::Closed) => break, + } + } + } + } + })?; + tasks.push("protocol bridge", task, true); + *retained = None; + Ok(()) + } + // Own the actual Controller scope before doing any fallible connection work. // The split also permits testing duplicate cleanup with an already-pending // real admission waiter, rather than relying on a scheduler race. @@ -4189,121 +4304,60 @@ where }; } #[cfg(feature = "ws-server")] + let bridge_failure = { + *tasks + .protocol_bridge + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(RuntimeProtocolBridge { + context: bridge_context, + streams: Arc::new(Mutex::new(Some(streams))), + snapshot_pending: restore_operation_id.is_some(), + }); + self.start_protocol_bridge(&handle, &worker_state, &tasks) + .err() + }; + #[cfg(not(feature = "ws-server"))] + let bridge_failure: Option = None; + if let Some(message) = bridge_failure + .as_ref() + .filter(|_| restore_operation_id.is_none()) { - let mut events = streams.events; - let mut entry_events = streams.log_entries; - let bridge_worker_state = worker_state.clone(); - let bridge_handle = handle.clone(); - let initial_publication = if restore_operation_id.is_some() { - bridge_context - .publish_protocol_event(streams.snapshot_event.clone()) - .map(|_| ()) - .map_err(|error| error.to_string()) - } else { - Ok(()) - }; - let bridge_task = match initial_publication.and_then(|()| self.spawn_on_adapter_runtime(async move { - loop { - tokio::select! { - event = events.recv() => { - match event { - Ok(event) => { - let mut event = match bridge_handle.alerter.freshen_event(event) { - Ok(event) => event, - Err(_) => { - // This shared bridge also carries Worker state: do - // not terminate it when a bounded Alert was evicted. - // Replace both receivers with the paired snapshot. - let replacement = subscribe_worker_protocol_session(&bridge_handle); - events = replacement.events; - entry_events = replacement.log_entries; - let mut snapshot = replacement.snapshot_event; - if apply_protocol_worker_state(&bridge_worker_state, &mut snapshot).is_err() { - break; - } - let _ = bridge_context.publish_protocol_event(snapshot); - for alert in replacement.alert_snapshot { - let _ = bridge_context.publish_protocol_event(Event::Alert(alert)); - } - continue; - } - }; - match apply_protocol_worker_state(&bridge_worker_state, &mut event) { - Ok(true) => { - let _ = bridge_context.publish_protocol_event(event); - } - Ok(false) => {} - Err(message) => { - let _ = bridge_context.publish_protocol_event(Event::Error { - code: protocol::ErrorCode::Internal, - message: format!("worker state stream rejected: {message}"), - }); - break; - } - } - } - Err(broadcast::error::RecvError::Lagged(_)) => continue, - Err(broadcast::error::RecvError::Closed) => break, - } - } - entry = entry_events.recv() => { - match entry { - Ok(entry) => { - if let Some(event) = live_log_entry_event(entry) { - let _ = bridge_context.publish_protocol_event(event); - } - } - Err(broadcast::error::RecvError::Lagged(_)) => continue, - Err(broadcast::error::RecvError::Closed) => break, - } - } + drop(workers); + let cleanup = self.cleanup_unconnected_controller( + &handle, + &shutdown, + &tasks, + &shutdown_requested, + &worker_state, + ); + let result = WorkerExecutionResult::errored( + operation, + match &cleanup { + Ok(()) => message.clone(), + Err(cleanup) => format!("{message}; controller cleanup failed: {cleanup}"), + }, + ); + return match cleanup { + Ok(()) => WorkerExecutionSpawnResult::RolledBack(result), + Err(_) => { + self.retain_uncertain_unconnected_controller( + &worker_ref, + handle, + shutdown, + shutdown_requested, + tasks, + worker_state, + workspace_client, + restore_operation_id, + workdir_attachment_statuses.clone(), + ); + WorkerExecutionSpawnResult::ReconciliationRequired { + result, + worker_state: None, + workdir_attachments: workdir_attachment_statuses, } } - })) { - Ok(task) => task, - Err(message) => { - drop(workers); - let cleanup = self.cleanup_unconnected_controller( - &handle, - &shutdown, - &tasks, - &shutdown_requested, - &worker_state, - ); - let result = WorkerExecutionResult::errored( - operation, - match &cleanup { - Ok(()) => message, - Err(cleanup) => { - format!("{message}; controller cleanup failed: {cleanup}") - } - }, - ); - return match cleanup { - Ok(()) => WorkerExecutionSpawnResult::RolledBack(result), - Err(_) => { - let retained_state = worker_state.read().ok().map(|state| state.clone()); - self.retain_uncertain_unconnected_controller( - &worker_ref, - handle, - shutdown, - shutdown_requested, - tasks, - worker_state, - workspace_client, - restore_operation_id, - workdir_attachment_statuses.clone(), - ); - WorkerExecutionSpawnResult::ReconciliationRequired { - result, - worker_state: retained_state, - workdir_attachments: workdir_attachment_statuses, - } - } - }; - } }; - tasks.push("protocol bridge", bridge_task, true); } #[cfg(not(feature = "ws-server"))] { @@ -4332,6 +4386,15 @@ where }, ); + if let Some(message) = bridge_failure { + // A committed restore retains this exact execution and its unadmitted + // paired bridge in the structured scope. Cleanup is not rollback. + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored(operation, message), + worker_state: Some(connected_worker_state), + workdir_attachments: workdir_attachment_statuses, + }; + } WorkerExecutionSpawnResult::Connected { worker_state: connected_worker_state, workdir_attachments: workdir_attachment_statuses, @@ -6188,7 +6251,20 @@ where WorkerExecutionSpawnResult::Connected { .. } => { self.remove_pending_restore(worker_ref)? } - other => return Err(format!("restore activation connection failed: {other:?}")), + other => { + if self + .workers + .lock() + .map_err(|_| "worker registry lock is poisoned")? + .get(worker_ref) + .is_some_and(|execution| { + execution.restore_operation_id == Some(operation_id) + }) + { + self.remove_pending_restore(worker_ref)?; + } + return Err(format!("restore activation connection failed: {other:?}")); + } } } let workers = self @@ -6206,6 +6282,8 @@ where { return Err("restored Worker protocol endpoint is not attachable".into()); } + #[cfg(feature = "ws-server")] + self.start_protocol_bridge(&execution.handle, &execution.worker_state, &execution.tasks)?; self.factory.activate_restored_controller( worker_ref, execution.workspace_id.as_deref(), From 1fd2e099715344fea3f35c07f23c4de58ef3bdd7 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 14:28:37 +0900 Subject: [PATCH 18/28] test: cover real partial publication and terminal receipt durability faults --- crates/worker-runtime/src/execution.rs | 87 +- crates/worker-runtime/src/fs_store.rs | 52 + crates/worker-runtime/src/runtime.rs | 60 ++ .../src/runtime/restore_continuation.rs | 3 + crates/worker-runtime/src/worker_backend.rs | 37 + ...orker_backend_restore_publication_tests.rs | 2 + ...rker_backend_restore_registration_tests.rs | 889 ++++++++++++++++++ 7 files changed, 1128 insertions(+), 2 deletions(-) create mode 100644 crates/worker-runtime/src/worker_backend_restore_registration_tests.rs diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index ffa6e807f..f4c7d85e9 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -215,6 +215,38 @@ impl WorkerExecutionResult { } } +// Private one-shot synchronization at real publication boundaries. Disconnect +// releases the waiter during unwinding; timeout bounds a forgotten test owner. +#[cfg(test)] +#[derive(Debug)] +pub(crate) struct RestoreTestGate { + pub(crate) entered: std::sync::mpsc::Sender, + pub(crate) release: std::sync::mpsc::Receiver<()>, +} + +#[cfg(test)] +impl RestoreTestGate { + pub(crate) fn wait(self, evidence: T) { + let _ = self.entered.send(evidence); + assert!( + !matches!( + self.release + .recv_timeout(std::time::Duration::from_secs(30)), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) + ), + "publication test gate timed out" + ); + } +} + +#[cfg(all(test, feature = "ws-server"))] +#[derive(Debug)] +pub(crate) struct RestoreFlushFault { + pub(crate) remaining_successes: usize, + pub(crate) gate: RestoreTestGate, + pub(crate) restored: std::sync::mpsc::Sender>, +} + /// Runtime hooks available to an execution backend for one Worker. #[cfg(feature = "ws-server")] type WorkerObservationPublisher = Arc< @@ -249,6 +281,8 @@ pub struct WorkerExecutionContext { observation_publisher: WorkerObservationPublisher, #[cfg(feature = "ws-server")] publication_state: Arc>, + #[cfg(all(test, feature = "ws-server"))] + flush_fault: Arc>>, } impl WorkerExecutionContext { @@ -262,6 +296,8 @@ impl WorkerExecutionContext { restore_progress: None, restore_repair_authorizer: None, observation_publisher, + #[cfg(test)] + flush_fault: Arc::default(), publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Active)), } } @@ -276,6 +312,8 @@ impl WorkerExecutionContext { restore_progress: None, restore_repair_authorizer: None, observation_publisher, + #[cfg(test)] + flush_fault: Arc::default(), publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Pending( VecDeque::new(), ))), @@ -386,20 +424,65 @@ impl WorkerExecutionContext { let Some(event) = event else { return Ok(()); }; - if let Err(error) = (self.observation_publisher)(self.worker_ref.clone(), event.clone()) - { + #[cfg(test)] + let fault = { + let mut fault = self.flush_fault.lock().unwrap(); + if fault.as_ref().is_some_and(|f| f.remaining_successes == 0) { + fault.take() + } else { + if let Some(fault) = fault.as_mut() { + fault.remaining_successes -= 1; + } + None + } + }; + #[cfg(test)] + let mut restored = None; + #[cfg(test)] + let publication = if let Some(fault) = fault { + restored = Some(fault.restored); + fault.gate.wait(event.clone()); + Err(RuntimeError::InvalidRequest( + "injected buffered publication failure".into(), + )) + } else { + (self.observation_publisher)(self.worker_ref.clone(), event.clone()) + }; + #[cfg(not(test))] + let publication = (self.observation_publisher)(self.worker_ref.clone(), event.clone()); + if let Err(error) = publication { let mut state = self .publication_state .lock() .map_err(|_| RuntimeError::StatePoisoned)?; if let WorkerExecutionPublicationState::Activating(events) = &mut *state { events.push_front(event); + #[cfg(test)] + if let Some(restored) = restored { + let _ = restored.send(events.iter().cloned().collect()); + } } return Err(error); } } } + #[cfg(all(test, feature = "ws-server"))] + pub(crate) fn test_fail_flush(&self, fault: RestoreFlushFault) { + assert!(self.flush_fault.lock().unwrap().replace(fault).is_none()); + } + + #[cfg(all(test, feature = "ws-server"))] + pub(crate) fn test_buffered_events(&self) -> Vec { + match &*self.publication_state.lock().unwrap() { + WorkerExecutionPublicationState::Pending(events) + | WorkerExecutionPublicationState::Activating(events) => { + events.iter().cloned().collect() + } + _ => Vec::new(), + } + } + #[cfg(not(feature = "ws-server"))] pub(crate) fn activate_candidate(&self) -> Result<(), RuntimeError> { Ok(()) diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index d2b9c86de..1278f89f7 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -128,6 +128,13 @@ pub struct FsRuntimeStore { worker_write_fault: Arc, #[cfg(test)] worker_reconcile_sync_fault: Arc, + #[cfg(test)] + worker_reconcile_read_fault: Arc, + #[cfg(test)] + worker_reconcile_counts: Arc<( + std::sync::atomic::AtomicUsize, + std::sync::atomic::AtomicUsize, + )>, } impl PartialEq for FsRuntimeStore { @@ -174,6 +181,20 @@ impl FsRuntimeStore { .store(true, Ordering::Release); } + #[cfg(test)] + pub(crate) fn fail_next_worker_reconcile_read(&self) { + self.worker_reconcile_read_fault + .store(true, Ordering::Release); + } + + #[cfg(test)] + pub(crate) fn test_worker_reconcile_counts(&self) -> (usize, usize) { + ( + self.worker_reconcile_counts.0.load(Ordering::Acquire), + self.worker_reconcile_counts.1.load(Ordering::Acquire), + ) + } + pub(crate) fn open_or_create( root: PathBuf, runtime_id: &str, @@ -235,6 +256,10 @@ impl FsRuntimeStore { worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), #[cfg(test)] worker_reconcile_sync_fault: Default::default(), + #[cfg(test)] + worker_reconcile_read_fault: Default::default(), + #[cfg(test)] + worker_reconcile_counts: Default::default(), }; let state = if snapshot_exists { Some(store.load_runtime_state()?) @@ -298,8 +323,16 @@ impl FsRuntimeStore { &self, worker_ref: &WorkerRef, ) -> Result { + #[cfg(test)] + self.worker_reconcile_counts + .0 + .fetch_add(1, Ordering::AcqRel); self.read_durable_worker_record_with_sync(worker_ref, |file| { #[cfg(test)] + self.worker_reconcile_counts + .1 + .fetch_add(1, Ordering::AcqRel); + #[cfg(test)] if self .worker_reconcile_sync_fault .swap(false, Ordering::AcqRel) @@ -337,6 +370,17 @@ impl FsRuntimeStore { "Worker aggregate reconciliation requires the Runtime store owner lock".to_string(), )); } + #[cfg(test)] + if self + .worker_reconcile_read_fault + .swap(false, Ordering::AcqRel) + { + return Err(RuntimeError::StoreCommitOutcomeUnknown { + operation: "read reconciled Worker aggregate", + path: self.worker_dir(&worker_ref.worker_id).join(WORKER_FILE), + source: std::io::Error::other("injected reconciliation read failure"), + }); + } let (record, file) = self.read_worker_record_file(worker_ref)?; let worker_dir = self.worker_dir(&worker_ref.worker_id); let path = worker_dir.join(WORKER_FILE); @@ -1724,6 +1768,10 @@ fn validate_current_runtime_store(root: &Path) -> Result<(), RuntimeError> { worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), #[cfg(test)] worker_reconcile_sync_fault: Default::default(), + #[cfg(test)] + worker_reconcile_read_fault: Default::default(), + #[cfg(test)] + worker_reconcile_counts: Default::default(), }; store.load_runtime_state().map(|_| ()) } @@ -1735,6 +1783,10 @@ fn validate_completed_migration_staging(root: &Path) -> Result<(), RuntimeError> worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), #[cfg(test)] worker_reconcile_sync_fault: Default::default(), + #[cfg(test)] + worker_reconcile_read_fault: Default::default(), + #[cfg(test)] + worker_reconcile_counts: Default::default(), }; let state = store.load_runtime_state()?; if let Some(worker) = state.workers.values().find(|worker| { diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 4f9141cc1..755509f7b 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -2340,6 +2340,48 @@ impl Runtime { } } + #[cfg(test)] + pub(crate) fn test_terminal_receipt_gate( + &self, + gate: crate::execution::RestoreTestGate, + ) { + assert!( + self.restore_continuations + .terminal_receipt_gate + .lock() + .unwrap() + .replace(gate) + .is_none() + ); + } + + #[cfg(test)] + pub(crate) fn test_pending_restore_context( + &self, + worker_ref: &WorkerRef, + ) -> ( + WorkerLifecycleOperationId, + crate::execution::WorkerExecutionContext, + bool, + ) { + let state = self.lock().unwrap(); + let worker = state.worker(worker_ref).unwrap(); + assert!( + worker + .restore_guard + .owners + .values() + .all(|owner| owner.receipt.is_none()) + ); + ( + worker.pending_restore.unwrap().operation_id, + worker.restore_candidate_context.clone().unwrap(), + state + .unreconciled_worker_writes + .contains(&worker_ref.worker_id), + ) + } + // Seed durable crash checkpoints without launching a second Controller. // Tests then reopen the real store with empty backend resource maps. #[cfg(all(test, feature = "fs-store"))] @@ -2512,6 +2554,24 @@ impl Runtime { if result.state == WorkerRestoreState::ReconciliationRequired { return Ok(()); } + #[cfg(test)] + if result.state == WorkerRestoreState::Accepted { + let gate = self + .restore_continuations + .terminal_receipt_gate + .lock() + .unwrap() + .take(); + if let Some(gate) = gate { + let operation = self + .lock()? + .worker(worker_ref)? + .pending_restore + .expect("terminal receipt follows committed operation") + .operation_id; + gate.wait(operation); + } + } let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); if candidate.restore_guard.active_request_id.is_some() diff --git a/crates/worker-runtime/src/runtime/restore_continuation.rs b/crates/worker-runtime/src/runtime/restore_continuation.rs index ad50b4d2d..ba8e884c7 100644 --- a/crates/worker-runtime/src/runtime/restore_continuation.rs +++ b/crates/worker-runtime/src/runtime/restore_continuation.rs @@ -20,6 +20,9 @@ pub(super) struct RestoreContinuations { owners: Mutex>>, #[cfg(test)] paused: std::sync::atomic::AtomicBool, + #[cfg(test)] + pub(super) terminal_receipt_gate: + Mutex>>, } #[derive(Debug, Default)] diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 80d8118d6..fe7a65e7e 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -555,9 +555,17 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { /// `WorkerController`. #[derive(Default)] struct RuntimeWorkerObservationHub { + #[cfg(test)] + registration_gate: Mutex>, workers: Mutex>, } +#[cfg(test)] +struct RestoreHubRegistrationGate { + gate: crate::execution::RestoreTestGate, + fail: bool, +} + #[derive(Clone)] struct RuntimeObservedWorker { workspace_id: Option, @@ -584,6 +592,17 @@ impl RuntimeWorkerObservationHub { sink: handle.sink.clone(), }, ); + drop(workers); + #[cfg(test)] + { + let gate = self.registration_gate.lock().unwrap().take(); + if let Some(gate) = gate { + gate.gate.wait(handle.clone()); + if gate.fail { + return Err("injected post-insert hub registration failure".into()); + } + } + } Ok(()) } @@ -837,6 +856,7 @@ struct FailedRestoreResources { #[cfg(test)] #[derive(Default)] struct RestoreFaults { + effects_calls: std::sync::atomic::AtomicUsize, fail_at: Mutex>, executions: std::sync::atomic::AtomicUsize, cleanup_fail: AtomicBool, @@ -2694,6 +2714,11 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate { failure.io_kind = io_kind; failure }; + #[cfg(test)] + self.factory + .restore_faults + .effects_calls + .fetch_add(1, Ordering::SeqCst); prepared.apply_effects().await.map_err(&map_error)?; self.factory .failed_restore_resources @@ -3392,6 +3417,8 @@ struct SubjectControllerReservation { /// Worker execution backend backed by real worker crate Workers. pub struct WorkerRuntimeExecutionBackend { + #[cfg(test)] + bridge_fault: Mutex>>, backend_id: String, factory: Arc, working_directory_materializer: Option>, @@ -3427,6 +3454,8 @@ where .build() .map_err(|err| format!("failed to build worker adapter runtime: {err}"))?; Ok(Self { + #[cfg(test)] + bridge_fault: Mutex::default(), backend_id: DEFAULT_BACKEND_ID.to_string(), factory: Arc::new(factory), working_directory_materializer: None, @@ -4161,6 +4190,14 @@ where .map_err(|error| error.to_string())?; bridge.snapshot_pending = false; } + #[cfg(test)] + { + let fault = self.bridge_fault.lock().unwrap().take(); + if let Some(fault) = fault { + fault.wait(handle.clone()); + return Err("injected bridge task admission failure after snapshot".into()); + } + } let streams = bridge.streams.clone(); let bridge_context = bridge.context.clone(); let bridge_worker_state = worker_state.clone(); diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs index 9a905c4bd..42c1c9153 100644 --- a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -1499,4 +1499,6 @@ mod publication { fixture.assert_released(); fixture.assert_quiet(); } + + include!("worker_backend_restore_registration_tests.rs"); } diff --git a/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs b/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs new file mode 100644 index 000000000..a57d80dcf --- /dev/null +++ b/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs @@ -0,0 +1,889 @@ +// Included by the production Profile publication suite. Each fault is at a +// distinct partial-publication boundary; no factory result is substituted. + +fn publication_gate() -> ( + mpsc::Sender<()>, + mpsc::Receiver, + crate::execution::RestoreTestGate, +) { + let (release, receiver) = mpsc::channel(); + let (entered, evidence) = mpsc::channel(); + ( + release, + evidence, + crate::execution::RestoreTestGate { + entered, + release: receiver, + }, + ) +} + +fn assert_single_publication(fixture: &Fixture, expected: usize) { + let faults = &fixture.backend.factory.restore_faults; + assert_eq!( + faults.executions.load(Ordering::SeqCst), + expected, + "construction reran" + ); + assert_eq!( + faults.effects_calls.load(Ordering::SeqCst), + expected, + "restore effects reran" + ); + assert_eq!( + faults.publication_attempts.load(Ordering::SeqCst), + expected, + "Controller start reran" + ); + fixture.assert_quiet(); +} + +fn assert_real_attach( + fixture: &Fixture, + handle: &WorkerHandle, +) -> crate::execution::WorkerProtocolTransport { + let worker_ref = &fixture.request.worker_ref; + assert!(matches!( + fixture + .backend + .worker_session(crate::execution::WorkerSessionObservationRequest { + worker_ref: worker_ref.clone(), + }), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(worker_ref) + .unwrap(); + transport.validate().unwrap(); + let paired = subscribe_worker_protocol_session(handle); + assert_eq!( + serde_json::to_value(&transport.snapshot).unwrap(), + serde_json::to_value(paired.snapshot_event).unwrap() + ); + assert!( + matches!(&transport.snapshot, Event::Snapshot { session, greeting, state, in_flight, .. } + if session == &session_store::public_snapshot::project_current_session_snapshot(&fixture.entries()) + && !greeting.tools.is_empty() && state.state == protocol::WorkerState::Idle && in_flight.is_empty()) + ); + transport +} + +fn observe_publication_markers( + fixture: &Fixture, + observations: tokio::sync::broadcast::Receiver, + markers: &'static [&'static str], +) -> tokio::sync::broadcast::Receiver { + fixture + .backend + .run_on_adapter_runtime(async move { + let mut observations = observations; + let mut counts = vec![0; markers.len()]; + let mut snapshots = 0; + tokio::time::timeout(WAIT, async { + loop { + let observed = observations.recv().await.unwrap(); + if matches!(observed.payload, Event::Snapshot { .. }) { + snapshots += 1; + } + for (index, marker) in markers.iter().enumerate() { + if is_marker(&observed.payload, marker) { + counts[index] += 1; + } + } + if counts.last() == Some(&1) { + break; + } + } + }) + .await + .map_err(|_| format!("lost original subscription markers: {counts:?}"))?; + assert_eq!( + counts, + vec![1; markers.len()], + "lost or duplicated prefix on original receiver" + ); + assert_eq!(snapshots, 1, "initial full snapshot was lost or replayed"); + while let Ok(event) = observations.try_recv() { + assert!( + !markers + .iter() + .any(|marker| is_marker(&event.payload, marker)), + "duplicate after tail fence" + ); + } + Ok(observations) + }) + .unwrap() +} + +fn partial_registration_recovers_same_controller(hub: bool) { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let pause = runtime.test_pause_restore_continuation(); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let observations = runtime.subscribe_worker_observation().unwrap(); + let (started_release, started_entered) = activation_gate(&fixture, true); + let first = runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let started = started_entered.recv_timeout(WAIT).unwrap(); + let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started); + let old = started.handle.as_ref().unwrap().clone(); + let canonical = fixture.canonical_files(); + alert_marker(&old, "registration-original-paired-prefix"); + let (release, entered, fault) = publication_gate(); + if hub { + *fixture + .backend + .factory + .observation_hub + .registration_gate + .lock() + .unwrap() = Some(RestoreHubRegistrationGate { + gate: fault, + fail: true, + }); + } else { + *fixture.backend.bridge_fault.lock().unwrap() = Some(fault); + } + started_release.release(); + drop(pause); + let boundary_handle = entered.recv_timeout(WAIT).unwrap(); + assert!(boundary_handle.same_controller(&old)); + let durable = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!(durable.operation_id, started.operation_id); + assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed); + assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null()); + if hub { + let (_, shared, _) = fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .unwrap(); + assert!( + Arc::ptr_eq(&shared, &old.shared_state), + "real hub insert preceded fault" + ); + } else { + assert!( + fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .is_none() + ); + } + let (operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref); + assert_eq!(operation, started.operation_id); + assert!(!uncertain); + assert!( + context + .test_buffered_events() + .iter() + .any(|event| matches!(event, Event::Snapshot { .. })) + ); + alert_marker(&old, "registration-during-fault"); + let pause = runtime.test_pause_restore_continuation(); + release.send(()).unwrap(); + // This lock also observes the end of the actual registration attempt, not + // merely the test's entered notification. Its retained scope owns cleanup. + let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(old.same_controller(&execution.handle)); + assert_eq!(execution.restore_operation_id, Some(operation)); + assert_eq!( + execution.tasks.tasks.lock().unwrap().len(), + if hub { 2 } else { 1 } + ); + let bridge = execution.tasks.protocol_bridge.lock().unwrap(); + if hub { + assert!(bridge.is_none()); + } else { + let bridge = bridge.as_ref().unwrap(); + assert!( + !bridge.snapshot_pending, + "snapshot already registered, do not replay it" + ); + let streams = bridge.streams.lock().unwrap(); + assert!( + streams.is_some(), + "original paired streams survived failed task admission" + ); + assert_eq!( + serde_json::to_value(&streams.as_ref().unwrap().snapshot_event).unwrap(), + serde_json::to_value(&started.snapshot).unwrap() + ); + } + drop(bridge); + assert!( + !old.protocol_is_closed(), + "publication failure must not clean a committed Controller" + ); + assert_single_publication(&fixture, 1); + drop(pause); + runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT); + assert!(durable_operation(&runtime, &worker_ref).is_none()); + let owner = restore_owner(&runtime, &worker_ref, &request.request_id); + assert_eq!( + owner["operation_id"], + serde_json::to_value(operation).unwrap() + ); + assert_eq!(owner["receipt"]["state"], "accepted"); + assert!( + fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .is_none() + ); + assert!(!pending.task_in_flight.load(Ordering::Acquire)); + assert!(pending.activated.lock().unwrap().is_none()); + assert!(pending.result.lock().unwrap().is_none()); + assert!(pending.task.try_lock().unwrap().is_none()); + assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); + assert!(old.same_controller(&fixture.backend.workers.lock().unwrap()[&worker_ref].handle)); + let transport = assert_real_attach(&fixture, &old); + assert_eq!(fixture.canonical_files(), canonical); + assert_single_publication(&fixture, 1); + alert_marker(&old, "registration-tail-fence"); + let observations = observe_publication_markers( + &fixture, + observations, + &[ + "registration-original-paired-prefix", + "registration-during-fault", + "registration-tail-fence", + ], + ); + assert_eq!( + runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_single_publication(&fixture, 1); + // Retain the original transport/handle and scope through Stop/replacement. + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + assert!(old.protocol_is_closed()); + assert!(transport.validate().is_err()); + let replacement_request = runtime.test_restore_request(&worker_ref); + assert_eq!( + runtime + .restore_worker_operation_scoped(scope, &worker_ref, replacement_request.clone()) + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + let replacement = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(!old.same_controller(&replacement.handle)); + assert_ne!(replacement.restore_operation_id, Some(operation)); + assert!( + transport + .dispatch(Method::Pause { + command: protocol::WorkerCommandEnvelope::new(801) + }) + .is_err() + ); + assert!( + transport + .dispatch(Method::Shutdown { + command: protocol::WorkerCommandEnvelope::new(802) + }) + .is_err() + ); + for method in [ + Method::Pause { + command: protocol::WorkerCommandEnvelope::new(803), + }, + Method::Shutdown { + command: protocol::WorkerCommandEnvelope::new(804), + }, + ] { + let old_command = old.clone(); + assert!( + fixture + .backend + .run_on_adapter_runtime(async move { + old_command + .send(method) + .await + .map_err(|error| error.to_string()) + }) + .is_err(), + "closed original Controller command must not route to replacement" + ); + } + fixture + .backend + .factory + .deactivate_stopped_controller(&worker_ref, &old) + .unwrap(); + let (_, shared, _) = fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .unwrap(); + assert!(Arc::ptr_eq(&shared, &replacement.handle.shared_state)); + alert_marker(&old, "registration-stopped-event-must-not-leak"); + alert_marker(&replacement.handle, "registration-replacement-fence"); + fixture + .backend + .run_on_adapter_runtime(async move { + let mut observations = observations; + tokio::time::timeout(WAIT, async { + loop { + let event = observations.recv().await.unwrap(); + assert!(!is_marker( + &event.payload, + "registration-stopped-event-must-not-leak" + )); + if is_marker(&event.payload, "registration-replacement-fence") { + break; + } + } + }) + .await + .map_err(|_| "replacement observation fence missing".to_string())?; + drop(transport); + Ok(()) + }) + .unwrap(); + assert_single_publication(&fixture, 2); + assert_eq!( + runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!(restore_owner_count(&runtime, &worker_ref), 2); + let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(current.handle.same_controller(&replacement.handle)); + assert_eq!( + current.restore_operation_id, + replacement.restore_operation_id + ); + assert_eq!( + replacement.handle.shared_state.snapshot().state, + protocol::WorkerState::Idle + ); + drop(current); + drop(execution); + drop(replacement); + drop(pending); + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); +} + +fn restore_owner_count(runtime: &EmbeddedRuntime, worker_ref: &WorkerRef) -> usize { + let saved = runtime + .test_runtime_store() + .read_worker_record(worker_ref) + .unwrap(); + serde_json::to_value(saved.restore_guard).unwrap()["owners"] + .as_object() + .unwrap() + .len() +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn bridge_partial_snapshot_registration_retains_original_streams_and_autonomously_recovers() { + partial_registration_recovers_same_controller(false); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn hub_partial_insert_registration_retains_original_controller_and_autonomously_recovers() { + partial_registration_recovers_same_controller(true); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn context_partial_flush_retains_only_undelivered_queue_and_recovers_without_repost() { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let pause = runtime.test_pause_restore_continuation(); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let observations = runtime.subscribe_worker_observation().unwrap(); + let (started_release, started_entered) = activation_gate(&fixture, true); + assert_eq!( + runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap() + .state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let started = started_entered.recv_timeout(WAIT).unwrap(); + let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started); + let old = started.handle.unwrap(); + let (operation, context, _) = runtime.test_pending_restore_context(&worker_ref); + // Stop after the actual bridge and hub registration, without failing that + // boundary, so the flush begins with its real original full snapshot. + let (hub_release, hub_entered, hub_gate) = publication_gate(); + *fixture + .backend + .factory + .observation_hub + .registration_gate + .lock() + .unwrap() = Some(RestoreHubRegistrationGate { + gate: hub_gate, + fail: false, + }); + started_release.release(); + drop(pause); + assert!( + hub_entered + .recv_timeout(WAIT) + .unwrap() + .same_controller(&old) + ); + let buffered_prefix = context.test_buffered_events(); + assert!(matches!( + buffered_prefix.first(), + Some(Event::Snapshot { .. }) + )); + let prefix_length = buffered_prefix.len(); + let marker = |text: &str| Event::Error { + code: protocol::ErrorCode::Internal, + message: text.into(), + }; + context + .publish_protocol_event(marker("flush-delivered")) + .unwrap(); + context + .publish_protocol_event(marker("flush-failed")) + .unwrap(); + context + .publish_protocol_event(marker("flush-pending")) + .unwrap(); + let (release, entered, gate) = publication_gate(); + let (restored, restored_queue) = mpsc::channel(); + context.test_fail_flush(crate::execution::RestoreFlushFault { + remaining_successes: prefix_length + 1, + gate, + restored, + }); + drop(hub_release); + assert!( + matches!(entered.recv_timeout(WAIT).unwrap(), Event::Error { message, .. } if message == "flush-failed") + ); + let partial_execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(old.same_controller(&partial_execution.handle)); + assert_eq!(partial_execution.restore_operation_id, Some(operation)); + assert_eq!(partial_execution.tasks.tasks.lock().unwrap().len(), 2); + let (_, shared, _) = fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .unwrap(); + assert!(Arc::ptr_eq(&shared, &old.shared_state)); + let pause = runtime.test_pause_restore_continuation(); + context + .publish_protocol_event(marker("flush-arrived-during-error")) + .unwrap(); + release.send(()).unwrap(); + let restored = restored_queue.recv_timeout(WAIT).unwrap(); + let undelivered: Vec<_> = restored + .iter() + .filter_map(|event| match event { + Event::Error { message, .. } if message.starts_with("flush-") => Some(message.as_str()), + _ => None, + }) + .collect(); + assert_eq!( + undelivered, + [ + "flush-failed", + "flush-pending", + "flush-arrived-during-error" + ] + ); + assert!( + !restored + .iter() + .any(|event| matches!(event, Event::Snapshot { .. })), + "delivered snapshot must not rejoin the failed suffix" + ); + assert_eq!( + serde_json::to_value(context.test_buffered_events()).unwrap(), + serde_json::to_value(&restored).unwrap() + ); + assert_eq!( + durable_operation(&runtime, &worker_ref) + .unwrap() + .operation_id, + operation + ); + assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null()); + // The next terminal gate observes successful retry only after the same + // queued owner has repaired its failed callback and remaining suffix. + let (terminal_release, terminal_entered, gate) = publication_gate(); + runtime.test_terminal_receipt_gate(gate); + drop(pause); + assert_eq!(terminal_entered.recv_timeout(WAIT).unwrap(), operation); + assert!(context.test_buffered_events().is_empty()); + assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null()); + assert_eq!( + durable_operation(&runtime, &worker_ref).unwrap().commit, + crate::runtime::RestoreCommitFact::Committed + ); + let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(old.same_controller(&execution.handle)); + assert_eq!(execution.restore_operation_id, Some(operation)); + drop(terminal_release); // RAII disconnect, not a second request. + runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT); + assert_eq!( + restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"], + "accepted" + ); + assert_real_attach(&fixture, &old); + fixture + .backend + .run_on_adapter_runtime(async move { + let mut observations = observations; + let mut messages = Vec::new(); + let mut snapshots = 0; + while let Ok(event) = observations.try_recv() { + match event.payload { + Event::Error { message, .. } if message.starts_with("flush-") => { + messages.push(message) + } + Event::Snapshot { .. } => snapshots += 1, + _ => {} + } + } + assert_eq!( + messages, + [ + "flush-delivered", + "flush-failed", + "flush-pending", + "flush-arrived-during-error" + ] + ); + assert_eq!(snapshots, 1); + Ok(()) + }) + .unwrap(); + assert_single_publication(&fixture, 1); + drop(partial_execution); + drop(execution); + drop(pending); + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); +} + +#[derive(Clone, Copy, Debug)] +enum TerminalFault { + BeforeRename, + AfterRename, + SyncUnknown, + ReadUnknown, +} + +fn published_terminal_receipt_fault(fault: TerminalFault) { + let mut fixture = Fixture::no_model(); + // Reuse the existing ordinary non-network model boundary, not the product + // Codex profile. All fault/convergence assertions precede the sole input. + let script = MockClient::sequential(vec![MockResponse::Complete(simple_text_events())]); + let injected = script.clone(); + let factory = fixture + .backend + .factory + .as_ref() + .clone() + .with_spawn_model_client_factory(move || Box::new(injected.clone())); + fixture.backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap()); + let runtime = Arc::new(stopped_runtime(&fixture)); + let pause = runtime.test_pause_restore_continuation(); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let observations = runtime.subscribe_worker_observation().unwrap(); + let (release, entered, gate) = publication_gate(); + runtime.test_terminal_receipt_gate(gate); + let restoring = { + let runtime = runtime.clone(); + let worker_ref = worker_ref.clone(); + let scope = scope.clone(); + let request = request.clone(); + std::thread::spawn(move || { + runtime.restore_worker_operation_scoped(&scope, &worker_ref, request) + }) + }; + let operation = entered.recv_timeout(WAIT).unwrap(); + // Restore legitimately repairs derived metadata before publication. Receipt + // failures must not rerun those effects; compare from this actual boundary. + let canonical = fixture.canonical_files(); + let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + let old = execution.handle.clone(); + assert_eq!(execution.restore_operation_id, Some(operation)); + assert!( + fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .is_none() + ); + assert_eq!(execution.tasks.tasks.lock().unwrap().len(), 2); + assert!(execution.tasks.protocol_bridge.lock().unwrap().is_none()); + assert_eq!( + durable_operation(&runtime, &worker_ref) + .unwrap() + .operation_id, + operation + ); + assert_eq!( + durable_operation(&runtime, &worker_ref).unwrap().commit, + crate::runtime::RestoreCommitFact::Committed + ); + let owner = restore_owner(&runtime, &worker_ref, &request.request_id); + assert!(owner["receipt"].is_null()); + let (same_operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref); + assert_eq!(same_operation, operation); + assert!(!uncertain); + assert!(context.test_buffered_events().is_empty()); + let transport = assert_real_attach(&fixture, &old); + assert_single_publication(&fixture, 1); + alert_marker(&old, "terminal-before-receipt"); + let store = runtime.test_runtime_store(); + let reconcile_before = store.test_worker_reconcile_counts(); + store.fail_next_worker_write(if matches!(fault, TerminalFault::BeforeRename) { + AtomicWriteFault::BeforeRename + } else { + AtomicWriteFault::AfterRename + }); + match fault { + TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(), + TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(), + _ => {} + } + release.send(()).unwrap(); + let first = restoring.join().unwrap().unwrap(); + let reconcile_after = store.test_worker_reconcile_counts(); + let reconciliation = ( + reconcile_after.0 - reconcile_before.0, + reconcile_after.1 - reconcile_before.1, + ); + assert_eq!( + reconciliation, + match fault { + TerminalFault::BeforeRename => (0, 0), + TerminalFault::AfterRename => (1, 4), // canonical file plus Worker/workers/root directories + TerminalFault::SyncUnknown => (1, 1), + TerminalFault::ReadUnknown => (1, 0), + }, + "{fault:?}: actual canonical read/sync attempts" + ); + assert_eq!( + first.state, + if matches!(fault, TerminalFault::AfterRename) { + server_api::WorkerRestoreState::Accepted + } else { + server_api::WorkerRestoreState::ReconciliationRequired + }, + "{fault:?}: {first:?}" + ); + assert_eq!(fixture.canonical_files(), canonical); + assert!(!old.protocol_is_closed()); + transport.validate().unwrap(); + assert_single_publication(&fixture, 1); + let disk_owner = restore_owner(&runtime, &worker_ref, &request.request_id); + assert_eq!(disk_owner["operation_id"], owner["operation_id"]); + match fault { + TerminalFault::BeforeRename => { + assert!(disk_owner["receipt"].is_null()); + assert!(!runtime.test_pending_restore_context(&worker_ref).2); + } + TerminalFault::AfterRename => { + assert_eq!(disk_owner["receipt"]["state"], "accepted"); + assert!(durable_operation(&runtime, &worker_ref).is_none()); + } + TerminalFault::SyncUnknown | TerminalFault::ReadUnknown => { + assert_eq!( + disk_owner["receipt"]["state"], "accepted", + "visible rename alone is not durable proof" + ); + let (same_operation, _, fenced) = runtime.test_pending_restore_context(&worker_ref); + assert_eq!(same_operation, operation); + assert!( + fenced, + "unknown canonical outcome must retain the memory owner/write fence" + ); + // Keep durability unproved for one competing Stop as well: it must + // fail before any Shutdown, never manufacture a Stopped judgment. + match fault { + TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(), + TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(), + _ => unreachable!(), + } + assert!(matches!( + runtime.stop_worker_scoped(&scope, &worker_ref, None), + Err(crate::error::RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); + assert!(!old.protocol_is_closed()); + assert!(!execution.shutdown_requested.load(Ordering::Acquire)); + assert!(runtime.test_pending_restore_context(&worker_ref).2); + assert_single_publication(&fixture, 1); + } + } + // Recovery is autonomous after the failed write / confirmed unknown fence. + // No POST, fresh request, or input is needed to make it converge. + drop(pause); + if !matches!(fault, TerminalFault::AfterRename) { + runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT); + } + assert!(durable_operation(&runtime, &worker_ref).is_none()); + assert_eq!( + restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"], + "accepted" + ); + let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(old.same_controller(¤t.handle)); + assert_eq!(current.restore_operation_id, Some(operation)); + assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); + assert_real_attach(&fixture, &old); + assert_single_publication(&fixture, 1); + alert_marker(&old, "terminal-after-recovery"); + let _observations = observe_publication_markers( + &fixture, + observations, + &["terminal-before-receipt", "terminal-after-recovery"], + ); + let accepted = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!(accepted.state, server_api::WorkerRestoreState::Accepted); + assert_eq!( + runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request.clone()) + .unwrap(), + accepted + ); + assert_eq!(restore_owner_count(&runtime, &worker_ref), 1); + let mut stale = request; + stale.request_id = "terminal-fresh-stale-intent".into(); + assert!(matches!( + runtime.restore_worker_operation_scoped(&scope, &worker_ref, stale), + Err(crate::error::RuntimeError::RestoreObservationConflict { .. }) + )); + assert_single_publication(&fixture, 1); + assert_eq!( + script.call_count.load(Ordering::SeqCst), + 0, + "receipt recovery cannot trigger input or a model call" + ); + // This transport was paired before the failed receipt write. Exercise its + // normal log-entry receiver after autonomous settlement, not a new attach. + transport + .dispatch(Method::submit_text( + "terminal-ordinary-input", + "terminal-ordinary-input", + )) + .unwrap(); + fixture + .backend + .run_on_adapter_runtime(async move { + let mut transport = transport; + let (mut admitted, mut committed, mut text, mut ended) = (false, false, false, false); + tokio::time::timeout(WAIT, async { + while !(admitted && committed && text && ended) { + match transport.events.recv().await.unwrap() { + Event::SubmissionAccepted { + submission_request_id, + .. + } if submission_request_id == "terminal-ordinary-input" => admitted = true, + Event::UserMessage { segments, .. } => { + assert_eq!( + segments, + vec![protocol::Segment::text("terminal-ordinary-input")] + ); + assert!(!committed, "paired log receiver duplicated input"); + committed = true; + } + Event::TextDone { text: output } => { + assert_eq!(output, "hello from worker"); + text = true; + } + Event::RunEnd { .. } => ended = true, + Event::SubmissionRejected { .. } | Event::Error { .. } => { + panic!("ordinary input failed after receipt recovery") + } + _ => {} + } + } + }) + .await + .map_err(|_| { + "ordinary input failed to settle on original paired transport".to_string() + })?; + drop(transport); + Ok(()) + }) + .unwrap(); + assert_eq!(script.call_count.load(Ordering::SeqCst), 1); + assert_eq!(fixture.entries().iter().filter(|entry| matches!(entry, + LogEntry::AnnotatedUserInput { segments, .. } if segments == &vec![protocol::Segment::text("terminal-ordinary-input")] + )).count(), 1); + assert_single_publication(&fixture, 1); + drop(execution); + drop(current); + runtime + .stop_worker_scoped(&scope, &worker_ref, None) + .unwrap(); + assert!(old.protocol_is_closed()); + fixture.assert_released(); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn published_controller_terminal_receipt_before_rename_autonomously_settles_without_reconstruction() +{ + published_terminal_receipt_fault(TerminalFault::BeforeRename); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn published_controller_terminal_receipt_after_rename_is_canonically_reread_and_synced() { + published_terminal_receipt_fault(TerminalFault::AfterRename); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn published_controller_terminal_receipt_sync_unknown_retains_owner_and_write_fence() { + published_terminal_receipt_fault(TerminalFault::SyncUnknown); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn published_controller_terminal_receipt_read_unknown_retains_owner_and_write_fence() { + published_terminal_receipt_fault(TerminalFault::ReadUnknown); +} From 1e88893ef92574a69be665b29718b30ccb8b67d9 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 14:31:45 +0900 Subject: [PATCH 19/28] fix: preserve non-restore cleanup evidence through bridge refactoring --- crates/worker-runtime/src/worker_backend.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index fe7a65e7e..da734f13a 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -4377,6 +4377,7 @@ where return match cleanup { Ok(()) => WorkerExecutionSpawnResult::RolledBack(result), Err(_) => { + let retained_state = worker_state.read().ok().map(|state| state.clone()); self.retain_uncertain_unconnected_controller( &worker_ref, handle, @@ -4390,7 +4391,7 @@ where ); WorkerExecutionSpawnResult::ReconciliationRequired { result, - worker_state: None, + worker_state: retained_state, workdir_attachments: workdir_attachment_statuses, } } From b19de45a8237ce0f75347a832235d293f6808d43 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 14:40:58 +0900 Subject: [PATCH 20/28] test: verify original catalog projection after receipt reconciliation --- ...rker_backend_restore_registration_tests.rs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs b/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs index a57d80dcf..b6767d951 100644 --- a/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs @@ -628,6 +628,12 @@ fn published_terminal_receipt_fault(fault: TerminalFault) { let scope = fixture.request.workspace_scope.clone().unwrap(); let request = runtime.test_restore_request(&worker_ref); let observations = runtime.subscribe_worker_observation().unwrap(); + let mut catalog = runtime + .subscribe_event_selector_scoped( + &scope, + protocol::subscription::EventSubscriptionSelector::RuntimeWorkers, + ) + .unwrap(); let (release, entered, gate) = publication_gate(); runtime.test_terminal_receipt_gate(gate); let restoring = { @@ -772,6 +778,38 @@ fn published_terminal_receipt_fault(fault: TerminalFault) { assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); assert_real_attach(&fixture, &old); assert_single_publication(&fixture, 1); + // A canonical terminal reread may bypass the final redundant upsert. The + // original subscriber must still have the current state and observation token. + let fresh_catalog = runtime + .subscribe_event_selector_scoped( + &scope, + protocol::subscription::EventSubscriptionSelector::RuntimeWorkers, + ) + .unwrap(); + let protocol::subscription::SubscriptionSnapshot::Workers { workers } = + fresh_catalog.snapshot() + else { + panic!("Runtime catalog must return Workers"); + }; + assert_eq!(workers.len(), 1); + let mut last_applied = None; + use futures::FutureExt; + while let Some(update) = catalog.recv().now_or_never() { + match update.unwrap().payload { + protocol::subscription::SubscriptionEventPayload::WorkerUpserted { worker } => { + last_applied = Some(worker); + } + payload => panic!("unexpected catalog update: {payload:?}"), + } + } + assert!( + serde_json::to_value(last_applied.expect("restore must publish its catalog state")) + .unwrap() + == serde_json::to_value(&workers[0]).unwrap(), + "original catalog subscriber diverged after {fault:?}" + ); + drop(fresh_catalog); + drop(catalog); alert_marker(&old, "terminal-after-recovery"); let _observations = observe_publication_markers( &fixture, From 8d47d2670bba4999c15ddea3a57d650383364f14 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 14:43:38 +0900 Subject: [PATCH 21/28] fix: initialize reconciliation fault evidence in offline store construction --- crates/worker-runtime/src/test_support.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/worker-runtime/src/test_support.rs b/crates/worker-runtime/src/test_support.rs index ca9192f8f..229815241 100644 --- a/crates/worker-runtime/src/test_support.rs +++ b/crates/worker-runtime/src/test_support.rs @@ -378,6 +378,10 @@ impl StoppedWorkerTemplate { worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), #[cfg(test)] worker_reconcile_sync_fault: Default::default(), + #[cfg(test)] + worker_reconcile_read_fault: Default::default(), + #[cfg(test)] + worker_reconcile_counts: Default::default(), }; let result = (|| { let worker = PersistedWorkerRecord { From 6f5a42466137868b3032f7ba4e7a50efef49e96c Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 19:37:26 +0900 Subject: [PATCH 22/28] fix: reclaim stale allocation authority before restoring committed run artifacts --- crates/worker-runtime/src/worker_backend.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index da734f13a..927e26b1a 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -2922,8 +2922,15 @@ impl ProfileRuntimeWorkerFactory { .map_err(|_| "restore run artifacts cannot be validated")?; let path = worker::runtime::worker_allocation::default_allocation_path() .map_err(|_| "restore allocation authority unavailable")?; - let table = worker::runtime::worker_allocation::LockFileGuard::open(&path) + let mut table = worker::runtime::worker_allocation::LockFileGuard::open(&path) .map_err(|_| "restore allocation authority unavailable")?; + // Activated process death leaves a durable allocation entry. + // Use the same stale-owner authority as normal registration; + // a live or uninspectable owner must still fence reclamation. + worker::runtime::worker_allocation::reclaim_stale(&mut table); + table + .save() + .map_err(|_| "restore stale allocation cleanup unconfirmed")?; if table .data() .find(&Self::runtime_worker_name_for_ref(&request.worker_ref)) From b3209c2012f07a33cd87f436f78df41c6e88cefa Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 19:38:24 +0900 Subject: [PATCH 23/28] test: verify real orphan pin repair across owned restore process death --- .../worker-runtime/src/restore_checkpoints.rs | 1 + crates/worker-runtime/src/runtime.rs | 6 + .../src/runtime/restore_checkpoint.rs | 3 +- .../src/runtime/restore_checkpoint_tests.rs | 12 +- .../worker_backend_restore_candidate_tests.rs | 34 +++ tests/e2e/README.md | 4 +- tests/e2e/tests/worker_restore/checkpoint.rs | 2 + tests/e2e/tests/worker_restore/repair.rs | 246 ++++++++++++++++++ tests/e2e/tests/worker_restore/seed.rs | 18 +- 9 files changed, 321 insertions(+), 5 deletions(-) create mode 100644 tests/e2e/tests/worker_restore/repair.rs diff --git a/crates/worker-runtime/src/restore_checkpoints.rs b/crates/worker-runtime/src/restore_checkpoints.rs index 91321afa6..0232eb976 100644 --- a/crates/worker-runtime/src/restore_checkpoints.rs +++ b/crates/worker-runtime/src/restore_checkpoints.rs @@ -25,6 +25,7 @@ pub const MAX_RESTORE_CHECKPOINT_ACK_BYTES: usize = 4096; pub enum RestoreCheckpointStage { PreparedNotCommitted, CommittedPreterminal, + ActivatedPreterminal, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 755509f7b..8eef2a877 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -2871,6 +2871,12 @@ impl Runtime { sanitize_worker_create_failure_message(&error), )); } + #[cfg(feature = "restore-checkpoints")] + self.test_restore_checkpoint( + worker_ref, + operation_id, + crate::restore_checkpoints::RestoreCheckpointStage::ActivatedPreterminal, + ); if let Err(error) = self.activate_restore_candidate_context(worker_ref, &candidate_context) { diff --git a/crates/worker-runtime/src/runtime/restore_checkpoint.rs b/crates/worker-runtime/src/runtime/restore_checkpoint.rs index 96da2eeb9..d94aa4fa1 100644 --- a/crates/worker-runtime/src/runtime/restore_checkpoint.rs +++ b/crates/worker-runtime/src/runtime/restore_checkpoint.rs @@ -75,7 +75,8 @@ impl Runtime { }; let expected = match config.stage { RestoreCheckpointStage::PreparedNotCommitted => RestoreCommitFact::NotCommitted, - RestoreCheckpointStage::CommittedPreterminal => RestoreCommitFact::Committed, + RestoreCheckpointStage::CommittedPreterminal + | RestoreCheckpointStage::ActivatedPreterminal => RestoreCommitFact::Committed, }; if config.worker_id != worker_ref.worker_id || pending.operation_id != operation_id diff --git a/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs b/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs index 7369d6c6b..8103c2fe0 100644 --- a/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs +++ b/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs @@ -14,6 +14,7 @@ fn checkpoint_stages_ack_canonical_decision_and_eof_resumes_same_operation_once( (RestoreCheckpointStage::PreparedNotCommitted, "release"), (RestoreCheckpointStage::CommittedPreterminal, "eof"), (RestoreCheckpointStage::CommittedPreterminal, "timeout"), + (RestoreCheckpointStage::ActivatedPreterminal, "release"), ] { let dir = tempfile::tempdir().unwrap(); let backend = Arc::new(TestExecutionBackend::default()); @@ -79,14 +80,19 @@ fn checkpoint_stages_ack_canonical_decision_and_eof_resumes_same_operation_once( match stage { RestoreCheckpointStage::PreparedNotCommitted => RestoreCheckpointCommit::NotCommitted, - RestoreCheckpointStage::CommittedPreterminal => RestoreCheckpointCommit::Committed, + RestoreCheckpointStage::CommittedPreterminal + | RestoreCheckpointStage::ActivatedPreterminal => RestoreCheckpointCommit::Committed, } ); assert_eq!( backend.restore_operation_ids.lock().unwrap().as_slice(), &[ack.operation_id] ); - assert!(backend.activation_operation_ids.lock().unwrap().is_empty()); + if stage == RestoreCheckpointStage::ActivatedPreterminal { + assert_eq!(backend.activation_operation_ids.lock().unwrap().as_slice(), &[ack.operation_id]); + } else { + assert!(backend.activation_operation_ids.lock().unwrap().is_empty()); + } assert!(received.try_recv().is_err()); let canonical = runtime_store(&runtime) .read_durable_worker_record(&worker.worker_ref) @@ -255,6 +261,8 @@ fn checkpoint_verification_requires_canonical_worker_operation_commit_and_absent .restore_checkpoint_ack(&config, &worker.worker_ref, operation) .is_err() ); + config.stage = RestoreCheckpointStage::ActivatedPreterminal; + assert!(runtime.restore_checkpoint_ack(&config, &worker.worker_ref, operation).is_err()); config.stage = RestoreCheckpointStage::PreparedNotCommitted; runtime_store(&runtime).fail_next_worker_reconcile_sync(); assert!( diff --git a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs index 690a4025f..894beee9b 100644 --- a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs @@ -557,6 +557,40 @@ mod restore_candidate { } } + #[test] + #[serial_test::serial(worker_allocation)] + fn restore_run_reclamation_preserves_live_allocation_owner() { + let fixture = Fixture::no_model(); + let factory = fixture.backend.factory.clone(); + let request = fixture.request.clone(); + let run = factory + .worker_restore_run_dir(&request.worker_ref, request.operation_id) + .unwrap(); + fs::create_dir_all(&run).unwrap(); + fs::write(run.join("retained-owner"), b"live allocation").unwrap(); + let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&request.worker_ref); + let allocation = worker::runtime::worker_allocation::install_top_level( + name.clone(), std::process::id(), run.join("worker.sock"), + Vec::new(), fixture.segment_id, + ).unwrap(); + let failure = fixture.backend.run_on_adapter_runtime(async move { + match factory.prepare_restore(request).await { + Err(failure) => Ok(failure), + Ok(_) => Err("live allocation unexpectedly reclaimed".into()), + } + }).unwrap(); + assert_eq!(failure.stage, RuntimeRestoreStage::Controller); + assert_eq!(failure.cause, "restore run directory still has an allocation owner"); + let table = worker::runtime::worker_allocation::LockFileGuard::open( + &worker::runtime::worker_allocation::default_allocation_path().unwrap(), + ).unwrap(); + assert_eq!(table.data().find(&name).unwrap().pid, std::process::id()); + assert_eq!(fs::read(run.join("retained-owner")).unwrap(), b"live allocation"); + drop(table); + drop(allocation); + fixture.assert_quiet(); + } + #[test] #[serial_test::serial(worker_allocation)] fn session_availability_requires_attachable_current_controller() { diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 4622758ab..857a28a09 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -41,7 +41,7 @@ process終了ケースはそのAccepted receipt確認後に環境が所有する ## Opt-in Runtime checkpoint process試験 -上のfeatureless製品4case（target内の既存seed projection試験1件とは別）に加え、`restore-checkpoints`を明示した実Runtimeで2つのlive windowを試験する。このfeatureはoffline fixture-supportと独立しており、通常RuntimeではCLI入口も環境変数による入口も無い。 +上のfeatureless製品4case（target内の既存seed projection試験1件とは別）に加え、`restore-checkpoints`を明示した実Runtimeでdecision前後とactivation後・terminal前のlive windowを試験する。このfeatureはoffline fixture-supportと独立しており、通常RuntimeではCLI入口も環境変数による入口も無い。 ```sh # 同期を有効にしたRuntimeだけを専用targetへ。通常製品binaryを上書きしない。 @@ -55,6 +55,8 @@ cargo test --locked -p yoi-e2e --features e2e,restore-checkpoints \ 共有harnessはloopback ackのstage/Worker/operation/PID/nonceを確認し、直ちに所有childをkill/reapする。**offline正本を確認してから**同roots/identity/trustを再起動するため、ack後のdeadlineとの競合を成功扱いしない。NotCommittedの実私有候補は同operationのcleanup-only→RolledBack/Stopped、Committed/preterminalは同じdecision→実protocol/full Session snapshot→Acceptedへ回収する。元のrequest/token、owner数1、同operation、stale fresh intent拒否、保存source hash、restore run path数、最終cleanupを検証する。JSONでpending状態を作らず、seedは初回のみ。 +orphan upload pinの代表repairは、隔離copyへ正規Session storeのupload/pin APIで作った未結付けpinを、実Profileのcommit後activationが解放する経路を使う。Prepared窓は未修復のsource保持、Activated窓は実修復bytes/hashの保持を区別し、kill/reap後に正本を読んでから再起動する。materialized headの固定資産を巻き戻さないため、この試験は先行repair済みNotCommittedのprocess証明ではない。Activated後のstale allocationは通常登録と同じallocation authorityで回収し、live ownerが残る場合はrun artifactsを回収しない。 + 明示解除・peer EOF・observer scopeの意図的panicも試験する。gateの8秒deadlineとscoped HTTP callの15秒deadlineが待機を制限し、通常operationの継続後に環境のRAIIがprocess/rootを回収する。これはtest scope失敗の回収証明であり、OSによるharness強制終了や電源断の一般的な保証ではない。原fixtureへのSubmit/Notifyや実モデル呼出しは行わない。checkpoint ack/at-death/recovery、kill/reap、binary/hashとcleanup artifactはfeaturelessログと分けて報告する。 通常のharness回帰試験（製品processは起動しない）: diff --git a/tests/e2e/tests/worker_restore/checkpoint.rs b/tests/e2e/tests/worker_restore/checkpoint.rs index e55e815be..312c5beaa 100644 --- a/tests/e2e/tests/worker_restore/checkpoint.rs +++ b/tests/e2e/tests/worker_restore/checkpoint.rs @@ -1,4 +1,6 @@ //! Opt-in real-process checkpoint cases; never substitute the featureless lane. +#[path = "repair.rs"] +mod repair; use super::*; use std::collections::BTreeMap; use std::io::{Read, Write}; diff --git a/tests/e2e/tests/worker_restore/repair.rs b/tests/e2e/tests/worker_restore/repair.rs new file mode 100644 index 000000000..9e502d642 --- /dev/null +++ b/tests/e2e/tests/worker_restore/repair.rs @@ -0,0 +1,246 @@ +//! Real orphan-pin repair on an installed fixed-asset copy, not a synthetic +//! pending head. Only the postcommit activation path can reconcile these pins. +use super::*; +use session_store::{Store, UploadedFileLimits, WorkerSessionStore}; + +const PIN_OWNER: &str = "interrupted-pin-before-queue-checkpoint"; +const UPLOAD: &[u8] = b"isolated orphan upload; never model input"; + +#[test] +fn orphan_pin_not_committed_process_death_preserves_unrepaired_source() -> Result<()> { + repair_crash(RestoreCheckpointStage::PreparedNotCommitted) +} + +#[test] +fn orphan_pin_activated_process_death_preserves_real_repair_and_committed_decision() -> Result<()> { + repair_crash(RestoreCheckpointStage::ActivatedPreterminal) +} + +fn repair_crash(stage: RestoreCheckpointStage) -> Result<()> { + let fixture = Fixture::load()?; + let result = (|| { + let mut environment = WorkerE2eEnvironment::new()?; + let mut peer = Peer::configure(&mut environment, &fixture, stage)?; + let session_id = fixture::support(fixture.manifest.session_id.parse())?; + let mut upload = None; + let mut installed = seed::start_in_with_source_setup(&fixture, environment, |aggregate| { + let store = fixture::support(WorkerSessionStore::new(aggregate.join("session")))?; + let reference = fixture::support(store.write_uploaded_file( + session_id, + "orphan.txt", + "text/plain", + UPLOAD, + UploadedFileLimits::default(), + ))?; + fixture::support(store.pin_uploaded_file(session_id, &reference, PIN_OWNER))?; + upload = Some(reference); + Ok(()) + })?; + let upload = upload.ok_or_else(|| failure("offline upload setup not called"))?; + let relative = format!("session/artifacts/paste/{}.file.json", upload.artifact_id); + let pin_path = aggregate_path(&installed.environment, &fixture).join(&relative); + let pin_before: serde_json::Value = serde_json::from_slice(&fs::read(&pin_path)?)?; + assert_eq!(pin_before["pending_owner_id"], PIN_OWNER); + let before = saved_source_hashes(&installed.environment, &fixture)?; + let initial = canonical(&installed.environment, &fixture)?; + let token = assert_single_state(&installed.environment, &fixture, "stopped")? + .restore_observation_token + .ok_or_else(|| failure("missing original token"))?; + let request_id = "real-orphan-repair-crash-request"; + let committed = stage == RestoreCheckpointStage::ActivatedPreterminal; + let call = installed.environment.pending_restore_call( + &fixture.manifest.worker_id, + request_id, + &token, + )?; + let (ack, at_death) = std::thread::scope(|scope| -> Result<_> { + let request = scope.spawn(call); + let ack = peer.acknowledge(&installed.environment)?; + installed.environment.crash_runtime()?; + let saved = canonical(&installed.environment, &fixture)?; + let op = &saved["execution_state"]["execution"]["intent"]; + let owner = &saved["restore_guard"]["owners"][request_id]; + assert_eq!(saved["restore_guard"]["active_request_id"], request_id); + assert_eq!(op["operation_id"], serde_json::to_value(ack.operation_id)?); + assert_eq!(owner["operation_id"], op["operation_id"]); + assert_eq!(owner["request"]["expected_observation_token"], token); + assert!(owner["receipt"].is_null()); + assert_eq!( + op["commit"], + if committed { + "committed" + } else { + "not_committed" + } + ); + assert_eq!( + ack.commit, + if committed { + RestoreCheckpointCommit::Committed + } else { + RestoreCheckpointCommit::NotCommitted + } + ); + // The durable Committed decision authorizes activation effects. The + // precommit repair callback is not used for a materialized head. + assert_eq!(op["source_repair_possible"], false); + let at_death = saved_source_hashes(&installed.environment, &fixture)?; + let pin_after: serde_json::Value = serde_json::from_slice(&fs::read(&pin_path)?)?; + let mut expected_pin = pin_before.clone(); + if committed { + expected_pin + .as_object_mut() + .unwrap() + .remove("pending_owner_id"); + } + assert_eq!( + pin_after, expected_pin, + "real repair changes only orphan ownership" + ); + if committed { + assert_ne!( + at_death[&relative], before[&relative], + "actual pin bytes must change" + ); + } else { + assert_eq!( + at_death, before, + "preparation cannot repair pin or metadata" + ); + } + // In both windows the original conversation is immutable. + for (path, hash) in &before { + if path == "session/session.json" || path.starts_with("session/segments/") { + assert_eq!(at_death.get(path), Some(hash), "{path}"); + } + } + let store = fixture::support(WorkerSessionStore::open_read_only( + aggregate_path(&installed.environment, &fixture).join("session"), + ))?; + assert_eq!( + fixture::support(store.read_uploaded_file(session_id, &upload))?, + UPLOAD + ); + assert_eq!( + restore_runs(&installed.environment, &fixture)?, + vec![format!("restore-{}", ack.operation_id)] + ); + fs::write( + installed + .environment + .artifacts_dir + .join("repair-at-death.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "ack":ack, "initial_execution":initial["execution_state"], + "canonical_commit":op["commit"], "source_repair_possible":op["source_repair_possible"], + "operation_id":op["operation_id"], "owner_operation_id":owner["operation_id"], + "receipt_absent":owner["receipt"].is_null(), "original_request_and_token":true, + "pin_pending_owner_before":pin_before["pending_owner_id"], + "pin_pending_owner_after":pin_after["pending_owner_id"], + "hashes_before":before, "hashes_at_death":at_death, + "restore_runs":restore_runs(&installed.environment, &fixture)?, + }))?, + )?; + let outcome = request + .join() + .map_err(|_| failure("restore caller panicked"))?; + if let Ok(value) = outcome { + let receipt: WorkerRestoreResponse = serde_json::from_value(value)?; + assert_ne!(receipt.result.state, WorkerRestoreState::Accepted); + } + installed.environment.restart_runtime_after_crash()?; + assert_ne!(installed.environment.runtime_pid(), Some(ack.pid)); + Ok((ack, at_death)) + })?; + drop(peer); + let recovered = installed.environment.restore_worker( + &fixture.manifest.worker_id, + request_id, + &token, + )?; + let receipt: WorkerRestoreResponse = serde_json::from_value(recovered.clone())?; + fs::write( + installed + .environment + .artifacts_dir + .join("repair-first-recovery.json"), + serde_json::to_vec_pretty(&recovered)?, + )?; + assert_eq!( + receipt.result.state, + if committed { + WorkerRestoreState::Accepted + } else { + WorkerRestoreState::RolledBack + } + ); + assert_eq!( + installed.environment.restore_worker( + &fixture.manifest.worker_id, + request_id, + &token + )?, + recovered + ); + let final_saved = canonical(&installed.environment, &fixture)?; + let owners = final_saved["restore_guard"]["owners"] + .as_object() + .ok_or_else(|| failure("missing owners"))?; + assert_eq!(owners.len(), 1); + let owner = &owners[request_id]; + assert_eq!( + owner["operation_id"], + serde_json::to_value(ack.operation_id)? + ); + assert_eq!(owner["request"]["expected_observation_token"], token); + assert_eq!( + owner["receipt"]["state"], + if committed { "accepted" } else { "rolled_back" } + ); + let after = saved_source_hashes(&installed.environment, &fixture)?; + assert_eq!( + after, at_death, + "restart/replay cannot restore the orphan pin or change saved source" + ); + let stale = installed.environment.restore_worker( + &fixture.manifest.worker_id, + "fresh-stale-repair-intent", + &token, + ); + assert!(matches!(stale, Err(error) if error.to_string().contains("HTTP 409"))); + fs::write( + installed + .environment + .artifacts_dir + .join("repair-recovery.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "ack":ack, "state":receipt.result.state, "diagnostics":receipt.result.diagnostics, + "operation_id":owner["operation_id"], "owner_count":owners.len(), + "hashes_after_recovery":after, "restore_runs":restore_runs(&installed.environment, &fixture)?, + "same_original_request_and_token":true, "fresh_stale_rejected":true, + }))?, + )?; + if committed { + assert_single_state(&installed.environment, &fixture, "idle")?; + assert_eq!( + restore_runs(&installed.environment, &fixture)?, + vec![format!("restore-{}", ack.operation_id)] + ); + let mut subscription = WorkerSessionSubscription::connect( + &installed.environment, + &fixture.manifest.worker_id, + &installed.snapshot, + )?; + subscription.assert_alive()?; + stop_and_cleanup(installed, vec![subscription], &fixture) + } else { + assert_single_state(&installed.environment, &fixture, "stopped")?; + assert!(restore_runs(&installed.environment, &fixture)?.is_empty()); + let report = installed.environment.cleanup()?; + assert!(report.cleanup_success && !report.fixture_root_exists_after); + Ok(()) + } + })(); + fixture.source.assert_unchanged()?; + result +} diff --git a/tests/e2e/tests/worker_restore/seed.rs b/tests/e2e/tests/worker_restore/seed.rs index f95085b06..3640db189 100644 --- a/tests/e2e/tests/worker_restore/seed.rs +++ b/tests/e2e/tests/worker_restore/seed.rs @@ -45,7 +45,17 @@ pub fn start(fixture: &Fixture) -> Result { start_in(fixture, WorkerE2eEnvironment::new()?) } -pub fn start_in(fixture: &Fixture, mut environment: WorkerE2eEnvironment) -> Result { +pub fn start_in(fixture: &Fixture, environment: WorkerE2eEnvironment) -> Result { + start_in_with_source_setup(fixture, environment, |_| Ok(())) +} + +// The callback runs only on the installed isolation copy, while Runtime is offline. +// Restart never calls the seed or this setup again. +pub fn start_in_with_source_setup( + fixture: &Fixture, + mut environment: WorkerE2eEnvironment, + setup: impl FnOnce(&Path) -> Result<()>, +) -> Result { super::phase(&environment, "fixture-copy-verify")?; let copy = environment.root.join("provided-fixture"); fixture.source.copy_to(©)?; @@ -114,6 +124,12 @@ pub fn start_in(fixture: &Fixture, mut environment: WorkerE2eEnvironment) -> Res backend_job: None, }; support(template.install(&options, request, &bootstrap.manifest))?; + setup( + &options + .root + .join("workers") + .join(template.worker_id().to_string()), + )?; let sessions = support(session_store::WorkerSessionStore::open_read_only( options .root From 15c3006d09029ccdc9e437d9bc4fd55a576334b6 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 19:59:56 +0900 Subject: [PATCH 24/28] fix: propagate fallible restore publication through server factory adapters --- .../src/server/tests/restored_parent_control_tests.rs | 4 ++-- .../src/server/tests/subjektiv_journal_connected_tests.rs | 4 ++-- .../workspace-server/src/server_workspace_config_wip_tests.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs index d1a4d6c61..e8249cf8b 100644 --- a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs +++ b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs @@ -138,9 +138,9 @@ impl RuntimeWorkerFactory for CapturingFactory { worker_ref: &worker_runtime::identity::WorkerRef, workspace_id: Option<&str>, handle: &worker::WorkerHandle, - ) { + ) -> std::result::Result<(), String> { self.inner - .activate_restored_controller(worker_ref, workspace_id, handle); + .activate_restored_controller(worker_ref, workspace_id, handle) } } diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs index 3b9215113..7c5fe0442 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs @@ -1084,9 +1084,9 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for RestoreAuthFactory worker_ref: &worker_runtime::identity::WorkerRef, workspace_id: Option<&str>, handle: &worker::WorkerHandle, - ) { + ) -> std::result::Result<(), String> { self.inner - .activate_restored_controller(worker_ref, workspace_id, handle); + .activate_restored_controller(worker_ref, workspace_id, handle) } } diff --git a/crates/workspace-server/src/server_workspace_config_wip_tests.rs b/crates/workspace-server/src/server_workspace_config_wip_tests.rs index 9e970e17d..7c719a648 100644 --- a/crates/workspace-server/src/server_workspace_config_wip_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_wip_tests.rs @@ -636,9 +636,9 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto worker_ref: &worker_runtime::identity::WorkerRef, workspace_id: Option<&str>, handle: &worker::WorkerHandle, - ) { + ) -> std::result::Result<(), String> { self.inner - .activate_restored_controller(worker_ref, workspace_id, handle); + .activate_restored_controller(worker_ref, workspace_id, handle) } } From e84057f402649f7c4f2316db396fd96ea940d63e Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 20:39:28 +0900 Subject: [PATCH 25/28] fix: retain exact restore ownership in test execution backends --- crates/tui/src/backend_worker_picker.rs | 25 ++++++++++++++++++++++++ crates/workspace-server/src/server.rs | 26 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/crates/tui/src/backend_worker_picker.rs b/crates/tui/src/backend_worker_picker.rs index 7812f85e3..f8fa5336c 100644 --- a/crates/tui/src/backend_worker_picker.rs +++ b/crates/tui/src/backend_worker_picker.rs @@ -785,6 +785,12 @@ mod tests { #[derive(Default)] struct PendingRuntimeBackend { operations: std::sync::Mutex>, + owners: std::sync::Mutex< + std::collections::HashMap< + worker_runtime::execution::WorkerLifecycleOperationId, + worker_runtime::identity::WorkerRef, + >, + >, } impl worker_runtime::execution::WorkerExecutionBackend for PendingRuntimeBackend { @@ -806,6 +812,21 @@ mod tests { ) } + fn restore_recovery_disposition( + &self, + operation_id: worker_runtime::execution::WorkerLifecycleOperationId, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> Option { + use worker_runtime::execution::RestoreRecoveryDisposition as D; + Some( + if self.owners.lock().unwrap().get(&operation_id) == Some(worker_ref) { + D::Continue + } else { + D::Absent + }, + ) + } + fn restore_worker( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, @@ -813,6 +834,10 @@ mod tests { use worker_runtime::execution::*; let mut operations = self.operations.lock().unwrap(); operations.push(request.operation_id); + self.owners + .lock() + .unwrap() + .insert(request.operation_id, request.worker_ref.clone()); if operations.len() == 1 { WorkerExecutionSpawnResult::ReconciliationRequired { result: WorkerExecutionResult::errored( diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index a8ddde657..175feb3e8 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -48392,6 +48392,12 @@ mod tests { std::sync::Mutex>, reconcile_operations: std::sync::Mutex>, + restore_owners: std::sync::Mutex< + std::collections::HashMap< + worker_runtime::execution::WorkerLifecycleOperationId, + worker_runtime::identity::WorkerRef, + >, + >, inputs: std::sync::Mutex>, protocol_methods: Arc>>, @@ -48425,6 +48431,7 @@ mod tests { restore_failure: std::sync::Mutex::new(None), restore_operations: std::sync::Mutex::new(Vec::new()), reconcile_operations: std::sync::Mutex::new(Vec::new()), + restore_owners: Default::default(), inputs: std::sync::Mutex::new(Vec::new()), protocol_methods: Arc::new(std::sync::Mutex::new(Vec::new())), stops: std::sync::Mutex::new(Vec::new()), @@ -48629,6 +48636,10 @@ mod tests { .lock() .unwrap() .push(request.operation_id); + self.restore_owners + .lock() + .unwrap() + .insert(request.operation_id, request.worker_ref.clone()); if let Some(result) = self.restore_failure.lock().unwrap().take() { return result; } @@ -48664,6 +48675,21 @@ mod tests { } } + fn restore_recovery_disposition( + &self, + operation_id: worker_runtime::execution::WorkerLifecycleOperationId, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> Option { + use worker_runtime::execution::RestoreRecoveryDisposition as D; + Some( + if self.restore_owners.lock().unwrap().get(&operation_id) == Some(worker_ref) { + D::Continue + } else { + D::Absent + }, + ) + } + fn reconcile_restore( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, From eb24cb04dca26c9b1bded604fca4afd8459e8d0c Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 20:39:28 +0900 Subject: [PATCH 26/28] fix: revalidate Host authority when publishing prepared restore controllers --- crates/worker/src/bootstrap.rs | 7 ++ .../subjektiv_journal_connected_tests.rs | 64 +++++++++++++++++-- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/crates/worker/src/bootstrap.rs b/crates/worker/src/bootstrap.rs index 2a591cb3f..cb712e11d 100644 --- a/crates/worker/src/bootstrap.rs +++ b/crates/worker/src/bootstrap.rs @@ -465,6 +465,13 @@ where WorkerError::RestoreCandidateClosing, )); } + // Preparation's read-only authorization is not a publication grant. + // Revalidate current Host authority after commit, before effects/input, + // including a publication retry on this same private Controller. + initialized + .worker + .validate_subjektiv_body_publication() + .map_err(WorkerBootstrapError::Worker)?; if !self.effects_applied { self.activation_failed = true; self.activation diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs index 7c5fe0442..cc8895777 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs @@ -1088,6 +1088,14 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for RestoreAuthFactory self.inner .activate_restored_controller(worker_ref, workspace_id, handle) } + + fn deactivate_stopped_controller( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + handle: &worker::WorkerHandle, + ) -> std::result::Result<(), String> { + self.inner.deactivate_stopped_controller(worker_ref, handle) + } } struct RestoreAuthCandidate { @@ -1101,6 +1109,14 @@ impl worker_runtime::worker_backend::RuntimeRestoreCandidate for RestoreAuthCand self.inner.initial_state() } + fn has_durable_effects(&self) -> bool { + self.inner.has_durable_effects() + } + + fn can_retry_publication(&self) -> bool { + self.inner.can_retry_publication() + } + async fn activate( &mut self, ) -> std::result::Result< @@ -1346,6 +1362,32 @@ enabled = false ) -> worker_runtime::execution::WorkerExecutionSpawnResult { self.inner.restore_worker(request) } + fn restore_recovery_disposition( + &self, + id: worker_runtime::execution::WorkerLifecycleOperationId, + worker: &worker_runtime::identity::WorkerRef, + ) -> Option { + self.inner.restore_recovery_disposition(id, worker) + } + // This caller-replay fixture gates repair/prepare to inspect the durable + // lost-response state before its next explicit call. Runtime continuation + // itself is covered with the unwrapped backend in the Runtime suite. + fn supports_restore_continuation(&self) -> bool { + false + } + fn reclaim_uncommitted_restore( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.inner.reclaim_uncommitted_restore(request) + } + fn discard_uncommitted_restore( + &self, + id: worker_runtime::execution::WorkerLifecycleOperationId, + worker: &worker_runtime::identity::WorkerRef, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.inner.discard_uncommitted_restore(id, worker) + } fn reconcile_restore( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, @@ -1798,6 +1840,7 @@ async fn restore_after_real_prepare_rejects_revoked_host_trust_without_publicati "the same production signer must be accepted before trust revocation" ); requests.lock().unwrap().clear(); + fixture.callbacks.lock().unwrap().clear(); let runtime = api.runtime.clone(); let target = worker.clone(); let intent = request.clone(); @@ -1816,13 +1859,20 @@ async fn restore_after_real_prepare_rejects_revoked_host_trust_without_publicati "preparation must not consume saved input or append history" ); assert_eq!(metadata.read_by_name(&name).unwrap().unwrap(), saved); - assert!( - requests - .lock() - .unwrap() - .iter() - .all(|(path, _)| !path.ends_with("/subjektiv/sessions")), - "publication authorization belongs after candidate preparation" + { + let callbacks = fixture.callbacks.lock().unwrap(); + assert_eq!( + callbacks.len(), + 1, + "prepare verifies current Host authority" + ); + assert_eq!(callbacks[0]["create_if_missing"], false); + assert_eq!(callbacks[0]["session_id"], active.session_id.to_string()); + } + assert_eq!(probe.publications.load(SeqCst), 0); + assert_eq!( + store.subject_session_attributions(subject_id).unwrap(), + attributions ); let prepared_record: Value = serde_json::from_slice(&std::fs::read(aggregate.join("worker.json")).unwrap()).unwrap(); From e8d4e964cb1943402671f9e10834efd0cf983b1c Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 21:25:46 +0900 Subject: [PATCH 27/28] fix: refresh Workdir API generated contract digest --- web/workspace/src/lib/generated/workdir-api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/workspace/src/lib/generated/workdir-api.ts b/web/workspace/src/lib/generated/workdir-api.ts index 99db0b0a5..04bfe3967 100644 --- a/web/workspace/src/lib/generated/workdir-api.ts +++ b/web/workspace/src/lib/generated/workdir-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-workdir-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7) +// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e) // Options: workdir-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/workdir-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_workdir_api_types -- --check From dddb21bd70f4097073712c1e5cb47bb4048b8361 Mon Sep 17 00:00:00 2001 From: Hare Date: Tue, 27 Oct 2026 01:24:50 +0900 Subject: [PATCH 28/28] fix: preserve possible repair during v9 Restore migration --- crates/worker-runtime/src/fs_store.rs | 131 ++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index 1278f89f7..cbc549383 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -1245,6 +1245,7 @@ fn migrate_worker_record( migrate_legacy_request_evidence(&mut identity_document) .map_err(|message| runtime_store_corrupt(identity_path, message.into()))?; migrate_restore_commit_fact(&mut identity_document, identity_path)?; + migrate_restore_repair_fact(&mut identity_document, identity_path)?; identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION); let aggregate: WorkerAggregateRecord = serde_json::from_value(identity_document).map_err(|error| { @@ -4097,6 +4098,136 @@ mod tests { (temp, opened.store, worker, path) } + #[test] + fn schema_v9_pending_restore_migrates_conservatively_and_survives_reopen() { + assert_v9_pending_restore_migration(false); + } + + #[test] + fn schema_v9_pending_stop_restore_migrates_conservatively_and_survives_reopen() { + assert_v9_pending_restore_migration(true); + } + + fn assert_v9_pending_restore_migration(under_stop: bool) { + // Real store migration/open/writer/reopen, with frozen v9 operation and + // admission shapes. No execution backend or recovery is substituted. + let (_temp, store, worker, path) = canonical_worker_store(); + let root = store.root.clone(); + let mut legacy = + serde_json::to_value(WorkerAggregateRecord::from_persisted(&worker)).unwrap(); + let execution = legacy["execution_state"]["execution"].clone(); + let restore_id = WorkerLifecycleOperationId::new(); + let restore = serde_json::json!({ + "operation_id": restore_id, + "mode": "explicit", + "request": execution["request"], + "binding": execution["binding"], + "last_settled_status": legacy["status"] + }); + let operation = if under_stop { + serde_json::json!({"operation": "stop", "intent": { + "operation_id": WorkerLifecycleOperationId::new(), + "pending_restore": restore, + "request": execution["request"], + "binding": execution["binding"], + "restore_intent": execution["restore_intent"], + "last_settled_status": legacy["status"] + }}) + } else { + serde_json::json!({"operation": "restore", "intent": restore}) + }; + legacy["execution_state"] = serde_json::json!({ + "state": "reconciliation_required", "execution": operation + }); + let token = legacy["restore_guard"]["observation_token"].clone(); + let owner = |request_id: &str, operation_id, receipt| { + serde_json::json!({ + "fingerprint": format!("intent-{request_id}"), + "operation_id": operation_id, + "request": {"expected_observation_token": token, "request_id": request_id}, + "preparation_applied": true, + "execution_started": true, + "receipt": receipt + }) + }; + legacy["restore_guard"] = serde_json::json!({ + "generation": token, + "active_request_id": "pending-restore", + "owners": { + "pending-restore": owner("pending-restore", serde_json::to_value(restore_id).unwrap(), serde_json::Value::Null), + "earlier-restore": owner("earlier-restore", serde_json::to_value(WorkerLifecycleOperationId::new()).unwrap(), + serde_json::json!({"state": "rolled_back", "worker": null, + "reason_code": "worker_restore_failed", "message": "retained failure"})) + } + }); + legacy["schema_version"] = serde_json::json!(AGGREGATE_SCHEMA_VERSION); + fs::write(&path, serde_json::to_vec(&legacy).unwrap()).unwrap(); + write_empty_schema_v8_store(&root); + let runtime_path = root.join(RUNTIME_FILE); + let mut runtime: serde_json::Value = read_json(&runtime_path, "read fixture").unwrap(); + runtime["schema_version"] = serde_json::json!(AGGREGATE_SCHEMA_VERSION); + fs::write(&runtime_path, serde_json::to_vec(&runtime).unwrap()).unwrap(); + let metadata_path = path.with_file_name(WORKER_METADATA_FILE); + let history_path = path.parent().unwrap().join("session/history.json"); + fs::create_dir_all(history_path.parent().unwrap()).unwrap(); + let metadata = serde_json::to_vec(&serde_json::json!({ + "worker_name": runtime_worker_name(worker.worker_id), + "retained_extension": {"source": "unchanged"} + })) + .unwrap(); + let history = b"[{\"role\":\"user\",\"content\":\"retained source\"}]\n"; + fs::write(&metadata_path, &metadata).unwrap(); + fs::write(&history_path, history).unwrap(); + drop(store); + + let mut expected = legacy; + expected["schema_version"] = serde_json::json!(SCHEMA_VERSION); + let guard = expected["restore_guard"].as_object_mut().unwrap(); + let token = guard.remove("generation").unwrap(); + guard.insert("observation_token".into(), token); + let intent = &mut expected["execution_state"]["execution"]["intent"]; + let restore = if under_stop { + &mut intent["pending_restore"] + } else { + intent + }; + restore["commit"] = serde_json::json!("legacy_unknown"); + restore["source_repair_possible"] = serde_json::json!(true); + + let opened = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); + let persisted = opened.store.read_worker_record(&worker.worker_ref).unwrap(); + // Compare the persisted aggregate, including both operation IDs, full + // request/source references, token, pending owner and terminal receipt. + assert_eq!( + read_json::(&path, "read migrated Worker").unwrap(), + expected + ); + opened.store.write_worker_record(&persisted).unwrap(); + assert_eq!( + read_json::(&path, "read written Worker").unwrap(), + expected + ); + drop(opened); + let bytes = fs::read(&path).unwrap(); + let reopened = FsRuntimeStore::open_or_create(root, "test-runtime").unwrap(); + reopened + .store + .read_worker_record(&worker.worker_ref) + .unwrap(); + assert_eq!( + read_json::(&path, "read reopened Worker").unwrap(), + expected + ); + assert_eq!(fs::read(&path).unwrap(), bytes); + // v9 metadata is canonically reformatted; its retained values, unlike + // whitespace, are the preservation contract. + assert_eq!( + read_json::(&metadata_path, "read retained metadata").unwrap(), + serde_json::from_slice::(&metadata).unwrap() + ); + assert_eq!(fs::read(&history_path).unwrap(), history); + } + #[test] fn legacy_pending_restore_migrates_as_unknown_and_preserves_stop_ownership() { use crate::runtime::RestoreCommitFact;