From 73a3e674c9656f419632a7d8be970ff5440a917e Mon Sep 17 00:00:00 2001 From: Hare Date: Thu, 22 Oct 2026 22:58:57 +0900 Subject: [PATCH] feat(worker): separate restored candidates, repair, and activation --- .../session-store/src/worker_session_store.rs | 15 + crates/worker-runtime/src/fs_store.rs | 21 +- crates/worker-runtime/src/lib.rs | 3 + .../src/retention/runtime_v5_tests.rs | 6 +- crates/worker-runtime/src/runtime.rs | 926 ++++++---- crates/worker-runtime/src/worker_backend.rs | 1141 ++++++++---- .../src/worker_backend/restore_recovery.rs | 173 ++ .../worker_backend_restore_candidate_tests.rs | 898 ++++++++++ .../worker_backend_restore_failure_tests.rs | 340 +++- .../worker-runtime/src/working_directory.rs | 27 +- crates/worker/src/bootstrap.rs | 33 + crates/worker/src/controller.rs | 29 + crates/worker/src/discovery.rs | 11 +- crates/worker/src/lib.rs | 3 + crates/worker/src/worker.rs | 1034 ++++++++++- crates/worker/tests/controller_test.rs | 37 +- crates/workspace-server/src/lib.rs | 3 + .../tests/restored_parent_control_tests.rs | 90 +- .../subjektiv_journal_connected_tests.rs | 1580 ++++++++++++++++- .../src/server_assignment_attention_tests.rs | 60 +- .../src/server_assignment_delivery_tests.rs | 142 +- .../src/server_workspace_config_wip_tests.rs | 89 +- deno.json | 4 +- tests/support/restore_environment.rs | 129 ++ tools/test.test.ts | 369 +++- tools/test.ts | 216 ++- 26 files changed, 6592 insertions(+), 787 deletions(-) create mode 100644 crates/worker-runtime/src/worker_backend/restore_recovery.rs create mode 100644 crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs create mode 100644 tests/support/restore_environment.rs diff --git a/crates/session-store/src/worker_session_store.rs b/crates/session-store/src/worker_session_store.rs index a19f1b2db..3a5dad1e8 100644 --- a/crates/session-store/src/worker_session_store.rs +++ b/crates/session-store/src/worker_session_store.rs @@ -194,6 +194,21 @@ impl WorkerSessionStore { Ok(store) } + /// Open a current retained Session without creating or migrating anything. + /// The live constraint is checked here and again by every later publication; + /// attaching the control store itself does not acquire publication authority. + pub fn open_existing_with_retention( + root: impl Into, + retention: crate::retention::SessionRetentionStore, + ) -> Result { + let mut store = Self::open_read_only(root)?; + if let Some(session) = *store.session_id.lock().unwrap() { + retention.ensure_live(session)?; + } + store.retention = Some(retention); + Ok(store) + } + fn lock_publication( &self, session: SessionId, diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index de799b323..21ecfeca8 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -2301,9 +2301,28 @@ impl WorkerAggregateRecord { }); } if let Some(request) = request { + let mut validation_request = request.clone(); + // Pending admission may supply a missing Workspace connection + // without applying it to the saved execution. Validate that + // exact owner's intent against identity; do not widen the + // Available-record rule or mutate the canonical request. + if validation_request.workspace_api.is_none() + && let PersistedWorkerLifecycleOperation::Restore(restore) = &operation + && let Some(api) = restore_guard + .pending_preparation(restore.operation_id) + .and_then(|preparation| preparation.workspace_api.as_ref()) + { + validation_request.workspace_api = + Some( + serde_json::from_value(serde_json::to_value(api).map_err( + |error| runtime_store_corrupt(path, error.to_string()), + )?) + .map_err(|error| runtime_store_corrupt(path, error.to_string()))?, + ); + } WorkerExecutionRecord { schema_version: SCHEMA_VERSION, - request: request.clone(), + request: validation_request, binding: binding.clone(), restore_intent, } diff --git a/crates/worker-runtime/src/lib.rs b/crates/worker-runtime/src/lib.rs index 1e160a6f5..f4a5b02ff 100644 --- a/crates/worker-runtime/src/lib.rs +++ b/crates/worker-runtime/src/lib.rs @@ -16,6 +16,9 @@ pub mod management; pub mod observation; pub mod profile_archive; pub mod resource; +#[cfg(test)] +#[path = "../../../tests/support/restore_environment.rs"] +mod restore_test_environment; #[cfg(feature = "fs-store")] pub mod retention; mod runtime; diff --git a/crates/worker-runtime/src/retention/runtime_v5_tests.rs b/crates/worker-runtime/src/retention/runtime_v5_tests.rs index d9a3bccdd..47d75c940 100644 --- a/crates/worker-runtime/src/retention/runtime_v5_tests.rs +++ b/crates/worker-runtime/src/retention/runtime_v5_tests.rs @@ -682,8 +682,10 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() { #[cfg(unix)] #[test] fn current_runtime_session_migration_still_rejects_live_worker_sockets() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().join("runtime"); + // The same TempDir owns the relative bind/connect reference. No cwd change + // or alias is needed, and the real live-socket migration refusal stays tested. + let temp = tempfile::tempdir_in(".").unwrap(); + let root = Path::new(temp.path().file_name().unwrap()).join("runtime"); let (worker, _, _) = runtime_fixture(&root, 12); let before = tree_bytes(&root); let runs = root.join("workers").join(worker.to_string()).join("runs"); diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 19c17b053..eaec092dc 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -1886,70 +1886,7 @@ impl Runtime { let previous = state.worker(worker_ref)?.clone(); { let worker = state.worker_mut(worker_ref)?; - if !attachments.is_empty() && worker.workspace_id.is_none() { - return Err(RuntimeError::InvalidRequest( - "logical Workdir attachments require Workspace authority".to_string(), - )); - } - if worker.request.is_none() { - return Err(RuntimeError::WorkerExecutionUnavailable { - worker_id: worker.worker_id, - message: "persisted Worker restore request is unavailable".to_string(), - }); - } - let logical_by_alias = attachments - .iter() - .map(|attachment| { - ( - attachment.alias.clone(), - attachment.working_directory_id.as_str(), - ) - }) - .collect::>(); - for local in &worker.workdir_attachments { - let local_id = local - .working_directory - .summary - .working_directory_id - .as_str(); - if let Some(logical_id) = logical_by_alias.get(&local.alias) - && *logical_id != local_id - { - return Err(RuntimeError::InvalidRequest(format!( - "logical Workdir attachment alias `{}` conflicts with Runtime-local Workdir {}", - local.alias, local_id - ))); - } - } - let retained_local = worker - .workdir_attachments - .iter() - .filter_map(|local| { - let local_id = local - .working_directory - .summary - .working_directory_id - .as_str(); - (logical_by_alias.get(&local.alias).copied() == Some(local_id)) - .then(|| (local.alias.clone(), local_id.to_string())) - }) - .collect::>(); - worker - .workdir_attachments - .retain(|local| retained_local.contains_key(&local.alias)); - let request = worker - .request - .as_mut() - .expect("restore request existence checked above"); - request - .workdir_attachment_requests - .retain(|local| retained_local.contains_key(&local.alias)); - request.workdir_attachments.retain(|local| { - retained_local - .get(&local.alias) - .is_some_and(|workdir_id| workdir_id == &local.working_directory_id) - }); - worker.logical_workdir_attachments = attachments; + worker.apply_logical_workdir_attachments(attachments)?; } if let Err(error) = state.persist_worker(&worker_ref.worker_id) { state.workers.insert(worker_ref.worker_id, previous); @@ -2417,54 +2354,6 @@ impl Runtime { return Ok(result); } } - if !owner.preparation_applied { - let uncertain = |message: String| { - RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::ReconciliationRequired, - "worker_restore_preparation_reconciliation_required", - message, - ) - }; - // Apply reference updates under the original pending_restore ID; no provider is opened. - let mut effects_started = false; - if let Err(error) = self.apply_restore_preparation_under_lock( - worker_ref, - owner.request.preparation, - owner.operation_id, - &mut effects_started, - ) { - if effects_started - || matches!(error, RuntimeError::StoreCommitOutcomeUnknown { .. }) - { - return Ok(uncertain(sanitize_worker_create_failure_message( - &error.to_string(), - ))); - } - let result = RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::Rejected, - "worker_restore_preparation_rejected", - sanitize_worker_create_failure_message(&error.to_string()), - ); - self.finish_restore_receipt(worker_ref, &result)?; - return Ok(result); - } - let mut state = self.lock()?; - let mut candidate = state.worker(worker_ref)?.clone(); - candidate - .restore_guard - .owners - .get_mut(&owner.request.request_id) - .expect("admitted owner") - .preparation_applied = true; - if let Err(error) = state.persist_worker_record(&candidate) { - // Keep the started (not applied) record in memory. Even a - // known failed completion write follows possible effects. - return Ok(uncertain(sanitize_worker_create_failure_message( - &error.to_string(), - ))); - } - state.workers.insert(worker_ref.worker_id, candidate); - } } let result = self.restore_worker_under_lock(worker_ref, mode)?; self.finish_restore_receipt(worker_ref, &result)?; @@ -2481,95 +2370,18 @@ impl Runtime { } let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); - if candidate.restore_guard.active_request_id.is_some() { + if candidate.restore_guard.active_request_id.is_some() + || candidate.pending_restore.is_some() + { candidate.restore_guard.finish(result.clone()); candidate.pending_restore = None; + candidate.restore_candidate_context = None; state.persist_worker_record(&candidate)?; state.workers.insert(worker_ref.worker_id, candidate); - } - Ok(()) - } - - fn apply_restore_preparation_under_lock( - &self, - worker_ref: &WorkerRef, - preparation: Option, - operation_id: WorkerLifecycleOperationId, - effects_started: &mut bool, - ) -> Result<(), RuntimeError> { - let Some(preparation) = preparation else { - return Ok(()); - }; - // Validate all read-only structure before any binding effect. - if let Some(attachments) = &preparation.workdir_attachments { - let attachments: Vec = serde_json::from_value( - serde_json::to_value(attachments) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, - ) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; - validate_logical_workdir_attachments(&attachments)?; - } - // DTO conversion and every Runtime preparation effect occur AFTER admission. - if let Some(api) = preparation.workspace_api { - let api: WorkspaceApiRef = serde_json::from_value( - serde_json::to_value(api) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, - ) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; - if self.lock()?.worker(worker_ref)?.workspace_id.as_deref() - != Some(api.workspace_id.as_str()) - { - return Err(RuntimeError::InvalidRequest( - "Prepared Workspace API does not match Worker Workspace identity".into(), - )); - } - let mut state = self.lock()?; - let mut candidate = state.worker(worker_ref)?.clone(); - if candidate.pending_stop.is_some() - || candidate - .pending_restore - .is_none_or(|pending| pending.operation_id != operation_id) - { - return Err(RuntimeError::RestoreObservationConflict { - worker_id: worker_ref.worker_id, - }); - } - let persisted = candidate.request.as_mut().ok_or_else(|| { - RuntimeError::InvalidRequest("Restore specification is missing".into()) - })?; - if persisted.workspace_api.as_ref().is_some_and(|existing| { - existing.workspace_id != api.workspace_id - || existing.base_url.trim_end_matches('/') != api.base_url.trim_end_matches('/') - }) { - return Err(RuntimeError::InvalidRequest( - "Workspace API preparation cannot change Worker identity or base URL".into(), - )); - } - persisted.workspace_api = Some(api); - *effects_started = true; - match state.persist_worker_record(&candidate) { - Ok(()) => { - state.workers.insert(worker_ref.worker_id, candidate); - } - Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { - state.workers.insert(worker_ref.worker_id, candidate); - return Err(error); - } - Err(error) => return Err(error), + if result.state == WorkerRestoreState::Accepted { + state.publish_worker_upsert(worker_ref.worker_id)?; } } - if let Some(attachments) = preparation.workdir_attachments { - let attachments: Vec = serde_json::from_value( - serde_json::to_value(attachments) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, - ) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; - validate_logical_workdir_attachments(&attachments)?; - *effects_started = true; - self.replace_worker_workdir_attachments_under_lock(worker_ref, attachments, true)?; - } - // Retained legacy SSH preparation is not a prerequisite or an effect - // of restoring a Worker. Sessions resolve live access when used. Ok(()) } @@ -2621,6 +2433,28 @@ impl Runtime { { return Ok(RuntimeWorkerRestoreResult::accepted(worker.detail())); } + let preparation = worker + .restore_guard + .active_request_id + .as_ref() + .and_then(|id| worker.restore_guard.owners.get(id)) + .filter(|owner| !owner.preparation_applied) + .and_then(|owner| owner.request.preparation.as_ref()); + let prepared_worker = match worker.prepared_restore_record(preparation) { + Ok(prepared) => prepared, + Err(error) => { + return Ok(RuntimeWorkerRestoreResult::failed( + if reconcile { + WorkerRestoreState::ReconciliationRequired + } else { + WorkerRestoreState::Rejected + }, + "worker_restore_preparation_rejected", + sanitize_worker_create_failure_message(&error.to_string()), + )); + } + }; + let worker = &prepared_worker; let Some(worker_request) = worker.request.clone() else { return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::Rejected, @@ -2645,6 +2479,16 @@ impl Runtime { .restore_candidate_context .clone() .unwrap_or_else(|| self.restore_candidate_execution_context(worker_ref.clone())); + // First-head repair needs the original launch Prompt material. Use + // only the exact cached snapshot, never current Profile resolution; + // ordinary saved-head restore still works without that cache. + let config_bundle = worker_request.config_bundle.as_ref().and_then(|reference| { + state + .config_bundles + .get(&reference.id) + .filter(|bundle| bundle.metadata.digest == reference.digest) + .cloned() + }); let request = WorkerExecutionRestoreRequest { operation_id, worker_ref: worker_ref.clone(), @@ -2655,7 +2499,7 @@ impl Runtime { logical_workdir_attachments: worker.logical_workdir_attachments.clone(), workdir_attachments: BTreeMap::new(), deferred_workdir_sessions: BTreeMap::new(), - config_bundle: None, + config_bundle, }; ( backend, @@ -2746,11 +2590,15 @@ impl Runtime { if let Err(error) = backend.activate_restored_worker(operation_id, worker_ref) { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %error, - "Worker restore committed but Backend observation activation failed" - ); + self.retain_restore_reconciliation_pending( + worker_ref, + candidate_context.clone(), + )?; + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_activation_pending", + sanitize_worker_create_failure_message(&error), + )); } if let Err(error) = self.activate_restore_candidate_context(worker_ref, &candidate_context) @@ -2766,13 +2614,21 @@ impl Runtime { sanitize_worker_create_failure_message(&error.to_string()), )); } - Ok(RuntimeWorkerRestoreResult::accepted(worker)) + let result = RuntimeWorkerRestoreResult::accepted(worker); + if let Err(error) = self.finish_restore_receipt(worker_ref, &result) { + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_receipt_commit_pending", + sanitize_worker_create_failure_message(&error.to_string()), + )); + } + Ok(result) } - Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + Err(error) => { tracing::error!( worker_id = %worker_ref.worker_id, error = %error, - "Worker restore commit outcome is unknown; preserving execution evidence" + "Worker restore activated but commit did not confirm success; preserving operation and execution" ); if let Err(retain_error) = self.retain_restore_execution_evidence( worker_ref, @@ -2783,70 +2639,15 @@ impl Runtime { tracing::error!( worker_id = %worker_ref.worker_id, error = %retain_error, - "failed to persist restore execution evidence after an uncertain commit" + "failed to persist restore execution evidence; original operation remains pending" ); } Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, - "worker_restore_commit_outcome_unknown", - "Worker restore commit outcome is uncertain; reread or retry the Worker restore", + "worker_restore_commit_reconciliation_required", + sanitize_worker_create_failure_message(&error.to_string()), )) } - Err(error) => match self.cleanup_failed_restore(&backend, worker_ref) { - Ok(()) => { - self.discard_restore_candidate_context(worker_ref, &candidate_context)?; - tracing::warn!( - worker_id = %worker_ref.worker_id, - error = %error, - "Worker restore commit failed and was rolled back" - ); - let receipt = RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::RolledBack, - "worker_restore_commit_rolled_back", - "Worker restore commit failed and cleanup completed", - ); - if let Err(settle_error) = - self.settle_restore_rollback(worker_ref, &receipt) - { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %settle_error, - "Worker restore cleanup completed but its durable operation could not be settled" - ); - return Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::ReconciliationRequired, - "worker_restore_rollback_commit_failed", - "Worker restore cleanup completed but durable reconciliation is still required", - )); - } - Ok(receipt) - } - Err(cleanup_error) => { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %error, - cleanup_error = %cleanup_error, - "Worker restore requires reconciliation after commit and cleanup failures" - ); - if let Err(retain_error) = self.retain_restore_execution_evidence( - worker_ref, - worker_state, - workdir_attachments, - candidate_context.clone(), - ) { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %retain_error, - "failed to persist restore execution evidence; in-process reconciliation remains required" - ); - } - Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::ReconciliationRequired, - "worker_restore_reconciliation_required", - "Worker restore result is uncertain; reread or retry the Worker restore", - )) - } - }, } } WorkerExecutionSpawnResult::Rejected(result) => { @@ -2962,13 +2763,11 @@ impl Runtime { fn activate_restore_candidate_context( &self, - worker_ref: &WorkerRef, + _worker_ref: &WorkerRef, context: &crate::execution::WorkerExecutionContext, ) -> Result<(), RuntimeError> { + // Retain the publication owner until terminal receipt persistence succeeds. context.activate_candidate()?; - self.lock()? - .worker_mut(worker_ref)? - .restore_candidate_context = None; Ok(()) } @@ -4581,13 +4380,43 @@ impl Runtime { ) -> Result { let mut state = self.lock()?; state.ensure_worker_ref(worker_ref)?; - let mut candidate = state.worker(worker_ref)?.clone(); + let previous = state.worker(worker_ref)?; + if previous.pending_stop.is_some() { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + let mut candidate = if let Some(id) = &previous.restore_guard.active_request_id { + let owner = previous.restore_guard.owners.get(id).ok_or_else(|| { + RuntimeError::InvalidRequest("Active Restore request has no persisted owner".into()) + })?; + if previous + .pending_restore + .is_none_or(|pending| pending.operation_id != owner.operation_id) + { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + let mut candidate = + previous.prepared_restore_record(owner.request.preparation.as_ref())?; + candidate + .restore_guard + .owners + .get_mut(id) + .expect("admitted owner") + .preparation_applied = true; + candidate + } else { + previous.clone() + }; candidate.restore_guard.rotate(); candidate.execution_metadata_available = true; candidate.execution_bound = true; - candidate.pending_restore = None; + // Keep pending/owner until buffered observations also activate. Otherwise + // a retry could replay Accepted after a failed publication activation. + // Activation follows this pending decision; terminal success is still absent. candidate.pending_stop = None; - // Keep the actual publication owner until buffered observations activate. candidate.status = status; let mut worker_state = worker_state; if worker_state.last_finished_submission_request_id.is_none() { @@ -4600,6 +4429,9 @@ impl Runtime { candidate.last_finished_submission_request_id.clone(); } let _ = candidate.apply_worker_state(&worker_state); + if let Some(pending) = candidate.pending_restore.as_mut() { + pending.last_settled_status = candidate.status; + } candidate.restore_intent = restore_intent_for_status(candidate.status); candidate.workdir_attachments = workdir_attachments; @@ -4607,17 +4439,14 @@ impl Runtime { // subsequent publish uses the same pure projection while the Worker // operation lock keeps this candidate stable. candidate.restore_guard.rotate(); - let accepted = RuntimeWorkerRestoreResult::accepted(candidate.detail()); - candidate.restore_guard.finish(accepted); let _subscription_projection = state.subscription_worker(&candidate)?; - // Persist the complete candidate before making it observable. A failed - // write therefore leaves both the in-memory catalog and subscriptions - // on the pre-restore record, so cleanup can produce a true rollback. + // Persist the execution projection before publishing observations. Terminal + // receipt settlement follows successful observation activation separately. state.persist_worker_record(&candidate)?; let detail = candidate.detail(); state.workers.insert(worker_ref.worker_id, candidate); - state.publish_worker_upsert(worker_ref.worker_id)?; + if let Err(error) = state.persist_runtime_snapshot() { tracing::warn!( worker_id = %worker_ref.worker_id, @@ -4723,27 +4552,6 @@ impl Runtime { Ok(()) } - fn cleanup_failed_restore( - &self, - backend: &WorkerExecutionBackendRef, - worker_ref: &WorkerRef, - ) -> Result<(), RuntimeError> { - let result = backend.stop_worker(worker_ref); - if !result.is_accepted() { - return Err(RuntimeError::WorkerExecutionRejected { - worker_id: worker_ref.worker_id, - operation: result.operation, - outcome: result.outcome, - message: format!( - "restored execution cleanup failed after catalog commit failure: {}", - result.message_or_default() - ), - result, - }); - } - Ok(()) - } - /// Bind the Backend registry identity once. Retention evidence fails closed /// until the Runtime host supplies this trusted configuration. pub fn bind_runtime_identity(&self, runtime_id: &str) -> Result<(), RuntimeError> { @@ -6494,6 +6302,21 @@ impl Default for RestoreGuard { } impl RestoreGuard { + pub(crate) fn pending_preparation( + &self, + operation_id: WorkerLifecycleOperationId, + ) -> Option<&runtime_api::WorkerRestorePreparation> { + let id = self.active_request_id.as_ref()?; + let owner = self.owners.get(id)?; + if owner.operation_id != operation_id + || owner.receipt.is_some() + || owner.request.request_id != *id + { + return None; + } + owner.request.preparation.as_ref() + } + fn rotate(&mut self) { self.observation_token = uuid::Uuid::now_v7().to_string(); } @@ -6535,6 +6358,123 @@ struct WorkerRecord { } impl WorkerRecord { + fn apply_logical_workdir_attachments( + &mut self, + mut attachments: Vec, + ) -> Result<(), RuntimeError> { + validate_logical_workdir_attachments(&attachments)?; + attachments.sort_by(|left, right| left.alias.cmp(&right.alias)); + if !attachments.is_empty() && self.workspace_id.is_none() { + return Err(RuntimeError::InvalidRequest( + "logical Workdir attachments require Workspace authority".to_string(), + )); + } + if self.request.is_none() { + return Err(RuntimeError::WorkerExecutionUnavailable { + worker_id: self.worker_id, + message: "persisted Worker restore request is unavailable".to_string(), + }); + } + let logical_by_alias = attachments + .iter() + .map(|attachment| { + ( + attachment.alias.clone(), + attachment.working_directory_id.as_str(), + ) + }) + .collect::>(); + for local in &self.workdir_attachments { + let local_id = local + .working_directory + .summary + .working_directory_id + .as_str(); + if let Some(logical_id) = logical_by_alias.get(&local.alias) + && *logical_id != local_id + { + return Err(RuntimeError::InvalidRequest(format!( + "logical Workdir attachment alias `{}` conflicts with Runtime-local Workdir {}", + local.alias, local_id + ))); + } + } + let retained_local = self + .workdir_attachments + .iter() + .filter_map(|local| { + let local_id = local + .working_directory + .summary + .working_directory_id + .as_str(); + (logical_by_alias.get(&local.alias).copied() == Some(local_id)) + .then(|| (local.alias.clone(), local_id.to_string())) + }) + .collect::>(); + self.workdir_attachments + .retain(|local| retained_local.contains_key(&local.alias)); + let request = self + .request + .as_mut() + .expect("restore request existence checked above"); + request + .workdir_attachment_requests + .retain(|local| retained_local.contains_key(&local.alias)); + request.workdir_attachments.retain(|local| { + retained_local + .get(&local.alias) + .is_some_and(|workdir_id| workdir_id == &local.working_directory_id) + }); + self.logical_workdir_attachments = attachments; + Ok(()) + } + + // Overlay preparation on an unpublished clone, never on Runtime authority. + // The same validation is repeated at the effect boundary under the Worker lock. + fn prepared_restore_record( + &self, + preparation: Option<&runtime_api::WorkerRestorePreparation>, + ) -> Result { + let mut candidate = self.clone(); + let Some(preparation) = preparation else { + return Ok(candidate); + }; + if let Some(api) = &preparation.workspace_api { + let api: WorkspaceApiRef = serde_json::from_value( + serde_json::to_value(api) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + if candidate.workspace_id.as_deref() != Some(api.workspace_id.as_str()) { + return Err(RuntimeError::InvalidRequest( + "Prepared Workspace API does not match Worker Workspace identity".into(), + )); + } + let persisted = candidate.request.as_mut().ok_or_else(|| { + RuntimeError::InvalidRequest("Restore specification is missing".into()) + })?; + if persisted.workspace_api.as_ref().is_some_and(|existing| { + existing.workspace_id != api.workspace_id + || existing.base_url.trim_end_matches('/') != api.base_url.trim_end_matches('/') + }) { + return Err(RuntimeError::InvalidRequest( + "Workspace API preparation cannot change Worker identity or base URL".into(), + )); + } + persisted.workspace_api = Some(api); + } + if let Some(attachments) = &preparation.workdir_attachments { + let attachments: Vec = serde_json::from_value( + serde_json::to_value(attachments) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + candidate.apply_logical_workdir_attachments(attachments)?; + } + Ok(candidate) + } + fn apply_worker_state(&mut self, incoming: &protocol::WorkerStateSnapshot) { if self.worker_state.as_ref().map(|state| &state.state) != Some(&incoming.state) { self.restore_guard.rotate(); @@ -8201,6 +8141,10 @@ mod tests { restore_gate: Mutex>>, restore_count: Mutex, restore_operation_ids: Mutex>, + activation_result: Mutex>>, + activation_gate: Mutex>>, + activation_operation_ids: Mutex>, + restore_requests: Mutex>, #[cfg(feature = "ws-server")] restore_event: Mutex>, config_bundles: Mutex>>, @@ -8448,6 +8392,7 @@ mod tests { &self, request: WorkerExecutionRestoreRequest, ) -> WorkerExecutionSpawnResult { + self.restore_requests.lock().unwrap().push(request.clone()); self.job_bindings .lock() .unwrap() @@ -8493,6 +8438,54 @@ mod tests { } } + fn reconcile_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + let known = self + .restore_requests + .lock() + .unwrap() + .iter() + .any(|previous| { + previous.operation_id == request.operation_id + && previous.worker_ref == request.worker_ref + }); + if known + && self + .contexts + .lock() + .unwrap() + .contains_key(&request.worker_ref.worker_id) + { + return WorkerExecutionSpawnResult::Connected { + worker_state: protocol::WorkerStatus::Idle.into(), + workdir_attachments: request.previous_workdir_attachments, + }; + } + self.restore_worker(request) + } + + fn activate_restored_worker( + &self, + operation_id: WorkerLifecycleOperationId, + _worker_ref: &WorkerRef, + ) -> Result<(), String> { + self.activation_operation_ids + .lock() + .unwrap() + .push(operation_id); + let gate = self.activation_gate.lock().unwrap().clone(); + if let Some(gate) = gate { + gate.enter_and_wait(); + } + self.activation_result + .lock() + .unwrap() + .clone() + .unwrap_or(Ok(())) + } + fn worker_completions( &self, _worker_ref: &WorkerRef, @@ -9755,7 +9748,7 @@ mod tests { } #[test] - fn synced_config_bundle_is_stored_checked_and_used_for_worker_creation() { + fn synced_config_bundle_is_stored_checked_and_used_for_worker_creation_and_restore() { let backend = Arc::new(TestExecutionBackend::default()); let runtime = Runtime::with_execution_backend(RuntimeOptions::default(), backend.clone()).unwrap(); @@ -9807,7 +9800,7 @@ mod tests { .unwrap(); assert_eq!( backend.config_bundles.lock().unwrap().as_slice(), - &[Some(bundle), None] + &[Some(bundle.clone()), Some(bundle)] ); } @@ -11725,6 +11718,343 @@ mod tests { } } + #[test] + fn restore_activation_failure_never_records_or_replays_accepted() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("activation failure")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let stops = *backend.stop_count.lock().unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + *backend.activation_result.lock().unwrap() = Some(Err("partial activation".into())); + let mut operation_id = None; + for _ in 0..2 { + let result = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(result.state, WorkerRestoreState::ReconciliationRequired); + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + let pending = record.pending_restore.unwrap(); + assert_eq!( + *operation_id.get_or_insert(pending.operation_id), + pending.operation_id + ); + assert!(record.restore_candidate_context.is_some()); + let owner = &record.restore_guard.owners[&request.request_id]; + assert_eq!(owner.operation_id, pending.operation_id); + assert!(owner.receipt.is_none()); + assert_eq!( + record.restore_guard.active_request_id.as_ref(), + Some(&request.request_id) + ); + assert_eq!(*backend.stop_count.lock().unwrap(), stops); + } + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + *backend.activation_result.lock().unwrap() = None; + let accepted = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(accepted.state, WorkerRestoreState::Accepted); + let activations = backend.activation_operation_ids.lock().unwrap().len(); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + accepted + ); + assert_eq!( + backend.activation_operation_ids.lock().unwrap().len(), + activations + ); + assert!( + backend + .activation_operation_ids + .lock() + .unwrap() + .iter() + .all(|id| Some(*id) == operation_id) + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + } + + #[cfg(feature = "fs-store")] + #[test] + fn activated_restore_commit_failure_preserves_pending_owner_without_stopping_execution() { + for fault in [ + crate::fs_store::AtomicWriteFault::BeforeRename, + crate::fs_store::AtomicWriteFault::AfterRename, + ] { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Arc::new( + Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("activation-commit"), + backend.clone(), + ) + .unwrap(), + ); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("activated commit failure")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let stops = *backend.stop_count.lock().unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let gate = Arc::new(RestoreGate::default()); + *backend.activation_gate.lock().unwrap() = Some(gate.clone()); + let restoring_runtime = runtime.clone(); + let restoring_ref = worker.worker_ref.clone(); + let restoring_request = request.clone(); + let restoring = std::thread::spawn(move || { + restoring_runtime.restore_worker_operation(&restoring_ref, restoring_request) + }); + assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + runtime_store(&runtime).fail_next_worker_write(fault); + gate.release(); + assert_eq!( + restoring.join().unwrap().unwrap().state, + WorkerRestoreState::ReconciliationRequired + ); + { + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert_eq!(record.pending_restore.unwrap().operation_id, operation_id); + assert!(record.restore_candidate_context.is_some()); + assert!( + record.restore_guard.owners[&request.request_id] + .receipt + .is_none() + ); + } + let persisted: serde_json::Value = serde_json::from_slice( + &std::fs::read( + dir.path() + .join("runtime/workers") + .join(worker.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap(); + let saved_receipt = + &persisted["restore_guard"]["owners"][&request.request_id]["receipt"]; + if matches!(fault, crate::fs_store::AtomicWriteFault::BeforeRename) { + assert!(saved_receipt.is_null()); + } else { + // Activation was confirmed before this uncertain terminal write. + assert_eq!(saved_receipt["state"], "accepted"); + } + assert_eq!(*backend.stop_count.lock().unwrap(), stops); + *backend.activation_gate.lock().unwrap() = None; + let accepted = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(accepted.state, WorkerRestoreState::Accepted); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + accepted + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!(*backend.stop_count.lock().unwrap(), stops); + } + } + + #[cfg(all(feature = "fs-store", feature = "ws-server"))] + #[test] + fn buffered_restore_state_write_failure_keeps_valid_pending_owner_and_replays_once() { + for fault in [ + crate::fs_store::AtomicWriteFault::BeforeRename, + crate::fs_store::AtomicWriteFault::AfterRename, + ] { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Arc::new( + Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("buffered-state"), + backend.clone(), + ) + .unwrap(), + ); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("buffered live state")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let gate = Arc::new(RestoreGate::default()); + *backend.activation_gate.lock().unwrap() = Some(gate.clone()); + *backend.restore_event.lock().unwrap() = Some(protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Running.into(), + }); + let restoring_runtime = runtime.clone(); + let restoring_ref = worker.worker_ref.clone(); + let restoring_request = request.clone(); + let restoring = std::thread::spawn(move || { + restoring_runtime.restore_worker_operation(&restoring_ref, restoring_request) + }); + assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + // The pending execution projection is committed, but its live + // Running observation is buffered until backend activation returns. + runtime_store(&runtime).fail_next_worker_write(fault); + gate.release(); + assert_eq!( + restoring.join().unwrap().unwrap().state, + WorkerRestoreState::ReconciliationRequired + ); + let persisted = runtime_store(&runtime).load_runtime_state().unwrap(); + let record = &persisted.workers[&worker.worker_id]; + assert_eq!(record.status, WorkerStatus::Idle); + let PersistedWorkerExecutionState::ReconciliationRequired( + PersistedWorkerLifecycleOperation::Restore(pending), + ) = &record.execution_state + else { + panic!("pending owner disappeared") + }; + assert_eq!(pending.last_settled_status, record.status); + assert_eq!(pending.operation_id, operation_id); + assert!( + record.restore_guard.owners[&request.request_id] + .receipt + .is_none() + ); + // Live protocol state and settled catalog state are deliberately + // different during activation. Persistence must remain readable. + assert_eq!( + runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .worker_state + .unwrap() + .catalog_status(), + protocol::WorkerStatus::Running + ); + *backend.activation_gate.lock().unwrap() = None; + let accepted = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(accepted.state, WorkerRestoreState::Accepted); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + accepted + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert!( + backend + .activation_operation_ids + .lock() + .unwrap() + .iter() + .all(|id| *id == operation_id) + ); + assert!(matches!( + runtime_store(&runtime) + .load_runtime_state() + .unwrap() + .workers[&worker.worker_id] + .execution_state, + PersistedWorkerExecutionState::Available(_) + )); + } + } + + #[test] + fn restore_candidate_receives_prepared_bindings_without_mutating_runtime_authority() { + let (runtime, backend) = runtime_and_backend(); + let runtime = Arc::new(runtime); + let workspace_scope = scope("workspace-a", "server-a"); + let mut create = scoped_task_request("read-only candidate bindings", "workspace-a"); + create.workspace_api = None; + let worker = runtime + .create_worker_scoped(&workspace_scope, create) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "workspace-a", "base_url": "https://prepared.example" }, + "workdir_attachments": [], "repository_access": [] + })).unwrap()); + let gate = Arc::new(RestoreGate::default()); + *backend.restore_gate.lock().unwrap() = Some(gate.clone()); + let restoring_runtime = runtime.clone(); + let restoring_ref = worker.worker_ref.clone(); + let restoring_scope = workspace_scope.clone(); + let restoring = std::thread::spawn(move || { + restoring_runtime.restore_worker_operation_scoped( + &restoring_scope, + &restoring_ref, + request, + ) + }); + assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); + { + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert!(record.request.as_ref().unwrap().workspace_api.is_none()); + assert!( + !record + .restore_guard + .owners + .values() + .next() + .unwrap() + .preparation_applied + ); + } + assert_eq!( + backend.restore_requests.lock().unwrap()[0] + .request + .workspace_api + .as_ref() + .unwrap() + .base_url, + "https://prepared.example" + ); + assert!(backend.activation_operation_ids.lock().unwrap().is_empty()); + gate.release(); + assert_eq!( + restoring.join().unwrap().unwrap().state, + WorkerRestoreState::Accepted + ); + let state = runtime.lock().unwrap(); + assert_eq!( + state + .worker(&worker.worker_ref) + .unwrap() + .request + .as_ref() + .unwrap() + .workspace_api + .as_ref() + .unwrap() + .base_url, + "https://prepared.example" + ); + } + #[cfg(feature = "ws-server")] #[test] fn committed_restore_retries_retained_observation_activation() { diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index af4933bc3..b5ace780f 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -16,6 +16,7 @@ //! Manifests, credentials, and handles are not passed to browser/API projections. mod deferred_workdir; +mod restore_recovery; use std::collections::{BTreeMap, HashMap}; use std::future::Future; @@ -176,12 +177,25 @@ pub struct RuntimeWorkerController { pub workspace_client: Arc, } +/// An unpublished restore resource owner. No protocol endpoint is available +/// until activation succeeds; discard settles the actual candidate resources +/// directly, without sending Shutdown to an ordinary Controller. +#[async_trait] +pub trait RuntimeRestoreCandidate: Send { + fn initial_state(&self) -> protocol::WorkerStateSnapshot; + + async fn activate(&mut self) -> Result; + + async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure>; +} + /// Failure facts supplied by the resource owner, never inferred from error text. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RuntimeRestoreStage { Preflight, Metadata, Reconstruction, + Recovery, Binding, Feature, Controller, @@ -228,6 +242,7 @@ pub struct RuntimeRestoreFailure { cleanup: RuntimeRestoreCleanup, io_kind: Option, cleanup_failure: Option, + durable_effects_possible: bool, } impl RuntimeRestoreFailure { @@ -239,6 +254,7 @@ impl RuntimeRestoreFailure { cleanup: RuntimeRestoreCleanup::NotNeeded, io_kind: None, cleanup_failure: None, + durable_effects_possible: false, } } @@ -250,6 +266,7 @@ impl RuntimeRestoreFailure { cleanup: RuntimeRestoreCleanup::Required, io_kind: None, cleanup_failure: None, + durable_effects_possible: false, } } @@ -261,6 +278,7 @@ impl RuntimeRestoreFailure { cleanup: RuntimeRestoreCleanup::Required, io_kind: None, cleanup_failure: None, + durable_effects_possible: false, } } @@ -285,7 +303,8 @@ impl std::fmt::Display for RuntimeRestoreFailure { fn restore_worker_cause(error: &WorkerError) -> &'static str { match error { - WorkerError::ConstructionCleanup { source, .. } => restore_worker_cause(source), + WorkerError::ConstructionCleanup { source, .. } + | WorkerError::RestoreActivationEffects { source } => restore_worker_cause(source), WorkerError::SegmentEmpty { .. } => "saved segment empty", WorkerError::SegmentSystemPromptMissing { .. } => "saved segment system prompt missing", WorkerError::WorkerMetadataPending { .. } => "saved Worker metadata pending", @@ -325,6 +344,52 @@ async fn finish_restore_failure( failure } +async fn prepare_restore_operation( + factory: &F, + request: WorkerExecutionRestoreRequest, +) -> Result, RuntimeRestoreFailure> { + factory.preflight_restore(&request).await.map_err(|_| { + RuntimeRestoreFailure::refused(RuntimeRestoreStage::Preflight, "restore preflight refused") + })?; + let repaired = factory.recover_restore_source(&request).await?; + let candidate = factory + .prepare_restore(request) + .await + .map_err(|mut failure| { + failure.durable_effects_possible |= repaired; + failure + })?; + if repaired { + Ok(Box::new(RepairedRestoreCandidate { candidate })) + } else { + Ok(candidate) + } +} + +// A completed repair is a canonical effect even when later activation fails +// before its own mutation boundary. Carry that evidence with the resource owner. +struct RepairedRestoreCandidate { + candidate: Box, +} + +#[async_trait] +impl RuntimeRestoreCandidate for RepairedRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.candidate.initial_state() + } + + async fn activate(&mut self) -> Result { + self.candidate.activate().await.map_err(|mut failure| { + failure.durable_effects_possible = true; + failure + }) + } + + async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure> { + self.candidate.discard().await + } +} + /// Factory seam used by [`WorkerRuntimeExecutionBackend`] to construct a real /// controller-backed Worker for a Runtime catalog entry. #[async_trait] @@ -382,10 +447,28 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { Ok(()) } + /// Separate idempotent recovery, bound to the accepted operation and its + /// canonical Session. A true result records that preparation's no-mutation + /// guarantee cannot be used to describe this operation as a whole. + async fn recover_restore_source( + &self, + _request: &WorkerExecutionRestoreRequest, + ) -> Result { + Ok(false) + } + + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure>; + async fn restore_controller( &self, request: WorkerExecutionRestoreRequest, - ) -> Result; + ) -> Result { + let mut candidate = prepare_restore_operation(self, request).await?; + candidate.activate().await + } /// Prove cleanup of a restore future that completed without returning a /// controller. An error is not absence evidence; unsupported factories keep @@ -667,6 +750,7 @@ pub struct ProfileRuntimeWorkerFactory { struct FailedRestoreResources { operation_id: crate::execution::WorkerLifecycleOperationId, sessions: Vec>, + candidate_workdir_cleanup: Option, allocation: Option, feature_tasks: Option, } @@ -690,6 +774,10 @@ impl ProfileRuntimeWorkerFactory { request: &WorkerExecutionRestoreRequest, error: &WorkerError, ) { + if let WorkerError::RestoreActivationEffects { source } = error { + self.retain_construction_failure(request, source); + return; + } let mut resources = self .failed_restore_resources .lock() @@ -2032,10 +2120,22 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { Ok(()) } - async fn restore_controller( + async fn recover_restore_source( + &self, + request: &WorkerExecutionRestoreRequest, + ) -> Result { + self.recover_saved_restore_source(request) + .await + .map_err(|mut failure| { + failure.durable_effects_possible = true; + failure + }) + } + + async fn prepare_restore( &self, request: WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result, RuntimeRestoreFailure> { self.preflight_restore(&request).await.map_err(|_| { RuntimeRestoreFailure::refused( RuntimeRestoreStage::Preflight, @@ -2047,10 +2147,9 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .executions .fetch_add(1, Ordering::SeqCst); let mut stage = RuntimeRestoreStage::Metadata; - let mut io_kind = None; - // Reconstruction can write before failing; only the resource owner can - // confirm cleanup, even when no Worker value was returned. - let result: Result = async { + // Reconstruction owns read-only resources; cleanup confirmation must + // come from that owner even when it cannot return a candidate. + let result: Result, &'static str> = async { #[cfg(test)] self.restore_checkpoint(stage) .map_err(|_| "injected restore boundary failure")?; @@ -2070,14 +2169,6 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { &request.request, self.runtime_id.as_deref(), ); - let observation_runtime_id = self.runtime_id.clone(); - let observation_workspace_id = request - .request - .workspace_api - .as_ref() - .map(|api| api.workspace_id.clone()); - let observation_grants = request.request.worker_observation_grants.clone(); - let observation_enabled = request.request.worker_observation_enabled; let workspace_request_client = request .request .workspace_api @@ -2096,14 +2187,14 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .worker_aggregate_dir(&request.worker_ref) .map_err(|_| "Worker aggregate root unavailable")?; let session_dir = worker_aggregate_dir.join("session"); - let session_store = WorkerSessionStore::new_with_retention( + let session_store = WorkerSessionStore::open_existing_with_retention( &session_dir, self.session_retention() .map_err(|_| "Session retention unavailable")?, ) .map_err(|_| "canonical Session store initialization failed")?; let worker_metadata_store = - WorkerAggregateStore::new(&worker_aggregate_dir, worker_name.clone()) + WorkerAggregateStore::open_read_only(&worker_aggregate_dir, worker_name.clone()) .map_err(|_| "canonical metadata store initialization failed")?; let metadata = worker_metadata_store .read_by_name(&worker_name) @@ -2122,6 +2213,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { FailedRestoreResources { operation_id: request.operation_id, sessions: Vec::new(), + candidate_workdir_cleanup: None, allocation: None, feature_tasks: None, }, @@ -2129,70 +2221,29 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { #[cfg(test)] self.restore_checkpoint(stage) .map_err(|_| "injected restore boundary failure")?; - let mut worker = match Worker::restore_from_worker_metadata_with_context( - &worker_name, - manifest.clone(), - store.clone(), - loader.clone(), - workspace_context.clone(), - filesystem_authority.clone(), - ) - .await - { - Ok(worker) => worker, - Err(WorkerError::WorkerMetadataPending { .. }) - if request.request.initial_input.is_none() => - { - let pending_loader = if workspace_context.workspace_id().is_some() { - let bundle = request - .config_bundle - .as_ref() - .ok_or("pending restore launch material missing")?; - let resolution = self - .observe_bundle_prompt_projection( - bundle, - observation_workspace_id.as_deref(), - ) - .map_err(|_| "pending restore Prompt projection invalid")? - .ok_or("pending restore Prompt projection missing")?; - loader - .clone() - .with_effective_catalog(resolution.projection.catalog.clone()) - } else { - loader.clone() - }; - Worker::restore_pending_from_worker_metadata_with_context( - &worker_name, - manifest.clone(), - store, - pending_loader, - workspace_context, - filesystem_authority, - ) - .await - .map_err(|error| { - self.retain_construction_failure(&request, &error); - restore_worker_cause(&error) - })? - } - Err(error) => { + let candidate = + Worker::prepare_restored_from_worker_metadata_with_context_and_model_client( + &worker_name, + manifest, + store, + loader, + workspace_context, + filesystem_authority, + self.spawn_model_client_factory + .as_ref() + .map(|factory| factory()), + ) + .await + .map_err(|error| { self.retain_construction_failure(&request, &error); - return Err(restore_worker_cause(&error)); - } - }; - let feature_tasks = worker.retain_startup_feature_cleanup(); + restore_worker_cause(&error) + })?; self.failed_restore_resources .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert( - request.worker_ref.clone(), - FailedRestoreResources { - operation_id: request.operation_id, - sessions: vec![worker.workdir_sessions()], - allocation: worker.restore_allocation_cleanup_handle(), - feature_tasks: Some(feature_tasks), - }, - ); + .get_mut(&request.worker_ref) + .expect("retained restore owner") + .candidate_workdir_cleanup = Some(candidate.workdir_cleanup()); #[cfg(test)] { let gate = self.restore_faults.factory_gate.lock().unwrap().take(); @@ -2209,126 +2260,16 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { #[cfg(test)] self.restore_checkpoint(stage) .map_err(|_| "injected restore boundary failure")?; - if let Some(binding) = request.request.backend_job.as_ref() { - worker - .bind_backend_job(binding.clone()) - .map_err(|_| "restored Backend Job binding failed")?; - } - validate_worker_memory_settings(worker.manifest(), &request.request) + validate_worker_memory_settings(candidate.manifest(), &request.request) .map_err(|_| "saved memory settings mismatch")?; - worker - .finalize_subjektiv_session_attribution( - request.request.subjektiv_attached, - SubjektivSessionAttributionLifecycle::RestoredSession, - ) - .map_err(|error| restore_worker_cause(&error))?; - let flow_transition_enabled = worker.manifest().feature.flow.enabled; - let workdir_sessions = restored_workdir_router( - &request.workdir_attachments, - &request.logical_workdir_attachments, - &request.deferred_workdir_sessions, - &request.request.workdir_attachments, - worker.workspace_client_handle(), - ) - .map_err(|_| "restored Workdir binding failed")?; - self.failed_restore_resources - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .get_mut(&request.worker_ref) - .expect("retained restore owner") - .sessions - .push(workdir_sessions.clone()); - worker - .workdir_sessions() - .close_all() - .await - .map_err(|_| "initial Workdir close failed")?; - worker.bind_workdir_sessions(workdir_sessions); - if let (Some(runtime_id), Some(workspace_id)) = - (observation_runtime_id, observation_workspace_id.clone()) - && observation_enabled - { - let mut providers: Vec> = vec![Arc::new( - WorkspaceClientWorkerObservationProvider::new(worker.workspace_client_handle()), - )]; - if !observation_grants.is_empty() { - providers.push(Arc::new(RuntimeGrantedWorkerObservationProvider { - runtime_id, - workspace_id, - grants: observation_grants.into_iter().take(100).collect(), - hub: self.observation_hub.clone(), - })); - } - worker.bind_worker_observation_provider(Some(Arc::new( - CompositeWorkerObservationProvider::new(providers), - ))); - } - stage = RuntimeRestoreStage::Feature; - #[cfg(test)] - self.restore_checkpoint(stage) - .map_err(|_| "injected restore boundary failure")?; - if flow_transition_enabled { - let report = worker - .install_runtime_flow_transition_feature() - .map_err(|_| "Flow transition installation failed")?; - if report.reports.iter().any(|report| !report.installed) { - return Err("Flow transition installation refused"); - } - } - - stage = RuntimeRestoreStage::Controller; - #[cfg(test)] - self.restore_checkpoint(stage) - .map_err(|_| "injected restore boundary failure")?; - let workspace_client = worker.workspace_client_handle(); - let run_dir = self - .worker_restore_run_dir(&request.worker_ref, request.operation_id) - .map_err(|_| "restore run directory unavailable")?; - let transport = self.controller_transport; - #[cfg(test)] - let transport = if self.restore_faults.socket_failure.load(Ordering::SeqCst) { - WorkerControllerTransport::UnixSocket - } else { - transport - }; - let started = PreparedWorker::new( - worker, - WorkerBootstrapLayout::RuntimeManagedRun { - run_dir: run_dir.clone(), - }, - transport, - ) - .start() - .await - .map_err(|error| match error { - WorkerBootstrapError::Worker(error) => restore_worker_cause(&error), - WorkerBootstrapError::Controller { source, .. } => { - io_kind = Some(source.kind()); - "restored Worker controller startup failed" - } - })?; - self.failed_restore_resources - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(&request.worker_ref); - let (handle, shutdown_rx, controller_task) = - (started.handle, started.shutdown, started.controller_task); - if flow_transition_enabled { - handle.shared_state.enable_flow_transition(); - } - Ok(RuntimeWorkerController { - handle, - shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), - controller_task, - workspace_client, - }) + Ok(Box::new(ProfileRestoreCandidate { + prepared: Some(candidate), + factory: self.clone(), + request: request.clone(), + }) as Box) } .await; - result.map_err(|cause| { - let mut failure = RuntimeRestoreFailure::started(stage, cause); - failure.io_kind = io_kind; - failure - }) + result.map_err(|cause| RuntimeRestoreFailure::started(stage, cause)) } async fn cleanup_failed_restore( @@ -2364,6 +2305,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .await .map_err(|_| RuntimeRestoreCleanupFailure::Feature)?; } + if let Some(cleanup) = resources.candidate_workdir_cleanup.as_ref() { + cleanup + .close() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; + } for sessions in &resources.sessions { sessions .close_all() @@ -2434,6 +2381,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .await .map_err(|_| "failed restore Feature cleanup".to_string())?; } + if let Some(cleanup) = resources.candidate_workdir_cleanup.as_ref() { + cleanup + .close() + .await + .map_err(|_| "failed candidate Workdir cleanup".to_string())?; + } for sessions in &resources.sessions { sessions .close_all() @@ -2472,6 +2425,216 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { } } +type ProfileRestoreStore = CombinedStore; + +struct ProfileRestoreCandidate { + prepared: Option>, + factory: ProfileRuntimeWorkerFactory, + request: WorkerExecutionRestoreRequest, +} + +#[async_trait] +impl RuntimeRestoreCandidate for ProfileRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + protocol::WorkerStateSnapshot::initial() + } + + async fn activate(&mut self) -> Result { + let prepared = self.prepared.take().ok_or_else(|| { + RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "restore candidate already consumed", + ) + })?; + let worker = prepared.activate().await.map_err(|error| { + self.factory + .retain_construction_failure(&self.request, &error); + let mut failure = RuntimeRestoreFailure::started( + RuntimeRestoreStage::Reconstruction, + restore_worker_cause(&error), + ); + failure.durable_effects_possible = + matches!(error, WorkerError::RestoreActivationEffects { .. }); + failure + })?; + self.factory + .activate_prepared_restore(worker, &self.request) + .await + .map_err(|mut failure| { + failure.durable_effects_possible = true; + failure + }) + } + + async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure> { + if let Some(prepared) = self.prepared.as_mut() { + prepared + .discard() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; + } + self.prepared = None; + self.factory.cleanup_failed_restore(&self.request).await + } +} + +impl ProfileRuntimeWorkerFactory { + async fn activate_prepared_restore( + &self, + mut worker: Worker< + Box, + CombinedStore, + >, + request: &WorkerExecutionRestoreRequest, + ) -> Result { + let mut stage = RuntimeRestoreStage::Binding; + let mut io_kind = None; + let result: Result = async { + let feature_tasks = worker.retain_startup_feature_cleanup(); + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: vec![worker.workdir_sessions()], + candidate_workdir_cleanup: None, + allocation: worker.restore_allocation_cleanup_handle(), + feature_tasks: Some(feature_tasks), + }, + ); + let observation_runtime_id = self.runtime_id.clone(); + let observation_workspace_id = request + .request + .workspace_api + .as_ref() + .map(|api| api.workspace_id.clone()); + let observation_grants = request.request.worker_observation_grants.clone(); + let observation_enabled = request.request.worker_observation_enabled; + if let Some(binding) = request.request.backend_job.as_ref() { + worker + .bind_backend_job(binding.clone()) + .map_err(|_| "restored Backend Job binding failed")?; + } + validate_worker_memory_settings(worker.manifest(), &request.request) + .map_err(|_| "saved memory settings mismatch")?; + let workdir_sessions = restored_workdir_router( + &request.workdir_attachments, + &request.logical_workdir_attachments, + &request.deferred_workdir_sessions, + &request.request.workdir_attachments, + worker.workspace_client_handle(), + ) + .map_err(|_| "restored Workdir binding failed")?; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get_mut(&request.worker_ref) + .expect("retained restore owner") + .sessions + .push(workdir_sessions.clone()); + worker + .workdir_sessions() + .close_all() + .await + .map_err(|_| "initial Workdir close failed")?; + worker.bind_workdir_sessions(workdir_sessions); + if let (Some(runtime_id), Some(workspace_id)) = + (observation_runtime_id, observation_workspace_id.clone()) + && observation_enabled + { + let mut providers: Vec> = vec![Arc::new( + WorkspaceClientWorkerObservationProvider::new(worker.workspace_client_handle()), + )]; + if !observation_grants.is_empty() { + providers.push(Arc::new(RuntimeGrantedWorkerObservationProvider { + runtime_id, + workspace_id, + grants: observation_grants.into_iter().take(100).collect(), + hub: self.observation_hub.clone(), + })); + } + worker.bind_worker_observation_provider(Some(Arc::new( + CompositeWorkerObservationProvider::new(providers), + ))); + } + worker + .finalize_subjektiv_session_attribution( + request.request.subjektiv_attached, + SubjektivSessionAttributionLifecycle::RestoredSession, + ) + .map_err(|error| restore_worker_cause(&error))?; + let flow_transition_enabled = worker.manifest().feature.flow.enabled; + stage = RuntimeRestoreStage::Feature; + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + if flow_transition_enabled { + let report = worker + .install_runtime_flow_transition_feature() + .map_err(|_| "Flow transition installation failed")?; + if report.reports.iter().any(|report| !report.installed) { + return Err("Flow transition installation refused"); + } + } + + stage = RuntimeRestoreStage::Controller; + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + let workspace_client = worker.workspace_client_handle(); + let run_dir = self + .worker_restore_run_dir(&request.worker_ref, request.operation_id) + .map_err(|_| "restore run directory unavailable")?; + let transport = self.controller_transport; + #[cfg(test)] + let transport = if self.restore_faults.socket_failure.load(Ordering::SeqCst) { + WorkerControllerTransport::UnixSocket + } else { + transport + }; + let started = PreparedWorker::new( + worker, + WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: run_dir.clone(), + }, + transport, + ) + .start() + .await + .map_err(|error| match error { + WorkerBootstrapError::Worker(error) => restore_worker_cause(&error), + WorkerBootstrapError::Controller { source, .. } => { + io_kind = Some(source.kind()); + "restored Worker controller startup failed" + } + })?; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&request.worker_ref); + let (handle, shutdown_rx, controller_task) = + (started.handle, started.shutdown, started.controller_task); + if flow_transition_enabled { + handle.shared_state.enable_flow_transition(); + } + Ok(RuntimeWorkerController { + handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), + controller_task, + workspace_client, + }) + } + .await; + result.map_err(|cause| { + let mut failure = RuntimeRestoreFailure::started(stage, cause); + failure.io_kind = io_kind; + failure + }) + } +} + #[derive(Clone)] struct RuntimeExecutionTaskScope { tasks: Arc>>>, @@ -2788,7 +2951,11 @@ struct RuntimeWorkerExecution { struct PendingRuntimeRestore { request: WorkerExecutionRestoreRequest, task: tokio::sync::Mutex>>, - result: Mutex>>, + result: Mutex, RuntimeRestoreFailure>>>, + activated: Mutex>>, + activation_started: AtomicBool, + discard_started: AtomicBool, + discard_result: Mutex>>, failure: Mutex>, } @@ -3081,6 +3248,7 @@ where fn retry_pending_restore_cleanup( &self, pending: &Arc, + resume_activation: bool, ) -> Result<(), String> { let failure = pending .result @@ -3088,7 +3256,12 @@ where .map_err(|_| "restore result lock is poisoned")? .as_ref() .and_then(|result| result.as_ref().err()) - .filter(|failure| failure.needs_cleanup()) + .filter(|failure| { + failure.needs_cleanup() + || (resume_activation + && (failure.durable_effects_possible + || pending.activation_started.load(Ordering::Acquire))) + }) .cloned(); if let Some(failure) = failure { let owner = Arc::clone(pending); @@ -3101,8 +3274,43 @@ where return Err("restore task has not been joined".into()); } *task = Some(self.spawn_on_adapter_runtime(async move { + if resume_activation + && !failure.needs_cleanup() + && (failure.durable_effects_possible + || owner.activation_started.load(Ordering::Acquire)) + { + let result = + match prepare_restore_operation(factory.as_ref(), owner.request.clone()) + .await + { + Ok(candidate) => Ok(candidate), + Err(failure) => Err(finish_restore_failure( + factory.as_ref(), + &owner.request, + failure, + ) + .await), + }; + owner.activation_started.store(false, Ordering::Release); + *owner + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = None; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(result); + return; + } let failure = finish_restore_failure(factory.as_ref(), &owner.request, failure).await; + if owner.activation_started.load(Ordering::Acquire) { + *owner + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure.clone())); + } *owner .result .lock() @@ -3794,64 +4002,148 @@ where return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } if let Err(message) = self - .retry_pending_restore_cleanup(&pending) + .retry_pending_restore_cleanup(&pending, false) .and_then(|()| self.wait_pending_restore(&pending, Duration::from_secs(5))) { return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } - if self.workers.lock().is_err() { + if let Some(Err(failure)) = pending + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + && failure.needs_cleanup() + { return WorkerExecutionResult::errored( WorkerExecutionOperation::Stop, - "worker adapter registry lock is poisoned", + failure.to_string(), ); } - let mut retained = match pending.result.lock() { - Ok(retained) => retained, - Err(_) => { - return WorkerExecutionResult::errored( - WorkerExecutionOperation::Stop, - "restore result lock is poisoned", - ); - } - }; - match retained.as_ref() { - Some(Ok(_)) => { - let controller = match retained.take().unwrap() { - Ok(controller) => controller, - Err(_) => unreachable!(), - }; - let state = Arc::new(RwLock::new(controller.handle.shared_state.snapshot())); - self.retain_uncertain_unconnected_controller( - worker_ref, - controller.handle, - controller.shutdown, - Arc::new(AtomicBool::new(false)), - RuntimeExecutionTaskScope::new(controller.controller_task), - state, - Some(controller.workspace_client), - Some(pending.request.operation_id), - Vec::new(), - ); - } - Some(Err(failure)) => { - if !matches!( - failure.cleanup, - RuntimeRestoreCleanup::NotNeeded | RuntimeRestoreCleanup::Confirmed - ) { + if pending.activation_started.load(Ordering::Acquire) { + let mut activated = match pending.activated.lock() { + Ok(result) => result, + Err(_) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "restore activation result lock is poisoned", + ); + } + }; + match activated.as_ref() { + Some(Ok(_)) => { + let controller = + activated.take().unwrap().unwrap_or_else(|_| unreachable!()); + let state = + Arc::new(RwLock::new(controller.handle.shared_state.snapshot())); + self.retain_uncertain_unconnected_controller( + worker_ref, + controller.handle, + controller.shutdown, + Arc::new(AtomicBool::new(false)), + RuntimeExecutionTaskScope::new(controller.controller_task), + state, + Some(controller.workspace_client), + Some(pending.request.operation_id), + Vec::new(), + ); + } + Some(Err(failure)) + if matches!( + failure.cleanup, + RuntimeRestoreCleanup::Confirmed | RuntimeRestoreCleanup::NotNeeded + ) => {} + Some(Err(failure)) => { return WorkerExecutionResult::errored( WorkerExecutionOperation::Stop, failure.to_string(), ); } + None => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "restore activation ownership is unproven", + ); + } } - None => { - return WorkerExecutionResult::errored( - WorkerExecutionOperation::Stop, - "restore factory has no completion evidence; cleanup is unproven", - ); + } else { + if !pending.discard_started.swap(true, Ordering::AcqRel) { + let owner = pending.clone(); + let task = match self.spawn_on_adapter_runtime(async move { + let result = owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let cleanup = match result { + Some(Ok(mut candidate)) => { + let result = candidate.discard().await; + if result.is_err() { + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Ok(candidate)); + } + result + } + Some(Err(failure)) => { + let result = if matches!( + failure.cleanup, + RuntimeRestoreCleanup::Confirmed + | RuntimeRestoreCleanup::NotNeeded + ) { + Ok(()) + } else { + Err(RuntimeRestoreCleanupFailure::Ownership) + }; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure)); + result + } + None => Err(RuntimeRestoreCleanupFailure::Ownership), + }; + *owner + .discard_result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(cleanup); + }) { + Ok(task) => task, + Err(message) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + message, + ); + } + }; + *pending.task.try_lock().expect("joined candidate task") = Some(task); + } + if let Err(message) = self.wait_pending_restore(&pending, Duration::from_secs(5)) { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + let cleanup = *pending + .discard_result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match cleanup { + Some(Ok(())) => {} + Some(Err(error)) => { + pending.discard_started.store(false, Ordering::Release); + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + error.to_string(), + ); + } + None => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "candidate discard has no completion evidence", + ); + } } } - drop(retained); if let Err(message) = self.remove_pending_restore(worker_ref) { return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } @@ -4729,9 +5021,10 @@ where } drop(workers); - // An already accepted operation owns its bindings. Re-observing mutable - // Workdir materialization must not turn an unknown operation into a new - // read-only refusal or allocate another candidate. + // The operation retains its resources across timeouts, but its saved + // bindings never override a later authoritative attachment/owner change. + // Compare before substituting the owned request or retrying any effects; + // drift retains the same owner for reconciliation/Stop, not a new intent. let admitted_pending = match self.pending_restore(&request.worker_ref) { Ok(Some(pending)) if pending.request.operation_id == request.operation_id => { Some(pending) @@ -4759,6 +5052,22 @@ where } }; if let Some(pending) = admitted_pending.as_ref() { + if request.request != pending.request.request + || request.workspace_scope != pending.request.workspace_scope + || request.logical_workdir_attachments + != pending.request.logical_workdir_attachments + || request.previous_workdir_attachments + != pending.request.previous_workdir_attachments + { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "current Worker authority differs from the owned restore candidate", + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } request = pending.request.clone(); } else { if let Err(result) = self.preflight_restore_inputs(&request) { @@ -4806,6 +5115,10 @@ where request: request.clone(), task: tokio::sync::Mutex::new(None), result: Mutex::new(None), + activated: Mutex::new(None), + activation_started: AtomicBool::new(false), + discard_started: AtomicBool::new(false), + discard_result: Mutex::new(None), failure: Mutex::new(None), }); let mut resources = match self.pending_restores.lock() { @@ -4823,7 +5136,9 @@ where let factory = Arc::clone(&self.factory); let result_owner = Arc::clone(&pending); let task = self.spawn_on_adapter_runtime(async move { - let result = match factory.restore_controller(request.clone()).await { + let result = match prepare_restore_operation(factory.as_ref(), request.clone()) + .await + { Ok(controller) => Ok(controller), Err(failure) => { *result_owner @@ -4877,7 +5192,7 @@ where } if existing_pending { if let Err(message) = self - .retry_pending_restore_cleanup(&pending) + .retry_pending_restore_cleanup(&pending, true) .and_then(|()| self.wait_pending_restore(&pending, self.spawn_restore_timeout)) { return WorkerExecutionSpawnResult::ReconciliationRequired { @@ -4890,8 +5205,30 @@ where }; } } - let controller = { - let mut retained = match pending.result.lock() { + if pending.activation_started.load(Ordering::Acquire) { + let activated = pending + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(Ok(controller)) = activated.as_ref() { + return WorkerExecutionSpawnResult::Connected { + worker_state: controller.handle.shared_state.snapshot(), + workdir_attachments: pending.request.previous_workdir_attachments.clone(), + }; + } + let message = match activated.as_ref() { + Some(Err(failure)) => failure.to_string(), + _ => "restore activation is owned by the same operation; reconcile activation" + .to_string(), + }; + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message), + worker_state: None, + workdir_attachments: pending.request.previous_workdir_attachments.clone(), + }; + } + let worker_state = { + let retained = match pending.result.lock() { Ok(retained) => retained, Err(_) => { return WorkerExecutionSpawnResult::ReconciliationRequired { @@ -4905,16 +5242,23 @@ where } }; match retained.as_ref() { - Some(Ok(_)) => match retained.take().unwrap() { - Ok(controller) => controller, - Err(_) => unreachable!(), - }, + Some(Ok(candidate)) => candidate.initial_state(), Some(Err(failure)) => { let result = WorkerExecutionResult::errored( WorkerExecutionOperation::Restore, failure.to_string(), ); let cleanup = failure.cleanup; + if failure.durable_effects_possible { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result, + worker_state: None, + workdir_attachments: pending + .request + .previous_workdir_attachments + .clone(), + }; + } if matches!( cleanup, RuntimeRestoreCleanup::NotNeeded | RuntimeRestoreCleanup::Confirmed @@ -4959,25 +5303,10 @@ where } }; - let result = self.connect_handle( - WorkerExecutionOperation::Restore, - worker_ref.clone(), - pending.request.context.clone(), - controller.handle, - controller.shutdown, - controller.controller_task, - Some(operation_id), - pending.request.previous_workdir_attachments.clone(), - Some(controller.workspace_client), - ); - if let Err(message) = self.remove_pending_restore(&worker_ref) { - return WorkerExecutionSpawnResult::ReconciliationRequired { - result: WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message), - worker_state: None, - workdir_attachments: Vec::new(), - }; + WorkerExecutionSpawnResult::Connected { + worker_state, + workdir_attachments: pending.request.previous_workdir_attachments.clone(), } - result } fn activate_restored_worker( @@ -4996,15 +5325,109 @@ where return Err(message); } }; + if let Some(pending) = self.pending_restore(worker_ref)? { + if pending.request.operation_id != operation_id { + return Err("restored Worker operation identity does not match".into()); + } + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + if !pending.activation_started.swap(true, Ordering::AcqRel) { + let owner = pending.clone(); + let factory = self.factory.clone(); + let task = self.spawn_on_adapter_runtime(async move { + let candidate = owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let result = match candidate { + Some(Ok(mut candidate)) => { + let result = candidate.activate().await; + match result { + Ok(controller) => Ok(controller), + Err(mut failure) => { + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure.clone())); + match candidate.discard().await { + Ok(()) => { + failure.cleanup = RuntimeRestoreCleanup::Confirmed + } + Err(error) => { + failure.cleanup_failure.get_or_insert(error); + failure.cleanup = RuntimeRestoreCleanup::Failed(error); + } + } + let failure = finish_restore_failure( + factory.as_ref(), + &owner.request, + failure, + ) + .await; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure.clone())); + Err(failure) + } + } + } + Some(Err(failure)) => Err(failure), + None => Err(RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "restore candidate ownership unavailable", + )), + }; + *owner + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(result); + })?; + *pending + .task + .try_lock() + .map_err(|_| "restore activation task is already owned")? = Some(task); + } + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + let controller = { + let mut result = pending + .activated + .lock() + .map_err(|_| "restore activation result lock is poisoned")?; + match result.as_ref() { + Some(Ok(_)) => result.take().unwrap().map_err(|error| error.to_string())?, + Some(Err(failure)) => return Err(failure.to_string()), + None => return Err("restore activation has no completion evidence".into()), + } + }; + match self.connect_handle( + WorkerExecutionOperation::Restore, + worker_ref.clone(), + pending.request.context.clone(), + controller.handle, + controller.shutdown, + controller.controller_task, + Some(operation_id), + pending.request.previous_workdir_attachments.clone(), + Some(controller.workspace_client), + ) { + WorkerExecutionSpawnResult::Connected { .. } => { + self.remove_pending_restore(worker_ref)? + } + other => return Err(format!("restore activation connection failed: {other:?}")), + } + } let workers = self .workers .lock() - .map_err(|_| "worker adapter registry lock is poisoned".to_string())?; + .map_err(|_| "worker adapter registry lock is poisoned")?; let execution = workers .get(worker_ref) - .ok_or_else(|| "restored Worker execution is not registered".to_string())?; + .ok_or("restored Worker execution is not registered")?; if execution.restore_operation_id != Some(operation_id) { - return Err("restored Worker operation identity does not match".to_string()); + return Err("restored Worker operation identity does not match".into()); } self.factory.activate_restored_controller( worker_ref, @@ -5702,6 +6125,7 @@ mod tests { root: &Path, materialize_head: bool, ) -> WorkerExecutionRestoreRequest { + require_restore_test_environment(); let worker_ref = WorkerRef::new(WorkerId::now_v7()); let worker_name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref); let aggregate_dir = root.join("workers").join(worker_ref.worker_id.to_string()); @@ -5901,10 +6325,17 @@ mod tests { }) .unwrap(); backend.preflight_restore(&request).unwrap(); - let expected = match backend.restore_worker(request.clone()) { + let _prepared = match backend.restore_worker(request.clone()) { WorkerExecutionSpawnResult::Connected { worker_state, .. } => worker_state, other => panic!("real Profile backend restore failed: {other:?}"), }; + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); + let expected = backend.workers.lock().unwrap()[&request.worker_ref] + .handle + .shared_state + .snapshot(); assert!(matches!( backend.worker_session(WorkerSessionObservationRequest { worker_ref: request.worker_ref.clone(), @@ -6035,6 +6466,9 @@ mod tests { backend.restore_worker(request.clone()), WorkerExecutionSpawnResult::Connected { .. } )); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); let mut transport_a = Arc::clone(&backend) .attach_worker_protocol(&request.worker_ref) .unwrap(); @@ -6072,6 +6506,9 @@ mod tests { backend.restore_worker(request.clone()), WorkerExecutionSpawnResult::Connected { .. } )); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); let current_b = backend .workers .lock() @@ -6166,6 +6603,9 @@ mod tests { backend.restore_worker(request.clone()), WorkerExecutionSpawnResult::Connected { .. } )); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); let execution = backend.workers.lock().unwrap()[&request.worker_ref].clone(); let mut transport = Arc::clone(&backend) .attach_worker_protocol(&request.worker_ref) @@ -6227,6 +6667,9 @@ mod tests { backend.restore_worker(request.clone()), WorkerExecutionSpawnResult::Connected { .. } )); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); (backend, request) } @@ -6748,6 +7191,10 @@ mod tests { request: direct_restore_request(worker_ref.clone()), task: tokio::sync::Mutex::new(None), result: Mutex::new(None), + activated: Mutex::new(None), + activation_started: AtomicBool::new(false), + discard_started: AtomicBool::new(false), + discard_result: Mutex::new(None), failure: Mutex::new(Some("task completion unknown".into())), }), ); @@ -6787,6 +7234,10 @@ mod tests { request: request.clone(), task: tokio::sync::Mutex::new(Some(failed)), result: Mutex::new(None), + activated: Mutex::new(None), + activation_started: AtomicBool::new(false), + discard_started: AtomicBool::new(false), + discard_result: Mutex::new(None), failure: Mutex::new(None), }); backend @@ -7072,6 +7523,9 @@ mod tests { matches!(restored, WorkerExecutionSpawnResult::Connected { .. }), "{restored:?}" ); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); let session = backend.workers.lock().unwrap()[&request.worker_ref] .handle .shared_state @@ -8139,6 +8593,23 @@ mod tests { ); } + struct TestRestoreCandidate(Option); + #[async_trait] + impl RuntimeRestoreCandidate for TestRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.0.as_ref().unwrap().handle.shared_state.snapshot() + } + async fn activate(&mut self) -> Result { + Ok(self.0.take().unwrap()) + } + async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure> { + if let Some(controller) = self.0.take() { + shutdown_profile_controller(controller).await; + } + Ok(()) + } + } + struct PendingCreateFactory { started: mpsc::Sender<()>, dropped: mpsc::Sender<()>, @@ -8154,6 +8625,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for PendingCreateFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, _request: WorkerExecutionSpawnRequest, @@ -8324,6 +8804,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for MockFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, request: WorkerExecutionSpawnRequest, @@ -8738,6 +9227,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for FailingFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, _request: WorkerExecutionSpawnRequest, @@ -8766,6 +9264,15 @@ mod tests { #[cfg(feature = "fs-store")] #[async_trait] impl RuntimeWorkerFactory for RestartArtifactFailureFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, request: WorkerExecutionSpawnRequest, @@ -8807,7 +9314,9 @@ mod tests { #[cfg(feature = "fs-store")] #[test] fn fs_restart_stop_cleans_prior_restore_artifacts_and_retains_journal_on_failure() { - let root = tempfile::tempdir().unwrap(); + require_restore_test_environment(); + let root = tempfile::tempdir_in(".").unwrap(); + let socket_base = Path::new(root.path().file_name().unwrap()).join("controllers"); let runtime_store = root.path().join("runtime"); let options = crate::fs_store::FsRuntimeStoreOptions { root: runtime_store.clone(), @@ -8819,7 +9328,7 @@ mod tests { WorkerRuntimeExecutionBackend::new(RestartArtifactFailureFactory { inner: MockFactory { client: MockClient::sequential(vec![]), - runtime_base: root.path().join("controllers"), + runtime_base: socket_base, cwd: root.path().to_path_buf(), store_dir: root.path().join("sessions"), worker_metadata_dir: root.path().join("metadata"), @@ -8965,6 +9474,7 @@ mod tests { FailedRestoreResources { operation_id: request.operation_id, sessions: vec![Arc::clone(&sessions)], + candidate_workdir_cleanup: None, allocation: None, feature_tasks: None, }, @@ -9063,6 +9573,7 @@ mod tests { #[cfg(feature = "fs-store")] include!("worker_backend_restore_failure_tests.rs"); + include!("worker_backend_restore_candidate_tests.rs"); struct PendingFailureFactory { restore_calls: Arc, @@ -9072,6 +9583,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for PendingFailureFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, _request: WorkerExecutionSpawnRequest, @@ -9199,6 +9719,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for LateControllerFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, request: WorkerExecutionSpawnRequest, @@ -9812,67 +10341,6 @@ mod tests { assert!(!socket_path.exists()); } - #[test] - fn profile_runtime_factory_uses_shared_worker_bootstrap_seams() { - let source = include_str!("worker_backend.rs"); - let production = source - .split_once("#[cfg(test)]\nmod tests") - .map(|(production, _)| production) - .expect("worker backend test module marker"); - let factory = production - .split_once("impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory") - .map(|(_, factory)| factory) - .expect("profile runtime factory implementation"); - let (fresh, restore) = factory - .split_once("async fn restore_controller") - .expect("fresh and restore factory paths"); - let assert_in_order = |path: &str, markers: &[&str]| { - let mut offset = 0; - for marker in markers { - let relative = path[offset..] - .find(marker) - .unwrap_or_else(|| panic!("missing ordered factory marker {marker}")); - offset += relative + marker.len(); - } - }; - assert_in_order( - fresh, - &[ - "WorkerBootstrap::new(", - ".prepare()", - "finalize_subjektiv_session_attribution(", - "worker.bind_workdir_sessions(", - "worker.bind_worker_observation_provider(", - "install_runtime_flow_transition_feature()", - "prepared.start()", - ], - ); - assert_in_order( - restore, - &[ - "Worker::restore_from_worker_metadata_with_context(", - "finalize_subjektiv_session_attribution(", - "worker.bind_workdir_sessions(", - "worker.bind_worker_observation_provider(", - "install_runtime_flow_transition_feature()", - "PreparedWorker::new(", - ".start()", - ], - ); - assert!( - production.contains("WorkerBootstrap::new("), - "fresh runtime Workers must use the shared construction bootstrap" - ); - assert!( - production.contains("PreparedWorker::new("), - "restored runtime Workers must use the shared pre-exposure lifecycle" - ); - assert!( - !production.contains("WorkerController::spawn_runtime_managed_run_with_transport"), - "runtime factory paths must not bypass the shared controller lifecycle" - ); - } - #[test] #[serial_test::serial(worker_allocation)] fn shared_bootstrap_preserves_in_process_transport_for_fresh_and_restored_runtime_workers() { @@ -10018,6 +10486,9 @@ mod tests { factory_calls_after_restore, "reconciliation must not construct a second controller" ); + backend + .activate_restored_worker(operation_id, &worker.worker_ref) + .unwrap(); let workers = backend.workers.lock().unwrap(); let execution = workers.get(&worker.worker_ref).unwrap(); assert_eq!(execution.restore_operation_id, Some(operation_id)); diff --git a/crates/worker-runtime/src/worker_backend/restore_recovery.rs b/crates/worker-runtime/src/worker_backend/restore_recovery.rs new file mode 100644 index 000000000..8dd47ee15 --- /dev/null +++ b/crates/worker-runtime/src/worker_backend/restore_recovery.rs @@ -0,0 +1,173 @@ +//! Explicit, operation-owned repair of saved Session material. This is not +//! read-only candidate preparation: the canonical Session/attribution is the +//! receipt for replay after a lost response, and effects are reported as such. + +use super::*; + +impl ProfileRuntimeWorkerFactory { + pub(super) async fn recover_saved_restore_source( + &self, + request: &WorkerExecutionRestoreRequest, + ) -> Result { + let cause = |cause| RuntimeRestoreFailure::started(RuntimeRestoreStage::Recovery, cause); + let name = Self::runtime_worker_name_for_ref(&request.worker_ref); + let root = self + .worker_aggregate_dir(&request.worker_ref) + .map_err(|_| cause("restore recovery aggregate unavailable"))?; + let metadata_store = WorkerAggregateStore::open_read_only(&root, &name) + .map_err(|_| cause("restore recovery metadata open failed"))?; + let metadata = metadata_store + .read_by_name(&name) + .map_err(|_| cause("restore recovery metadata read failed"))? + .ok_or_else(|| cause("restore recovery metadata missing"))?; + let active = metadata + .active + .as_ref() + .ok_or_else(|| cause("restore recovery Session missing"))?; + let pending = active.segment_id.is_none(); + let incomplete = request.request.subjektiv_attached + && !matches!( + metadata.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Confirmed { .. }) + ); + if !pending && !incomplete { + return Ok(false); + } + // The accepted Runtime operation already owns this Session. Its stable + // Session identity and sole committed seed, not a new restore request, + // are used to reconcile an interrupted first-head publication. + let session_store = WorkerSessionStore::new_with_retention( + root.join("session"), + self.session_retention() + .map_err(|_| cause("restore recovery retention unavailable"))?, + ) + .map_err(|_| cause("restore recovery Session open failed"))?; + let (manifest, mut loader) = Self::manifest_for_restore(&metadata, &request.request) + .map_err(|_| cause("restore recovery saved manifest invalid"))?; + let workspace = RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + self.runtime_id.as_deref(), + ); + let request_client = request + .request + .workspace_api + .as_ref() + .and_then(|api| self.workspace_request_clients.get(&api.workspace_id)); + let context = workspace.worker_context( + &request.worker_ref, + request.workspace_scope.as_ref(), + self.worker_mutation_identity.as_ref(), + request_client, + self.embedded_worker_mutation_dispatcher.as_ref(), + Some(self.prompt_projection_cache.clone()), + ); + if pending && context.workspace_id().is_some() { + let bundle = request + .config_bundle + .as_ref() + .ok_or_else(|| cause("pending recovery launch material missing"))?; + let projection = self + .observe_bundle_prompt_projection( + bundle, + request + .request + .workspace_api + .as_ref() + .map(|api| api.workspace_id.as_str()), + ) + .map_err(|_| cause("pending recovery Prompt projection invalid"))? + .ok_or_else(|| cause("pending recovery Prompt projection missing"))?; + loader = loader.with_effective_catalog(projection.projection.catalog.clone()); + } + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: Vec::new(), + candidate_workdir_cleanup: None, + allocation: None, + feature_tasks: None, + }, + ); + let store = CombinedStore::new(session_store, metadata_store); + let worker_result = Worker::restore_pending_from_worker_metadata_with_context( + &name, + manifest, + store, + loader, + context, + WorkerFilesystemAuthority::None, + ) + .await; + let mut worker = worker_result.map_err(|error| { + self.retain_construction_failure(request, &error); + cause(restore_worker_cause(&error)) + })?; + let tasks = worker.retain_startup_feature_cleanup(); + let sessions = worker.workdir_sessions(); + let allocation = worker.restore_allocation_cleanup_handle(); + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: vec![sessions.clone()], + candidate_workdir_cleanup: None, + allocation: allocation.clone(), + feature_tasks: Some(tasks.clone()), + }, + ); + let result = async { + if let Some(binding) = request.request.backend_job.as_ref() { + worker + .bind_backend_job(binding.clone()) + .map_err(|_| cause("recovery Backend Job binding failed"))?; + } + worker + .finalize_subjektiv_session_attribution( + request.request.subjektiv_attached, + SubjektivSessionAttributionLifecycle::RestoredSession, + ) + .map_err(|error| cause(restore_worker_cause(&error)))?; + if pending { + worker::WorkerController::recover_pending_head(&mut worker, &root) + .await + .map_err(|_| cause("pending Session head recovery failed"))?; + } + Ok::<_, RuntimeRestoreFailure>(()) + } + .await; + // Never release allocation while the Worker guard is attached. Retained + // handles in the same operation remain available on every cleanup error. + drop(worker); + let cleanup = async { + tasks + .shutdown() + .await + .map_err(|_| cause("recovery Feature settlement failed"))?; + sessions + .close_all() + .await + .map_err(|_| cause("recovery Workdir close failed"))?; + if let Some(allocation) = allocation { + allocation + .release() + .map_err(|_| cause("recovery allocation release failed"))?; + } + Ok::<_, RuntimeRestoreFailure>(()) + } + .await; + result?; + cleanup?; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&request.worker_ref); + Ok(true) + } +} diff --git a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs new file mode 100644 index 000000000..a5b0312f9 --- /dev/null +++ b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs @@ -0,0 +1,898 @@ +// Included inside worker_backend::tests, alongside the restore failure suite. +// These integration tests connect the real Profile factory, canonical stores, +// restore owner, and Controller. Only model I/O and the Workspace HTTP transport +// are substituted. The loopback Host is not an authentication/policy test. +// Run in the ordinary worker-runtime lib suite with the default product features +// and the filter `restore_candidate::`. No live service is used. Like the +// failure suite, this requires the existing Yoi path settings to resolve into +// target/restore-test-environment; the common fixture rejects unsafe fallbacks +// before any factory, Git, or allocation-table operation. + +#[cfg(feature = "http-server")] +mod restore_candidate { + use super::*; + + const FIRST: &str = "candidate-saved-first"; + const SECOND: &str = "candidate-saved-second"; + const NOTICE: &str = "candidate-saved-notice"; + const FRESH: &str = "candidate-fresh-input"; + const WAIT: Duration = Duration::from_secs(5); + + struct HostProbe { + calls: Arc>>, + task: Option>, + } + + impl Drop for HostProbe { + fn drop(&mut self) { + if let Some(task) = self.task.take() { + task.abort(); + } + } + } + + // Aborting on unwind prevents a failed setup assertion from leaving its + // hanging model turn alive. Successful setup uses the normal shutdown/join. + struct SeedController(Option); + + impl Drop for SeedController { + fn drop(&mut self) { + if let Some(controller) = self.0.as_ref() { + controller.controller_task.abort(); + } + } + } + + async fn receive_matching( + events: &mut tokio::sync::broadcast::Receiver, + phase: &str, + mut matches: impl FnMut(&Event) -> bool, + ) -> Event { + let mut seen = Vec::new(); + tokio::time::timeout(WAIT, async { + loop { + let event = events.recv().await.expect("Controller event stream"); + seen.push(format!("{event:?}")); + if matches(&event) { + return event; + } + assert!( + !matches!( + event, + Event::Error { .. } + | Event::SubmissionRejected { .. } + | Event::NotificationRejected { .. } + ), + "unexpected Controller rejection: {event:?}" + ); + } + }) + .await + .unwrap_or_else(|_| { + panic!("Controller acknowledgement deadline during {phase}; observed {seen:?}") + }) + } + + async fn submit(handle: &WorkerHandle, id: &str) -> Event { + let mut events = handle.subscribe(); + handle + .send(Method::Submit { + submission_request_id: id.into(), + input: vec![protocol::Segment::Text { content: id.into() }], + }) + .await + .unwrap(); + receive_matching(&mut events, id, |event| { + matches!(event, Event::SubmissionAccepted { submission_request_id, .. } + if submission_request_id == id) + }) + .await + } + + async fn notify(handle: &WorkerHandle) { + let mut events = handle.subscribe(); + handle + .send(Method::Notify { + notification_request_id: NOTICE.into(), + message: NOTICE.into(), + }) + .await + .unwrap(); + receive_matching(&mut events, "notification acceptance", |event| { + matches!(event, Event::NotificationAccepted { notification_request_id } + if notification_request_id == NOTICE) + }) + .await; + } + + async fn pending(handle: &WorkerHandle) -> protocol::PendingSubmissionsSnapshot { + let mut events = handle.subscribe(); + handle.send(Method::ListPendingSubmissions).await.unwrap(); + match receive_matching(&mut events, "pending queue snapshot", |event| { + matches!(event, Event::PendingSubmissionsChanged { .. }) + }) + .await + { + Event::PendingSubmissionsChanged { pending } => pending, + _ => unreachable!(), + } + } + + async fn finished(handle: &WorkerHandle, id: &str) { + let mut events = handle.subscribe(); + let is_finished = |snapshot: &protocol::WorkerStateSnapshot| { + snapshot.last_finished_submission_request_id.as_deref() == Some(id) + && snapshot.state == protocol::WorkerState::Idle + }; + if !is_finished(&handle.shared_state.snapshot()) { + receive_matching( + &mut events, + id, + |event| matches!(event, Event::WorkerState { snapshot } if is_finished(snapshot)), + ) + .await; + } + } + + fn queue_checkpoint(entries: &[LogEntry]) -> serde_json::Value { + entries + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == "worker.pending_activations.v1" => Some(payload.clone()), + _ => None, + }) + .expect("real Controller must have committed its pending activation checkpoint") + } + + struct Fixture { + backend: Arc, + request: WorkerExecutionRestoreRequest, + model: MockClient, + session: WorkerSessionStore, + session_id: session_store::SessionId, + segment_id: session_store::SegmentId, + receipts: Vec, + host: HostProbe, + _root: tempfile::TempDir, + } + + impl Fixture { + fn new() -> Self { + Self::with_transport(WorkerControllerTransport::InProcess) + } + + fn with_transport(transport: WorkerControllerTransport) -> Self { + let root = restore_failure_tempdir(); + let model = MockClient::sequential(vec![ + MockResponse::Complete(vec![ + LlmEvent::tool_use_start(0, "candidate-task-probe", "TaskCreate"), + LlmEvent::tool_input_delta( + 0, + r#"{"subject":"Activated candidate","description":"Execution probe"}"#, + ), + LlmEvent::tool_use_stop(0), + LlmEvent::Status(StatusEvent { + status: ResponseStatus::Completed, + }), + ]), + MockResponse::Complete(simple_text_events()), + MockResponse::Complete(simple_text_events()), + MockResponse::Complete(simple_text_events()), + ]); + let injected = model.clone(); + let mut factory = ProfileRuntimeWorkerFactory::new(root.path()) + .with_runtime_store_dir(root.path()) + .with_runtime_id("candidate-runtime") + .with_spawn_model_client_factory(move || Box::new(injected.clone())); + factory.controller_transport = transport; + let catalog = worker::PromptCatalog::builtins_only() + .unwrap() + .projection() + .clone(); + factory + .observe_workspace_prompt_projection( + worker::WorkspacePromptProjection::new( + "workspace-saved-subjektiv", + "candidate-fixture", + catalog.catalog_digest.clone(), + catalog, + ) + .unwrap(), + ) + .unwrap(); + let backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap()); + let fixture_root = root.path().to_path_buf(); + let (request, session, session_id, segment_id, receipts, host) = backend + .run_on_adapter_runtime(async move { + let calls = Arc::new(Mutex::new(Vec::new())); + let authorization_calls = calls.clone(); + let other_calls = calls.clone(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let authorization_path = + "/api/w/workspace-saved-subjektiv/authorization/check"; + let app = axum::Router::new() + .route( + authorization_path, + axum::routing::post(move |axum::Json(input): axum::Json< + server_api::authorization::OperationAuthorizationRequest, + >| { + let calls = authorization_calls.clone(); + async move { + calls.lock().unwrap().push(( + "POST".into(), authorization_path.into(), + )); + let response = if input.operation == "task" + && input.resource == server_api::authorization::PermissionResource::Feature + { + server_api::authorization::OperationAuthorizationResponse { + scope: Some(server_api::authorization::PermissionResource::Feature), + } + } else { + saved_subjektiv_authorization_fixture(input) + .expect("only the saved fixture Feature grants are available") + }; + axum::Json(response) + } + }), + ) + .fallback(move |request: axum::extract::Request| { + let calls = other_calls.clone(); + async move { + calls.lock().unwrap().push(( + request.method().to_string(), request.uri().path().to_owned(), + )); + axum::http::StatusCode::FORBIDDEN + } + }); + let host = HostProbe { + calls, + task: Some(tokio::spawn(async move { axum::serve(listener, app).await.unwrap() })), + }; + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let (mut manifest, mut create) = + saved_subjektiv_policy(&fixture_root, &worker_ref); + // Normal restore deliberately excludes Subject repair. Keep + // a real Workspace client so accidental Host effects are visible. + manifest.feature.subjektiv = manifest::ResolvedSubjektivFeatureConfig::default(); + manifest.feature.task.enabled = true; + create.subjektiv_attached = false; + create.workspace_api.as_mut().unwrap().base_url = base_url; + let aggregate = fixture_root.join("workers").join(worker_ref.worker_id.to_string()); + let session = WorkerSessionStore::new(aggregate.join("session")).unwrap(); + let metadata = WorkerAggregateStore::new(&aggregate, &manifest.worker.name).unwrap(); + let mut engine = Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated( + MockClient::sequential(vec![MockResponse::Hang(simple_text_events().into_iter().take(2).collect())]), + ); + engine.set_system_prompt("Committed restore candidate fixture prompt"); + let mut worker = Worker::new( + manifest.clone(), + engine, + CombinedStore::new(session.clone(), metadata.clone()), + RuntimeWorkspaceBackendRef::from_worker_request(&create, Some("candidate-runtime")) + .worker_context(&worker_ref, Some(&crate::runtime::RuntimeWorkspaceScope::new( + "workspace-saved-subjektiv", "server-main", + )), None, None, None, None), + WorkerFilesystemAuthority::None, + Scope::empty(), + ).await.unwrap(); + worker.enable_worker_metadata_write_through().unwrap(); + worker.materialize_durable_session_head().await.unwrap(); + let (session_id, segment_id) = (worker.session_id(), worker.segment_id()); + let workspace_client = worker.workspace_client_handle(); + let started = PreparedWorker::new( + worker, + WorkerBootstrapLayout::RuntimeManagedRun { run_dir: fixture_root.join("seed") }, + WorkerControllerTransport::InProcess, + ).start().await.unwrap(); + let mut seed = SeedController(Some(RuntimeWorkerController { + handle: started.handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), + controller_task: started.controller_task, + workspace_client, + })); + let handle = seed.0.as_ref().unwrap().handle.clone(); + let mut started_events = handle.subscribe(); + submit(&handle, "candidate-setup-turn").await; + // Running and streaming observations can arrive in either + // order. Do not discard TextDelta while awaiting WorkerState. + let mut running = handle.shared_state.catalog_status() == WorkerStatus::Running; + let mut streaming = false; + receive_matching(&mut started_events, "seed running and streaming", |event| { + if let Event::WorkerState { snapshot } = event { + running |= snapshot.catalog_status() == protocol::WorkerStatus::Running; + } + streaming |= matches!(event, Event::TextDelta { .. }); + running && streaming + }).await; + let state = Arc::new(RwLock::new(handle.shared_state.snapshot())); + let mut events = handle.subscribe(); + handle.send(Method::Pause { command: next_internal_command(&state).unwrap() }) + .await.unwrap(); + receive_matching(&mut events, "seed pause acknowledgement", |event| { + matches!(event, Event::WorkerState { snapshot } + if snapshot.catalog_status() == protocol::WorkerStatus::Paused) + }).await; + let mut receipts = Vec::new(); + for id in [FIRST, SECOND] { + let receipt = submit(&handle, id).await; + assert!(matches!(receipt, Event::SubmissionAccepted { + disposition: protocol::SubmissionDisposition::Queued, .. + })); + receipts.push(serde_json::to_value(receipt).unwrap()); + if id == FIRST { notify(&handle).await; } + } + assert_eq!(pending(&handle).await.submissions.len(), 2); + shutdown_profile_controller(seed.0.take().unwrap()).await; + // This orphan pin would be removed by the normal activation + // reconciliation. Preparation and discard must leave it alone. + let orphan = session.write_uploaded_file( + session_id, "orphan.txt", "text/plain", b"retained orphan", + session_store::UploadedFileLimits::default(), + ).unwrap(); + session.pin_uploaded_file(session_id, &orphan, "uncommitted-acceptance").unwrap(); + let mut saved = metadata.read_by_name(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref)) + .unwrap().unwrap(); + saved.resolved_manifest_snapshot = Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap()); + saved.workspace_root = Some(fixture_root.join("stale-derived-root")); + metadata.write(&saved).unwrap(); + let mut request = direct_restore_request(worker_ref); + request.request = create; + request.workspace_scope = Some(crate::runtime::RuntimeWorkspaceScope::new( + "workspace-saved-subjektiv", "server-main", + )); + Ok((request, session, session_id, segment_id, receipts, host)) + }) + .unwrap(); + let fixture = Self { + backend, + request, + model, + session, + session_id, + segment_id, + receipts, + host, + _root: root, + }; + let checkpoint = queue_checkpoint(&fixture.entries()); + assert_eq!(checkpoint["pending"].as_array().unwrap().len(), 2); + assert_eq!( + checkpoint["pending_notifications"] + .as_array() + .unwrap() + .len(), + 1 + ); + fixture.assert_quiet(); + fixture + } + + fn entries(&self) -> Vec { + self.session + .read_all(self.session_id, self.segment_id) + .unwrap() + } + + fn canonical_files(&self) -> BTreeMap> { + let aggregate = self + .backend + .factory + .worker_aggregate_dir(&self.request.worker_ref) + .unwrap(); + let mut files = persisted_files(&aggregate); + files.retain(|path, _| !path.components().any(|part| part.as_os_str() == "runs")); + files + } + + fn assert_no_modifying_host_calls(&self) { + assert!( + self.host + .calls + .lock() + .unwrap() + .iter() + .all(|(method, path)| { + method == "GET" + || method == "HEAD" + || (method == "POST" + && path == "/api/w/workspace-saved-subjektiv/authorization/check") + }), + "unexpected modifying Host call" + ); + } + + fn assert_quiet(&self) { + assert_eq!(self.model.call_count.load(Ordering::SeqCst), 0); + assert!(self.model.captured.lock().unwrap().is_empty()); + self.assert_no_modifying_host_calls(); + assert!( + session_store::collect_state(&self.entries()) + .annotated_history + .iter() + .all(|entry| { + !matches!( + entry.item, + LoggedItem::ToolCall { .. } | LoggedItem::ToolResult { .. } + ) + }), + "prepare committed tool execution" + ); + } + + fn prepare(&self) { + assert!(matches!( + self.backend.restore_worker(self.request.clone()), + WorkerExecutionSpawnResult::Connected { .. } + )); + // Connected here is the backend's prepared snapshot, not publication. + assert!( + self.backend + .pending_restore(&self.request.worker_ref) + .unwrap() + .is_some() + ); + assert!(self.backend.workers.lock().unwrap().is_empty()); + assert!( + self.backend + .factory + .observation_hub + .get(&self.request.worker_ref) + .is_none() + ); + assert!( + !self + .backend + .factory + .worker_restore_run_dir(&self.request.worker_ref, self.request.operation_id,) + .unwrap() + .exists() + ); + let table = worker::runtime::worker_allocation::LockFileGuard::open( + &worker::runtime::worker_allocation::default_allocation_path().unwrap(), + ) + .unwrap(); + assert!( + table + .data() + .find(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref( + &self.request.worker_ref, + )) + .is_none() + ); + } + + fn assert_no_allocation(&self) { + let table = worker::runtime::worker_allocation::LockFileGuard::open( + &worker::runtime::worker_allocation::default_allocation_path().unwrap(), + ) + .unwrap(); + assert!( + table + .data() + .find(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref( + &self.request.worker_ref, + )) + .is_none() + ); + } + + fn assert_released(&self) { + self.assert_no_allocation(); + assert!( + self.backend + .pending_restore(&self.request.worker_ref) + .unwrap() + .is_none() + ); + assert!(self.backend.workers.lock().unwrap().is_empty()); + assert!( + self.backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty() + ); + assert!( + self.backend + .factory + .observation_hub + .get(&self.request.worker_ref) + .is_none() + ); + } + } + + impl Drop for Fixture { + fn drop(&mut self) { + // Keep the storage and loopback transport alive until owned tasks stop. + let _ = self.backend.stop_worker(&self.request.worker_ref); + if let Some(task) = self.host.task.take() { + task.abort(); + let _ = self.backend.run_on_adapter_runtime(async move { + let _ = task.await; + Ok(()) + }); + } + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn prepare_saved_inputs_rejects_new_input_and_discard_preserves_authority() { + // Exercise the configured socket transport without ever publishing or + // binding it; activation coverage below uses the normal InProcess path. + let mut fixture = Fixture::with_transport(WorkerControllerTransport::UnixSocket); + let opens = Arc::new(AtomicUsize::new(0)); + let counted = opens.clone(); + let raw = deferred_workdir::DeferredWorkdirSession::handle( + "candidate-discard-resource", + workdir::WorkdirSessionCapabilities::READ_ONLY, + move || { + counted.fetch_add(1, Ordering::SeqCst); + Err("not opened during prepare".into()) + }, + ); + fixture.request.deferred_workdir_sessions.insert( + WorkdirAttachmentAlias::new("checkout").unwrap(), + raw.clone(), + ); + let before = fixture.canonical_files(); + fixture.prepare(); + for input in [ + WorkerInput::user("must-not-accept"), + WorkerInput::notify("must-not-notify"), + ] { + assert!( + !fixture + .backend + .dispatch_input(&fixture.request.worker_ref, input) + .is_accepted() + ); + } + #[cfg(feature = "ws-server")] + assert!( + fixture + .backend + .clone() + .attach_worker_protocol(&fixture.request.worker_ref) + .is_err() + ); + assert!(!matches!( + fixture + .backend + .worker_session(WorkerSessionObservationRequest { + worker_ref: fixture.request.worker_ref.clone(), + }), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before); + assert!( + fixture + .backend + .factory + .failed_restore_resources + .lock() + .unwrap()[&fixture.request.worker_ref] + .candidate_workdir_cleanup + .is_some(), + "retain the real prepared Worker's close-only resource owner" + ); + assert!( + fixture + .backend + .stop_worker(&fixture.request.worker_ref) + .is_accepted() + ); + fixture.assert_released(); + + let factory = fixture.backend.factory.clone(); + let request = fixture.request.clone(); + fixture + .backend + .run_on_adapter_runtime(async move { + // The adapter resolves and owns its own raw handles. Separately + // exercise the factory seam with the exact supplied cleanup handle. + let mut candidate = factory + .prepare_restore(request) + .await + .map_err(|error| error.to_string())?; + candidate + .discard() + .await + .map_err(|error| error.to_string())?; + assert!(matches!( + raw.read(workdir::ReadRequest { + path: workdir::WorkdirPath::new("probe.txt").unwrap(), + offset: 0, + limit: 1, + max_bytes: 32, + }) + .await, + Err(workdir::WorkdirError::SessionClosed) + )); + Ok(()) + }) + .unwrap(); + assert_eq!(opens.load(Ordering::SeqCst), 0); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before); + assert!( + fixture + .backend + .stop_worker(&fixture.request.worker_ref) + .is_accepted() + ); + assert_eq!(fixture.canonical_files(), before); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn prepare_failure_at_each_pre_activation_checkpoint_preserves_saved_authority() { + // Feature and Controller are activation checkpoints, not preparation. + for stage in [ + RuntimeRestoreStage::Metadata, + RuntimeRestoreStage::Reconstruction, + RuntimeRestoreStage::Binding, + ] { + let fixture = Fixture::new(); + let before = fixture.canonical_files(); + *fixture + .backend + .factory + .restore_faults + .fail_at + .lock() + .unwrap() = Some(stage); + let result = fixture.backend.restore_worker(fixture.request.clone()); + assert!( + matches!(result, WorkerExecutionSpawnResult::RolledBack(_)), + "{stage:?}: {result:?}" + ); + fixture.assert_released(); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before, "{stage:?}"); + let completed = fixture.backend.completed_restore_failures.lock().unwrap(); + let (_, failure) = &completed[&fixture.request.worker_ref]; + assert_eq!(failure.stage, stage); + assert_eq!(failure.cleanup, RuntimeRestoreCleanup::Confirmed); + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn activate_saved_inputs_drains_fifo_once_and_replays_durable_receipts() { + let fixture = Fixture::new(); + let before = fixture.canonical_files(); + fixture.prepare(); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before); + fixture + .backend + .activate_restored_worker(fixture.request.operation_id, &fixture.request.worker_ref) + .unwrap(); + let handle = fixture.backend.workers.lock().unwrap()[&fixture.request.worker_ref] + .handle + .clone(); + let model = fixture.model.clone(); + let receipts = fixture.receipts.clone(); + fixture + .backend + .run_on_adapter_runtime(async move { + finished(&handle, SECOND).await; + assert_eq!(model.call_count.load(Ordering::SeqCst), 3); + { + let requests = model.captured.lock().unwrap(); + let first = serde_json::to_string(&requests[0].items).unwrap(); + let second = serde_json::to_string(&requests[2].items).unwrap(); + assert!( + requests[0] + .tools + .iter() + .any(|tool| tool.name == "TaskCreate") + ); + assert!(first.contains(FIRST)); + assert!(!first.contains(SECOND), "second Submit overtook the first"); + assert_eq!(first.matches(NOTICE).count(), 1); + assert!(second.find(FIRST).unwrap() < second.find(SECOND).unwrap()); + assert_eq!(second.matches(NOTICE).count(), 1); + } + let committed = serde_json::to_value(handle.committed_entries()).unwrap(); + for (id, mut expected) in [FIRST, SECOND].into_iter().zip(receipts) { + // Activation advances the durable receipt from Queued to Started; + // replay must preserve its original submission identity. + expected["data"]["disposition"] = serde_json::json!("started"); + assert_eq!( + serde_json::to_value(submit(&handle, id).await).unwrap(), + expected + ); + } + notify(&handle).await; + assert!(pending(&handle).await.submissions.is_empty()); + assert_eq!(model.call_count.load(Ordering::SeqCst), 3); + assert_eq!( + serde_json::to_value(handle.committed_entries()).unwrap(), + committed + ); + Ok(()) + }) + .unwrap(); + let history = session_store::collect_state(&fixture.entries()).annotated_history; + assert!( + history.iter().any(|entry| matches!( + &entry.item, + LoggedItem::ToolResult { call_id, is_error: false, .. } + if call_id == "candidate-task-probe" + )), + "activation must execute the real TaskCreate tool, not just decode a call" + ); + for marker in [FIRST, SECOND, NOTICE] { + let count = history + .iter() + .filter(|entry| { + match &entry.item { + LoggedItem::Message { content, .. } => content.iter().any(|part| { + matches!(part, LoggedContentPart::Text { text } if text.contains(marker)) + }), + _ => false, + } + }) + .count(); + assert_eq!(count, 1, "{marker} must commit exactly once"); + } + let mut fresh = WorkerInput::user(FRESH); + fresh.submission_request_id = Some(FRESH.into()); + assert!( + fixture + .backend + .dispatch_input(&fixture.request.worker_ref, fresh) + .is_accepted() + ); + let handle = fixture.backend.workers.lock().unwrap()[&fixture.request.worker_ref] + .handle + .clone(); + fixture + .backend + .run_on_adapter_runtime(async move { + finished(&handle, FRESH).await; + assert!(pending(&handle).await.submissions.is_empty()); + Ok(()) + }) + .unwrap(); + assert_eq!(fixture.model.call_count.load(Ordering::SeqCst), 4); + assert!( + !fixture.host.calls.lock().unwrap().is_empty(), + "activation must reach the configured Host transport" + ); + fixture.assert_no_modifying_host_calls(); + assert!( + fixture + .backend + .stop_worker(&fixture.request.worker_ref) + .is_accepted() + ); + fixture.assert_released(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn explicit_repair_reconciles_lost_attribution_response_without_reseeding() { + for materialized in [true, false] { + let root = restore_failure_tempdir(); + let model = MockClient::new(Vec::new()); + let injected = model.clone(); + let backend = WorkerRuntimeExecutionBackend::new( + ProfileRuntimeWorkerFactory::new(root.path()) + .with_runtime_store_dir(root.path()) + .with_runtime_id("repair-runtime") + .with_spawn_model_client_factory(move || Box::new(injected.clone())), + ) + .unwrap(); + let factory = backend.factory.clone(); + let fixture_root = root.path().to_path_buf(); + backend.run_on_adapter_runtime(async move { + let mut request = saved_subjektiv_restore_fixture(&fixture_root, materialized).await; + let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&request.worker_ref); + let aggregate = fixture_root.join("workers").join(request.worker_ref.worker_id.to_string()); + let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap(); + let mut saved = metadata.read_by_name(&name).unwrap().unwrap(); + let session_id = saved.active.as_ref().unwrap().session_id; + saved.subjektiv_session_attribution = Some(session_store::SubjektivSessionAttributionState::OutcomeUnknown { session_id, diagnostic: "saved interrupted attribution".into() }); + metadata.write(&saved).unwrap(); + // Explicit repair may acquire artifact locks; compare canonical + // contents separately from that lock-file side effect. + let session_files = || { + let mut files = persisted_files(&aggregate.join("session")); + files.remove(Path::new("artifacts/paste/.aggregate.lock")); + files + }; + let original = session_files(); + let mut bundle = test_bundle(); + bundle.metadata.workspace_id = "workspace-saved-subjektiv".into(); + bundle.prompt_catalog = Some(worker::PromptCatalog::builtins_only().unwrap().projection().clone()); + request.config_bundle = Some(bundle.with_computed_digest()); + + // Substitute only the Host transport. A committed attribution + // with an undecodable response models lost acknowledgement; + // its durable Session-keyed receipt survives the retry. This + // does not test the real Host's signature or Subject policy. + let attribution_path = fixture_root.join("host-attribution.json"); + let stored = attribution_path.clone(); + let calls = Arc::new(AtomicUsize::new(0)); + let record_calls = calls.clone(); + let startup_calls = Arc::new(AtomicUsize::new(0)); + let startups = startup_calls.clone(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + request.request.workspace_api.as_mut().unwrap().base_url = format!("http://{}", listener.local_addr().unwrap()); + let app = axum::Router::new() + .route("/api/w/workspace-saved-subjektiv/authorization/check", axum::routing::post( + |axum::Json(input): axum::Json| async move { + axum::Json(saved_subjektiv_authorization_fixture(input).unwrap()) + })) + .route("/api/w/workspace-saved-subjektiv/subjektiv/sessions", axum::routing::post( + move |axum::Json(input): axum::Json| { + let stored = stored.clone(); + let calls = record_calls.clone(); + async move { + assert_eq!(input.session_id, session_id.to_string()); + let ordinal = calls.fetch_add(1, Ordering::SeqCst); + if !stored.exists() { + assert!(input.create_if_missing); + fs::write(&stored, serde_json::to_vec(&server_api::SubjektivRecordSessionResponse { + session_id: input.session_id, subject_id: "saved-subject".into(), + }).unwrap()).unwrap(); + } + (axum::http::StatusCode::OK, [(axum::http::header::CONTENT_TYPE, "application/json")], + if ordinal == 0 { "{}".to_string() } else { fs::read_to_string(stored).unwrap() }) + } + })) + .route("/api/w/workspace-saved-subjektiv/subjektiv/journal/startup", axum::routing::get(move || { + let startups = startups.clone(); + async move { + startups.fetch_add(1, Ordering::SeqCst); + axum::Json(serde_json::json!({ "subject_id": "saved-subject", "journals": [], "omitted": false })) + } + })); + let host = HostProbe { calls: Arc::new(Mutex::new(Vec::new())), task: Some(tokio::spawn(async move { axum::serve(listener, app).await.unwrap() })) }; + // Same real factory and same operation for both repairs. No + // candidate/controller is constructed by the test itself. + assert!(factory.recover_saved_restore_source(&request).await.is_err()); + assert!(attribution_path.exists(), "Host committed before response loss"); + assert_eq!(session_files(), original); + let failed = metadata.read_by_name(&name).unwrap().unwrap(); + assert!(matches!(failed.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::OutcomeUnknown { session_id: id, .. }) if id == session_id)); + assert_eq!(startup_calls.load(Ordering::SeqCst), 0); + factory.cleanup_failed_restore(&request).await.unwrap(); + assert!(factory.recover_saved_restore_source(&request).await.unwrap()); + let repaired = metadata.read_by_name(&name).unwrap().unwrap(); + assert_eq!(repaired.active.as_ref().unwrap().session_id, session_id); + assert!(matches!(repaired.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Confirmed { session_id: id, .. }) if id == session_id)); + let segments = WorkerSessionStore::new(aggregate.join("session")).unwrap() + .list_segments(session_id).unwrap(); + assert_eq!(segments.len(), 1, "retry must retain exactly one seed"); + if materialized { + assert_eq!(session_files(), original); + assert_eq!(startup_calls.load(Ordering::SeqCst), 0); + } else { + assert_eq!(startup_calls.load(Ordering::SeqCst), 1); + } + let after_repair = persisted_files(&aggregate); + let calls_after_repair = calls.load(Ordering::SeqCst); + assert!(!factory.recover_saved_restore_source(&request).await.unwrap()); + assert_eq!(persisted_files(&aggregate), after_repair); + assert_eq!(calls.load(Ordering::SeqCst), calls_after_repair); + assert!(factory.failed_restore_resources.lock().unwrap().is_empty()); + drop(host); + Ok(()) + }).unwrap(); + assert_eq!(model.call_count.load(Ordering::SeqCst), 0); + } + } +} diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs index 8b870cd62..40de8168c 100644 --- a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -1,8 +1,11 @@ // Integration coverage through EmbeddedRuntime -> real Profile factory -> real -// Worker/bootstrap, local Git Workdir provider, and allocation table. Model I/O -// is not invoked; this is a deterministic ordinary suite, not product E2E. +// Worker/bootstrap, local Git Workdir provider, and allocation table. External +// model I/O is replaced for saved-input coverage; this is not product E2E. + +use crate::restore_test_environment::require_restore_test_environment; fn restore_failure_tempdir() -> tempfile::TempDir { + require_restore_test_environment(); // Explicitly stay inside this checkout, even when TMPDIR points elsewhere. // Keep fixture components short; no symlink/alias to evade socket limits. let target = Path::new(env!("CARGO_MANIFEST_DIR")) @@ -102,11 +105,16 @@ struct ProfileFailureFixture { backend: Arc, worker: crate::catalog::WorkerDetail, provider_opens: Arc, + seed_handle: Option, root: tempfile::TempDir, } impl ProfileFailureFixture { fn new(timeout: Duration) -> Self { + Self::with_saved_inputs(timeout, None) + } + + fn with_saved_inputs(timeout: Duration, model: Option) -> Self { let root = restore_failure_tempdir(); let repo = root.path().join("g"); fs::create_dir(&repo).unwrap(); @@ -118,16 +126,20 @@ impl ProfileFailureFixture { git(&repo, &["commit", "-m", "init"]); let store = root.path().join("s"); let provider_opens = Arc::new(AtomicUsize::new(0)); + let mut factory = + ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store); + let queue_inputs = model.is_some(); + if let Some(model) = model { + factory = factory.with_spawn_model_client_factory(move || Box::new(model.clone())); + } let backend = Arc::new( - WorkerRuntimeExecutionBackend::new( - ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store), - ) - .unwrap() - .with_spawn_restore_timeout(timeout) - .with_working_directory_materializer(RestoreOpenCountingMaterializer { - inner: RuntimeGitMaterializer::new(root.path().join("w")), - opens: provider_opens.clone(), - }), + WorkerRuntimeExecutionBackend::new(factory) + .unwrap() + .with_spawn_restore_timeout(timeout) + .with_working_directory_materializer(RestoreOpenCountingMaterializer { + inner: RuntimeGitMaterializer::new(root.path().join("w")), + opens: provider_opens.clone(), + }), ); let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( crate::fs_store::FsRuntimeStoreOptions { @@ -146,12 +158,41 @@ impl ProfileFailureFixture { working_directory: working_directory_request(&repo), }]; let worker = runtime.create_worker(request).unwrap(); + let seed_handle = queue_inputs.then(|| { + backend.workers.lock().unwrap()[&worker.worker_ref] + .handle + .clone() + }); + if let Some(handle) = seed_handle.clone() { + backend.run_on_adapter_runtime(async move { + async fn event(events: &mut tokio::sync::broadcast::Receiver, predicate: impl Fn(&Event) -> bool) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let next = events.recv().await.unwrap(); + assert!(!matches!(next, Event::Error { .. } | Event::SubmissionRejected { .. } | Event::NotificationRejected { .. }), "{next:?}"); + if predicate(&next) { break; } + } + }).await.expect("saved-input fixture acknowledgement"); + } + let mut events = handle.subscribe(); + handle.send(Method::submit_text("revocation-seed", "hold for saved input setup")).await.unwrap(); + event(&mut events, |next| matches!(next, Event::TextDelta { .. })).await; + handle.send(Method::Pause { command: protocol::WorkerCommandEnvelope::new(handle.shared_state.snapshot().last_command_id + 1) }).await.unwrap(); + event(&mut events, |next| matches!(next, Event::WorkerState { snapshot } if snapshot.catalog_status() == protocol::WorkerStatus::Paused)).await; + handle.send(Method::submit_text("revocation-saved-submit", "must stay queued")).await.unwrap(); + event(&mut events, |next| matches!(next, Event::SubmissionAccepted { submission_request_id, disposition: protocol::SubmissionDisposition::Queued, .. } if submission_request_id == "revocation-saved-submit")).await; + handle.send(Method::Notify { notification_request_id: "revocation-saved-notice".into(), message: "must also stay queued".into() }).await.unwrap(); + event(&mut events, |next| matches!(next, Event::NotificationAccepted { notification_request_id } if notification_request_id == "revocation-saved-notice")).await; + Ok(()) + }).unwrap(); + } runtime.stop_worker(&worker.worker_ref, None).unwrap(); // Count restore provider access, not the initial create/stop setup. provider_opens.store(0, Ordering::SeqCst); Self { root, provider_opens, + seed_handle, runtime, backend, worker, @@ -241,6 +282,68 @@ impl ProfileFailureFixture { assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0); } + fn resume_same_operation( + &self, + request: runtime_api::WorkerRestoreRequest, + operation_id: crate::execution::WorkerLifecycleOperationId, + ) { + let result = self + .runtime + .restore_worker_operation(&self.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::Accepted, + "{result:?}" + ); + assert_eq!( + self.backend.workers.lock().unwrap()[&self.worker.worker_ref].restore_operation_id, + Some(operation_id) + ); + let calls = self + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst); + let replay = self + .runtime + .restore_worker_operation(&self.worker.worker_ref, request) + .unwrap(); + assert_eq!( + serde_json::to_value(replay).unwrap(), + serde_json::to_value(result).unwrap() + ); + assert_eq!( + self.backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + calls + ); + self.runtime + .stop_worker(&self.worker.worker_ref, None) + .unwrap(); + assert!( + self.backend + .factory + .observation_hub + .get(&self.worker.worker_ref) + .is_none() + ); + assert!(self.backend.pending_restores.lock().unwrap().is_empty()); + assert!(self.backend.workers.lock().unwrap().is_empty()); + assert!( + self.backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty() + ); + } + fn fresh_restore_and_old_replay( &self, request: runtime_api::WorkerRestoreRequest, @@ -319,6 +422,171 @@ impl ProfileFailureFixture { } } +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_timeout_attachment_revocation_cannot_reactivate_old_candidate() { + let model = MockClient::sequential(vec![MockResponse::Hang( + simple_text_events().into_iter().take(2).collect(), + )]); + let fixture = + ProfileFailureFixture::with_saved_inputs(Duration::from_secs(2), Some(model.clone())); + let body_calls = model.call_count.load(Ordering::SeqCst); + assert_eq!(body_calls, 1); + let aggregate = fixture + .backend + .factory + .worker_aggregate_dir(&fixture.worker.worker_ref) + .unwrap(); + let session_before = persisted_files(&aggregate.join("session")); + let metadata = WorkerAggregateStore::open_read_only( + &aggregate, + ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&fixture.worker.worker_ref), + ) + .unwrap(); + let saved = metadata + .read_by_name(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref( + &fixture.worker.worker_ref, + )) + .unwrap() + .unwrap(); + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .factory_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .factory_entered + .lock() + .unwrap() = Some(entered_tx); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let first = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + entered.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + assert!(!pending.request.logical_workdir_attachments.is_empty()); + let updated = fixture + .runtime + .replace_worker_workdir_attachments(&fixture.worker.worker_ref, Vec::new()) + .unwrap(); + assert!(updated.workdir_attachments.is_empty()); + let authority = || -> serde_json::Value { + serde_json::from_slice( + &fs::read( + fixture + .root + .path() + .join("s/workers") + .join(fixture.worker.worker_ref.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap() + }; + assert!(authority().get("logical_workdir_attachments").is_none()); + release.notify_one(); + let replay = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "current attachment revocation must fence the old candidate: {replay:?}" + ); + assert!( + replay + .message + .as_deref() + .unwrap_or_default() + .contains("current Worker authority differs"), + "{replay:?}" + ); + let record = authority(); + let owner = &record["restore_guard"]["owners"][&request.request_id]; + assert_eq!( + owner["operation_id"], + serde_json::to_value(pending.request.operation_id).unwrap() + ); + assert!(owner["receipt"].is_null()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!(Arc::ptr_eq( + &pending, + &fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap() + )); + assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0); + assert!(!pending.activation_started.load(Ordering::SeqCst)); + assert_eq!(model.call_count.load(Ordering::SeqCst), body_calls); + assert_eq!(persisted_files(&aggregate.join("session")), session_before); + assert_eq!( + metadata.read_by_name(&saved.worker_name).unwrap().unwrap(), + saved + ); + let old = fixture.seed_handle.clone().unwrap(); + fixture + .backend + .run_on_adapter_runtime(async move { + assert!(old.protocol_is_closed()); + let old_command = Method::Resume { + command: protocol::WorkerCommandEnvelope::new( + old.shared_state.snapshot().last_command_id + 1, + ), + }; + assert!( + old.send(old_command).await.is_err(), + "old execution command must not enter the restore candidate" + ); + Ok(()) + }) + .unwrap(); + assert!(authority().get("logical_workdir_attachments").is_none()); + let input = fixture.runtime.send_input( + &fixture.worker.worker_ref, + WorkerInput::user("must-not-enter-revoked-candidate"), + ); + assert!( + input.is_err(), + "pending candidate must not admit fresh input: {input:?}" + ); + // Stop owns cleanup of the original operation, never a newly rebound alias. + fixture + .runtime + .stop_worker(&fixture.worker.worker_ref, None) + .unwrap(); + fixture.assert_closed_deferred_sessions(&pending); + assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0); + assert!(fixture.backend.pending_restores.lock().unwrap().is_empty()); + let old = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request) + .unwrap(); + assert_ne!(old.state, server_api::WorkerRestoreState::Accepted); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); +} + #[test] #[serial_test::serial(worker_allocation)] fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() { @@ -327,8 +595,6 @@ fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() { for stage in [ RuntimeRestoreStage::Reconstruction, RuntimeRestoreStage::Binding, - RuntimeRestoreStage::Feature, - RuntimeRestoreStage::Controller, ] { let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); fixture.fail(stage); @@ -474,7 +740,7 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer .unwrap(); assert_eq!( result.state, - server_api::WorkerRestoreState::RolledBack, + server_api::WorkerRestoreState::ReconciliationRequired, "{result:?}" ); assert!( @@ -488,12 +754,11 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer .iter() .all(|sessions| sessions.aliases().is_empty()) ); - fixture.assert_released(); fixture.assert_closed_deferred_sessions(&pending); eprintln!( - "restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=RolledBack" + "restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=ReconciliationRequired" ); - fixture.fresh_restore_and_old_replay(request, result); + fixture.resume_same_operation(request, pending.request.operation_id); } #[test] @@ -585,6 +850,7 @@ async fn failed_restore_raw_cleanup_is_operation_bound_and_does_not_follow_reuse FailedRestoreResources { operation_id: current.operation_id, sessions: Vec::new(), + candidate_workdir_cleanup: None, allocation: None, feature_tasks: None, }, @@ -715,14 +981,14 @@ fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconcile .unwrap(); assert_eq!( result.state, - server_api::WorkerRestoreState::RolledBack, + server_api::WorkerRestoreState::Accepted, "{result:?}" ); - fixture.assert_released(); + assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); eprintln!( "restore-fault cleanup=timeout retained-task=true late-completion=joined executions=1" ); - fixture.fresh_restore_and_old_replay(request, result); + fixture.resume_same_operation(request, pending.request.operation_id); } #[cfg(unix)] @@ -745,7 +1011,7 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( .unwrap(); assert_eq!( result.state, - server_api::WorkerRestoreState::RolledBack, + server_api::WorkerRestoreState::ReconciliationRequired, "{result:?}" ); let diagnostic = serde_json::to_string(&result).unwrap(); @@ -757,8 +1023,12 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( // This case deliberately exercises the real Unix socket path-limit error; // positive restore cases retain the factory's normal InProcess transport. // Short fixture roots must not accidentally turn it into a successful bind. - let operation_id = - fixture.backend.completed_restore_failures.lock().unwrap()[&fixture.worker.worker_ref].0; + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + let operation_id = pending.request.operation_id; let socket = fixture .backend .factory @@ -767,7 +1037,25 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( .join("worker.sock"); assert!(socket.as_os_str().as_encoded_bytes().len() >= 108); assert!(!diagnostic.contains(&fixture.root.path().to_string_lossy().to_string())); - fixture.assert_released(); + assert!(!socket.exists()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!( + fixture + .backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty() + ); + assert!( + fixture + .backend + .completed_restore_failures + .lock() + .unwrap() + .is_empty() + ); fixture .backend .factory @@ -777,7 +1065,7 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( eprintln!( "restore-fault stage=socket-start real-bootstrap=true partial-run=removed allocation=released" ); - fixture.fresh_restore_and_old_replay(request, result); + fixture.resume_same_operation(request, operation_id); } #[test] @@ -1178,6 +1466,8 @@ fn profile_restore_missing_committed_system_prompt_rolls_back_and_replays_classi "#, ) .unwrap(); + let mut manifest = manifest; + manifest.worker.name = worker_name.clone(); metadata.resolved_manifest_snapshot = Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap()); metadata_store.write(&metadata).unwrap(); diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index a541ca2e1..97cdff43a 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -3676,8 +3676,12 @@ mod tests { #[test] fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() { + crate::restore_test_environment::require_restore_test_environment(); let repo = create_clean_repo(); - let runtime_root = tempfile::tempdir().unwrap(); + // Both ends use a short reference into this TempDir-owned tree. Keep + // real agent/broker sockets and the existing bind/cleanup assertions. + let runtime_root = tempfile::tempdir_in(".").unwrap(); + let runtime_path = Path::new(runtime_root.path().file_name().unwrap()); let key_root = tempfile::tempdir().unwrap(); let key_path = key_root.path().join("id_ed25519"); let fallback_key_path = key_root.path().join("id_ed25519_fallback"); @@ -3689,7 +3693,7 @@ mod tests { .unwrap(); assert!(status.success()); } - let materializer = RuntimeGitMaterializer::new(runtime_root.path()); + let materializer = RuntimeGitMaterializer::new(runtime_path); let mut request = request(repo.path()); request.materialization = Some(crate::catalog::RepositoryMaterializationContext { workspace_id: "workspace-1".to_string(), @@ -3731,7 +3735,7 @@ mod tests { kind: server_api::RepositorySourceKind::Ssh, uri: "ssh://git@example.test/repo.git".to_string(), }; - let command_access = RepositoryCommandAccess::prepare(runtime_root.path(), &ssh_operation) + let command_access = RepositoryCommandAccess::prepare(runtime_path, &ssh_operation) .unwrap() .unwrap(); assert!(!command_access.root.join("identity").exists()); @@ -3873,18 +3877,17 @@ mod tests { .unwrap(); // Restore preflight must validate without consuming this one-shot SSH // delivery or creating agent/session resources needed by actual bind. - let record_before = - fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(); + let record_before = fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap(); materializer .preflight_bind_working_directory(&id, None) .unwrap(); assert_eq!( - fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(), + fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap(), record_before ); assert_eq!(materializer.list_working_directories().unwrap().len(), 1); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 0 @@ -3898,7 +3901,7 @@ mod tests { assert!(ssh_policy.contains("__repository-ssh")); assert!(!ssh_policy.contains("SSH_AUTH_SOCK")); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 1 @@ -3916,7 +3919,7 @@ mod tests { "consumed delivery expiry must not revoke a live Workdir session" ); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 1, @@ -3925,7 +3928,7 @@ mod tests { drop(binding); assert!(!ssh_command.exists()); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 0 @@ -3971,7 +3974,7 @@ mod tests { ], &read_only_command_policy, )); - let broker_socket = fs::read_dir(runtime_root.path().join(REPOSITORY_ACCESS_DIR)) + let broker_socket = fs::read_dir(runtime_path.join(REPOSITORY_ACCESS_DIR)) .unwrap() .filter_map(Result::ok) .map(|entry| entry.path().join("b.sock")) @@ -4104,7 +4107,7 @@ mod tests { .code, "working_directory_repository_access_expired" ); - let restored = RuntimeGitMaterializer::new(runtime_root.path()); + let restored = RuntimeGitMaterializer::new(runtime_path); assert_eq!( restored.bind_working_directory(&id, None).unwrap_err().code, "working_directory_remote_repository_access_required" diff --git a/crates/worker/src/bootstrap.rs b/crates/worker/src/bootstrap.rs index daf86ee25..00e6c5ddf 100644 --- a/crates/worker/src/bootstrap.rs +++ b/crates/worker/src/bootstrap.rs @@ -9,6 +9,7 @@ use crate::PromptCatalogSource; use crate::controller::{ ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle, }; +pub use crate::worker::{PreparedRestoredWorker, RestoreCandidateWorkdirCleanup}; use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext}; use manifest::WorkerManifest; @@ -168,6 +169,38 @@ where Ok(PreparedWorker::new(worker, self.layout, self.transport)) } + /// Reconstruct a saved materialized Worker without allocation, attribution + /// repair, Feature installation, or controller exposure. The caller owns the + /// later controller layout/transport and must activate before starting it. + pub async fn prepare_restored_candidate( + self, + worker_name: &str, + ) -> Result, WorkerBootstrapError> { + let mut candidate = + Worker::prepare_restored_from_worker_metadata_with_context_and_model_client( + worker_name, + self.manifest, + self.store, + self.prompt_catalog, + self.workspace_context, + self.filesystem_authority, + self.model_client, + ) + .await + .map_err(WorkerBootstrapError::Worker)?; + if let Some(workdir_session) = self.workdir_session { + candidate + .worker_mut() + .map_err(WorkerBootstrapError::Worker)? + .bind_single_workdir_session(Some(workdir_session)); + } + candidate + .worker_mut() + .map_err(WorkerBootstrapError::Worker)? + .bind_subjektiv_host(self.subjektiv_host); + Ok(candidate) + } + pub async fn prepare_restored( self, worker_name: &str, diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index 3ff170dd9..6721753d6 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -689,6 +689,35 @@ pub enum WorkerControllerTransport { pub struct WorkerController; impl WorkerController { + /// Explicit recovery of an unmaterialized Session head. This installs the + /// saved Feature composition needed to produce the seed, but creates no + /// Controller, input channel, socket, observer, or queue consumer. The caller + /// owns the same operation's Feature/session/allocation cleanup on every exit. + pub async fn recover_pending_head( + worker: &mut Worker, + runtime_base: &Path, + ) -> std::io::Result<()> + where + C: LlmClient + Clone + 'static, + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + if worker.needs_initial_session_head_materialization() { + worker + .validate_subjektiv_body_publication() + .map_err(std::io::Error::other)?; + let registry = SpawnedWorkerRegistry::new_internal( + worker.manifest().worker.name.clone(), + worker.scope().clone(), + ); + register_worker_tools(worker, runtime_base.to_path_buf(), registry, None, None).await?; + worker + .materialize_durable_session_head() + .await + .map_err(std::io::Error::other)?; + } + Ok(()) + } + pub async fn spawn( worker: Worker, runtime_base: &Path, diff --git a/crates/worker/src/discovery.rs b/crates/worker/src/discovery.rs index dd7ecb8e8..e8354a499 100644 --- a/crates/worker/src/discovery.rs +++ b/crates/worker/src/discovery.rs @@ -1066,7 +1066,8 @@ mod tests { use tokio::net::UnixListener; use crate::runtime::dir::RuntimeDir; - use crate::runtime::worker_allocation::test_util::RuntimeDirSandbox; + + use crate::restore_test_environment::require_restore_test_environment; #[derive(Clone)] struct FailTargetPeerStore { @@ -1110,11 +1111,13 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn state_backed_visibility_and_restore_planning() { - let root = TempDir::new().unwrap(); + require_restore_test_environment(); + let root = tempfile::tempdir_in(".").unwrap(); let store_dir = root.path().join("store"); - let runtime_base = root.path().join("runtime"); + // Bind, durable child metadata, and discovery probes all use this same + // short reference into the tree still owned by root. No cwd mutation. + let runtime_base = Path::new(root.path().file_name().unwrap()).join("runtime"); std::fs::create_dir_all(&runtime_base).unwrap(); - let _runtime_sandbox = RuntimeDirSandbox::new(&runtime_base); let store = FsWorkerStore::new(&store_dir).unwrap(); let session_id = new_session_id(); diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index e04ec36a4..2f3df7f9b 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -20,6 +20,9 @@ mod metadata_lifecycle; pub mod model_client; mod paste_artifact_tool; pub mod prompt; +#[cfg(test)] +#[path = "../../../tests/support/restore_environment.rs"] +mod restore_test_environment; pub mod runtime; pub mod runtime_command; pub mod segment_log_sink; diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 1ef93cd0e..59eb439a7 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -3386,6 +3386,208 @@ impl WorkerStartupFeatureCleanup { } } +/// A close-only candidate resource owner, not filesystem execution authority. +#[derive(Clone)] +pub struct RestoreCandidateWorkdirCleanup { + sessions: Arc, +} + +impl RestoreCandidateWorkdirCleanup { + pub async fn close(&self) -> Result<(), WorkerError> { + self.sessions + .close_all() + .await + .map_err(|error| WorkerError::FeatureLifecycle(error.to_string())) + } +} + +/// An unpublished, read-only reconstruction of an existing materialized Worker. +/// +/// The host must hold its per-Worker operation lock through activation. Snapshot +/// checks reject drift; they do not replace that lock or current host authorization. +/// No controller, allocation, Feature installation, or repair runs during prepare. +/// The public candidate exposes no Worker execution or mutation handles. Binding +/// inside this crate is limited to the open candidate; once discard begins, only +/// cleanup retry remains valid, even if its previous attempt failed or was cancelled. +/// Use `discard` for confirmed cleanup; Drop only schedules best-effort cleanup on +/// the current Tokio runtime and cannot attest that asynchronous cleanup completed. +pub struct PreparedRestoredWorker { + worker: Option, St>>, + closing: bool, + metadata: Option, + entries: serde_json::Value, + manifest: serde_json::Value, +} + +impl PreparedRestoredWorker +where + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, +{ + pub fn manifest(&self) -> &WorkerManifest { + &self.worker().manifest + } + + /// Retain the same Workdir resource owner for explicit cleanup, including + /// after discard fails. This does not grant Worker input or execution access. + pub fn workdir_cleanup(&self) -> RestoreCandidateWorkdirCleanup { + RestoreCandidateWorkdirCleanup { + sessions: self.worker().workdir_sessions.clone(), + } + } + + #[cfg(test)] + fn workdir_sessions(&self) -> Arc { + self.worker().workdir_sessions.clone() + } + + fn worker(&self) -> &Worker, St> { + self.worker + .as_ref() + .expect("restore candidate already discarded") + } + + /// Bootstrap may bind host services before returning an open candidate. + /// Never reopen a candidate whose cleanup has begun. + pub(crate) fn worker_mut( + &mut self, + ) -> Result<&mut Worker, St>, WorkerError> { + if self.closing { + return Err(WorkerError::RestoreCandidateClosing); + } + self.worker + .as_mut() + .ok_or(WorkerError::RestoreCandidateClosing) + } + + fn revalidate(&self, worker: &Worker, St>) -> Result<(), WorkerError> { + let changed = |component| WorkerError::RestoreCandidateChanged { + worker_name: worker.manifest.worker.name.clone(), + component, + }; + if worker.store.read_by_name(&worker.manifest.worker.name)? != self.metadata { + return Err(changed("metadata")); + } + if restore_snapshot_value(&worker.manifest)? != self.manifest { + return Err(changed("manifest")); + } + let entries = worker + .store + .read_all(worker.session_id(), worker.segment_id())?; + if restore_snapshot_value(&entries)? != self.entries { + return Err(changed("segment")); + } + Ok(()) + } + + /// Acquire writer allocation only after exact source validation, validate + /// again under that allocation, then apply the existing restore mutations. + /// An error may retain partial durable effects; ConstructionCleanup carries + /// resource ownership and is not evidence of rollback. + pub async fn activate(self) -> Result, St>, WorkerError> { + self.activate_with_allocation(|worker| { + let socket_path = dir::default_base() + .map_err(ScopeLockError::from)? + .join(&worker.manifest.worker.name) + .join("sock"); + Ok(worker_allocation::install_top_level_with_deny( + worker.manifest.worker.name.clone(), + std::process::id(), + socket_path, + worker.scope.snapshot().allow_rules(), + worker.scope.snapshot().deny_rules(), + worker.segment_id(), + )?) + }) + .await + } + + async fn activate_with_allocation( + mut self, + allocate: impl FnOnce( + &Worker, St>, + ) -> Result, + ) -> Result, St>, WorkerError> { + let mut worker = self + .worker + .take() + .ok_or(WorkerError::RestoreCandidateClosing)?; + if self.closing { + // Preserve the same cleanup owner without validating or acquiring + // allocation for a partially closed candidate. + return Err(worker.construction_failure(WorkerError::RestoreCandidateClosing)); + } + self.revalidate(&worker) + .map_err(|error| worker.construction_failure(error))?; + worker.scope_allocation = + Some(allocate(&worker).map_err(|error| worker.construction_failure(error))?); + self.revalidate(&worker) + .map_err(|error| worker.construction_failure(error))?; + worker + .finish_restored_construction() + .await + .map_err(|source| WorkerError::RestoreActivationEffects { + source: Box::new(source), + }) + } + + /// Directly settle candidate resources, without Controller Shutdown. Failed + /// cleanup retains the candidate so the same owner can retry it. + pub async fn discard(&mut self) -> Result<(), WorkerError> { + self.closing = true; + if let Some(worker) = self.worker.as_mut() { + worker + .retain_startup_feature_cleanup() + .shutdown() + .await + .map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?; + worker + .workdir_sessions + .close_all() + .await + .map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?; + // An unpublished candidate has no allocation guard. Activation moves + // the Worker out and uses ConstructionCleanup on failure; never try + // releasing a cleanup handle while its guard is still attached. + debug_assert!(worker.scope_allocation.is_none()); + } + self.worker = None; + Ok(()) + } +} + +impl Drop for PreparedRestoredWorker { + fn drop(&mut self) { + let Some(worker) = self.worker.as_mut() else { + return; + }; + let sessions = worker.workdir_sessions.clone(); + let features = worker.retain_startup_feature_cleanup(); + // Candidates do not acquire allocation until consuming activation. + debug_assert!(worker.scope_allocation.is_none()); + match tokio::runtime::Handle::try_current() { + Ok(runtime) => { + runtime.spawn(async move { + if let Err(error) = features.shutdown().await { + tracing::warn!(%error, "restore candidate Feature cleanup unconfirmed"); + return; + } + if let Err(error) = sessions.close_all().await { + tracing::warn!(%error, "restore candidate Workdir cleanup unconfirmed"); + } + }); + } + Err(error) => { + tracing::warn!(%error, "restore candidate dropped without async cleanup runtime") + } + } + } +} + +fn restore_snapshot_value(value: &impl serde::Serialize) -> Result { + serde_json::to_value(value) + .map_err(|error| WorkerError::InvalidState(format!("encode restore snapshot: {error}"))) +} + /// An independent agent execution unit. /// /// Holds a [`Engine`] directly and persists session state via @@ -3405,6 +3607,7 @@ pub struct Worker { /// constructors install this from the same FsStore that owns the session /// logs; low-level `Worker::new` tests leave it absent. worker_metadata_writer: Option, + persist_restored_manifest: bool, worker_metadata_segment_cas: Option, /// Shared session pointer. Source of truth for the Worker's current /// `segment_id` and append tally. `self.segment_id()` is a thin @@ -3820,6 +4023,7 @@ impl Worker { last_run_interrupted: false, store, worker_metadata_writer: None, + persist_restored_manifest: false, worker_metadata_segment_cas: None, segment_state: SegmentState::new(session_id, segment_id, 0), filesystem_authority, @@ -4534,14 +4738,21 @@ impl Worker { } fn worker_metadata(&self, active: Option) -> WorkerMetadata { - worker_metadata_for_manifest( + let mut metadata = worker_metadata_for_manifest( &self.manifest, self.workspace_id(), self.filesystem_authority .as_local() .map(|local| local.root.as_path()), active, - ) + ); + // A repaired/restored Session must not lose the saved execution policy + // merely because its current composition also permits legacy local use. + if self.persist_restored_manifest { + metadata.resolved_manifest_snapshot = + manifest::write_persisted_worker_manifest_snapshot(&self.manifest).ok(); + } + metadata } fn write_worker_metadata_pending(&self) -> Result<(), WorkerError> { @@ -8445,6 +8656,7 @@ where last_run_interrupted: false, store, worker_metadata_writer, + persist_restored_manifest: existing_session_id.is_some(), worker_metadata_segment_cas, segment_state: SegmentState::new(session_id, segment_id, 0), filesystem_authority: common.filesystem_authority, @@ -8852,6 +9064,7 @@ where last_run_interrupted: false, store, worker_metadata_writer: None, + persist_restored_manifest: false, worker_metadata_segment_cas: None, segment_state: SegmentState::new(session_id, segment_id, 0), filesystem_authority: common.filesystem_authority, @@ -8987,6 +9200,7 @@ where last_run_interrupted: false, store, worker_metadata_writer, + persist_restored_manifest: true, worker_metadata_segment_cas, segment_state: SegmentState::new(session_id, segment_id, 0), filesystem_authority: common.filesystem_authority, @@ -9080,6 +9294,33 @@ where workspace_context: WorkerWorkspaceContext, filesystem_authority: WorkerFilesystemAuthority, ) -> Result { + Self::prepare_restored_from_worker_metadata_with_context_and_model_client( + worker_name, + manifest, + store, + loader, + workspace_context, + filesystem_authority, + None, + ) + .await? + .activate() + .await + } + + /// Prepare only a materialized saved Worker. Pending Session creation and + /// Subject attribution repair must be performed explicitly by the host + /// before entering this read-only path. + #[allow(clippy::too_many_arguments)] + pub async fn prepare_restored_from_worker_metadata_with_context_and_model_client( + worker_name: &str, + fallback: WorkerManifest, + store: St, + loader: PromptCatalogSource, + workspace_context: WorkerWorkspaceContext, + filesystem_authority: WorkerFilesystemAuthority, + model_client: Option>, + ) -> Result, WorkerError> { let metadata = store .read_by_name(worker_name)? @@ -9093,11 +9334,13 @@ where worker_name: worker_name.to_string(), }); } - let active = metadata - .active - .ok_or_else(|| WorkerError::WorkerMetadataInactive { - worker_name: worker_name.to_string(), - })?; + let active = + metadata + .active + .as_ref() + .ok_or_else(|| WorkerError::WorkerMetadataInactive { + worker_name: worker_name.to_string(), + })?; let segment_id = active .segment_id .ok_or_else(|| WorkerError::WorkerMetadataPending { @@ -9106,10 +9349,15 @@ where })?; let manifest = restore_manifest_from_worker_metadata_snapshot( worker_name, - metadata.resolved_manifest_snapshot, - manifest, + metadata.resolved_manifest_snapshot.clone(), + fallback, )?; - Self::restore_from_manifest_with_context( + if manifest.worker.name != worker_name { + return Err(WorkerError::InvalidState( + "restore manifest Worker name mismatch".into(), + )); + } + Self::prepare_restored_from_manifest_snapshot( active.session_id, segment_id, manifest, @@ -9117,6 +9365,8 @@ where loader, workspace_context, filesystem_authority, + model_client, + Some(metadata), ) .await } @@ -9303,10 +9553,41 @@ where filesystem_authority: WorkerFilesystemAuthority, model_client: Option>, ) -> Result { + let metadata = store.read_by_name(&manifest.worker.name)?; + Self::prepare_restored_from_manifest_snapshot( + session_id, + segment_id, + manifest, + store, + loader, + workspace_context, + filesystem_authority, + model_client, + metadata, + ) + .await? + .activate() + .await + } + + #[allow(clippy::too_many_arguments)] + async fn prepare_restored_from_manifest_snapshot( + session_id: SessionId, + segment_id: SegmentId, + manifest: WorkerManifest, + store: St, + loader: PromptCatalogSource, + workspace_context: WorkerWorkspaceContext, + filesystem_authority: WorkerFilesystemAuthority, + model_client: Option>, + metadata: Option, + ) -> Result, WorkerError> { // Read raw entries once so we can both reconstruct state and // seed the broadcast sink's mirror with the same prefix that // sits on disk. let raw_entries = store.read_all(session_id, segment_id)?; + let entries_snapshot = restore_snapshot_value(&raw_entries)?; + let manifest_snapshot = restore_snapshot_value(&manifest)?; let state = session_store::collect_state(&raw_entries); if state.entries_count == 0 { return Err(WorkerError::SegmentEmpty { segment_id }); @@ -9332,24 +9613,6 @@ where model_client, )?; - // Atomic: register_worker inside install_top_level rejects when - // another live allocation already holds `segment_id`. Wrapping - // the lookup + install inside a single `LockFileGuard` is what - // makes "no two live Workers write to the same session log" - // actually structural rather than a hopeful pre-check. - let socket_path = dir::default_base() - .map_err(ScopeLockError::from)? - .join(&manifest.worker.name) - .join("sock"); - let scope_allocation = worker_allocation::install_top_level_with_deny( - manifest.worker.name.clone(), - std::process::id(), - socket_path, - common.scope.allow_rules(), - common.scope.deny_rules(), - segment_id, - )?; - // Build the worker and apply the manifest defaults first, then // overwrite the pieces the session log is authoritative for. let mut worker = @@ -9398,6 +9661,7 @@ where last_run_interrupted: state.last_run_interrupted, store, worker_metadata_writer, + persist_restored_manifest: true, worker_metadata_segment_cas, segment_state: SegmentState::new(session_id, segment_id, state.entries_count), filesystem_authority: common.filesystem_authority, @@ -9436,7 +9700,7 @@ where pending_attachments: Arc::new(Mutex::new(Vec::::new())), pending_committed_history: Arc::new(Mutex::new(std::collections::VecDeque::new())), failed_activation_authority: Mutex::new(None), - scope_allocation: Some(scope_allocation), + scope_allocation: None, callback_socket: None, runtime_ticket_role: None, prompts: common.prompts, @@ -9461,7 +9725,20 @@ where worker .session .restore_pending_activations(&state.extensions); - worker.finish_restored_construction().await + worker.apply_permissions_from_manifest(); + worker.apply_prune_from_manifest(); + let mut candidate = PreparedRestoredWorker { + worker: Some(worker), + closing: false, + metadata, + entries: entries_snapshot, + manifest: manifest_snapshot, + }; + if let Err(error) = candidate.revalidate(candidate.worker()) { + let worker = candidate.worker.take().expect("candidate just constructed"); + return Err(worker.construction_failure(error)); + } + Ok(candidate) } async fn finish_restored_construction(mut self) -> Result { @@ -10329,6 +10606,23 @@ pub enum WorkerError { sessions: Arc, }, + /// Activation entered canonical pin/metadata/delegation updates. Cleanup + /// ownership in the source does not establish rollback of those effects. + #[error("{source}")] + RestoreActivationEffects { + #[source] + source: Box, + }, + + #[error("restore candidate is closing; only cleanup retry is allowed")] + RestoreCandidateClosing, + + #[error("restore candidate source changed for Worker {worker_name}: {component}")] + RestoreCandidateChanged { + worker_name: String, + component: &'static str, + }, + #[error("invalid durable Worker state: {0}")] InvalidState(String), @@ -12998,6 +13292,686 @@ mod build_summary_prompt_tests { assert!(table.data().find("test-worker").is_none()); } + // Real Session/metadata stores, allocation and Workdir providers; only the + // model transport is substituted. Profile/Runtime publication belongs to the + // factory integration suite, not this reconstruction component suite. + mod restored_candidate_tests { + use super::*; + + type CandidateStore = + session_store::CombinedStore; + + #[derive(Clone)] + struct CountedClient(Arc); + + #[async_trait] + impl LlmClient for CountedClient { + async fn stream( + &self, + request: agen::llm_client::Request, + ) -> Result< + std::pin::Pin< + Box< + dyn futures::Stream< + Item = Result< + agen::llm_client::event::Event, + agen::llm_client::ClientError, + >, + > + Send, + >, + >, + agen::llm_client::ClientError, + > { + self.0.fetch_add(1, Ordering::SeqCst); + NoopClient.stream(request).await + } + + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + } + + #[derive(Debug)] + struct FailCloseOnce { + inner: workdir::WorkdirSessionHandle, + attempts: AtomicUsize, + } + + #[async_trait] + impl workdir::WorkdirSession for FailCloseOnce { + fn workdir(&self) -> &workdir::Workdir { + self.inner.workdir() + } + fn capabilities(&self) -> workdir::WorkdirSessionCapabilities { + self.inner.capabilities() + } + async fn stat( + &self, + request: workdir::StatRequest, + ) -> Result { + self.inner.stat(request).await + } + async fn read( + &self, + request: workdir::ReadRequest, + ) -> Result { + self.inner.read(request).await + } + async fn write( + &self, + request: workdir::WriteRequest, + ) -> Result { + self.inner.write(request).await + } + async fn edit( + &self, + request: workdir::EditRequest, + ) -> Result { + self.inner.edit(request).await + } + async fn list( + &self, + request: workdir::ListRequest, + ) -> Result { + self.inner.list(request).await + } + async fn glob( + &self, + request: workdir::GlobRequest, + ) -> Result { + self.inner.glob(request).await + } + async fn grep( + &self, + request: workdir::GrepRequest, + ) -> Result { + self.inner.grep(request).await + } + async fn start_command( + &self, + request: workdir::CommandRequest, + ) -> Result { + self.inner.start_command(request).await + } + async fn command_status( + &self, + handle: workdir::CommandHandle, + ) -> Result { + self.inner.command_status(handle).await + } + async fn command_output( + &self, + request: workdir::CommandOutputRequest, + ) -> Result { + self.inner.command_output(request).await + } + async fn cancel_command( + &self, + handle: workdir::CommandHandle, + ) -> Result<(), workdir::WorkdirError> { + self.inner.cancel_command(handle).await + } + async fn close(&self) -> Result<(), workdir::WorkdirError> { + self.inner.close().await?; + if self.attempts.fetch_add(1, Ordering::SeqCst) == 0 { + // The real provider has closed, but its first response is + // uncertain: the candidate must not become activatable again. + Err(workdir::WorkdirError::Unavailable( + "close response lost".into(), + )) + } else { + Ok(()) + } + } + } + + struct Fixture { + temp: tempfile::TempDir, + store: CandidateStore, + manifest: WorkerManifest, + session_id: SessionId, + segment_id: SegmentId, + orphan: protocol::UploadedFileRef, + model_calls: Arc, + } + + impl Fixture { + async fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let store = session_store::CombinedStore::new( + session_store::FsStore::new(temp.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(temp.path().join("workers")).unwrap(), + ); + let mut manifest = minimal_manifest(); + manifest.scope = ScopeConfig { + allow: vec![ScopeRule { + target: temp.path().canonicalize().unwrap(), + permission: Permission::Write, + recursive: true, + symlink_policy: Default::default(), + }], + deny: Vec::new(), + }; + let model_calls = Arc::new(AtomicUsize::new(0)); + let mut worker = Worker::new( + manifest.clone(), + Engine::, Mutable, SessionHistoryMetadata>::new_annotated( + Box::new(CountedClient(model_calls.clone())), + ), + store.clone(), + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + Scope::empty(), + ) + .await + .unwrap(); + worker + .engine_mut() + .set_system_prompt("saved candidate prompt"); + worker.ensure_segment_head().await.unwrap(); + let session_id = worker.session_id(); + let segment_id = worker.segment_id(); + let queue = worker.pending_submission_handle(); + queue + .accept( + "saved-submit".into(), + vec![Segment::Text { + content: "saved input".into(), + }], + false, + ) + .unwrap(); + queue + .accept_notification("saved-notification".into(), "saved notice".into()) + .unwrap(); + let orphan = store + .write_uploaded_file( + session_id, + "orphan.txt", + "text/plain", + b"orphan", + session_store::UploadedFileLimits::default(), + ) + .unwrap(); + store + .pin_uploaded_file(session_id, &orphan, "acceptance-without-checkpoint") + .unwrap(); + let mut metadata = WorkerMetadata::new( + &manifest.worker.name, + Some(WorkerActiveSegmentRef { + session_id, + segment_id: Some(segment_id), + }), + ); + // Activation replaces this stale derived path; preparation must not. + metadata.workspace_root = Some(temp.path().join("old-root")); + metadata.spawned_children.push(WorkerSpawnedChild { + worker_name: "missing-child".into(), + socket_path: temp.path().join("missing-child.sock"), + scope_delegated: Vec::new(), + callback_address: temp.path().join("unused-parent.sock"), + }); + metadata.resolved_manifest_snapshot = + Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap()); + store.write(&metadata).unwrap(); + drop(worker); + Self { + temp, + store, + manifest, + session_id, + segment_id, + orphan, + model_calls, + } + } + + async fn prepare(&self) -> PreparedRestoredWorker { + Worker::prepare_restored_from_worker_metadata_with_context_and_model_client( + &self.manifest.worker.name, + self.manifest.clone(), + self.store.clone(), + PromptCatalogSource::builtins_only(), + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::local( + self.temp.path().to_path_buf(), + self.temp.path().to_path_buf(), + ), + Some(Box::new(CountedClient(self.model_calls.clone()))), + ) + .await + .unwrap() + } + + fn allocate( + &self, + worker: &Worker, CandidateStore>, + ) -> Result { + Ok(worker_allocation::install_top_level_at_for_test( + self.temp.path().join("allocation.json"), + worker.manifest.worker.name.clone(), + std::process::id(), + self.temp.path().join("worker.sock"), + worker.scope.snapshot().allow_rules(), + worker.segment_id(), + )?) + } + + fn snapshot(&self) -> std::collections::BTreeMap> { + fn visit( + root: &Path, + path: &Path, + files: &mut std::collections::BTreeMap>, + ) { + for entry in std::fs::read_dir(path).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + visit(root, &path, files); + } else { + files.insert( + path.strip_prefix(root).unwrap().to_path_buf(), + std::fs::read(path).unwrap(), + ); + } + } + } + let mut files = std::collections::BTreeMap::new(); + for name in ["sessions", "workers"] { + visit(self.temp.path(), &self.temp.path().join(name), &mut files); + } + files + } + } + + #[tokio::test] + async fn prepare_and_discard_preserve_session_queue_pins_and_metadata() { + let fixture = Fixture::new().await; + let before = fixture.snapshot(); + let mut candidate = fixture.prepare().await; + assert!( + candidate + .worker() + .restore_allocation_cleanup_handle() + .is_none() + ); + assert!(!fixture.temp.path().join("worker.sock").exists()); + assert_eq!( + candidate.worker().pending_submissions().submissions.len(), + 1 + ); + assert_eq!( + candidate + .worker() + .session + .pending_activations + .lock() + .unwrap() + .pending_notifications + .len(), + 1 + ); + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + let session = candidate + .worker() + .workdir_sessions() + .resolve(None) + .unwrap() + .session; + assert!( + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root() + }) + .await + .is_ok() + ); + candidate.discard().await.unwrap(); + candidate.discard().await.unwrap(); + assert!( + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root() + }) + .await + .is_err() + ); + assert_eq!(fixture.snapshot(), before); + } + + #[tokio::test] + async fn failed_discard_allows_only_cleanup_retry_not_binding_or_activation() { + for retry_discard in [true, false] { + let fixture = Fixture::new().await; + let before = fixture.snapshot(); + let mut candidate = fixture.prepare().await; + let provider = Arc::new(FailCloseOnce { + inner: candidate.workdir_sessions().resolve(None).unwrap().session, + attempts: AtomicUsize::new(0), + }); + candidate + .worker_mut() + .unwrap() + .bind_single_workdir_session(Some(provider.clone())); + let cleanup = candidate.workdir_sessions(); + assert!(matches!( + candidate.discard().await, + Err(WorkerError::FeatureLifecycle(_)) + )); + assert_eq!(provider.attempts.load(Ordering::SeqCst), 1); + assert!(!cleanup.is_empty()); + assert_eq!(fixture.snapshot(), before); + assert!(matches!( + candidate.worker_mut(), + Err(WorkerError::RestoreCandidateClosing) + )); + assert_eq!( + candidate.manifest().worker.name, + fixture.manifest.worker.name + ); + assert!(Arc::ptr_eq(&candidate.workdir_sessions(), &cleanup)); + if retry_discard { + candidate.discard().await.unwrap(); + assert!(cleanup.is_empty()); + assert_eq!(provider.attempts.load(Ordering::SeqCst), 2); + assert!(matches!( + candidate.worker_mut(), + Err(WorkerError::RestoreCandidateClosing) + )); + candidate.discard().await.unwrap(); + } else { + let allocations = AtomicUsize::new(0); + let error = match candidate + .activate_with_allocation(|worker| { + allocations.fetch_add(1, Ordering::SeqCst); + fixture.allocate(worker) + }) + .await + { + Err(error) => error, + Ok(_) => panic!("partially closed candidate was activated"), + }; + let WorkerError::ConstructionCleanup { + source, + allocation, + sessions, + } = error + else { + panic!("cleanup ownership lost"); + }; + assert!(matches!(*source, WorkerError::RestoreCandidateClosing)); + assert!(allocation.is_none()); + assert_eq!(allocations.load(Ordering::SeqCst), 0); + assert_eq!(provider.attempts.load(Ordering::SeqCst), 1); + assert!(Arc::ptr_eq(&sessions, &cleanup)); + sessions.close_all().await.unwrap(); + assert!(cleanup.is_empty()); + assert_eq!(provider.attempts.load(Ordering::SeqCst), 2); + } + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + } + } + + #[tokio::test] + async fn dropping_candidate_closes_workdir_without_consuming_saved_inputs() { + let fixture = Fixture::new().await; + let before = fixture.snapshot(); + let candidate = fixture.prepare().await; + let sessions = candidate.worker().workdir_sessions(); + let session = sessions.resolve(None).unwrap().session; + drop(candidate); + tokio::time::timeout(Duration::from_secs(5), async { + while !sessions.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("candidate Drop must finish direct close"); + assert!( + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root() + }) + .await + .is_err() + ); + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn activation_alone_reconciles_orphan_pins_and_updates_active_metadata() { + let fixture = Fixture::new().await; + let before_metadata = fixture + .store + .read_by_name(&fixture.manifest.worker.name) + .unwrap() + .unwrap(); + let before_entries = restore_snapshot_value( + &fixture + .store + .read_all(fixture.session_id, fixture.segment_id) + .unwrap(), + ) + .unwrap(); + let candidate = fixture.prepare().await; + assert!(matches!( + fixture + .store + .delete_uploaded_file(fixture.session_id, &fixture.orphan.artifact_id), + Err(StoreError::ArtifactAlreadyCommitted) + )); + let worker = candidate + .activate_with_allocation(|worker| fixture.allocate(worker)) + .await + .unwrap(); + assert!(worker.restore_allocation_cleanup_handle().is_some()); + let metadata = fixture + .store + .read_by_name(&fixture.manifest.worker.name) + .unwrap() + .unwrap(); + assert_ne!(metadata.workspace_root, before_metadata.workspace_root); + assert_eq!(metadata.active, before_metadata.active); + assert_eq!(before_metadata.spawned_children.len(), 1); + assert!(metadata.spawned_children.is_empty()); + assert_eq!(metadata.reclaimed_children.len(), 1); + assert!( + fixture + .store + .delete_uploaded_file(fixture.session_id, &fixture.orphan.artifact_id) + .unwrap() + ); + assert_eq!(worker.pending_submissions().submissions.len(), 1); + assert_eq!( + worker + .session + .pending_activations + .lock() + .unwrap() + .pending_notifications + .len(), + 1 + ); + assert_eq!( + restore_snapshot_value( + &fixture + .store + .read_all(fixture.session_id, fixture.segment_id) + .unwrap() + ) + .unwrap(), + before_entries + ); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + worker.workdir_sessions().close_all().await.unwrap(); + let allocation = worker.restore_allocation_cleanup_handle().unwrap(); + assert!(matches!( + allocation.release(), + Err(ScopeLockError::AllocationGuardAttached { .. }) + )); + drop(worker); + allocation.release().unwrap(); + } + + #[tokio::test] + async fn metadata_or_same_entry_count_history_drift_rejects_before_allocation() { + for component in ["metadata", "segment", "manifest"] { + let fixture = Fixture::new().await; + let mut candidate = fixture.prepare().await; + match component { + "metadata" => { + fixture + .store + .update_by_name(&fixture.manifest.worker.name, |metadata| { + metadata.workspace_id = Some("changed-workspace".into()); + }) + .unwrap(); + } + "segment" => { + let mut entries = fixture + .store + .read_all(fixture.session_id, fixture.segment_id) + .unwrap(); + let LogEntry::AnnotatedSegmentStart { system_prompt, .. } = &mut entries[0] + else { + panic!("fixture seed"); + }; + *system_prompt = Some("different same-length history".into()); + fixture + .store + .create_segment(fixture.session_id, fixture.segment_id, &entries) + .unwrap(); + } + "manifest" => { + candidate.worker_mut().unwrap().manifest.model.model_id = + Some("changed-model".into()); + } + _ => unreachable!(), + } + let before = fixture.snapshot(); + let allocations = AtomicUsize::new(0); + let error = match candidate + .activate_with_allocation(|worker| { + allocations.fetch_add(1, Ordering::SeqCst); + fixture.allocate(worker) + }) + .await + { + Err(error) => error, + Ok(_) => panic!("stale candidate accepted"), + }; + let WorkerError::ConstructionCleanup { + source, + allocation, + sessions, + } = error + else { + panic!("cleanup ownership lost"); + }; + assert!( + matches!(*source, WorkerError::RestoreCandidateChanged { component: changed, .. } if changed == component) + ); + assert!(allocation.is_none()); + assert_eq!(allocations.load(Ordering::SeqCst), 0); + sessions.close_all().await.unwrap(); + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + } + } + + #[tokio::test] + async fn post_allocation_metadata_drift_retains_cleanup_without_mutating_pins() { + let fixture = Fixture::new().await; + let candidate = fixture.prepare().await; + let error = match candidate + .activate_with_allocation(|worker| { + let guard = fixture.allocate(worker)?; + fixture + .store + .update_by_name(&fixture.manifest.worker.name, |metadata| { + metadata.workspace_id = Some("changed-under-allocation".into()); + })?; + Ok(guard) + }) + .await + { + Err(error) => error, + Ok(_) => panic!("stale candidate accepted"), + }; + let before = fixture.snapshot(); + let WorkerError::ConstructionCleanup { + source, + allocation, + sessions, + } = error + else { + panic!("cleanup ownership lost"); + }; + assert!(matches!( + *source, + WorkerError::RestoreCandidateChanged { + component: "metadata", + .. + } + )); + let allocation = + allocation.expect("allocation retained after second validation failure"); + assert!( + worker_allocation::LockFileGuard::open( + &fixture.temp.path().join("allocation.json") + ) + .unwrap() + .data() + .find(&fixture.manifest.worker.name) + .is_some() + ); + assert!(matches!( + fixture + .store + .delete_uploaded_file(fixture.session_id, &fixture.orphan.artifact_id), + Err(StoreError::ArtifactAlreadyCommitted) + )); + sessions.close_all().await.unwrap(); + worker_allocation::fail_next_save_for_test( + &fixture.temp.path().join("allocation.json"), + ); + assert!(allocation.release().is_err()); + allocation.release().unwrap(); + assert_eq!(fixture.snapshot(), before); + } + + #[tokio::test] + async fn pending_session_is_rejected_without_repair_or_session_creation() { + let fixture = Fixture::new().await; + fixture + .store + .update_by_name(&fixture.manifest.worker.name, |metadata| { + metadata.active.as_mut().unwrap().segment_id = None; + }) + .unwrap(); + let before = fixture.snapshot(); + let result = + Worker::prepare_restored_from_worker_metadata_with_context_and_model_client( + &fixture.manifest.worker.name, + fixture.manifest.clone(), + fixture.store.clone(), + PromptCatalogSource::builtins_only(), + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + Some(Box::new(CountedClient(fixture.model_calls.clone()))), + ) + .await; + assert!(matches!( + result, + Err(WorkerError::WorkerMetadataPending { .. }) + )); + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + } + } + // Constructor finalization uses real Worker, allocation, Workdir, and storage // implementations. Only the model and metadata-write failure boundary are substituted. mod construction_cleanup_tests { diff --git a/crates/worker/tests/controller_test.rs b/crates/worker/tests/controller_test.rs index 1009b79fd..2fe26e3d6 100644 --- a/crates/worker/tests/controller_test.rs +++ b/crates/worker/tests/controller_test.rs @@ -1,3 +1,4 @@ +use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -150,6 +151,19 @@ impl LlmClient for MockClient { // Helpers // --------------------------------------------------------------------------- +#[path = "../../../tests/support/restore_environment.rs"] +mod restore_test_environment; +use restore_test_environment::require_restore_test_environment; + +fn owned_relative_runtime_base() -> (tempfile::TempDir, PathBuf) { + require_restore_test_environment(); + let temp = tempfile::tempdir_in(".").unwrap(); + // The relative reference and the TempDir name the same owned tree, without + // putting the checkout prefix into sockaddr_un or changing process cwd. + let runtime_base = Path::new(temp.path().file_name().unwrap()).join("runtime"); + (temp, runtime_base) +} + fn simple_text_events() -> Vec { vec![ LlmEvent::text_block_start(0), @@ -295,6 +309,7 @@ async fn make_worker_with_manifest_and_workspace_context( manifest: WorkerManifest, workspace_context: WorkerWorkspaceContext, ) -> (Worker, std::path::PathBuf) { + require_restore_test_environment(); let store_tmp = tempfile::tempdir().unwrap(); let store = CombinedStore::new( FsStore::new(store_tmp.path()).unwrap(), @@ -320,18 +335,21 @@ async fn make_worker_with_manifest_and_workspace_context( } async fn spawn_controller(worker: Worker) -> WorkerHandle { - let tmp = tempfile::tempdir().unwrap(); - let runtime_base = tmp.path().to_owned(); - std::mem::forget(tmp); - let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base) + let (tmp, runtime_base) = owned_relative_runtime_base(); + let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base) .await .unwrap(); + // Retain ownership until controller shutdown (or test-runtime teardown). + tokio::spawn(async move { + let _ = shutdown_rx.await; + drop(tmp); + }); handle } #[tokio::test] async fn canonical_worker_handle_uploads_and_submits_attachment() { - let root = tempfile::tempdir().unwrap(); + let (root, runtime_base) = owned_relative_runtime_base(); let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap(); let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap(); let store = CombinedStore::new( @@ -353,7 +371,7 @@ async fn canonical_worker_handle_uploads_and_submits_attachment() { ) .await .unwrap(); - let (handle, shutdown_rx) = WorkerController::spawn(worker, &root.path().join("runtime")) + let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base) .await .unwrap(); let mut rx = handle.subscribe(); @@ -1761,11 +1779,12 @@ permission = "write" "#; let client = MockClient::new(simple_text_events()); let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0; - let tmp = tempfile::tempdir().unwrap(); - let result = WorkerController::spawn(worker, tmp.path()).await; + let (_tmp, runtime_base) = owned_relative_runtime_base(); + let result = WorkerController::spawn(worker, &runtime_base).await; assert!( result.is_ok(), - "feature exposure must not imply delegation authority" + "feature exposure must not imply delegation authority: {:?}", + result.as_ref().err() ); } diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index 02c40beb4..dd117ae7b 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -19,6 +19,9 @@ pub mod repositories; pub mod repository_access; pub mod repository_source; pub mod resource_broker; +#[cfg(test)] +#[path = "../../../tests/support/restore_environment.rs"] +mod restore_test_environment; pub mod retention; pub mod runtime_settings; pub mod runtime_subscription; diff --git a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs index 91093f5fc..d1a4d6c61 100644 --- a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs +++ b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs @@ -11,6 +11,20 @@ use worker_runtime::worker_backend::{ WorkerRuntimeExecutionBackend, }; +pub(super) fn connected_restore_tempdir() -> tempfile::TempDir { + crate::restore_test_environment::require_restore_test_environment(); + let checkout = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap(); + tempfile::Builder::new() + .prefix("c") + .rand_bytes(3) + .tempdir_in(checkout.join("target")) + .unwrap() +} + const WATCHDOG: Duration = Duration::from_secs(45); #[derive(Clone)] @@ -82,27 +96,91 @@ impl RuntimeWorkerFactory for CapturingFactory { ) -> std::result::Result<(), String> { self.inner.preflight_restore(request).await } - async fn restore_controller( + async fn recover_restore_source( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.inner.recover_restore_source(request).await + } + + async fn prepare_restore( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + let worker_ref = request.worker_ref.clone(); + let inner = self.inner.prepare_restore(request).await?; + Ok(Box::new(CapturingCandidate { + inner, + worker_ref, + captures: self.captures.clone(), + })) + } + + async fn cleanup_failed_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.cleanup_failed_restore(request).await + } + + async fn reconcile_stopped_worker( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.reconcile_stopped_worker(worker_ref).await + } + + fn activate_restored_controller( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + workspace_id: Option<&str>, + handle: &worker::WorkerHandle, + ) { + self.inner + .activate_restored_controller(worker_ref, workspace_id, handle); + } +} + +struct CapturingCandidate { + inner: Box, + worker_ref: worker_runtime::identity::WorkerRef, + captures: Arc, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for CapturingCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.inner.initial_state() + } + + async fn activate( + &mut self, ) -> std::result::Result< RuntimeWorkerController, worker_runtime::worker_backend::RuntimeRestoreFailure, > { - let worker_ref = request.worker_ref.clone(); - let controller = self.inner.restore_controller(request).await?; + let controller = self.inner.activate().await?; self.captures .handles .lock() .unwrap() - .push((worker_ref.clone(), controller.handle.clone())); + .push((self.worker_ref.clone(), controller.handle.clone())); self.captures .restored_clients .lock() .unwrap() - .push((worker_ref, controller.workspace_client.clone())); + .push((self.worker_ref.clone(), controller.workspace_client.clone())); Ok(controller) } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.discard().await + } } struct Resources { @@ -262,7 +340,7 @@ async fn call(tools: &BTreeMap>, name: &str, args: Value) #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn restored_runtime_parent_lists_and_inputs_existing_child() { - let root = tempfile::tempdir().unwrap(); + let root = connected_restore_tempdir(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let base = format!("http://{}", listener.local_addr().unwrap()); let mut config = test_server_config(root.path()); diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs index 4cc4b5c24..3b9215113 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs @@ -11,6 +11,8 @@ use agen::llm_client::{ use serde_json::{Value, json}; use std::sync::{Arc, Mutex}; +use super::super::restored_parent_control_tests::connected_restore_tempdir; + #[derive(Clone, Default)] struct ConnectedJournalModel { dialogs: Arc>>, @@ -320,7 +322,7 @@ async fn connected_wait_for_recording( #[tokio::test(flavor = "multi_thread", worker_threads = 6)] async fn backend_real_runtime_journal_release_and_next_body_persisted_startup() { - let dir = tempfile::tempdir().unwrap(); + let dir = connected_restore_tempdir(); init_clean_git_workspace(dir.path()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let mut config = test_server_config(dir.path()); @@ -896,3 +898,1579 @@ fn connected_host_reader( ) .unwrap() } + +// This wrapper changes timing only. Production owns reconstruction, recovery, +// activation and cleanup; the real SQLite trust binding supplies the denial. +#[derive(Default)] +struct RestoreAuthProbe { + prepared: tokio::sync::Notify, + release: tokio::sync::Notify, + prepares: std::sync::atomic::AtomicUsize, + activations: std::sync::atomic::AtomicUsize, + publications: std::sync::atomic::AtomicUsize, + handles: Mutex>, + controller_tasks: Mutex>, + operations: Mutex>, + repairs: Mutex>, + repair_operations: Mutex>, + gate_repair: std::sync::atomic::AtomicBool, + repairing: tokio::sync::Notify, + release_repair: tokio::sync::Notify, +} + +struct RestoreAuthFactory { + inner: worker_runtime::worker_backend::ProfileRuntimeWorkerFactory, + probe: Arc, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeWorkerFactory for RestoreAuthFactory { + fn observe_workspace_prompt_projection( + &self, + projection: worker::WorkspacePromptProjection, + ) -> std::result::Result<(), String> { + self.inner.observe_workspace_prompt_projection(projection) + } + + async fn fetch_workspace_config( + &self, + request: worker_runtime::execution::WorkspaceConfigFetchRequest, + ) -> std::result::Result { + self.inner.fetch_workspace_config(request).await + } + + fn retained_session_snapshot( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result< + session_store::RetainedSessionSnapshot, + session_store::RetainedSnapshotReadError, + > { + self.inner.retained_session_snapshot(worker_ref) + } + + fn retained_session_history_page( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + cursor: Option<&str>, + limit: Option, + ) -> std::result::Result + { + self.inner + .retained_session_history_page(worker_ref, cursor, limit) + } + + fn retained_session_attachment( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + session_id: &str, + attachment_id: &str, + ) -> std::result::Result< + session_store::RetainedSessionAttachment, + session_store::RetainedAttachmentReadError, + > { + self.inner + .retained_session_attachment(worker_ref, session_id, attachment_id) + } + + async fn spawn_controller( + &self, + mut request: worker_runtime::execution::WorkerExecutionSpawnRequest, + ) -> std::result::Result { + if request.request.backend_job.is_some() { + return self.inner.spawn_controller(request).await; + } + // Save a no-auth model too, so a restore regression cannot reach developer + // credentials. Only the external model stream is replaced, not Host I/O. + let archive = worker_runtime::profile_archive::ProfileSourceArchive::build( + worker_runtime::profile_archive::ProfileSourceArchiveInput { + id: "profile-source-archive:restore-host-auth".into(), + entrypoints: BTreeMap::from([("builtin:companion".into(), "profiles/body.dcdl".into())]), + imports: BTreeMap::new(), + sources: BTreeMap::from([("profiles/body.dcdl".into(), r#"{ + slug = "body"; description = "Connected restore Host authority"; scope = "workspace_read"; + model = { scheme = "anthropic"; model_id = "test-model"; auth = { kind = "none"; }; }; + engine = { max_tokens = 100; }; + feature = { subjektiv = { enabled = true; extraction = { enabled = false; }; }; }; + }"#.into())]), + }, + ).map_err(|error| error.to_string())?; + request.request.profile_source = + worker_runtime::catalog::ProfileSourceArchiveSource::Embedded { archive }; + let controller = self.inner.spawn_controller(request).await?; + self.probe + .controller_tasks + .lock() + .unwrap() + .push(controller.controller_task.abort_handle()); + self.probe + .handles + .lock() + .unwrap() + .push(controller.handle.clone()); + Ok(controller) + } + + async fn preflight_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), String> { + self.inner.preflight_restore(request).await + } + + async fn recover_restore_source( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.probe + .repair_operations + .lock() + .unwrap() + .push(serde_json::to_value(request.operation_id).unwrap()); + if self + .probe + .gate_repair + .swap(false, std::sync::atomic::Ordering::SeqCst) + { + self.probe.repairing.notify_one(); + self.probe.release_repair.notified().await; + } + let repaired = self.inner.recover_restore_source(request).await?; + self.probe.repairs.lock().unwrap().push(repaired); + Ok(repaired) + } + + async fn prepare_restore( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + let operation = serde_json::to_value(request.operation_id).unwrap(); + let inner = self.inner.prepare_restore(request).await?; + self.probe.operations.lock().unwrap().push(operation); + if self + .probe + .prepares + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + == 0 + { + self.probe.prepared.notify_one(); + self.probe.release.notified().await; + } + Ok(Box::new(RestoreAuthCandidate { + inner, + probe: self.probe.clone(), + })) + } + + async fn cleanup_failed_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.cleanup_failed_restore(request).await + } + + async fn reconcile_stopped_worker( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.reconcile_stopped_worker(worker_ref).await + } + + fn activate_restored_controller( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + workspace_id: Option<&str>, + handle: &worker::WorkerHandle, + ) { + self.inner + .activate_restored_controller(worker_ref, workspace_id, handle); + } +} + +struct RestoreAuthCandidate { + inner: Box, + probe: Arc, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for RestoreAuthCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.inner.initial_state() + } + + async fn activate( + &mut self, + ) -> std::result::Result< + worker_runtime::worker_backend::RuntimeWorkerController, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + self.probe + .activations + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let controller = self.inner.activate().await?; + self.probe + .publications + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.probe + .controller_tasks + .lock() + .unwrap() + .push(controller.controller_task.abort_handle()); + self.probe + .handles + .lock() + .unwrap() + .push(controller.handle.clone()); + Ok(controller) + } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.discard().await + } +} + +#[derive(Clone, Default)] +struct RestoreAuthModel { + calls: Arc, + jobs: ConnectedJournalModel, +} + +#[async_trait::async_trait] +impl LlmClient for RestoreAuthModel { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + + async fn stream( + &self, + request: Request, + ) -> std::result::Result { + use futures::StreamExt; + if request.tools.iter().any(|tool| { + matches!( + tool.name.as_str(), + "ReadJournalState" | "SubmitBackendJobResult" | "SubmitMemorySurface" + ) + }) { + return self.jobs.stream(request).await; + } + // Measure the saved body, not independent Journal jobs admitted by its + // seed turn before Stop; those have a different execution owner. + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(Box::pin( + futures::stream::iter([ + Ok(Event::text_block_start(0)), + Ok(Event::text_delta( + 0, + "fixture turn held for durable queue setup", + )), + ]) + .chain(futures::stream::pending()), + )) + } +} + +struct ConnectedRestoreServer { + server: tokio::task::JoinHandle<()>, + probe: Arc, +} + +impl Drop for ConnectedRestoreServer { + fn drop(&mut self) { + // Unblock a prepared candidate even if an assertion unwinds at the gate. + self.probe.release.notify_one(); + self.probe.release_repair.notify_one(); + self.server.abort(); + for task in self.probe.controller_tasks.lock().unwrap().iter() { + task.abort(); + } + } +} + +async fn restore_auth_event( + events: &mut tokio::sync::broadcast::Receiver, + predicate: impl Fn(&protocol::Event) -> bool, +) -> protocol::Event { + tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let event = events.recv().await.expect("real Controller event stream"); + assert!( + !matches!( + event, + protocol::Event::Error { .. } + | protocol::Event::SubmissionRejected { .. } + | protocol::Event::NotificationRejected { .. } + ), + "{event:?}" + ); + if predicate(&event) { + return event; + } + } + }) + .await + .expect("real Controller acknowledgement timed out") +} + +// Only initial creation is substituted for the pending-head case: it writes a +// genuine first-Session Worker, but publishes no Controller/head. Every restore, +// repair, Host authorization, activation and Stop delegates to production. +struct PendingHeadSeedBackend { + inner: Arc>, + seed: std::sync::atomic::AtomicBool, + root: std::path::PathBuf, + model: RestoreAuthModel, +} + +impl worker_runtime::execution::WorkerExecutionBackend for PendingHeadSeedBackend { + fn backend_id(&self) -> &str { + self.inner.backend_id() + } + fn spawn_worker( + &self, + request: worker_runtime::execution::WorkerExecutionSpawnRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + if !self.seed.swap(false, std::sync::atomic::Ordering::SeqCst) { + return self.inner.spawn_worker(request); + } + let root = self + .root + .join("workers") + .join(request.worker_ref.worker_id.to_string()); + let name = format!("worker-runtime-{}", request.worker_ref.worker_id); + let model = self.model.clone(); + let settings = request.request.memory_settings.clone().unwrap(); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + use session_store::WorkerMetadataStore as _; + let mut manifest = manifest::WorkerManifest::from_toml(&format!( + r#" +[worker] +name = "{name}" +pwd = "./" +[scope] +allow = [] +[model] +scheme = "anthropic" +model_id = "test-model" +auth = {{ kind = "none" }} +[engine] +max_tokens = 100 +[feature.subjektiv.profile] +enabled = true +[feature.subjektiv.profile.extraction] +enabled = false +"# + )) + .unwrap(); + manifest + .feature + .subjektiv + .bind_workspace_settings(settings) + .unwrap(); + let sessions = + session_store::WorkerSessionStore::new(root.join("session")).unwrap(); + let metadata = session_store::WorkerAggregateStore::new(&root, &name).unwrap(); + let engine = agen::Engine::< + _, + agen::state::Mutable, + worker::SessionHistoryMetadata, + >::new_annotated(model); + let mut head = worker::Worker::new( + manifest.clone(), + engine, + session_store::CombinedStore::new(sessions.clone(), metadata.clone()), + worker::WorkerWorkspaceContext::unavailable( + Some(worker::WorkspaceId::new(TEST_WORKSPACE_ID).unwrap()), + "initial creation interrupted before first head", + ), + worker::WorkerFilesystemAuthority::None, + manifest::Scope::empty(), + ) + .await + .unwrap(); + head.enable_worker_metadata_write_through().unwrap(); + let mut saved = metadata.read_by_name(&name).unwrap().unwrap(); + assert!(saved.active.as_ref().unwrap().segment_id.is_none()); + saved.resolved_manifest_snapshot = Some( + manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap(), + ); + metadata.write(&saved).unwrap(); + drop(head); + }); + }) + .join() + .unwrap(); + worker_runtime::execution::WorkerExecutionSpawnResult::Connected { + worker_state: protocol::WorkerStateSnapshot::initial(), + workdir_attachments: Vec::new(), + } + } + fn fetch_workspace_config( + &self, + request: worker_runtime::execution::WorkspaceConfigFetchRequest, + ) -> std::result::Result { + self.inner.fetch_workspace_config(request) + } + fn observe_workspace_prompt_projection( + &self, + projection: worker::WorkspacePromptProjection, + ) -> std::result::Result<(), String> { + self.inner.observe_workspace_prompt_projection(projection) + } + fn preflight_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), worker_runtime::execution::WorkerExecutionResult> { + self.inner.preflight_restore(request) + } + fn completed_restore_failure( + &self, + id: worker_runtime::execution::WorkerLifecycleOperationId, + worker: &worker_runtime::identity::WorkerRef, + ) -> Option { + self.inner.completed_restore_failure(id, worker) + } + fn restore_worker( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.inner.restore_worker(request) + } + fn reconcile_restore( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.inner.reconcile_restore(request) + } + fn activate_restored_worker( + &self, + id: worker_runtime::execution::WorkerLifecycleOperationId, + worker: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.activate_restored_worker(id, worker) + } + fn dispatch_input( + &self, + worker: &worker_runtime::identity::WorkerRef, + input: worker_runtime::interaction::WorkerInput, + ) -> worker_runtime::execution::WorkerExecutionResult { + self.inner.dispatch_input(worker, input) + } + fn dispatch_method( + &self, + worker: &worker_runtime::identity::WorkerRef, + method: protocol::Method, + ) -> worker_runtime::execution::WorkerExecutionResult { + self.inner.dispatch_method(worker, method) + } + fn stop_worker_operation( + &self, + request: worker_runtime::execution::WorkerExecutionStopRequest, + ) -> worker_runtime::execution::WorkerExecutionResult { + self.inner.stop_worker_operation(request) + } + fn worker_snapshot( + &self, + worker: &worker_runtime::identity::WorkerRef, + ) -> Option { + self.inner.worker_snapshot(worker) + } +} + +// One real connected seed and stopped queue setup shared by the publication +// denial and explicit-repair regressions. Only timing and response delivery are +// intercepted: every callback still passes the production signer/router/store. +struct ConnectedRestoreFixture { + api: WorkspaceApi, + execution: + Arc>, + probe: Arc, + model: RestoreAuthModel, + worker: RuntimeWorkerRef, + subject_id: String, + requests: Arc>>, + callbacks: Arc>>, + lose_response: Arc, + _server: ConnectedRestoreServer, + _dir: tempfile::TempDir, +} + +async fn connected_restore_fixture() -> ConnectedRestoreFixture { + connected_restore_fixture_with_pending_head(false).await +} + +async fn connected_restore_fixture_with_pending_head( + pending_head: bool, +) -> ConnectedRestoreFixture { + use std::sync::atomic::Ordering::SeqCst; + + // Guard the resolved global paths before factory/config/allocation access. + // No Git or Workdir attachment is needed for this connected regression. + let dir = connected_restore_tempdir(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut config = test_server_config(dir.path()); + config.repositories.clear(); + config.backend_base_url = Some(format!("http://{}", listener.local_addr().unwrap())); + let control = Arc::new(test_control_store(&config)); + seed_test_registered_workspace(control.as_ref(), &config) + .await + .unwrap(); + let broker = BackendResourceBroker::default(); + let dispatcher = Arc::new( + crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new( + config.clone(), + control.clone(), + ), + ); + let model = RestoreAuthModel::default(); + let model_factory = model.clone(); + let audience = embedded_runtime_request_audience(&config).unwrap(); + let probe = Arc::new(RestoreAuthProbe::default()); + let factory = worker_runtime::worker_backend::ProfileRuntimeWorkerFactory::new( + config.workspace_execution_root.clone(), + ) + .with_controller_transport(worker::WorkerControllerTransport::InProcess) + .with_embedded_worker_mutation_dispatcher(EMBEDDED_WORKER_RUNTIME_ID, dispatcher.clone()) + .with_workspace_request_client( + worker_runtime::workspace_request::RuntimeWorkspaceRequestClient::new( + TEST_WORKSPACE_ID, + audience.clone(), + EMBEDDED_WORKER_RUNTIME_ID, + ) + .with_runtime_request_source(&EMBEDDED_RUNTIME_REQUEST_IDENTITY, audience), + ) + .with_runtime_store_dir(config.embedded_runtime_store_root.clone()) + .with_resource_client(Arc::new(broker.clone())) + .with_spawn_model_client_factory(move || Box::new(model_factory.clone())); + let execution = Arc::new( + worker_runtime::worker_backend::WorkerRuntimeExecutionBackend::new(RestoreAuthFactory { + inner: factory, + probe: probe.clone(), + }) + .unwrap(), + ); + let seeded_execution = Arc::new(PendingHeadSeedBackend { + inner: execution.clone(), + seed: std::sync::atomic::AtomicBool::new(pending_head), + root: config.embedded_runtime_store_root.clone(), + model: model.clone(), + }); + let api = WorkspaceApi::new_with_execution_backend_and_broker( + config, + control, + seeded_execution, + broker, + HostExecutors::default(), + Some(dispatcher), + ) + .await + .unwrap(); + seed_test_delivery_binding(&api); + let requests = Arc::new(Mutex::new(Vec::new())); + let observed = requests.clone(); + let callbacks = Arc::new(Mutex::new(Vec::new())); + let recorded = callbacks.clone(); + let lose_response = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let lose_once = lose_response.clone(); + let app = build_router(api.clone()).layer(axum::middleware::from_fn( + move |request: axum::extract::Request, next: axum::middleware::Next| { + let observed = observed.clone(); + let recorded = recorded.clone(); + let lose_once = lose_once.clone(); + async move { + let path = request.uri().path().to_owned(); + let is_session = path.ends_with("/subjektiv/sessions"); + let request = if is_session { + let (parts, body) = request.into_parts(); + let bytes = to_bytes(body, usize::MAX).await.unwrap(); + recorded + .lock() + .unwrap() + .push(serde_json::from_slice(&bytes).unwrap()); + axum::http::Request::from_parts(parts, Body::from(bytes)) + } else { + request + }; + let response = next.run(request).await; + observed.lock().unwrap().push((path, response.status())); + if is_session + && response.status() == StatusCode::OK + && lose_once.swap(false, SeqCst) + { + // The real handler has already persisted its immutable SQLite + // attribution. Lose only its acknowledgement, never fake a Host. + axum::http::Response::builder() + .status(StatusCode::BAD_GATEWAY) + .body(Body::from("fixture lost committed Host response")) + .unwrap() + } else { + response + } + } + }, + )); + let server = ConnectedRestoreServer { + server: tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }), + probe: probe.clone(), + }; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject(crate::subjektiv::SubjectRole::new("restore-host-auth").unwrap()) + .unwrap(); + let owner = + seed_test_workspace_owner_token(api.store.as_ref(), "restore-host-auth-owner").await; + let update = server_api::authorization::SubjectPermissionsUpdate { + operation_id: "restore-host-auth-permissions".into(), + expected_change_id: None, + execution: [ + "task", + "subjektiv", + "web", + "image", + "sub_worker", + "worker", + "workspace_worker_discovery", + "objective", + "manage_workdir", + "workdir_catalog", + "drive", + "ticket", + ] + .into_iter() + .map(|name| server_api::authorization::ScopedPermission { + operation: name.into(), + resource: server_api::authorization::PermissionResource::Feature, + }) + .collect(), + delegation: vec![], + }; + let response = build_router(api.clone()) + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(format!( + "/api/w/{TEST_WORKSPACE_ID}/subjektiv/subjects/{}/permissions", + subject.id + )) + .header("authorization", format!("Bearer {owner}")) + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_vec(&update).unwrap())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let Json(created) = scoped_create_workspace_worker( + State(api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.into(), + }), + HeaderMap::new(), + Json(CreateWorkspaceWorkerRequest { + runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(), + display_name: "restore-auth-body".into(), + singleton_key: None, + profile: Some("builtin:companion".into()), + ticket_assignment: None, + initial_submit: vec![], + workdir_attachments: vec![], + feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest { + subjektiv: Some(server_api::SubjektivWorkerConnectionRequest { + subject_id: subject.id.clone(), + }), + }, + control_operation_id: None, + execution_grant: None, + }), + ) + .await + .unwrap(); + let worker = RuntimeWorkerRef::new(created.runtime_id, created.worker_id); + if pending_head { + assert_eq!( + api.runtime + .stop_worker(&worker, WorkerLifecycleRequest::default()) + .unwrap() + .state, + InternalWorkerOperationState::Accepted + ); + } else { + let handle = probe.handles.lock().unwrap()[0].clone(); + let mut events = handle.subscribe(); + handle + .send(protocol::Method::submit_text( + "setup-running", + "hold for queue setup", + )) + .await + .unwrap(); + restore_auth_event(&mut events, |event| { + matches!(event, protocol::Event::TextDelta { .. }) + }) + .await; + let command = protocol::WorkerCommandEnvelope::new( + handle.shared_state.snapshot().last_command_id + 1, + ); + handle + .send(protocol::Method::Pause { command }) + .await + .unwrap(); + restore_auth_event(&mut events, |event| { + matches!(event, protocol::Event::WorkerState { snapshot } + if snapshot.catalog_status() == protocol::WorkerStatus::Paused) + }) + .await; + handle + .send(protocol::Method::submit_text( + "saved-submit", + "must remain queued", + )) + .await + .unwrap(); + restore_auth_event(&mut events, |event| { + matches!(event, protocol::Event::SubmissionAccepted { + submission_request_id, disposition: protocol::SubmissionDisposition::Queued, .. + } if submission_request_id == "saved-submit") + }) + .await; + handle + .send(protocol::Method::Notify { + notification_request_id: "saved-notice".into(), + message: "must remain queued too".into(), + }) + .await + .unwrap(); + restore_auth_event(&mut events, |event| { + matches!(event, protocol::Event::NotificationAccepted { + notification_request_id + } if notification_request_id == "saved-notice") + }) + .await; + let stopped = api + .runtime + .stop_worker(&worker, WorkerLifecycleRequest::default()) + .unwrap(); + assert_eq!(stopped.state, InternalWorkerOperationState::Accepted); + assert!(handle.protocol_is_closed()); + // Pause/Shutdown can admit real Journal work for the seed turn. Finish that + // owned work before measuring the restore window; it is not restore I/O. + let attr = store + .subject_session_attributions(&subject.id) + .unwrap() + .into_iter() + .find(|attr| attr.worker_id == worker.worker_id) + .unwrap(); + let host = JournalHostContext::validated_session(&store, attr.clone()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(30), async { + let mut ticks = tokio::time::interval(std::time::Duration::from_millis(25)); + loop { + ticks.tick().await; + let jobs = api + .store + .list_backend_journal_jobs(TEST_WORKSPACE_ID, &attr.session_id, None, 100) + .unwrap(); + let closed = store + .journal_for_session(&host) + .unwrap() + .is_none_or(|journal| { + store + .active_journal_generation(&host, &journal) + .unwrap() + .is_none() + }); + if closed + && !store.journal_retention_required(&host).unwrap() + && jobs + .iter() + .all(|job| job.state == BackendJobState::Completed) + { + break; + } + } + }) + .await + .expect("seed Journal work must settle before measuring restore"); + } + + ConnectedRestoreFixture { + api, + execution, + probe, + model, + worker, + subject_id: subject.id, + requests, + callbacks, + lose_response, + _server: server, + _dir: dir, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn restore_after_real_prepare_rejects_revoked_host_trust_without_publication_or_queue_consumption() + { + use session_store::WorkerMetadataStore as _; + use std::sync::atomic::Ordering::SeqCst; + use worker_runtime::execution::WorkerExecutionBackend as _; + + let fixture = connected_restore_fixture().await; + let api = &fixture.api; + let execution = &fixture.execution; + let probe = &fixture.probe; + let model = &fixture.model; + let worker = fixture.worker.clone(); + let requests = &fixture.requests; + let store = open_subjektiv_store(api).unwrap(); + let subject_id = &fixture.subject_id; + let aggregate = api + .config + .embedded_runtime_store_root + .join("workers") + .join(&worker.worker_id); + let name = format!("worker-runtime-{}", worker.worker_id); + let metadata = session_store::WorkerAggregateStore::open_read_only(&aggregate, &name).unwrap(); + let saved = metadata.read_by_name(&name).unwrap().unwrap(); + assert!(matches!( + saved.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Confirmed { .. }) + )); + let active = saved.active.as_ref().unwrap(); + let session = + session_store::WorkerSessionStore::open_read_only(aggregate.join("session")).unwrap(); + let entries = || { + session + .read_all_read_only(active.session_id, active.segment_id.unwrap()) + .unwrap() + }; + let before = serde_json::to_value(entries()).unwrap(); + let checkpoint = entries() + .into_iter() + .rev() + .find_map(|entry| match entry { + session_store::LogEntry::Extension { + domain, payload, .. + } if domain == "worker.pending_activations.v1" => Some(payload), + _ => None, + }) + .expect("real Controller persisted pending activation checkpoint"); + assert_eq!( + checkpoint["pending"][0]["submission_request_id"], + "saved-submit" + ); + assert_eq!( + checkpoint["pending_notifications"][0]["notification_request_id"], + "saved-notice" + ); + let attributions = store.subject_session_attributions(subject_id).unwrap(); + let key = crate::subjektiv::subject_worker_singleton_key(subject_id).unwrap(); + let singleton = api + .store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &key) + .unwrap(); + let model_before = model.calls.load(SeqCst); + let request = runtime_api::WorkerRestoreRequest { + expected_observation_token: api + .runtime + .worker(&worker) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: "restore-revoked-after-real-prepare".into(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }; + assert!( + requests.lock().unwrap().iter().any( + |(path, status)| path.ends_with("/subjektiv/sessions") && *status == StatusCode::OK + ), + "the same production signer must be accepted before trust revocation" + ); + requests.lock().unwrap().clear(); + let runtime = api.runtime.clone(); + let target = worker.clone(); + let intent = request.clone(); + let restoring = tokio::task::spawn_blocking(move || runtime.restore_worker(&target, intent)); + tokio::time::timeout( + std::time::Duration::from_secs(10), + probe.prepared.notified(), + ) + .await + .expect("real prepare must complete before revocation"); + assert_eq!(probe.activations.load(SeqCst), 0); + assert_eq!(model.calls.load(SeqCst), model_before); + assert_eq!( + serde_json::to_value(entries()).unwrap(), + before, + "preparation must not consume saved input or append history" + ); + assert_eq!(metadata.read_by_name(&name).unwrap().unwrap(), saved); + assert!( + requests + .lock() + .unwrap() + .iter() + .all(|(path, _)| !path.ends_with("/subjektiv/sessions")), + "publication authorization belongs after candidate preparation" + ); + let prepared_record: Value = + serde_json::from_slice(&std::fs::read(aggregate.join("worker.json")).unwrap()).unwrap(); + let operation = prepared_record["restore_guard"]["owners"]["restore-revoked-after-real-prepare"]["operation_id"].clone(); + assert!( + !operation.is_null(), + "admission must persist the operation owner before prepare" + ); + assert!( + api.config_store + .revoke_workspace_runtime_binding( + TEST_WORKSPACE_ID, + EMBEDDED_WORKER_RUNTIME_ID, + &now_registry_timestamp(), + ) + .unwrap() + ); + probe.release.notify_one(); + let first = tokio::time::timeout(std::time::Duration::from_secs(20), restoring) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{first:?}" + ); + assert!( + requests + .lock() + .unwrap() + .iter() + .any(|(path, status)| path.ends_with("/subjektiv/sessions") + && *status == StatusCode::UNAUTHORIZED), + "actual signed Host publication must see revoked persisted trust" + ); + assert_eq!(probe.prepares.load(SeqCst), 1); + assert_eq!(probe.activations.load(SeqCst), 1); + let runtime = api.runtime.clone(); + let target = worker.clone(); + let replay = tokio::time::timeout( + std::time::Duration::from_secs(20), + tokio::task::spawn_blocking(move || runtime.restore_worker(&target, request)), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{replay:?}" + ); + // Reconciliation may reconstruct after confirmed cleanup, but must never + // invent a new operation or repair an already confirmed attribution. + assert!( + probe + .operations + .lock() + .unwrap() + .iter() + .all(|id| *id == operation) + ); + assert!( + probe + .repairs + .lock() + .unwrap() + .iter() + .all(|repaired| !repaired) + ); + let worker_ref = worker_runtime::identity::WorkerRef::try_from(&worker).unwrap(); + assert!( + execution.worker_snapshot(&worker_ref).is_none(), + "no live Controller may be published" + ); + assert!( + requests + .lock() + .unwrap() + .iter() + .all(|(path, _)| !path.ends_with("/authorization/check")), + "restoring saved Features must not re-admit them" + ); + assert_eq!(probe.publications.load(SeqCst), 0); + assert_eq!( + probe.handles.lock().unwrap().len(), + 1, + "only stopped seed Controller existed" + ); + assert_eq!(model.calls.load(SeqCst), model_before); + assert_eq!( + serde_json::to_value(entries()).unwrap(), + before, + "Host denial must not consume either saved queue or execute Tools" + ); + assert_eq!( + metadata + .read_by_name(&name) + .unwrap() + .unwrap() + .subjektiv_session_attribution, + saved.subjektiv_session_attribution + ); + assert_eq!( + store.subject_session_attributions(subject_id).unwrap(), + attributions + ); + assert_eq!( + api.store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &key) + .unwrap(), + singleton + ); + // Activation may already reconcile metadata/pins/allocation. Denial is not a + // fictitious rollback/Stopped: the same operation must retain its owner. + let record: Value = + serde_json::from_slice(&std::fs::read(aggregate.join("worker.json")).unwrap()).unwrap(); + assert_eq!( + record["restore_guard"]["active_request_id"], + "restore-revoked-after-real-prepare" + ); + let owner = &record["restore_guard"]["owners"]["restore-revoked-after-real-prepare"]; + assert_eq!( + owner["operation_id"], operation, + "the durable operation owner must survive denial and replay" + ); + assert_eq!( + record["execution_state"]["state"], + "reconciliation_required" + ); + assert_eq!( + record["execution_state"]["execution"]["operation"], + "restore" + ); + assert_eq!( + record["execution_state"]["execution"]["intent"]["operation_id"], + operation + ); + assert!( + owner["receipt"].is_null(), + "no saved Accepted receipt may escape on replay" + ); + // Only this fixture's retained operation is cleaned up, after its durable + // evidence has been checked. Cleanup is not the restore outcome above. + let cleanup = execution.stop_worker(&worker_ref); + assert!( + cleanup.is_accepted(), + "fixture restore cleanup: {cleanup:?}" + ); + shutdown_workspace_feature_storage(&api).await.unwrap(); +} + +#[derive(Clone, Copy)] +enum ConnectedRepair { + IncompleteAttribution, + PendingHead, + RevokedAttribution, +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn explicit_repair_existing_attribution_replays_lost_real_host_response_once() { + connected_explicit_repair(ConnectedRepair::IncompleteAttribution).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn explicit_repair_pending_head_creates_one_session_after_lost_real_host_response() { + connected_explicit_repair(ConnectedRepair::PendingHead).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn explicit_repair_replay_rejects_revoked_real_host_trust_without_false_accepted() { + connected_explicit_repair(ConnectedRepair::RevokedAttribution).await; +} + +async fn connected_explicit_repair(scenario: ConnectedRepair) { + use session_store::{Store as _, WorkerMetadataStore as _}; + use std::sync::atomic::Ordering::SeqCst; + use worker_runtime::execution::WorkerExecutionBackend as _; + + let pending = matches!(scenario, ConnectedRepair::PendingHead); + let fixture = connected_restore_fixture_with_pending_head(pending).await; + let api = &fixture.api; + let probe = &fixture.probe; + let worker = &fixture.worker; + let revoked = matches!(scenario, ConnectedRepair::RevokedAttribution); + let aggregate = api + .config + .embedded_runtime_store_root + .join("workers") + .join(&worker.worker_id); + let name = format!("worker-runtime-{}", worker.worker_id); + let metadata = session_store::WorkerAggregateStore::new(&aggregate, &name).unwrap(); + let mut saved = metadata.read_by_name(&name).unwrap().unwrap(); + let original_active = saved.active.clone().unwrap(); + // The genuine pending fixture has no Session manifest yet. Open its empty + // store before measuring repair; after publication always reread authority + // rather than the original Store's cached empty lineage. + let pending_store = + pending.then(|| session_store::WorkerSessionStore::new(aggregate.join("session")).unwrap()); + let sessions = + || match session_store::WorkerSessionStore::open_read_only(aggregate.join("session")) { + Ok(store) => store, + Err(session_store::StoreError::Io(error)) + if pending && error.kind() == std::io::ErrorKind::NotFound => + { + pending_store.as_ref().unwrap().clone() + } + Err(error) => panic!("read repaired Session: {error}"), + }; + let original_entries = match original_active.segment_id { + Some(segment_id) => serde_json::to_value( + sessions() + .read_all_read_only(original_active.session_id, segment_id) + .unwrap(), + ) + .unwrap(), + None => { + assert!(pending); + assert!( + sessions() + .list_segments(original_active.session_id) + .unwrap() + .is_empty() + ); + serde_json::json!([]) + } + }; + let store = open_subjektiv_store(api).unwrap(); + let initial_attributions = store + .subject_session_attributions(&fixture.subject_id) + .unwrap(); + let singleton_key = + crate::subjektiv::subject_worker_singleton_key(&fixture.subject_id).unwrap(); + let singleton = api + .store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &singleton_key) + .unwrap(); + + let session_id = saved.active.as_ref().unwrap().session_id; + assert_eq!( + initial_attributions + .iter() + .filter(|attr| attr.session_id == session_id.to_string()) + .count(), + usize::from(!pending), + "existing attribution repair is not first creation; pending-head repair is" + ); + // Deliberate initial interruption fixture, before admitting any restore. + // Subsequent transitions, including OutcomeUnknown, are production writes. + saved.subjektiv_session_attribution = Some( + session_store::SubjektivSessionAttributionState::OutcomeUnknown { + session_id, + diagnostic: "fixture interrupted Session attribution acknowledgement".into(), + }, + ); + metadata.write(&saved).unwrap(); + let model_before = fixture.model.calls.load(SeqCst); + fixture.requests.lock().unwrap().clear(); + fixture.callbacks.lock().unwrap().clear(); + fixture.lose_response.store(true, SeqCst); + let request = runtime_api::WorkerRestoreRequest { + expected_observation_token: api + .runtime + .worker(worker) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: "connected-explicit-repair".into(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }; + let runtime = api.runtime.clone(); + let target = worker.clone(); + let intent = request.clone(); + let first = tokio::time::timeout( + std::time::Duration::from_secs(20), + tokio::task::spawn_blocking(move || runtime.restore_worker(&target, intent)), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{first:?}" + ); + assert!( + !fixture.lose_response.load(SeqCst), + "lose acknowledgement only after real Host OK" + ); + assert_eq!( + probe.prepares.load(SeqCst), + 0, + "repair precedes candidate construction" + ); + assert_eq!(probe.activations.load(SeqCst), 0); + assert_eq!(probe.publications.load(SeqCst), 0); + assert_eq!(fixture.model.calls.load(SeqCst), model_before); + let failed = metadata.read_by_name(&name).unwrap().unwrap(); + assert_eq!( + failed.active, saved.active, + "response loss cannot publish a first head" + ); + assert!(matches!(failed.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::OutcomeUnknown { session_id: id, .. }) if id == session_id)); + let committed_attributions = store + .subject_session_attributions(&fixture.subject_id) + .unwrap(); + assert_eq!( + committed_attributions.len(), + initial_attributions.len() + usize::from(pending) + ); + assert_eq!( + committed_attributions + .iter() + .filter(|attr| attr.session_id == session_id.to_string() + && attr.worker_id == worker.worker_id) + .count(), + 1, + "actual SQLite attribution persisted before response loss" + ); + if !pending { + assert_eq!( + committed_attributions, initial_attributions, + "reconcile the existing immutable row" + ); + } + assert_eq!(fixture.callbacks.lock().unwrap().len(), 1); + { + let callbacks = fixture.callbacks.lock().unwrap(); + assert_eq!(callbacks[0]["session_id"], session_id.to_string()); + assert_eq!(callbacks[0]["create_if_missing"], true); + } + let record = || -> Value { + serde_json::from_slice(&std::fs::read(aggregate.join("worker.json")).unwrap()).unwrap() + }; + let first_record = record(); + let operation = + first_record["restore_guard"]["owners"]["connected-explicit-repair"]["operation_id"] + .clone(); + assert!(!operation.is_null()); + assert!( + first_record["restore_guard"]["owners"]["connected-explicit-repair"]["receipt"].is_null() + ); + assert_eq!( + first_record["restore_guard"]["active_request_id"], + "connected-explicit-repair" + ); + assert_eq!( + first_record["execution_state"]["state"], + "reconciliation_required" + ); + assert!( + fixture + .requests + .lock() + .unwrap() + .iter() + .any( + |(path, status)| path.ends_with("/subjektiv/sessions") && *status == StatusCode::OK + ) + ); + assert!( + !fixture + .requests + .lock() + .unwrap() + .iter() + .any(|(path, _)| path.ends_with("/subjektiv/journal/startup")), + "no startup before attribution acknowledgement" + ); + if pending { + assert!(sessions().list_segments(session_id).unwrap().is_empty()); + } + + // Re-enter the same public Runtime intent and durable operation. The repair + // gate changes timing only; trust revocation is the real persisted binding. + if revoked { + probe.gate_repair.store(true, SeqCst); + } + let mut repaired_seed = None; + let runtime = api.runtime.clone(); + let target = worker.clone(); + let intent = request.clone(); + let retry = tokio::task::spawn_blocking(move || runtime.restore_worker(&target, intent)); + if revoked { + tokio::time::timeout( + std::time::Duration::from_secs(10), + probe.repairing.notified(), + ) + .await + .unwrap(); + assert!( + api.config_store + .revoke_workspace_runtime_binding( + TEST_WORKSPACE_ID, + EMBEDDED_WORKER_RUNTIME_ID, + &now_registry_timestamp(), + ) + .unwrap() + ); + probe.release_repair.notify_one(); + } else { + tokio::time::timeout( + std::time::Duration::from_secs(10), + probe.prepared.notified(), + ) + .await + .unwrap(); + assert_eq!(probe.activations.load(SeqCst), 0); + assert_eq!(fixture.model.calls.load(SeqCst), model_before); + assert_eq!(*probe.repairs.lock().unwrap(), vec![true]); + let repaired = metadata.read_by_name(&name).unwrap().unwrap(); + assert_eq!(repaired.active.as_ref().unwrap().session_id, session_id); + assert!(matches!(repaired.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Confirmed { session_id: id, ref subject_id }) + if id == session_id && subject_id == &fixture.subject_id)); + assert_eq!( + sessions().list_segments(session_id).unwrap().len(), + 1, + "exactly one durable seed" + ); + let segment_id = repaired.active.unwrap().segment_id.unwrap(); + let entries = sessions() + .read_all_read_only(session_id, segment_id) + .unwrap(); + let seeds = entries + .iter() + .filter(|entry| matches!(entry, session_store::LogEntry::AnnotatedSegmentStart { .. })) + .collect::>(); + assert_eq!(seeds.len(), 1); + repaired_seed = Some((segment_id, serde_json::to_value(seeds[0]).unwrap())); + if !pending { + assert_eq!( + serde_json::to_value(entries).unwrap(), + original_entries, + "attribution-only repair and normal prepare cannot reseed or consume saved input" + ); + } + assert_eq!( + fixture + .requests + .lock() + .unwrap() + .iter() + .filter(|(path, _)| path.ends_with("/subjektiv/journal/startup")) + .count(), + usize::from(pending) + ); + probe.release.notify_one(); + } + let replay = tokio::time::timeout(std::time::Duration::from_secs(20), retry) + .await + .unwrap() + .unwrap() + .unwrap(); + if revoked { + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{replay:?}" + ); + assert!( + fixture + .requests + .lock() + .unwrap() + .iter() + .any(|(path, status)| path.ends_with("/subjektiv/sessions") + && *status == StatusCode::UNAUTHORIZED) + ); + assert_eq!(probe.prepares.load(SeqCst), 0); + assert_eq!(probe.activations.load(SeqCst), 0); + assert_eq!(probe.publications.load(SeqCst), 0); + assert_eq!(fixture.model.calls.load(SeqCst), model_before); + assert!( + matches!(metadata.read_by_name(&name).unwrap().unwrap().subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Failed { session_id: id, .. }) if id == session_id) + ); + let denied = record(); + assert_eq!( + denied["restore_guard"]["active_request_id"], + "connected-explicit-repair" + ); + assert_eq!( + denied["restore_guard"]["owners"]["connected-explicit-repair"]["operation_id"], + operation + ); + assert!( + denied["restore_guard"]["owners"]["connected-explicit-repair"]["receipt"].is_null() + ); + assert_eq!( + denied["execution_state"]["state"], + "reconciliation_required" + ); + assert_eq!( + serde_json::to_value( + sessions() + .read_all_read_only( + original_active.session_id, + original_active.segment_id.unwrap(), + ) + .unwrap() + ) + .unwrap(), + original_entries + ); + } else { + assert_eq!( + replay.state, + server_api::WorkerRestoreState::Accepted, + "{replay:?}" + ); + let callbacks_after = fixture.callbacks.lock().unwrap().len(); + let repairs_after = probe.repair_operations.lock().unwrap().len(); + let runtime = api.runtime.clone(); + let target = worker.clone(); + let accepted = tokio::time::timeout( + std::time::Duration::from_secs(20), + tokio::task::spawn_blocking(move || runtime.restore_worker(&target, request)), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + accepted.state, + server_api::WorkerRestoreState::Accepted, + "{accepted:?}" + ); + assert_eq!(probe.prepares.load(SeqCst), 1); + assert_eq!(probe.publications.load(SeqCst), 1); + assert_eq!(probe.repair_operations.lock().unwrap().len(), repairs_after); + assert_eq!( + fixture.callbacks.lock().unwrap().len(), + callbacks_after, + "receipt replay performs no Host callback" + ); + assert_eq!(sessions().list_segments(session_id).unwrap().len(), 1); + let repaired = metadata.read_by_name(&name).unwrap().unwrap(); + let active = repaired.active.unwrap(); + let (segment_id, seed) = repaired_seed.unwrap(); + assert_eq!(active.session_id, session_id); + assert_eq!(active.segment_id, Some(segment_id)); + let entries = sessions() + .read_all_read_only(session_id, segment_id) + .unwrap(); + let seeds = entries + .iter() + .filter(|entry| matches!(entry, session_store::LogEntry::AnnotatedSegmentStart { .. })) + .collect::>(); + assert_eq!(seeds.len(), 1); + assert_eq!( + serde_json::to_value(seeds[0]).unwrap(), + seed, + "activation and receipt replay preserve the exact repaired seed" + ); + } + if pending { + assert_eq!( + fixture.model.calls.load(SeqCst), + model_before, + "pending first-head repair has no body input to execute" + ); + assert_eq!( + session_id, original_active.session_id, + "first head belongs to the originally pending Session" + ); + assert!(original_active.segment_id.is_none()); + } + assert_eq!( + store + .subject_session_attributions(&fixture.subject_id) + .unwrap(), + committed_attributions + ); + assert_eq!( + api.store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &singleton_key) + .unwrap(), + singleton + ); + assert!( + probe + .repair_operations + .lock() + .unwrap() + .iter() + .all(|id| *id == operation) + ); + assert!( + probe + .operations + .lock() + .unwrap() + .iter() + .all(|id| *id == operation) + ); + assert_eq!( + probe.repair_operations.lock().unwrap().len(), + 2, + "one failed repair and one same-operation replay" + ); + assert!( + fixture + .requests + .lock() + .unwrap() + .iter() + .all(|(path, _)| !path.ends_with("/authorization/check")), + "saved Features must not be re-admitted" + ); + { + let callbacks = fixture.callbacks.lock().unwrap(); + assert!( + callbacks + .iter() + .all(|callback| callback["session_id"] == session_id.to_string()) + ); + assert_eq!( + callbacks + .iter() + .filter(|callback| callback["create_if_missing"] == true) + .count(), + 2, + "repair retries the Session-keyed receipt; publication only verifies it" + ); + } + let worker_ref = worker_runtime::identity::WorkerRef::try_from(worker).unwrap(); + if revoked { + assert!(fixture.execution.worker_snapshot(&worker_ref).is_none()); + assert!( + fixture.execution.stop_worker(&worker_ref).is_accepted(), + "fixture-owned repair resource cleanup" + ); + } else { + assert!(fixture.execution.worker_snapshot(&worker_ref).is_some()); + // Teardown is outside the measured restore window: Stop may admit an + // independent Journal job, not a second execution of the restored body. + assert_eq!( + api.runtime + .stop_worker(worker, WorkerLifecycleRequest::default()) + .unwrap() + .state, + InternalWorkerOperationState::Accepted + ); + } + shutdown_workspace_feature_storage(api).await.unwrap(); +} diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs index dece1c3f4..cb88a2636 100644 --- a/crates/workspace-server/src/server_assignment_attention_tests.rs +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -490,21 +490,61 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFac self.controller(request.worker_ref.worker_id.to_string(), false) .await } - async fn restore_controller( + async fn prepare_restore( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + Ok(Box::new(AttentionRestoreCandidate { + inputs: Some(( + AttentionWorkerFactory { + root: self.root.clone(), + }, + request.worker_ref.worker_id.to_string(), + )), + })) + } +} + +// Preparation owns only fixture inputs; reconstruction and Controller startup +// remain lazy so discard never needs to shut down a started Worker. +struct AttentionRestoreCandidate { + inputs: Option<(AttentionWorkerFactory, String)>, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for AttentionRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + protocol::WorkerStateSnapshot::initial() + } + + async fn activate( + &mut self, ) -> std::result::Result< worker_runtime::worker_backend::RuntimeWorkerController, worker_runtime::worker_backend::RuntimeRestoreFailure, > { - self.controller(request.worker_ref.worker_id.to_string(), true) - .await - .map_err(|_| { - worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( - worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, - "attention fixture reconstruction failed", - ) - }) + let (factory, worker_id) = self.inputs.take().ok_or_else(|| { + worker_runtime::worker_backend::RuntimeRestoreFailure::refused( + worker_runtime::worker_backend::RuntimeRestoreStage::Controller, + "attention fixture candidate already consumed", + ) + })?; + factory.controller(worker_id, true).await.map_err(|_| { + worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( + worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, + "attention fixture reconstruction failed", + ) + }) + } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inputs.take(); + Ok(()) } } @@ -611,7 +651,7 @@ async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() { async fn real_attention_worker_start_restore(lose_response: bool) { use session_store::{LogEntry, Store, WorkerSessionStore}; - let dir = tempfile::tempdir().unwrap(); + let dir = restored_parent_control_tests::connected_restore_tempdir(); init_clean_git_workspace(dir.path()); let config = test_server_config(dir.path()); let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs index 43bddd74f..25031c8f6 100644 --- a/crates/workspace-server/src/server_assignment_delivery_tests.rs +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -14,17 +14,37 @@ async fn notification_diagnostics_http_auth_scope_and_readonly_evidence() { seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID); let ticket = delivery_ticket(&api, "Notification diagnostics auth fixture"); delivery_assign(&api, ticket.id.as_str(), "diagnostics-assignment").await; - let path = format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",ticket.id.as_str()); - let page = delivery_http(&api,"GET",&path,None,StatusCode::OK).await; - assert_eq!(page["items"].as_array().unwrap().len(),1); - assert_eq!(page["items"][0]["model_processing_confirmed"],false); + let path = format!( + "/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1", + ticket.id.as_str() + ); + let page = delivery_http(&api, "GET", &path, None, StatusCode::OK).await; + assert_eq!(page["items"].as_array().unwrap().len(), 1); + assert_eq!(page["items"][0]["model_processing_confirmed"], false); assert!(page["items"][0].get("content").is_none()); - let again=delivery_http(&api,"GET",&path,None,StatusCode::OK).await; - assert_eq!(again,page); - let response=build_router(api.clone()).oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap()).await.unwrap(); - assert_eq!(response.status(),StatusCode::UNAUTHORIZED); - delivery_http(&api,"GET","/api/w/other-workspace/notification-diagnostics",None,StatusCode::NOT_FOUND).await; - delivery_http(&api,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),None,StatusCode::BAD_REQUEST).await; + let again = delivery_http(&api, "GET", &path, None, StatusCode::OK).await; + assert_eq!(again, page); + let response = build_router(api.clone()) + .oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + delivery_http( + &api, + "GET", + "/api/w/other-workspace/notification-diagnostics", + None, + StatusCode::NOT_FOUND, + ) + .await; + delivery_http( + &api, + "GET", + &format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"), + None, + StatusCode::BAD_REQUEST, + ) + .await; } fn delivery_assignment_path(ticket: &str) -> String { @@ -828,21 +848,62 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFact self.controller(request.worker_ref.worker_id.to_string(), false) .await } - async fn restore_controller( + async fn prepare_restore( &self, request: delivery_execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + Ok(Box::new(DeliveryRestoreCandidate { + inputs: Some(( + DeliveryWorkerFactory { + root: self.root.clone(), + client: self.client.clone(), + }, + request.worker_ref.worker_id.to_string(), + )), + })) + } +} + +// Retain the same scripted client/receipt store without opening either store or +// starting a Controller until activation. Discard drops only these owned inputs. +struct DeliveryRestoreCandidate { + inputs: Option<(DeliveryWorkerFactory, String)>, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for DeliveryRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + protocol::WorkerStateSnapshot::initial() + } + + async fn activate( + &mut self, ) -> std::result::Result< worker_runtime::worker_backend::RuntimeWorkerController, worker_runtime::worker_backend::RuntimeRestoreFailure, > { - self.controller(request.worker_ref.worker_id.to_string(), true) - .await - .map_err(|_| { - worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( - worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, - "delivery fixture reconstruction failed", - ) - }) + let (factory, worker_id) = self.inputs.take().ok_or_else(|| { + worker_runtime::worker_backend::RuntimeRestoreFailure::refused( + worker_runtime::worker_backend::RuntimeRestoreStage::Controller, + "delivery fixture candidate already consumed", + ) + })?; + factory.controller(worker_id, true).await.map_err(|_| { + worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( + worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, + "delivery fixture reconstruction failed", + ) + }) + } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inputs.take(); + Ok(()) } } @@ -941,6 +1002,7 @@ async fn delivery_real_fixture( Arc, DeliveryScriptedClient, ) { + let _environment = restored_parent_control_tests::connected_restore_tempdir(); let config = test_server_config(root); let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) @@ -1146,9 +1208,22 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi "uncertain delivery changed request id or payload: {attempts:?}" ); assert_eq!(delivery_receipts(&api, &worker).len(), 1); - let unknown = api.store.notification_diagnostics(TEST_WORKSPACE_ID, &server_api::NotificationDiagnosticsQuery { - assignment_id: Some(assignment.into()), ..Default::default() - }).unwrap().items.into_iter().find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown).expect("saved unknown acceptance must remain visible while Runtime already has its receipt"); + let unknown = api + .store + .notification_diagnostics( + TEST_WORKSPACE_ID, + &server_api::NotificationDiagnosticsQuery { + assignment_id: Some(assignment.into()), + ..Default::default() + }, + ) + .unwrap() + .items + .into_iter() + .find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown) + .expect( + "saved unknown acceptance must remain visible while Runtime already has its receipt", + ); assert!(!unknown.model_processing_confirmed); // Change mutable rendering context while the acknowledgement is unknown. // A new self-report is not a new input and cannot change the frozen retry. @@ -1209,11 +1284,24 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi } delivery_wait_idle(&reopened, &worker, 2).await; assert_eq!(delivery_receipts(&reopened, &worker).len(), 2); - let evidence = delivery_http(&reopened,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",unknown.source_id),None,StatusCode::OK).await; - assert_eq!(evidence["items"][0]["stage"],"runtime_accepted"); - assert_eq!(evidence["items"][0]["delivery_id"],json!(unknown.delivery_id)); - assert_eq!(evidence["items"][0]["model_processing_confirmed"],false); - assert_eq!(evidence["items"][0]["retry_candidate"],false); + let evidence = delivery_http( + &reopened, + "GET", + &format!( + "/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}", + unknown.source_id + ), + None, + StatusCode::OK, + ) + .await; + assert_eq!(evidence["items"][0]["stage"], "runtime_accepted"); + assert_eq!( + evidence["items"][0]["delivery_id"], + json!(unknown.delivery_id) + ); + assert_eq!(evidence["items"][0]["model_processing_confirmed"], false); + assert_eq!(evidence["items"][0]["retry_candidate"], false); assert_eq!( delivery_commits(&reopened, &worker) .iter() diff --git a/crates/workspace-server/src/server_workspace_config_wip_tests.rs b/crates/workspace-server/src/server_workspace_config_wip_tests.rs index 6c3fd0fd6..9e970e17d 100644 --- a/crates/workspace-server/src/server_workspace_config_wip_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_wip_tests.rs @@ -187,7 +187,14 @@ async fn config_call( runtime .invoke( path.into(), - if matches!(operation,"edit"|"write") {wip_protocol::InterfaceReference {scope:"/".into(),name:format!("yoi.file/{operation}-text/v1")}} else {config_interface(path)}, + if matches!(operation, "edit" | "write") { + wip_protocol::InterfaceReference { + scope: "/".into(), + name: format!("yoi.file/{operation}-text/v1"), + } + } else { + config_interface(path) + }, operation.into(), args, agen::tool::ToolExecutionContext::direct(), @@ -581,14 +588,88 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto *self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle()); Ok(controller) } - async fn restore_controller( + async fn preflight_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), String> { + self.inner.preflight_restore(request).await + } + + async fn recover_restore_source( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.inner.recover_restore_source(request).await + } + + async fn prepare_restore( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + let inner = self.inner.prepare_restore(request).await?; + Ok(Box::new(ConfigCaptureCandidate { + inner, + client: self.client.clone(), + controller_abort: self.controller_abort.clone(), + })) + } + + async fn cleanup_failed_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.cleanup_failed_restore(request).await + } + + async fn reconcile_stopped_worker( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.reconcile_stopped_worker(worker_ref).await + } + + fn activate_restored_controller( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + workspace_id: Option<&str>, + handle: &worker::WorkerHandle, + ) { + self.inner + .activate_restored_controller(worker_ref, workspace_id, handle); + } +} + +struct ConfigCaptureCandidate { + inner: Box, + client: Arc>>>, + controller_abort: Arc>>, +} + +#[async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for ConfigCaptureCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.inner.initial_state() + } + + async fn activate( + &mut self, ) -> std::result::Result< worker_runtime::worker_backend::RuntimeWorkerController, worker_runtime::worker_backend::RuntimeRestoreFailure, > { - self.inner.restore_controller(request).await + let controller = self.inner.activate().await?; + *self.client.lock().unwrap() = Some(controller.workspace_client.clone()); + *self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle()); + Ok(controller) + } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.discard().await } } @@ -717,6 +798,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION, decode_runtime_request_source_claims, }; + let root = restored_parent_control_tests::connected_restore_tempdir(); // Remote Runtime-owned HTTP adapter, including its real request signer. let mut fixture = manual_worker_assignment_fixture().await; let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap(); @@ -796,7 +878,6 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w // Actual embedded Runtime factory -> real Worker controller -> captured // production WorkspaceClient -> signed HTTP -> production Backend middleware. - let root = tempfile::tempdir().unwrap(); init_clean_git_workspace(root.path()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); diff --git a/deno.json b/deno.json index d10675008..b006ccf53 100644 --- a/deno.json +++ b/deno.json @@ -1,8 +1,8 @@ { "tasks": { - "test": "deno run --allow-read --allow-write=target --allow-run --allow-env tools/test.ts", + "test": "deno run --allow-read --allow-write=target --allow-run --allow-env --allow-sys=uid tools/test.ts", "test:ticket:real-api": "cargo test -p yoi-workspace-server --lib ticket_web_roundtrip_tests -- --ignored", - "test:runner": "deno test --allow-read --allow-write --allow-run --allow-env tools/test.test.ts" + "test:runner": "deno test --allow-read --allow-write --allow-run --allow-env --allow-sys=uid tools/test.test.ts" }, "fmt": { "proseWrap": "never" diff --git a/tests/support/restore_environment.rs b/tests/support/restore_environment.rs new file mode 100644 index 000000000..c045a2ed5 --- /dev/null +++ b/tests/support/restore_environment.rs @@ -0,0 +1,129 @@ +//! Fail-closed preflight shared by restore fixtures across Worker, Runtime and Server. +//! +//! The normal runner supplies a separate namespace per process; a manually +//! configured invocation may use the checkout-owned base itself. This module +//! never sets process-wide environment or creates directories/tables. + +use std::fs; +use std::path::{Component, Path, PathBuf}; + +fn validate_restore_test_paths( + base: &Path, + resolved: [Option; 5], +) -> Result { + let root = resolved[0] + .as_deref() + .filter(|config| config.file_name().is_some_and(|name| name == "config")) + .and_then(Path::parent) + .ok_or("config")?; + let namespace = root.strip_prefix(base).map_err(|_| "config")?; + if namespace + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err("config"); + } + for ((label, suffix), actual) in [ + ("config", "config"), + ("data", "data"), + ("secrets", "data"), + ("runtime", "run"), + ("allocation", "run/workers.json"), + ] + .into_iter() + .zip(resolved.iter()) + { + if actual.as_deref() != Some(root.join(suffix).as_path()) { + return Err(label); + } + } + Ok(root.to_path_buf()) +} + +fn require_owned_directory(checkout: &Path, directory: &Path) { + let mut checked = checkout.to_path_buf(); + for part in directory.strip_prefix(checkout).unwrap().components() { + assert!(matches!(part, Component::Normal(_))); + checked.push(part); + let metadata = fs::symlink_metadata(&checked) + .expect("create isolated restore test directories before launching the test process"); + assert!( + metadata.is_dir() && !metadata.file_type().is_symlink(), + "restore test directory must not traverse symlinks" + ); + } +} + +pub(crate) fn require_restore_test_environment() { + let checkout = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap(); + let base = checkout.join("target/restore-test-environment"); + // Compare all resolver results before inspecting even an absent allocation + // table (its lookup can create a lock). Never inspect a live fallback path. + let root = validate_restore_test_paths( + &base, + [ + manifest::paths::config_dir(), + manifest::paths::data_dir(), + manifest::paths::secret_data_dir(), + manifest::paths::runtime_dir(), + manifest::paths::worker_allocation_path(), + ], + ) + .unwrap_or_else(|label| { + panic!( + "restore fixture requires isolated {label}: run deno task test, or configure YOI_CONFIG_DIR=/target/restore-test-environment/config, YOI_DATA_DIR=/target/restore-test-environment/data, YOI_RUNTIME_DIR=/target/restore-test-environment/run before launch" + ) + }); + assert_eq!(fs::canonicalize(checkout).unwrap(), checkout); + for directory in [ + root.clone(), + root.join("config"), + root.join("data"), + root.join("run"), + ] { + require_owned_directory(checkout, &directory); + } + for name in ["workers.json", "workers.json.lock"] { + match fs::symlink_metadata(root.join("run").join(name)) { + Ok(metadata) => assert!( + metadata.is_file() && !metadata.file_type().is_symlink(), + "owned allocation paths must be regular files, not symlinks" + ), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => panic!("inspect owned restore allocation path: {error}"), + } + } +} + +#[test] +fn restore_fixture_path_validation_rejects_fallback_before_resource_access() { + let base = Path::new("/owned/test-environment"); + for root in [base.to_path_buf(), base.join("run-1/worker-shard-2")] { + let safe = ["config", "data", "data", "run", "run/workers.json"] + .map(|suffix| Some(root.join(suffix))); + assert_eq!( + validate_restore_test_paths(base, safe.clone()), + Ok(root.clone()) + ); + for index in 0..safe.len() { + for fallback in [None, Some(PathBuf::from("/not-a-fixture"))] { + let mut unsafe_paths = safe.clone(); + unsafe_paths[index] = fallback; + assert!(validate_restore_test_paths(base, unsafe_paths).is_err()); + } + } + } + for root in [ + base.join("../escape"), + base.join("safe/../../escape"), + PathBuf::from("/outside"), + ] { + let paths = ["config", "data", "data", "run", "run/workers.json"] + .map(|suffix| Some(root.join(suffix))); + assert!(validate_restore_test_paths(base, paths).is_err()); + } +} diff --git a/tools/test.test.ts b/tools/test.test.ts index aa8c69d12..bad5ee0c1 100644 --- a/tools/test.test.ts +++ b/tools/test.test.ts @@ -6,6 +6,7 @@ import { mapConcurrent, parseJobs, parseRustListing, + prepareTestIsolation, resultSummary, rustTestCommands, shardRustCommand, @@ -202,6 +203,39 @@ Deno.test("round-robin shards retain every listed name exactly once at threshold assertThrows(() => shardRustCommand(task, ["same", "same"])); }); +Deno.test("global runtime-path packages serialize every normal target without losing names", () => { + for (const pkg of ["worker", "worker-runtime", "yoi-workspace-server"]) { + for (const target of ["lib", "controller_restore_test"]) { + const task = { + name: `${pkg}/${target}`, + command: "/test-binary", + args: [], + cwd: "/pkg", + }; + for (const count of [0, 1, 59, 72, 73, 1135]) { + const names = listedNames(count); + const shards = shardRustCommand(task, names); + assert( + shards.flatMap((shard) => shard.selectedNames).sort().join() === + [...names].sort().join(), + ); + for (const shard of shards) { + const args = shard.task.args; + assert(args[args.indexOf("--test-threads") + 1] === "1"); + if (count > 72) { + assert(args[0] === "--exact"); + assert(args.slice(3).join() === shard.selectedNames.join()); + } else { + assert(args.join() === "--test-threads,1"); + } + assert(!args.includes("--ignored") && !args.includes("--skip")); + } + } + assert(task.args.length === 0, "do not mutate discovered commands"); + } + } +}); + Deno.test("scheduler drains active workers before propagating a callback error", async () => { let drained = false; let failed = false; @@ -266,6 +300,276 @@ function emergencyKill(pid: number) { } } +async function assertRejects( + run: () => Promise, + message: string, +) { + try { + await run(); + } catch (error) { + assert(String(error).includes(message), String(error)); + return; + } + throw new Error(`expected rejection containing ${message}`); +} + +Deno.test({ + name: + "isolated real children use bounded disjoint tables and sanitized Git/compiler environments", + ignore: Deno.build.os !== "linux", + async fn() { + const checkout = await Deno.realPath(await Deno.makeTempDir()); + const environmentSnapshot = () => + JSON.stringify( + Object.entries(process.env).sort(([a], [b]) => a.localeCompare(b)), + ); + const before = environmentSnapshot(); + try { + // An ancestor repository must not make an uninitialized tempfile look like + // a repository. No developer repository or configuration is borrowed. + for (const dir of [".git/objects", ".git/refs"]) { + await Deno.mkdir(`${checkout}/${dir}`, { recursive: true }); + } + await Deno.writeTextFile( + `${checkout}/.git/HEAD`, + "ref: refs/heads/main\n", + ); + const [first, second] = await Promise.all([ + prepareTestIsolation(checkout, "same/output/id"), + prepareTestIsolation(checkout, "same/output/id"), + ]); + assert( + first.root !== second.root, + "concurrent runner ids must not collide", + ); + const task = { + ...fixtureTask("isolation-env", checkout), + env: { + CARGO_MANIFEST_DIR: `${checkout}/crates/example`, + CARGO_HOME: `${checkout}/compiler-cache`, + RUSTUP_HOME: `${checkout}/toolchain-cache`, + YOI_HOME: "/live/yoi", + YOI_CONFIG_DIR: "/live/config", + YOI_DATA_DIR: "/live/data", + YOI_RUNTIME_DIR: "/live/run", + HOME: "/live/home", + TMPDIR: "/deleted/compiler-tmp", + GIT_DIR: "/live/git", + GIT_WORK_TREE: "/live/tree", + GIT_CONFIG: "/live/gitconfig", + GIT_CONFIG_GLOBAL: "/live/gitconfig", + GIT_CONFIG_SYSTEM: "/live/system-gitconfig", + GIT_TEMPLATE_DIR: "/live/template", + GIT_CONFIG_PARAMETERS: "'core.hooksPath'='/live/hooks'", + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "core.hooksPath", + GIT_CONFIG_VALUE_0: "/live/hooks", + HTTP_PROXY: "http://live.invalid", + https_proxy: "http://live.invalid", + ALL_PROXY: "http://live.invalid", + NO_PROXY: "live.invalid", + RUSTC_WRAPPER: "/live/sccache", + RUSTC_WORKSPACE_WRAPPER: "/live/wrapper", + SCCACHE_DISABLE: "0", + }, + }; + const taskBefore = JSON.stringify(task); + const commands = await Promise.all([ + first.wrapCommand(task, "pkg/a"), + first.wrapCommand(task, "pkg?a"), // same sanitized label + first.wrapCommand(task, "pkg/a"), // same exact label + second.wrapCommand(task, "pkg/a"), + ]); + assert(JSON.stringify(task) === taskBefore); + assert( + environmentSnapshot() === before, + "host environment changed", + ); + const results = await Promise.all( + commands.map((command, index) => + execute(command, { + output: checkout, + label: `environment-${index}`, + timeoutMs: 10_000, + }) + ), + ); + const tables = new Set(); + for (const [index, result] of results.entries()) { + assert(result.code === 0 && !result.error, JSON.stringify(result)); + const observed = JSON.parse(result.stdout); + const env = observed.env as Record; + assert(observed.cwd === checkout && commands[index].cwd === task.cwd); + assert(commands[index].args === task.args); + assert(env.CARGO_MANIFEST_DIR === task.env.CARGO_MANIFEST_DIR); + assert(env.CARGO_HOME === task.env.CARGO_HOME); + assert(env.RUSTUP_HOME === task.env.RUSTUP_HOME); + const namespace = env.YOI_RUNTIME_DIR.replace(/\/run$/, ""); + const prefix = `${checkout}/target/restore-test-environment/`; + assert(namespace.startsWith(prefix)); + const suffix = namespace.slice(prefix.length); + assert( + suffix.split("/").every((part) => /^[a-zA-Z0-9_-]+$/.test(part)), + ); + assert(env.YOI_CONFIG_DIR === `${namespace}/config`); + assert(env.YOI_DATA_DIR === `${namespace}/data`); + assert( + env.YOI_HOME === env.YOI_DATA_DIR, + "secret data must equal data", + ); + assert(env.HOME === `${namespace}/home`); + for ( + const dir of [ + namespace, + env.YOI_CONFIG_DIR, + env.YOI_DATA_DIR, + env.YOI_RUNTIME_DIR, + env.HOME, + ] + ) { + assert((await Deno.lstat(dir)).isDirectory); + assert(await Deno.realPath(dir) === dir); + } + for (const key of ["TMPDIR", "TMP", "TEMP"]) { + assert(env[key] === `${checkout}/target/t`); + } + for ( + const key of [ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_CONFIG", + "GIT_CONFIG_PARAMETERS", + "GIT_CONFIG_COUNT", + "GIT_CONFIG_KEY_0", + "GIT_CONFIG_VALUE_0", + "HTTP_PROXY", + "https_proxy", + "ALL_PROXY", + "NO_PROXY", + ] + ) { + assert(!(key in env), `${key} leaked to child`); + } + assert(env.GIT_CONFIG_GLOBAL === "/dev/null"); + assert(env.GIT_CONFIG_SYSTEM === "/dev/null"); + assert(env.GIT_CONFIG_NOSYSTEM === "1"); + assert(env.GIT_CEILING_DIRECTORIES === checkout); + assert(env.GIT_TEMPLATE_DIR === `${namespace}/git-template`); + assert(observed.globalConfig === ""); + assert(observed.ancestorDiscovered === false); + assert(observed.hooks.length === 0 && observed.info.length === 0); + assert(observed.author.includes("normal-tests@example.invalid")); + assert(env.GIT_AUTHOR_NAME === env.GIT_COMMITTER_NAME); + assert(env.GIT_AUTHOR_EMAIL === env.GIT_COMMITTER_EMAIL); + assert(env.RUSTC_WRAPPER === "" && env.RUSTC_WORKSPACE_WRAPPER === ""); + assert(env.SCCACHE_DISABLE === "1"); + const table = `${env.YOI_RUNTIME_DIR}/workers.json`; + tables.add(table); + assert(await Deno.readTextFile(table) === String(observed.pid)); + } + assert( + tables.size === commands.length, + "independent commands shared a table", + ); + assert(environmentSnapshot() === before); + } finally { + await Deno.remove(checkout, { recursive: true }); + } + }, +}); + +Deno.test("isolation refuses lexical aliases and symlinked owned paths before external writes", async () => { + for ( + const unsafe of ["target", "target/restore-test-environment", "target/t"] + ) { + const checkout = await Deno.realPath(await Deno.makeTempDir()); + const outside = await Deno.realPath(await Deno.makeTempDir()); + try { + await Deno.writeTextFile(`${outside}/sentinel`, "untouched"); + if (unsafe !== "target") await Deno.mkdir(`${checkout}/target`); + await Deno.symlink(outside, `${checkout}/${unsafe}`); + await assertRejects( + () => prepareTestIsolation(checkout, "run"), + "symlink", + ); + assert( + Array.from(Deno.readDirSync(outside)).map((entry) => entry.name) + .join() === + "sentinel", + ); + assert(await Deno.readTextFile(`${outside}/sentinel`) === "untouched"); + } finally { + await Deno.remove(checkout, { recursive: true }); + await Deno.remove(outside, { recursive: true }); + } + } + const checkout = await Deno.realPath(await Deno.makeTempDir()); + try { + await Deno.symlink(checkout, `${checkout}/alias`); + await assertRejects( + () => prepareTestIsolation(`${checkout}/alias`, "run"), + "symlink", + ); + for (const suffix of ["/../escape", "/./child", "//child"]) { + await assertRejects( + () => prepareTestIsolation(`${checkout}${suffix}`, "run"), + "normal-only", + ); + } + assert( + Array.from(Deno.readDirSync(checkout)).map((entry) => entry.name) + .join() === + "alias", + ); + } finally { + await Deno.remove(checkout, { recursive: true }); + } +}); + +Deno.test("command wrapping rechecks owned ancestors and refuses symlinked namespace children", async () => { + for ( + const unsafe of [ + "ancestor", + "tmp", + "config", + "data", + "run", + "home", + "git-template", + "git-template/info", + "git-template/hooks", + ] + ) { + const checkout = await Deno.realPath(await Deno.makeTempDir()); + const outside = await Deno.realPath(await Deno.makeTempDir()); + try { + const isolation = await prepareTestIsolation(checkout, "run"); + let path: string; + if (unsafe === "ancestor") { + path = isolation.root; + await Deno.remove(path); + } else if (unsafe === "tmp") { + path = `${checkout}/target/t`; + await Deno.remove(path); + } else { + const namespace = `${isolation.root}/unsafe-0`; + await Deno.mkdir(namespace); + if (unsafe.includes("/")) await Deno.mkdir(`${namespace}/git-template`); + path = `${namespace}/${unsafe}`; + } + await Deno.symlink(outside, path); + await assertRejects( + () => isolation.wrapCommand(fixtureTask("success", checkout), "unsafe"), + "symlink", + ); + assert(Array.from(Deno.readDirSync(outside)).length === 0); + } finally { + await Deno.remove(checkout, { recursive: true }); + await Deno.remove(outside, { recursive: true }); + } + } +}); + for (const [mode, code] of [["success", 0], ["nonzero", 7]] as const) { Deno.test({ name: `execute streams real ${mode} child output and records its exit`, @@ -436,15 +740,28 @@ Deno.test({ "runtime listing persists complete name inventory for large and small binaries", ignore: Deno.build.os !== "linux", async fn() { - const output = await Deno.makeTempDir(); + const output = await Deno.realPath(await Deno.makeTempDir()); try { + const isolation = await prepareTestIsolation(output, "discovery"); + const wrappedTables = new Set(); const discovery = await discoverRustCommands( [ fixtureTask("list-large", output), fixtureTask("list-small", output), ], 2, - { output }, + { + output, + run: async (task, label) => { + const wrapped = await isolation.wrapCommand(task, label); + wrappedTables.add(`${wrapped.env!.YOI_RUNTIME_DIR}/workers.json`); + return await execute(wrapped, { output, label, timeoutMs: 10_000 }); + }, + }, + ); + assert( + wrappedTables.size === 2, + "all discovery launches must use the callback", ); assert(discovery.inventory.complete); assert( @@ -635,6 +952,54 @@ if (import.meta.main) { ); } Deno.exit(mode === "list-nonzero" ? 7 : 0); + } else if (mode === "isolation-env") { + const env = Deno.env.toObject(); + const scratch = await Deno.makeTempDir({ dir: env.TMPDIR }); + const text = (bytes: Uint8Array) => new TextDecoder().decode(bytes); + async function git(args: string[]) { + return await new Deno.Command("git", { + args, + cwd: scratch, + stdout: "piped", + stderr: "piped", + }).output(); + } + try { + const ownedRepo = await git([ + "-C", + output, + "rev-parse", + "--show-toplevel", + ]); + assert( + ownedRepo.success && text(ownedRepo.stdout).trim() === output, + "owned ancestor repository fixture must be discoverable at its root", + ); + const ancestor = await git(["rev-parse", "--show-toplevel"]); + const init = await git(["init", "--quiet"]); + assert(init.success, text(init.stderr)); + const global = await git(["config", "--global", "--list"]); + assert(global.success, text(global.stderr)); + const author = await git(["var", "GIT_AUTHOR_IDENT"]); + assert(author.success, text(author.stderr)); + await Deno.writeTextFile( + `${env.YOI_RUNTIME_DIR}/workers.json`, + String(Deno.pid), + ); + console.log(JSON.stringify({ + env, + cwd: Deno.cwd(), + pid: Deno.pid, + ancestorDiscovered: ancestor.success, + globalConfig: text(global.stdout), + author: text(author.stdout), + hooks: Array.from(Deno.readDirSync(`${scratch}/.git/hooks`)), + info: Array.from(Deno.readDirSync(`${scratch}/.git/info`)), + })); + } finally { + await Deno.remove(scratch, { recursive: true }); + } + Deno.exit(0); } else if (mode === "runner-parent") { const cancellation = installCancellationHandlers(); try { diff --git a/tools/test.ts b/tools/test.ts index 14cb347d4..35bf8c769 100644 --- a/tools/test.ts +++ b/tools/test.ts @@ -1,9 +1,10 @@ -// Bound top-level suites/shards, not total processes; large shards use two test threads. +// Bound top-level suites/shards, not total processes; large shards normally use two threads. // Cargo selects targets/features. Retain all listed names; never skip cached successes. import { spawn } from "node:child_process"; import process from "node:process"; import { availableParallelism } from "node:os"; import { fileURLToPath } from "node:url"; +import { posix } from "node:path"; // Safety limits, NOT the separate 60s performance goal. export const SUITE_TIMEOUT_MS = 180_000; @@ -65,7 +66,19 @@ export function shardRustCommand( if (new Set(names).size !== names.length) { throw new Error("cannot shard duplicate test names"); } - if (names.length <= 72) return [{ task, selectedNames: [...names] }]; + // RuntimeDirSandbox changes process-global paths. Do not overlap its writes + // with restore guards in the same process, including small integration targets. + const serial = ["worker", "worker-runtime", "yoi-workspace-server"].includes( + task.name.split("/")[0], + ); + if (names.length <= 72) { + return [{ + task: serial + ? { ...task, args: [...task.args, "--test-threads", "1"] } + : task, + selectedNames: [...names], + }]; + } const count = Math.min(32, Math.ceil(names.length / 36)); const shards: string[][] = Array.from({ length: count }, () => []); names.forEach((name, index) => shards[index % count].push(name)); @@ -75,7 +88,13 @@ export function shardRustCommand( name: `${task.name}/shard-${ String(index + 1).padStart(2, "0") }-of-${count}`, - args: [...task.args, "--exact", "--test-threads", "2", ...selectedNames], + args: [ + ...task.args, + "--exact", + "--test-threads", + serial ? "1" : "2", + ...selectedNames, + ], }, selectedNames, })); @@ -84,18 +103,27 @@ export function shardRustCommand( export async function discoverRustCommands( commands: TestCommand[], jobs: number, - options: { output: string; signal?: AbortSignal }, + options: { + output: string; + signal?: AbortSignal; + run?: (task: TestCommand, label: string) => Promise; + }, ) { const binaries = await mapConcurrent(commands, jobs, async (task, index) => { - const listing = await execute({ + const listingTask = { ...task, name: `${task.name}/list`, args: [...task.args, "--list", "--format", "terse"], - }, { - ...options, - label: `list-${String(index).padStart(3, "0")}`, - timeoutMs: SUITE_TIMEOUT_MS, - }); + }; + const label = `list-${String(index).padStart(3, "0")}`; + const listing = await (options.run + ? options.run(listingTask, label) + : execute(listingTask, { + output: options.output, + signal: options.signal, + label, + timeoutMs: SUITE_TIMEOUT_MS, + })); let shards: RustShard[] = []; let error: string | undefined; try { @@ -124,7 +152,8 @@ export async function discoverRustCommands( error, selectedCases: error ? null - : shards.reduce((sum, shard) => sum + shard.selectedNames.length, 0), + : shards.reduce((sum, shard) => + sum + shard.selectedNames.length, 0), shards, }; }); @@ -426,12 +455,149 @@ export interface TestCommand { command: string; args: string[]; cwd: string; - env?: Record; + env?: Record; // Exact link from each execution result into inventory.json, even when two // Cargo targets happen to share the same package/target display name. inventoryRef?: { binaryIndex: number; shardIndex: number }; } +// Walk with lstat before creating any child: recursive mkdir/realPath alone +// would follow an existing symlink first. Reject aliases rather than silently +// canonicalizing a caller's checkout. As with the Rust guard, this assumes the +// owned checkout is not concurrently replaced by an adversarial filesystem actor. +async function ownedTestDirectory(checkout: string, relative: string) { + if ( + !posix.isAbsolute(checkout) || checkout === "/" || + posix.normalize(checkout) !== checkout + ) throw new Error("test checkout must be an absolute normal-only path"); + + async function checkDirectory(path: string, owned: boolean) { + const stat = await Deno.lstat(path); + if (!stat.isDirectory || stat.isSymlink) { + throw new Error( + `test isolation refuses non-directory or symlink: ${path}`, + ); + } + if (owned && process.getuid && stat.uid !== process.getuid()) { + throw new Error( + `test isolation directory is not owned by this user: ${path}`, + ); + } + if (await Deno.realPath(path) !== path) { + throw new Error( + `test isolation refuses non-canonical directory: ${path}`, + ); + } + } + let checkoutPath = ""; + for (const part of checkout.slice(1).split("/")) { + checkoutPath += `/${part}`; + await checkDirectory(checkoutPath, false); + } + let path = checkout; + for (const part of relative.split("/")) { + if (!part || part === "." || part === "..") { + throw new Error("invalid owned test directory component"); + } + path += `/${part}`; + try { + await Deno.mkdir(path, { mode: 0o700 }); + } catch (error) { + if (!(error instanceof Deno.errors.AlreadyExists)) throw error; + } + await checkDirectory(path, true); + } + return path; +} + +/** Prepare child-only isolation for the maintained normal-test entrypoint. + * Each wrap gets a fresh allocation table, even for repeated/sanitized labels or + * concurrent runners using the same output id. Resources stay under target for + * diagnosis; callers with temporary checkouts own their recursive cleanup. + */ +export async function prepareTestIsolation(checkout: string, runId: string) { + const ownedDirectory = (relative: string) => + ownedTestDirectory(checkout, relative); + const prefix = "target/restore-test-environment"; + await ownedDirectory(prefix); + // Keep tempfile Unix-socket references short. tempfile owns unique resources + // here; neither runners nor suites remove/replace this shared directory. + const tmp = await ownedDirectory("target/t"); + const digest = new Uint8Array( + await crypto.subtle.digest("SHA-256", new TextEncoder().encode(runId)), + ); + const id = Array.from( + digest.slice(0, 6), + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""); + const relativeRoot = `${prefix}/runner-${id}-${crypto.randomUUID()}`; + const root = await ownedDirectory(relativeRoot); + let next = 0; + return { + root, + async wrapCommand(task: TestCommand, label: string): Promise { + const safeLabel = label.replaceAll(/[^a-zA-Z0-9_-]/g, "-").slice(0, 40) || + "command"; + const relative = `${relativeRoot}/${safeLabel}-${next++}`; + // Recheck shared ancestors as well as the command namespace on every wrap. + await ownedDirectory("target/t"); + const namespace = await ownedDirectory(relative); + for ( + const dir of [ + "config", + "data", + "run", + "home", + "git-template/info", + "git-template/hooks", + ] + ) { + await ownedDirectory(`${relative}/${dir}`); + } + const inherited = { ...process.env, ...task.env }; + const env: NonNullable = { ...task.env }; + // Undefined shadows inherited entries in execute's merge; node spawn omits + // them. Strip indexed Git config and less common Git path overrides too. + for (const key of Object.keys(inherited)) { + if (/^GIT_/i.test(key) || /(^|_)proxy$/i.test(key)) { + env[key] = undefined; + } + } + Object.assign(env, { + HOME: `${namespace}/home`, + YOI_HOME: `${namespace}/data`, + YOI_CONFIG_DIR: `${namespace}/config`, + YOI_DATA_DIR: `${namespace}/data`, + YOI_RUNTIME_DIR: `${namespace}/run`, + TMPDIR: tmp, + TMP: tmp, + TEMP: tmp, + GIT_TEMPLATE_DIR: `${namespace}/git-template`, + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_SYSTEM: "/dev/null", + GIT_CONFIG_NOSYSTEM: "1", + GIT_ATTR_NOSYSTEM: "1", + GIT_CEILING_DIRECTORIES: checkout, + GIT_AUTHOR_NAME: "Normal test runner", + GIT_AUTHOR_EMAIL: "normal-tests@example.invalid", + GIT_COMMITTER_NAME: "Normal test runner", + GIT_COMMITTER_EMAIL: "normal-tests@example.invalid", + RUSTC_WRAPPER: "", + RUSTC_WORKSPACE_WRAPPER: "", + SCCACHE_DISABLE: "1", + }); + // HOME is isolated, but toolchain/cache discovery must keep its ordinary + // host locations (these are compiler settings, not live product state). + for (const key of ["CARGO_HOME", "RUSTUP_HOME"] as const) { + const fallback = key === "CARGO_HOME" ? ".cargo" : ".rustup"; + env[key] = inherited[key] || + (process.env.HOME ? `${process.env.HOME}/${fallback}` : undefined); + } + return { ...task, env }; + }, + }; +} + export function rustTestCommands( artifacts: CargoArtifact[], packages: { id: string; name: string; manifest_path: string }[], @@ -497,16 +663,26 @@ async function main() { "", ); const jobs = parseJobs(Deno.args); - const output = `${root}/target/normal-tests/${ + const outputId = `${ new Date().toISOString().replaceAll(":", "-") - }`; - await Deno.mkdir(output, { recursive: true }); + }-${crypto.randomUUID()}`; + const isolation = await prepareTestIsolation(root, outputId); + const output = await ownedTestDirectory( + root, + `target/normal-tests/${outputId}`, + ); const cancellation = installCancellationHandlers(); - const run = ( + const run = async ( task: TestCommand, label: string, timeoutMs = SUITE_TIMEOUT_MS, - ) => execute(task, { output, label, timeoutMs, signal: cancellation.signal }); + ) => + execute(await isolation.wrapCommand(task, label), { + output, + label, + timeoutMs, + signal: cancellation.signal, + }); const summary: Record = { phase: "metadata", failed: false }; let executionStart: number | undefined; const writeSummary = () => @@ -516,7 +692,7 @@ async function main() { { jobs, concurrency: - "top-level suites/shards; large Rust shards use 2 test threads, other suites retain internal parallelism", + "top-level suites/shards; worker, worker-runtime and yoi-workspace-server use 1 test thread; other large Rust shards use 2, other suites retain internal parallelism", platform: "linux", suiteTimeoutMs: SUITE_TIMEOUT_MS, buildTimeoutMs: BUILD_TIMEOUT_MS, @@ -600,6 +776,7 @@ async function main() { const discovery = await discoverRustCommands(rust, jobs, { output, signal: cancellation.signal, + run, }); summary.discoverySeconds = (performance.now() - discoveryStart) / 1000; summary.inventory = "inventory.json"; @@ -655,6 +832,7 @@ async function main() { "--allow-write", "--allow-run", "--allow-env", + "--allow-sys=uid", "tools/test.test.ts", "tools/check-state-contract-names.test.ts", ], @@ -663,7 +841,7 @@ async function main() { ...discovery.tasks, ]; console.log( - `Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (large Rust shards: 2 test threads; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`, + `Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (worker/worker-runtime/yoi-workspace-server: 1 test thread; other large Rust shards: 2; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`, ); const results: ReturnType[] = []; summary.results = results;