From b7a22146a0c2ea4e0a2c4bcb0324ecefa42ce077 Mon Sep 17 00:00:00 2001 From: Hare Date: Tue, 20 Oct 2026 06:00:58 +0900 Subject: [PATCH 01/46] fix: recover confirmed worker restore failures --- crates/worker-runtime/src/execution.rs | 19 + crates/worker-runtime/src/runtime.rs | 16 +- crates/worker-runtime/src/worker_backend.rs | 1510 ++++++++++++----- .../worker_backend_restore_failure_tests.rs | 945 +++++++++++ crates/worker/src/bootstrap.rs | 6 +- crates/worker/src/controller.rs | 20 +- crates/worker/src/lib.rs | 11 +- .../worker/src/runtime/worker_allocation.rs | 5 +- .../src/runtime/worker_allocation/error.rs | 20 + .../runtime/worker_allocation/lifecycle.rs | 557 +++++- .../src/runtime/worker_allocation/mutate.rs | 25 +- .../src/runtime/worker_allocation/table.rs | 9 + crates/worker/src/worker.rs | 788 ++++++++- crates/worker/tests/controller_test.rs | 205 +++ .../tests/restored_parent_control_tests.rs | 5 +- .../src/server_assignment_attention_tests.rs | 11 +- .../src/server_assignment_delivery_tests.rs | 11 +- .../src/server_workspace_config_wip_tests.rs | 5 +- 18 files changed, 3621 insertions(+), 547 deletions(-) create mode 100644 crates/worker-runtime/src/worker_backend_restore_failure_tests.rs diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index f15d5ca4e..56892006b 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -630,6 +630,16 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { Ok(()) } + /// Completed resource-owner evidence retained while Runtime settlement may + /// still need retry. This is not a new request or execution authorization. + fn completed_restore_failure( + &self, + _operation_id: WorkerLifecycleOperationId, + _worker_ref: &WorkerRef, + ) -> Option { + None + } + fn reconcile_restore( &self, request: WorkerExecutionRestoreRequest, @@ -898,6 +908,15 @@ impl WorkerExecutionBackendRef { self.backend.preflight_restore(request) } + pub(crate) fn completed_restore_failure( + &self, + operation_id: WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Option { + self.backend + .completed_restore_failure(operation_id, worker_ref) + } + pub(crate) fn reconcile_restore( &self, request: WorkerExecutionRestoreRequest, diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 2b671c6e5..9f41c41f4 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -2489,6 +2489,14 @@ impl Runtime { .map(|result| (Some(result), None)) } + #[cfg(all(test, feature = "fs-store"))] + pub(crate) fn test_runtime_store(&self) -> FsRuntimeStore { + match &self.lock().unwrap().persistence { + RuntimePersistence::Fs(store) => store.clone(), + RuntimePersistence::Memory => panic!("expected filesystem Runtime"), + } + } + #[cfg(test)] pub(crate) fn test_restore_request( &self, @@ -2944,7 +2952,11 @@ impl Runtime { return Err(error); } - let restore_result = if reconcile { + let completed_failure = backend.completed_restore_failure(operation_id, worker_ref); + let has_completed_failure = completed_failure.is_some(); + let restore_result = if let Some(result) = completed_failure { + result + } else if reconcile { backend.reconcile_restore(request) } else { backend.restore_worker(request) @@ -3069,7 +3081,7 @@ impl Runtime { } } WorkerExecutionSpawnResult::Rejected(result) => { - if reconcile { + if reconcile && !has_completed_failure { return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_reconciliation_rejected", diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 74044cde9..55617d312 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -175,6 +175,163 @@ pub struct RuntimeWorkerController { pub workspace_client: Arc, } +/// Failure facts supplied by the resource owner, never inferred from error text. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimeRestoreStage { + Preflight, + Metadata, + Reconstruction, + Binding, + Feature, + Controller, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimeRestoreResources { + None, + WorkerStartup, + WorkdirBinding, + Unproven, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum RuntimeRestoreCleanupFailure { + #[error("restore resource ownership unavailable")] + Ownership, + #[error("restore Workdir close failed")] + Workdir, + #[error("restore run artifact cleanup failed")] + Artifacts, + #[error("restore allocation release unconfirmed")] + Allocation, + #[error("restore Feature task settlement unconfirmed")] + Feature, + #[error("factory cannot confirm restore cleanup")] + Unsupported, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RuntimeRestoreCleanup { + NotNeeded, + Required, + Confirmed, + Failed(RuntimeRestoreCleanupFailure), +} + +/// Diagnostics are fixed owner-selected causes, not arbitrary provider messages, +/// paths, configuration, or credentials. Cleanup cannot overwrite the first cause. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeRestoreFailure { + pub stage: RuntimeRestoreStage, + pub resources: RuntimeRestoreResources, + cause: &'static str, + cleanup: RuntimeRestoreCleanup, + io_kind: Option, + cleanup_failure: Option, +} + +impl RuntimeRestoreFailure { + pub fn refused(stage: RuntimeRestoreStage, cause: &'static str) -> Self { + Self { + stage, + resources: RuntimeRestoreResources::None, + cause, + cleanup: RuntimeRestoreCleanup::NotNeeded, + io_kind: None, + cleanup_failure: None, + } + } + + pub fn unproven(stage: RuntimeRestoreStage, cause: &'static str) -> Self { + Self { + stage, + resources: RuntimeRestoreResources::Unproven, + cause, + cleanup: RuntimeRestoreCleanup::Required, + io_kind: None, + cleanup_failure: None, + } + } + + fn started(stage: RuntimeRestoreStage, cause: &'static str) -> Self { + Self { + stage, + resources: RuntimeRestoreResources::WorkerStartup, + cause, + cleanup: RuntimeRestoreCleanup::Required, + io_kind: None, + cleanup_failure: None, + } + } + + fn needs_cleanup(&self) -> bool { + matches!( + self.cleanup, + RuntimeRestoreCleanup::Required | RuntimeRestoreCleanup::Failed(_) + ) + } +} + +impl std::fmt::Display for RuntimeRestoreFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let cause: String = self.cause.chars().take(160).collect(); + write!( + f, + "restore {:?}: {}; io={:?}; resources={:?}; cleanup={:?}; prior_cleanup={:?}", + self.stage, cause, self.io_kind, self.resources, self.cleanup, self.cleanup_failure + ) + } +} + +fn restore_worker_cause(error: &WorkerError) -> &'static str { + match error { + WorkerError::ConstructionCleanup { source, .. } => restore_worker_cause(source), + WorkerError::SegmentEmpty { .. } => "saved segment empty", + WorkerError::SegmentSystemPromptMissing { .. } => "saved segment system prompt missing", + WorkerError::WorkerMetadataPending { .. } => "saved Worker metadata pending", + WorkerError::WorkerMetadataMissing { .. } => "saved Worker metadata missing", + WorkerError::WorkerMetadataInactive { .. } => "saved Worker metadata inactive", + WorkerError::WorkerMetadataManifestSnapshotMissing { .. } => { + "saved manifest snapshot missing" + } + WorkerError::ScopeLock(_) => "Worker allocation acquisition failed", + WorkerError::Store(_) => "saved segment read failed", + WorkerError::WorkerStore(_) => "Worker metadata read failed", + WorkerError::Provider(_) => "saved model provider initialization failed", + WorkerError::SubjektivSessionAttribution { .. } => "restored attribution failed", + WorkerError::FeatureInstall(_) | WorkerError::FeatureLifecycle(_) => { + "restored Feature installation failed" + } + _ => "saved Worker reconstruction failed", + } +} + +async fn finish_restore_failure( + factory: &F, + request: &WorkerExecutionRestoreRequest, + mut failure: RuntimeRestoreFailure, +) -> RuntimeRestoreFailure { + if failure.needs_cleanup() { + let cleanup = if failure.resources == RuntimeRestoreResources::WorkdirBinding { + // Mutable materialization is outside the factory's cleanup authority. + // Preserve the exact accepted operation for its resource owner; never + // infer rollback from a factory that did not acquire these bindings. + Err(RuntimeRestoreCleanupFailure::Unsupported) + } else { + factory.cleanup_failed_restore(request).await + }; + failure.cleanup = match cleanup { + Ok(()) => RuntimeRestoreCleanup::Confirmed, + Err(error) => { + failure.cleanup_failure.get_or_insert(error); + RuntimeRestoreCleanup::Failed(error) + } + }; + } + tracing::warn!(failure = %failure, "Worker restore failed; operation-owned cleanup evidence retained"); + failure +} + /// Factory seam used by [`WorkerRuntimeExecutionBackend`] to construct a real /// controller-backed Worker for a Runtime catalog entry. #[async_trait] @@ -235,7 +392,7 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { async fn restore_controller( &self, request: WorkerExecutionRestoreRequest, - ) -> Result; + ) -> Result; /// Prove cleanup of a restore future that completed without returning a /// controller. An error is not absence evidence; unsupported factories keep @@ -243,8 +400,8 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { async fn cleanup_failed_restore( &self, _request: &WorkerExecutionRestoreRequest, - ) -> Result<(), String> { - Err("Runtime Worker factory cannot prove failed restore cleanup".to_string()) + ) -> Result<(), RuntimeRestoreCleanupFailure> { + Err(RuntimeRestoreCleanupFailure::Unsupported) } /// Verify/release factory-owned resources when the backend registry has no @@ -508,20 +665,73 @@ pub struct ProfileRuntimeWorkerFactory { workspace_request_clients: Arc>, embedded_worker_mutation_dispatcher: Option>, controller_transport: WorkerControllerTransport, - failed_restore_sessions: Arc< - Mutex< - HashMap< - WorkerRef, - ( - crate::execution::WorkerLifecycleOperationId, - Arc, - ), - >, - >, - >, + failed_restore_resources: Arc>>, + #[cfg(test)] + restore_faults: Arc, +} + +#[derive(Clone)] +struct FailedRestoreResources { + operation_id: crate::execution::WorkerLifecycleOperationId, + sessions: Vec>, + allocation: Option, + feature_tasks: Option, +} + +#[cfg(test)] +#[derive(Default)] +struct RestoreFaults { + fail_at: Mutex>, + executions: std::sync::atomic::AtomicUsize, + cleanup_fail: AtomicBool, + socket_failure: AtomicBool, + factory_gate: Mutex>>, + factory_entered: Mutex>>, + cleanup_gate: Mutex>>, + cleanup_entered: Mutex>>, } impl ProfileRuntimeWorkerFactory { + fn retain_construction_failure( + &self, + request: &WorkerExecutionRestoreRequest, + error: &WorkerError, + ) { + let mut resources = self + .failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(owner) = resources.get_mut(&request.worker_ref) else { + return; + }; + if owner.operation_id != request.operation_id { + return; + } + match error { + WorkerError::ConstructionCleanup { + allocation, + sessions, + .. + } => { + owner.allocation = allocation.clone(); + owner.sessions.push(sessions.clone()); + } + WorkerError::ScopeLock(error) => owner.allocation = error.allocation_cleanup_handle(), + _ => {} + } + } + + #[cfg(test)] + fn restore_checkpoint(&self, stage: RuntimeRestoreStage) -> Result<(), String> { + let mut failure = self.restore_faults.fail_at.lock().unwrap(); + if *failure == Some(stage) { + failure.take(); + Err("injected restore boundary failure".into()) + } else { + Ok(()) + } + } + pub fn new(profile_base_dir: impl Into) -> Self { let profile_base_dir = profile_base_dir.into(); Self { @@ -536,7 +746,9 @@ impl ProfileRuntimeWorkerFactory { workspace_request_clients: Arc::new(HashMap::new()), embedded_worker_mutation_dispatcher: None, controller_transport: WorkerControllerTransport::InProcess, - failed_restore_sessions: Arc::new(Mutex::new(HashMap::new())), + failed_restore_resources: Arc::new(Mutex::new(HashMap::new())), + #[cfg(test)] + restore_faults: Arc::new(RestoreFaults::default()), } } @@ -1708,265 +1920,403 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { async fn restore_controller( &self, request: WorkerExecutionRestoreRequest, - ) -> Result { - self.preflight_restore(&request).await?; - let worker_name = Self::runtime_worker_name_for_ref(&request.worker_ref); - let only_binding = (request.workdir_attachments.len() == 1) - .then(|| request.workdir_attachments.values().next()) - .flatten(); - let filesystem_authority = only_binding - .map(|binding| { - WorkerFilesystemAuthority::local( - binding.root().to_path_buf(), - binding.cwd().to_path_buf(), - ) - }) - .unwrap_or(WorkerFilesystemAuthority::None); - let workspace_backend_ref = RuntimeWorkspaceBackendRef::from_worker_request( - &request.request, - self.runtime_id.as_deref(), - ); - let observation_runtime_id = self.runtime_id.clone(); - let observation_workspace_id = request - .request - .workspace_api - .as_ref() - .map(|api| api.workspace_id.clone()); - let observation_grants = request.request.worker_observation_grants.clone(); - let observation_enabled = request.request.worker_observation_enabled; - let workspace_request_client = request - .request - .workspace_api - .as_ref() - .and_then(|api| self.workspace_request_clients.get(&api.workspace_id)); - let workspace_context = workspace_backend_ref.worker_context( - &request.worker_ref, - request.workspace_scope.as_ref(), - self.worker_mutation_identity.as_ref(), - workspace_request_client, - self.embedded_worker_mutation_dispatcher.as_ref(), - Some(self.prompt_projection_cache.clone()), - ); - - let worker_aggregate_dir = self.worker_aggregate_dir(&request.worker_ref)?; - let session_dir = worker_aggregate_dir.join("session"); - let session_store = - WorkerSessionStore::new_with_retention(&session_dir, self.session_retention()?) - .map_err(|err| { - format!( - "failed to initialize canonical Worker Session store at {}: {err}", - session_dir.display() - ) - })?; - let worker_metadata_store = - WorkerAggregateStore::new(&worker_aggregate_dir, worker_name.clone()).map_err( - |err| { - format!( - "failed to initialize canonical Worker metadata store at {}: {err}", - worker_aggregate_dir.display() - ) - }, - )?; - let metadata = worker_metadata_store - .read_by_name(&worker_name) - .map_err(|error| format!("failed to read Worker metadata: {error}"))? - .ok_or_else(|| "Persisted Worker metadata is unavailable".to_string())?; - let (manifest, loader) = Self::manifest_for_restore(&metadata, &request.request)?; - let store = CombinedStore::new(session_store, worker_metadata_store); - - let mut worker = match Worker::restore_from_worker_metadata_with_context( - &worker_name, - manifest.clone(), - store.clone(), - loader.clone(), - workspace_context.clone(), - filesystem_authority.clone(), - ) - .await - { - Ok(worker) => worker, - Err(WorkerError::WorkerMetadataPending { .. }) - if request.request.initial_input.is_none() => - { - let pending_loader = if workspace_context.workspace_id().is_some() { - let bundle = request.config_bundle.as_ref().ok_or_else(|| { - "pending Workspace Worker restore requires operation-owned launch material; generic restore must not reconstruct it from current Workspace config" - .to_string() - })?; - let resolution = self - .observe_bundle_prompt_projection( - bundle, - observation_workspace_id.as_deref(), - )? - .ok_or_else(|| { - "pending Workspace Worker restore requires a saved Workspace Prompt projection" - .to_string() - })?; - loader - .clone() - .with_effective_catalog(resolution.projection.catalog.clone()) - } else { - loader.clone() - }; - Worker::restore_pending_from_worker_metadata_with_context( - &worker_name, - manifest.clone(), - store, - pending_loader, - workspace_context, - filesystem_authority, - ) - .await - .map_err(|err| format!("failed to recreate pending Worker from metadata: {err}"))? - } - Err(err) => return Err(format!("failed to restore Worker from metadata: {err}")), - }; - if let Some(binding) = request.request.backend_job.as_ref() { - worker - .bind_backend_job(binding.clone()) - .map_err(str::to_string)?; - } - validate_worker_memory_settings(worker.manifest(), &request.request)?; - worker - .finalize_subjektiv_session_attribution( - request.request.subjektiv_attached, - SubjektivSessionAttributionLifecycle::RestoredSession, + ) -> Result { + self.preflight_restore(&request).await.map_err(|_| { + RuntimeRestoreFailure::refused( + RuntimeRestoreStage::Preflight, + "restore preflight refused", ) - .map_err(|error| format!("finalize restored Worker Session attribution: {error}"))?; - let flow_transition_enabled = worker.manifest().feature.flow.enabled; - let workdir_sessions = restored_workdir_router( - &request.workdir_attachments, - &request.logical_workdir_attachments, - &request.request.workdir_attachments, - worker.scope().clone(), - worker.workspace_client_handle(), - )?; - worker.bind_workdir_sessions(workdir_sessions); - if let (Some(runtime_id), Some(workspace_id)) = - (observation_runtime_id, observation_workspace_id.clone()) - && observation_enabled - { - let mut providers: Vec> = vec![Arc::new( - WorkspaceClientWorkerObservationProvider::new(worker.workspace_client_handle()), - )]; - if !observation_grants.is_empty() { - providers.push(Arc::new(RuntimeGrantedWorkerObservationProvider { - runtime_id, - workspace_id, - grants: observation_grants.into_iter().take(100).collect(), - hub: self.observation_hub.clone(), - })); - } - worker.bind_worker_observation_provider(Some(Arc::new( - CompositeWorkerObservationProvider::new(providers), - ))); - } - if flow_transition_enabled { - let report = worker - .install_runtime_flow_transition_feature() - .map_err(|error| format!("install Flow transition feature: {error}"))?; - if report.reports.iter().any(|report| !report.installed) { - return Err(format!( - "install Flow transition feature failed: {:?}", - report.reports - )); - } - } - - let workspace_client = worker.workspace_client_handle(); - let run_dir = self.worker_restore_run_dir(&request.worker_ref, request.operation_id)?; - let bash_output_dir = bash_output_dir_for_worker_id(&request.worker_ref.worker_id); - let cleanup_sessions = worker.workdir_sessions(); - let started = PreparedWorker::new( - worker, - WorkerBootstrapLayout::RuntimeManagedRun { - run_dir: run_dir.clone(), - bash_output_dir, - }, - self.controller_transport, - ) - .start() - .await - .map_err(|error| match error { - WorkerBootstrapError::Worker(source) => { - format!("failed to prepare restored Worker: {source}") - } - WorkerBootstrapError::Controller { - source, - cleanup_failed, - } => { - if cleanup_failed { - self.failed_restore_sessions - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert( - request.worker_ref.clone(), - (request.operation_id, cleanup_sessions), - ); - } - format!( - "failed to spawn restored Worker controller in {}: {source}", - run_dir.display() - ) - } })?; - let (handle, shutdown_rx, controller_task) = - (started.handle, started.shutdown, started.controller_task); - if flow_transition_enabled { - handle.shared_state.enable_flow_transition(); + #[cfg(test)] + self.restore_faults + .executions + .fetch_add(1, Ordering::SeqCst); + let mut stage = RuntimeRestoreStage::Metadata; + let mut io_kind = None; + // Reconstruction can write before failing; only the resource owner can + // confirm cleanup, even when no Worker value was returned. + let result: Result = async { + let worker_name = Self::runtime_worker_name_for_ref(&request.worker_ref); + let only_binding = (request.workdir_attachments.len() == 1) + .then(|| request.workdir_attachments.values().next()) + .flatten(); + let filesystem_authority = only_binding + .map(|binding| { + WorkerFilesystemAuthority::local( + binding.root().to_path_buf(), + binding.cwd().to_path_buf(), + ) + }) + .unwrap_or(WorkerFilesystemAuthority::None); + let workspace_backend_ref = RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + self.runtime_id.as_deref(), + ); + let observation_runtime_id = self.runtime_id.clone(); + let observation_workspace_id = request + .request + .workspace_api + .as_ref() + .map(|api| api.workspace_id.clone()); + let observation_grants = request.request.worker_observation_grants.clone(); + let observation_enabled = request.request.worker_observation_enabled; + let workspace_request_client = request + .request + .workspace_api + .as_ref() + .and_then(|api| self.workspace_request_clients.get(&api.workspace_id)); + let workspace_context = workspace_backend_ref.worker_context( + &request.worker_ref, + request.workspace_scope.as_ref(), + self.worker_mutation_identity.as_ref(), + workspace_request_client, + self.embedded_worker_mutation_dispatcher.as_ref(), + Some(self.prompt_projection_cache.clone()), + ); + + let worker_aggregate_dir = self + .worker_aggregate_dir(&request.worker_ref) + .map_err(|_| "Worker aggregate root unavailable")?; + let session_dir = worker_aggregate_dir.join("session"); + let session_store = WorkerSessionStore::new_with_retention( + &session_dir, + self.session_retention() + .map_err(|_| "Session retention unavailable")?, + ) + .map_err(|_| "canonical Session store initialization failed")?; + let worker_metadata_store = + WorkerAggregateStore::new(&worker_aggregate_dir, worker_name.clone()) + .map_err(|_| "canonical metadata store initialization failed")?; + let metadata = worker_metadata_store + .read_by_name(&worker_name) + .map_err(|_| "Worker metadata read failed")? + .ok_or("saved Worker metadata unavailable")?; + let (manifest, loader) = Self::manifest_for_restore(&metadata, &request.request) + .map_err(|_| "saved manifest resolution failed")?; + let store = CombinedStore::new(session_store, worker_metadata_store); + + stage = RuntimeRestoreStage::Reconstruction; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: Vec::new(), + allocation: None, + feature_tasks: None, + }, + ); + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + let mut worker = match Worker::restore_from_worker_metadata_with_context( + &worker_name, + manifest.clone(), + store.clone(), + loader.clone(), + workspace_context.clone(), + filesystem_authority.clone(), + ) + .await + { + Ok(worker) => worker, + Err(WorkerError::WorkerMetadataPending { .. }) + if request.request.initial_input.is_none() => + { + let pending_loader = if workspace_context.workspace_id().is_some() { + let bundle = request + .config_bundle + .as_ref() + .ok_or("pending restore launch material missing")?; + let resolution = self + .observe_bundle_prompt_projection( + bundle, + observation_workspace_id.as_deref(), + ) + .map_err(|_| "pending restore Prompt projection invalid")? + .ok_or("pending restore Prompt projection missing")?; + loader + .clone() + .with_effective_catalog(resolution.projection.catalog.clone()) + } else { + loader.clone() + }; + Worker::restore_pending_from_worker_metadata_with_context( + &worker_name, + manifest.clone(), + store, + pending_loader, + workspace_context, + filesystem_authority, + ) + .await + .map_err(|error| { + self.retain_construction_failure(&request, &error); + restore_worker_cause(&error) + })? + } + Err(error) => { + self.retain_construction_failure(&request, &error); + return Err(restore_worker_cause(&error)); + } + }; + let feature_tasks = worker.retain_startup_feature_cleanup(); + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: vec![worker.workdir_sessions()], + allocation: worker.restore_allocation_cleanup_handle(), + feature_tasks: Some(feature_tasks), + }, + ); + #[cfg(test)] + { + let gate = self.restore_faults.factory_gate.lock().unwrap().take(); + if let Some(gate) = gate { + if let Some(entered) = + self.restore_faults.factory_entered.lock().unwrap().take() + { + let _ = entered.send(()); + } + gate.notified().await; + } + } + stage = RuntimeRestoreStage::Binding; + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + if let Some(binding) = request.request.backend_job.as_ref() { + worker + .bind_backend_job(binding.clone()) + .map_err(|_| "restored Backend Job binding failed")?; + } + validate_worker_memory_settings(worker.manifest(), &request.request) + .map_err(|_| "saved memory settings mismatch")?; + worker + .finalize_subjektiv_session_attribution( + request.request.subjektiv_attached, + SubjektivSessionAttributionLifecycle::RestoredSession, + ) + .map_err(|error| restore_worker_cause(&error))?; + let flow_transition_enabled = worker.manifest().feature.flow.enabled; + let workdir_sessions = restored_workdir_router( + &request.workdir_attachments, + &request.logical_workdir_attachments, + &request.request.workdir_attachments, + worker.scope().clone(), + worker.workspace_client_handle(), + ) + .map_err(|_| "restored Workdir binding failed")?; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get_mut(&request.worker_ref) + .expect("retained restore owner") + .sessions + .push(workdir_sessions.clone()); + worker + .workdir_sessions() + .close_all() + .await + .map_err(|_| "initial Workdir close failed")?; + worker.bind_workdir_sessions(workdir_sessions); + if let (Some(runtime_id), Some(workspace_id)) = + (observation_runtime_id, observation_workspace_id.clone()) + && observation_enabled + { + let mut providers: Vec> = vec![Arc::new( + WorkspaceClientWorkerObservationProvider::new(worker.workspace_client_handle()), + )]; + if !observation_grants.is_empty() { + providers.push(Arc::new(RuntimeGrantedWorkerObservationProvider { + runtime_id, + workspace_id, + grants: observation_grants.into_iter().take(100).collect(), + hub: self.observation_hub.clone(), + })); + } + worker.bind_worker_observation_provider(Some(Arc::new( + CompositeWorkerObservationProvider::new(providers), + ))); + } + stage = RuntimeRestoreStage::Feature; + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + if flow_transition_enabled { + let report = worker + .install_runtime_flow_transition_feature() + .map_err(|_| "Flow transition installation failed")?; + if report.reports.iter().any(|report| !report.installed) { + return Err("Flow transition installation refused"); + } + } + + stage = RuntimeRestoreStage::Controller; + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + let workspace_client = worker.workspace_client_handle(); + let run_dir = self + .worker_restore_run_dir(&request.worker_ref, request.operation_id) + .map_err(|_| "restore run directory unavailable")?; + let bash_output_dir = bash_output_dir_for_worker_id(&request.worker_ref.worker_id); + let transport = self.controller_transport; + #[cfg(test)] + let transport = if self.restore_faults.socket_failure.load(Ordering::SeqCst) { + WorkerControllerTransport::UnixSocket + } else { + transport + }; + let started = PreparedWorker::new( + worker, + WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: run_dir.clone(), + bash_output_dir, + }, + transport, + ) + .start() + .await + .map_err(|error| match error { + WorkerBootstrapError::Worker(error) => restore_worker_cause(&error), + WorkerBootstrapError::Controller { source, .. } => { + io_kind = Some(source.kind()); + "restored Worker controller startup failed" + } + })?; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&request.worker_ref); + let (handle, shutdown_rx, controller_task) = + (started.handle, started.shutdown, started.controller_task); + if flow_transition_enabled { + handle.shared_state.enable_flow_transition(); + } + Ok(RuntimeWorkerController { + handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), + controller_task, + workspace_client, + }) } - Ok(RuntimeWorkerController { - handle, - shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), - controller_task, - workspace_client, + .await; + result.map_err(|cause| { + let mut failure = RuntimeRestoreFailure::started(stage, cause); + failure.io_kind = io_kind; + failure }) } async fn cleanup_failed_restore( &self, request: &WorkerExecutionRestoreRequest, - ) -> Result<(), String> { + ) -> Result<(), RuntimeRestoreCleanupFailure> { + #[cfg(test)] + { + let gate = self.restore_faults.cleanup_gate.lock().unwrap().take(); + if let Some(gate) = gate { + if let Some(entered) = self.restore_faults.cleanup_entered.lock().unwrap().take() { + let _ = entered.send(()); + } + gate.notified().await; + } + if self.restore_faults.cleanup_fail.load(Ordering::SeqCst) { + return Err(RuntimeRestoreCleanupFailure::Workdir); + } + } let retained = self - .failed_restore_sessions + .failed_restore_resources .lock() - .map_err(|_| "failed restore sessions lock is poisoned".to_string())? + .map_err(|_| RuntimeRestoreCleanupFailure::Ownership)? .get(&request.worker_ref) .cloned(); - if let Some((operation_id, sessions)) = retained { - if operation_id != request.operation_id { - return Err("failed restore sessions belong to another operation".to_string()); + if let Some(resources) = retained { + if resources.operation_id != request.operation_id { + return Err(RuntimeRestoreCleanupFailure::Ownership); + } + if let Some(feature_tasks) = resources.feature_tasks { + feature_tasks + .shutdown() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Feature)?; + } + for sessions in resources.sessions { + sessions + .close_all() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; + } + if let Some(allocation) = resources.allocation { + allocation + .release() + .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; + } else { + // Reconstruction may fail before returning its allocation guard. + // Absence is checked at the allocation authority, not inferred from Err. + let path = worker::runtime::worker_allocation::default_allocation_path() + .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; + let table = worker::runtime::worker_allocation::LockFileGuard::open(&path) + .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; + if table + .data() + .find(&Self::runtime_worker_name_for_ref(&request.worker_ref)) + .is_some() + { + return Err(RuntimeRestoreCleanupFailure::Allocation); + } } - sessions - .close_all() - .await - .map_err(|error| format!("failed restore session cleanup: {error}"))?; } // restore_controller starts the Controller only as its final fallible // step. A completed error leaves no live Controller; remove only this // operation's disposable run artifacts, never Session/metadata authority. - self.stopped_restore_run_dirs(&request.worker_ref)?; - let run_dir = self.worker_restore_run_dir(&request.worker_ref, request.operation_id)?; - Self::remove_restore_run_artifacts(&run_dir)?; - self.failed_restore_sessions + self.stopped_restore_run_dirs(&request.worker_ref) + .map_err(|_| RuntimeRestoreCleanupFailure::Artifacts)?; + let run_dir = self + .worker_restore_run_dir(&request.worker_ref, request.operation_id) + .map_err(|_| RuntimeRestoreCleanupFailure::Artifacts)?; + Self::remove_restore_run_artifacts(&run_dir) + .map_err(|_| RuntimeRestoreCleanupFailure::Artifacts)?; + self.failed_restore_resources .lock() - .map_err(|_| "failed restore sessions lock is poisoned".to_string())? + .map_err(|_| RuntimeRestoreCleanupFailure::Ownership)? .remove(&request.worker_ref); + self.observation_hub.get(&request.worker_ref); Ok(()) } async fn reconcile_stopped_worker(&self, worker_ref: &WorkerRef) -> Result<(), String> { let retained = self - .failed_restore_sessions + .failed_restore_resources .lock() .map_err(|_| "failed restore sessions lock is poisoned".to_string())? .get(worker_ref) .cloned(); - if let Some((_, sessions)) = retained.as_ref() { - sessions - .close_all() - .await - .map_err(|error| format!("failed restore session cleanup: {error}"))?; + if let Some(resources) = retained.as_ref() { + if let Some(feature_tasks) = resources.feature_tasks.as_ref() { + feature_tasks + .shutdown() + .await + .map_err(|_| "failed restore Feature cleanup".to_string())?; + } + for sessions in &resources.sessions { + sessions + .close_all() + .await + .map_err(|_| "failed restore Workdir cleanup".to_string())?; + } + if let Some(allocation) = resources.allocation.as_ref() { + allocation + .release() + .map_err(|_| "failed restore allocation cleanup".to_string())?; + } } // A restart loses the map, not the operation-owned filesystem artifacts. // Validate every candidate before deleting any, and retain map/journal @@ -1975,7 +2325,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { Self::remove_restore_run_artifacts(&run_dir)?; } if retained.is_some() { - self.failed_restore_sessions + self.failed_restore_resources .lock() .map_err(|_| "failed restore sessions lock is poisoned".to_string())? .remove(worker_ref); @@ -2310,7 +2660,7 @@ struct RuntimeWorkerExecution { struct PendingRuntimeRestore { request: WorkerExecutionRestoreRequest, task: tokio::sync::Mutex>>, - result: Mutex>>, + result: Mutex>>, failure: Mutex>, } @@ -2332,8 +2682,14 @@ impl PendingRuntimeRestore { } Ok(Err(error)) => { task.take(); - let message = - format!("restore factory task failed: {error}; cleanup is unproven"); + let message = format!( + "restore factory task failed ({}); cleanup is unproven", + if error.is_panic() { + "panic" + } else { + "cancelled" + } + ); *self .failure .lock() @@ -2363,6 +2719,18 @@ pub struct WorkerRuntimeExecutionBackend { runtime: Mutex>, workers: Mutex>, pending_restores: Mutex>>, + // One completed execution receipt per Worker, separate from live resources. + // Runtime owns durable request replay; retain this until the next admitted + // operation so a failed canonical settlement cannot re-execute the factory. + completed_restore_failures: Mutex< + HashMap< + WorkerRef, + ( + crate::execution::WorkerLifecycleOperationId, + RuntimeRestoreFailure, + ), + >, + >, worker_locks: Mutex>>>, spawn_restore_timeout: Duration, subject_reservations: Mutex>, @@ -2386,6 +2754,7 @@ where runtime: Mutex::new(Some(runtime)), workers: Mutex::new(HashMap::new()), pending_restores: Mutex::new(HashMap::new()), + completed_restore_failures: Mutex::new(HashMap::new()), worker_locks: Mutex::new(HashMap::new()), spawn_restore_timeout: SPAWN_RESTORE_TASK_TIMEOUT, subject_reservations: Mutex::new(HashMap::new()), @@ -2519,6 +2888,82 @@ where .map_err(|err| format!("worker adapter task did not complete: {err}"))? } + fn preflight_restore_inputs( + &self, + request: &WorkerExecutionRestoreRequest, + ) -> Result<(), WorkerExecutionResult> { + if !request.previous_workdir_attachments.is_empty() { + let materializer = self + .working_directory_materializer + .as_ref() + .ok_or_else(|| { + WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "Persisted Worker Workdir attachments cannot be restored by this Runtime", + ) + })?; + for previous in &request.previous_workdir_attachments { + let relative_cwd = request + .request + .workdir_attachments + .iter() + .find(|claim| claim.alias == previous.alias) + .and_then(|claim| claim.relative_cwd.as_deref()); + // Recovery attachments remain CleanupPending across restart. + // Validate current bindability/identity, not the old state name. + // Preflight is read-only; actual restore binds again below. + materializer.preflight_bind_working_directory( + &previous.working_directory.summary.working_directory_id, + relative_cwd, + ).map_err(|message| { + WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + format!("Persisted Worker Workdir attachment `{}` is unavailable: {message}", previous.alias), + ) + })?; + } + } + if request.previous_workdir_attachments.is_empty() + && !request.request.workdir_attachment_requests.is_empty() + { + return Err(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "Persisted Worker Workdir allocations are unavailable", + )); + } + if request.previous_workdir_attachments.is_empty() + && !request.request.workdir_attachments.is_empty() + && self.working_directory_materializer.is_none() + { + return Err(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "Persisted Worker Workdir claims cannot be restored by this Runtime", + )); + } + if let Some(workspace_api) = request.request.workspace_api.as_ref() { + let Some(scope) = request.workspace_scope.as_ref() else { + return Err(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "Persisted Workspace restore authorization is unavailable", + )); + }; + if scope.workspace_id != workspace_api.workspace_id { + return Err(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "Persisted Workspace restore authorization does not match Worker authority", + )); + } + } + + let factory = Arc::clone(&self.factory); + let request = request.clone(); + self.run_on_adapter_runtime(async move { factory.preflight_restore(&request).await }) + .map_err(|message| { + WorkerExecutionResult::rejected(WorkerExecutionOperation::Restore, message) + })?; + Ok(()) + } + fn pending_restore( &self, worker_ref: &WorkerRef, @@ -2534,10 +2979,94 @@ where pending: &Arc, timeout: Duration, ) -> Result<(), String> { - let pending = Arc::clone(pending); + let owner = Arc::clone(pending); self.run_cancellable_on_adapter_runtime(timeout + Duration::from_secs(1), async move { - pending.wait(timeout).await + owner.wait(timeout).await }) + .map_err(|message| { + match pending.result.lock().ok().and_then(|result| { + result + .as_ref() + .and_then(|result| result.as_ref().err()) + .cloned() + }) { + Some(failure) => format!("{failure}; {message}"), + None => message, + } + }) + } + + // Binding failures cannot borrow the factory's cleanup proof. Retain the + // exact operation and any already acquired bindings for their actual owner. + fn retain_restore_binding_failure( + &self, + mut request: WorkerExecutionRestoreRequest, + bindings: BTreeMap, + ) -> WorkerExecutionSpawnResult { + request.workdir_attachments = bindings; + let mut failure = RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Binding, + "restore Workdir binding failed; materializer reconciliation required", + ); + failure.resources = RuntimeRestoreResources::WorkdirBinding; + failure.cleanup = RuntimeRestoreCleanup::Failed(RuntimeRestoreCleanupFailure::Unsupported); + failure.cleanup_failure = Some(RuntimeRestoreCleanupFailure::Unsupported); + self.pending_restores + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + Arc::new(PendingRuntimeRestore { + request, + task: tokio::sync::Mutex::new(None), + result: Mutex::new(Some(Err(failure.clone()))), + failure: Mutex::new(None), + }), + ); + WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + failure.to_string(), + ), + worker_state: None, + workdir_attachments: Vec::new(), + } + } + + // Called only after joining the previous owned task, under the Worker lock. + // A caller timeout never cancels this close/join future. + fn retry_pending_restore_cleanup( + &self, + pending: &Arc, + ) -> Result<(), String> { + let failure = pending + .result + .lock() + .map_err(|_| "restore result lock is poisoned")? + .as_ref() + .and_then(|result| result.as_ref().err()) + .filter(|failure| failure.needs_cleanup()) + .cloned(); + if let Some(failure) = failure { + let owner = Arc::clone(pending); + let factory = Arc::clone(&self.factory); + let mut task = pending + .task + .try_lock() + .map_err(|_| "restore task is still owned")?; + if task.is_some() { + return Err("restore task has not been joined".into()); + } + *task = Some(self.spawn_on_adapter_runtime(async move { + let failure = + finish_restore_failure(factory.as_ref(), &owner.request, failure).await; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Err(failure)); + })?); + } + Ok(()) } fn remove_pending_restore(&self, worker_ref: &WorkerRef) -> Result<(), String> { @@ -3230,6 +3759,12 @@ where if let Err(message) = self.wait_pending_restore(&pending, Duration::from_secs(5)) { return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } + if let Err(message) = self + .retry_pending_restore_cleanup(&pending) + .and_then(|()| self.wait_pending_restore(&pending, Duration::from_secs(5))) + { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } if self.workers.lock().is_err() { return WorkerExecutionResult::errored( WorkerExecutionOperation::Stop, @@ -3264,17 +3799,14 @@ where Vec::new(), ); } - Some(Err(_)) => { - let factory = Arc::clone(&self.factory); - let request = pending.request.clone(); - if let Err(message) = self - .run_cancellable_on_adapter_runtime(RUNTIME_TASK_TIMEOUT, async move { - factory.cleanup_failed_restore(&request).await - }) - { + Some(Err(failure)) => { + if !matches!( + failure.cleanup, + RuntimeRestoreCleanup::NotNeeded | RuntimeRestoreCleanup::Confirmed + ) { return WorkerExecutionResult::errored( WorkerExecutionOperation::Stop, - message, + failure.to_string(), ); } } @@ -4067,76 +4599,29 @@ where } drop(workers); - if !request.previous_workdir_attachments.is_empty() { - let materializer = self - .working_directory_materializer - .as_ref() - .ok_or_else(|| { - WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - "Persisted Worker Workdir attachments cannot be restored by this Runtime", - ) - })?; - for previous in &request.previous_workdir_attachments { - let relative_cwd = request - .request - .workdir_attachments - .iter() - .find(|claim| claim.alias == previous.alias) - .and_then(|claim| claim.relative_cwd.as_deref()); - // Recovery attachments remain CleanupPending across restart. - // Validate current bindability/identity, not the old state name. - // Preflight is read-only; actual restore binds again below. - materializer.preflight_bind_working_directory( - &previous.working_directory.summary.working_directory_id, - relative_cwd, - ).map_err(|message| { - WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - format!("Persisted Worker Workdir attachment `{}` is unavailable: {message}", previous.alias), - ) - })?; - } - } - if request.previous_workdir_attachments.is_empty() - && !request.request.workdir_attachment_requests.is_empty() - { - return Err(WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - "Persisted Worker Workdir allocations are unavailable", - )); - } - if request.previous_workdir_attachments.is_empty() - && !request.request.workdir_attachments.is_empty() - && self.working_directory_materializer.is_none() - { - return Err(WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - "Persisted Worker Workdir claims cannot be restored by this Runtime", - )); - } - if let Some(workspace_api) = request.request.workspace_api.as_ref() { - let Some(scope) = request.workspace_scope.as_ref() else { - return Err(WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - "Persisted Workspace restore authorization is unavailable", - )); - }; - if scope.workspace_id != workspace_api.workspace_id { - return Err(WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - "Persisted Workspace restore authorization does not match Worker authority", - )); - } - } + self.preflight_restore_inputs(request) + } - let factory = Arc::clone(&self.factory); - let request = request.clone(); - self.run_on_adapter_runtime(async move { factory.preflight_restore(&request).await }) - .map_err(|message| { - WorkerExecutionResult::rejected(WorkerExecutionOperation::Restore, message) - })?; - Ok(()) + fn completed_restore_failure( + &self, + operation_id: crate::execution::WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Option { + let completed = self + .completed_restore_failures + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let (completed_id, failure) = completed.get(worker_ref)?; + if *completed_id != operation_id { + return None; + } + let result = + WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, failure.to_string()); + Some(if failure.cleanup == RuntimeRestoreCleanup::NotNeeded { + WorkerExecutionSpawnResult::Rejected(result) + } else { + WorkerExecutionSpawnResult::RolledBack(result) + }) } fn restore_worker( @@ -4162,6 +4647,11 @@ where )); } }; + if let Some(result) = + self.completed_restore_failure(request.operation_id, &request.worker_ref) + { + return result; + } let workers = self.workers.lock().map_err(|_| { WorkerExecutionResult::errored( WorkerExecutionOperation::Restore, @@ -4206,93 +4696,138 @@ where } drop(workers); - let mut workdir_attachments = BTreeMap::new(); - if !request.previous_workdir_attachments.is_empty() { - let Some(materializer) = self.working_directory_materializer.as_ref() else { - return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - "persisted Worker has Workdir attachments, but no materializer is configured for this Runtime backend", - )); - }; - for status in request.previous_workdir_attachments.clone() { - let relative_cwd = request + // An already accepted operation owns its bindings. Re-observing mutable + // Workdir materialization must not turn an unknown operation into a new + // read-only refusal or allocate another candidate. + let admitted_pending = match self.pending_restore(&request.worker_ref) { + Ok(Some(pending)) if pending.request.operation_id == request.operation_id => { + Some(pending) + } + Ok(Some(_)) => { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::busy( + WorkerExecutionOperation::Restore, + "another restore operation still owns pending resources", + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + Ok(None) => None, + Err(message) => { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + message, + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + }; + if let Some(pending) = admitted_pending.as_ref() { + request = pending.request.clone(); + } else { + if let Err(result) = self.preflight_restore_inputs(&request) { + return WorkerExecutionSpawnResult::Rejected(result); + } + let aliases = if request.previous_workdir_attachments.is_empty() { + request .request .workdir_attachments .iter() - .find(|claim| claim.alias == status.alias) - .and_then(|claim| claim.relative_cwd.as_deref()); - match materializer.bind_working_directory( - &status.working_directory.summary.working_directory_id, - relative_cwd, - ) { - Ok(binding) => { - if workdir_attachments - .insert(status.alias.clone(), binding) - .is_some() - { - return WorkerExecutionSpawnResult::Rejected( - WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - format!( - "duplicate persisted Workdir attachment alias `{}`", - status.alias - ), - ), - ); - } - } - Err(error) => { - return WorkerExecutionSpawnResult::Rejected( - WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - error.to_string(), - ), - ); - } - } - } - } else if !request.request.workdir_attachments.is_empty() { - let Some(materializer) = self.working_directory_materializer.as_ref() else { + .map(|claim| &claim.alias) + .collect::>() + } else { + request + .previous_workdir_attachments + .iter() + .map(|status| &status.alias) + .collect::>() + }; + if aliases + .iter() + .collect::>() + .len() + != aliases.len() + { return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( WorkerExecutionOperation::Restore, - "persisted Worker has Workdir claims, but no materializer is configured for this Runtime backend", + "duplicate restore Workdir attachment alias", )); - }; - for claim in request.request.workdir_attachments.clone() { - match materializer.bind_working_directory( - &claim.working_directory_id, - claim.relative_cwd.as_deref(), - ) { - Ok(binding) => { - if workdir_attachments - .insert(claim.alias.clone(), binding) - .is_some() - { - return WorkerExecutionSpawnResult::Rejected( - WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - format!("duplicate Workdir attachment alias `{}`", claim.alias), - ), - ); + } + let mut workdir_attachments = BTreeMap::new(); + if !request.previous_workdir_attachments.is_empty() { + let Some(materializer) = self.working_directory_materializer.as_ref() else { + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "persisted Worker has Workdir attachments, but no materializer is configured for this Runtime backend", + )); + }; + for status in request.previous_workdir_attachments.clone() { + let relative_cwd = request + .request + .workdir_attachments + .iter() + .find(|claim| claim.alias == status.alias) + .and_then(|claim| claim.relative_cwd.as_deref()); + match materializer.bind_working_directory( + &status.working_directory.summary.working_directory_id, + relative_cwd, + ) { + Ok(binding) => { + if workdir_attachments + .insert(status.alias.clone(), binding) + .is_some() + { + return self + .retain_restore_binding_failure(request, workdir_attachments); + } + } + Err(_) => { + return self + .retain_restore_binding_failure(request, workdir_attachments); } } - Err(error) => { - return WorkerExecutionSpawnResult::Rejected( - WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - error.to_string(), - ), - ); + } + } else if !request.request.workdir_attachments.is_empty() { + let Some(materializer) = self.working_directory_materializer.as_ref() else { + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "persisted Worker has Workdir claims, but no materializer is configured for this Runtime backend", + )); + }; + for claim in request.request.workdir_attachments.clone() { + match materializer.bind_working_directory( + &claim.working_directory_id, + claim.relative_cwd.as_deref(), + ) { + Ok(binding) => { + if workdir_attachments + .insert(claim.alias.clone(), binding) + .is_some() + { + return self + .retain_restore_binding_failure(request, workdir_attachments); + } + } + Err(_) => { + return self + .retain_restore_binding_failure(request, workdir_attachments); + } } } } + request.workdir_attachments = workdir_attachments.clone(); } - request.workdir_attachments = workdir_attachments.clone(); - let worker_ref = request.worker_ref.clone(); let operation_id = request.operation_id; + let mut existing_pending = false; let pending = match self.pending_restore(&worker_ref) { - Ok(Some(pending)) if pending.request.operation_id == operation_id => pending, + Ok(Some(pending)) if pending.request.operation_id == operation_id => { + existing_pending = true; + pending + } Ok(Some(_)) => { return WorkerExecutionSpawnResult::ReconciliationRequired { result: WorkerExecutionResult::busy( @@ -4304,16 +4839,10 @@ where }; } Ok(None) => { - let factory = Arc::clone(&self.factory); - let preflight_request = request.clone(); - if let Err(message) = self.run_on_adapter_runtime(async move { - factory.preflight_restore(&preflight_request).await - }) { - return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - message, - )); - } + self.completed_restore_failures + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&worker_ref); let pending = Arc::new(PendingRuntimeRestore { request: request.clone(), task: tokio::sync::Mutex::new(None), @@ -4335,7 +4864,23 @@ where let factory = Arc::clone(&self.factory); let result_owner = Arc::clone(&pending); let task = self.spawn_on_adapter_runtime(async move { - let result = factory.restore_controller(request).await; + let result = match factory.restore_controller(request.clone()).await { + Ok(controller) => Ok(controller), + Err(mut failure) => { + if failure.resources == RuntimeRestoreResources::None + && !request.workdir_attachments.is_empty() + { + failure.resources = RuntimeRestoreResources::WorkdirBinding; + failure.cleanup = RuntimeRestoreCleanup::Required; + } + *result_owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure.clone())); + Err(finish_restore_failure(factory.as_ref(), &request, failure).await) + } + }; let mut retained = result_owner .result .lock() @@ -4377,6 +4922,21 @@ where workdir_attachments: Vec::new(), }; } + if existing_pending { + if let Err(message) = self + .retry_pending_restore_cleanup(&pending) + .and_then(|()| self.wait_pending_restore(&pending, self.spawn_restore_timeout)) + { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + message, + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + } let controller = { let mut retained = match pending.result.lock() { Ok(retained) => retained, @@ -4396,12 +4956,39 @@ where Ok(controller) => controller, Err(_) => unreachable!(), }, - Some(Err(message)) => { + Some(Err(failure)) => { + let result = WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + failure.to_string(), + ); + let cleanup = failure.cleanup; + if matches!( + cleanup, + RuntimeRestoreCleanup::NotNeeded | RuntimeRestoreCleanup::Confirmed + ) { + self.completed_restore_failures + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(worker_ref.clone(), (operation_id, failure.clone())); + drop(retained); + if let Err(message) = self.remove_pending_restore(&worker_ref) { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + message, + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + return if cleanup == RuntimeRestoreCleanup::NotNeeded { + WorkerExecutionSpawnResult::Rejected(result) + } else { + WorkerExecutionSpawnResult::RolledBack(result) + }; + } return WorkerExecutionSpawnResult::ReconciliationRequired { - result: WorkerExecutionResult::errored( - WorkerExecutionOperation::Restore, - message.clone(), - ), + result, worker_state: None, workdir_attachments: Vec::new(), }; @@ -6508,7 +7095,7 @@ mod tests { } Err(error) => error, }; - assert!(error.contains(expected), "{error}"); + assert_eq!(error.stage, RuntimeRestoreStage::Preflight); assert_eq!(persisted_files(&runtime_store), before); assert!( factory @@ -6647,10 +7234,7 @@ mod tests { } Err(error) => error, }; - assert!( - error.contains("subjektiv settings snapshot mismatch"), - "{error}" - ); + assert_eq!(error.stage, RuntimeRestoreStage::Preflight); assert_eq!(persisted_files(&runtime_store), before); assert!(factory.observation_hub.workers.lock().unwrap().is_empty()); let run_dir = factory @@ -7421,7 +8005,7 @@ mod tests { async fn restore_controller( &self, _request: WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result { panic!("create timeout must not restore a Worker") } } @@ -7584,7 +8168,7 @@ mod tests { async fn restore_controller( &self, request: WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result { let request = WorkerExecutionSpawnRequest { worker_ref: request.worker_ref, request: request.request, @@ -7593,7 +8177,12 @@ mod tests { workdir_attachments: request.workdir_attachments, config_bundle: request.config_bundle, }; - self.spawn_controller(request).await + self.spawn_controller(request).await.map_err(|_| { + RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "mock controller startup failed", + ) + }) } } @@ -7901,8 +8490,11 @@ mod tests { async fn restore_controller( &self, _request: WorkerExecutionRestoreRequest, - ) -> Result { - Err("restore failed".to_string()) + ) -> Result { + Err(RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Reconstruction, + "restore failed", + )) } } @@ -7926,7 +8518,7 @@ mod tests { async fn restore_controller( &self, request: WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result { *self.restore_id.lock().unwrap() = Some(request.operation_id); let runs = self .runtime_store @@ -7940,14 +8532,17 @@ mod tests { b"incomplete restore artifact", ) .unwrap(); - Err("injected restore failure with persisted artifacts".to_string()) + Err(RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "injected restore failure with persisted artifacts", + )) } async fn cleanup_failed_restore( &self, _: &WorkerExecutionRestoreRequest, - ) -> Result<(), String> { - Err("injected restore artifact cleanup failure".to_string()) + ) -> Result<(), RuntimeRestoreCleanupFailure> { + Err(RuntimeRestoreCleanupFailure::Artifacts) } } @@ -8054,7 +8649,7 @@ mod tests { assert!( backend .factory - .failed_restore_sessions + .failed_restore_resources .lock() .unwrap() .is_empty() @@ -8107,16 +8702,18 @@ mod tests { fs::create_dir_all(run.parent().unwrap()).unwrap(); fs::write(&run, "invalid restore artifact").unwrap(); let sessions = Arc::new(WorkdirSessionRouter::new()); - factory.failed_restore_sessions.lock().unwrap().insert( + factory.failed_restore_resources.lock().unwrap().insert( worker_ref.clone(), - (request.operation_id, Arc::clone(&sessions)), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: vec![Arc::clone(&sessions)], + allocation: None, + feature_tasks: None, + }, ); assert!( - factory - .cleanup_failed_restore(&request) - .await - .unwrap_err() - .contains("unsafe restore artifact") + factory.cleanup_failed_restore(&request).await.unwrap_err() + == RuntimeRestoreCleanupFailure::Artifacts ); assert!( factory @@ -8126,20 +8723,20 @@ mod tests { .contains("unsafe restore artifact") ); let retained = factory - .failed_restore_sessions + .failed_restore_resources .lock() .unwrap() .get(&worker_ref) .cloned() .unwrap(); - assert_eq!(retained.0, request.operation_id); - assert!(Arc::ptr_eq(&retained.1, &sessions)); + assert_eq!(retained.operation_id, request.operation_id); + assert!(Arc::ptr_eq(&retained.sessions[0], &sessions)); fs::remove_file(&run).unwrap(); fs::create_dir(&run).unwrap(); fs::write(run.join("worker.out.log"), "stale restore artifact").unwrap(); factory.reconcile_stopped_worker(&worker_ref).await.unwrap(); assert!(!run.exists()); - assert!(factory.failed_restore_sessions.lock().unwrap().is_empty()); + assert!(factory.failed_restore_resources.lock().unwrap().is_empty()); } #[cfg(unix)] @@ -8206,6 +8803,9 @@ mod tests { ); } + #[cfg(feature = "fs-store")] + include!("worker_backend_restore_failure_tests.rs"); + struct PendingFailureFactory { restore_calls: Arc, cleanup_calls: Arc, @@ -8223,20 +8823,23 @@ mod tests { async fn restore_controller( &self, _request: WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result { self.restore_calls.fetch_add(1, Ordering::SeqCst); tokio::time::sleep(Duration::from_millis(50)).await; - Err("injected uncertain restore side effect".to_string()) + Err(RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Reconstruction, + "injected uncertain restore side effect", + )) } async fn cleanup_failed_restore( &self, _request: &WorkerExecutionRestoreRequest, - ) -> Result<(), String> { + ) -> Result<(), RuntimeRestoreCleanupFailure> { self.cleanup_calls.fetch_add(1, Ordering::SeqCst); if self.cleanup_allowed.load(Ordering::SeqCst) { Ok(()) } else { - Err("injected pending restore cleanup failure".to_string()) + Err(RuntimeRestoreCleanupFailure::Workdir) } } } @@ -8307,12 +8910,7 @@ mod tests { failed.outcome, crate::execution::WorkerExecutionOutcome::Errored ); - assert!( - failed - .message - .unwrap() - .contains("pending restore cleanup failure") - ); + assert!(failed.message.unwrap().contains("Workdir")); assert_eq!( backend .pending_restore(&worker_ref) @@ -8330,7 +8928,7 @@ mod tests { "cleanup then catalog-commit retry must converge" ); assert_eq!(restore_calls.load(Ordering::SeqCst), 1); - assert_eq!(cleanup_calls.load(Ordering::SeqCst), 2); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 3); } struct LateControllerFactory { @@ -8351,7 +8949,7 @@ mod tests { async fn restore_controller( &self, request: WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result { let first = self.restores.fetch_add(1, Ordering::SeqCst) == 0; if first { self.release.notified().await; @@ -8831,7 +9429,7 @@ mod tests { Ok(_) => panic!("legacy Workspace Worker restore unexpectedly succeeded"), Err(error) => error, }; - assert!(error.contains("replacement Worker is required"), "{error}"); + assert_eq!(error.stage, RuntimeRestoreStage::Preflight); } #[test] diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs new file mode 100644 index 000000000..af39d3619 --- /dev/null +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -0,0 +1,945 @@ +// Integration coverage through EmbeddedRuntime -> real Profile factory -> real +// Worker/bootstrap, local Git Workdir provider, and allocation table. Model I/O +// is not invoked; this is a deterministic ordinary suite, not product E2E. + +struct ProfileFailureFixture { + runtime: EmbeddedRuntime, + backend: Arc, + worker: crate::catalog::WorkerDetail, + _repo: tempfile::TempDir, + root: tempfile::TempDir, +} + +impl ProfileFailureFixture { + fn new(timeout: Duration) -> Self { + let root = tempfile::tempdir().unwrap(); + let repo = create_clean_repo(); + let store = root.path().join("private-restore-test/runtime"); + let backend = Arc::new( + WorkerRuntimeExecutionBackend::new( + ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store), + ) + .unwrap() + .with_spawn_restore_timeout(timeout) + .with_working_directory_materializer(RuntimeGitMaterializer::new( + root.path().join("workdirs"), + )), + ); + let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( + crate::fs_store::FsRuntimeStoreOptions { + root: store, + runtime_id: "restore-failure-test".into(), + display_name: None, + }, + backend.clone(), + ) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let mut request = create_request("restore failure recovery"); + request.profile = ProfileSelector::Builtin("default".into()); + request.workdir_attachment_requests = vec![WorkingDirectoryAttachmentRequest { + alias: WorkdirAttachmentAlias::new("checkout").unwrap(), + working_directory: working_directory_request(repo.path()), + }]; + let worker = runtime.create_worker(request).unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + Self { + root, + _repo: repo, + runtime, + backend, + worker, + } + } + + fn fail(&self, stage: RuntimeRestoreStage) { + *self.backend.factory.restore_faults.fail_at.lock().unwrap() = Some(stage); + } + + fn assert_released(&self) { + assert!(self.backend.pending_restores.lock().unwrap().is_empty()); + assert!(self.backend.workers.lock().unwrap().is_empty()); + assert!( + self.backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty() + ); + assert!( + self.backend + .factory + .observation_hub + .workers + .lock() + .unwrap() + .is_empty() + ); + let table = worker::runtime::worker_allocation::LockFileGuard::open( + &worker::runtime::worker_allocation::default_allocation_path().unwrap(), + ) + .unwrap(); + assert!( + table + .data() + .find(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref( + &self.worker.worker_ref + )) + .is_none() + ); + let aggregate = self + .backend + .factory + .worker_aggregate_dir(&self.worker.worker_ref) + .unwrap(); + assert!(fs::read_dir(aggregate.join("runs")).unwrap().all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("restore-") + })); + assert_eq!( + self.runtime + .worker_detail(&self.worker.worker_ref) + .unwrap() + .status, + crate::catalog::WorkerStatus::Stopped + ); + } + + fn fresh_restore_and_old_replay( + &self, + request: runtime_api::WorkerRestoreRequest, + failed: crate::runtime::RuntimeWorkerRestoreResult, + ) { + let calls = self + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst); + let replay = self + .runtime + .restore_worker_operation(&self.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + serde_json::to_value(&replay).unwrap(), + serde_json::to_value(&failed).unwrap() + ); + assert_eq!( + self.backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + calls + ); + let fresh = self.runtime.test_restore_request(&self.worker.worker_ref); + assert_ne!(fresh.request_id, request.request_id); + assert_ne!( + fresh.expected_observation_token, + request.expected_observation_token + ); + let restored = self + .runtime + .restore_worker_operation(&self.worker.worker_ref, fresh) + .unwrap(); + assert_eq!( + restored.state, + server_api::WorkerRestoreState::Accepted, + "{restored:?}" + ); + assert_eq!( + self.backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + calls + 1 + ); + assert_eq!(self.backend.workers.lock().unwrap().len(), 1); + let replay = self + .runtime + .restore_worker_operation(&self.worker.worker_ref, request) + .unwrap(); + assert_eq!( + serde_json::to_value(&replay).unwrap(), + serde_json::to_value(&failed).unwrap() + ); + assert_eq!( + self.backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + calls + 1 + ); + // Only tear down after the fresh explicit Restore has already succeeded. + self.runtime + .stop_worker(&self.worker.worker_ref, None) + .unwrap(); + } +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() { + for stage in [ + RuntimeRestoreStage::Reconstruction, + RuntimeRestoreStage::Binding, + RuntimeRestoreStage::Feature, + RuntimeRestoreStage::Controller, + ] { + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + fixture.fail(stage); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{stage:?}: {result:?}" + ); + let diagnostic = serde_json::to_string(&result).unwrap(); + assert!( + diagnostic.contains("injected restore boundary failure"), + "{diagnostic}" + ); + assert!(diagnostic.contains(&format!("{stage:?}")), "{diagnostic}"); + assert!(!diagnostic.contains(&fixture.root.path().to_string_lossy().to_string())); + assert!(diagnostic.len() < 1024); + fixture.assert_released(); + eprintln!( + "restore-fault stage={stage:?} outcome=RolledBack pending=0 controllers=0 allocations=0" + ); + fixture.fresh_restore_and_old_replay(request, result); + } +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_cleanup_failure_retains_owner_and_original_cause_until_same_operation_recovers() +{ + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + fixture.fail(RuntimeRestoreStage::Feature); + fixture + .backend + .factory + .restore_faults + .cleanup_fail + .store(true, Ordering::SeqCst); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let failed = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + failed.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + let message = pending + .result + .lock() + .unwrap() + .as_ref() + .unwrap() + .as_ref() + .err() + .unwrap() + .to_string(); + assert!( + message.contains("injected restore boundary failure"), + "{message}" + ); + assert!(message.contains("Workdir"), "{message}"); + let owned = fixture + .backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .get(&fixture.worker.worker_ref) + .cloned() + .unwrap(); + assert_eq!(owned.operation_id, pending.request.operation_id); + assert!(owned.allocation.is_some()); + assert!( + owned + .sessions + .iter() + .any(|sessions| !sessions.aliases().is_empty()) + ); + let repeat = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + repeat.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + assert!(Arc::ptr_eq( + &pending, + &fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap() + )); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + fixture + .backend + .factory + .restore_faults + .cleanup_fail + .store(false, Ordering::SeqCst); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{result:?}" + ); + assert!( + serde_json::to_string(&result) + .unwrap() + .contains("injected restore boundary failure") + ); + assert!( + owned + .sessions + .iter() + .all(|sessions| sessions.aliases().is_empty()) + ); + fixture.assert_released(); + eprintln!( + "restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=RolledBack" + ); + fixture.fresh_restore_and_old_replay(request, result); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconciles_without_execution() + { + let fixture = ProfileFailureFixture::new(Duration::from_secs(2)); + fixture.fail(RuntimeRestoreStage::Feature); + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .cleanup_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .cleanup_entered + .lock() + .unwrap() = Some(entered_tx); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + entered.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + assert!( + pending + .result + .lock() + .unwrap() + .as_ref() + .unwrap() + .as_ref() + .err() + .unwrap() + .to_string() + .contains("injected restore boundary failure") + ); + assert!(pending.task.try_lock().unwrap().is_some()); + for _ in 0..2 { + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + assert!(Arc::ptr_eq( + &pending, + &fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap() + )); + } + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + release.notify_one(); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{result:?}" + ); + fixture.assert_released(); + eprintln!( + "restore-fault cleanup=timeout retained-task=true late-completion=joined executions=1" + ); + fixture.fresh_restore_and_old_replay(request, result); +} + +#[cfg(unix)] +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation() { + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + fixture + .backend + .factory + .restore_faults + .socket_failure + .store(true, Ordering::SeqCst); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{result:?}" + ); + let diagnostic = serde_json::to_string(&result).unwrap(); + assert!( + diagnostic.contains("controller startup failed"), + "{diagnostic}" + ); + assert!(diagnostic.contains("InvalidInput"), "{diagnostic}"); + assert!(!diagnostic.contains("private-restore-test")); + fixture.assert_released(); + fixture + .backend + .factory + .restore_faults + .socket_failure + .store(false, Ordering::SeqCst); + eprintln!( + "restore-fault stage=socket-start real-bootstrap=true partial-run=removed allocation=released" + ); + fixture.fresh_restore_and_old_replay(request, result); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_read_only_refusal_replays_and_fresh_observation_can_restore() { + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + let aggregate = fixture + .backend + .factory + .worker_aggregate_dir(&fixture.worker.worker_ref) + .unwrap(); + fs::rename(aggregate.join("session"), aggregate.join("saved-session")).unwrap(); + let before = persisted_files(&aggregate); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::Rejected, + "{result:?}" + ); + let mut after = persisted_files(&aggregate); + let mut before = before; + // The Runtime rejection receipt is expected to commit; source authority is not. + before.remove(Path::new("worker.json")); + after.remove(Path::new("worker.json")); + assert_eq!(after, before); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 0 + ); + fs::rename(aggregate.join("saved-session"), aggregate.join("session")).unwrap(); + // Creation's stopped observation entry is weak; querying it evicts it. + assert!( + fixture + .backend + .factory + .observation_hub + .get(&fixture.worker.worker_ref) + .is_none() + ); + fixture.assert_released(); + fixture.fresh_restore_and_old_replay(request, result); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_real_artifact_cleanup_failure_retains_same_owner_and_cause() { + let fixture = ProfileFailureFixture::new(Duration::from_secs(2)); + fixture.fail(RuntimeRestoreStage::Binding); + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .factory_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .factory_entered + .lock() + .unwrap() = Some(entered_tx); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + entered.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + let run = fixture + .backend + .factory + .worker_restore_run_dir(&fixture.worker.worker_ref, pending.request.operation_id) + .unwrap(); + fs::write(&run, "invalid disposable run artifact").unwrap(); + release.notify_one(); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{result:?}" + ); + let cause = pending + .result + .lock() + .unwrap() + .as_ref() + .unwrap() + .as_ref() + .err() + .unwrap() + .clone(); + assert_eq!(cause.stage, RuntimeRestoreStage::Binding); + assert_eq!( + cause.cleanup_failure, + Some(RuntimeRestoreCleanupFailure::Artifacts) + ); + assert!( + fixture + .backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .contains_key(&fixture.worker.worker_ref) + ); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + fs::remove_file(&run).unwrap(); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{result:?}" + ); + assert!(result.message.as_ref().unwrap().contains("Artifacts")); + fixture.assert_released(); + eprintln!( + "restore-fault cleanup=real-artifact-failure late-factory=true owner=retained original-cause=preserved" + ); + fixture.fresh_restore_and_old_replay(request, result); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_stop_race_joins_late_factory_failure_without_second_controller() { + let fixture = ProfileFailureFixture::new(Duration::from_secs(2)); + fixture.fail(RuntimeRestoreStage::Binding); + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .factory_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .factory_entered + .lock() + .unwrap() = Some(entered_tx); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request) + .unwrap(); + entered.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + assert!(pending.result.lock().unwrap().is_none()); + let runtime = fixture.runtime.clone(); + let worker_ref = fixture.worker.worker_ref.clone(); + let (started_tx, started) = mpsc::channel(); + let stopping = std::thread::spawn(move || { + started_tx.send(()).unwrap(); + runtime.stop_worker(&worker_ref, None) + }); + started.recv_timeout(Duration::from_secs(5)).unwrap(); + // The factory still owns the allocation and cannot attest stopped yet. + assert!( + !pending + .task + .try_lock() + .map(|task| task.as_ref().is_some_and(|task| task.is_finished())) + .unwrap_or(false) + ); + release.notify_one(); + stopping.join().unwrap().unwrap(); + fixture.assert_released(); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + let fresh = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, fresh) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::Accepted, + "{result:?}" + ); + assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); + fixture + .runtime + .stop_worker(&fixture.worker.worker_ref, None) + .unwrap(); + eprintln!("restore-fault race=Stop late-factory=joined controllers-at-most=1"); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_failure_settlement_write_retry_preserves_completed_cause_without_execution() { + for fault in [ + crate::fs_store::AtomicWriteFault::BeforeRename, + crate::fs_store::AtomicWriteFault::AfterRename, + ] { + let fixture = ProfileFailureFixture::new(Duration::from_secs(2)); + fixture.fail(RuntimeRestoreStage::Binding); + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .factory_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .factory_entered + .lock() + .unwrap() = Some(entered_tx); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + entered.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let operation_id = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap() + .request + .operation_id; + fixture + .runtime + .test_runtime_store() + .fail_next_worker_write(fault); + release.notify_one(); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{result:?}" + ); + assert_eq!( + result.reason_code.as_deref(), + Some("worker_restore_rollback_commit_failed") + ); + assert!( + fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .is_none() + ); + assert!( + fixture + .backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty() + ); + let saved = fixture + .backend + .completed_restore_failure(operation_id, &fixture.worker.worker_ref) + .unwrap(); + assert!( + matches!(saved, WorkerExecutionSpawnResult::RolledBack(ref failure) + if failure.message.as_ref().unwrap().contains("injected restore boundary failure")) + ); + let result = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{result:?}" + ); + assert!( + result + .message + .as_ref() + .unwrap() + .contains("injected restore boundary failure") + ); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + fixture.assert_released(); + eprintln!( + "restore-fault settlement={fault:?} completed-cause=retained replay-executions=1" + ); + fixture.fresh_restore_and_old_replay(request, result); + } +} + +struct RacedBindingFactory { + changed_checkout: PathBuf, + preflights: AtomicUsize, +} + +#[async_trait] +impl RuntimeWorkerFactory for RacedBindingFactory { + async fn spawn_controller( + &self, + _: WorkerExecutionSpawnRequest, + ) -> Result { + panic!("binding-only failure must not create a controller") + } + async fn preflight_restore(&self, _: &WorkerExecutionRestoreRequest) -> Result<(), String> { + // Inject another owner's checkout change AFTER the real materializer's + // read-only checks, but before its mutable bind. Recovery stays real. + if self.preflights.fetch_add(1, Ordering::SeqCst) == 0 { + fs::remove_dir_all(&self.changed_checkout).unwrap(); + } + Ok(()) + } + async fn restore_controller( + &self, + _: WorkerExecutionRestoreRequest, + ) -> Result { + panic!("factory execution must not run after partial binding failed") + } +} + +#[test] +fn partial_restore_binding_failure_keeps_exact_operation_without_factory_cleanup_claim() { + let root = tempfile::tempdir().unwrap(); + let repo = create_clean_repo(); + let materializer = RuntimeGitMaterializer::new(root.path()); + let first = materializer + .create(&working_directory_request(repo.path())) + .unwrap(); + let second = materializer + .create(&working_directory_request(repo.path())) + .unwrap(); + let mut request = direct_restore_request(WorkerRef::new(WorkerId::now_v7())); + request.previous_workdir_attachments = vec![ + WorkingDirectoryAttachmentStatus { + alias: WorkdirAttachmentAlias::new("first").unwrap(), + working_directory: first.status(), + }, + WorkingDirectoryAttachmentStatus { + alias: WorkdirAttachmentAlias::new("second").unwrap(), + working_directory: second.status(), + }, + ]; + let backend = WorkerRuntimeExecutionBackend::new(RacedBindingFactory { + changed_checkout: second.root().to_path_buf(), + preflights: AtomicUsize::new(0), + }) + .unwrap() + .with_working_directory_materializer(materializer); + assert!(matches!( + backend.restore_worker(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + let pending = backend + .pending_restore(&request.worker_ref) + .unwrap() + .unwrap(); + assert_eq!(pending.request.operation_id, request.operation_id); + assert_eq!(pending.request.workdir_attachments.len(), 1); + assert_eq!( + pending + .request + .workdir_attachments + .values() + .next() + .unwrap() + .root(), + first.root() + ); + for _ in 0..2 { + assert!(matches!( + backend.reconcile_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + } + assert_eq!(backend.factory.preflights.load(Ordering::SeqCst), 1); + assert!(!backend.stop_worker(&request.worker_ref).is_accepted()); + assert!(Arc::ptr_eq( + &pending, + &backend + .pending_restore(&request.worker_ref) + .unwrap() + .unwrap() + )); + assert!(backend.workers.lock().unwrap().is_empty()); + eprintln!( + "restore-fault stage=mutable-binding outcome=ReconciliationRequired partial-bindings=retained factory-executions=0" + ); + // Fixture teardown drops the retained passive local binding. This is not + // evidence that credential/policy materialization has been rolled back. +} diff --git a/crates/worker/src/bootstrap.rs b/crates/worker/src/bootstrap.rs index d027aa8c9..ba8afef39 100644 --- a/crates/worker/src/bootstrap.rs +++ b/crates/worker/src/bootstrap.rs @@ -271,6 +271,7 @@ where St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { let cleanup_sessions = worker.workdir_sessions(); + let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some(); let controller = match layout { WorkerBootstrapLayout::Direct { runtime_base, @@ -305,7 +306,10 @@ where controller_task, }), Err(source) => { - let cleanup_failed = cleanup_sessions.close_all().await.is_err(); + // Factory-owned task scopes must settle before sessions close. + // Cleanup remains unconfirmed here; preserve the startup source. + let cleanup_failed = + factory_owned_cleanup || cleanup_sessions.close_all().await.is_err(); Err(WorkerBootstrapError::Controller { source, cleanup_failed, diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index d8be6da1a..8c4640bf9 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -854,6 +854,7 @@ impl WorkerController { St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { let sessions = worker.workdir_sessions(); + let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some(); let result = Self::spawn_initialized( worker, runtime_base, @@ -863,7 +864,10 @@ impl WorkerController { transport, ) .await; + // An opted-in factory must quiesce its captured Feature tasks before + // closing their Workdir sessions; this wrapper cannot confirm that order. if result.is_err() + && !factory_owned_cleanup && let Err(error) = sessions.close_all().await { tracing::warn!(%error, "Workdir sessions close after controller startup failure failed"); @@ -1092,6 +1096,7 @@ impl WorkerController { let pause_tx = worker.engine_mut().pause_sender(); let notify_buffer = worker.notify_buffer_handle(); + worker.finish_startup_feature_cleanup_capture(); let controller_task = tokio::spawn(controller_loop( worker, method_rx, @@ -4424,21 +4429,6 @@ mod tests { assert!(feature_install < materialize_gate); assert!(materialize_gate < materialize); assert!(materialize < exposure); - let observer = startup.find("let command_observer = fs_for_view").unwrap(); - let recover = startup.find(".recover_unfinished_compaction()").unwrap(); - let files = startup - .find("runtime_dir.write_status(&shared_state).await?") - .unwrap(); - let loop_start = startup.find("tokio::spawn(controller_loop(").unwrap(); - assert!(recover < observer && files < observer && observer < loop_start); - let socket_startup = &startup[observer..loop_start]; - let abort = socket_startup.find("observer.abort()").unwrap(); - let join = socket_startup.find("observer.await").unwrap(); - let failure = socket_startup.find("return Err(error)").unwrap(); - assert!( - abort < join && join < failure, - "startup cannot detach its observer on error" - ); assert!( startup[materialize_gate..materialize] .contains("needs_initial_session_head_materialization") diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index 10cf5a6c5..54145dbf9 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -69,9 +69,10 @@ pub use session_history::{ pub use shared_state::WorkerSharedState; pub use worker::{ LocalWorkingDirectory, SubjektivSessionAttributionLifecycle, Worker, WorkerError, - WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, WorkspaceBinaryRequest, - WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError, - WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse, - WorkspaceServerOperation, WorkspaceWorkerDiscoveryRequest, apply_worker_manifest, - marker_workspace_client, unavailable_workspace_client, + WorkerFilesystemAuthority, WorkerRunResult, WorkerStartupFeatureCleanup, + WorkerWorkspaceContext, WorkspaceBinaryRequest, WorkspaceBinaryResponse, WorkspaceClient, + WorkspaceClientError, WorkspaceId, WorkspaceIdError, WorkspacePromptCatalogResolution, + WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse, WorkspaceServerOperation, + WorkspaceWorkerDiscoveryRequest, apply_worker_manifest, marker_workspace_client, + unavailable_workspace_client, }; diff --git a/crates/worker/src/runtime/worker_allocation.rs b/crates/worker/src/runtime/worker_allocation.rs index c7445cdaa..ab6bc3455 100644 --- a/crates/worker/src/runtime/worker_allocation.rs +++ b/crates/worker/src/runtime/worker_allocation.rs @@ -21,8 +21,9 @@ pub use error::ScopeLockError; #[cfg(test)] pub(crate) use lifecycle::install_top_level_at_for_test; pub use lifecycle::{ - ScopeAllocationGuard, SegmentActivationCommitError, SegmentActivationGuard, SegmentLockInfo, - adopt_allocation, install_top_level, install_top_level_with_deny, lookup_segment, + ScopeAllocationCleanup, ScopeAllocationGuard, SegmentActivationCommitError, + SegmentActivationGuard, SegmentLockInfo, adopt_allocation, install_top_level, + install_top_level_with_deny, lookup_segment, }; pub use mutate::{ delegate_scope, reclaim_delegated_scope, reclaim_stale, reclaim_stale_with, register_worker, diff --git a/crates/worker/src/runtime/worker_allocation/error.rs b/crates/worker/src/runtime/worker_allocation/error.rs index 601202693..334e0125f 100644 --- a/crates/worker/src/runtime/worker_allocation/error.rs +++ b/crates/worker/src/runtime/worker_allocation/error.rs @@ -11,6 +11,14 @@ use session_store::SegmentId; pub enum ScopeLockError { #[error("I/O error on workers.json: {0}")] Io(#[from] io::Error), + #[error("worker `{worker_name}` allocation still has a live guard")] + AllocationGuardAttached { worker_name: String }, + #[error("allocation acquisition was published but its save failed: {source}")] + AcquisitionSaveFailed { + #[source] + source: io::Error, + cleanup: super::lifecycle::ScopeAllocationCleanup, + }, #[error("worker `{0}` is already allocated")] DuplicateWorkerName(String), #[error("requested scope `{}` conflicts with worker allocation `{competitor}` rule `{}`", .rule.target.display(), .competitor_rule.target.display())] @@ -46,3 +54,15 @@ pub enum ScopeLockError { actual: Option, }, } + +impl ScopeLockError { + /// Retain the release authority for an owned acquisition whose table rename + /// succeeded before save returned an error. Refusals and unpublished saves + /// do not confer ownership of any preexisting allocation. + pub fn allocation_cleanup_handle(&self) -> Option { + match self { + Self::AcquisitionSaveFailed { cleanup, .. } => Some(cleanup.clone()), + _ => None, + } + } +} diff --git a/crates/worker/src/runtime/worker_allocation/lifecycle.rs b/crates/worker/src/runtime/worker_allocation/lifecycle.rs index cab09ec18..ba6505008 100644 --- a/crates/worker/src/runtime/worker_allocation/lifecycle.rs +++ b/crates/worker/src/runtime/worker_allocation/lifecycle.rs @@ -3,6 +3,7 @@ //! up on drop. use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; use manifest::ScopeRule; use session_store::SegmentId; @@ -11,13 +12,82 @@ use super::error::ScopeLockError; use super::mutate::release_worker; use super::table::{LockFileGuard, default_allocation_path}; -/// Owned allocation: on drop, opens the lock file and releases this -/// Worker's entry. The guard keeps only the name + lock-file path; it -/// does not hold the `flock` for the Worker's lifetime. +/// Owned allocation shared with any retained cleanup handles. Only the final +/// owner's drop attempts best-effort release; no `flock` is held for its lifetime. #[derive(Debug)] pub struct ScopeAllocationGuard { + owner: Arc, +} + +/// Retains an allocation's release authority even after its Worker is consumed +/// or dropped. A successful controller transfer should drop this handle without +/// explicitly releasing the live Worker's allocation. +#[derive(Debug, Clone)] +pub struct ScopeAllocationCleanup { + owner: Arc, +} + +#[derive(Debug)] +struct ScopeAllocationOwner { worker_name: String, lock_path: PathBuf, + state: Mutex, +} + +#[derive(Debug)] +struct ScopeAllocationOwnerState { + guard_attached: bool, + released: bool, +} + +impl ScopeAllocationOwner { + fn release(&self) -> Result<(), ScopeLockError> { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + if state.guard_attached { + return Err(ScopeLockError::AllocationGuardAttached { + worker_name: self.worker_name.clone(), + }); + } + if state.released { + return Ok(()); + } + let mut guard = LockFileGuard::open(&self.lock_path)?; + if guard.data().find(&self.worker_name).is_some() { + release_worker(&mut guard, &self.worker_name)?; + } else { + // A previous save may have removed the entry but failed after rename. + // Complete the durable save before reporting release as confirmed. + guard.save()?; + } + state.released = true; + Ok(()) + } +} + +impl Drop for ScopeAllocationOwner { + fn drop(&mut self) { + let _ = self.release(); + } +} + +impl Drop for ScopeAllocationGuard { + fn drop(&mut self) { + self.owner + .state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .guard_attached = false; + } +} + +impl ScopeAllocationCleanup { + /// Release only after the factory has returned an error and no live + /// Controller can still use this allocation. An attached guard rejects + /// release. Success is idempotent across all clones; failure retains the + /// authority for an explicit retry. + pub fn release(&self) -> Result<(), ScopeLockError> { + self.owner.release() + } } /// Exclusive authority to activate a replacement Segment for one allocated Worker. @@ -56,12 +126,58 @@ impl SegmentActivationCommitError { } impl ScopeAllocationGuard { + fn new(worker_name: String, lock_path: PathBuf) -> Self { + Self { + owner: Arc::new(ScopeAllocationOwner { + worker_name, + lock_path, + state: Mutex::new(ScopeAllocationOwnerState { + guard_attached: true, + released: false, + }), + }), + } + } + + fn commit_acquisition(self, table: &mut LockFileGuard) -> Result { + let mut published = false; + match table.save_with_publication(&mut published) { + Ok(()) => Ok(self), + Err(source) => { + let cleanup = if published { + Some(self.cleanup_handle()) + } else { + // A staged entry is not ours until rename. Disarm before + // detaching so Drop cannot remove a preexisting allocation + // (or reopen the table lock still held by this caller). + self.owner + .state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .released = true; + None + }; + drop(self); + match cleanup { + Some(cleanup) => Err(ScopeLockError::AcquisitionSaveFailed { source, cleanup }), + None => Err(ScopeLockError::Io(source)), + } + } + } + } + pub fn worker_name(&self) -> &str { - &self.worker_name + &self.owner.worker_name } pub fn lock_path(&self) -> &Path { - &self.lock_path + &self.owner.lock_path + } + + pub fn cleanup_handle(&self) -> ScopeAllocationCleanup { + ScopeAllocationCleanup { + owner: Arc::clone(&self.owner), + } } /// Lock the machine-wide table and validate a Segment replacement before @@ -71,24 +187,25 @@ impl ScopeAllocationGuard { expected: SegmentId, replacement: SegmentId, ) -> Result { - let guard = LockFileGuard::open(&self.lock_path)?; + let guard = LockFileGuard::open(self.lock_path())?; let actual = guard .data() .allocations .iter() - .find(|allocation| allocation.worker_name == self.worker_name) + .find(|allocation| allocation.worker_name == self.worker_name()) .and_then(|allocation| allocation.segment_id); if actual != Some(expected) { return Err(ScopeLockError::SegmentChanged { - worker_name: self.worker_name.clone(), + worker_name: self.worker_name().to_owned(), expected, actual, }); } if let Some(existing) = guard.data().allocations.iter().find(|allocation| { - allocation.segment_id == Some(replacement) && allocation.worker_name != self.worker_name + allocation.segment_id == Some(replacement) + && allocation.worker_name != self.worker_name() }) { return Err(ScopeLockError::SegmentConflict { segment_id: replacement, @@ -99,7 +216,7 @@ impl ScopeAllocationGuard { Ok(SegmentActivationGuard { guard, - worker_name: self.worker_name.clone(), + worker_name: self.worker_name().to_owned(), expected, replacement, }) @@ -145,14 +262,6 @@ impl SegmentActivationGuard { } } -impl Drop for ScopeAllocationGuard { - fn drop(&mut self) { - if let Ok(mut guard) = LockFileGuard::open(&self.lock_path) { - let _ = release_worker(&mut guard, &self.worker_name); - } - } -} - /// Open the default lock file, register a top-level Worker, and return a /// guard that will release the allocation on drop. pub fn install_top_level( @@ -185,7 +294,7 @@ pub fn install_top_level_with_deny( ) -> Result { let lock_path = default_allocation_path()?; let mut guard = LockFileGuard::open(&lock_path)?; - super::mutate::register_worker_with_deny( + super::mutate::prepare_worker_with_deny( &mut guard, worker_name.clone(), pid, @@ -194,10 +303,7 @@ pub fn install_top_level_with_deny( scope_deny, segment_id, )?; - Ok(ScopeAllocationGuard { - worker_name, - lock_path, - }) + ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard) } #[cfg(test)] @@ -210,7 +316,7 @@ pub(crate) fn install_top_level_at_for_test( segment_id: SegmentId, ) -> Result { let mut guard = LockFileGuard::open(&lock_path)?; - super::mutate::register_worker_with_deny( + super::mutate::prepare_worker_with_deny( &mut guard, worker_name.clone(), pid, @@ -219,10 +325,7 @@ pub(crate) fn install_top_level_at_for_test( Vec::new(), segment_id, )?; - Ok(ScopeAllocationGuard { - worker_name, - lock_path, - }) + ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard) } /// Take ownership of an existing allocation that was pre-registered by @@ -240,6 +343,15 @@ pub fn adopt_allocation( segment_id: SegmentId, ) -> Result { let lock_path = default_allocation_path()?; + adopt_allocation_at(lock_path, worker_name, new_pid, segment_id) +} + +fn adopt_allocation_at( + lock_path: PathBuf, + worker_name: String, + new_pid: u32, + segment_id: SegmentId, +) -> Result { let mut guard = LockFileGuard::open(&lock_path)?; let alloc = guard .data_mut() @@ -247,11 +359,7 @@ pub fn adopt_allocation( .ok_or_else(|| ScopeLockError::UnknownWorker(worker_name.clone()))?; alloc.pid = new_pid; alloc.segment_id = Some(segment_id); - guard.save()?; - Ok(ScopeAllocationGuard { - worker_name, - lock_path, - }) + ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard) } /// Information about a Worker that currently holds an allocation for a @@ -329,6 +437,388 @@ mod tests { } } + #[test] + fn retained_cleanup_handle_defers_release_until_final_owner_drop() { + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let guard = install_top_level_at_for_test( + lock_path.clone(), + "retained".into(), + std::process::id(), + sock("retained"), + Vec::new(), + sid(), + ) + .unwrap(); + let cleanup = guard.cleanup_handle(); + let retained = cleanup.clone(); + + drop(guard); + drop(cleanup); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("retained").is_some()); + } + drop(retained); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("retained").is_none()); + } + + #[test] + fn dropping_cleanup_handle_after_transfer_preserves_live_guard() { + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let segment = sid(); + let guard = install_top_level_at_for_test( + lock_path.clone(), + "transferred".into(), + std::process::id(), + sock("transferred"), + Vec::new(), + segment, + ) + .unwrap(); + + drop(guard.cleanup_handle()); + // A successful controller transfer leaves the guard usable, including + // the existing activation protocol, without retaining a cleanup handle. + let replacement = sid(); + guard + .begin_segment_activation(segment, replacement) + .unwrap() + .commit() + .unwrap(); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert_eq!( + table.data().find("transferred").unwrap().segment_id, + Some(replacement) + ); + } + drop(guard); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("transferred").is_none()); + } + + #[test] + fn cleanup_rejects_live_guard_then_releases_only_detached_allocation() { + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let guard = install_top_level_at_for_test( + lock_path.clone(), + "live".into(), + std::process::id(), + sock("live"), + Vec::new(), + sid(), + ) + .unwrap(); + let _competitor = install_top_level_at_for_test( + lock_path.clone(), + "competitor".into(), + std::process::id(), + sock("competitor"), + Vec::new(), + sid(), + ) + .unwrap(); + let cleanup = guard.cleanup_handle(); + assert!(matches!( + cleanup.release(), + Err(ScopeLockError::AllocationGuardAttached { worker_name }) if worker_name == "live" + )); + drop(cleanup.clone()); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("live").is_some()); + assert!(table.data().find("competitor").is_some()); + } + + drop(guard); + cleanup.release().unwrap(); + cleanup.clone().release().unwrap(); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("live").is_none()); + assert!(table.data().find("competitor").is_some()); + } + + #[test] + fn published_install_failure_retains_cleanup_across_error_drop_and_retry() { + use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test}; + + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let _competitor = install_top_level_at_for_test( + lock_path.clone(), + "competitor".into(), + std::process::id(), + sock("competitor"), + Vec::new(), + sid(), + ) + .unwrap(); + let segment = sid(); + fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename); + let error = install_top_level_at_for_test( + lock_path.clone(), + "failed".into(), + std::process::id(), + sock("failed"), + Vec::new(), + segment, + ) + .unwrap_err(); + assert!(matches!( + &error, + ScopeLockError::AcquisitionSaveFailed { .. } + )); + let cleanup = error.allocation_cleanup_handle().unwrap(); + let retry = error.allocation_cleanup_handle().unwrap(); + drop(error); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert_eq!( + table.data().find("failed").unwrap().segment_id, + Some(segment) + ); + assert!(table.data().find("competitor").is_some()); + } + + fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename); + assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_)))); + drop(cleanup); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("failed").is_some()); + } + retry.release().unwrap(); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("failed").is_none()); + assert!(table.data().find("competitor").is_some()); + } + + #[test] + fn refused_install_has_no_cleanup_authority_over_existing_allocation() { + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let segment = sid(); + let _competitor = install_top_level_at_for_test( + lock_path.clone(), + "competitor".into(), + std::process::id(), + sock("competitor"), + vec![write_rule("/shared", true)], + segment, + ) + .unwrap(); + let cases = [ + ("competitor", sid(), Vec::new()), + ("other", segment, Vec::new()), + ("other", sid(), vec![write_rule("/shared", true)]), + ]; + for (name, requested_segment, scope) in cases { + let error = install_top_level_at_for_test( + lock_path.clone(), + name.into(), + std::process::id(), + sock("refused"), + scope, + requested_segment, + ) + .unwrap_err(); + assert!(matches!( + &error, + ScopeLockError::DuplicateWorkerName(_) + | ScopeLockError::SegmentConflict { .. } + | ScopeLockError::WriteConflict { .. } + )); + assert!(error.allocation_cleanup_handle().is_none()); + drop(error); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert_eq!(table.data().allocations.len(), 1); + assert_eq!( + table.data().find("competitor").unwrap().socket, + sock("competitor") + ); + assert_eq!( + table.data().find("competitor").unwrap().segment_id, + Some(segment) + ); + } + } + + #[test] + fn unpublished_install_failure_has_no_cleanup_authority() { + use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test}; + + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let _competitor = install_top_level_at_for_test( + lock_path.clone(), + "competitor".into(), + std::process::id(), + sock("competitor"), + Vec::new(), + sid(), + ) + .unwrap(); + fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename); + let error = install_top_level_at_for_test( + lock_path.clone(), + "failed".into(), + std::process::id(), + sock("failed"), + Vec::new(), + sid(), + ) + .unwrap_err(); + assert!(matches!(&error, ScopeLockError::Io(_))); + assert!(error.allocation_cleanup_handle().is_none()); + drop(error); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("failed").is_none()); + assert!(table.data().find("competitor").is_some()); + } + + #[test] + fn published_adoption_failure_retains_cleanup_for_retry() { + use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test}; + + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + { + let mut table = LockFileGuard::open(&lock_path).unwrap(); + delegate_placeholder(&mut table, "child", std::process::id()); + delegate_placeholder(&mut table, "competitor", std::process::id()); + } + let segment = sid(); + fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename); + let error = adopt_allocation_at( + lock_path.clone(), + "child".into(), + std::process::id(), + segment, + ) + .unwrap_err(); + let cleanup = error.allocation_cleanup_handle().unwrap(); + drop(error); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert_eq!( + table.data().find("child").unwrap().segment_id, + Some(segment) + ); + } + fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename); + assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_)))); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("child").is_none()); + assert!(table.data().find("competitor").is_some()); + } + cleanup.release().unwrap(); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("child").is_none()); + assert!(table.data().find("competitor").is_some()); + } + + #[test] + fn unpublished_adoption_failure_preserves_placeholder_without_cleanup_authority() { + use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test}; + + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let placeholder_pid = std::process::id(); + { + let mut table = LockFileGuard::open(&lock_path).unwrap(); + delegate_placeholder(&mut table, "child", placeholder_pid); + } + fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename); + let error = adopt_allocation_at( + lock_path.clone(), + "child".into(), + placeholder_pid.wrapping_add(1), + sid(), + ) + .unwrap_err(); + assert!(matches!(&error, ScopeLockError::Io(_))); + assert!(error.allocation_cleanup_handle().is_none()); + drop(error); + let table = LockFileGuard::open(&lock_path).unwrap(); + let allocation = table.data().find("child").unwrap(); + assert_eq!(allocation.pid, placeholder_pid); + assert_eq!(allocation.segment_id, None); + } + + #[test] + fn failed_cleanup_release_retains_authority_for_retry() { + use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test}; + + for point in [SaveFaultPoint::BeforeRename, SaveFaultPoint::AfterRename] { + let dir = TempDir::new().unwrap(); + let lock_path = dir.path().join("workers.json"); + let guard = install_top_level_at_for_test( + lock_path.clone(), + "failed".into(), + std::process::id(), + sock("failed"), + vec![read_rule("/shared", true)], + sid(), + ) + .unwrap(); + let _other = install_top_level_at_for_test( + lock_path.clone(), + "other".into(), + std::process::id(), + sock("other"), + vec![read_rule("/shared", true)], + sid(), + ) + .unwrap(); + let cleanup = guard.cleanup_handle(); + let retry = cleanup.clone(); + drop(guard); + fail_next_save_at_for_test(&lock_path, point); + + assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_)))); + drop(cleanup); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert_eq!( + table.data().find("failed").is_some(), + point == SaveFaultPoint::BeforeRename + ); + assert!(table.data().find("other").is_some()); + } + retry.release().unwrap(); + { + let table = LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("failed").is_none()); + assert!(table.data().find("other").is_some()); + } + + // Success disarms every clone and final Drop, even if the name is + // subsequently allocated to a different Worker. + let _replacement = install_top_level_at_for_test( + lock_path.clone(), + "failed".into(), + std::process::id(), + sock("replacement"), + vec![read_rule("/shared", true)], + sid(), + ) + .unwrap(); + retry.clone().release().unwrap(); + drop(retry); + let table = LockFileGuard::open(&lock_path).unwrap(); + assert_eq!( + table.data().find("failed").unwrap().socket, + sock("replacement") + ); + assert!(table.data().find("other").is_some()); + } + } + #[test] fn adopt_allocation_rewrites_pid_and_releases_on_drop() { let dir = TempDir::new().unwrap(); @@ -358,6 +848,7 @@ mod tests { let dir = TempDir::new().unwrap(); let _sandbox = RuntimeDirSandbox::new(dir.path()); let err = adopt_allocation("ghost".into(), 42, sid()).unwrap_err(); + assert!(err.allocation_cleanup_handle().is_none()); assert!(matches!(err, ScopeLockError::UnknownWorker(ref n) if n == "ghost")); } diff --git a/crates/worker/src/runtime/worker_allocation/mutate.rs b/crates/worker/src/runtime/worker_allocation/mutate.rs index 846adca98..9ba22f057 100644 --- a/crates/worker/src/runtime/worker_allocation/mutate.rs +++ b/crates/worker/src/runtime/worker_allocation/mutate.rs @@ -57,6 +57,30 @@ pub fn register_worker_with_deny( scope_allow: Vec, scope_deny: Vec, segment_id: SegmentId, +) -> Result<(), ScopeLockError> { + prepare_worker_with_deny( + guard, + worker_name, + pid, + socket, + scope_allow, + scope_deny, + segment_id, + )?; + guard.save()?; + Ok(()) +} + +/// Admit and stage a registration without publishing it. Owned installation +/// constructs its cleanup authority after admission and before committing. +pub(super) fn prepare_worker_with_deny( + guard: &mut LockFileGuard, + worker_name: String, + pid: u32, + socket: PathBuf, + scope_allow: Vec, + scope_deny: Vec, + segment_id: SegmentId, ) -> Result<(), ScopeLockError> { reclaim_stale(guard); if guard.data().find(&worker_name).is_some() { @@ -101,7 +125,6 @@ pub fn register_worker_with_deny( delegated_from: None, segment_id: Some(segment_id), }); - guard.save()?; Ok(()) } diff --git a/crates/worker/src/runtime/worker_allocation/table.rs b/crates/worker/src/runtime/worker_allocation/table.rs index b9d0a16ae..a8420551d 100644 --- a/crates/worker/src/runtime/worker_allocation/table.rs +++ b/crates/worker/src/runtime/worker_allocation/table.rs @@ -245,6 +245,14 @@ impl LockFileGuard { /// Persist with atomic replacement while the separate allocation lock stays /// held. A crash exposes either the old complete table or the new one. pub fn save(&mut self) -> io::Result<()> { + let mut published = false; + self.save_with_publication(&mut published) + } + + /// Report this save's rename while the caller still holds the table lock. + /// In-memory mutation alone is not evidence that an acquisition took place. + pub(super) fn save_with_publication(&mut self, published: &mut bool) -> io::Result<()> { + *published = false; let json = serde_json::to_vec_pretty(&self.data).map_err(io::Error::other)?; let mut temp_os = self.data_path.as_os_str().to_owned(); temp_os.push(".tmp"); @@ -264,6 +272,7 @@ impl LockFileGuard { )); } fs::rename(&temp_path, &self.data_path)?; + *published = true; #[cfg(test)] if take_save_fault(&self.data_path, SaveFaultPoint::AfterRename) { return Err(io::Error::other( diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 6af8d337b..92a7e7047 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -533,7 +533,7 @@ use crate::prompt::source::PromptCatalogSource; use crate::prompt::system::{SystemPromptContext, SystemPromptError, SystemPromptTemplate}; use crate::runtime::dir; use crate::runtime::worker_allocation::{ - self, ScopeAllocationGuard, ScopeLockError, SegmentActivationGuard, + self, ScopeAllocationCleanup, ScopeAllocationGuard, ScopeLockError, SegmentActivationGuard, }; use crate::skill::{SkillActivationResponse, SkillClientError}; #[cfg(test)] @@ -2951,6 +2951,69 @@ struct PendingCompactionCleanup { compaction_id: String, } +/// Retains startup task scopes after a consuming controller startup fails. +/// +/// Call `shutdown` only after startup has failed with no live Controller. Keep +/// this handle on errors or timeout and retry; it owns any in-flight settlement +/// future, including join handles drained from the registries. On successful +/// controller transfer, drop the factory's handle without shutting it down. +#[derive(Clone)] +pub struct WorkerStartupFeatureCleanup { + inner: Arc, +} + +struct StartupFeatureCleanupInner { + registries: Mutex>, + settlement: tokio::sync::Mutex< + Option>>, + >, +} + +impl WorkerStartupFeatureCleanup { + fn new(registry: FeatureBackgroundTaskRegistry) -> Self { + Self { + inner: Arc::new(StartupFeatureCleanupInner { + registries: Mutex::new(vec![registry]), + settlement: tokio::sync::Mutex::new(None), + }), + } + } + + fn retain(&self, registry: FeatureBackgroundTaskRegistry) { + self.inner + .registries + .lock() + .expect("startup Feature cleanup poisoned") + .push(registry); + } + + /// Stops and joins every captured registry. Cancelling this call preserves + /// its settlement future for the next call, rather than dropping task joins. + pub async fn shutdown(&self) -> Result<(), crate::hook::HookError> { + let mut settlement = self.inner.settlement.lock().await; + let pending = settlement.get_or_insert_with(|| { + let registries = self + .inner + .registries + .lock() + .expect("startup Feature cleanup poisoned") + .clone(); + Box::pin(async move { + let mut failure = None; + for registry in registries { + if let Err(error) = registry.shutdown().await { + failure.get_or_insert(error); + } + } + failure.map_or(Ok(()), Err) + }) + }); + let result = pending.as_mut().await; + *settlement = None; + result + } +} + /// An independent agent execution unit. /// /// Holds a [`Engine`] directly and persists session state via @@ -2997,6 +3060,8 @@ pub struct Worker { hook_registry: Option>, /// Executable background tasks registered by successfully installed features. feature_background_tasks: FeatureBackgroundTaskRegistry, + /// Present only while an opted-in factory is capturing startup task scopes. + startup_feature_cleanup: Option, /// Enabled chat invocation metadata and executable handlers. feature_invocations: FeatureInvocationRegistry, /// Internal Workers install an explicit Feature composition and disable @@ -3087,9 +3152,8 @@ pub struct Worker { /// Workers built via `from_manifest` / `from_manifest_spawned` / /// `restore_from_manifest` (production paths); `None` for the /// low-level `Worker::new` constructor used in tests, which bypasses - /// the registry. Kept purely for its `Drop` impl, which releases - /// the allocation when the Worker is dropped. - #[allow(dead_code)] + /// the registry. Cleanup handles can retain the allocation beyond Worker + /// drop; only the final shared owner attempts best-effort release. scope_allocation: Option, /// Socket path of the spawning Worker. `Some` only for Workers built via /// `from_manifest_spawned`. Consumed by the controller to fire @@ -3155,7 +3219,7 @@ pub struct Worker { impl Drop for Worker { fn drop(&mut self) { // A failed activation authority nests the same machine-wide lock used by - // `ScopeAllocationGuard::drop`. Release the nested guard first so Worker + // allocation owner's final Drop. Release the nested guard first so Worker // teardown cannot self-deadlock while removing its outer allocation. let authority = match self.failed_activation_authority.get_mut() { Ok(authority) => authority.take(), @@ -3165,6 +3229,47 @@ impl Drop for Worker { } } +impl Worker { + /// Opt in before a consuming startup call. The handle captures the current + /// task registry and each subsequently installed registry until successful + /// controller transfer ends capture. Repeated calls share the same capture. + pub fn retain_startup_feature_cleanup(&mut self) -> WorkerStartupFeatureCleanup { + self.startup_feature_cleanup + .get_or_insert_with(|| { + WorkerStartupFeatureCleanup::new(self.feature_background_tasks.clone()) + }) + .clone() + } + + /// Observes explicit factory cleanup ownership without enabling capture. + pub fn startup_feature_cleanup_handle(&self) -> Option { + self.startup_feature_cleanup.clone() + } + + /// End factory capture immediately before transferring to a live Controller. + /// The factory must also drop its handle after startup returns success. + pub fn finish_startup_feature_cleanup_capture(&mut self) { + self.startup_feature_cleanup = None; + } + + /// Retain cleanup authority before passing the reconstructed Worker into a + /// consuming factory. Explicit release is only valid after factory failure + /// with no live Controller; drop the handle on successful controller transfer. + pub fn restore_allocation_cleanup_handle(&self) -> Option { + self.scope_allocation + .as_ref() + .map(ScopeAllocationGuard::cleanup_handle) + } + + fn construction_failure(&self, error: WorkerError) -> WorkerError { + WorkerError::ConstructionCleanup { + source: Box::new(error), + allocation: self.restore_allocation_cleanup_handle(), + sessions: self.workdir_sessions.clone(), + } + } +} + impl Worker { pub async fn stop_feature_runtime(&mut self, reason: impl Into) { if let Some(hooks) = self.hook_registry.clone() { @@ -3352,6 +3457,7 @@ impl Worker { hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + startup_feature_cleanup: None, feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: true, interceptor_installed: false, @@ -3850,6 +3956,9 @@ impl Worker { return report; } self.feature_background_tasks = report.background_tasks.clone(); + if let Some(cleanup) = &self.startup_feature_cleanup { + cleanup.retain(self.feature_background_tasks.clone()); + } self.feature_invocations = report.chat_invocations.clone(); for instruction in report.installed_instruction_contributions() { self.register_feature_instruction(instruction); @@ -7906,7 +8015,7 @@ where let workdir_sessions = workdir_sessions_from_authority(&common.filesystem_authority, &scope); - let mut worker = Self { + let worker = Self { backend_job: std::sync::OnceLock::new(), manifest, engine: Some(worker), @@ -7925,6 +8034,7 @@ where hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + startup_feature_cleanup: None, feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: true, interceptor_installed: false, @@ -7966,10 +8076,15 @@ where log_writer: None, notification_committer: None, }; - worker.apply_permissions_from_manifest(); - worker.apply_prune_from_manifest(); - worker.write_worker_metadata_pending()?; - Ok(worker) + worker.finish_pending_construction() + } + + fn finish_pending_construction(mut self) -> Result { + self.apply_permissions_from_manifest(); + self.apply_prune_from_manifest(); + self.write_worker_metadata_pending() + .map_err(|error| self.construction_failure(error))?; + Ok(self) } /// Complete the Host-owned Session-to-Subject attribution boundary before a @@ -8306,6 +8421,7 @@ where hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + startup_feature_cleanup: None, feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: false, interceptor_installed: false, @@ -8438,6 +8554,7 @@ where hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + startup_feature_cleanup: None, feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: true, interceptor_installed: false, @@ -8751,6 +8868,11 @@ where if state.system_prompt.is_none() { return Err(WorkerError::SegmentSystemPromptMissing { segment_id }); } + let restored_history_entries = + restore_history_entries(session_id, segment_id, &raw_entries).map_err(|error| { + WorkerError::InvalidState(format!("restore typed Worker session history: {error}")) + })?; + let flow_runtime_state = restored_flow_runtime_state(&state.extensions)?; let mirror_entries: Vec = raw_entries.clone(); let scope_config = effective_restore_scope_config(&store, &manifest)?; @@ -8797,10 +8919,6 @@ where // (the Worker's one and only write path that prepends a summary at // history[0]). Restoring the anchor lets Anthropic re-use a // stable cache prefix for long-lived restored sessions. - let restored_history_entries = - restore_history_entries(session_id, segment_id, &raw_entries).map_err(|error| { - WorkerError::InvalidState(format!("restore typed Worker session history: {error}")) - })?; let restored_history = restored_history_entries .iter() .map(|entry| entry.item.clone()) @@ -8839,14 +8957,13 @@ where filesystem_authority: common.filesystem_authority, workdir_sessions, workspace_context: common.workspace_context, - flow_runtime_state: Arc::new(Mutex::new(restored_flow_runtime_state( - &state.extensions, - )?)), + flow_runtime_state: Arc::new(Mutex::new(flow_runtime_state)), flow_feature_enabled: false, scope, hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + startup_feature_cleanup: None, feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: true, interceptor_installed: false, @@ -8896,17 +9013,24 @@ where worker .session .restore_pending_activations(&state.extensions); - worker - .pending_submission_handle() - .reconcile_uploaded_file_pins()?; - worker.apply_permissions_from_manifest(); - worker.apply_prune_from_manifest(); - worker.write_worker_metadata_active(SegmentLocation { - session_id, - segment_id, - })?; - worker.reconcile_restored_delegations().await?; - Ok(worker) + worker.finish_restored_construction().await + } + + async fn finish_restored_construction(mut self) -> Result { + self.pending_submission_handle() + .reconcile_uploaded_file_pins() + .map_err(|error| self.construction_failure(error.into()))?; + self.apply_permissions_from_manifest(); + self.apply_prune_from_manifest(); + self.write_worker_metadata_active(SegmentLocation { + session_id: self.session_id(), + segment_id: self.segment_id(), + }) + .map_err(|error| self.construction_failure(error))?; + self.reconcile_restored_delegations() + .await + .map_err(|error| self.construction_failure(error))?; + Ok(self) } async fn reconcile_restored_delegations(&mut self) -> Result<(), WorkerError> { @@ -9745,6 +9869,16 @@ fn compact_failure_category(error: &WorkerError) -> CompactFailureCategory { #[derive(Debug, thiserror::Error)] pub enum WorkerError { + /// A constructor failed after acquiring resources. Retain these handles until + /// explicit cleanup succeeds; the Worker has dropped before this is returned. + #[error("{source}")] + ConstructionCleanup { + #[source] + source: Box, + allocation: Option, + sessions: Arc, + }, + #[error("invalid durable Worker state: {0}")] InvalidState(String), @@ -12224,6 +12358,604 @@ mod build_summary_prompt_tests { assert_eq!(restored.active_run_turn_count, None); } + #[tokio::test] + async fn restore_cleanup_handle_retains_allocation_after_worker_drop() { + let dir = tempfile::tempdir().unwrap(); + let store = session_store::FsStore::new(dir.path().join("sessions")).unwrap(); + let mut worker = Worker::new( + minimal_manifest(), + Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient), + store, + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + Scope::empty(), + ) + .await + .unwrap(); + assert!(worker.restore_allocation_cleanup_handle().is_none()); + let lock_path = dir.path().join("workers.json"); + worker.scope_allocation = Some( + worker_allocation::install_top_level_at_for_test( + lock_path.clone(), + "test-worker".into(), + std::process::id(), + dir.path().join("worker.sock"), + Vec::new(), + worker.segment_id(), + ) + .unwrap(), + ); + let cleanup = worker.restore_allocation_cleanup_handle().unwrap(); + + drop(worker); + { + let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("test-worker").is_some()); + } + cleanup.release().unwrap(); + let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find("test-worker").is_none()); + } + + // Constructor finalization uses real Worker, allocation, Workdir, and storage + // implementations. Only the model and metadata-write failure boundary are substituted. + mod construction_cleanup_tests { + use super::*; + + type ConstructionStore = + session_store::CombinedStore; + + async fn constructed_worker( + temp: &tempfile::TempDir, + ) -> Worker, ConstructionStore> { + let store = session_store::CombinedStore::new( + session_store::FsStore::new(temp.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(temp.path().join("workers")).unwrap(), + ); + let mut worker = Worker::new( + minimal_manifest(), + Engine::, Mutable, SessionHistoryMetadata>::new_annotated( + Box::new(NoopClient), + ), + store, + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::local( + temp.path().to_path_buf(), + temp.path().to_path_buf(), + ), + Scope::writable(temp.path()).unwrap(), + ) + .await + .unwrap(); + worker.scope_allocation = Some( + worker_allocation::install_top_level_at_for_test( + temp.path().join("workers.json"), + worker.manifest.worker.name.clone(), + std::process::id(), + temp.path().join("worker.sock"), + Vec::new(), + worker.segment_id(), + ) + .unwrap(), + ); + worker + } + + #[tokio::test] + async fn post_construction_failures_retain_resources_through_failed_release_retry() { + for stage in [ + "pending_metadata", + "restore_pins", + "restore_metadata", + "restore_delegations", + ] { + let temp = tempfile::tempdir().unwrap(); + let mut worker = constructed_worker(&temp).await; + if stage != "pending_metadata" { + worker.ensure_segment_head().await.unwrap(); + } + let name = worker.manifest.worker.name.clone(); + let lock_path = temp.path().join("workers.json"); + let sessions = Arc::downgrade(&worker.workdir_sessions()); + let session = worker.workdir_sessions().resolve(None).unwrap().session; + let writes = Arc::new(AtomicUsize::new(0)); + let writer_calls = writes.clone(); + let metadata_path = temp + .path() + .join("workers") + .join(&name) + .join("metadata.json"); + worker.worker_metadata_writer = Some(Arc::new(move |_| { + writer_calls.fetch_add(1, Ordering::SeqCst); + if stage == "restore_delegations" { + // A successful publication followed by an unreadable retained + // record forces the subsequent delegation reconciliation to fail. + std::fs::create_dir_all(&metadata_path)?; + Ok(()) + } else { + Err(WorkerStoreError::Io(std::io::Error::other( + "metadata write refused", + ))) + } + })); + if stage == "restore_pins" { + let pins_dir = temp + .path() + .join("sessions") + .join(worker.session_id().to_string()) + .join("artifacts") + .join("paste"); + std::fs::create_dir_all(pins_dir.join(".aggregate.lock")).unwrap(); + } + let result = if stage == "pending_metadata" { + worker.finish_pending_construction() + } else { + worker.finish_restored_construction().await + }; + let error = match result { + Err(error) => error, + Ok(_) => panic!("{stage} must fail after Worker construction"), + }; + let WorkerError::ConstructionCleanup { + source, + allocation, + sessions: retained, + } = &error + else { + panic!("{stage} lost construction cleanup authority: {error:?}"); + }; + if stage == "restore_pins" { + assert!(matches!( + source.as_ref(), + WorkerError::Store(StoreError::Io(_)) + )); + assert_eq!(writes.load(Ordering::SeqCst), 0); + } else { + assert!(matches!( + source.as_ref(), + WorkerError::WorkerStore(WorkerStoreError::Io(_)) + )); + assert_eq!(writes.load(Ordering::SeqCst), 1); + } + assert_eq!(error.to_string(), source.to_string()); + assert_eq!( + std::error::Error::source(&error).unwrap().to_string(), + source.to_string() + ); + assert!(Arc::ptr_eq(&sessions.upgrade().unwrap(), retained)); + let allocation = allocation.clone().expect("constructed Worker's allocation"); + let retained = retained.clone(); + drop(error); + { + let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap(); + assert!( + table.data().find(&name).is_some(), + "{stage} released during Worker Drop" + ); + } + assert!( + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root() + }) + .await + .is_ok() + ); + retained.close_all().await.unwrap(); + assert!(retained.is_empty()); + assert!( + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root() + }) + .await + .is_err() + ); + + worker_allocation::fail_next_save_for_test(&lock_path); + assert!(matches!(allocation.release(), Err(ScopeLockError::Io(_)))); + { + let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap(); + assert!( + table.data().find(&name).is_some(), + "{stage} lost failed release" + ); + } + allocation.release().unwrap(); + allocation.clone().release().unwrap(); + let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap(); + assert!(table.data().find(&name).is_none()); + } + } + + #[tokio::test] + async fn malformed_restore_state_is_rejected_before_filesystem_preparation() { + for corrupt_flow in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let store = session_store::CombinedStore::new( + session_store::FsStore::new(temp.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(temp.path().join("workers")).unwrap(), + ); + let session_id = session_store::new_session_id(); + let segment_id = session_store::new_segment_id(); + let mut entries = vec![LogEntry::AnnotatedSegmentStart { + ts: 0, + session_id, + system_prompt: Some("saved prompt".into()), + config: RequestConfig::default(), + history: Vec::new(), + forked_from: None, + compacted_from: None, + }]; + if corrupt_flow { + entries.push(LogEntry::Extension { + ts: 1, + domain: FLOW_RUNTIME_EXTENSION_DOMAIN.into(), + payload: serde_json::Value::Null, + }); + } else { + entries.push(LogEntry::ToolResultCorrected { + ts: 1, + entry: to_logged_history_entry(&history_entry( + Item::user_message("correction without a target"), + WorkerHistoryProvenance::LegacyUnknown, + )), + }); + } + store + .create_segment(session_id, segment_id, &entries) + .unwrap(); + let result = Worker::, _>::restore_from_manifest_with_context_and_model_client( + session_id, + segment_id, + minimal_manifest(), + store, + PromptCatalogSource::builtins_only(), + WorkerWorkspaceContext::no_workspace(), + // If preparation runs first, it must reject this missing root, + // without touching the process-global allocation table. + WorkerFilesystemAuthority::local(temp.path().join("missing"), temp.path().join("missing")), + Some(Box::new(NoopClient)), + ).await; + let error = match result { + Err(error) => error, + Ok(_) => panic!("corrupt restore state must be rejected"), + }; + let WorkerError::InvalidState(message) = error else { + panic!("restore parsed state too late: {error:?}"); + }; + assert!(message.contains(if corrupt_flow { + "Flow runtime state" + } else { + "typed Worker session history" + })); + } + } + } + + // Real Feature installation, task registries, Workdir provider/router, and + // PreparedWorker startup; only the model and task completion barrier are test implementations. + mod startup_feature_cleanup_tests { + use super::*; + use crate::feature::background::{ + BackgroundTaskCancellation, BackgroundTaskContext, BackgroundTaskSpec, + FeatureBackgroundTask, + }; + use crate::feature::{ + BackgroundTaskDeclaration, FeatureDescriptor, FeatureId, FeatureInstallContext, + FeatureInstallError, FeatureModule, + }; + use crate::hook::{HookError, HookErrorCategory}; + use tokio::sync::Notify; + + type StartupStore = + session_store::CombinedStore; + + #[derive(Default)] + struct TaskSignals { + entered: Notify, + cancelled: AtomicBool, + release: Notify, + exited: AtomicBool, + session: Mutex>, + } + + #[derive(Clone)] + struct BarrierTask { + signals: Arc, + _lifetime: Arc<()>, + panic_on_release: bool, + } + + #[async_trait] + impl FeatureBackgroundTask for BarrierTask { + async fn run( + &self, + _context: BackgroundTaskContext, + cancellation: BackgroundTaskCancellation, + ) -> Result<(), HookError> { + self.signals.entered.notify_one(); + cancellation.cancelled().await; + self.signals.cancelled.store(true, Ordering::Release); + self.signals.release.notified().await; + assert!(!self.panic_on_release, "injected startup task join failure"); + let session = self.signals.session.lock().unwrap().clone(); + if let Some(session) = session { + assert!( + session_open(&session).await, + "Feature task must retain Workdir access until it is quiescent" + ); + } + self.signals.exited.store(true, Ordering::Release); + Ok(()) + } + } + + struct BarrierFeature(BarrierTask); + + impl FeatureModule for BarrierFeature { + fn descriptor(&self) -> FeatureDescriptor { + FeatureDescriptor::builtin("startup-barrier", "Startup barrier") + .with_background_task(BackgroundTaskDeclaration::worker_managed( + "barrier", + "Startup task completion barrier", + )) + } + + fn install( + &self, + context: &mut FeatureInstallContext<'_>, + ) -> Result<(), FeatureInstallError> { + context.background_tasks().register( + BackgroundTaskSpec::single_flight( + BackgroundTaskDeclaration::worker_managed( + "barrier", + "Startup task completion barrier", + ), + Duration::from_secs(60), + ), + self.0.clone(), + ) + } + } + + async fn worker(temp: &tempfile::TempDir) -> Worker { + let store = session_store::CombinedStore::new( + session_store::FsStore::new(temp.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(temp.path().join("workers")).unwrap(), + ); + Worker::new( + minimal_manifest(), + Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient), + store, + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + Scope::empty(), + ) + .await + .unwrap() + } + + fn install( + worker: &mut Worker, + panic_on_release: bool, + ) -> (Arc, std::sync::Weak<()>) { + let signals = Arc::new(TaskSignals::default()); + let lifetime = Arc::new(()); + let weak = Arc::downgrade(&lifetime); + let report = worker.install_features(FeatureRegistryBuilder::new().with_module( + BarrierFeature(BarrierTask { + signals: signals.clone(), + _lifetime: lifetime, + panic_on_release, + }), + )); + assert!(!report.has_errors()); + (signals, weak) + } + + async fn start(worker: &Worker, signals: &TaskSignals) { + worker + .feature_background_tasks + .start( + &FeatureId::builtin("startup-barrier"), + "barrier", + HookInvocationContext { + workspace_id: None, + worker_id: "startup-owner".into(), + session_id: worker.session_id().to_string(), + run_id: None, + turn_index: None, + call_id: None, + }, + ) + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), signals.entered.notified()) + .await + .unwrap(); + } + + fn bind_session( + worker: &mut Worker, + temp: &tempfile::TempDir, + ) -> workdir::WorkdirSessionHandle { + let session: workdir::WorkdirSessionHandle = + Arc::new(workdir::LocalWorkdirSession::materialized_bound( + workdir::Workdir::new("startup-task-cleanup"), + temp.path().to_path_buf(), + temp.path().to_path_buf(), + SharedScope::new(Scope::writable(temp.path()).unwrap()), + workdir::WorkdirSessionCapabilities::ALL, + )); + worker.bind_single_workdir_session(Some(session.clone())); + session + } + + async fn session_open(session: &workdir::WorkdirSessionHandle) -> bool { + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root(), + }) + .await + .is_ok() + } + + async fn fail_consuming_start( + worker: Worker, + temp: &tempfile::TempDir, + ) { + let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some(); + let blocked_parent = temp.path().join("blocked-runtime"); + std::fs::write(&blocked_parent, "not a directory").unwrap(); + let result = crate::PreparedWorker::new( + worker, + crate::WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: blocked_parent.join("1"), + bash_output_dir: temp.path().join("spill"), + }, + crate::WorkerControllerTransport::InProcess, + ) + .start() + .await; + let Err(crate::WorkerBootstrapError::Controller { + source, + cleanup_failed, + }) = result + else { + panic!("runtime parent file must reject controller startup"); + }; + assert!(matches!( + source.kind(), + std::io::ErrorKind::NotADirectory | std::io::ErrorKind::AlreadyExists + )); + assert_eq!(cleanup_failed, factory_owned_cleanup); + } + + #[tokio::test] + async fn consumed_worker_failure_retains_all_task_joins_across_cleanup_timeout() { + let temp = tempfile::tempdir().unwrap(); + let mut worker = worker(&temp).await; + let session = bind_session(&mut worker, &temp); + let sessions = worker.workdir_sessions(); + let (first, first_lifetime) = install(&mut worker, false); + *first.session.lock().unwrap() = Some(session.clone()); + start(&worker, &first).await; + let cleanup = worker.retain_startup_feature_cleanup(); + let retry_owner = cleanup.clone(); + let (second, second_lifetime) = install(&mut worker, false); + *second.session.lock().unwrap() = Some(session.clone()); + start(&worker, &second).await; + fail_consuming_start(worker, &temp).await; + + assert!(!sessions.aliases().is_empty()); + assert!(session_open(&session).await); + assert!(first_lifetime.upgrade().is_some()); + assert!(second_lifetime.upgrade().is_some()); + assert!( + tokio::time::timeout(Duration::from_millis(50), cleanup.shutdown()) + .await + .is_err() + ); + assert!(first.cancelled.load(Ordering::Acquire)); + drop(cleanup); + + // A fresh shutdown of the drained registry would falsely succeed + // once the second task finishes, abandoning the first task's join. + second.release.notify_one(); + assert!( + tokio::time::timeout(Duration::from_millis(50), retry_owner.shutdown()) + .await + .is_err() + ); + assert!(!first.exited.load(Ordering::Acquire)); + assert!(session_open(&session).await); + first.release.notify_one(); + tokio::time::timeout(Duration::from_secs(5), retry_owner.shutdown()) + .await + .unwrap() + .unwrap(); + assert!(first.exited.load(Ordering::Acquire)); + assert!(second.exited.load(Ordering::Acquire)); + assert!(session_open(&session).await); + sessions.close_all().await.unwrap(); + assert!(sessions.aliases().is_empty()); + assert!(!session_open(&session).await); + drop(retry_owner); + assert!(first_lifetime.upgrade().is_none()); + assert!(second_lifetime.upgrade().is_none()); + } + + #[tokio::test] + async fn startup_without_factory_cleanup_owner_still_closes_sessions_on_error() { + let temp = tempfile::tempdir().unwrap(); + let mut worker = worker(&temp).await; + let session = bind_session(&mut worker, &temp); + let sessions = worker.workdir_sessions(); + assert!(session_open(&session).await); + fail_consuming_start(worker, &temp).await; + assert!(sessions.aliases().is_empty()); + assert!(!session_open(&session).await); + } + + #[tokio::test] + async fn cleanup_error_settles_other_registries_and_keeps_owner_for_retry() { + let temp = tempfile::tempdir().unwrap(); + let mut worker = worker(&temp).await; + let (first, first_lifetime) = install(&mut worker, true); + start(&worker, &first).await; + let cleanup = worker.retain_startup_feature_cleanup(); + let (second, second_lifetime) = install(&mut worker, false); + start(&worker, &second).await; + fail_consuming_start(worker, &temp).await; + first.release.notify_one(); + second.release.notify_one(); + + let error = tokio::time::timeout(Duration::from_secs(5), cleanup.shutdown()) + .await + .unwrap() + .unwrap_err(); + assert_eq!(error.category, HookErrorCategory::Internal); + assert!(second.exited.load(Ordering::Acquire)); + assert!(first_lifetime.upgrade().is_some()); + assert!(second_lifetime.upgrade().is_some()); + tokio::time::timeout(Duration::from_secs(5), cleanup.shutdown()) + .await + .unwrap() + .unwrap(); + drop(cleanup); + assert!(first_lifetime.upgrade().is_none()); + assert!(second_lifetime.upgrade().is_none()); + } + + #[tokio::test] + async fn successful_controller_transfer_releases_old_startup_task_scopes() { + let temp = tempfile::tempdir().unwrap(); + let mut worker = worker(&temp).await; + let (_signals, old_scope) = install(&mut worker, false); + let cleanup = worker.retain_startup_feature_cleanup(); + let started = crate::PreparedWorker::new( + worker, + crate::WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: temp.path().join("runs/1"), + bash_output_dir: temp.path().join("spill"), + }, + crate::WorkerControllerTransport::InProcess, + ) + .start() + .await + .unwrap(); + + assert!(old_scope.upgrade().is_some()); + drop(cleanup); + let released = old_scope.upgrade().is_none(); + stop_body_publication_controller(started).await; + assert!( + released, + "live Controller must not retain factory startup capture" + ); + } + } + #[tokio::test] async fn post_metadata_allocation_failure_fences_appends_and_restart_converges() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/worker/tests/controller_test.rs b/crates/worker/tests/controller_test.rs index 1938f0b18..e4b8d7041 100644 --- a/crates/worker/tests/controller_test.rs +++ b/crates/worker/tests/controller_test.rs @@ -759,6 +759,211 @@ async fn controller_startup_failure_closes_bound_workdir_session() { )); } +mod socket_startup_failure { + //! Connect the real controller startup, command observer, socket bind and session rollback. + //! Only the Workdir telemetry boundary and LLM are substituted. This ordinary controller + //! integration target uses an isolated directory at the socket path to reject the real bind. + use super::*; + use std::path::PathBuf; + use tokio::sync::broadcast; + use workdir::WorkdirSession; + + #[derive(Debug)] + struct ObservedSession { + local: LocalWorkdirSession, + socket_path: PathBuf, + events: broadcast::Sender, + subscriptions: AtomicUsize, + receivers_at_close: Mutex>, + } + + #[async_trait] + impl WorkdirSession for ObservedSession { + fn workdir(&self) -> &Workdir { + self.local.workdir() + } + + fn capabilities(&self) -> WorkdirSessionCapabilities { + self.local.capabilities() + } + + async fn stat( + &self, + request: workdir::StatRequest, + ) -> Result { + self.local.stat(request).await + } + + async fn read( + &self, + request: workdir::ReadRequest, + ) -> Result { + self.local.read(request).await + } + + async fn write( + &self, + request: workdir::WriteRequest, + ) -> Result { + self.local.write(request).await + } + + async fn edit( + &self, + request: workdir::EditRequest, + ) -> Result { + self.local.edit(request).await + } + + async fn list( + &self, + request: workdir::ListRequest, + ) -> Result { + self.local.list(request).await + } + + async fn glob( + &self, + request: workdir::GlobRequest, + ) -> Result { + self.local.glob(request).await + } + + async fn grep( + &self, + request: workdir::GrepRequest, + ) -> Result { + self.local.grep(request).await + } + + async fn start_command( + &self, + request: CommandRequest, + ) -> Result { + self.local.start_command(request).await + } + + async fn command_status( + &self, + handle: workdir::CommandHandle, + ) -> Result { + self.local.command_status(handle).await + } + + async fn command_output( + &self, + request: CommandOutputRequest, + ) -> Result { + self.local.command_output(request).await + } + + async fn cancel_command(&self, handle: workdir::CommandHandle) -> Result<(), WorkdirError> { + self.local.cancel_command(handle).await + } + + fn subscribe_command_events(&self) -> Option> { + // Runtime creation and initial file writes must already have succeeded. A directory + // survives SocketServer's stale-file removal and makes the subsequent Unix bind fail. + std::fs::create_dir(&self.socket_path).unwrap(); + self.subscriptions.fetch_add(1, Ordering::SeqCst); + Some(self.events.subscribe()) + } + + async fn close(&self) -> Result<(), WorkdirError> { + // Record before any await: aborting without joining must not pass by getting a + // scheduler turn during the provider's asynchronous close. The sender stays alive + // even after close, so channel closure cannot accidentally dispose of the observer. + self.receivers_at_close + .lock() + .unwrap() + .push(self.events.receiver_count()); + self.local.close().await + } + } + + #[tokio::test(flavor = "current_thread")] + async fn socket_bind_failure_joins_command_observer_before_closing_session() { + let root = tempfile::tempdir().unwrap(); + let run_dir = root.path().join("runs/1"); + let socket_path = run_dir.join("worker.sock"); + let client = MockClient::new(simple_text_events()); + let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap(); + let store = CombinedStore::new( + FsStore::new(root.path().join("sessions")).unwrap(), + FsWorkerStore::new(root.path().join("workers")).unwrap(), + ); + let engine = + Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated( + client.clone(), + ); + let mut worker = Worker::new( + manifest, + engine, + store, + WorkerWorkspaceContext::local_filesystem(None), + WorkerFilesystemAuthority::local(root.path().into(), root.path().into()), + manifest::Scope::writable(root.path()).unwrap(), + ) + .await + .unwrap(); + let (events, unused_receiver) = broadcast::channel(8); + drop(unused_receiver); + let session = Arc::new(ObservedSession { + local: LocalWorkdirSession::materialized_bound( + Workdir::new("socket-startup-failure"), + root.path().into(), + root.path().into(), + worker.scope().clone(), + WorkdirSessionCapabilities::ALL, + ), + socket_path: socket_path.clone(), + events, + subscriptions: AtomicUsize::new(0), + receivers_at_close: Mutex::new(Vec::new()), + }); + worker.bind_single_workdir_session(Some(session.clone())); + let sessions = worker.workdir_sessions(); + + let result = tokio::time::timeout( + std::time::Duration::from_secs(5), + WorkerController::spawn_runtime_managed_run( + worker, + &run_dir, + &root.path().join("bash-output"), + ), + ) + .await + .expect("socket startup and rollback must settle"); + let error = match result { + Err(error) => error, + Ok(_) => panic!("a directory at the Unix socket path must reject controller startup"), + }; + assert_eq!(error.kind(), std::io::ErrorKind::AddrInUse); + assert_eq!(session.subscriptions.load(Ordering::SeqCst), 1); + assert_eq!(*session.receivers_at_close.lock().unwrap(), vec![0]); + assert_eq!(session.events.receiver_count(), 0); + assert!(sessions.aliases().is_empty()); + // No controller retains the Worker/Engine after the failed startup. + assert_eq!(Arc::strong_count(&client.call_count), 1); + assert_eq!(client.call_count.load(Ordering::SeqCst), 0); + assert!(socket_path.is_dir()); + assert!(tokio::net::UnixStream::connect(&socket_path).await.is_err()); + assert!(matches!( + session + .start_command(CommandRequest { + command: "printf unreachable".to_owned(), + timeout_secs: 5, + output_limit: 1024, + cwd: workdir::WorkdirPath::root(), + spill_dir: None, + tool_call_id: None, + }) + .await, + Err(WorkdirError::Unavailable(_)) + )); + } +} + async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) { let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); loop { diff --git a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs index 0c5ca25dc..8e5c4d779 100644 --- a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs +++ b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs @@ -85,7 +85,10 @@ impl RuntimeWorkerFactory for CapturingFactory { async fn restore_controller( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, - ) -> std::result::Result { + ) -> std::result::Result< + RuntimeWorkerController, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { let worker_ref = request.worker_ref.clone(); let controller = self.inner.restore_controller(request).await?; self.captures diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs index 2f5c36ca8..da46ba67d 100644 --- a/crates/workspace-server/src/server_assignment_attention_tests.rs +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -494,9 +494,18 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFac async fn restore_controller( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, - ) -> std::result::Result { + ) -> std::result::Result< + worker_runtime::worker_backend::RuntimeWorkerController, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { self.controller(request.worker_ref.worker_id.to_string(), true) .await + .map_err(|_| { + worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( + worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, + "attention fixture reconstruction failed", + ) + }) } } diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs index 5d397814a..ff5adfca2 100644 --- a/crates/workspace-server/src/server_assignment_delivery_tests.rs +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -832,9 +832,18 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFact async fn restore_controller( &self, request: delivery_execution::WorkerExecutionRestoreRequest, - ) -> std::result::Result { + ) -> std::result::Result< + worker_runtime::worker_backend::RuntimeWorkerController, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { self.controller(request.worker_ref.worker_id.to_string(), true) .await + .map_err(|_| { + worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( + worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, + "delivery fixture reconstruction failed", + ) + }) } } diff --git a/crates/workspace-server/src/server_workspace_config_wip_tests.rs b/crates/workspace-server/src/server_workspace_config_wip_tests.rs index 1e989edd8..0714b0f22 100644 --- a/crates/workspace-server/src/server_workspace_config_wip_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_wip_tests.rs @@ -584,7 +584,10 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto async fn restore_controller( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, - ) -> std::result::Result { + ) -> std::result::Result< + worker_runtime::worker_backend::RuntimeWorkerController, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { self.inner.restore_controller(request).await } } From b5c105545c2fddd1849150bf0355832d1e9a301c Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 04:55:38 +0900 Subject: [PATCH 02/46] fix: close unattached deferred sessions in restore rollback Logical Workdir routing can use a Workspace session instead of the operation's raw Deferred handle, and reconstruction can fail before routing. Close the exact handles frozen in the accepted operation request after router cleanup and before releasing allocation ownership. Do not resolve aliases against a new execution or consult current Host grants for cleanup. Reproduced on 61c20b099 with diagnostic-only test output: rollback emptied routers with provider opens 0, but raw read succeeded and opened the provider (1). Retain the SessionClosed and zero-open regression, add pre-router Metadata/Reconstruction/Binding recovery and operation-mismatch/same-alias isolation coverage. Focused Profile and deferred cases pass; full exact-source validation follows this commit before publication. --- crates/worker-runtime/src/worker_backend.rs | 21 +- .../worker_backend_restore_failure_tests.rs | 179 ++++++++++++++++-- 2 files changed, 176 insertions(+), 24 deletions(-) diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 5583848ad..af4933bc3 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -2051,6 +2051,9 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { // Reconstruction can write before failing; only the resource owner can // confirm cleanup, even when no Worker value was returned. let result: Result = async { + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; let worker_name = Self::runtime_worker_name_for_ref(&request.worker_ref); let only_binding = (request.workdir_attachments.len() == 1) .then(|| request.workdir_attachments.values().next()) @@ -2351,22 +2354,34 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .map_err(|_| RuntimeRestoreCleanupFailure::Ownership)? .get(&request.worker_ref) .cloned(); - if let Some(resources) = retained { + if let Some(resources) = retained.as_ref() { if resources.operation_id != request.operation_id { return Err(RuntimeRestoreCleanupFailure::Ownership); } - if let Some(feature_tasks) = resources.feature_tasks { + if let Some(feature_tasks) = resources.feature_tasks.as_ref() { feature_tasks .shutdown() .await .map_err(|_| RuntimeRestoreCleanupFailure::Feature)?; } - for sessions in resources.sessions { + for sessions in &resources.sessions { sessions .close_all() .await .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; } + } + // Logical attachments can route through a different Workspace session, + // and reconstruction can fail before any router exists. Close the exact + // raw handles retained by this operation, never a current alias lookup. + // An unopened Deferred session closes without contacting its provider. + for session in request.deferred_workdir_sessions.values() { + session + .close() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; + } + if let Some(resources) = retained { if let Some(allocation) = resources.allocation { allocation .release() diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs index f7c021837..8b870cd62 100644 --- a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -216,6 +216,31 @@ impl ProfileFailureFixture { ); } + fn assert_closed_deferred_sessions(&self, pending: &PendingRuntimeRestore) { + let sessions = pending.request.deferred_workdir_sessions.clone(); + assert!(!sessions.is_empty()); + self.backend + .run_on_adapter_runtime(async move { + for (alias, session) in sessions { + let read = session + .read(workdir::ReadRequest { + path: workdir::WorkdirPath::new("README.md").unwrap(), + offset: 0, + limit: 10, + max_bytes: 1024, + }) + .await; + assert!( + matches!(read, Err(workdir::WorkdirError::SessionClosed)), + "raw deferred attachment {alias} remained usable: {read:?}" + ); + } + Ok(()) + }) + .unwrap(); + assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0); + } + fn fresh_restore_and_old_replay( &self, request: runtime_api::WorkerRestoreRequest, @@ -464,33 +489,145 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer .all(|sessions| sessions.aliases().is_empty()) ); fixture.assert_released(); - let unopened = pending.request.deferred_workdir_sessions.clone(); - fixture - .backend - .run_on_adapter_runtime(async move { - for session in unopened.values() { - assert!(matches!( - session - .read(workdir::ReadRequest { - path: workdir::WorkdirPath::new("README.md").unwrap(), - offset: 0, - limit: 10, - max_bytes: 1024, - }) - .await, - Err(workdir::WorkdirError::SessionClosed) - )); - } - Ok(()) - }) - .unwrap(); - assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0); + fixture.assert_closed_deferred_sessions(&pending); eprintln!( "restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=RolledBack" ); fixture.fresh_restore_and_old_replay(request, result); } +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_failure_before_router_binding_closes_unattached_raw_sessions() { + for stage in [ + RuntimeRestoreStage::Metadata, + RuntimeRestoreStage::Reconstruction, + RuntimeRestoreStage::Binding, + ] { + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + fixture.fail(stage); + fixture + .backend + .factory + .restore_faults + .cleanup_fail + .store(true, Ordering::SeqCst); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let failed = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + failed.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + assert!(pending.request.workdir_attachments.is_empty()); + assert!(!pending.request.logical_workdir_attachments.is_empty()); + assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0); + fixture + .backend + .factory + .restore_faults + .cleanup_fail + .store(false, Ordering::SeqCst); + let rolled_back = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + rolled_back.state, + server_api::WorkerRestoreState::RolledBack, + "{stage:?}: {rolled_back:?}" + ); + fixture.assert_released(); + fixture.assert_closed_deferred_sessions(&pending); + fixture.fresh_restore_and_old_replay(request, rolled_back); + // Retained old cleanup authority must not become the fresh execution's + // same-alias connection, even after the new execution has stopped. + fixture.assert_closed_deferred_sessions(&pending); + } +} + +#[tokio::test] +#[serial_test::serial(worker_allocation)] +async fn failed_restore_raw_cleanup_is_operation_bound_and_does_not_follow_reused_alias() { + let root = restore_failure_tempdir(); + let factory = ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(root.path()); + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let alias = WorkdirAttachmentAlias::new("checkout").unwrap(); + let opens = Arc::new(AtomicUsize::new(0)); + let raw = |opens: Arc| { + deferred_workdir::DeferredWorkdirSession::handle( + "same-workdir", + workdir::WorkdirSessionCapabilities::READ_ONLY, + move || { + opens.fetch_add(1, Ordering::SeqCst); + Err("offline fixture".into()) + }, + ) + }; + let mut old = direct_restore_request(worker_ref.clone()); + old.deferred_workdir_sessions + .insert(alias.clone(), raw(opens.clone())); + let mut current = direct_restore_request(worker_ref.clone()); + current + .deferred_workdir_sessions + .insert(alias.clone(), raw(opens.clone())); + factory.failed_restore_resources.lock().unwrap().insert( + worker_ref.clone(), + FailedRestoreResources { + operation_id: current.operation_id, + sessions: Vec::new(), + allocation: None, + feature_tasks: None, + }, + ); + assert_eq!( + factory.cleanup_failed_restore(&old).await, + Err(RuntimeRestoreCleanupFailure::Ownership) + ); + assert_eq!(opens.load(Ordering::SeqCst), 0); + assert_eq!( + factory.failed_restore_resources.lock().unwrap()[&worker_ref].operation_id, + current.operation_id + ); + factory.cleanup_failed_restore(¤t).await.unwrap(); + assert_eq!(opens.load(Ordering::SeqCst), 0); + // Old raw ownership was not routed into, or closed with, the newer operation. + let request = workdir::ReadRequest { + path: workdir::WorkdirPath::new("README.md").unwrap(), + offset: 0, + limit: 10, + max_bytes: 1024, + }; + assert!(matches!( + current.deferred_workdir_sessions[&alias] + .read(request.clone()) + .await, + Err(workdir::WorkdirError::SessionClosed) + )); + assert!(matches!( + old.deferred_workdir_sessions[&alias] + .read(request.clone()) + .await, + Err(workdir::WorkdirError::Unavailable(_)) + )); + assert_eq!(opens.load(Ordering::SeqCst), 1); + factory.cleanup_failed_restore(&old).await.unwrap(); + assert!(matches!( + old.deferred_workdir_sessions[&alias].read(request).await, + Err(workdir::WorkdirError::SessionClosed) + )); + assert_eq!(opens.load(Ordering::SeqCst), 1); +} + #[test] #[serial_test::serial(worker_allocation)] fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconciles_without_execution() From 62e2b4430588bacf90a2812a0a88772fc0a00c58 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 05:32:07 +0900 Subject: [PATCH 03/46] fix: retain safe Journal compaction failure diagnostics Keep stage/reason classification in the Journal Feature without changing frozen input, metadata or fallback behavior. Exercise connection, Host read failures, size budget and secret-bearing fixtures through compaction persistence. Validation: old freeze function with new tests fails connection and snapshot/size diagnostics (3 failures); new code passes all six selected cases and cargo test -p worker. Serialization is currently unreachable for the derived JSON-safe snapshot fields; no artificial runtime failure is claimed. --- Cargo.lock | 1 + crates/worker/Cargo.toml | 1 + crates/worker/src/feature/builtin/journal.rs | 152 ++++++++++++++-- crates/worker/tests/common/diagnostics.rs | 33 ++++ crates/worker/tests/compact_events_test.rs | 175 ++++++++++++++++--- 5 files changed, 324 insertions(+), 38 deletions(-) create mode 100644 crates/worker/tests/common/diagnostics.rs diff --git a/Cargo.lock b/Cargo.lock index f2901cb1f..ec2016a90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6558,6 +6558,7 @@ dependencies = [ "toml", "tools", "tracing", + "tracing-subscriber", "tungstenite 0.28.0", "uuid", "wip-client", diff --git a/crates/worker/Cargo.toml b/crates/worker/Cargo.toml index 9cdbc33c9..a96239ba3 100644 --- a/crates/worker/Cargo.toml +++ b/crates/worker/Cargo.toml @@ -58,6 +58,7 @@ wip-protocol = { workspace = true } wip-text-view = { workspace = true } [dev-dependencies] +tracing-subscriber.workspace = true axum = { workspace = true } workdir = { workspace = true, features = ["http-client"] } dotenv = "0.15.0" diff --git a/crates/worker/src/feature/builtin/journal.rs b/crates/worker/src/feature/builtin/journal.rs index e5de44038..7db8b5e35 100644 --- a/crates/worker/src/feature/builtin/journal.rs +++ b/crates/worker/src/feature/builtin/journal.rs @@ -59,6 +59,53 @@ impl JournalReadStatus { } } +// Only bounded classifications cross into diagnostics. Host messages/codes and +// io::Error payloads can contain Journal text, credentials, proofs or host paths. +// These diagnostics belong to the Feature, not persisted/model-visible input. +enum JournalCompactionFailure { + Connection(std::io::ErrorKind), + SnapshotConflict, + SnapshotInvalidArgument, + SnapshotUnavailable, + Serialization, + ByteBudget, +} + +impl JournalCompactionFailure { + fn snapshot(error: SubjektivHostError) -> Self { + match error { + SubjektivHostError::Conflict { .. } => Self::SnapshotConflict, + SubjektivHostError::InvalidArgument(_) => Self::SnapshotInvalidArgument, + SubjektivHostError::Unavailable(_) => Self::SnapshotUnavailable, + } + } + + fn stage(&self) -> &'static str { + match self { + Self::Connection(_) => "connection", + Self::SnapshotConflict | Self::SnapshotInvalidArgument | Self::SnapshotUnavailable => { + "snapshot" + } + Self::Serialization => "serialization", + Self::ByteBudget => "byte_budget", + } + } + + fn reason(&self) -> &'static str { + match self { + Self::Connection(std::io::ErrorKind::PermissionDenied) => "permission_denied", + Self::Connection(std::io::ErrorKind::InvalidInput) => "invalid_input", + Self::Connection(std::io::ErrorKind::NotConnected) => "not_connected", + Self::Connection(_) => "io_error", + Self::SnapshotConflict => "host_conflict", + Self::SnapshotInvalidArgument => "host_invalid_argument", + Self::SnapshotUnavailable => "host_unavailable", + Self::Serialization => "json_serialization_failed", + Self::ByteBudget => "startup_max_bytes_exceeded", + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct JournalCompactionInput { @@ -203,33 +250,46 @@ pub(crate) fn freeze_compaction_journal( .and_then(|connection| connection.host.workspace_id()) }); let read = match connection { - Ok(Some(connection)) if connection.host.journal_compaction_granted() => { - Some(connection.journal_compaction_context().map_err(|_| ())) - } + Ok(Some(connection)) if connection.host.journal_compaction_granted() => Some( + connection + .journal_compaction_context() + .map_err(JournalCompactionFailure::snapshot), + ), Ok(_) => None, - Err(_) => Some(Err(())), + Err(error) => Some(Err(JournalCompactionFailure::Connection(error.kind()))), }; let snapshot = read.as_ref().and_then(|read| read.as_ref().ok()).cloned(); - let (status, json) = match read { - None => (Disabled, String::new()), + let prepared = match read { + None => Ok((Disabled, String::new())), Some(Ok(snapshot)) => { let status = if snapshot.journals.is_empty() { Absent } else { Available }; - match serde_json::to_string(&snapshot) { - Ok(json) if json.len() <= subjektiv::journal::startup::STARTUP_MAX_BYTES => { - (status, json) - } - _ => (Unavailable, String::new()), - } + // The current snapshot has only derived, JSON-safe fields (strings, + // integers, vectors, enums and a UUID). There is no known serialization + // failure fixture; retain this guard if the domain type changes. + serde_json::to_string(&snapshot) + .map_err(|_| JournalCompactionFailure::Serialization) + .and_then(|json| { + if json.len() <= subjektiv::journal::startup::STARTUP_MAX_BYTES { + Ok((status, json)) + } else { + Err(JournalCompactionFailure::ByteBudget) + } + }) } - Some(Err(())) => (Unavailable, String::new()), + Some(Err(failure)) => Err(failure), }; - if status == Unavailable { - tracing::warn!("Journal compaction snapshot unavailable; continuing without Journal"); - } + let (status, json) = prepared.unwrap_or_else(|failure| { + tracing::warn!( + stage = failure.stage(), + reason = failure.reason(), + "Journal compaction snapshot unavailable; continuing without Journal" + ); + (Unavailable, String::new()) + }); use sha2::{Digest, Sha256}; let snapshot_sha256: Option = (!json.is_empty()).then(|| { Sha256::digest(json.as_bytes()) @@ -726,12 +786,72 @@ fn tool_schema(name: &str) -> Value { } } +#[cfg(test)] +#[path = "../../../tests/common/diagnostics.rs"] +mod diagnostic_test_support; + #[cfg(test)] mod tests { use super::*; use std::sync::Mutex; use subjektiv::journal::JournalReceipt; + #[test] + fn compaction_connection_failure_keeps_safe_reason_without_exposing_error_or_changing_input() { + let prompts = crate::prompt::catalog::PromptCatalog::builtins_only().unwrap(); + for (kind, reason) in [ + (std::io::ErrorKind::PermissionDenied, "permission_denied"), + (std::io::ErrorKind::InvalidInput, "invalid_input"), + (std::io::ErrorKind::NotConnected, "not_connected"), + (std::io::ErrorKind::Other, "io_error"), + ] { + let logs = diagnostic_test_support::CapturedWarnings::default(); + let input = tracing::subscriber::with_default(logs.subscriber(), || { + freeze_compaction_journal( + Err(std::io::Error::new( + kind, + "PRIVATE_JOURNAL_BODY /private/host/path proof=PRIVATE_PROOF credential=PRIVATE_CREDENTIAL raw_error=PRIVATE_ERROR", + )), + &prompts, + ) + .unwrap() + }); + let log = logs.text(); + assert!(log.contains("stage=\"connection\""), "{log}"); + assert!(log.contains(&format!("reason=\"{reason}\"")), "{log}"); + assert_eq!(log.matches("continuing without Journal").count(), 1); + assert!(!log.contains("PRIVATE_")); + assert!(!log.contains("/private/host/path")); + assert!(input.body().contains("Journal read status: unavailable.")); + let persisted = serde_json::to_string(&input).unwrap(); + assert!(!persisted.contains("PRIVATE_")); + assert!(!persisted.contains("/private/host/path")); + assert!(!persisted.contains(reason)); + let payload: JournalCompactionInput = input + .metadata() + .feature_payload() + .unwrap() + .decode("journal:compaction", 1) + .unwrap(); + assert!(payload.snapshot.is_none()); + assert!(payload.snapshot_sha256.is_none()); + assert_eq!( + payload.prompt_provenance.source_digest, + prompts.projection().source_digest + ); + assert_eq!( + payload.prompt_provenance.projection_digest, + prompts.projection().catalog_digest + ); + } + let logs = diagnostic_test_support::CapturedWarnings::default(); + let input = tracing::subscriber::with_default(logs.subscriber(), || { + freeze_compaction_journal(Ok(None), &prompts).unwrap() + }); + assert!(input.body().contains("Journal read status: disabled.")); + assert!(logs.text().is_empty()); + } + #[derive(Default)] struct Host { requests: Mutex>, diff --git a/crates/worker/tests/common/diagnostics.rs b/crates/worker/tests/common/diagnostics.rs new file mode 100644 index 000000000..d5768f599 --- /dev/null +++ b/crates/worker/tests/common/diagnostics.rs @@ -0,0 +1,33 @@ +use std::io::{self, Write}; +use std::sync::{Arc, Mutex}; + +/// Per-test warning capture; never installs a global subscriber or writes fixture secrets. +#[derive(Clone, Default)] +pub struct CapturedWarnings(Arc>>); + +impl CapturedWarnings { + pub fn subscriber(&self) -> impl tracing::Subscriber + Send + Sync + 'static { + let output = self.clone(); + tracing_subscriber::fmt() + .with_writer(move || output.clone()) + .without_time() + .with_ansi(false) + .with_max_level(tracing::Level::WARN) + .finish() + } + + pub fn text(&self) -> String { + String::from_utf8(self.0.lock().unwrap().clone()).unwrap() + } +} + +impl Write for CapturedWarnings { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/crates/worker/tests/compact_events_test.rs b/crates/worker/tests/compact_events_test.rs index 0869c0b5c..c4de54e11 100644 --- a/crates/worker/tests/compact_events_test.rs +++ b/crates/worker/tests/compact_events_test.rs @@ -26,6 +26,9 @@ use tokio::sync::broadcast; use worker::{Worker, WorkerController, WorkerError}; +#[path = "common/diagnostics.rs"] +mod diagnostics; + type TestStore = CombinedStore; #[derive(Clone)] @@ -2876,7 +2879,7 @@ const COMPACTION_HOST_WORKSPACE_ID: &str = "journal-transport-host-workspace"; // Host authorization/selection or live-model semantic continuation quality. struct CompactionJournalHost { granted: bool, - unavailable: bool, + failure: Option<&'static str>, snapshot: Mutex, reads: Mutex>, startup_reads: AtomicUsize, @@ -2886,7 +2889,7 @@ impl CompactionJournalHost { fn new(snapshot: subjektiv::journal::startup::JournalStartupSnapshot) -> Self { Self { granted: true, - unavailable: false, + failure: None, snapshot: Mutex::new(snapshot), reads: Mutex::new(vec![]), startup_reads: AtomicUsize::new(0), @@ -2917,10 +2920,18 @@ impl worker::subjektiv::SubjektivHost for CompactionJournalHost { worker::subjektiv::SubjektivHostError, > { self.reads.lock().unwrap().push(context.clone()); - if self.unavailable { - return Err(worker::subjektiv::SubjektivHostError::Unavailable( - "injected Journal read failure".into(), - )); + if let Some(failure) = self.failure { + use worker::subjektiv::SubjektivHostError; + let secret = "PRIVATE_JOURNAL_BODY /private/host/path proof=PRIVATE_PROOF credential=PRIVATE_CREDENTIAL raw_error=PRIVATE_ERROR"; + return Err(match failure { + "unavailable" => SubjektivHostError::Unavailable(secret.into()), + "invalid-argument" => SubjektivHostError::InvalidArgument(secret.into()), + "conflict" => SubjektivHostError::Conflict { + code: secret.into(), + message: secret.into(), + }, + _ => panic!("unknown failure fixture"), + }); } Ok(self.snapshot.lock().unwrap().clone()) } @@ -3466,26 +3477,37 @@ async fn journal_three_compactions_freeze_one_snapshot_preserve_full_summary_tai } #[tokio::test] -async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversation_compaction() { - for case in [ - "absent", - "disabled", - "unavailable", - "over-byte-budget", - "no-host", - ] { +async fn journal_empty_disabled_and_available_compaction_preserve_inputs_without_failure_warnings() +{ + assert_journal_compaction_cases(&["absent", "available", "disabled", "no-host"]).await; +} + +#[tokio::test] +async fn journal_snapshot_failures_keep_safe_diagnostics_and_continue_compaction() { + assert_journal_compaction_cases(&["unavailable", "invalid-argument", "conflict"]).await; +} + +#[tokio::test] +async fn journal_over_byte_budget_keeps_safe_diagnostic_and_continues_compaction() { + assert_journal_compaction_cases(&["over-byte-budget"]).await; +} + +async fn assert_journal_compaction_cases(cases: &[&'static str]) { + for &case in cases { let mut snapshot = compaction_snapshot(0); if case == "absent" { snapshot.journals.clear(); } if case == "over-byte-budget" { snapshot.journals[0].current_position = - "x".repeat(subjektiv::journal::startup::STARTUP_MAX_BYTES); + "PRIVATE_JOURNAL_BODY".repeat(subjektiv::journal::startup::STARTUP_MAX_BYTES); } - let expected_json = (case == "absent").then(|| serde_json::to_string(&snapshot).unwrap()); + let expected_json = matches!(case, "absent" | "available") + .then(|| serde_json::to_string(&snapshot).unwrap()); let mut host = CompactionJournalHost::new(snapshot); host.granted = case != "disabled"; - host.unavailable = case == "unavailable"; + host.failure = + matches!(case, "unavailable" | "invalid-argument" | "conflict").then_some(case); let host = Arc::new(host); let client = MockClient::new(vec![ single_text_events("seed"), @@ -3517,7 +3539,51 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat true, ) .await; - fixture.worker.compact(0).await.unwrap(); + // Scope the subscriber to every poll, not a thread-local guard across await. + // This observes the real Feature warning while compacting into FsStore. + let logs = diagnostics::CapturedWarnings::default(); + use tracing::instrument::WithSubscriber; + fixture + .worker + .compact(0) + .with_subscriber(logs.subscriber()) + .await + .unwrap(); + let log = logs.text(); + let failure = match case { + "unavailable" => Some(("snapshot", "host_unavailable")), + "invalid-argument" => Some(("snapshot", "host_invalid_argument")), + "conflict" => Some(("snapshot", "host_conflict")), + "over-byte-budget" => Some(("byte_budget", "startup_max_bytes_exceeded")), + _ => None, + }; + if let Some((stage, reason)) = failure { + assert!(log.contains(&format!("stage=\"{stage}\"")), "{case}: {log}"); + assert!( + log.contains(&format!("reason=\"{reason}\"")), + "{case}: {log}" + ); + assert_eq!( + log.matches("continuing without Journal").count(), + 1, + "{case}" + ); + } else { + assert!( + !log.contains("Journal compaction snapshot unavailable"), + "{case}: {log}" + ); + } + for secret in [ + "PRIVATE_", + "/private/host/path", + "JOURNAL_POSITION_0", + "JOURNAL_BODY_0", + "state-token-0", + "fixed-event-reference-0", + ] { + assert!(!log.contains(secret), "{case}: diagnostic leaked {secret}"); + } let input = first_compactor_input(&observed.requests()[1]); let body = if matches!(case, "disabled" | "no-host") { assert_eq!( @@ -3532,8 +3598,8 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat String::new() } else { let body = journal_body(&fixture.worker); - let expected = if case == "absent" { - "absent" + let expected = if matches!(case, "absent" | "available") { + case } else { "unavailable" }; @@ -3548,6 +3614,71 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat &fixture.worker.prompts().load_full(), Some(COMPACTION_HOST_WORKSPACE_ID), ); + let seed = replacement_seed(&fixture.worker); + let metadata = seed[1].metadata.input_metadata.as_ref().unwrap(); + assert_eq!( + metadata.usage().recording_material, + session_store::InputEvidenceUse::Exclude + ); + assert_eq!( + metadata.usage().memory_evidence, + session_store::InputEvidenceUse::Exclude + ); + assert_eq!( + metadata.usage().observation, + session_store::InputObservationUse::Hidden + ); + let payload: serde_json::Value = metadata + .feature_payload() + .unwrap() + .decode("journal:compaction", 1) + .unwrap(); + assert_eq!(payload["status"], expected); + assert_eq!( + payload["prompt_provenance"]["workspace_id"], + COMPACTION_HOST_WORKSPACE_ID + ); + let projection = fixture.worker.prompts().load_full().projection().clone(); + assert_eq!( + payload["prompt_provenance"]["source_digest"], + projection.source_digest + ); + assert_eq!( + payload["prompt_provenance"]["projection_digest"], + projection.catalog_digest + ); + if let Some(json) = expected_json.as_deref() { + use sha2::{Digest, Sha256}; + assert_eq!( + payload["snapshot"], + serde_json::from_str::(json).unwrap() + ); + assert_eq!( + payload["snapshot_sha256"], + Sha256::digest(json.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + ); + } else { + assert!(payload["snapshot"].is_null()); + assert!(payload["snapshot_sha256"].is_null()); + } + // Feature diagnostics do not become model input or a new metadata contract. + let saved = serde_json::to_string(&seed[1]).unwrap(); + for forbidden in [ + "PRIVATE_", + "/private/host/path", + "host_unavailable", + "host_conflict", + "host_invalid_argument", + "startup_max_bytes_exceeded", + ] { + assert!( + !saved.contains(forbidden), + "{case}: saved input leaked {forbidden}" + ); + } body }; assert_full_initial_summary(&input, &inherited); @@ -3559,8 +3690,8 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat .is_some_and(|text| text.contains("PREVIOUS_JOURNAL_DELEGATED_CONTEXT"))), "fallback retains only the newly frozen reference, not an old snapshot as latest" ); - assert!(!body.contains("JOURNAL_POSITION_0")); - assert!(!body.contains("journal-reference-0")); + assert_eq!(body.contains("JOURNAL_POSITION_0"), case == "available"); + assert_eq!(body.contains("journal-reference-0"), case == "available"); assert_eq!( host.reads.lock().unwrap().len(), usize::from(!matches!(case, "disabled" | "no-host")), From 62b2d27b4a35fe6d3f7f96261bf0ca4e545ed94b Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 05:40:39 +0900 Subject: [PATCH 04/46] fix: retain command failure stages and output collection state --- Cargo.lock | 2 + crates/tools/src/bash.rs | 8 +- crates/tools/src/bash_output.rs | 28 +- crates/tools/tests/integration.rs | 103 ++++++- crates/workdir/Cargo.toml | 1 + crates/workdir/src/external.rs | 4 + crates/workdir/src/local/command_transport.rs | 220 +++++++++++---- .../src/local/command_transport/tests.rs | 263 +++++++++++++++++ .../src/local/command_transport/tests/http.rs | 238 ++++++++++++++++ crates/workdir/src/operation.rs | 267 +++++++++++++++++- crates/worker-runtime/Cargo.toml | 1 + .../tests/command_transport_tests.rs | 264 ++++++++++++++++- crates/workspace-server/src/server.rs | 6 + .../server/tests/command_transport_tests.rs | 2 + 14 files changed, 1329 insertions(+), 78 deletions(-) create mode 100644 crates/workdir/src/local/command_transport/tests/http.rs diff --git a/Cargo.lock b/Cargo.lock index f2901cb1f..fd2babc6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6497,6 +6497,7 @@ name = "workdir" version = "0.1.0" dependencies = [ "async-trait", + "axum", "fs-operation", "libc", "manifest", @@ -6597,6 +6598,7 @@ dependencies = [ "tokio", "tokio-tungstenite 0.29.0", "toml", + "tools", "tower", "tracing", "tracing-subscriber", diff --git a/crates/tools/src/bash.rs b/crates/tools/src/bash.rs index efeaa3acf..c3c3a6311 100644 --- a/crates/tools/src/bash.rs +++ b/crates/tools/src/bash.rs @@ -367,7 +367,7 @@ impl BashTool { guard.handle = None; let timed_out = terminal.status == workdir::CommandStatus::TimedOut; - let summary = + let mut summary = if cancellation_requested || terminal.status == workdir::CommandStatus::Cancelled { format!("$ {cmd_summary} (cancelled)") } else if timed_out { @@ -379,6 +379,12 @@ impl BashTool { None => format!("$ {cmd_summary} (terminated)"), } }; + if matches!( + terminal.output_collection, + workdir::CommandOutputCollection::Incomplete { .. } + ) { + summary.push_str(" (output incomplete)"); + } Ok(BashResult { summary, stdout_preview: String::from_utf8_lossy(&stdout_preview).into_owned(), diff --git a/crates/tools/src/bash_output.rs b/crates/tools/src/bash_output.rs index 538b50433..fad3b02e9 100644 --- a/crates/tools/src/bash_output.rs +++ b/crates/tools/src/bash_output.rs @@ -68,8 +68,10 @@ pub struct OutputRange { pub struct OutputTerminal { pub status: Option, pub exit_code: Option, - /// False on a forcibly dropped collector or failed transport: received bytes are retained, - /// but they are not a claim about the unobserved process tail. + pub process_failure: Option, + pub output_collection: Option, + /// True only for confirmed EOF collection (quota is reported separately). A terminal + /// fence or exit zero alone does not confirm the unobserved process tail. pub transport_complete: bool, } @@ -405,12 +407,9 @@ impl BashOutputWriter { terminal: &workdir::CommandTerminal, ) -> Result { let mut r = self.record.lock().unwrap(); - if terminal.stdout_delivered_bytes != r.stdout_seen + if !terminal.metadata_valid() + || terminal.stdout_delivered_bytes != r.stdout_seen || terminal.stderr_delivered_bytes != r.stderr_seen - || terminal.stdout_bytes < r.stdout_seen - || terminal.stderr_bytes < r.stderr_seen - || ((terminal.stdout_bytes != r.stdout_seen || terminal.stderr_bytes != r.stderr_seen) - != terminal.truncation_reason.is_some()) { return Err("command terminal counts do not fence delivered bytes".into()); } @@ -419,7 +418,12 @@ impl BashOutputWriter { r.terminal = Some(OutputTerminal { status: Some(terminal.status), exit_code: terminal.exit_code, - transport_complete: true, + transport_complete: matches!( + terminal.output_collection, + workdir::CommandOutputCollection::Complete + ), + process_failure: terminal.process_failure, + output_collection: Some(terminal.output_collection), }); Ok(r.metadata()) } @@ -439,6 +443,8 @@ impl Drop for BashOutputWriter { status: None, exit_code: None, transport_complete: false, + process_failure: None, + output_collection: None, }); } } @@ -456,6 +462,8 @@ mod tests { status: Some(CommandStatus::Completed), exit_code: Some(0), transport_complete: true, + process_failure: None, + output_collection: Some(workdir::CommandOutputCollection::Complete), }) .id } @@ -514,6 +522,8 @@ mod tests { status: Some(CommandStatus::Completed), exit_code: Some(0), transport_complete: true, + process_failure: None, + output_collection: Some(workdir::CommandOutputCollection::Complete), }); assert!(root.metadata(&metadata.id).is_some()); let _replacement = sibling.begin().unwrap(); @@ -582,7 +592,7 @@ mod tests { let root = BashOutputView::new(); let mut writer = root.begin().unwrap(); writer.append(CommandStream::Stdout, b"abc"); - let mut terminal = serde_json::json!({"status":"completed","exit_code":0,"stdout_bytes":9,"stderr_bytes":0,"stdout_delivered_bytes":4,"stderr_delivered_bytes":0,"truncation_reason":"byte_budget"}); + let mut terminal = serde_json::json!({"status":"completed","exit_code":0,"process_failure":null,"output_collection":{"state":"complete"},"stdout_bytes":9,"stderr_bytes":0,"stdout_delivered_bytes":4,"stderr_delivered_bytes":0,"truncation_reason":"byte_budget"}); assert!( writer .finish_transport(&serde_json::from_value(terminal.clone()).unwrap()) diff --git a/crates/tools/tests/integration.rs b/crates/tools/tests/integration.rs index f7b1dd2da..50ddfd32d 100644 --- a/crates/tools/tests/integration.rs +++ b/crates/tools/tests/integration.rs @@ -416,6 +416,7 @@ struct ExternalValidatingSession { truncate_first_binary_response: bool, command_response_losses: std::sync::atomic::AtomicUsize, lose_terminal_only: bool, + collection_override: Option, mutate_after_first_binary_response: Option, } @@ -534,7 +535,11 @@ impl workdir::WorkdirSession for ExternalValidatingSession { &self, request: workdir::CommandOutputRequest, ) -> Result { - let output = self.inner.command_output(request).await?; + let mut output = self.inner.command_output(request).await?; + if let (Some(collection), Some(terminal)) = (self.collection_override, &mut output.terminal) + { + terminal.output_collection = collection; + } if (!self.lose_terminal_only || output.terminal.is_some()) && self .command_response_losses @@ -575,6 +580,7 @@ async fn external_view_image_uses_a_protocol_bounded_request() { truncate_first_binary_response: false, command_response_losses: std::sync::atomic::AtomicUsize::new(0), lose_terminal_only: false, + collection_override: None, mutate_after_first_binary_response: None, }); let png = include_bytes!("fixtures/view-image/valid.png"); @@ -600,6 +606,7 @@ async fn external_view_image_rejects_truncated_and_cross_version_chunks() { truncate_first_binary_response: true, command_response_losses: std::sync::atomic::AtomicUsize::new(0), lose_terminal_only: false, + collection_override: None, mutate_after_first_binary_response: None, }); let (_meta, tool) = view_image_tool(truncated)(); @@ -618,6 +625,7 @@ async fn external_view_image_rejects_truncated_and_cross_version_chunks() { truncate_first_binary_response: false, command_response_losses: std::sync::atomic::AtomicUsize::new(0), lose_terminal_only: false, + collection_override: None, mutate_after_first_binary_response: Some(image_path), }); let (_meta, tool) = view_image_tool(changing)(); @@ -1104,7 +1112,20 @@ async fn bash_cancellation_returns_bounded_progress_as_terminal_output() { let execution = tokio::spawn(async move { executing.execute(r#"{"command":"printf 'before'; printf 'err-before' >&2; while :; do :; done","cwd":"."}"#, execution_context).await }); - await_command_output(&mut events).await; + // Cancellation must follow both writes, not merely the first stdout chunk. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + let mut ready = [false; 2]; + while !ready.iter().all(|seen| *seen) { + if let workdir::CommandEvent::Output { stream, .. } = events.recv().await.unwrap() { + ready[match stream { + workdir::CommandStream::Stdout => 0, + workdir::CommandStream::Stderr => 1, + }] = true; + } + } + }) + .await + .expect("both streams written before cancellation"); bash.cancel_execution(&context).await.unwrap(); let ToolError::Cancelled(output) = tokio::time::timeout(std::time::Duration::from_secs(5), execution) @@ -1281,6 +1302,7 @@ async fn lost_page_and_terminal_responses_replay_without_restarting_or_double_ap mutate_after_first_binary_response: None, command_response_losses: std::sync::atomic::AtomicUsize::new(1), lose_terminal_only, + collection_override: None, }); let view = tools::bash_output::BashOutputView::new(); let definitions = core_builtin_tools(session.clone(), Tracker::new(), view.clone()); @@ -1315,3 +1337,80 @@ async fn lost_page_and_terminal_responses_replay_without_restarting_or_double_ap session.close().await.unwrap(); } } + +// Provider metadata is the fixture here; actual read/drain injection and HTTP +// forwarding are tested by workdir's production-supervisor suite. +#[tokio::test] +async fn exit_zero_with_incomplete_output_keeps_partial_bytes_and_safe_failure_in_retained_evidence() + { + use workdir::{ + CommandFailure, CommandFailureStage as Stage, CommandIoErrorKind, CommandOutputCollection, + }; + for stage in [ + Stage::StdoutRead, + Stage::StderrRead, + Stage::StdoutDrain, + Stage::StderrDrain, + Stage::OutputDelivery, + ] { + let root = TempDir::new().unwrap(); + let failure = CommandFailure { + stage, + reason: CommandIoErrorKind::PermissionDenied, + os_code: None, + }; + let collection = CommandOutputCollection::Incomplete { failure }; + let session: WorkdirSessionHandle = Arc::new(ExternalValidatingSession { + inner: LocalWorkdirSession::new( + Scope::writable(root.path()).unwrap(), + root.path().into(), + ), + truncate_first_binary_response: false, + mutate_after_first_binary_response: None, + command_response_losses: std::sync::atomic::AtomicUsize::new(0), + lose_terminal_only: false, + collection_override: Some(collection), + }); + let view = tools::bash_output::BashOutputView::new(); + let (_, bash) = tools::bash_tool(session.clone(), view.clone())(); + let out = call( + &bash, + json!({"cwd":".","command":"printf partial; printf err >&2"}), + ) + .await; + assert!(out.summary.ends_with("(output incomplete)")); + let body: serde_json::Value = serde_json::from_str(&out.content.unwrap()).unwrap(); + let metadata = &body["output"]; + assert_eq!(body["stdout_preview"], "partial"); + assert_eq!(body["stderr_preview"], "err"); + assert_eq!(body["preview_omitted"], false); + assert_eq!(metadata["terminal"]["exit_code"], 0); + assert_eq!(metadata["terminal"]["status"], "completed"); + assert_eq!(metadata["terminal"]["transport_complete"], false); + assert_eq!( + metadata["terminal"]["output_collection"], + serde_json::to_value(collection).unwrap() + ); + assert_eq!(metadata["storage_truncated"], false); + assert_eq!(metadata["transport_truncation"], serde_json::Value::Null); + assert_eq!(metadata["stdout_seen"], 7); + assert_eq!(metadata["stdout_delivered"], 7); + let id = metadata["id"].as_str().unwrap(); + assert_eq!( + serde_json::to_value(view.metadata(id).unwrap()).unwrap(), + *metadata + ); + assert_eq!( + view.read(id, workdir::CommandStream::Stdout, 0, 100) + .unwrap(), + b"partial" + ); + assert_eq!( + view.read(id, workdir::CommandStream::Stderr, 0, 100) + .unwrap(), + b"err" + ); + assert!(!metadata.to_string().contains(root.path().to_str().unwrap())); + session.close().await.unwrap(); + } +} diff --git a/crates/workdir/Cargo.toml b/crates/workdir/Cargo.toml index b8541897a..c972355ec 100644 --- a/crates/workdir/Cargo.toml +++ b/crates/workdir/Cargo.toml @@ -23,5 +23,6 @@ thiserror.workspace = true tokio = { workspace = true, features = ["io-util", "process", "rt", "sync", "time"] } [dev-dependencies] +axum.workspace = true serde_json.workspace = true tempfile.workspace = true diff --git a/crates/workdir/src/external.rs b/crates/workdir/src/external.rs index c1c00bfd8..e1d37d21c 100644 --- a/crates/workdir/src/external.rs +++ b/crates/workdir/src/external.rs @@ -924,6 +924,8 @@ mod tests { chunks: vec![], next_cursor: 0, terminal: Some(crate::CommandTerminal { + process_failure: None, + output_collection: crate::CommandOutputCollection::Complete, status: crate::CommandStatus::Completed, exit_code: Some(0), stdout_bytes: 0, @@ -1082,6 +1084,8 @@ mod tests { }], next_cursor: 3, terminal: Some(crate::CommandTerminal { + process_failure: None, + output_collection: crate::CommandOutputCollection::Complete, status: crate::CommandStatus::Completed, exit_code: Some(0), stdout_bytes: 100 * 1024 * 1024, diff --git a/crates/workdir/src/local/command_transport.rs b/crates/workdir/src/local/command_transport.rs index 1af30b1fc..0d5a261e4 100644 --- a/crates/workdir/src/local/command_transport.rs +++ b/crates/workdir/src/local/command_transport.rs @@ -3,6 +3,7 @@ //! A replay page survives a lost HTTP response until its next cursor is sent. use super::*; +use crate::{CommandFailure, CommandFailureStage, CommandIoErrorKind, CommandOutputCollection}; use tokio::io::{AsyncRead, AsyncReadExt}; use tokio::sync::mpsc; @@ -146,6 +147,33 @@ pub(super) fn start( telemetry: CommandTelemetry, environment: BTreeMap, resources: Vec>, +) -> LocalCommand { + let execution_id = command_id.clone(); + let execution_telemetry = telemetry.clone(); + start_with( + command_id, + telemetry, + resources, + move |cancel, sender| async move { + execute( + cwd, + request, + &execution_id, + &execution_telemetry, + environment, + cancel, + &sender, + ) + .await + }, + ) +} + +fn start_with + Send + 'static>( + command_id: String, + telemetry: CommandTelemetry, + resources: Vec>, + execution: impl FnOnce(watch::Receiver, mpsc::Sender) -> F + Send + 'static, ) -> LocalCommand { let (sender, receiver) = mpsc::channel(QUEUE_CHUNKS); let (cancel, cancel_rx) = watch::channel(false); @@ -160,16 +188,7 @@ pub(super) fn start( })); let task = tokio::spawn(async move { let _resources = resources; - let terminal = execute( - cwd, - request, - &command_id, - &telemetry, - environment, - cancel_rx, - &sender, - ) - .await; + let terminal = execution(cancel_rx, sender.clone()).await; telemetry.terminal(&command_id, terminal.status, terminal.exit_code); // Closing the byte queue fences this already-published terminal, without // requiring spare queue capacity or a receiver to release task resources. @@ -192,10 +211,18 @@ struct ByteAccounting { } impl ByteAccounting { - fn terminal(&self, status: CommandStatus, exit_code: Option) -> crate::CommandTerminal { + fn terminal( + &self, + status: CommandStatus, + exit_code: Option, + process_failure: Option, + output_collection: CommandOutputCollection, + ) -> crate::CommandTerminal { crate::CommandTerminal { status, exit_code, + process_failure, + output_collection, stdout_bytes: self.stdout, stderr_bytes: self.stderr, stdout_delivered_bytes: self.stdout_delivered, @@ -215,7 +242,7 @@ async fn pump( command_id: &str, mut remaining: u64, accounting: &mut ByteAccounting, -) -> std::io::Result<()> { +) -> Result<(), CommandFailure> { let mut stdout_done = false; let mut stderr_done = false; let mut stdout_decoder = CommandOutputDecoder::default(); @@ -228,7 +255,11 @@ async fn pump( // Before quota, reserve first to retain bounded pipe backpressure. let permit = if remaining > 0 { let Ok(permit) = sender.reserve().await else { - return Ok(()); + return Err(CommandFailure { + stage: CommandFailureStage::OutputDelivery, + reason: CommandIoErrorKind::Unknown, + os_code: None, + }); }; Some(permit) } else { @@ -242,7 +273,15 @@ async fn pump( CommandStream::Stdout => &stdout_buffer, CommandStream::Stderr => &stderr_buffer, }; - let count = result?; + let count = result.map_err(|error| { + CommandFailure::io( + match stream { + CommandStream::Stdout => CommandFailureStage::StdoutRead, + CommandStream::Stderr => CommandFailureStage::StderrRead, + }, + &error, + ) + })?; let decoder = match stream { CommandStream::Stdout => &mut stdout_decoder, CommandStream::Stderr => &mut stderr_decoder, @@ -288,7 +327,7 @@ async fn execute( command_id: &str, telemetry: &CommandTelemetry, environment: BTreeMap, - mut cancel: watch::Receiver, + cancel: watch::Receiver, sender: &mpsc::Sender, ) -> crate::CommandTerminal { telemetry.started(command_id, request.tool_call_id); @@ -304,74 +343,159 @@ async fn execute( .kill_on_drop(true); #[cfg(unix)] command.as_std_mut().process_group(0); - let mut accounting = ByteAccounting::default(); - let Ok(mut child) = command.spawn() else { - return accounting.terminal(CommandStatus::Failed, None); + let mut child = match command.spawn() { + Ok(child) => child, + Err(error) => { + return ByteAccounting::default().terminal( + CommandStatus::Failed, + None, + Some(CommandFailure::io(CommandFailureStage::Spawn, &error)), + CommandOutputCollection::NotStarted, + ); + } }; - let mut process_group = CommandProcessGroupGuard::new(child.id()); let stdout = child.stdout.take().expect("piped stdout"); let stderr = child.stderr.take().expect("piped stderr"); - let (status, exit_code) = { + supervise( + child, + stdout, + stderr, + request.timeout_secs, + request.transport_byte_budget, + command_id, + telemetry, + cancel, + sender, + ) + .await +} + +// Substitute only syscalls and readers: injected faults run the production supervisor. +trait CommandProcess: Send { + fn id(&self) -> Option; + fn start_kill(&mut self); + fn wait( + &mut self, + ) -> impl std::future::Future> + Send; +} + +impl CommandProcess for tokio::process::Child { + fn id(&self) -> Option { + self.id() + } + fn start_kill(&mut self) { + let _ = self.start_kill(); + } + async fn wait(&mut self) -> std::io::Result { + self.wait().await + } +} + +fn process_result( + result: std::io::Result, +) -> (CommandStatus, Option, Option) { + match result { + Ok(exit) => ( + if exit.success() { + CommandStatus::Completed + } else { + CommandStatus::Failed + }, + exit.code(), + None, + ), + Err(error) => ( + CommandStatus::Failed, + None, + Some(CommandFailure::io(CommandFailureStage::Wait, &error)), + ), + } +} + +#[allow(clippy::too_many_arguments)] +async fn supervise( + mut child: impl CommandProcess, + stdout: impl AsyncRead + Unpin, + stderr: impl AsyncRead + Unpin, + timeout_secs: u64, + byte_budget: u64, + command_id: &str, + telemetry: &CommandTelemetry, + mut cancel: watch::Receiver, + sender: &mpsc::Sender, +) -> crate::CommandTerminal { + let mut process_group = CommandProcessGroupGuard::new(child.id()); + let mut accounting = ByteAccounting::default(); + let (status, exit_code, process_failure, output_collection) = { let output = pump( stdout, stderr, sender, telemetry, command_id, - request.transport_byte_budget, + byte_budget, &mut accounting, ); tokio::pin!(output); - let mut drained = false; - let timeout = tokio::time::sleep(Duration::from_secs(request.timeout_secs.max(1))); + let mut collected = None; + let timeout = tokio::time::sleep(Duration::from_secs(timeout_secs.max(1))); tokio::pin!(timeout); let terminal = loop { tokio::select! { - result = &mut output, if !drained => { - drained = true; - if result.is_err() || sender.is_closed() { + result = &mut output, if collected.is_none() => { + let failed = result.is_err(); + collected = Some(result); + if failed { process_group.terminate(); - let _ = child.start_kill(); - let _ = child.wait().await; - break (CommandStatus::Failed, None); + child.start_kill(); + break process_result(child.wait().await); } } exit = child.wait() => { process_group.terminate(); - break match exit { - Ok(exit) => (if exit.success() { CommandStatus::Completed } else { CommandStatus::Failed }, exit.code()), - Err(_) => (CommandStatus::Failed, None), - }; + break process_result(exit); } _ = &mut timeout => { process_group.terminate(); - let _ = child.start_kill(); - let code = child.wait().await.ok().and_then(|exit| exit.code()); - break (CommandStatus::TimedOut, code); + child.start_kill(); + let (_, code, failure) = process_result(child.wait().await); + break (CommandStatus::TimedOut, code, failure); } _ = sender.closed() => { process_group.terminate(); - let _ = child.start_kill(); - let _ = child.wait().await; - break (CommandStatus::Cancelled, None); + child.start_kill(); + let (_, code, failure) = process_result(child.wait().await); + break (CommandStatus::Cancelled, code, failure); } changed = cancel.changed() => { if changed.is_err() || *cancel.borrow() { process_group.terminate(); - let _ = child.start_kill(); - let code = child.wait().await.ok().and_then(|exit| exit.code()); - break (CommandStatus::Cancelled, code); + child.start_kill(); + let (_, code, failure) = process_result(child.wait().await); + break (CommandStatus::Cancelled, code, failure); } } } }; - // Even after kill, preserve the bounded pipe tail before publishing terminal. - if !drained { - let _ = output.await; - } - terminal + // Exit is not pipe EOF. Retain read failures first observed during drain. + let result = match collected { + Some(result) => result, + None => output.await.map_err(|mut failure| { + failure.stage = match failure.stage { + CommandFailureStage::StdoutRead => CommandFailureStage::StdoutDrain, + CommandFailureStage::StderrRead => CommandFailureStage::StderrDrain, + stage => stage, + }; + failure + }), + }; + let collection = match result { + Ok(()) => CommandOutputCollection::Complete, + Err(failure) => CommandOutputCollection::Incomplete { failure }, + }; + (terminal.0, terminal.1, terminal.2, collection) }; - accounting.terminal(status, exit_code) + accounting.terminal(status, exit_code, process_failure, output_collection) } #[cfg(test)] diff --git a/crates/workdir/src/local/command_transport/tests.rs b/crates/workdir/src/local/command_transport/tests.rs index 12f20b061..324ce614e 100644 --- a/crates/workdir/src/local/command_transport/tests.rs +++ b/crates/workdir/src/local/command_transport/tests.rs @@ -520,3 +520,266 @@ async fn slow_receiver_backpressures_before_quota_but_full_queue_does_not_block_ assert_eq!(queued_bytes, QUEUE_CHUNKS * COMMAND_EVENT_CHUNK_BYTES); } } + +#[derive(Clone, Default)] +struct Gate(Arc)>>); + +impl Gate { + fn open(&self) { + let mut state = self.0.lock().unwrap(); + state.0 = true; + if let Some(waker) = state.1.take() { + waker.wake(); + } + } + + fn poll(&self, cx: &mut std::task::Context<'_>) -> std::task::Poll<()> { + let mut state = self.0.lock().unwrap(); + if state.0 { + std::task::Poll::Ready(()) + } else { + state.1 = Some(cx.waker().clone()); + std::task::Poll::Pending + } + } +} + +struct FaultProcess { + ready: Gate, + exited: Gate, + wait_error: Option, +} + +impl CommandProcess for FaultProcess { + fn id(&self) -> Option { + None + } + fn start_kill(&mut self) { + self.ready.open(); + } + async fn wait(&mut self) -> std::io::Result { + std::future::poll_fn(|cx| self.ready.poll(cx)).await; + self.exited.open(); + match self.wait_error { + Some(kind) => Err(std::io::Error::new( + kind, + "/private/path TOKEN=secret command-body", + )), + None => Ok(std::process::ExitStatus::default()), + } + } +} + +struct FaultReader { + prefix: &'static [u8], + prefix_read: Gate, + fail_ready: Gate, +} + +impl AsyncRead for FaultReader { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buffer: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + if !self.prefix.is_empty() { + let count = self.prefix.len().min(buffer.remaining()); + buffer.put_slice(&self.prefix[..count]); + self.prefix = &self.prefix[count..]; + self.prefix_read.open(); + return std::task::Poll::Ready(Ok(())); + } + std::task::ready!(self.fail_ready.poll(cx)); + std::task::Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "/private/path TOKEN=secret command-body", + ))) + } +} + +// Faults replace only OS wait/read boundaries, not collection or finalization. +#[tokio::test] +async fn read_and_post_exit_drain_faults_keep_prefix_exit_and_safe_original_reason() { + for stream in [CommandStream::Stdout, CommandStream::Stderr] { + for after_exit in [false, true] { + let ready = Gate::default(); + let exited = Gate::default(); + let fail_ready = if after_exit { + exited.clone() + } else { + let gate = Gate::default(); + gate.open(); + gate + }; + let reader = FaultReader { + prefix: b"partial", + prefix_read: if after_exit { + ready.clone() + } else { + Gate::default() + }, + fail_ready, + }; + let process = FaultProcess { + ready, + exited, + wait_error: None, + }; + let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS); + let (_cancel, cancel_rx) = watch::channel(false); + let telemetry = CommandTelemetry::new(); + let terminal = tokio::time::timeout(Duration::from_secs(5), async { + match stream { + CommandStream::Stdout => { + supervise( + process, + reader, + &b""[..], + 10, + 100, + "fault", + &telemetry, + cancel_rx, + &sender, + ) + .await + } + CommandStream::Stderr => { + supervise( + process, + &b""[..], + reader, + 10, + 100, + "fault", + &telemetry, + cancel_rx, + &sender, + ) + .await + } + } + }) + .await + .unwrap(); + let Message::Bytes(chunk) = receiver.recv().await.unwrap(); + assert_eq!(chunk.bytes, b"partial"); + assert_eq!(chunk.stream, stream); + assert_eq!(chunk.sequence, 0); + assert_eq!(chunk.offset, 0); + assert!(receiver.try_recv().is_err()); + assert_eq!(terminal.status, CommandStatus::Completed); + assert_eq!(terminal.exit_code, Some(0)); + assert_eq!(terminal.process_failure, None); + assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7); + assert_eq!( + terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes, + 7 + ); + assert_eq!(terminal.truncation_reason, None); + assert_eq!( + terminal.output_collection, + CommandOutputCollection::Incomplete { + failure: CommandFailure { + stage: match (stream, after_exit) { + (CommandStream::Stdout, false) => CommandFailureStage::StdoutRead, + (CommandStream::Stdout, true) => CommandFailureStage::StdoutDrain, + (CommandStream::Stderr, false) => CommandFailureStage::StderrRead, + (CommandStream::Stderr, true) => CommandFailureStage::StderrDrain, + }, + reason: CommandIoErrorKind::PermissionDenied, + os_code: None + }, + } + ); + let wire = serde_json::to_string(&terminal).unwrap(); + for secret in ["/private", "TOKEN", "secret", "command-body"] { + assert!(!wire.contains(secret)); + } + } + } +} + +#[tokio::test] +async fn wait_fault_keeps_collected_bytes_and_does_not_invent_exit_information() { + let ready = Gate::default(); + ready.open(); + let process = FaultProcess { + ready, + exited: Gate::default(), + wait_error: Some(std::io::ErrorKind::InvalidData), + }; + let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS); + let (_cancel, cancel_rx) = watch::channel(false); + let terminal = supervise( + process, + &b"out"[..], + &b"err"[..], + 10, + 100, + "wait-fault", + &CommandTelemetry::new(), + cancel_rx, + &sender, + ) + .await; + assert_eq!(terminal.status, CommandStatus::Failed); + assert_eq!(terminal.exit_code, None); + assert_eq!( + terminal.process_failure, + Some(CommandFailure { + stage: CommandFailureStage::Wait, + reason: CommandIoErrorKind::InvalidData, + os_code: None + }) + ); + assert_eq!( + terminal.output_collection, + CommandOutputCollection::Complete + ); + assert_eq!(terminal.stdout_bytes, 3); + assert_eq!(terminal.stderr_bytes, 3); + let mut chunks = Vec::new(); + while let Ok(Message::Bytes(chunk)) = receiver.try_recv() { + chunks.push(chunk); + } + assert_eq!(stream_bytes(&chunks, CommandStream::Stdout), b"out"); + assert_eq!(stream_bytes(&chunks, CommandStream::Stderr), b"err"); + assert!(!serde_json::to_string(&terminal).unwrap().contains("secret")); +} + +#[tokio::test] +async fn spawn_fault_retains_safe_reason_without_claiming_output_collection() { + let root = TempDir::new().unwrap(); + let (sender, _receiver) = mpsc::channel(QUEUE_CHUNKS); + let (_cancel, cancel_rx) = watch::channel(false); + let terminal = execute( + root.path().join("missing-secret-path"), + Harness::request("secret-command", 10, 100), + "spawn-fault", + &CommandTelemetry::new(), + BTreeMap::new(), + cancel_rx, + &sender, + ) + .await; + assert_eq!( + terminal.process_failure, + Some(CommandFailure { + stage: CommandFailureStage::Spawn, + reason: CommandIoErrorKind::NotFound, + os_code: Some(libc::ENOENT) + }) + ); + assert_eq!( + terminal.output_collection, + CommandOutputCollection::NotStarted + ); + assert_eq!(terminal.exit_code, None); + assert_eq!(terminal.status, CommandStatus::Failed); + assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 0); + assert!(!serde_json::to_string(&terminal).unwrap().contains("secret")); +} + +#[cfg(feature = "http-client")] +mod http; diff --git a/crates/workdir/src/local/command_transport/tests/http.rs b/crates/workdir/src/local/command_transport/tests/http.rs new file mode 100644 index 000000000..0bed3d4dc --- /dev/null +++ b/crates/workdir/src/local/command_transport/tests/http.rs @@ -0,0 +1,238 @@ +//! Loopback integration of the real supervisor, byte queue/replay, local session, +//! HTTP dispatcher/client. OS wait/read are +//! deterministic faults; session open and start bind that injected process fixture. +//! Authorization is not asserted here (Runtime's signed HTTP suite owns it). + +use super::*; +use crate::http::*; +use axum::{Json, Router, extract::State, routing::post}; + +struct Server(tokio::task::JoinHandle<()>); +impl Drop for Server { + fn drop(&mut self) { + self.0.abort(); + } +} + +async fn operation( + State(session): State>, + Json(request): Json, +) -> Json { + if matches!(request.operation, WorkdirSessionOperation::CommandStart(_)) { + return Json(WorkdirSessionOperationResult::CommandStart(CommandHandle( + "injected".into(), + ))); + } + Json( + dispatch_workdir_session_operation(session.as_ref(), request.operation) + .await + .unwrap(), + ) +} + +#[tokio::test] +async fn pipe_and_wait_faults_cross_real_http_with_prefix_and_terminal_replay() { + tokio::time::timeout(Duration::from_secs(20), async { + for (stream, after_exit, wait_fault) in [ + (CommandStream::Stdout, false, false), + (CommandStream::Stderr, false, false), + (CommandStream::Stdout, true, false), + (CommandStream::Stderr, true, false), + (CommandStream::Stdout, false, true), + ] { + let root = TempDir::new().unwrap(); + let session = Arc::new(LocalWorkdirSession::new( + Scope::writable(root.path()).unwrap(), + root.path().into(), + )); + let ready = Gate::default(); + let exited = Gate::default(); + if wait_fault { + ready.open(); + } + let process = FaultProcess { + ready: ready.clone(), + exited: exited.clone(), + wait_error: wait_fault.then_some(std::io::ErrorKind::InvalidData), + }; + let fail_ready = if after_exit { + exited + } else { + let gate = Gate::default(); + gate.open(); + gate + }; + let reader = FaultReader { + prefix: b"partial", + prefix_read: if after_exit { ready } else { Gate::default() }, + fail_ready, + }; + let command = start_with( + "injected".into(), + CommandTelemetry::new(), + vec![], + move |cancel, sender| async move { + let telemetry = CommandTelemetry::new(); + if wait_fault { + supervise( + process, + &b"partial"[..], + &b""[..], + 10, + 100, + "injected", + &telemetry, + cancel, + &sender, + ) + .await + } else { + match stream { + CommandStream::Stdout => { + supervise( + process, + reader, + &b""[..], + 10, + 100, + "injected", + &telemetry, + cancel, + &sender, + ) + .await + } + CommandStream::Stderr => { + supervise( + process, + &b""[..], + reader, + 10, + 100, + "injected", + &telemetry, + cancel, + &sender, + ) + .await + } + } + } + }, + ); + session + .inner + .commands + .lock() + .await + .insert("injected".into(), command); + let app = Router::new() + .route( + "/v1/working-directories/{id}/sessions", + post(|| async { + Json(OpenWorkdirSessionResponse { + session_id: WorkdirSessionId::new("fault-session").unwrap(), + workdir_id: crate::Workdir::new("fault-workdir").id().clone(), + capabilities: crate::WorkdirSessionCapabilities::ALL, + }) + }), + ) + .route("/v1/workdir-sessions/{id}/operations", post(operation)) + .with_state(session.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let _server = Server(tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + })); + let remote: crate::WorkdirSessionHandle = Arc::new( + RemoteWorkdirSession::open( + reqwest::Client::builder().no_proxy().build().unwrap(), + base.parse().unwrap(), + "fixture-token", + crate::Workdir::new("fault-workdir").id().clone(), + OpenWorkdirSessionRequest::default(), + ) + .await + .unwrap(), + ); + let handle = CommandHandle("injected".into()); + let mut request = CommandOutputRequest { + handle, + cursor: 0, + limit: 1, + wait: true, + }; + let mut bytes = Vec::new(); + let terminal = loop { + let page = remote.command_output(request.clone()).await.unwrap(); + assert_eq!(remote.command_output(request.clone()).await.unwrap(), page); + for chunk in &page.chunks { + assert_eq!(chunk.sequence, request.cursor); + assert_eq!(chunk.offset, request.cursor); + assert_eq!(chunk.stream, stream); + bytes.extend_from_slice(&chunk.bytes); + request.cursor += chunk.bytes.len() as u64; + } + assert_eq!(page.next_cursor, request.cursor); + if let Some(terminal) = page.terminal { + break terminal; + } + }; + assert_eq!(bytes, b"partial"); + assert_eq!(terminal.truncation_reason, None); + assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7); + assert_eq!( + terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes, + 7 + ); + if wait_fault { + assert_eq!(terminal.status, CommandStatus::Failed); + assert_eq!(terminal.exit_code, None); + assert_eq!( + terminal.process_failure, + Some(CommandFailure { + stage: CommandFailureStage::Wait, + reason: CommandIoErrorKind::InvalidData, + os_code: None + }) + ); + assert_eq!( + terminal.output_collection, + CommandOutputCollection::Complete + ); + } else { + assert_eq!(terminal.status, CommandStatus::Completed); + assert_eq!(terminal.exit_code, Some(0)); + let stage = match (stream, after_exit) { + (CommandStream::Stdout, false) => CommandFailureStage::StdoutRead, + (CommandStream::Stderr, false) => CommandFailureStage::StderrRead, + (CommandStream::Stdout, true) => CommandFailureStage::StdoutDrain, + (CommandStream::Stderr, true) => CommandFailureStage::StderrDrain, + }; + assert_eq!( + terminal.output_collection, + CommandOutputCollection::Incomplete { + failure: CommandFailure { + stage, + reason: CommandIoErrorKind::PermissionDenied, + os_code: None + } + } + ); + } + let wire = serde_json::to_string(&terminal).unwrap(); + for secret in [ + "/private", + "TOKEN", + "secret", + "command-body", + "fixture-token", + ] { + assert!(!wire.contains(secret)); + } + session.close().await.unwrap(); + } + }) + .await + .expect("bounded loopback fault projection"); +} diff --git a/crates/workdir/src/operation.rs b/crates/workdir/src/operation.rs index 7e180e7aa..8cd671b53 100644 --- a/crates/workdir/src/operation.rs +++ b/crates/workdir/src/operation.rs @@ -132,12 +132,151 @@ pub enum CommandOutputTruncation { ByteBudget, } -/// Raw totals include discarded pipe bytes. Delivered totals count only prefixes -/// admitted to the transport queue, not page reads or retries by a consumer. +/// Closed diagnostic vocabulary: never serialize OS messages, paths, or errno text. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CommandIoErrorKind { + NotFound, + PermissionDenied, + ConnectionRefused, + ConnectionReset, + ConnectionAborted, + NotConnected, + BrokenPipe, + AlreadyExists, + WouldBlock, + InvalidInput, + InvalidData, + TimedOut, + WriteZero, + Interrupted, + UnexpectedEof, + OutOfMemory, + HostUnreachable, + NetworkUnreachable, + AddrInUse, + AddrNotAvailable, + NetworkDown, + NotADirectory, + IsADirectory, + DirectoryNotEmpty, + ReadOnlyFilesystem, + StaleNetworkFileHandle, + StorageFull, + NotSeekable, + QuotaExceeded, + FileTooLarge, + ResourceBusy, + ExecutableFileBusy, + Deadlock, + CrossesDevices, + TooManyLinks, + InvalidFilename, + ArgumentListTooLong, + Unsupported, + Other, + Unknown, +} + +impl From for CommandIoErrorKind { + fn from(kind: std::io::ErrorKind) -> Self { + use std::io::ErrorKind as Io; + match kind { + Io::NotFound => Self::NotFound, + Io::PermissionDenied => Self::PermissionDenied, + Io::ConnectionRefused => Self::ConnectionRefused, + Io::ConnectionReset => Self::ConnectionReset, + Io::ConnectionAborted => Self::ConnectionAborted, + Io::NotConnected => Self::NotConnected, + Io::BrokenPipe => Self::BrokenPipe, + Io::AlreadyExists => Self::AlreadyExists, + Io::WouldBlock => Self::WouldBlock, + Io::InvalidInput => Self::InvalidInput, + Io::InvalidData => Self::InvalidData, + Io::TimedOut => Self::TimedOut, + Io::WriteZero => Self::WriteZero, + Io::Interrupted => Self::Interrupted, + Io::UnexpectedEof => Self::UnexpectedEof, + Io::OutOfMemory => Self::OutOfMemory, + Io::HostUnreachable => Self::HostUnreachable, + Io::NetworkUnreachable => Self::NetworkUnreachable, + Io::AddrInUse => Self::AddrInUse, + Io::AddrNotAvailable => Self::AddrNotAvailable, + Io::NetworkDown => Self::NetworkDown, + Io::NotADirectory => Self::NotADirectory, + Io::IsADirectory => Self::IsADirectory, + Io::DirectoryNotEmpty => Self::DirectoryNotEmpty, + Io::ReadOnlyFilesystem => Self::ReadOnlyFilesystem, + Io::StaleNetworkFileHandle => Self::StaleNetworkFileHandle, + Io::StorageFull => Self::StorageFull, + Io::NotSeekable => Self::NotSeekable, + Io::QuotaExceeded => Self::QuotaExceeded, + Io::FileTooLarge => Self::FileTooLarge, + Io::ResourceBusy => Self::ResourceBusy, + Io::ExecutableFileBusy => Self::ExecutableFileBusy, + Io::Deadlock => Self::Deadlock, + Io::CrossesDevices => Self::CrossesDevices, + Io::TooManyLinks => Self::TooManyLinks, + Io::InvalidFilename => Self::InvalidFilename, + Io::ArgumentListTooLong => Self::ArgumentListTooLong, + Io::Unsupported => Self::Unsupported, + Io::Other => Self::Other, + _ => Self::Unknown, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum CommandFailureStage { + Spawn, + Wait, + StdoutRead, + StderrRead, + StdoutDrain, + StderrDrain, + OutputDelivery, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CommandFailure { + pub stage: CommandFailureStage, + pub reason: CommandIoErrorKind, + /// Observed numeric OS code, if supplied by the source. Never inferred from text. + pub os_code: Option, +} + +impl CommandFailure { + pub(crate) fn io(stage: CommandFailureStage, error: &std::io::Error) -> Self { + Self { + stage, + reason: error.kind().into(), + os_code: error.raw_os_error(), + } + } +} + +/// EOF collection is independent of exit status and the explicit byte budget. +/// Incomplete counts are lower bounds on bytes actually produced by the process. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +pub enum CommandOutputCollection { + NotStarted, + Complete, + Incomplete { failure: CommandFailure }, +} + +/// Raw totals include discarded *observed* pipe bytes, not an unknown failed tail. +/// Delivered totals count only prefixes admitted to the transport queue, not page +/// reads or retries. Neither terminal arrival nor exit zero implies EOF collection. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] pub struct CommandTerminal { pub status: CommandStatus, pub exit_code: Option, + pub process_failure: Option, + pub output_collection: CommandOutputCollection, pub stdout_bytes: u64, pub stderr_bytes: u64, pub stdout_delivered_bytes: u64, @@ -145,6 +284,54 @@ pub struct CommandTerminal { pub truncation_reason: Option, } +impl CommandTerminal { + /// Validate provider-authored metadata before publishing it as collected evidence. + pub fn metadata_valid(&self) -> bool { + let truncated = self.stdout_bytes > self.stdout_delivered_bytes + || self.stderr_bytes > self.stderr_delivered_bytes; + if self.status == CommandStatus::Running + || self.stdout_delivered_bytes > self.stdout_bytes + || self.stderr_delivered_bytes > self.stderr_bytes + || self.truncation_reason != truncated.then_some(CommandOutputTruncation::ByteBudget) + { + return false; + } + match self.process_failure { + Some(CommandFailure { + stage: CommandFailureStage::Spawn, + .. + }) => { + self.status == CommandStatus::Failed + && self.exit_code.is_none() + && self.output_collection == CommandOutputCollection::NotStarted + && self.stdout_bytes == 0 + && self.stderr_bytes == 0 + } + Some(CommandFailure { + stage: CommandFailureStage::Wait, + .. + }) => { + self.status != CommandStatus::Completed + && self.exit_code.is_none() + && self.collection_started() + } + Some(_) => false, + None => self.collection_started(), + } + } + + fn collection_started(&self) -> bool { + match self.output_collection { + CommandOutputCollection::NotStarted => false, + CommandOutputCollection::Complete => true, + CommandOutputCollection::Incomplete { failure } => !matches!( + failure.stage, + CommandFailureStage::Spawn | CommandFailureStage::Wait + ), + } + } +} + /// Sending next_cursor acknowledges this page. Only the current page may be /// retried; acknowledged bytes expire. Terminal fences all captured bytes, /// including on timeout/cancel. Consumers must pull while the process runs. @@ -186,19 +373,13 @@ impl CommandOutput { self.next_cursor <= MAX_COMMAND_TRANSPORT_BYTES && end.is_none_or(|end| end == self.next_cursor) && self.terminal.as_ref().is_none_or(|terminal| { - let truncated = terminal.stdout_bytes > terminal.stdout_delivered_bytes - || terminal.stderr_bytes > terminal.stderr_delivered_bytes; - stream_ends[0].is_none_or(|end| end == terminal.stdout_delivered_bytes) + terminal.metadata_valid() + && stream_ends[0].is_none_or(|end| end == terminal.stdout_delivered_bytes) && stream_ends[1].is_none_or(|end| end == terminal.stderr_delivered_bytes) - && terminal.status != CommandStatus::Running - && terminal.stdout_delivered_bytes <= terminal.stdout_bytes - && terminal.stderr_delivered_bytes <= terminal.stderr_bytes && terminal .stdout_delivered_bytes .checked_add(terminal.stderr_delivered_bytes) == Some(self.next_cursor) - && terminal.truncation_reason - == truncated.then_some(CommandOutputTruncation::ByteBudget) }) } } @@ -262,4 +443,70 @@ mod tests { .contains(&serde_json::json!("transport_byte_budget")) ); } + + #[test] + fn failure_wire_is_closed_and_does_not_infer_os_codes_from_custom_messages() { + let error = std::io::Error::new( + std::io::ErrorKind::Other, + "/host/private TOKEN=secret EACCES command", + ); + let failure = CommandFailure::io(CommandFailureStage::Wait, &error); + assert_eq!( + serde_json::to_value(failure).unwrap(), + serde_json::json!({"stage":"wait","reason":"other","os_code":null}) + ); + let os_error = std::io::Error::from_raw_os_error(libc::EIO); + let actual = CommandFailure::io(CommandFailureStage::StdoutDrain, &os_error); + assert_eq!(actual.os_code, Some(libc::EIO)); + for invalid in [ + serde_json::json!({"stage":"wait","reason":"/host/private TOKEN=secret"}), + serde_json::json!({"stage":"/host/private","reason":"other"}), + serde_json::json!({"stage":"wait","reason":"other","message":"secret"}), + ] { + assert!(serde_json::from_value::(invalid).is_err()); + } + } + + #[test] + fn provider_terminal_rejects_inconsistent_failure_domains_and_unstarted_byte_claims() { + let wire = serde_json::json!({"status":"completed","exit_code":0,"process_failure":null,"output_collection":{"state":"incomplete","failure":{"stage":"stdout_drain","reason":"other","os_code":null}},"stdout_bytes":3,"stderr_bytes":0,"stdout_delivered_bytes":3,"stderr_delivered_bytes":0,"truncation_reason":null}); + let terminal: CommandTerminal = serde_json::from_value(wire.clone()).unwrap(); + assert!(terminal.metadata_valid()); + for (field, value) in [ + ( + "output_collection", + serde_json::json!({"state":"not_started"}), + ), + ( + "output_collection", + serde_json::json!({"state":"incomplete","failure":{"stage":"wait","reason":"other"}}), + ), + ( + "process_failure", + serde_json::json!({"stage":"stdout_read","reason":"other"}), + ), + ( + "process_failure", + serde_json::json!({"stage":"spawn","reason":"not_found"}), + ), + ("truncation_reason", serde_json::json!("byte_budget")), + ] { + let mut invalid = wire.clone(); + invalid[field] = value; + assert!( + !serde_json::from_value::(invalid) + .unwrap() + .metadata_valid() + ); + } + let mut unclassified = wire.clone(); + unclassified + .as_object_mut() + .unwrap() + .remove("output_collection"); + assert!(serde_json::from_value::(unclassified).is_err()); + let mut raw = wire; + raw["message"] = serde_json::json!("/private TOKEN=secret"); + assert!(serde_json::from_value::(raw).is_err()); + } } diff --git a/crates/worker-runtime/Cargo.toml b/crates/worker-runtime/Cargo.toml index 0eba6a589..9b1c4ab49 100644 --- a/crates/worker-runtime/Cargo.toml +++ b/crates/worker-runtime/Cargo.toml @@ -56,6 +56,7 @@ server-api = { path = "../server-api" } workdir.workspace = true [dev-dependencies] +tools.workspace = true workdir = { workspace = true, features = ["http-client"] } futures.workspace = true agen.workspace = true diff --git a/crates/worker-runtime/src/http_server/tests/command_transport_tests.rs b/crates/worker-runtime/src/http_server/tests/command_transport_tests.rs index 902694621..fb4c3ffd7 100644 --- a/crates/worker-runtime/src/http_server/tests/command_transport_tests.rs +++ b/crates/worker-runtime/src/http_server/tests/command_transport_tests.rs @@ -3,6 +3,7 @@ //! pre-seeded session is a fixture; no Runtime Worker/materializer is claimed here. use super::*; +use std::collections::BTreeMap; use std::time::Duration; use workdir::http::{ RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture, @@ -70,14 +71,23 @@ impl Harness { } async fn new() -> Self { - let root = tempfile::tempdir().unwrap(); - let session: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::materialized_bound( - Workdir::new("wd-transport"), - root.path().to_path_buf(), - root.path().to_path_buf(), - SharedScope::new(Scope::writable(root.path()).unwrap()), - WorkdirSessionCapabilities::ALL, - )); + Self::with_environment(tempfile::tempdir().unwrap(), BTreeMap::new()).await + } + + async fn with_environment( + root: tempfile::TempDir, + command_environment: BTreeMap, + ) -> Self { + let session: WorkdirSessionHandle = + Arc::new(LocalWorkdirSession::materialized_bound_with_environment( + Workdir::new("wd-transport"), + root.path().to_path_buf(), + root.path().to_path_buf(), + SharedScope::new(Scope::writable(root.path()).unwrap()), + WorkdirSessionCapabilities::ALL, + command_environment, + Vec::new(), + )); let identity = RuntimeIdentityMaterial::generate("transport-key").unwrap(); let trust = workspace_trust(&identity); let state = RuntimeHttpState { @@ -222,6 +232,234 @@ async fn real_http_waiting_command_survives_long_poll_expiry() { .expect("bounded long-poll continuation over HTTP"); } +#[tokio::test] +async fn real_http_spawn_failure_preserves_safe_cause_counts_and_terminal_replay() { + tokio::time::timeout(Duration::from_secs(10), async { + let root = tempfile::tempdir().unwrap(); + let missing_path = root.path().join("private-spawn-path-must-not-exist"); + assert!(!missing_path.exists()); + let private_path = missing_path.to_str().unwrap().to_owned(); + let environment_secret = "spawn-environment-secret-must-not-leak"; + let command_secret = "spawn-command-secret-must-not-leak"; + let tool_call_secret = "spawn-tool-call-secret-must-not-leak"; + // This is an OS spawn failure in the production local provider, not a + // fabricated terminal: PATH contains no bash. The test process environment + // is untouched, so parallel command tests retain their normal PATH. + let harness = Harness::with_environment( + root, + BTreeMap::from([ + ("PATH".into(), private_path.clone()), + ("PRIVATE_AUTH_TOKEN".into(), environment_secret.into()), + ]), + ) + .await; + let command = format!("printf '{command_secret}'; printf ran > must-not-run"); + let handle = harness + .remote + .start_command(CommandRequest { + transport_byte_budget: 1024, + command: command.clone(), + timeout_secs: 5, + cwd: WorkdirPath::root(), + tool_call_id: Some(tool_call_secret.into()), + }) + .await + .unwrap(); + let request = CommandOutputRequest { + handle, + cursor: 0, + limit: 16, + wait: true, + }; + let page = harness.remote.command_output(request.clone()).await.unwrap(); + assert!(page.chunks.is_empty()); + assert_eq!(page.next_cursor, 0); + let terminal = page.terminal.as_ref().expect("spawn failure is terminal"); + assert_eq!(terminal.status, workdir::CommandStatus::Failed); + assert_eq!(terminal.exit_code, None); + assert_eq!( + terminal.process_failure, + Some(workdir::CommandFailure { + stage: workdir::CommandFailureStage::Spawn, + reason: workdir::CommandIoErrorKind::NotFound, + os_code: Some(libc::ENOENT), + }) + ); + assert_eq!( + terminal.output_collection, + workdir::CommandOutputCollection::NotStarted + ); + assert_eq!(terminal.stdout_bytes, 0); + assert_eq!(terminal.stderr_bytes, 0); + assert_eq!(terminal.stdout_delivered_bytes, 0); + assert_eq!(terminal.stderr_delivered_bytes, 0); + assert_eq!(terminal.truncation_reason, None); + + // A zero-byte terminal page remains replayable at its unchanged cursor; + // neither a retry nor a smaller nonwaiting read changes its accounting. + assert_eq!( + harness.remote.command_output(request.clone()).await.unwrap(), + page + ); + let replay_request = CommandOutputRequest { + wait: false, + limit: 1, + ..request + }; + assert_eq!( + harness + .remote + .command_output(replay_request.clone()) + .await + .unwrap(), + page + ); + let response = harness + .raw_operation( + serde_json::to_value(WorkdirSessionOperation::CommandOutput( + replay_request.clone(), + )) + .unwrap(), + ) + .await; + assert_eq!(response.status(), reqwest::StatusCode::OK); + let body = response.text().await.unwrap(); + for secret in [ + private_path.as_str(), + harness._root.path().to_str().unwrap(), + "private-spawn-path-must-not-exist", + "PRIVATE_AUTH_TOKEN", + environment_secret, + command_secret, + command.as_str(), + tool_call_secret, + ] { + assert!( + !body.contains(secret), + "private input leaked into HTTP terminal" + ); + } + let wire: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!( + wire, + serde_json::json!({ + "operation": "command_output", + "result": { + "chunks": [], + "next_cursor": 0, + "terminal": { + "status": "failed", + "exit_code": null, + "process_failure": {"stage": "spawn", "reason": "not_found", "os_code": libc::ENOENT}, + "output_collection": {"state": "not_started"}, + "stdout_bytes": 0, + "stderr_bytes": 0, + "stdout_delivered_bytes": 0, + "stderr_delivered_bytes": 0, + "truncation_reason": null + } + } + }) + ); + assert_eq!( + harness.remote.command_output(replay_request).await.unwrap(), + page + ); + // Exercise the same real spawn failure through Bash, not a terminal + // fixture. Opening another remote client reuses the pre-seeded provider + // session and still passes through the signed HTTP middleware. + let bash_remote: WorkdirSessionHandle = Arc::new( + RemoteWorkdirSession::open_with_authorization( + harness.client.clone(), + harness.operation_url.join("/").unwrap(), + harness.authorization.clone(), + Workdir::new("wd-transport").id().clone(), + OpenWorkdirSessionRequest::default(), + ) + .await + .unwrap(), + ); + let view = tools::bash_output::BashOutputView::new(); + let (_, bash) = tools::bash_tool(bash_remote, view.clone())(); + let input = serde_json::json!({ + "command": command, + "cwd": ".", + "timeout": 5 + }) + .to_string(); + let output = bash + .execute( + &input, + agen::tool::ToolExecutionContext::new(tool_call_secret, "http-spawn-attempt", 0), + ) + .await + .expect("Bash returns a structured failed-command result"); + let content = output.content.as_deref().expect("Bash ToolOutput body"); + let tool_body: serde_json::Value = serde_json::from_str(content).unwrap(); + assert_eq!(tool_body["stdout_preview"], ""); + assert_eq!(tool_body["stderr_preview"], ""); + assert_eq!(tool_body["preview_omitted"], false); + assert_eq!( + tool_body["output"]["terminal"], + serde_json::json!({ + "status": "failed", + "exit_code": null, + "process_failure": {"stage": "spawn", "reason": "not_found", "os_code": libc::ENOENT}, + "output_collection": {"state": "not_started"}, + "transport_complete": false + }) + ); + let output_id = tool_body["output"]["id"].as_str().unwrap(); + let retained = view.metadata(output_id).expect("retained spawn failure"); + assert_eq!(retained.terminal.status, Some(workdir::CommandStatus::Failed)); + assert_eq!(retained.terminal.exit_code, None); + assert_eq!(retained.terminal.process_failure, terminal.process_failure); + assert_eq!( + retained.terminal.output_collection, + Some(workdir::CommandOutputCollection::NotStarted) + ); + assert!(!retained.terminal.transport_complete); + assert_eq!(retained.stdout_saved, 0); + assert_eq!(retained.stderr_saved, 0); + assert_eq!(retained.stdout_seen, 0); + assert_eq!(retained.stderr_seen, 0); + assert_eq!(retained.stdout_delivered, 0); + assert_eq!(retained.stderr_delivered, 0); + assert!(!retained.storage_truncated); + assert!(retained.retention_limit.is_none()); + assert_eq!(retained.transport_truncation, None); + let retained_wire = serde_json::to_value(&retained).unwrap(); + assert_eq!(retained_wire, tool_body["output"]); + for stream in [CommandStream::Stdout, CommandStream::Stderr] { + assert_eq!(view.read(output_id, stream, 0, 1), Some(Vec::new())); + } + for secret in [ + private_path.as_str(), + harness._root.path().to_str().unwrap(), + "private-spawn-path-must-not-exist", + "PRIVATE_AUTH_TOKEN", + environment_secret, + command_secret, + command.as_str(), + tool_call_secret, + ] { + assert!( + !content.contains(secret), + "private input leaked into Bash ToolOutput body" + ); + } + assert!(!harness._root.path().join("must-not-run").exists()); + harness.remote.close().await.unwrap(); + assert!(harness.session.command_snapshot().is_empty()); + assert_eq!( + serde_json::to_value(view.metadata(output_id).unwrap()).unwrap(), + retained_wire + ); + }) + .await + .expect("bounded real spawn failure over signed HTTP"); +} + #[tokio::test] async fn real_http_closed_session_retains_failure_reason() { let harness = Harness::new().await; @@ -291,6 +529,11 @@ async fn real_http_command_pages_preserve_binary_streams_replay_and_nonzero_exit assert_eq!(stderr, b"stderr\xe6\x97\xa5"); assert_eq!(terminal.status, workdir::CommandStatus::Failed); assert_eq!(terminal.exit_code, Some(9)); + assert_eq!(terminal.process_failure, None); + assert_eq!( + terminal.output_collection, + workdir::CommandOutputCollection::Complete + ); assert!(harness.remote.command_output(CommandOutputRequest { handle, cursor: 0, limit: 1, wait: false }).await.is_err()); harness.remote.close().await.unwrap(); assert!(harness.session.command_snapshot().is_empty()); @@ -435,6 +678,11 @@ async fn real_http_command_budget_bounds_unique_payload_and_reports_larger_seen_ ); assert_eq!(terminal.status, workdir::CommandStatus::Failed); assert_eq!(terminal.exit_code, Some(9)); + assert_eq!(terminal.process_failure, None); + assert_eq!( + terminal.output_collection, + workdir::CommandOutputCollection::Complete + ); assert!( harness .remote diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 76d394da4..a1b36997f 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -38045,6 +38045,8 @@ mod tests { chunks: Vec::new(), next_cursor: 0, terminal: Some(workdir::CommandTerminal { + process_failure: None, + output_collection: workdir::CommandOutputCollection::Complete, status: workdir::CommandStatus::Cancelled, exit_code: None, @@ -38147,6 +38149,8 @@ mod tests { chunks: Vec::new(), next_cursor: 0, terminal: Some(workdir::CommandTerminal { + process_failure: None, + output_collection: workdir::CommandOutputCollection::Complete, status: workdir::CommandStatus::Cancelled, exit_code: None, @@ -39239,6 +39243,8 @@ mod tests { chunks: Vec::new(), next_cursor: 0, terminal: Some(workdir::CommandTerminal { + process_failure: None, + output_collection: workdir::CommandOutputCollection::Complete, status: workdir::CommandStatus::Cancelled, exit_code: None, diff --git a/crates/workspace-server/src/server/tests/command_transport_tests.rs b/crates/workspace-server/src/server/tests/command_transport_tests.rs index af9d6416b..e27e764f0 100644 --- a/crates/workspace-server/src/server/tests/command_transport_tests.rs +++ b/crates/workspace-server/src/server/tests/command_transport_tests.rs @@ -39,6 +39,8 @@ fn page(cursor: u64, bytes: &[u8], terminal: bool) -> workdir::CommandOutput { }, next_cursor: cursor + bytes.len() as u64, terminal: terminal.then_some(workdir::CommandTerminal { + process_failure: None, + output_collection: workdir::CommandOutputCollection::Complete, status: workdir::CommandStatus::Cancelled, exit_code: None, stdout_bytes: cursor + bytes.len() as u64, From a1748b552c52fd5f68de6ba852421fd778aec010 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 05:52:48 +0900 Subject: [PATCH 05/46] fix: validate closed-session diagnostics and fault quotas --- crates/tools/src/bash_output.rs | 37 +++++++++ .../src/local/command_transport/tests.rs | 83 +++++++++++++++++++ crates/worker-runtime/src/http_server.rs | 6 +- 3 files changed, 125 insertions(+), 1 deletion(-) diff --git a/crates/tools/src/bash_output.rs b/crates/tools/src/bash_output.rs index fad3b02e9..6e9db7783 100644 --- a/crates/tools/src/bash_output.rs +++ b/crates/tools/src/bash_output.rs @@ -612,4 +612,41 @@ mod tests { Some(workdir::CommandOutputTruncation::ByteBudget) ); } + + #[test] + fn incomplete_collection_with_quota_retains_both_causes_without_claiming_complete_output() { + let root = BashOutputView::new(); + let mut writer = root.begin().unwrap(); + writer.append(CommandStream::Stdout, b"abc"); + let terminal: workdir::CommandTerminal = serde_json::from_value(serde_json::json!({ + "status":"completed", "exit_code":0, "process_failure":null, + "output_collection":{"state":"incomplete","failure":{"stage":"stdout_drain","reason":"other","os_code":null}}, + "stdout_bytes":9, "stderr_bytes":0, "stdout_delivered_bytes":3, "stderr_delivered_bytes":0, + "truncation_reason":"byte_budget" + })).unwrap(); + let metadata = writer.finish_transport(&terminal).unwrap(); + assert!(!metadata.terminal.transport_complete); + assert_eq!(metadata.terminal.exit_code, Some(0)); + assert_eq!( + metadata.terminal.output_collection, + Some(terminal.output_collection) + ); + assert_eq!( + metadata.transport_truncation, + Some(workdir::CommandOutputTruncation::ByteBudget) + ); + assert_eq!(metadata.stdout_seen, 9); + assert_eq!(metadata.stdout_saved, 3); + assert!(metadata.storage_truncated); + assert!(metadata.retention_limit.is_none()); + assert_eq!( + root.read(&metadata.id, CommandStream::Stdout, 0, 100) + .unwrap(), + b"abc" + ); + assert_eq!( + serde_json::to_value(root.metadata(&metadata.id).unwrap()).unwrap(), + serde_json::to_value(metadata).unwrap() + ); + } } diff --git a/crates/workdir/src/local/command_transport/tests.rs b/crates/workdir/src/local/command_transport/tests.rs index 324ce614e..647e1db16 100644 --- a/crates/workdir/src/local/command_transport/tests.rs +++ b/crates/workdir/src/local/command_transport/tests.rs @@ -783,3 +783,86 @@ async fn spawn_fault_retains_safe_reason_without_claiming_output_collection() { #[cfg(feature = "http-client")] mod http; + +#[tokio::test] +async fn drain_failure_and_byte_budget_keep_independent_loss_diagnostics() { + for stream in [CommandStream::Stdout, CommandStream::Stderr] { + let ready = Gate::default(); + let exited = Gate::default(); + let process = FaultProcess { + ready: ready.clone(), + exited: exited.clone(), + wait_error: None, + }; + let reader = FaultReader { + prefix: b"partial", + prefix_read: ready, + fail_ready: exited, + }; + let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS); + let (_cancel, cancel_rx) = watch::channel(false); + let telemetry = CommandTelemetry::new(); + let terminal = tokio::time::timeout(Duration::from_secs(5), async { + match stream { + CommandStream::Stdout => { + supervise( + process, + reader, + &b""[..], + 10, + 3, + "quota-fault", + &telemetry, + cancel_rx, + &sender, + ) + .await + } + CommandStream::Stderr => { + supervise( + process, + &b""[..], + reader, + 10, + 3, + "quota-fault", + &telemetry, + cancel_rx, + &sender, + ) + .await + } + } + }) + .await + .unwrap(); + let Message::Bytes(chunk) = receiver.recv().await.unwrap(); + assert_eq!(chunk.stream, stream); + assert_eq!(chunk.bytes, b"par"); + assert!(receiver.try_recv().is_err()); + assert_eq!(terminal.exit_code, Some(0)); + assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7); + assert_eq!( + terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes, + 3 + ); + assert_eq!( + terminal.truncation_reason, + Some(crate::CommandOutputTruncation::ByteBudget) + ); + assert_eq!( + terminal.output_collection, + CommandOutputCollection::Incomplete { + failure: CommandFailure { + stage: match stream { + CommandStream::Stdout => CommandFailureStage::StdoutDrain, + CommandStream::Stderr => CommandFailureStage::StderrDrain, + }, + reason: CommandIoErrorKind::PermissionDenied, + os_code: None, + }, + } + ); + assert!(terminal.metadata_valid()); + } +} diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index a1faefc73..aefc66973 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -3206,9 +3206,13 @@ mod tests { .await .expect_err("close must be terminal"); assert!(matches!( - closed_error, + closed_error.classification_source(), workdir::WorkdirError::Unavailable(_) )); + assert_eq!( + closed_error.failure_reason(), + Some(workdir::WorkdirFailureReason::SessionClosed) + ); assert!( state .workdir_sessions From 6810f51f460ac0027628156bab5945423c148c3e Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 05:53:58 +0900 Subject: [PATCH 06/46] fix: retain safe Drive and config adapter failure diagnostics --- crates/worker/src/feature/builtin.rs | 1 + .../src/feature/builtin/backend_diagnostic.rs | 195 ++++++ .../src/feature/builtin/drive/backend.rs | 237 +++++-- .../feature/builtin/drive/backend_tests.rs | 644 ++++++++++++++++++ .../builtin/workspace_config/backend.rs | 117 +++- .../builtin/workspace_config/backend_tests.rs | 580 ++++++++++++++++ .../builtin/workspace_config/content.rs | 74 +- .../workspace_config/production_tests.rs | 161 ++++- .../feature/builtin/workspace_config/wip.rs | 9 +- 9 files changed, 1947 insertions(+), 71 deletions(-) create mode 100644 crates/worker/src/feature/builtin/backend_diagnostic.rs create mode 100644 crates/worker/src/feature/builtin/drive/backend_tests.rs create mode 100644 crates/worker/src/feature/builtin/workspace_config/backend_tests.rs diff --git a/crates/worker/src/feature/builtin.rs b/crates/worker/src/feature/builtin.rs index dac21d1d5..26b4d9a4e 100644 --- a/crates/worker/src/feature/builtin.rs +++ b/crates/worker/src/feature/builtin.rs @@ -4,6 +4,7 @@ //! same descriptor-approved registry path used by feature modules. They are not //! an external plugin-loading surface. +mod backend_diagnostic; pub mod backend_job_result; pub(crate) mod bash; diff --git a/crates/worker/src/feature/builtin/backend_diagnostic.rs b/crates/worker/src/feature/builtin/backend_diagnostic.rs new file mode 100644 index 000000000..ac7d7db44 --- /dev/null +++ b/crates/worker/src/feature/builtin/backend_diagnostic.rs @@ -0,0 +1,195 @@ +//! Internal adapter diagnostics are independent of public failure classifications. +//! +//! Only closed vocabularies, numeric status and opaque correlation tokens cross this boundary. +//! In particular, a transport error's Display/Debug and server-controlled text are never recorded. +use sha2::{Digest, Sha256}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Adapter { + Drive, + WorkspaceConfig, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Stage { + Request, + Response, + Completion, + Reconcile, + Attach, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Reason { + Deadline, + TaskJoin, + TransportUnavailable, + InvalidPath, + Communication, + Decode, + RequestSize, + ResponseSize, + ResponseLength, + Identity, + InvalidMetadata, + BackendRejected, + AccessCeiling, + AttachConversion, + BackendUnknown, + UnrecognizedResponse, + NotCommittedSnapshot, + IncompleteReceipt, + Reconciled, +} + +/// One logical call, including its original-request reconciliation. Request identities are hashed +/// rather than logged verbatim: callers and responses need not contain safe log text. +#[derive(Clone)] +pub(super) struct Diagnostic { + adapter: Adapter, + operation: &'static str, + mutation: bool, + correlation: String, + dispatch: tracing::Dispatch, + stage_override: Option, +} + +impl Diagnostic { + pub(super) fn new( + adapter: Adapter, + operation: &'static str, + mutation: bool, + request_id: Option<&str>, + ) -> Self { + Self { + adapter, + operation, + mutation, + correlation: request_id + .map(|id| { + let hash: String = Sha256::digest(id.as_bytes()) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + format!("request:{hash}") + }) + .unwrap_or_else(|| uuid::Uuid::now_v7().to_string()), + dispatch: tracing::dispatcher::get_default(Clone::clone), + stage_override: None, + } + } + + pub(super) fn at_stage(&self, stage: Stage) -> Self { + let mut diagnostic = self.clone(); + diagnostic.stage_override = Some(stage); + diagnostic + } + + pub(super) fn record(&self, stage: Stage, reason: Reason, status: Option) { + let stage = self.stage_override.unwrap_or(stage); + tracing::dispatcher::with_default(&self.dispatch, || { + tracing::warn!( + target: "worker::backend_diagnostic", + adapter = ?self.adapter, + operation = self.operation, + mutation = self.mutation, + correlation = %self.correlation, + stage = ?stage, + reason = ?reason, + status, + "Workspace adapter diagnostic" + ); + }); + } + + pub(super) fn transport(&self, error: &crate::worker::WorkspaceClientError) { + use crate::worker::WorkspaceClientError; + self.record( + Stage::Request, + match error { + WorkspaceClientError::Unavailable(_) => Reason::TransportUnavailable, + WorkspaceClientError::InvalidPath(_) => Reason::InvalidPath, + WorkspaceClientError::Request(_) => Reason::Communication, + }, + None, + ); + } +} + +#[cfg(test)] +pub(super) mod tests { + use super::{Adapter, Diagnostic, Reason, Stage}; + use std::sync::{Arc, Mutex}; + use tracing::{Event, Metadata, Subscriber, field::Visit, span}; + + #[test] + fn unsafe_request_identity_is_correlated_without_logging_its_value() { + let capture = Capture::default(); + tracing::dispatcher::with_default(&capture.dispatch(), || { + for id in [ + "secret /host/path\ncredential", + "secret /host/path\ncredential", + "different", + ] { + Diagnostic::new(Adapter::Drive, "mutate", true, Some(id)).record( + Stage::Request, + Reason::Communication, + None, + ); + } + }); + let text = capture.text(); + for secret in ["secret", "/host/path", "credential", "different"] { + assert!(!text.contains(secret)); + } + let correlations: Vec<_> = text + .lines() + .map(|line| { + line.split("correlation=") + .nth(1) + .unwrap() + .split(';') + .next() + .unwrap() + }) + .collect(); + assert_eq!(correlations.len(), 3); + assert_eq!(correlations[0], correlations[1]); + assert_ne!(correlations[0], correlations[2]); + } + + #[derive(Clone, Default)] + pub(crate) struct Capture(pub(crate) Arc>>); + impl Capture { + pub(crate) fn dispatch(&self) -> tracing::Dispatch { + tracing::Dispatch::new(self.clone()) + } + pub(crate) fn text(&self) -> String { + self.0.lock().unwrap().join("\n") + } + } + struct Fields(String); + impl Visit for Fields { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + use std::fmt::Write; + write!(&mut self.0, "{}={value:?};", field.name()).unwrap(); + } + } + impl Subscriber for Capture { + fn enabled(&self, metadata: &Metadata<'_>) -> bool { + metadata.target() == "worker::backend_diagnostic" + } + fn new_span(&self, _: &span::Attributes<'_>) -> span::Id { + span::Id::from_u64(1) + } + fn record(&self, _: &span::Id, _: &span::Record<'_>) {} + fn record_follows_from(&self, _: &span::Id, _: &span::Id) {} + fn event(&self, event: &Event<'_>) { + let mut fields = Fields(String::new()); + event.record(&mut fields); + self.0.lock().unwrap().push(fields.0); + } + fn enter(&self, _: &span::Id) {} + fn exit(&self, _: &span::Id) {} + } +} diff --git a/crates/worker/src/feature/builtin/drive/backend.rs b/crates/worker/src/feature/builtin/drive/backend.rs index 4c799d1ef..5730dcf0a 100644 --- a/crates/worker/src/feature/builtin/drive/backend.rs +++ b/crates/worker/src/feature/builtin/drive/backend.rs @@ -11,6 +11,8 @@ //! cases retain an unknown outcome with the original request ID. Never blindly resend with a new ID. use std::{sync::Arc, time::Duration}; +use super::super::backend_diagnostic::{Adapter, Diagnostic, Reason, Stage}; + use crate::worker::{ WorkspaceBinaryRequest, WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod, }; @@ -23,6 +25,7 @@ const RESPONSE_LIMIT: usize = 2 * 1024 * 1024; #[derive(Clone)] pub struct DriveBackend { client: Arc, + deadline: Duration, } #[derive(Debug, Clone, thiserror::Error)] @@ -49,7 +52,10 @@ pub enum DriveError { impl DriveBackend { pub fn new(client: Arc) -> Self { - Self { client } + Self { + client, + deadline: DEADLINE, + } } pub fn workspace(&self) -> Result<&str, DriveError> { self.client @@ -61,15 +67,28 @@ impl DriveBackend { Ok(format!("/api/w/{}/drive", encode(self.workspace()?))) } pub(super) fn validate_entry(&self, entry: &DriveEntry) -> Result<(), DriveError> { - self.validate_ref(&entry.entry)?; + self.validate_entry_at( + entry, + &Diagnostic::new(Adapter::Drive, "validate_entry", false, None), + ) + } + fn validate_entry_at( + &self, + entry: &DriveEntry, + diagnostic: &Diagnostic, + ) -> Result<(), DriveError> { + self.validate_ref(&entry.entry) + .inspect_err(|_| diagnostic.record(Stage::Completion, Reason::Identity, None))?; if let Some(parent) = &entry.parent { - self.validate_ref(parent)?; + self.validate_ref(parent) + .inspect_err(|_| diagnostic.record(Stage::Completion, Reason::Identity, None))?; } if (entry.last_mutation_id.len() > 128 || entry.last_mutation_id.chars().any(char::is_control) || (entry.parent.is_some() && entry.last_mutation_id.is_empty())) || entry.size.is_some_and(|n| n > DRIVE_FILE_MAX_BYTES) { + diagnostic.record(Stage::Completion, Reason::InvalidMetadata, None); return Err(DriveError::Unavailable); } Ok(()) @@ -90,11 +109,13 @@ impl DriveBackend { request: WorkspaceRequest, mutation: bool, request_id: &str, + diagnostic: &Diagnostic, ) -> Result { let client = self.client.clone(); + let deadline = self.deadline; let response = tokio::time::timeout( - DEADLINE, - tokio::task::spawn_blocking(move || client.execute_with_timeout(request, DEADLINE)), + deadline, + tokio::task::spawn_blocking(move || client.execute_with_timeout(request, deadline)), ) .await; let unknown = || { @@ -105,10 +126,20 @@ impl DriveBackend { } }; let response = response - .map_err(|_| unknown())? - .map_err(|_| unknown())? - .map_err(|_| unknown())?; + .map_err(|_| { + diagnostic.record(Stage::Request, Reason::Deadline, None); + unknown() + })? + .map_err(|_| { + diagnostic.record(Stage::Request, Reason::TaskJoin, None); + unknown() + })? + .map_err(|error| { + diagnostic.transport(&error); + unknown() + })?; if response.body.len() > RESPONSE_LIMIT { + diagnostic.record(Stage::Response, Reason::ResponseSize, None); return Err(if mutation { unknown() } else { @@ -121,42 +152,57 @@ impl DriveBackend { response.body.as_bytes(), mutation, request_id, + diagnostic, )); } - serde_json::from_str(&response.body).map_err(|_| unknown()) + serde_json::from_str(&response.body).map_err(|_| { + diagnostic.record(Stage::Response, Reason::Decode, None); + unknown() + }) } async fn get( &self, - suffix: &str, + suffix: &'static str, query: &Q, + diagnostic: &Diagnostic, ) -> Result { let query = query_string(query)?; self.json( WorkspaceRequest::get(format!("{}{suffix}{query}", self.base()?)), false, "", + diagnostic, ) .await } pub async fn capabilities(&self) -> Result { - self.get("/capabilities", &serde_json::json!({})).await + self.get( + "/capabilities", + &serde_json::json!({}), + &Diagnostic::new(Adapter::Drive, "capabilities", false, None), + ) + .await } pub async fn root(&self) -> Result { + let diagnostic = Diagnostic::new(Adapter::Drive, "root", false, None); let entry: DriveEntry = self .json( WorkspaceRequest::get(format!("{}/root", self.base()?)), false, "", + &diagnostic, ) .await?; - self.validate_entry(&entry)?; + self.validate_entry_at(&entry, &diagnostic)?; if entry.kind != DriveEntryKind::Folder || entry.parent.is_some() { + diagnostic.record(Stage::Completion, Reason::InvalidMetadata, None); return Err(DriveError::Unavailable); } Ok(entry) } pub async fn metadata(&self, entry: &DriveEntryRef) -> Result { self.validate_ref(entry)?; + let diagnostic = Diagnostic::new(Adapter::Drive, "metadata", false, None); let result: DriveEntry = self .get( "/metadata", @@ -164,10 +210,12 @@ impl DriveBackend { entry_workspace_id: entry.workspace_id.clone(), id: entry.node_id.clone(), }, + &diagnostic, ) .await?; - self.validate_entry(&result)?; + self.validate_entry_at(&result, &diagnostic)?; if result.entry != *entry { + diagnostic.record(Stage::Completion, Reason::Identity, None); return Err(DriveError::Unavailable); } Ok(result) @@ -179,6 +227,7 @@ impl DriveBackend { after: Option, ) -> Result { self.validate_ref(parent)?; + let diagnostic = Diagnostic::new(Adapter::Drive, "list", false, None); let result: DriveListResponse = self .get( "/list", @@ -188,6 +237,7 @@ impl DriveBackend { limit, after, }, + &diagnostic, ) .await?; self.validate_page( @@ -195,33 +245,43 @@ impl DriveBackend { limit .unwrap_or(DRIVE_PAGE_MAX_LIMIT) .min(DRIVE_PAGE_MAX_LIMIT), + &diagnostic, )?; if result .entries .iter() .any(|entry| entry.parent.as_ref() != Some(parent)) { + diagnostic.record(Stage::Completion, Reason::Identity, None); return Err(DriveError::Unavailable); } Ok(result) } pub async fn search(&self, query: DriveSearchQuery) -> Result { - let result: DriveListResponse = self.get("/search", &query).await?; + let diagnostic = Diagnostic::new(Adapter::Drive, "search", false, None); + let result: DriveListResponse = self.get("/search", &query, &diagnostic).await?; self.validate_page( &result, query .limit .unwrap_or(DRIVE_SEARCH_MAX_LIMIT) .min(DRIVE_SEARCH_MAX_LIMIT), + &diagnostic, )?; Ok(result) } - fn validate_page(&self, page: &DriveListResponse, limit: u32) -> Result<(), DriveError> { + fn validate_page( + &self, + page: &DriveListResponse, + limit: u32, + diagnostic: &Diagnostic, + ) -> Result<(), DriveError> { if page.entries.len() > limit as usize { + diagnostic.record(Stage::Completion, Reason::ResponseSize, None); return Err(DriveError::Limit); } for entry in &page.entries { - self.validate_entry(entry)?; + self.validate_entry_at(entry, diagnostic)?; } Ok(()) } @@ -231,6 +291,7 @@ impl DriveBackend { max_bytes: u32, ) -> Result { self.validate_ref(entry)?; + let diagnostic = Diagnostic::new(Adapter::Drive, "read_text", false, None); let result: DriveReadTextResponse = self .get( "/read-text", @@ -239,10 +300,16 @@ impl DriveBackend { id: entry.node_id.clone(), max_bytes, }, + &diagnostic, ) .await?; - self.validate_entry(&result.entry)?; - if result.entry.entry != *entry || result.text.len() > max_bytes as usize { + self.validate_entry_at(&result.entry, &diagnostic)?; + if result.entry.entry != *entry { + diagnostic.record(Stage::Completion, Reason::Identity, None); + return Err(DriveError::Unavailable); + } + if result.text.len() > max_bytes as usize { + diagnostic.record(Stage::Completion, Reason::ResponseSize, None); return Err(DriveError::Unavailable); } Ok(result) @@ -255,6 +322,7 @@ impl DriveBackend { if total as usize > limit { return Err(DriveError::Limit); } + let diagnostic = Diagnostic::new(Adapter::Drive, "read_chunk", false, None); let mut bytes = Vec::with_capacity(total as usize); // Even an empty file gets a read bound to the observed committed mutation and current authorization. loop { @@ -274,21 +342,42 @@ impl DriveBackend { max_response_bytes: DRIVE_CHUNK_MAX_BYTES as usize, }; let client = self.client.clone(); + let deadline = self.deadline; let response = tokio::time::timeout( - DEADLINE, + deadline, tokio::task::spawn_blocking(move || { - client.execute_binary_with_timeout(request, DEADLINE) + client.execute_binary_with_timeout(request, deadline) }), ) .await - .map_err(|_| DriveError::Unavailable)? - .map_err(|_| DriveError::Unavailable)? - .map_err(|_| DriveError::Unavailable)?; + .map_err(|_| { + diagnostic.record(Stage::Request, Reason::Deadline, None); + DriveError::Unavailable + })? + .map_err(|_| { + diagnostic.record(Stage::Request, Reason::TaskJoin, None); + DriveError::Unavailable + })? + .map_err(|error| { + diagnostic.transport(&error); + DriveError::Unavailable + })?; + if response.body.len() > DRIVE_CHUNK_MAX_BYTES as usize { + diagnostic.record(Stage::Response, Reason::ResponseSize, None); + return Err(DriveError::Unavailable); + } if !(200..300).contains(&response.status) { - return Err(classify(response.status, &response.body, false, "")); + return Err(classify( + response.status, + &response.body, + false, + "", + &diagnostic, + )); } let expected = total.saturating_sub(offset).min(length) as usize; if response.body.len() != expected { + diagnostic.record(Stage::Response, Reason::ResponseLength, None); return Err(DriveError::Unavailable); } bytes.extend_from_slice(&response.body); @@ -301,22 +390,40 @@ impl DriveBackend { &self, id: &str, result: Result, + diagnostic: &Diagnostic, ) -> Result { // A parseable success can still be an invalid completion. Classify it // before reconciliation so it also gets one original-request inquiry. let result = result.and_then(|response| { - self.validate_completion(id, &response) + self.validate_completion(id, &response, diagnostic) .map_err(|_| DriveError::OutcomeUnknown(id.into()))?; Ok(response) }); match result { - Err(DriveError::OutcomeUnknown(_)) => match self.status(id).await { + Err(DriveError::OutcomeUnknown(_)) => match self + .status_with_diagnostic(id, &diagnostic.at_stage(Stage::Reconcile)) + .await + { Ok(DriveRequestStatusResponse { state: DriveRequestState::Committed, response: Some(response), .. - }) => Ok(response), // status validates both IDs and entry metadata - _ => Err(DriveError::OutcomeUnknown(id.into())), + }) => { + diagnostic.record(Stage::Reconcile, Reason::Reconciled, None); + Ok(response) // status validates both IDs and entry metadata + } + Ok(status) => { + diagnostic.record( + Stage::Reconcile, + match status.state { + DriveRequestState::Uncommitted => Reason::NotCommittedSnapshot, + DriveRequestState::Committed => Reason::IncompleteReceipt, + }, + None, + ); + Err(DriveError::OutcomeUnknown(id.into())) + } + Err(_) => Err(DriveError::OutcomeUnknown(id.into())), }, other => other, } @@ -325,12 +432,14 @@ impl DriveBackend { &self, id: &str, response: &DriveMutationResponse, + diagnostic: &Diagnostic, ) -> Result<(), DriveError> { if response.request_id != id { + diagnostic.record(Stage::Completion, Reason::Identity, None); return Err(DriveError::Unavailable); } if let Some(entry) = &response.entry { - self.validate_entry(entry)?; + self.validate_entry_at(entry, diagnostic)?; } Ok(()) } @@ -338,9 +447,11 @@ impl DriveBackend { &self, request: DriveMutationRequest, ) -> Result { + let diagnostic = Diagnostic::new(Adapter::Drive, "mutate", true, Some(&request.request_id)); let body = serde_json::to_string(&request) .map_err(|_| DriveError::Invalid("invalid mutation".into()))?; if body.len() > 512 * 1024 { + diagnostic.record(Stage::Request, Reason::RequestSize, None); return Err(DriveError::Limit); } let result = self @@ -352,16 +463,20 @@ impl DriveBackend { ), true, &request.request_id, + &diagnostic, ) .await; - self.reconcile(&request.request_id, result).await + self.reconcile(&request.request_id, result, &diagnostic) + .await } pub async fn upload( &self, query: DriveUploadQuery, bytes: Vec, ) -> Result { + let diagnostic = Diagnostic::new(Adapter::Drive, "upload", true, Some(&query.request_id)); if bytes.len() > DRIVE_FILE_MAX_BYTES as usize { + diagnostic.record(Stage::Request, Reason::RequestSize, None); return Err(DriveError::Limit); } let path = format!("{}/upload{}", self.base()?, query_string(&query)?); @@ -372,29 +487,56 @@ impl DriveBackend { max_response_bytes: RESPONSE_LIMIT, }; let client = self.client.clone(); + let deadline = self.deadline; let result = tokio::time::timeout( - DEADLINE, + deadline, tokio::task::spawn_blocking(move || { - client.execute_binary_with_timeout(request, DEADLINE) + client.execute_binary_with_timeout(request, deadline) }), ) .await; let result = match result { + Ok(Ok(Ok(response))) if response.body.len() > RESPONSE_LIMIT => { + diagnostic.record(Stage::Response, Reason::ResponseSize, None); + Err(DriveError::OutcomeUnknown(query.request_id.clone())) + } Ok(Ok(Ok(response))) if (200..300).contains(&response.status) => { - serde_json::from_slice(&response.body) - .map_err(|_| DriveError::OutcomeUnknown(query.request_id.clone())) + serde_json::from_slice(&response.body).map_err(|_| { + diagnostic.record(Stage::Response, Reason::Decode, None); + DriveError::OutcomeUnknown(query.request_id.clone()) + }) } Ok(Ok(Ok(response))) => Err(classify( response.status, &response.body, true, &query.request_id, + &diagnostic, )), - _ => Err(DriveError::OutcomeUnknown(query.request_id.clone())), + Err(_) => { + diagnostic.record(Stage::Request, Reason::Deadline, None); + Err(DriveError::OutcomeUnknown(query.request_id.clone())) + } + Ok(Err(_)) => { + diagnostic.record(Stage::Request, Reason::TaskJoin, None); + Err(DriveError::OutcomeUnknown(query.request_id.clone())) + } + Ok(Ok(Err(error))) => { + diagnostic.transport(&error); + Err(DriveError::OutcomeUnknown(query.request_id.clone())) + } }; - self.reconcile(&query.request_id, result).await + self.reconcile(&query.request_id, result, &diagnostic).await } pub async fn status(&self, request_id: &str) -> Result { + let diagnostic = Diagnostic::new(Adapter::Drive, "request_status", false, Some(request_id)); + self.status_with_diagnostic(request_id, &diagnostic).await + } + async fn status_with_diagnostic( + &self, + request_id: &str, + diagnostic: &Diagnostic, + ) -> Result { if request_id.is_empty() || request_id.len() > 256 { return Err(DriveError::Invalid("invalid request ID".into())); } @@ -403,13 +545,15 @@ impl DriveBackend { WorkspaceRequest::get(format!("{}/requests/{}", self.base()?, encode(request_id))), false, "", + diagnostic, ) .await?; if response.request_id != request_id { + diagnostic.record(Stage::Reconcile, Reason::Identity, None); return Err(DriveError::Unavailable); } if let Some(result) = &response.response { - self.validate_completion(request_id, result)?; + self.validate_completion(request_id, result, diagnostic)?; } Ok(response) } @@ -463,11 +607,21 @@ fn query_string(value: &T) -> Result { format!("?{}", pairs.join("&")) }) } -fn classify(status: u16, body: &[u8], mutation: bool, id: &str) -> DriveError { +fn classify( + status: u16, + body: &[u8], + mutation: bool, + id: &str, + diagnostic: &Diagnostic, +) -> DriveError { if let Ok(error) = serde_json::from_slice::(body) { - if error.classification == DriveFailureClassification::Unknown { + if error.classification == DriveFailureClassification::Unknown + || error.code == DriveApiErrorCode::OutcomeUnknown + { + diagnostic.record(Stage::Response, Reason::BackendUnknown, Some(status)); return DriveError::OutcomeUnknown(id.into()); } + diagnostic.record(Stage::Response, Reason::BackendRejected, Some(status)); return match error.code { DriveApiErrorCode::Denied => DriveError::Denied, DriveApiErrorCode::NotFound => DriveError::NotFound, @@ -478,6 +632,7 @@ fn classify(status: u16, body: &[u8], mutation: bool, id: &str) -> DriveError { DriveApiErrorCode::OutcomeUnknown => DriveError::OutcomeUnknown(id.into()), }; } + diagnostic.record(Stage::Response, Reason::UnrecognizedResponse, Some(status)); if matches!(status, 401 | 403) { DriveError::Denied } else if mutation { @@ -486,3 +641,7 @@ fn classify(status: u16, body: &[u8], mutation: bool, id: &str) -> DriveError { DriveError::Unavailable } } + +#[cfg(test)] +#[path = "backend_tests.rs"] +mod tests; diff --git a/crates/worker/src/feature/builtin/drive/backend_tests.rs b/crates/worker/src/feature/builtin/drive/backend_tests.rs new file mode 100644 index 000000000..46109b3cc --- /dev/null +++ b/crates/worker/src/feature/builtin/drive/backend_tests.rs @@ -0,0 +1,644 @@ +//! Production Drive adapter fault injection. Only the Workspace transport boundary is substituted; +//! retained fixture effects and original-request inquiries are inspected independently of replies. +//! The HTTP case additionally connects the adapter to reqwest and an isolated loopback router. +use super::*; +use crate::feature::builtin::backend_diagnostic::{Reason, tests::Capture}; +use crate::worker::{WorkspaceBinaryResponse, WorkspaceClientError, WorkspaceResponse}; +use std::sync::{Condvar, Mutex}; +use tracing::instrument::WithSubscriber; + +const SECRET: &str = "secret-token /private/drive credential=unsafe"; +const ID: &str = "host-request-1"; + +fn entry() -> DriveEntry { + DriveEntry { + entry: DriveEntryRef { + workspace_id: "workspace".into(), + node_id: "2".into(), + }, + parent: Some(DriveEntryRef { + workspace_id: "workspace".into(), + node_id: "1".into(), + }), + name: SECRET.into(), + kind: DriveEntryKind::File, + last_mutation_id: "old-request".into(), + size: Some(5), + content_type: Some("text/plain".into()), + updated_by: SECRET.into(), + updated_at: SECRET.into(), + latest_url: SECRET.into(), + } +} +fn mutation() -> DriveMutationRequest { + DriveMutationRequest { + request_id: ID.into(), + mutation: DriveMutation::UpdateText { + id: entry().entry, + expected_mutation_id: "old-request".into(), + text: SECRET.into(), + content_type: "text/plain".into(), + }, + } +} +fn json(value: impl Serialize) -> WorkspaceResponse { + WorkspaceResponse { + status: 200, + body: serde_json::to_string(&value).unwrap(), + } +} +#[derive(Clone, Copy, Debug)] +enum Fault { + Communication, + Unavailable, + InvalidPath, + Decode, + Size, + Length, + Identity, + Metadata, + Rejected, + Unknown, + Unrecognized, + Incomplete, + Panic, + Deadline, + None, +} +#[derive(Debug, Default)] +struct State { + mutations: usize, + saved: Option, + requests: Vec, + binary: Vec, + released: bool, + started: bool, + finished: bool, +} +#[derive(Debug)] +struct Client { + fault: Fault, + receipt: Fault, + state: Mutex, + changed: Condvar, + started: tokio::sync::Notify, +} +impl Client { + fn new(fault: Fault, receipt: Fault) -> Arc { + Arc::new(Self { + fault, + receipt, + state: Mutex::new(State::default()), + changed: Condvar::new(), + started: tokio::sync::Notify::new(), + }) + } + fn fail( + &self, + fault: Fault, + success: WorkspaceResponse, + ) -> Result { + match fault { + Fault::Communication => Err(WorkspaceClientError::Request(SECRET.into())), + Fault::Unavailable => Err(WorkspaceClientError::Unavailable(SECRET.into())), + Fault::InvalidPath => Err(WorkspaceClientError::InvalidPath(SECRET.into())), + Fault::Panic => panic!("injected adapter task panic"), + Fault::Decode => Ok(WorkspaceResponse { + status: 200, + body: SECRET.into(), + }), + Fault::Size => Ok(WorkspaceResponse { + status: 200, + body: SECRET.repeat(RESPONSE_LIMIT / SECRET.len() + 1), + }), + Fault::Rejected | Fault::Unknown => Ok(WorkspaceResponse { + status: 409, + body: serde_json::to_string(&DriveApiError { + code: DriveApiErrorCode::Conflict, + message: SECRET.into(), + classification: if matches!(fault, Fault::Unknown) { + DriveFailureClassification::Unknown + } else { + DriveFailureClassification::NotCommitted + }, + }) + .unwrap(), + }), + Fault::Unrecognized => Ok(WorkspaceResponse { + status: 503, + body: SECRET.into(), + }), + _ => Ok(success), + } + } + fn release(&self) { + self.state.lock().unwrap().released = true; + self.changed.notify_all(); + } + fn wait(&self, finished: bool) { + let (state, timeout) = self + .changed + .wait_timeout_while(self.state.lock().unwrap(), Duration::from_secs(5), |s| { + if finished { !s.finished } else { !s.started } + }) + .unwrap(); + assert!(!timeout.timed_out()); + assert!(if finished { + state.finished + } else { + state.started + }); + } +} +struct Release(Arc); +impl Drop for Release { + fn drop(&mut self) { + self.0.release(); + } +} +impl WorkspaceClient for Client { + fn workspace_id(&self) -> Option<&str> { + Some("workspace") + } + fn kind(&self) -> &str { + "drive-diagnostic-fixture" + } + fn is_available(&self) -> bool { + true + } + fn execute( + &self, + request: WorkspaceRequest, + ) -> Result { + let is_status = request.path.contains("/requests/"); + let fault = if is_status { self.receipt } else { self.fault }; + let mutation = request.method == WorkspaceRequestMethod::Post; + { + let mut state = self.state.lock().unwrap(); + if mutation + && !matches!( + fault, + Fault::Rejected | Fault::InvalidPath | Fault::Unavailable + ) + { + let input: DriveMutationRequest = + serde_json::from_str(request.body.as_ref().unwrap()).unwrap(); + assert_eq!(input, super::tests::mutation()); + state.mutations += 1; + if let DriveMutation::UpdateText { text, .. } = input.mutation { + state.saved = Some(text); + } + } + state.requests.push(request); + state.started = true; + self.started.notify_one(); + self.changed.notify_all(); + if matches!(fault, Fault::Deadline) { + let (_state, timeout) = self + .changed + .wait_timeout_while(state, Duration::from_secs(5), |s| !s.released) + .unwrap(); + assert!( + !timeout.timed_out(), + "transport must be explicitly released" + ); + } + } + let mut metadata = entry(); + if matches!(fault, Fault::Identity) { + metadata.entry.node_id = "3".into(); + } + if matches!(fault, Fault::Metadata) { + metadata.size = Some(DRIVE_FILE_MAX_BYTES + 1); + } + let completion = DriveMutationResponse { + request_id: if matches!(fault, Fault::Identity) { + "other-request" + } else { + ID + } + .into(), + entry: Some(metadata.clone()), + }; + let value = if is_status { + json(DriveRequestStatusResponse { + request_id: if matches!(fault, Fault::Identity) { + "other-request" + } else { + ID + } + .into(), + state: if matches!( + fault, + Fault::None | Fault::Identity | Fault::Metadata | Fault::Incomplete + ) { + DriveRequestState::Committed + } else { + DriveRequestState::Uncommitted + }, + response: if matches!(fault, Fault::None | Fault::Identity | Fault::Metadata) { + Some(completion) + } else { + None + }, + }) + } else if mutation { + json(completion) + } else { + json(metadata) + }; + if !is_status { + self.state.lock().unwrap().finished = true; + } + self.changed.notify_all(); + self.fail(fault, value) + } + fn execute_binary( + &self, + request: WorkspaceBinaryRequest, + ) -> Result { + let upload = request.method == WorkspaceRequestMethod::Put; + { + let mut state = self.state.lock().unwrap(); + if upload { + state.mutations += 1; + state.saved = Some(SECRET.into()); + } + state.binary.push(request); + } + let success = if upload { + json(DriveMutationResponse { + request_id: ID.into(), + entry: Some(entry()), + }) + } else { + WorkspaceResponse { + status: 200, + body: if matches!(self.fault, Fault::Length) { + "x" + } else { + "12345" + } + .into(), + } + }; + self.fail(self.fault, success) + .map(|r| WorkspaceBinaryResponse { + status: r.status, + body: r.body.into_bytes(), + }) + } +} +fn safe(capture: &Capture, reason: Reason, public: &str) { + let text = capture.text(); + assert!(text.contains(&format!("reason={reason:?}")), "{text}"); + assert!(text.contains("adapter=Drive"), "{text}"); + assert!(text.contains("correlation="), "{text}"); + for secret in [ + SECRET, + "secret-token", + "/private/drive", + "credential=unsafe", + ] { + assert!(!text.contains(secret), "internal diagnostic leaked content"); + assert!(!public.contains(secret), "public error leaked content"); + } +} +fn reason(fault: Fault) -> Reason { + match fault { + Fault::Communication => Reason::Communication, + Fault::Unavailable => Reason::TransportUnavailable, + Fault::InvalidPath => Reason::InvalidPath, + Fault::Decode => Reason::Decode, + Fault::Size => Reason::ResponseSize, + Fault::Length => Reason::ResponseLength, + Fault::Identity => Reason::Identity, + Fault::Metadata => Reason::InvalidMetadata, + Fault::Rejected => Reason::BackendRejected, + Fault::Unknown => Reason::BackendUnknown, + Fault::Unrecognized => Reason::UnrecognizedResponse, + Fault::Panic => Reason::TaskJoin, + Fault::Deadline => Reason::Deadline, + Fault::Incomplete => Reason::IncompleteReceipt, + Fault::None => panic!("no fault"), + } +} +#[tokio::test] +async fn production_read_failures_retain_public_classification_and_safe_cause() { + for fault in [ + Fault::Communication, + Fault::Unavailable, + Fault::InvalidPath, + Fault::Decode, + Fault::Size, + Fault::Identity, + Fault::Metadata, + Fault::Rejected, + Fault::Unknown, + Fault::Unrecognized, + Fault::Panic, + ] { + let client = Client::new(fault, Fault::None); + let backend = DriveBackend::new(client.clone()); + let capture = Capture::default(); + let error = backend + .metadata(&entry().entry) + .with_subscriber(capture.dispatch()) + .await + .unwrap_err(); + match fault { + Fault::Size => assert!(matches!(error, DriveError::Limit)), + Fault::Rejected => assert!(matches!(error, DriveError::Conflict)), + Fault::Unknown => assert!(matches!(error, DriveError::OutcomeUnknown(_))), + _ => assert!(matches!(error, DriveError::Unavailable)), + } + safe(&capture, reason(fault), &error.to_string()); + let state = client.state.lock().unwrap(); + assert_eq!(state.requests.len(), 1); + assert_eq!(state.mutations, 0); + } +} +#[tokio::test] +async fn production_mutation_loss_reconciles_only_original_identity_and_never_resends() { + for fault in [ + Fault::Communication, + Fault::Decode, + Fault::Size, + Fault::Identity, + Fault::Metadata, + Fault::Unknown, + Fault::Unrecognized, + Fault::Panic, + ] { + for receipt in [ + Fault::None, + Fault::Identity, + Fault::Metadata, + Fault::Decode, + Fault::Communication, + Fault::Unrecognized, + Fault::Length, + Fault::Incomplete, + ] { + let client = Client::new(fault, receipt); + let backend = DriveBackend::new(client.clone()); + let capture = Capture::default(); + let result = backend + .mutate(mutation()) + .with_subscriber(capture.dispatch()) + .await; + if matches!(receipt, Fault::None) { + assert_eq!(result.unwrap().request_id, ID); + } else { + assert!(matches!(result, Err(DriveError::OutcomeUnknown(ref id)) if id == ID)); + } + safe(&capture, reason(fault), ""); + let text = capture.text(); + if matches!(receipt, Fault::Length | Fault::Incomplete) { + safe( + &capture, + if matches!(receipt, Fault::Length) { + Reason::NotCommittedSnapshot + } else { + Reason::IncompleteReceipt + }, + "", + ); + } + assert!(text.contains("stage=Reconcile"), "{text}"); + let correlations: std::collections::BTreeSet<_> = text + .lines() + .filter_map(|l| l.split("correlation=").nth(1)) + .filter_map(|l| l.split(';').next()) + .collect(); + assert_eq!( + correlations.len(), + 1, + "original loss and inquiry must correlate" + ); + assert!( + !text.contains(ID), + "request identity must be opaque in logs" + ); + let state = client.state.lock().unwrap(); + assert_eq!(state.mutations, 1); + assert_eq!(state.saved.as_deref(), Some(SECRET)); + assert_eq!(state.requests.len(), 2); + assert_eq!(state.requests[1].method, WorkspaceRequestMethod::Get); + assert!(state.requests[1].path.ends_with(&format!("/requests/{ID}"))); + } + } +} +#[tokio::test] +async fn production_rejection_has_no_effect_or_status_inquiry() { + let client = Client::new(Fault::Rejected, Fault::None); + let backend = DriveBackend::new(client.clone()); + let capture = Capture::default(); + let error = backend + .mutate(mutation()) + .with_subscriber(capture.dispatch()) + .await + .unwrap_err(); + assert!(matches!(error, DriveError::Conflict)); + safe(&capture, Reason::BackendRejected, &error.to_string()); + let state = client.state.lock().unwrap(); + assert_eq!(state.mutations, 0); + assert_eq!(state.requests.len(), 1); +} +#[tokio::test] +async fn production_binary_length_transport_and_upload_loss_are_diagnosed() { + for fault in [ + Fault::Length, + Fault::Communication, + Fault::Unavailable, + Fault::Panic, + ] { + let client = Client::new(fault, Fault::None); + let backend = DriveBackend::new(client.clone()); + let capture = Capture::default(); + let error = backend + .bytes(&entry(), 5) + .with_subscriber(capture.dispatch()) + .await + .unwrap_err(); + assert!(matches!(error, DriveError::Unavailable)); + safe(&capture, reason(fault), &error.to_string()); + assert_eq!(client.state.lock().unwrap().binary.len(), 1); + } + for fault in [Fault::Decode, Fault::Communication, Fault::Size] { + let client = Client::new(fault, Fault::Unrecognized); + let backend = DriveBackend::new(client.clone()); + let capture = Capture::default(); + let query = DriveUploadQuery { + operation: DriveUploadOperation::Update, + request_id: ID.into(), + entry_workspace_id: "workspace".into(), + parent_id: None, + name: None, + id: Some("2".into()), + expected_mutation_id: Some("old-request".into()), + content_type: "text/plain".into(), + size: 5, + sha256: "hash".into(), + }; + let error = backend + .upload(query, b"12345".to_vec()) + .with_subscriber(capture.dispatch()) + .await + .unwrap_err(); + assert!(matches!(error, DriveError::OutcomeUnknown(ref id) if id == ID)); + safe(&capture, reason(fault), &error.to_string()); + let state = client.state.lock().unwrap(); + assert_eq!(state.mutations, 1); + assert_eq!(state.binary.len(), 1); + assert_eq!(state.requests.len(), 1); + assert!(state.requests[0].path.ends_with(&format!("/requests/{ID}"))); + } +} +#[tokio::test] +async fn production_deadline_does_not_cancel_effect_or_infer_uncommitted_from_snapshot() { + for mutation_call in [false, true] { + let client = Client::new(Fault::Deadline, Fault::Unrecognized); + let release = Release(client.clone()); + let mut backend = DriveBackend::new(client.clone()); + backend.deadline = Duration::from_secs(1); + let capture = Capture::default(); + let request = async { + if mutation_call { + backend.mutate(mutation()).await.map(|_| ()) + } else { + backend.metadata(&entry().entry).await.map(|_| ()) + } + } + .with_subscriber(capture.dispatch()); + tokio::pin!(request); + tokio::select! { + biased; + _ = client.started.notified() => {}, + _ = &mut request => panic!("request must start before timeout"), + } + { + let state = client.state.lock().unwrap(); + assert_eq!(state.mutations, usize::from(mutation_call)); + assert!(!state.released && !state.finished); + } + let result = request.await; + let error = result.unwrap_err(); + assert!(if mutation_call { + matches!(error, DriveError::OutcomeUnknown(ref id) if id == ID) + } else { + matches!(error, DriveError::Unavailable) + }); + safe(&capture, Reason::Deadline, &error.to_string()); + drop(release); + let waiting = client.clone(); + tokio::task::spawn_blocking(move || waiting.wait(true)) + .await + .unwrap(); + let state = client.state.lock().unwrap(); + assert_eq!(state.mutations, usize::from(mutation_call)); + assert_eq!(state.saved.is_some(), mutation_call); + assert_eq!(state.requests.len(), if mutation_call { 2 } else { 1 }); + } +} + +#[tokio::test] +async fn http_reply_loss_keeps_original_receipt_and_retained_effect_without_reexecution() { + use axum::{ + Router, + extract::State as HttpState, + routing::{get, post}, + }; + #[derive(Default)] + struct Stored { + updates: usize, + content: String, + ids: Vec, + } + struct Server(tokio::task::JoinHandle<()>); + impl Drop for Server { + fn drop(&mut self) { + self.0.abort(); + } + } + let stored = Arc::new(Mutex::new(Stored::default())); + let router = Router::new() + .route("/api/w/workspace/drive/metadata", get(|| async { SECRET })) + .route( + "/api/w/workspace/drive/mutate", + post( + |HttpState(state): HttpState>>, + body: axum::body::Bytes| async move { + let request: DriveMutationRequest = serde_json::from_slice(&body).unwrap(); + let mut state = state.lock().unwrap(); + state.ids.push(request.request_id); + if let DriveMutation::UpdateText { + text, + expected_mutation_id, + .. + } = request.mutation + { + assert_eq!(expected_mutation_id, "old-request"); + state.content = text; + state.updates += 1; + } else { + panic!("unexpected mutation"); + } + // The effect is retained; only the success response is malformed. + SECRET + }, + ), + ) + .route( + "/api/w/workspace/drive/requests/{id}", + get( + |HttpState(state): HttpState>>, + axum::extract::Path(id): axum::extract::Path| async move { + let state = state.lock().unwrap(); + assert_eq!(state.ids, [id.clone()]); + axum::Json(DriveRequestStatusResponse { + request_id: id.clone(), + state: DriveRequestState::Committed, + response: Some(DriveMutationResponse { + request_id: id, + entry: Some(entry()), + }), + }) + }, + ), + ) + .with_state(stored.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = Server(tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + })); + let client = Arc::new(crate::worker::TestWorkspaceHttpClient::new( + "workspace", + format!("http://{address}"), + )); + let backend = DriveBackend::new(client); + let capture = Capture::default(); + let error = backend + .metadata(&entry().entry) + .with_subscriber(capture.dispatch()) + .await + .unwrap_err(); + assert!(matches!(error, DriveError::Unavailable)); + safe(&capture, Reason::Decode, &error.to_string()); + let capture = Capture::default(); + let response = backend + .mutate(mutation()) + .with_subscriber(capture.dispatch()) + .await + .unwrap(); + assert_eq!(response.request_id, ID); + safe(&capture, Reason::Decode, ""); + assert!(capture.text().contains("reason=Reconciled")); + let state = stored.lock().unwrap(); + assert_eq!(state.updates, 1); + assert_eq!(state.content, SECRET); + assert_eq!(state.ids, [ID]); + drop(state); + drop(server); +} diff --git a/crates/worker/src/feature/builtin/workspace_config/backend.rs b/crates/worker/src/feature/builtin/workspace_config/backend.rs index f10c60eda..d5d58e376 100644 --- a/crates/worker/src/feature/builtin/workspace_config/backend.rs +++ b/crates/worker/src/feature/builtin/workspace_config/backend.rs @@ -7,6 +7,7 @@ //! undecodable mutation responses do not prove non-commit; dispatched requests are never resent. //! This adapter does not reimplement Backend source-persistence transactions or activation. use std::sync::Arc; +use std::time::Duration; use async_trait::async_trait; use serde::{Serialize, de::DeserializeOwned}; @@ -20,6 +21,7 @@ use super::{ ATTACHMENT_ALIAS, CONTENT_ROOT, ConfigAccess, ConfigAttachError, ConfigAttachment, DEADLINE, WorkspaceConfigAttachmentBackend, }; +use crate::feature::builtin::backend_diagnostic::{Adapter, Diagnostic, Reason, Stage}; use crate::wip::WipOperationError; use crate::worker::{ WorkspaceClient, WorkspaceClientError, WorkspaceRequest, WorkspaceRequestMethod, @@ -34,11 +36,21 @@ const MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; #[derive(Clone)] pub struct WorkspaceConfigBackend { client: Arc, + deadline: Duration, } impl WorkspaceConfigBackend { pub fn new(client: Arc) -> Self { - Self { client } + Self { + client, + deadline: DEADLINE, + } + } + + #[cfg(test)] + pub(super) fn with_deadline(mut self, deadline: Duration) -> Self { + self.deadline = deadline; + self } fn base(&self) -> Result { @@ -47,6 +59,7 @@ impl WorkspaceConfigBackend { .workspace_id() .filter(|id| !id.is_empty()) .ok_or_else(|| { + diagnostic("authority", false).record(Stage::Request, Reason::Identity, None); failure( ProtocolErrorCode::PermissionDenied, "Workspace config authority unavailable", @@ -69,11 +82,13 @@ impl WorkspaceConfigBackend { &self, request: WorkspaceRequest, mutation: bool, + diagnostic: &Diagnostic, ) -> Result { let response = self .client - .execute_with_timeout(request, DEADLINE) + .execute_with_timeout(request, self.deadline) .map_err(|error| { + diagnostic.transport(&error); if mutation && !matches!(error, WorkspaceClientError::InvalidPath(_)) { unknown() } else { @@ -84,6 +99,7 @@ impl WorkspaceConfigBackend { } })?; if response.body.len() > MAX_RESPONSE_BYTES { + diagnostic.record(Stage::Response, Reason::ResponseSize, Some(response.status)); return Err(if mutation { unknown() } else { @@ -96,8 +112,18 @@ impl WorkspaceConfigBackend { if !response.is_success() { if let Ok(error) = serde_json::from_str::(&response.body) { if error.classification == WorkspaceConfigFailureClassification::Unknown { + diagnostic.record( + Stage::Response, + Reason::BackendUnknown, + Some(response.status), + ); return Err(unknown()); } + diagnostic.record( + Stage::Response, + Reason::BackendRejected, + Some(response.status), + ); // Do not reflect arbitrary transport/provider text into model output. let code = match error.status { 401 | 403 | 404 => ProtocolErrorCode::PermissionDenied, @@ -111,6 +137,11 @@ impl WorkspaceConfigBackend { "Workspace config request rejected without commit", )); } + diagnostic.record( + Stage::Response, + Reason::UnrecognizedResponse, + Some(response.status), + ); return Err(if mutation { unknown() } else { @@ -125,6 +156,7 @@ impl WorkspaceConfigBackend { }); } serde_json::from_str(&response.body).map_err(|_| { + diagnostic.record(Stage::Response, Reason::Decode, Some(response.status)); if mutation { unknown() } else { @@ -138,10 +170,17 @@ impl WorkspaceConfigBackend { /// GET checks the current grant even when the Worker has not attached yet. pub fn current(&self) -> Result, WipOperationError> { + self.current_diagnosed(&diagnostic("current", false)) + } + + fn current_diagnosed( + &self, + diagnostic: &Diagnostic, + ) -> Result, WipOperationError> { let result: Option = - self.request(WorkspaceRequest::get(self.base()?), false)?; + self.request(WorkspaceRequest::get(self.base()?), false, diagnostic)?; if let Some(attachment) = &result { - self.validate_attachment(attachment)?; + self.validate_attachment(attachment, diagnostic)?; } Ok(result) } @@ -149,6 +188,7 @@ impl WorkspaceConfigBackend { fn validate_attachment( &self, attachment: &WorkspaceConfigAttachment, + diagnostic: &Diagnostic, ) -> Result<(), WipOperationError> { if Some(attachment.workspace_id.as_str()) != self.client.workspace_id() || attachment.alias != ATTACHMENT_ALIAS @@ -160,6 +200,7 @@ impl WorkspaceConfigBackend { .bytes() .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_')) { + diagnostic.record(Stage::Response, Reason::Identity, None); return Err(failure( ProtocolErrorCode::PermissionDenied, "Invalid Workspace config attachment identity", @@ -173,14 +214,34 @@ impl WorkspaceConfigBackend { suffix: &str, request: &R, mutation: bool, + ) -> Result { + let operation = match suffix { + "/observe" => "observe", + "/read" => "read", + "/commit" => "commit", + "" => "attach", + _ => "post", + }; + self.post_diagnosed(suffix, request, mutation, diagnostic(operation, mutation)) + .await + } + + pub(super) async fn post_diagnosed( + &self, + suffix: &str, + request: &R, + mutation: bool, + diagnostic: Diagnostic, ) -> Result { let body = serde_json::to_string(request).map_err(|_| { + diagnostic.record(Stage::Request, Reason::Decode, None); failure( ProtocolErrorCode::InvalidArguments, "Invalid Workspace config request", ) })?; if body.len() > MAX_RESPONSE_BYTES { + diagnostic.record(Stage::Request, Reason::RequestSize, None); return Err(failure( ProtocolErrorCode::ResourceLimitExceeded, "Workspace config request exceeds limit", @@ -192,12 +253,16 @@ impl WorkspaceConfigBackend { body, ); let this = self.clone(); + let request_diagnostic = diagnostic.clone(); tokio::time::timeout( - DEADLINE, - tokio::task::spawn_blocking(move || this.request(request, mutation)), + self.deadline, + tokio::task::spawn_blocking(move || { + this.request(request, mutation, &request_diagnostic) + }), ) .await .map_err(|_| { + diagnostic.record(Stage::Completion, Reason::Deadline, None); if mutation { unknown() } else { @@ -208,6 +273,7 @@ impl WorkspaceConfigBackend { } })? .map_err(|_| { + diagnostic.record(Stage::Completion, Reason::TaskJoin, None); if mutation { unknown() } else { @@ -223,18 +289,22 @@ impl WorkspaceConfigBackend { &self, ) -> Result, WipOperationError> { let this = self.clone(); + let diagnostic = diagnostic("current", false); + let request_diagnostic = diagnostic.clone(); tokio::time::timeout( - DEADLINE, - tokio::task::spawn_blocking(move || this.current()), + self.deadline, + tokio::task::spawn_blocking(move || this.current_diagnosed(&request_diagnostic)), ) .await .map_err(|_| { + diagnostic.record(Stage::Completion, Reason::Deadline, None); failure( ProtocolErrorCode::ResourceLimitExceeded, "Workspace config request deadline exceeded", ) })? .map_err(|_| { + diagnostic.record(Stage::Completion, Reason::TaskJoin, None); failure( ProtocolErrorCode::Internal, "Workspace config transport unavailable", @@ -249,10 +319,14 @@ impl WorkspaceConfigAttachmentBackend for WorkspaceConfigBackend { self.client.is_available() } fn validate_access(&self, access: Option) -> Result<(), ConfigAttachError> { - let current = self.current().map_err(attach_error)?; + let diagnostic = diagnostic("validate_access", false); + let current = self + .current_diagnosed(&diagnostic) + .map_err(|error| attach_error(error, &diagnostic))?; if access == Some(ConfigAccess::ReadWrite) && current.is_some_and(|a| a.access == WorkspaceConfigAccess::ReadOnly) { + diagnostic.record(Stage::Attach, Reason::AccessCeiling, None); return Err(ConfigAttachError::Denied); } Ok(()) @@ -271,10 +345,16 @@ impl WorkspaceConfigAttachmentBackend for WorkspaceConfigBackend { ConfigAccess::ReadWrite => WorkspaceConfigAccess::ReadWrite, }), }; - let attachment: WorkspaceConfigAttachment = - self.post("", &request, true).await.map_err(attach_error)?; - self.validate_attachment(&attachment) - .map_err(|_| ConfigAttachError::OutcomeUnknown)?; + let diagnostic = diagnostic("attach", true); + let attachment: WorkspaceConfigAttachment = self + .post_diagnosed("", &request, true, diagnostic.clone()) + .await + .map_err(|error| attach_error(error, &diagnostic))?; + self.validate_attachment(&attachment, &diagnostic) + .map_err(|_| { + diagnostic.record(Stage::Attach, Reason::Identity, None); + ConfigAttachError::OutcomeUnknown + })?; Ok(ConfigAttachment { connection_id: attachment.connection_id, access: match attachment.access { @@ -286,7 +366,8 @@ impl WorkspaceConfigAttachmentBackend for WorkspaceConfigBackend { } } -fn attach_error(error: WipOperationError) -> ConfigAttachError { +fn attach_error(error: WipOperationError, diagnostic: &Diagnostic) -> ConfigAttachError { + diagnostic.record(Stage::Attach, Reason::AttachConversion, None); match error { WipOperationError::OutcomeUnknown(_) => ConfigAttachError::OutcomeUnknown, WipOperationError::Protocol(error) => match error.code { @@ -298,6 +379,10 @@ fn attach_error(error: WipOperationError) -> ConfigAttachError { } } +pub(super) fn diagnostic(operation: &'static str, mutation: bool) -> Diagnostic { + Diagnostic::new(Adapter::WorkspaceConfig, operation, mutation, None) +} + pub(super) fn failure(code: ProtocolErrorCode, message: &str) -> WipOperationError { WipOperationError::Protocol(ProtocolError { code, @@ -307,3 +392,7 @@ pub(super) fn failure(code: ProtocolErrorCode, message: &str) -> WipOperationErr pub(super) fn unknown() -> WipOperationError { WipOperationError::OutcomeUnknown("Workspace config outcome unknown; refresh before any subsequent operation; no automatic retry".into()) } + +#[cfg(test)] +#[path = "backend_tests.rs"] +mod tests; diff --git a/crates/worker/src/feature/builtin/workspace_config/backend_tests.rs b/crates/worker/src/feature/builtin/workspace_config/backend_tests.rs new file mode 100644 index 000000000..b617107c7 --- /dev/null +++ b/crates/worker/src/feature/builtin/workspace_config/backend_tests.rs @@ -0,0 +1,580 @@ +//! Production WorkspaceConfigBackend with only WorkspaceClient substituted. The transport fixture +//! applies effects before injected reply loss, retains its state for rereading, and gates blocking +//! completion explicitly. These ordinary worker lib tests do not claim Backend transaction coverage. +use super::*; +use crate::feature::builtin::backend_diagnostic::tests::Capture; +use crate::worker::WorkspaceResponse; +use std::sync::{Condvar, Mutex}; +use tracing::instrument::WithSubscriber; + +const UNSAFE: &str = "secret-token /private/config credential=unsafe"; + +#[derive(Clone, Copy, Debug)] +enum Fault { + Unavailable, + InvalidPath, + Communication, + Decode, + Size, + Rejected(u16), + Unknown, + Unrecognized, + Identity, + Panic, + Deadline, +} + +#[derive(Debug, Default)] +struct Effects { + requests: usize, + mutations: usize, + saved: Option, + completed: bool, + released: bool, +} + +#[derive(Debug)] +struct Client { + fault: Fault, + state: Mutex, + changed: Condvar, + started: tokio::sync::Notify, +} +impl Client { + fn new(fault: Fault) -> Arc { + Arc::new(Self { + fault, + state: Mutex::new(Effects::default()), + changed: Condvar::new(), + started: tokio::sync::Notify::new(), + }) + } + fn finish(&self) { + self.state.lock().unwrap().completed = true; + self.changed.notify_all(); + } + fn release(&self) { + self.state.lock().unwrap().released = true; + self.changed.notify_all(); + } + fn wait_completed(&self) { + let (state, timeout) = self + .changed + .wait_timeout_while(self.state.lock().unwrap(), Duration::from_secs(5), |s| { + !s.completed + }) + .unwrap(); + assert!( + !timeout.timed_out() && state.completed, + "blocking request did not finish" + ); + } +} +struct Release(Arc); +impl Drop for Release { + fn drop(&mut self) { + self.0.release(); + } +} + +fn attachment() -> WorkspaceConfigAttachment { + WorkspaceConfigAttachment { + workspace_id: "workspace".into(), + connection_id: "connection-1".into(), + alias: ATTACHMENT_ALIAS.into(), + content_path: CONTENT_ROOT.into(), + working_directory_id: "grant".into(), + access: WorkspaceConfigAccess::ReadWrite, + name: UNSAFE.into(), + purpose: UNSAFE.into(), + already_attached: false, + } +} + +impl WorkspaceClient for Client { + fn workspace_id(&self) -> Option<&str> { + Some("workspace") + } + fn kind(&self) -> &str { + "diagnostic-test-boundary" + } + fn is_available(&self) -> bool { + true + } + fn execute( + &self, + request: WorkspaceRequest, + ) -> Result { + { + let mut state = self.state.lock().unwrap(); + state.requests += 1; + if request.method == WorkspaceRequestMethod::Post + && !request.path.ends_with("/read") + && !matches!( + self.fault, + Fault::Unavailable | Fault::InvalidPath | Fault::Rejected(_) + ) + { + state.mutations += 1; + state.saved = request.body.clone(); + } + self.started.notify_one(); + if matches!(self.fault, Fault::Deadline) { + let (_state, timeout) = self + .changed + .wait_timeout_while(state, Duration::from_secs(5), |s| !s.released) + .unwrap(); + assert!(!timeout.timed_out(), "test must release blocked transport"); + } + } + self.finish(); + match self.fault { + Fault::Unavailable => Err(WorkspaceClientError::Unavailable(UNSAFE.into())), + Fault::InvalidPath => Err(WorkspaceClientError::InvalidPath(UNSAFE.into())), + Fault::Communication => Err(WorkspaceClientError::Request(UNSAFE.into())), + Fault::Panic => panic!("injected blocking task failure"), + Fault::Decode => Ok(WorkspaceResponse { + status: 200, + body: UNSAFE.into(), + }), + Fault::Size => Ok(WorkspaceResponse { + status: 200, + body: UNSAFE.repeat(MAX_RESPONSE_BYTES / UNSAFE.len() + 1), + }), + Fault::Rejected(_) | Fault::Unknown => { + let status = match self.fault { + Fault::Rejected(status) => status, + _ => 503, + }; + Ok(WorkspaceResponse { + status, + body: serde_json::to_string(&WorkspaceConfigApiError { + status, + code: UNSAFE.into(), + message: UNSAFE.into(), + classification: if matches!(self.fault, Fault::Unknown) { + WorkspaceConfigFailureClassification::Unknown + } else { + WorkspaceConfigFailureClassification::NotCommitted + }, + }) + .unwrap(), + }) + } + Fault::Unrecognized => Ok(WorkspaceResponse { + status: 503, + body: UNSAFE.into(), + }), + Fault::Identity | Fault::Deadline => { + let mut value = attachment(); + if matches!(self.fault, Fault::Identity) { + value.workspace_id = UNSAFE.into(); + } + Ok(WorkspaceResponse { + status: 200, + body: serde_json::to_string(&value).unwrap(), + }) + } + } + } +} + +fn assert_safe(capture: &Capture, reason: Reason, stage: Stage, public: &str) { + let text = capture.text(); + assert!( + text.contains(&format!("reason={reason:?}")), + "missing reason: {text}" + ); + assert!( + text.contains(&format!("stage={stage:?}")), + "missing stage: {text}" + ); + assert!(text.contains("adapter=WorkspaceConfig"), "{text}"); + assert!(text.contains("correlation="), "{text}"); + for secret in [ + UNSAFE, + "secret-token", + "/private/config", + "credential=unsafe", + ] { + assert!( + !text.contains(secret), + "diagnostics leaked unsafe boundary data" + ); + assert!( + !public.contains(secret), + "public result leaked unsafe boundary data" + ); + } +} +fn public_error( + result: Result, + code: Option, +) -> String { + match (result, code) { + (Err(WipOperationError::OutcomeUnknown(message)), None) => { + assert!(message.contains("no automatic retry")); + message + } + (Err(WipOperationError::Protocol(error)), Some(expected)) => { + assert_eq!(error.code, expected); + error.message + } + _ => panic!("unexpected public error classification"), + } +} + +#[tokio::test] +async fn production_reply_failures_retain_reasons_without_retrying_mutations() { + let cases = [ + ( + Fault::Unavailable, + Reason::TransportUnavailable, + Stage::Request, + ProtocolErrorCode::Internal, + false, + ), + ( + Fault::InvalidPath, + Reason::InvalidPath, + Stage::Request, + ProtocolErrorCode::Internal, + false, + ), + ( + Fault::Communication, + Reason::Communication, + Stage::Request, + ProtocolErrorCode::Internal, + true, + ), + ( + Fault::Decode, + Reason::Decode, + Stage::Response, + ProtocolErrorCode::Internal, + true, + ), + ( + Fault::Size, + Reason::ResponseSize, + Stage::Response, + ProtocolErrorCode::ResourceLimitExceeded, + true, + ), + ( + Fault::Unknown, + Reason::BackendUnknown, + Stage::Response, + ProtocolErrorCode::Internal, + true, + ), + ( + Fault::Unrecognized, + Reason::UnrecognizedResponse, + Stage::Response, + ProtocolErrorCode::Internal, + true, + ), + ( + Fault::Panic, + Reason::TaskJoin, + Stage::Completion, + ProtocolErrorCode::Internal, + true, + ), + ]; + for (fault, reason, stage, read_code, applies) in cases { + for mutation in [false, true] { + let client = Client::new(fault); + let backend = WorkspaceConfigBackend::new(client.clone()); + let capture = Capture::default(); + let result = backend + .post::( + if mutation { "/commit" } else { "/read" }, + &serde_json::json!({"content": UNSAFE}), + mutation, + ) + .with_subscriber(capture.dispatch()) + .await; + let code = if (mutation && !matches!(fault, Fault::InvalidPath)) + || matches!(fault, Fault::Unknown) + { + None + } else { + Some(read_code) + }; + let public = public_error(result, code); + assert_safe(&capture, reason, stage, &public); + let state = client.state.lock().unwrap(); + assert_eq!(state.requests, 1); + assert_eq!(state.mutations, usize::from(mutation && applies)); + assert_eq!(state.saved.is_some(), mutation && applies); + } + } +} + +#[tokio::test] +async fn production_typed_rejections_preserve_public_codes_and_never_apply() { + for (status, code) in [ + (403, ProtocolErrorCode::PermissionDenied), + (409, ProtocolErrorCode::ValidatorMismatch), + (413, ProtocolErrorCode::ResourceLimitExceeded), + (422, ProtocolErrorCode::InvalidArguments), + (500, ProtocolErrorCode::Internal), + ] { + for mutation in [false, true] { + let client = Client::new(Fault::Rejected(status)); + let backend = WorkspaceConfigBackend::new(client.clone()); + let capture = Capture::default(); + let public = public_error( + backend + .post::( + if mutation { "/commit" } else { "/read" }, + &(), + mutation, + ) + .with_subscriber(capture.dispatch()) + .await, + Some(code), + ); + assert_safe(&capture, Reason::BackendRejected, Stage::Response, &public); + assert!(capture.text().contains(&format!("status={status}"))); + let state = client.state.lock().unwrap(); + assert_eq!(state.requests, 1); + assert_eq!(state.mutations, 0); + assert!(state.saved.is_none()); + } + } +} + +#[tokio::test] +async fn production_deadline_is_unknown_after_effect_and_blocking_completion_is_not_retried() { + for mutation in [false, true] { + let client = Client::new(Fault::Deadline); + let release = Release(client.clone()); + let backend = + WorkspaceConfigBackend::new(client.clone()).with_deadline(Duration::from_secs(1)); + let capture = Capture::default(); + let body = serde_json::json!({"content": UNSAFE}); + let request = backend + .post::( + if mutation { "/commit" } else { "/read" }, + &body, + mutation, + ) + .with_subscriber(capture.dispatch()); + tokio::pin!(request); + tokio::select! { + biased; + _ = client.started.notified() => {}, + _ = &mut request => panic!("request must start and apply its effect before timeout is observed"), + } + { + let state = client.state.lock().unwrap(); + assert_eq!(state.mutations, usize::from(mutation)); + assert_eq!(state.saved.is_some(), mutation); + assert!(!state.completed && !state.released); + } + let result = request.await; + let public = public_error( + result, + if mutation { + None + } else { + Some(ProtocolErrorCode::ResourceLimitExceeded) + }, + ); + assert_safe(&capture, Reason::Deadline, Stage::Completion, &public); + drop(release); + let completed = client.clone(); + tokio::task::spawn_blocking(move || completed.wait_completed()) + .await + .unwrap(); + let state = client.state.lock().unwrap(); + assert_eq!(state.requests, 1); + assert_eq!(state.mutations, usize::from(mutation)); + assert_eq!(state.saved.is_some(), mutation); + } +} + +#[tokio::test] +async fn production_current_identity_and_task_loss_remain_safe() { + for (fault, reason, stage, code) in [ + ( + Fault::Identity, + Reason::Identity, + Stage::Response, + ProtocolErrorCode::PermissionDenied, + ), + ( + Fault::Panic, + Reason::TaskJoin, + Stage::Completion, + ProtocolErrorCode::Internal, + ), + ( + Fault::Decode, + Reason::Decode, + Stage::Response, + ProtocolErrorCode::Internal, + ), + ] { + let client = Client::new(fault); + let backend = WorkspaceConfigBackend::new(client.clone()); + let capture = Capture::default(); + let result = backend + .current_async() + .with_subscriber(capture.dispatch()) + .await; + let message = match result { + Err(WipOperationError::Protocol(error)) => { + assert_eq!(error.code, code); + error.message + } + _ => panic!("unexpected current classification"), + }; + assert_safe(&capture, reason, stage, &message); + assert_eq!(client.state.lock().unwrap().mutations, 0); + } +} + +#[tokio::test] +async fn production_attach_conversion_retains_original_reason_and_effect_count() { + for (fault, reason, expected, mutations) in [ + ( + Fault::Communication, + Reason::Communication, + ConfigAttachError::OutcomeUnknown, + 1, + ), + ( + Fault::Identity, + Reason::Identity, + ConfigAttachError::OutcomeUnknown, + 1, + ), + ( + Fault::Rejected(403), + Reason::BackendRejected, + ConfigAttachError::Denied, + 0, + ), + ( + Fault::Rejected(422), + Reason::BackendRejected, + ConfigAttachError::InvalidRequest, + 0, + ), + ( + Fault::Rejected(500), + Reason::BackendRejected, + ConfigAttachError::Unavailable, + 0, + ), + ] { + let client = Client::new(fault); + let backend = WorkspaceConfigBackend::new(client.clone()); + let capture = Capture::default(); + assert_eq!( + backend + .attach("host-key-not-logged", None) + .with_subscriber(capture.dispatch()) + .await, + Err(expected) + ); + assert_safe(&capture, reason, Stage::Attach, expected.message()); + let text = capture.text(); + if !matches!(fault, Fault::Identity) { + assert!(text.contains("reason=AttachConversion"), "{text}"); + } + assert!(!text.contains("host-key-not-logged")); + let correlations: std::collections::BTreeSet<_> = text + .lines() + .filter_map(|line| line.split("correlation=").nth(1)) + .filter_map(|tail| tail.split(';').next()) + .collect(); + assert_eq!( + correlations.len(), + 1, + "request and attach conversion must correlate" + ); + let state = client.state.lock().unwrap(); + assert_eq!(state.requests, 1); + assert_eq!(state.mutations, mutations); + assert_eq!(state.saved.is_some(), mutations == 1); + } +} + +#[tokio::test] +async fn production_oversized_request_is_rejected_before_dispatch() { + let client = Client::new(Fault::Communication); + let backend = WorkspaceConfigBackend::new(client.clone()); + let capture = Capture::default(); + let public = public_error( + backend + .post::("/commit", &"x".repeat(MAX_RESPONSE_BYTES), true) + .with_subscriber(capture.dispatch()) + .await, + Some(ProtocolErrorCode::ResourceLimitExceeded), + ); + assert_safe(&capture, Reason::RequestSize, Stage::Request, &public); + let state = client.state.lock().unwrap(); + assert_eq!(state.requests, 0); + assert_eq!(state.mutations, 0); +} + +#[tokio::test] +async fn production_current_deadline_remains_bounded_and_has_no_mutation() { + let client = Client::new(Fault::Deadline); + let release = Release(client.clone()); + let backend = WorkspaceConfigBackend::new(client.clone()).with_deadline(Duration::from_secs(1)); + let capture = Capture::default(); + let request = backend.current_async().with_subscriber(capture.dispatch()); + tokio::pin!(request); + tokio::select! { + biased; + _ = client.started.notified() => {}, + _ = &mut request => panic!("current request must enter the gate before timeout is observed"), + } + assert_eq!(client.state.lock().unwrap().mutations, 0); + let error = request.await; + let message = match error { + Err(WipOperationError::Protocol(error)) => { + assert_eq!(error.code, ProtocolErrorCode::ResourceLimitExceeded); + error.message + } + _ => panic!("current deadline must remain a public resource-limit error"), + }; + assert_safe(&capture, Reason::Deadline, Stage::Completion, &message); + drop(release); + let completed = client.clone(); + tokio::task::spawn_blocking(move || completed.wait_completed()) + .await + .unwrap(); + let state = client.state.lock().unwrap(); + assert_eq!(state.requests, 1); + assert_eq!(state.mutations, 0); +} + +#[tokio::test] +async fn production_request_encoding_failure_is_safe_and_never_dispatched() { + struct Unencodable; + impl Serialize for Unencodable { + fn serialize(&self, _: S) -> Result { + Err(serde::ser::Error::custom(UNSAFE)) + } + } + let client = Client::new(Fault::Communication); + let backend = WorkspaceConfigBackend::new(client.clone()); + let capture = Capture::default(); + let result = backend + .post::("/commit", &Unencodable, true) + .with_subscriber(capture.dispatch()) + .await; + let message = public_error(result, Some(ProtocolErrorCode::InvalidArguments)); + assert_safe(&capture, Reason::Decode, Stage::Request, &message); + let state = client.state.lock().unwrap(); + assert_eq!(state.requests, 0); + assert_eq!(state.mutations, 0); +} diff --git a/crates/worker/src/feature/builtin/workspace_config/content.rs b/crates/worker/src/feature/builtin/workspace_config/content.rs index b2f67d342..ae6c64b41 100644 --- a/crates/worker/src/feature/builtin/workspace_config/content.rs +++ b/crates/worker/src/feature/builtin/workspace_config/content.rs @@ -36,9 +36,11 @@ use wip_protocol::{ }; use super::backend::{ - MAX_DEPTH, MAX_NODES, MAX_PATH_BYTES, MAX_TEXT_BYTES, WorkspaceConfigBackend, failure, unknown, + MAX_DEPTH, MAX_NODES, MAX_PATH_BYTES, MAX_TEXT_BYTES, WorkspaceConfigBackend, diagnostic, + failure, unknown, }; use super::{CONTENT_ROOT, WorkspaceConfigFeature}; +use crate::feature::builtin::backend_diagnostic::{Diagnostic, Reason, Stage}; use crate::wip::{ WipCallContext, WipOperationError, WipOperationHandler, WipOperationOutput, WipProjection, WipProjectionKind, WipSubtreeProvider, contextual_reference, json_to_wip, @@ -138,6 +140,23 @@ async fn observe( connection: &str, paths: Vec, depth: u32, +) -> Result { + observe_diagnosed( + backend, + connection, + paths, + depth, + diagnostic("observe", false), + ) + .await +} + +async fn observe_diagnosed( + backend: &WorkspaceConfigBackend, + connection: &str, + paths: Vec, + depth: u32, + diagnostic: Diagnostic, ) -> Result { if paths.len() > MAX_NODES || depth > MAX_DEPTH { return Err(invalid()); @@ -148,7 +167,7 @@ async fn observe( } } let result: WorkspaceConfigObserveResponse = backend - .post( + .post_diagnosed( "/observe", &WorkspaceConfigObserveRequest { connection_id: connection.into(), @@ -156,6 +175,7 @@ async fn observe( depth, }, false, + diagnostic.clone(), ) .await?; if result.connection_id != connection @@ -164,24 +184,37 @@ async fn observe( || result.validator.len() > 4096 || result.digest.is_empty() { + diagnostic.record( + Stage::Response, + if result.connection_id != connection { + Reason::Identity + } else { + Reason::InvalidMetadata + }, + None, + ); return Err(failure( ProtocolErrorCode::Internal, "Invalid Workspace config metadata", )); } if result.entrypoints.is_empty() || result.entrypoints.len() > MAX_NODES { + diagnostic.record(Stage::Response, Reason::InvalidMetadata, None); return Err(failure( ProtocolErrorCode::Internal, "Invalid Workspace config entrypoints", )); } for path in &result.entrypoints { - validate_source(path)?; + validate_source(path) + .inspect_err(|_| diagnostic.record(Stage::Response, Reason::InvalidMetadata, None))?; } let mut seen = BTreeSet::new(); for node in &result.nodes { if !node.path.is_empty() { - validate_source(&node.path)?; + validate_source(&node.path).inspect_err(|_| { + diagnostic.record(Stage::Response, Reason::InvalidMetadata, None) + })?; } if !seen.insert(&node.path) || node.validator.is_empty() @@ -198,6 +231,7 @@ async fn observe( suffix.is_some_and(|s| s.split('/').count() <= depth as usize) }) { + diagnostic.record(Stage::Response, Reason::InvalidMetadata, None); return Err(failure( ProtocolErrorCode::Internal, "Invalid Workspace config metadata scope", @@ -490,9 +524,10 @@ fn content_type(value: Option<&Value>) -> Result Result { + let diagnostic = diagnostic("read", false); let response: WorkspaceConfigReadResponse = self .backend - .post( + .post_diagnosed( "/read", &WorkspaceConfigReadRequest { connection_id: self.attachment.connection_id.clone(), @@ -500,6 +535,7 @@ impl ConfigHandler { validator: self.node.validator.clone(), }, false, + diagnostic.clone(), ) .await?; if response.path != self.node.path @@ -507,12 +543,14 @@ impl ConfigHandler { || Some(&response.digest) != self.node.digest.as_ref() || Some(&response.content_type) != self.node.content_type.as_ref() { + diagnostic.record(Stage::Response, Reason::Identity, None); return Err(failure( ProtocolErrorCode::Internal, "Invalid Workspace config read response", )); } if response.content.len() > MAX_TEXT_BYTES { + diagnostic.record(Stage::Response, Reason::ResponseSize, None); return Err(failure( ProtocolErrorCode::ResourceLimitExceeded, "Workspace config source exceeds text limit", @@ -630,12 +668,17 @@ impl WipOperationHandler for ConfigHandler { context: WipCallContext, ) -> Result { let dispatched = std::sync::atomic::AtomicBool::new(false); + let diagnostic = diagnostic( + "operation", + !matches!(operation, "read" | "read_lines" | "search_text"), + ); tokio::time::timeout( super::DEADLINE, - self.execute(operation, arguments, context, &dispatched), + self.execute(operation, arguments, context, &dispatched, &diagnostic), ) .await .map_err(|_| { + diagnostic.record(Stage::Completion, Reason::Deadline, None); if dispatched.load(std::sync::atomic::Ordering::Acquire) { unknown() } else { @@ -655,6 +698,7 @@ impl ConfigHandler { arguments: &BTreeMap, context: WipCallContext, dispatched: &std::sync::atomic::AtomicBool, + operation_diagnostic: &Diagnostic, ) -> Result { if !self.operations.iter().any(|name| name == operation) { return Err(failure( @@ -798,9 +842,10 @@ impl ConfigHandler { }; let changes = self.canonical_changes(changes).await?; dispatched.store(true, std::sync::atomic::Ordering::Release); + let diagnostic = operation_diagnostic.clone(); let result: WorkspaceConfigCommitResponse = self .backend - .post( + .post_diagnosed( "/commit", &WorkspaceConfigCommitRequest { connection_id: self.attachment.connection_id.clone(), @@ -812,28 +857,37 @@ impl ConfigHandler { }, }, true, + diagnostic.clone(), ) .await?; // Commit response supplies ROOT state. Obtain the exact post-commit PATH // validator, never substitute the root validator for a file validator. - let updated = observe( + let updated = observe_diagnosed( &self.backend, &self.attachment.connection_id, vec![self.node.path.clone()], 0, + diagnostic.at_stage(Stage::Reconcile), ) .await .map_err(|_| unknown())?; if updated.digest != result.digest || updated.validator != result.validator { + diagnostic.record(Stage::Reconcile, Reason::Identity, None); return Err(unknown()); } let node = updated .nodes .iter() .find(|n| n.path == self.node.path) - .ok_or_else(unknown)?; + .ok_or_else(|| { + diagnostic.record(Stage::Reconcile, Reason::InvalidMetadata, None); + unknown() + })?; let value = json_to_wip(&json!({"digest": result.digest, - "activation": "Saved to canonical Workspace configuration; existing Worker manifests are not rewritten"})).map_err(|_| unknown())?; + "activation": "Saved to canonical Workspace configuration; existing Worker manifests are not rewritten"})).map_err(|_| { + diagnostic.record(Stage::Completion, Reason::Decode, None); + unknown() + })?; Ok(WipOperationOutput::native_with_validator( value, object_validator( diff --git a/crates/worker/src/feature/builtin/workspace_config/production_tests.rs b/crates/worker/src/feature/builtin/workspace_config/production_tests.rs index 71649555c..27ee6f161 100644 --- a/crates/worker/src/feature/builtin/workspace_config/production_tests.rs +++ b/crates/worker/src/feature/builtin/workspace_config/production_tests.rs @@ -1,8 +1,10 @@ -//! Worker-side E2E across real Feature invocation, WIP Client/Host and typed -//! Workspace transport. Canonical persistence/projection tests live in Backend; -//! this deterministic router models its published auth/CAS/response contract. +//! Worker-side integration across real Feature invocation, WIP Client/Host and the production +//! WorkspaceConfigBackend. Only WorkspaceClient transport is substituted by a deterministic router +//! retaining applied effects and modeling the published auth/CAS/response contract. These ordinary +//! worker lib tests do not replace canonical persistence/projection tests owned by Backend. use super::*; use crate::feature::FeatureRegistryBuilder; +use crate::feature::builtin::backend_diagnostic::tests::Capture; use crate::hook::HookRegistryBuilder; use crate::wip::{WipMountRegistry, WipRuntime}; use crate::worker::{ @@ -16,6 +18,7 @@ use serde_json::{Value as Json, json}; use server_api::*; use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex}; +use tracing::instrument::WithSubscriber; #[derive(Debug)] struct State { @@ -26,6 +29,10 @@ struct State { entries: BTreeMap, commits: Vec, requests: Vec, + applied_commits: usize, + malformed_read_reply: Option<&'static str>, + malformed_observe_reply: Option<&'static str>, + lose_postcommit_observe_reply: bool, fail_commit: bool, fail_current: bool, lose_commit_reply: bool, @@ -58,6 +65,10 @@ impl Router { ]), commits: Vec::new(), requests: Vec::new(), + applied_commits: 0, + malformed_read_reply: None, + malformed_observe_reply: None, + lose_postcommit_observe_reply: false, fail_commit: false, fail_current: false, lose_commit_reply: false, @@ -222,6 +233,11 @@ impl WorkspaceClient for Router { } match suffix { "/observe" => { + if state.lose_postcommit_observe_reply && state.applied_commits > 0 { + return Err(WorkspaceClientError::Request( + "secret-token /private/host/config".into(), + )); + } let request: WorkspaceConfigObserveRequest = serde_json::from_value(input).unwrap(); assert!(request.depth <= 8 && request.paths.len() <= 256); let mut kinds = @@ -268,13 +284,23 @@ impl WorkspaceClient for Router { ) }) .collect(); - Ok(response(&WorkspaceConfigObserveResponse { + let mut result = WorkspaceConfigObserveResponse { connection_id: state.active.as_ref().unwrap().connection_id.clone(), validator: state.validator(""), digest: state.tree_digest(), entrypoints: vec!["main.dcdl".into()], nodes, - })) + }; + match state.malformed_observe_reply { + Some("connection") => { + result.connection_id = "secret-token /private/host/config".into() + } + Some("entrypoints") => result.entrypoints.clear(), + Some("scope") => result.nodes[0].path = "outside/path.dcdl".into(), + None => {} + Some(_) => unreachable!(), + } + Ok(response(&result)) } "/read" => { let request: WorkspaceConfigReadRequest = serde_json::from_value(input).unwrap(); @@ -290,13 +316,20 @@ impl WorkspaceClient for Router { WorkspaceConfigFailureClassification::NotCommitted, )); }; - Ok(response(&WorkspaceConfigReadResponse { + let mut result = WorkspaceConfigReadResponse { path: request.path.clone(), content: content.clone(), content_type: *content_type, digest: digest(content), validator: state.validator(&request.path), - })) + }; + match state.malformed_read_reply { + Some("identity") => result.path = "secret-token /private/host/config".into(), + Some("size") => result.content = "x".repeat(256 * 1024 + 1), + None => {} + Some(_) => unreachable!(), + } + Ok(response(&result)) } "/commit" => { let request: WorkspaceConfigCommitRequest = serde_json::from_value(input).unwrap(); @@ -403,6 +436,7 @@ impl WorkspaceClient for Router { )); } state.entries = entries; + state.applied_commits += 1; if state.lose_commit_reply { state.lose_commit_reply = false; return Err(WorkspaceClientError::Request( @@ -797,14 +831,20 @@ async fn post_commit_observation_mismatch_reports_unknown_without_replaying_writ let path = "/workspace-config/main.dcdl"; observe_interface(&runtime, path, true).await; client.state.lock().unwrap().mismatched_commit_reply = Some(field); + let capture = Capture::default(); let error = call(&runtime, path, "write", json!({"content":"saved"})) + .with_subscriber(capture.dispatch()) .await .unwrap_err() .to_string(); assert!(error.contains("unknown"), "{field}: {error}"); + assert!(capture.text().contains("stage=Reconcile")); + assert!(capture.text().contains("reason=Identity")); + assert!(!capture.text().contains("saved")); let state = client.state.lock().unwrap(); assert_eq!(state.commits.len(), 1); assert_eq!(state.entries["main.dcdl"].1, "saved"); + assert_eq!(state.applied_commits, 1); } } @@ -1155,3 +1195,110 @@ async fn workspace_config_backend_validators_reject_path_and_connection_transpla assert!(result.content.unwrap().contains("model = updated")); assert_eq!(client.state.lock().unwrap().commits.len(), 1); } + +#[tokio::test] +async fn production_read_validation_retains_reason_without_committing() { + for (fault, reason) in [("identity", "Identity"), ("size", "ResponseSize")] { + let client = Router::new(); + let (feature, runtime) = runtime(client.clone()); + feature.attach("initial", None).await.unwrap(); + let path = "/workspace-config/main.dcdl"; + observe_interface(&runtime, path, true).await; + client.state.lock().unwrap().malformed_read_reply = Some(fault); + let capture = Capture::default(); + let error = call(&runtime, path, "read", json!({})) + .with_subscriber(capture.dispatch()) + .await + .unwrap_err() + .to_string(); + let text = capture.text(); + assert!(text.contains(&format!("reason={reason}")), "{text}"); + assert!(text.contains("stage=Response")); + assert!(!error.contains("unknown")); + assert!(!text.contains("secret-token") && !text.contains("/private/host")); + assert!(!error.contains("secret-token") && !error.contains("/private/host")); + let state = client.state.lock().unwrap(); + assert_eq!(state.applied_commits, 0); + assert_eq!(state.entries["main.dcdl"].1, "model = old"); + } +} + +#[tokio::test] +async fn production_observation_validation_retains_reason_without_committing() { + for (fault, reason) in [ + ("connection", "Identity"), + ("entrypoints", "InvalidMetadata"), + ("scope", "InvalidMetadata"), + ] { + let client = Router::new(); + let (feature, runtime) = runtime(client.clone()); + feature.attach("initial", None).await.unwrap(); + client.state.lock().unwrap().malformed_observe_reply = Some(fault); + let capture = Capture::default(); + let error = runtime + .tree("/workspace-config/main.dcdl".into(), 0, true) + .with_subscriber(capture.dispatch()) + .await + .unwrap_err() + .to_string(); + let text = capture.text(); + assert!(text.contains(&format!("reason={reason}")), "{text}"); + assert!(text.contains("stage=Response")); + assert!(!text.contains("secret-token") && !text.contains("/private/host")); + assert!(!error.contains("secret-token") && !error.contains("/private/host")); + assert_eq!(client.state.lock().unwrap().applied_commits, 0); + } +} + +#[tokio::test] +async fn production_postcommit_transport_loss_retains_cause_and_unknown_without_retry() { + let client = Router::new(); + let (feature, runtime) = runtime(client.clone()); + feature.attach("initial", None).await.unwrap(); + let path = "/workspace-config/main.dcdl"; + observe_interface(&runtime, path, true).await; + client.state.lock().unwrap().lose_postcommit_observe_reply = true; + let capture = Capture::default(); + let error = call( + &runtime, + path, + "write", + json!({"content":"saved-before-observation-loss"}), + ) + .with_subscriber(capture.dispatch()) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("unknown")); + let text = capture.text(); + assert!( + text.lines().any( + |event| event.contains("reason=Communication") && event.contains("stage=Reconcile") + ), + "{text}" + ); + assert!( + !text.contains("reason=UnrecognizedResponse"), + "must not invent a response-decode cause for transport loss" + ); + assert!(!text.contains("secret-token") && !text.contains("/private/host")); + assert!(!text.contains("saved-before-observation-loss")); + assert!(!error.contains("secret-token") && !error.contains("/private/host")); + let correlations: BTreeSet<_> = text + .lines() + .filter_map(|line| line.split("correlation=").nth(1)) + .filter_map(|tail| tail.split(';').next()) + .collect(); + assert_eq!( + correlations.len(), + 1, + "lost postcommit observation must correlate to original commit" + ); + let state = client.state.lock().unwrap(); + assert_eq!(state.commits.len(), 1); + assert_eq!(state.applied_commits, 1); + assert_eq!( + state.entries["main.dcdl"].1, + "saved-before-observation-loss" + ); +} diff --git a/crates/worker/src/feature/builtin/workspace_config/wip.rs b/crates/worker/src/feature/builtin/workspace_config/wip.rs index e03c90db3..0f77f854c 100644 --- a/crates/worker/src/feature/builtin/workspace_config/wip.rs +++ b/crates/worker/src/feature/builtin/workspace_config/wip.rs @@ -161,7 +161,14 @@ impl WipOperationHandler for AttachHandler { "attachment_path": format!("/workdir-attachments/{}", encode_identity(&result.connection_id)), "content_path": CONTENT_ROOT, "already_attached": result.already_attached, - })).map_err(|_| WipOperationError::OutcomeUnknown("Completed config attachment response unavailable".into()))?; + })).map_err(|_| { + super::backend::diagnostic("attach_output", true).record( + crate::feature::builtin::backend_diagnostic::Stage::Attach, + crate::feature::builtin::backend_diagnostic::Reason::Decode, + None, + ); + WipOperationError::OutcomeUnknown("Completed config attachment response unavailable".into()) + })?; Ok(WipOperationOutput::native(value)) } } From 4872cf2aac9f562dd531da08bcc2ad19daa4befc Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 05:55:48 +0900 Subject: [PATCH 07/46] fix: retain independent Bash Drop cleanup diagnostics --- crates/tools/src/bash.rs | 89 +----- crates/tools/src/bash/cleanup.rs | 121 ++++++++ crates/tools/src/bash/cleanup/tests.rs | 397 +++++++++++++++++++++++++ crates/tools/src/bash_output.rs | 75 ++++- 4 files changed, 607 insertions(+), 75 deletions(-) create mode 100644 crates/tools/src/bash/cleanup.rs create mode 100644 crates/tools/src/bash/cleanup/tests.rs diff --git a/crates/tools/src/bash.rs b/crates/tools/src/bash.rs index c3c3a6311..abb9700dc 100644 --- a/crates/tools/src/bash.rs +++ b/crates/tools/src/bash.rs @@ -19,6 +19,8 @@ use workdir::{ WorkdirSessionRouter, }; +mod cleanup; + const DEFAULT_TIMEOUT_SECS: u64 = 120; const MAX_TIMEOUT_SECS: u64 = 600; const INLINE_BYTE_BUDGET: usize = 12 * 1024; @@ -72,80 +74,19 @@ impl Drop for CommandGuard { let Some(handle) = self.handle.take() else { return; }; - let workdir = self.session.clone(); - let state = Arc::clone(&self.state); - let execution_id = self.execution_id.clone(); - let execution_nonce = self.execution_nonce; - let mut retained = self.retained.take(); - let cursor = self.cursor; - // A dropped provider future is not terminal confirmation. Keep the live - // execution registered until cleanup has both requested cancellation and - // observed terminal command output, so cancellation/session teardown - // cannot race with an apparently empty registry. - tokio::spawn(async move { - let result = tokio::time::timeout(std::time::Duration::from_secs(10), async { - let _ = workdir.cancel_command(handle.clone()).await; - let mut cursor = cursor; - loop { - let page = command_page( - &workdir, - CommandOutputRequest { - handle: handle.clone(), - cursor, - limit: workdir::MAX_COMMAND_READ_BYTES, - wait: true, - }, - ) - .await?; - if let Some(writer) = &mut retained { - let mut next = cursor; - for chunk in &page.chunks { - if chunk.sequence != next { - return Err(workdir::WorkdirError::Unavailable( - "command cleanup sequence gap".into(), - )); - } - writer - .append_chunk(chunk) - .map_err(workdir::WorkdirError::Unavailable)?; - next = next.checked_add(chunk.bytes.len() as u64).ok_or_else(|| { - workdir::WorkdirError::Unavailable( - "command cleanup cursor overflow".into(), - ) - })?; - } - if page.next_cursor != next { - return Err(workdir::WorkdirError::Unavailable( - "command cleanup cursor gap".into(), - )); - } - } - cursor = page.next_cursor; - if let Some(terminal) = page.terminal { - if let Some(writer) = &mut retained { - writer - .finish_transport(&terminal) - .map_err(workdir::WorkdirError::Unavailable)?; - } - return Ok::<(), workdir::WorkdirError>(()); - } - } - }) - .await; - // Bounded cleanup can end with an unknown provider outcome. The writer then - // terminalizes partial evidence on drop; retaining a dead collector in `active` - // would leak registrations and falsely suggest it could still collect output. - let _ = result; - let mut state = state.lock().unwrap(); - if state - .active - .get(&execution_id) - .is_some_and(|active| active.execution_nonce == execution_nonce) - { - state.active.remove(&execution_id); - state.cancellation_requested.remove(&execution_id); - } - }); + let cleanup = cleanup::Cleanup { + session: self.session.clone(), + state: Arc::clone(&self.state), + execution_id: self.execution_id.clone(), + execution_nonce: self.execution_nonce, + handle, + cursor: self.cursor, + retained: self + .retained + .take() + .expect("started command owns output writer"), + }; + tokio::spawn(cleanup.run()); } } diff --git a/crates/tools/src/bash/cleanup.rs b/crates/tools/src/bash/cleanup.rs new file mode 100644 index 000000000..ed118c898 --- /dev/null +++ b/crates/tools/src/bash/cleanup.rs @@ -0,0 +1,121 @@ +//! Finite Drop cleanup. A cancel acknowledgement is not termination evidence; only a +//! validated terminal fence confirms the process result. No provider text is logged. + +use super::*; +use crate::bash_output::{ + CleanupCancel, CleanupDiagnostics, CleanupDrain, CleanupFailure, CleanupProviderFailure, + CleanupRegistration, +}; +use std::time::Duration; + +const CLEANUP_TIMEOUT: Duration = Duration::from_secs(10); +// Reserve time for independent terminal observation even if cancel never responds. +const CANCEL_TIMEOUT: Duration = Duration::from_secs(2); + +pub(super) struct Cleanup { + pub session: WorkdirSessionHandle, + pub state: Arc>, + pub execution_id: String, + pub execution_nonce: u64, + pub handle: CommandHandle, + pub cursor: u64, + pub retained: BashOutputWriter, +} + +impl Cleanup { + pub async fn run(mut self) { + let deadline = tokio::time::Instant::now() + CLEANUP_TIMEOUT; + let cancel = match tokio::time::timeout( + CANCEL_TIMEOUT, + self.session.cancel_command(self.handle.clone()), + ) + .await + { + Ok(Ok(())) => CleanupCancel::Acknowledged, + Ok(Err(error)) => CleanupCancel::Failed { + reason: CleanupProviderFailure::from(&error), + }, + Err(_) => CleanupCancel::DeadlineExceeded, + }; + let drain = match tokio::time::timeout_at(deadline, self.drain()).await { + Ok(Ok(())) => CleanupDrain::TerminalConfirmed, + Ok(Err(reason)) => CleanupDrain::Failed { reason }, + Err(_) => CleanupDrain::DeadlineExceeded, + }; + let deadline_expired = matches!(drain, CleanupDrain::DeadlineExceeded); + // Finalize evidence before exposing an empty registry. Keep the lock through + // publication so an older cleanup can never remove a newer generation. + let mut state = self.state.lock().unwrap(); + let registration = if state + .active + .get(&self.execution_id) + .is_some_and(|active| active.execution_nonce == self.execution_nonce) + { + state.active.remove(&self.execution_id); + state.cancellation_requested.remove(&self.execution_id); + CleanupRegistration::Released + } else { + CleanupRegistration::Superseded + }; + let diagnostics = CleanupDiagnostics { + execution_nonce: self.execution_nonce, + cancel, + drain, + deadline_expired, + registration, + }; + tracing::info!( + output_id = %self.retained.id(), + cleanup = ?diagnostics, + "Bash Drop cleanup finished; cancellation and terminal observation are independent" + ); + self.retained.finish_cleanup(diagnostics); + drop(self.retained); + } + + async fn drain(&mut self) -> Result<(), CleanupFailure> { + loop { + let page = command_page( + &self.session, + CommandOutputRequest { + handle: self.handle.clone(), + cursor: self.cursor, + limit: workdir::MAX_COMMAND_READ_BYTES, + wait: true, + }, + ) + .await + .map_err(|error| CleanupFailure::Provider { + reason: CleanupProviderFailure::from(&error), + })?; + let mut next = self.cursor; + for chunk in &page.chunks { + if chunk.sequence != next { + return Err(CleanupFailure::SequenceGap); + } + next = next + .checked_add(chunk.bytes.len() as u64) + .filter(|next| *next <= crate::bash_output::COMMAND_BYTES as u64) + .ok_or(CleanupFailure::DeliveryBudgetExceeded)?; + self.retained + .append_chunk(chunk) + .map_err(|_| CleanupFailure::RetainedChunkInvalid)?; + } + if page.next_cursor != next { + return Err(CleanupFailure::CursorGap); + } + self.cursor = next; + if let Some(terminal) = page.terminal { + self.retained + .finish_transport(&terminal) + .map_err(|_| CleanupFailure::TerminalMetadataInvalid)?; + return Ok(()); + } + // An immediately-ready provider must not starve the cleanup deadline. + tokio::task::yield_now().await; + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/tools/src/bash/cleanup/tests.rs b/crates/tools/src/bash/cleanup/tests.rs new file mode 100644 index 000000000..9b04ec0c0 --- /dev/null +++ b/crates/tools/src/bash/cleanup/tests.rs @@ -0,0 +1,397 @@ +//! Component tests enter through the production CommandGuard Drop, cleanup, page +//! validation and retained store. Only the Workdir provider is replaced; paused time +//! and a cancel barrier make faults/deadlines deterministic without OS timing. + +use super::*; +use std::sync::atomic::{AtomicBool, Ordering}; +use workdir::*; + +const PRIVATE: &str = "SECRET command /private/host/path output password=poison"; + +#[derive(Clone, Copy, Debug)] +enum Fault { + Cancel, + CancelHang, + Drain, + TerminalTimeout, + Sequence, + Cursor, + RetainedChunk, + TerminalMetadata, + CollectionIncomplete, + Complete, +} + +#[derive(Debug)] +struct Provider { + local: LocalWorkdirSession, + fault: Fault, + cancelling: tokio::sync::Notify, + allow_drain: AtomicBool, +} + +#[async_trait] +impl WorkdirSession for Provider { + fn workdir(&self) -> &Workdir { + self.local.workdir() + } + fn capabilities(&self) -> WorkdirSessionCapabilities { + WorkdirSessionCapabilities::ALL + } + async fn stat(&self, _: StatRequest) -> Result { + Err(WorkdirError::OperationFailed) + } + async fn read(&self, _: ReadRequest) -> Result { + Err(WorkdirError::OperationFailed) + } + async fn write(&self, _: WriteRequest) -> Result { + Err(WorkdirError::OperationFailed) + } + async fn edit(&self, _: EditRequest) -> Result { + Err(WorkdirError::OperationFailed) + } + async fn list(&self, _: ListRequest) -> Result { + Err(WorkdirError::OperationFailed) + } + async fn glob(&self, _: GlobRequest) -> Result { + Err(WorkdirError::OperationFailed) + } + async fn grep(&self, _: GrepRequest) -> Result { + Err(WorkdirError::OperationFailed) + } + async fn start_command(&self, _: CommandRequest) -> Result { + Err(WorkdirError::OperationFailed) + } + async fn command_status(&self, _: CommandHandle) -> Result { + Err(WorkdirError::OperationFailed) + } + + async fn command_output( + &self, + request: CommandOutputRequest, + ) -> Result { + assert_eq!(request.cursor, 3); + while !self.allow_drain.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + match self.fault { + Fault::Drain => { + return Err(WorkdirError::Transport(PRIVATE.into()) + .with_failure_reason(WorkdirFailureReason::HttpTransfer)); + } + Fault::TerminalTimeout => return std::future::pending().await, + _ => {} + } + let mut chunk = CommandByteChunk { + sequence: 3, + offset: 3, + stream: CommandStream::Stdout, + bytes: b"tail".to_vec(), + }; + if matches!(self.fault, Fault::Sequence) { + chunk.sequence = 4; + } + if matches!(self.fault, Fault::RetainedChunk) { + chunk.offset = 42; + } + let mut terminal = terminal(); + if matches!(self.fault, Fault::TerminalMetadata) { + terminal.stdout_delivered_bytes = 6; + } + if matches!(self.fault, Fault::CollectionIncomplete) { + terminal.output_collection = CommandOutputCollection::Incomplete { + failure: CommandFailure { + stage: CommandFailureStage::StdoutDrain, + reason: CommandIoErrorKind::Other, + os_code: None, + }, + }; + } + Ok(CommandOutput { + chunks: vec![chunk], + next_cursor: if matches!(self.fault, Fault::Cursor) { + 8 + } else { + 7 + }, + terminal: Some(terminal), + }) + } + + async fn cancel_command(&self, _: CommandHandle) -> Result<(), WorkdirError> { + self.cancelling.notify_one(); + match self.fault { + Fault::Cancel => Err(WorkdirError::Unavailable(PRIVATE.into())), + Fault::CancelHang => std::future::pending().await, + _ => Ok(()), + } + } + async fn close(&self) -> Result<(), WorkdirError> { + Ok(()) + } +} + +fn terminal() -> CommandTerminal { + CommandTerminal { + status: CommandStatus::Completed, + exit_code: Some(0), + process_failure: None, + output_collection: CommandOutputCollection::Complete, + stdout_bytes: 7, + stderr_bytes: 0, + stdout_delivered_bytes: 7, + stderr_delivered_bytes: 0, + truncation_reason: None, + } +} + +struct Fixture { + _root: tempfile::TempDir, + provider: Arc, + state: Arc>, + view: BashOutputView, + id: String, +} + +impl Fixture { + fn start(fault: Fault) -> (Self, CommandGuard) { + let base = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/cleanup-fixtures"); + std::fs::create_dir_all(&base).unwrap(); + let root = tempfile::tempdir_in(base).unwrap(); + let provider = Arc::new(Provider { + local: LocalWorkdirSession::new( + manifest::Scope::writable(root.path()).unwrap(), + root.path().into(), + ), + fault, + cancelling: Default::default(), + allow_drain: AtomicBool::new(false), + }); + let view = BashOutputView::new(); + let mut retained = view.begin().unwrap(); + retained.append(CommandStream::Stdout, b"pre"); + let id = retained.id(); + let state = Arc::new(Mutex::new(BashExecutionState::default())); + let handle = CommandHandle(PRIVATE.into()); + state.lock().unwrap().active.insert( + PRIVATE.into(), + ActiveCommand { + call_id: PRIVATE.into(), + execution_nonce: 1, + session: provider.clone(), + handle: handle.clone(), + }, + ); + state + .lock() + .unwrap() + .cancellation_requested + .insert(PRIVATE.into()); + let guard = CommandGuard { + session: provider.clone(), + state: state.clone(), + execution_id: PRIVATE.into(), + execution_nonce: 1, + handle: Some(handle), + cursor: 3, + retained: Some(retained), + }; + ( + Self { + _root: root, + provider, + state, + view, + id, + }, + guard, + ) + } + + async fn finish(&self) -> BashOutputMetadata { + self.provider.allow_drain.store(true, Ordering::Release); + // The provider's read retry backoff and cleanup deadlines are virtual time. + for _ in 0..4 { + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(3)).await; + } + for _ in 0..1000 { + if let Some(metadata) = self.view.metadata(&self.id) { + return metadata; + } + tokio::task::yield_now().await; + } + panic!("finite cleanup did not publish partial evidence"); + } + + fn assert_evidence(&self, m: &BashOutputMetadata, terminal: bool, bytes: &[u8]) { + assert_eq!( + self.view + .read(&self.id, CommandStream::Stdout, 0, 64) + .unwrap(), + bytes + ); + assert_eq!(m.stdout_saved, bytes.len()); + assert_eq!(m.stdout_delivered, bytes.len() as u64); + assert_eq!(m.stdout_seen, bytes.len() as u64); + assert_eq!( + m.terminal.status, + terminal.then_some(CommandStatus::Completed) + ); + assert_eq!(m.terminal.exit_code, terminal.then_some(0)); + if !terminal { + assert!(m.terminal.output_collection.is_none()); + assert!(m.terminal.process_failure.is_none()); + assert!(!m.terminal.transport_complete); + } + let diagnostics = m.cleanup.as_ref().unwrap(); + assert_eq!(diagnostics.execution_nonce, 1); + let json = serde_json::to_string(diagnostics).unwrap(); + let debug = format!("{diagnostics:?}"); + for forbidden in [ + PRIVATE, "SECRET", "/private", "password", "poison", "tail", "pre", + ] { + assert!(!json.contains(forbidden), "diagnostics leaked data: {json}"); + assert!( + !debug.contains(forbidden), + "tracing payload leaked data: {debug}" + ); + } + } +} + +#[tokio::test(start_paused = true)] +async fn cancel_failure_does_not_preclude_confirmed_terminal() { + let (f, guard) = Fixture::start(Fault::Cancel); + drop(guard); + f.provider.cancelling.notified().await; + assert!(f.state.lock().unwrap().active.contains_key(PRIVATE)); + let m = f.finish().await; + f.assert_evidence(&m, true, b"pretail"); + assert!(m.terminal.transport_complete); + assert!(matches!( + m.cleanup.as_ref().unwrap().cancel, + CleanupCancel::Failed { .. } + )); + assert!(matches!( + m.cleanup.as_ref().unwrap().drain, + CleanupDrain::TerminalConfirmed + )); + assert!(!m.cleanup.as_ref().unwrap().deadline_expired); + assert!(f.state.lock().unwrap().active.is_empty()); + assert!(f.state.lock().unwrap().cancellation_requested.is_empty()); +} + +#[tokio::test(start_paused = true)] +async fn acknowledged_cancel_drain_faults_keep_distinct_reasons_and_partial_bytes() { + for (fault, reason, bytes) in [ + (Fault::Drain, "provider", b"pre".as_slice()), + (Fault::Sequence, "sequence_gap", b"pre"), + (Fault::Cursor, "cursor_gap", b"pretail"), + (Fault::RetainedChunk, "retained_chunk_invalid", b"pre"), + ( + Fault::TerminalMetadata, + "terminal_metadata_invalid", + b"pretail", + ), + ] { + let (f, guard) = Fixture::start(fault); + drop(guard); + f.provider.cancelling.notified().await; + let m = f.finish().await; + f.assert_evidence(&m, false, bytes); + let diagnostic = serde_json::to_value(m.cleanup.as_ref().unwrap()).unwrap(); + assert_eq!(diagnostic["cancel"]["outcome"], "acknowledged"); + assert_eq!(diagnostic["drain"]["outcome"], "failed"); + assert_eq!(diagnostic["drain"]["reason"]["kind"], reason); + assert_eq!(diagnostic["registration"], "released"); + assert_eq!(diagnostic["deadline_expired"], false); + assert!(f.state.lock().unwrap().active.is_empty()); + } +} + +#[tokio::test(start_paused = true)] +async fn terminal_deadline_never_claims_stop_or_leaves_active_collector() { + let (f, guard) = Fixture::start(Fault::TerminalTimeout); + drop(guard); + f.provider.cancelling.notified().await; + let m = f.finish().await; + f.assert_evidence(&m, false, b"pre"); + assert!(matches!( + m.cleanup.as_ref().unwrap().cancel, + CleanupCancel::Acknowledged + )); + assert!(matches!( + m.cleanup.as_ref().unwrap().drain, + CleanupDrain::DeadlineExceeded + )); + assert!(m.cleanup.as_ref().unwrap().deadline_expired); + assert!(f.state.lock().unwrap().active.is_empty()); +} + +#[tokio::test(start_paused = true)] +async fn cancel_deadline_still_attempts_independent_terminal_observation() { + let (f, guard) = Fixture::start(Fault::CancelHang); + drop(guard); + f.provider.cancelling.notified().await; + let m = f.finish().await; + f.assert_evidence(&m, true, b"pretail"); + assert!(matches!( + m.cleanup.as_ref().unwrap().cancel, + CleanupCancel::DeadlineExceeded + )); + assert!(matches!( + m.cleanup.as_ref().unwrap().drain, + CleanupDrain::TerminalConfirmed + )); + assert!(!m.cleanup.as_ref().unwrap().deadline_expired); + assert!(f.state.lock().unwrap().active.is_empty()); +} + +#[tokio::test(start_paused = true)] +async fn old_cleanup_preserves_new_execution_and_cancellation_registration() { + let (f, guard) = Fixture::start(Fault::Complete); + drop(guard); + f.provider.cancelling.notified().await; + f.state + .lock() + .unwrap() + .active + .get_mut(PRIVATE) + .unwrap() + .execution_nonce = 2; + let m = f.finish().await; + f.assert_evidence(&m, true, b"pretail"); + assert!(matches!( + m.cleanup.as_ref().unwrap().registration, + CleanupRegistration::Superseded + )); + assert_eq!(f.state.lock().unwrap().active[PRIVATE].execution_nonce, 2); + assert!( + f.state + .lock() + .unwrap() + .cancellation_requested + .contains(PRIVATE) + ); +} + +#[tokio::test(start_paused = true)] +async fn terminal_confirmation_is_independent_of_output_collection_completeness() { + let (f, guard) = Fixture::start(Fault::CollectionIncomplete); + drop(guard); + f.provider.cancelling.notified().await; + let m = f.finish().await; + f.assert_evidence(&m, true, b"pretail"); + assert!(!m.terminal.transport_complete); + assert!(matches!( + m.terminal.output_collection, + Some(CommandOutputCollection::Incomplete { .. }) + )); + assert!(matches!( + m.cleanup.as_ref().unwrap().drain, + CleanupDrain::TerminalConfirmed + )); + assert!(f.state.lock().unwrap().active.is_empty()); +} diff --git a/crates/tools/src/bash_output.rs b/crates/tools/src/bash_output.rs index fad3b02e9..080a87fec 100644 --- a/crates/tools/src/bash_output.rs +++ b/crates/tools/src/bash_output.rs @@ -19,7 +19,7 @@ use std::time::{Duration, Instant}; use serde::Serialize; use workdir::{CommandStatus, CommandStream}; -const COMMAND_BYTES: usize = 4 * 1024 * 1024; +pub(crate) const COMMAND_BYTES: usize = 4 * 1024 * 1024; const STORE_BYTES: usize = 32 * 1024 * 1024; const STORE_COUNT: usize = 64; const ORDER_COUNT: usize = 4096; @@ -55,6 +55,7 @@ struct Record { transport_counts: Option<(u64, u64)>, transport_truncation: Option, terminal: Option, + cleanup: Option, } #[derive(Clone, Debug, Serialize)] @@ -95,10 +96,72 @@ pub struct BashOutputMetadata { pub transport_truncation: Option, pub storage_truncated: bool, pub terminal: OutputTerminal, + pub cleanup: Option, pub text_encoding: &'static str, pub ordering: &'static str, } +/// Drop cleanup observations, not assertions that cancellation stopped a process. +/// Correlate with the enclosing output UUID and this host-assigned execution generation; +/// caller identifiers, provider handles, commands and error text are deliberately excluded. +#[derive(Clone, Debug, Serialize)] +pub struct CleanupDiagnostics { + pub execution_nonce: u64, + pub cancel: CleanupCancel, + pub drain: CleanupDrain, + pub deadline_expired: bool, + pub registration: CleanupRegistration, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum CleanupCancel { + Acknowledged, + Failed { reason: CleanupProviderFailure }, + DeadlineExceeded, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum CleanupDrain { + TerminalConfirmed, + Failed { reason: CleanupFailure }, + DeadlineExceeded, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum CleanupFailure { + Provider { reason: CleanupProviderFailure }, + SequenceGap, + CursorGap, + DeliveryBudgetExceeded, + RetainedChunkInvalid, + TerminalMetadataInvalid, +} + +#[derive(Clone, Debug, Serialize)] +pub struct CleanupProviderFailure { + pub failure_reason: Option, + pub denial_reason: Option, +} + +impl From<&workdir::WorkdirError> for CleanupProviderFailure { + fn from(error: &workdir::WorkdirError) -> Self { + Self { + failure_reason: error.failure_reason(), + denial_reason: error.denial_reason(), + } + } +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CleanupRegistration { + Released, + Superseded, +} + /// A bounded reservation acquired before command start. Drop terminalizes partial evidence. pub struct BashOutputWriter { record: Arc>, @@ -214,6 +277,7 @@ impl BashOutputView { transport_counts: None, transport_truncation: None, terminal: None, + cleanup: None, })); store.records.push_back(record.clone()); store.reserved += COMMAND_BYTES; @@ -313,6 +377,7 @@ impl Record { transport_truncation: self.transport_truncation.clone(), storage_truncated: self.truncated || self.transport_truncation.is_some(), terminal: self.terminal.clone().expect("finalized record"), + cleanup: self.cleanup.clone(), text_encoding: "utf8_lossy", ordering: "provider_read_observation", } @@ -342,6 +407,14 @@ async fn reap(weak: Weak>, mut shutdown: tokio::sync::watch::Receiv } impl BashOutputWriter { + pub(crate) fn id(&self) -> String { + self.record.lock().unwrap().id.clone() + } + + pub(crate) fn finish_cleanup(&mut self, diagnostics: CleanupDiagnostics) { + self.record.lock().unwrap().cleanup = Some(diagnostics); + } + pub fn append_chunk(&mut self, chunk: &workdir::CommandByteChunk) -> Result<(), String> { let expected = { let r = self.record.lock().unwrap(); From 28b7cc414bb66d065cded13ccd75f2997fd43352 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 06:04:39 +0900 Subject: [PATCH 08/46] fix: retain external workdir results independently of audit failures --- crates/server-api/src/lib.rs | 8 +- crates/tools/src/file_target.rs | 1 + crates/workdir/src/http.rs | 89 ++- crates/workdir/src/lib.rs | 6 + .../src/feature/builtin/manage_workdir.rs | 100 +++- crates/workspace-server/src/server.rs | 176 ++++-- .../src/server/external_operation_audit.rs | 36 ++ .../server/tests/workspace_admission_tests.rs | 8 +- .../workspace_admission_tests/audit_tests.rs | 546 ++++++++++++++++++ 9 files changed, 907 insertions(+), 63 deletions(-) create mode 100644 crates/workspace-server/src/server/external_operation_audit.rs create mode 100644 crates/workspace-server/src/server/tests/workspace_admission_tests/audit_tests.rs diff --git a/crates/server-api/src/lib.rs b/crates/server-api/src/lib.rs index 3b448d409..951a21566 100644 --- a/crates/server-api/src/lib.rs +++ b/crates/server-api/src/lib.rs @@ -125,6 +125,8 @@ pub struct WorkdirOperationApiError { pub message: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub diagnostics: Option>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub audit_failures: Vec, #[serde(skip, default = "default_repository_error_status")] status: u16, /// Server-local metadata retained by generated error response extensions, never the wire body. @@ -156,6 +158,7 @@ impl WorkdirOperationApiError { code: None, message: error.message, diagnostics: Some(error.diagnostics), + audit_failures: Vec::new(), status: error.status, workdir_log: None, denial_reason: None, @@ -170,6 +173,7 @@ impl WorkdirOperationApiError { code: Some(error.code), message: error.message, diagnostics: None, + audit_failures: Vec::new(), status, workdir_log: None, denial_reason: error.denial_reason, @@ -7522,7 +7526,9 @@ pub struct CurrentWorkerWorkdirOperationRequest { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(transparent)] -pub struct CurrentWorkerWorkdirOperationResponse(pub workdir::http::WorkdirSessionOperationResult); +pub struct CurrentWorkerWorkdirOperationResponse( + pub workdir::http::WorkdirSessionOperationResponse, +); #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] diff --git a/crates/tools/src/file_target.rs b/crates/tools/src/file_target.rs index c3bfb535e..af307e0cc 100644 --- a/crates/tools/src/file_target.rs +++ b/crates/tools/src/file_target.rs @@ -167,6 +167,7 @@ pub(crate) fn checked_error(error: workdir::WorkdirError) -> ToolError { | WorkdirError::IsDirectory { .. } | WorkdirError::InvalidGlob { .. } | WorkdirError::InvalidRegex { .. } => "checkout_invalid", + WorkdirError::NotDispatched => "checkout_not_dispatched", WorkdirError::Unavailable { .. } | WorkdirError::SessionClosed => "checkout_unavailable", // IO/transport failures may arrive after effects have begun. Do not infer // no-effects from their text or strip explicit OutcomeUnknown evidence. diff --git a/crates/workdir/src/http.rs b/crates/workdir/src/http.rs index f99f804c5..805aea2de 100644 --- a/crates/workdir/src/http.rs +++ b/crates/workdir/src/http.rs @@ -119,6 +119,51 @@ pub enum WorkdirSessionOperationResult { CommandCancel, } +/// Audit persistence is independent of the provider result and never authorizes a resend. +/// Only server-generated correlation and closed classifications cross this boundary. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct WorkdirAuditFailure { + pub operation_id: String, + pub stage: WorkdirAuditStage, + pub operation_outcome: WorkdirAuditOperationOutcome, + pub reason: WorkdirAuditFailureReason, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum WorkdirAuditStage { + BeforeDispatch, + DispatchRejected, + ResponseReceived, + ResponseUnknown, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum WorkdirAuditOperationOutcome { + NotDispatched, + Succeeded, + Failed { code: WorkdirTransportErrorCode }, + Unknown, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum WorkdirAuditFailureReason { + AuditStoreUnavailable, +} + +/// A successful operation can still have failed audit persistence. Consumers must +/// surface that degradation separately, without discarding the result or resending. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct WorkdirSessionOperationResponse { + #[serde(flatten)] + pub result: WorkdirSessionOperationResult, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub audit_failures: Vec, +} + /// Dispatch one typed operation against any provider-neutral Workdir session. /// /// Runtime HTTP handlers, outbound External providers, and Backend brokers can @@ -208,6 +253,7 @@ pub enum WorkdirTransportErrorCode { NotFound, Conflict, OutcomeUnknown, + NotDispatched, Unsupported, InvalidRequest, Denied, @@ -231,6 +277,7 @@ impl WorkdirTransportErrorCode { Self::NotFound => "not_found", Self::Conflict => "conflict", Self::OutcomeUnknown => "outcome_unknown", + Self::NotDispatched => "not_dispatched", Self::Unsupported => "unsupported", Self::InvalidRequest => "invalid_request", Self::Denied => "denied", @@ -261,7 +308,7 @@ impl WorkdirTransportErrorCode { | Self::SymlinkTargetIsDirectory | Self::IsDirectory | Self::SymlinkDirectoryNotTraversed => 400, - Self::Unavailable => 503, + Self::Unavailable | Self::NotDispatched => 503, Self::Io | Self::Internal | Self::OutcomeUnknown => 500, Self::Transport => 502, } @@ -298,6 +345,9 @@ impl WorkdirTransportError { Code::OutcomeUnknown, "Workdir operation may have effects; inspect before retrying", ), + WorkdirError::NotDispatched => { + (Code::NotDispatched, "Workdir operation was not dispatched") + } WorkdirError::NotFound(_) => (Code::NotFound, "Workdir path was not found"), WorkdirError::Conflict(_) => ( Code::Conflict, @@ -410,6 +460,7 @@ impl WorkdirTransportError { }, Code::UnknownCommand => WorkdirError::UnknownCommand("".to_string()), Code::Unavailable => WorkdirError::Unavailable(self.message), + Code::NotDispatched => WorkdirError::NotDispatched, Code::Io => WorkdirError::Io { path: "".into(), source: std::io::Error::other(self.message), @@ -1210,6 +1261,37 @@ pub use client::{ mod tests { use super::*; + #[test] + fn operation_response_keeps_success_and_independent_audit_failure() { + let original = + WorkdirSessionOperationResult::CommandStart(crate::CommandHandle("command-a".into())); + let response = WorkdirSessionOperationResponse { + result: original.clone(), + audit_failures: vec![WorkdirAuditFailure { + operation_id: "op-00000000-0000-7000-8000-000000000001".into(), + stage: WorkdirAuditStage::ResponseReceived, + operation_outcome: WorkdirAuditOperationOutcome::Succeeded, + reason: WorkdirAuditFailureReason::AuditStoreUnavailable, + }], + }; + let json = serde_json::to_value(&response).unwrap(); + assert_eq!(json["operation"], "command_start"); + assert_eq!(json["result"], "command-a"); + assert_eq!( + json["audit_failures"][0]["operation_outcome"]["kind"], + "succeeded" + ); + assert_eq!( + serde_json::from_value::(json).unwrap(), + response + ); + let normal = serde_json::to_value(&original).unwrap(); + let decoded: WorkdirSessionOperationResponse = + serde_json::from_value(normal.clone()).unwrap(); + assert!(decoded.audit_failures.is_empty()); + assert_eq!(serde_json::to_value(decoded).unwrap(), normal); + } + #[test] fn transport_error_round_trip_keeps_public_classification() { for (code, expected_status, expected_error) in [ @@ -1229,6 +1311,11 @@ mod tests { 409, "The target file's content or existence changed since it was last observed", ), + ( + WorkdirTransportErrorCode::NotDispatched, + 503, + "not dispatched", + ), (WorkdirTransportErrorCode::Unsupported, 400, "unsupported"), (WorkdirTransportErrorCode::Denied, 403, "denied"), ( diff --git a/crates/workdir/src/lib.rs b/crates/workdir/src/lib.rs index 1a515ac16..c208253f0 100644 --- a/crates/workdir/src/lib.rs +++ b/crates/workdir/src/lib.rs @@ -409,6 +409,7 @@ pub enum WorkdirFailureReason { SessionClosed, ProviderUnavailable, CommandTerminalMissing, + AuditStoreUnavailable, HttpTimeout, HttpConnect, HttpTransfer, @@ -422,6 +423,7 @@ impl WorkdirFailureReason { Self::SessionClosed => "session_closed", Self::ProviderUnavailable => "provider_unavailable", Self::CommandTerminalMissing => "command_terminal_missing", + Self::AuditStoreUnavailable => "audit_store_unavailable", Self::HttpTimeout => "http_timeout", Self::HttpConnect => "http_connect", Self::HttpTransfer => "http_transfer", @@ -466,6 +468,10 @@ pub enum WorkdirError { #[error("Workdir session is unavailable: {0}")] Unavailable(String), + /// The broker has proof that this operation never reached the provider. + #[error("Workdir operation was not dispatched")] + NotDispatched, + #[error("Workdir operation failed")] OperationFailed, diff --git a/crates/worker/src/feature/builtin/manage_workdir.rs b/crates/worker/src/feature/builtin/manage_workdir.rs index 12cfdb65d..d8f2302f2 100644 --- a/crates/worker/src/feature/builtin/manage_workdir.rs +++ b/crates/worker/src/feature/builtin/manage_workdir.rs @@ -291,6 +291,11 @@ impl WorkspaceAttachedWorkdirSession { .execute(request) .map_err(workspace_workdir_error)?; if !response.is_success() { + if let Ok(error) = + serde_json::from_str::(&response.body) + { + notify_workdir_audit_failures(&error.audit_failures); + } return Err( serde_json::from_str::(&response.body) .map(workdir::http::WorkdirTransportError::into_workdir_error) @@ -302,11 +307,14 @@ impl WorkspaceAttachedWorkdirSession { }), ); } - serde_json::from_str(&response.body).map_err(|error| { - WorkdirError::Transport(format!( - "failed to decode Workspace Workdir operation result: {error}" - )) - }) + let response: workdir::http::WorkdirSessionOperationResponse = + serde_json::from_str(&response.body).map_err(|_| { + WorkdirError::Transport( + "failed to decode Workspace Workdir operation result".into(), + ) + })?; + notify_workdir_audit_failures(&response.audit_failures); + Ok(response.result) } fn mismatch(expected: &str) -> WorkdirError { @@ -316,6 +324,27 @@ impl WorkspaceAttachedWorkdirSession { } } +// Notification is separate from the operation return: callers retain successful command +// handles, checkout observations and original failures, and must not resend for audit loss. +fn notify_workdir_audit_failures(failures: &[workdir::http::WorkdirAuditFailure]) { + use sha2::Digest; + for failure in failures { + // Response strings are untrusted. Never copy them into the receiving process log. + let operation_id_hash = sha2::Sha256::digest(failure.operation_id.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + tracing::error!( + event = "external_workdir_audit_failed", + operation_id_hash, + audit_stage = ?failure.stage, + audit_reason = ?failure.reason, + operation_outcome = ?failure.operation_outcome, + "External Workdir audit persistence failed; operation result is independent; do not resend" + ); + } +} + fn workspace_workdir_error(error: WorkspaceClientError) -> WorkdirError { match error { WorkspaceClientError::Unavailable(message) => WorkdirError::Unavailable(message), @@ -1653,6 +1682,67 @@ mod tests { } } + #[tokio::test] + async fn attached_session_keeps_operation_results_when_audit_persistence_failed() { + let failure = json!({ + "operation_id":"op-safe-id", "stage":"response_received", + "reason":"audit_store_unavailable", "operation_outcome":{"kind":"succeeded"} + }); + let mut responses = vec![response(json!({ + "operation":"command_start", "result":"retained-command", + "audit_failures":[failure] + }))]; + for (code, status, outcome) in [ + ("denied", 403, json!({"kind":"failed","code":"denied"})), + ("outcome_unknown", 500, json!({"kind":"unknown"})), + ("not_dispatched", 503, json!({"kind":"not_dispatched"})), + ] { + let mut failure = failure.clone(); + failure["operation_outcome"] = outcome; + responses.push(WorkspaceResponse { status, body: json!({ + "code":code, "message":"safe provider classification", "audit_failures":[failure] + }).to_string() }); + } + let client = Arc::new(RecordingWorkspaceClient::new(responses)); + let session = WorkspaceAttachedWorkdirSession::handle(client.clone(), "checkout"); + let handle = session + .start_command(CommandRequest { + command: "true".into(), + timeout_secs: 1, + transport_byte_budget: workdir::MAX_COMMAND_TRANSPORT_BYTES, + cwd: workdir::WorkdirPath::root(), + tool_call_id: None, + }) + .await + .unwrap(); + assert_eq!(handle.0, "retained-command"); + let request = workdir::CheckoutRequest { + target: workdir::WorkdirPath::new("file.txt").unwrap(), + validator: vec![1], + operation: workdir::CheckoutOperation::Write { + content: b"new".to_vec(), + expected_hash: [0; 32], + }, + }; + assert!(matches!( + session.checkout_execute(request.clone()).await, + Err(WorkdirError::Denied(_)) + )); + assert!(matches!( + session.checkout_execute(request.clone()).await, + Err(WorkdirError::OutcomeUnknown(_)) + )); + assert!(matches!( + session.checkout_execute(request).await, + Err(WorkdirError::NotDispatched) + )); + assert_eq!( + client.requests().len(), + 4, + "audit loss must not trigger a resend" + ); + } + #[tokio::test] async fn attached_session_does_not_send_worker_local_bash_spill_path() { let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({ diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 76d394da4..69a1b035d 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -26,6 +26,7 @@ use subjektiv::api::*; mod backend_job_profile; +mod external_operation_audit; mod subject_authorization; mod subjektiv_job; mod subjektiv_journal; @@ -933,6 +934,10 @@ struct ExternalProviderAuditContext { /// output confirmation from the old connection; unconfirmed cleanup is not reported as success. /// Only confirmed provider shutdown can replace per-command confirmation. Reconnection does /// not prove that missing commands have stopped. +/// +/// Required request auditing fails closed before dispatch. Completion audit failure cannot +/// undo a provider result: it is emitted independently and retained in the current API +/// response, including successful command handles. Audit loss never triggers a resend. struct ExternalProviderWorkdirSession { workdir: workdir::Workdir, connection: Arc, @@ -991,18 +996,22 @@ impl ExternalProviderWorkdirSession { async fn audit_operation( &self, operation_id: &ExternalWorkdirOperationId, - operation_kind: &str, + operation_kind: &'static str, outcome: &str, - ) -> std::result::Result<(), workdir::WorkdirError> { + stage: workdir::http::WorkdirAuditStage, + operation_outcome: workdir::http::WorkdirAuditOperationOutcome, + ) -> std::result::Result<(), workdir::http::WorkdirAuditFailure> { let Some(audit) = &self.audit else { return Ok(()); }; let audit = audit.clone(); + let operation_id_for_notification = operation_id.as_str().to_owned(); + let operation_kind_for_notification = operation_kind; let operation_id = operation_id.clone(); let operation_kind = operation_kind.to_owned(); let outcome = outcome.to_owned(); let connection = self.connection.clone(); - audit + let result = audit .store .db_executor() .run(move || { @@ -1024,12 +1033,17 @@ impl ExternalProviderWorkdirSession { &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), ) }) - .await - .map_err(|error| { - workdir::WorkdirError::Unavailable(format!( - "External Workdir operation audit failed: {error}" - )) - }) + .await; + result.map_err(|_| { + let failure = workdir::http::WorkdirAuditFailure { + operation_id: operation_id_for_notification, + stage, + operation_outcome, + reason: workdir::http::WorkdirAuditFailureReason::AuditStoreUnavailable, + }; + external_operation_audit::notify(failure.clone(), operation_kind_for_notification); + failure + }) } async fn operate( @@ -1079,8 +1093,19 @@ impl ExternalProviderWorkdirSession { } let operation_id = ExternalWorkdirOperationId::new(format!("op-{}", Uuid::now_v7())) .map_err(workdir::WorkdirError::InvalidArgument)?; - self.audit_operation(&operation_id, operation_kind, "requested") - .await?; + use workdir::http::{WorkdirAuditOperationOutcome as Outcome, WorkdirAuditStage as Stage}; + self.audit_operation( + &operation_id, + operation_kind, + "requested", + Stage::BeforeDispatch, + Outcome::NotDispatched, + ) + .await + .map_err(|_| { + workdir::WorkdirError::NotDispatched + .with_failure_reason(workdir::WorkdirFailureReason::AuditStoreUnavailable) + })?; let (response, receive) = tokio::sync::oneshot::channel(); let mut cancel_guard = ExternalOperationCancelGuard { sender: self.connection.sender.clone(), @@ -1097,39 +1122,72 @@ impl ExternalProviderWorkdirSession { .await .is_err() { - self.audit_operation(&operation_id, operation_kind, "provider_unavailable") - .await?; - return Err(workdir::WorkdirError::Unavailable( - "External Workdir provider disconnected".to_string(), - )); + cancel_guard.disarm(); + // A rejected channel send proves there was no provider dispatch. + let _audit_result = self + .audit_operation( + &operation_id, + operation_kind, + "not_dispatched", + Stage::DispatchRejected, + Outcome::NotDispatched, + ) + .await; + return Err(workdir::WorkdirError::NotDispatched + .with_failure_reason(workdir::WorkdirFailureReason::ProviderUnavailable)); } - let result = match tokio::time::timeout(operation_timeout, receive).await { + let (result, stage) = match tokio::time::timeout(operation_timeout, receive).await { Ok(result) => { cancel_guard.disarm(); match result { - Ok(result) => result.map_err(WorkdirTransportError::into_workdir_error), - Err(_) => Err(workdir::WorkdirError::Unavailable( - "External Workdir provider disconnected".to_string(), - )), + Ok(result) => ( + result.map_err(WorkdirTransportError::into_workdir_error), + Stage::ResponseReceived, + ), + Err(_) => ( + Err(workdir::WorkdirError::OutcomeUnknown( + "External Workdir provider response lost; inspect before retrying" + .to_string(), + )), + Stage::ResponseUnknown, + ), } } Err(_) => { cancel_guard.cancel(); - Err(workdir::WorkdirError::Unavailable( - "External Workdir provider operation timed out".to_string(), - )) + ( + Err(workdir::WorkdirError::OutcomeUnknown( + "External Workdir provider response timed out; inspect before retrying" + .to_string(), + )), + Stage::ResponseUnknown, + ) } }; - self.audit_operation( - &operation_id, - operation_kind, - if result.is_ok() { - "completed" - } else { - "failed" + let outcome = match &result { + Ok(_) => Outcome::Succeeded, + Err(error) + if matches!( + error.classification_source(), + workdir::WorkdirError::OutcomeUnknown(_) + ) => + { + Outcome::Unknown + } + Err(error) => Outcome::Failed { + code: WorkdirTransportError::from_workdir_error(error).code, }, - ) - .await?; + }; + let audit_outcome = match outcome { + Outcome::Succeeded => "completed", + Outcome::Failed { .. } => "failed", + Outcome::Unknown => "outcome_unknown", + Outcome::NotDispatched => unreachable!("provider result follows dispatch"), + }; + // Failure is notified independently, never substituted for the provider result. + let _audit_result = self + .audit_operation(&operation_id, operation_kind, audit_outcome, stage, outcome) + .await; result } @@ -13169,26 +13227,38 @@ impl server_api::ServerApi for ServerApiContractService { ), ..Default::default() }; - let Json(response) = scoped_execute_current_worker_workdir_operation( - State( - self.workspace_api() - .map_err(server_api::WorkdirOperationApiError::api)? - .clone(), - ), - AxumPath(ScopedWorkspacePath { workspace_id }), - current_worker_contract_headers(&context).map_err(|error| { - let mut error = server_api::WorkdirOperationApiError::api(error); - error.workdir_log = Some(fallback_log); - error - })?, - Json(WorkspaceWorkdirSessionOperationRequest { - target_workdir: request.target_workdir, - operation: request.operation, - }), - ) - .await - .map_err(InternalWorkdirOperationError::into_contract_error)?; - Ok(server_api::CurrentWorkerWorkdirOperationResponse(response)) + let (result, audit_failures) = + external_operation_audit::capture(scoped_execute_current_worker_workdir_operation( + State( + self.workspace_api() + .map_err(server_api::WorkdirOperationApiError::api)? + .clone(), + ), + AxumPath(ScopedWorkspacePath { workspace_id }), + current_worker_contract_headers(&context).map_err(|error| { + let mut error = server_api::WorkdirOperationApiError::api(error); + error.workdir_log = Some(fallback_log); + error + })?, + Json(WorkspaceWorkdirSessionOperationRequest { + target_workdir: request.target_workdir, + operation: request.operation, + }), + )) + .await; + match result { + Ok(Json(result)) => Ok(server_api::CurrentWorkerWorkdirOperationResponse( + workdir::http::WorkdirSessionOperationResponse { + result, + audit_failures, + }, + )), + Err(error) => { + let mut error = error.into_contract_error(); + error.audit_failures = audit_failures; + Err(error) + } + } } async fn external_workdir_grant_create( diff --git a/crates/workspace-server/src/server/external_operation_audit.rs b/crates/workspace-server/src/server/external_operation_audit.rs new file mode 100644 index 000000000..1b36808d9 --- /dev/null +++ b/crates/workspace-server/src/server/external_operation_audit.rs @@ -0,0 +1,36 @@ +//! Per-request audit notifications, independent of provider result classification. +//! +//! The scope follows the operation future, not a session-wide last-result slot: concurrent +//! requests cannot consume one another's diagnostics. Every failure is also emitted immediately +//! for direct session callers and requests cancelled before a response. This is notification, +//! not a replacement durable audit ledger or crash-recovery/effect-deduplication guarantee. + +use std::{cell::RefCell, future::Future}; +use workdir::http::WorkdirAuditFailure; + +tokio::task_local! { + static FAILURES: RefCell>; +} + +pub(super) async fn capture(future: impl Future) -> (T, Vec) { + FAILURES + .scope(RefCell::new(Vec::new()), async { + let result = future.await; + let failures = FAILURES.with(|failures| failures.take()); + (result, failures) + }) + .await +} + +pub(super) fn notify(failure: WorkdirAuditFailure, operation: &'static str) { + tracing::error!( + event = "external_workdir_audit_failed", + operation, + operation_id = %failure.operation_id, + audit_stage = ?failure.stage, + audit_reason = ?failure.reason, + operation_outcome = ?failure.operation_outcome, + "External Workdir audit persistence failed; operation result is independent; do not resend" + ); + let _ = FAILURES.try_with(|failures| failures.borrow_mut().push(failure)); +} diff --git a/crates/workspace-server/src/server/tests/workspace_admission_tests.rs b/crates/workspace-server/src/server/tests/workspace_admission_tests.rs index 877ce5dff..facd8fe80 100644 --- a/crates/workspace-server/src/server/tests/workspace_admission_tests.rs +++ b/crates/workspace-server/src/server/tests/workspace_admission_tests.rs @@ -1,3 +1,5 @@ +mod audit_tests; + use super::*; use sha2::Digest; use std::sync::atomic::AtomicI64; @@ -459,7 +461,7 @@ async fn provider_error_and_disconnect_release_dispatch_admission() { assert_json( result, if disconnect { - StatusCode::SERVICE_UNAVAILABLE + StatusCode::INTERNAL_SERVER_ERROR } else { StatusCode::FORBIDDEN }, @@ -575,8 +577,8 @@ async fn provider_timeout_releases_admission_without_wall_clock_wait() { let (long, _reply) = f.start_read("1", &mut a).await; tokio::time::pause(); tokio::time::advance(std::time::Duration::from_secs(31)).await; - let body = assert_json(long.await.unwrap(), StatusCode::SERVICE_UNAVAILABLE).await; - assert_eq!(body["code"], "unavailable"); + let body = assert_json(long.await.unwrap(), StatusCode::INTERNAL_SERVER_ERROR).await; + assert_eq!(body["code"], "outcome_unknown"); tokio::time::resume(); let gate = f.server.admission_for_workspace(TEST_WORKSPACE_ID).await; drop( diff --git a/crates/workspace-server/src/server/tests/workspace_admission_tests/audit_tests.rs b/crates/workspace-server/src/server/tests/workspace_admission_tests/audit_tests.rs new file mode 100644 index 000000000..212198b6f --- /dev/null +++ b/crates/workspace-server/src/server/tests/workspace_admission_tests/audit_tests.rs @@ -0,0 +1,546 @@ +//! Integration coverage for signed loopback HTTP -> production admission/auth -> external +//! Workdir session -> real SQLite audit persistence. Only the external provider boundary +//! is substituted with a controlled command receiver. Temporary DBs, listeners and bounded +//! requests are isolated per case; these run in the normal workspace-server library suite. + +use super::*; +use std::io::Write; +use tracing::instrument::WithSubscriber; + +const AUDIT_SECRET: &str = "audit-store-password-secret"; +const WRITE_SECRET: &[u8] = b"write-content-secret"; +const PATH_SECRET: &str = "private-request-path-secret.txt"; +const PROVIDER_SECRET: &str = "/host/private/provider-secret"; + +type ProviderReply = + oneshot::Sender>; + +#[derive(Clone, Default)] +struct CapturedAuditLog(Arc>>); + +impl Write for CapturedAuditLog { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +struct PendingHttpWrite { + task: tokio::task::JoinHandle<(StatusCode, Value)>, + proof: String, +} + +impl Drop for PendingHttpWrite { + fn drop(&mut self) { + self.task.abort(); + } +} + +struct ProviderProbe { + receiver: mpsc::Receiver, + dispatches: usize, +} + +impl ProviderProbe { + async fn receive_write(&mut self, pending: &mut PendingHttpWrite) -> (String, ProviderReply) { + let command = tokio::time::timeout(BOUND, async { + tokio::select! { + command = self.receiver.recv() => command.expect("provider channel closed"), + response = &mut pending.task => { + panic!("HTTP completed before write dispatch: {:?}", response.unwrap()); + } + } + }) + .await + .expect("signed HTTP write never reached provider"); + let ExternalProviderCommand::Operation { + operation_id, + operation, + response, + } = command + else { + panic!("expected write dispatch, received cancellation") + }; + self.dispatches += 1; + let WorkdirSessionOperation::Write(request) = operation else { + panic!("expected mutation, got {operation:?}") + }; + assert_eq!( + request.path, + workdir::WorkdirPath::new(PATH_SECRET).unwrap() + ); + assert_eq!(request.content, WRITE_SECRET); + assert_eq!(request.expected_hash, None); + (operation_id.as_str().to_owned(), response) + } + + fn assert_dispatches(&mut self, expected: usize) { + while let Ok(command) = self.receiver.try_recv() { + if matches!(command, ExternalProviderCommand::Operation { .. }) { + self.dispatches += 1; + } + } + assert_eq!( + self.dispatches, expected, + "unexpected mutation dispatch/retry" + ); + } +} + +impl Fixture { + fn attach_audited_writer(&self) -> ProviderProbe { + let receiver = self.attach("1"); + // Setup changes only: real signed requests still traverse production authorization. + // All three capability sources must agree before the session is first resolved. + let connection = rusqlite::Connection::open(&self.api.config.database_path).unwrap(); + assert_eq!( + connection + .execute( + "UPDATE external_workdir_grants SET permissions='read_write' \ + WHERE workspace_id=?1 AND grant_id='external-1'", + [TEST_WORKSPACE_ID], + ) + .unwrap(), + 1 + ); + assert_eq!( + connection + .execute( + "UPDATE worker_workdir_links SET capabilities='read_write' \ + WHERE workspace_id=?1 AND runtime_id='workdirless-runtime' \ + AND worker_id='1' AND alias='attachment' AND unlinked_at IS NULL", + [TEST_WORKSPACE_ID], + ) + .unwrap(), + 1 + ); + let mut registry = self.api.external_workdir_providers.lock().unwrap(); + let provider = registry.connections.get_mut("external-1").unwrap(); + Arc::get_mut(provider).unwrap().capabilities = + workdir::WorkdirSessionCapabilities::READ_WRITE; + ProviderProbe { + receiver, + dispatches: 0, + } + } +} + +struct AuditHttpFixture { + fixture: Fixture, + logs: CapturedAuditLog, + client: reqwest::Client, + base_url: String, + server: tokio::task::JoinHandle<()>, +} + +impl Drop for AuditHttpFixture { + fn drop(&mut self) { + self.server.abort(); + } +} + +impl AuditHttpFixture { + async fn new() -> (Self, ProviderProbe) { + let fixture = Fixture::new().await; + let provider = fixture.attach_audited_writer(); + let logs = CapturedAuditLog::default(); + let writer = logs.clone(); + let subscriber = tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .json() + .flatten_event(true) + .with_writer(move || writer.clone()) + .finish(); + let dispatch = tracing::Dispatch::new(subscriber); + // Axum spawns connection tasks: scope the subscriber around each request rather + // than relying on the listener task's thread-local subscriber being inherited. + let app = fixture.app.clone().layer(axum::middleware::from_fn( + move |request: Request, next: axum::middleware::Next| { + let dispatch = dispatch.clone(); + async move { next.run(request).with_subscriber(dispatch).await } + }, + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + let client = reqwest::Client::builder() + .no_proxy() + .timeout(BOUND) + .build() + .unwrap(); + ( + Self { + fixture, + logs, + client, + base_url, + server, + }, + provider, + ) + } + + fn fail_audit(&self, before_dispatch: bool) { + let filter = if before_dispatch { "=" } else { "<>" }; + rusqlite::Connection::open(&self.fixture.api.config.database_path) + .unwrap() + .execute_batch(&format!( + "CREATE TRIGGER fail_operation_audit BEFORE INSERT ON audit_events \ + WHEN NEW.action='external_workdir_operation' AND NEW.outcome {filter} 'requested' \ + BEGIN SELECT RAISE(FAIL, '{AUDIT_SECRET}'); END;" + )) + .unwrap(); + } + + fn start_write(&self) -> PendingHttpWrite { + let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-session/operations"); + let body = serde_json::to_vec(&WorkspaceWorkdirSessionOperationRequest { + target_workdir: "attachment".into(), + operation: WorkdirSessionOperation::Write(workdir::WriteRequest { + path: workdir::WorkdirPath::new(PATH_SECRET).unwrap(), + content: WRITE_SECRET.to_vec(), + expected_hash: None, + }), + }) + .unwrap(); + let request = self.fixture.request( + Some("1"), + &path, + body.clone(), + START, + worker_runtime::auth::WORKSPACE_REQUEST_PERMISSION, + ); + let proof = request.headers()[worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER] + .to_str() + .unwrap() + .to_owned(); + let url = format!("{}{path}", self.base_url); + let client = self.client.clone(); + let headers = request.headers().clone(); + let task = tokio::spawn(async move { + let response = client + .post(url) + .headers(headers) + .body(body) + .send() + .await + .unwrap(); + let status = response.status(); + assert_eq!(response.headers()[CONTENT_TYPE], "application/json"); + let bytes = response.bytes().await.unwrap(); + (status, serde_json::from_slice(&bytes).unwrap()) + }); + PendingHttpWrite { task, proof } + } + + async fn finish( + &self, + mut pending: PendingHttpWrite, + expected_status: StatusCode, + ) -> (Value, Vec) { + let (status, body) = tokio::time::timeout(BOUND, &mut pending.task) + .await + .expect("HTTP did not finish; possible automatic retry") + .unwrap(); + assert_eq!(status, expected_status, "{body}"); + let text = String::from_utf8(self.logs.0.lock().unwrap().clone()).unwrap(); + let encoded_content = serde_json::to_string(WRITE_SECRET).unwrap(); + for output in [body.to_string(), text.clone()] { + for secret in [ + AUDIT_SECRET, + std::str::from_utf8(WRITE_SECRET).unwrap(), + encoded_content.as_str(), + PATH_SECRET, + PROVIDER_SECRET, + pending.proof.as_str(), + self.fixture._root.path().to_str().unwrap(), + ] { + assert!(!output.contains(secret), "sensitive data escaped: {output}"); + } + } + let events = text + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + (body, events) + } + + fn assert_audits(&self, expected: &[(&str, &str)]) { + let connection = + rusqlite::Connection::open(&self.fixture.api.config.database_path).unwrap(); + let mut statement = connection + .prepare( + "SELECT request_id, outcome, summary FROM audit_events \ + WHERE workspace_id=?1 AND action='external_workdir_operation' \ + AND target_id='external-1' ORDER BY rowid", + ) + .unwrap(); + let rows: Vec<(String, String, String)> = statement + .query_map([TEST_WORKSPACE_ID], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + }) + .unwrap() + .collect::>() + .unwrap(); + let actual: Vec<(&str, &str)> = rows + .iter() + .map(|(operation_id, outcome, _)| (operation_id.as_str(), outcome.as_str())) + .collect(); + assert_eq!(actual, expected, "persisted audits: {rows:?}"); + for (_, _, summary) in rows { + assert!(summary.contains("operation_kind=write"), "{summary}"); + for secret in [ + AUDIT_SECRET, + PATH_SECRET, + PROVIDER_SECRET, + "write-content-secret", + ] { + assert!( + !summary.contains(secret), + "sensitive audit summary: {summary}" + ); + } + } + } + + fn apply_provider_write(&self) { + std::fs::write(self.effect_path(), WRITE_SECRET).unwrap(); + } + + fn effect_path(&self) -> std::path::PathBuf { + self.fixture._root.path().join("provider-effect") + } +} + +fn write_result() -> WorkdirSessionOperationResult { + WorkdirSessionOperationResult::Write(workdir::WriteResult { + bytes_written: WRITE_SECRET.len(), + created: true, + }) +} + +fn assert_audit_failure( + body: &Value, + events: &[Value], + stage: &str, + debug_stage: &str, + outcome: Value, + debug_outcome: &str, +) -> String { + let failures = body["audit_failures"] + .as_array() + .expect("independent audit receipt"); + assert_eq!(failures.len(), 1, "{body}"); + let operation_id = failures[0]["operation_id"].as_str().unwrap(); + assert!(operation_id.starts_with("op-"), "{body}"); + assert_eq!( + failures[0], + json!({ + "operation_id": operation_id, + "stage": stage, + "reason": "audit_store_unavailable", + "operation_outcome": outcome, + }) + ); + let notifications: Vec<&Value> = events + .iter() + .filter(|event| event["event"] == "external_workdir_audit_failed") + .collect(); + assert_eq!(notifications.len(), 1, "{events:?}"); + let event = notifications[0]; + assert_eq!(event["operation_id"], operation_id); + assert_eq!(event["operation"], "write"); + assert_eq!(event["audit_stage"], debug_stage); + assert_eq!(event["audit_reason"], "AuditStoreUnavailable"); + assert_eq!(event["operation_outcome"], debug_outcome); + operation_id.to_owned() +} + +fn assert_failure_reason(events: &[Value], reason: &str) { + let errors: Vec<&Value> = events + .iter() + .filter(|event| event["event"] == "api_error") + .collect(); + assert_eq!(errors.len(), 1, "{events:?}"); + assert_eq!(errors[0]["failure_reason"], reason, "{events:?}"); +} + +#[tokio::test] +async fn successful_write_after_audit_failure_preserves_result_and_does_not_repeat_mutation() { + let (h, mut provider) = AuditHttpFixture::new().await; + h.fail_audit(false); + let mut pending = h.start_write(); + let (dispatched_id, reply) = provider.receive_write(&mut pending).await; + h.apply_provider_write(); + reply.send(Ok(write_result())).unwrap(); + let (mut body, events) = h.finish(pending, StatusCode::OK).await; + let receipt_id = assert_audit_failure( + &body, + &events, + "response_received", + "ResponseReceived", + json!({"kind":"succeeded"}), + "Succeeded", + ); + assert_eq!(receipt_id, dispatched_id); + body.as_object_mut().unwrap().remove("audit_failures"); + assert_eq!(body, serde_json::to_value(write_result()).unwrap()); + assert_eq!(std::fs::read(h.effect_path()).unwrap(), WRITE_SECRET); + provider.assert_dispatches(1); + h.assert_audits(&[(&dispatched_id, "requested")]); +} + +#[tokio::test] +async fn denied_write_after_audit_failure_preserves_provider_error_and_separate_receipt() { + let (h, mut provider) = AuditHttpFixture::new().await; + h.fail_audit(false); + let mut pending = h.start_write(); + let (dispatched_id, reply) = provider.receive_write(&mut pending).await; + let mut original = WorkdirTransportError::from_workdir_error(&workdir::WorkdirError::Denied( + PROVIDER_SECRET.into(), + )); + let expected_message = original.message.clone(); + original.message = PROVIDER_SECRET.into(); + reply.send(Err(original)).unwrap(); + let (body, events) = h.finish(pending, StatusCode::FORBIDDEN).await; + assert_eq!(body["code"], "denied"); + assert_eq!(body["message"], expected_message); + let receipt_id = assert_audit_failure( + &body, + &events, + "response_received", + "ResponseReceived", + json!({"kind":"failed", "code":"denied"}), + "Failed { code: Denied }", + ); + assert_eq!(receipt_id, dispatched_id); + assert!(!h.effect_path().exists()); + provider.assert_dispatches(1); + h.assert_audits(&[(&dispatched_id, "requested")]); +} + +#[tokio::test] +async fn lost_write_response_after_audit_failure_stays_unknown_without_repeating_effect() { + let (h, mut provider) = AuditHttpFixture::new().await; + h.fail_audit(false); + let mut pending = h.start_write(); + let (dispatched_id, reply) = provider.receive_write(&mut pending).await; + h.apply_provider_write(); + // The provider applied the effect, but the server cannot infer that from response loss. + drop(reply); + let (body, events) = h.finish(pending, StatusCode::INTERNAL_SERVER_ERROR).await; + assert_eq!(body["code"], "outcome_unknown"); + let receipt_id = assert_audit_failure( + &body, + &events, + "response_unknown", + "ResponseUnknown", + json!({"kind":"unknown"}), + "Unknown", + ); + assert_eq!(receipt_id, dispatched_id); + assert_eq!(std::fs::read(h.effect_path()).unwrap(), WRITE_SECRET); + provider.assert_dispatches(1); + h.assert_audits(&[(&dispatched_id, "requested")]); +} + +#[tokio::test] +async fn before_write_audit_failure_is_not_dispatched_and_has_no_provider_effect() { + let (h, mut provider) = AuditHttpFixture::new().await; + h.fail_audit(true); + let (body, events) = h + .finish(h.start_write(), StatusCode::SERVICE_UNAVAILABLE) + .await; + assert_eq!(body["code"], "not_dispatched"); + assert_audit_failure( + &body, + &events, + "before_dispatch", + "BeforeDispatch", + json!({"kind":"not_dispatched"}), + "NotDispatched", + ); + assert_failure_reason(&events, "audit_store_unavailable"); + assert!( + body.get("failure_reason").is_none(), + "internal reason leaked: {body}" + ); + assert!(!h.effect_path().exists()); + provider.assert_dispatches(0); + h.assert_audits(&[]); +} + +#[tokio::test] +async fn rejected_write_send_with_audit_failure_is_not_dispatched_not_unknown() { + let (h, mut provider) = AuditHttpFixture::new().await; + h.fail_audit(false); + provider.receiver.close(); + let (body, events) = h + .finish(h.start_write(), StatusCode::SERVICE_UNAVAILABLE) + .await; + assert_eq!(body["code"], "not_dispatched"); + let receipt_id = assert_audit_failure( + &body, + &events, + "dispatch_rejected", + "DispatchRejected", + json!({"kind":"not_dispatched"}), + "NotDispatched", + ); + assert_failure_reason(&events, "provider_unavailable"); + assert!(!h.effect_path().exists()); + provider.assert_dispatches(0); + h.assert_audits(&[(&receipt_id, "requested")]); +} + +#[tokio::test] +async fn successful_write_persists_requested_and_completed_with_same_operation_id() { + let (h, mut provider) = AuditHttpFixture::new().await; + let mut pending = h.start_write(); + let (operation_id, reply) = provider.receive_write(&mut pending).await; + h.apply_provider_write(); + reply.send(Ok(write_result())).unwrap(); + let (body, events) = h.finish(pending, StatusCode::OK).await; + assert_eq!(body, serde_json::to_value(write_result()).unwrap()); + assert!( + events + .iter() + .all(|event| event["event"] != "external_workdir_audit_failed") + ); + assert_eq!(std::fs::read(h.effect_path()).unwrap(), WRITE_SECRET); + provider.assert_dispatches(1); + h.assert_audits(&[(&operation_id, "requested"), (&operation_id, "completed")]); +} + +#[tokio::test] +async fn provider_reported_unknown_preserves_received_response_and_unknown_effects() { + let (h, mut provider) = AuditHttpFixture::new().await; + h.fail_audit(false); + let mut pending = h.start_write(); + let (dispatched_id, reply) = provider.receive_write(&mut pending).await; + reply + .send(Err(WorkdirTransportError::from_workdir_error( + &workdir::WorkdirError::OutcomeUnknown(PROVIDER_SECRET.into()), + ))) + .unwrap(); + let (body, events) = h.finish(pending, StatusCode::INTERNAL_SERVER_ERROR).await; + assert_eq!(body["code"], "outcome_unknown"); + let receipt_id = assert_audit_failure( + &body, + &events, + "response_received", + "ResponseReceived", + json!({"kind":"unknown"}), + "Unknown", + ); + assert_eq!(receipt_id, dispatched_id); + provider.assert_dispatches(1); + h.assert_audits(&[(&dispatched_id, "requested")]); +} From 7394ec6784727e44d9f4b10776631627abda3b1c Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 06:13:07 +0900 Subject: [PATCH 09/46] fix: retain safe Workdir open authorization and response failure causes Keep bounded reasons and derived failure stages separate from public classification. Reject provider-authored raw error text on transport decoding. Preserve identity mismatch when cleanup fails and retain its handle for explicit close without opening again. Add signed Runtime HTTP, Registry/API logging, remote mismatch, authorization no-dispatch and deferred fault coverage. Focused tests and Workdir suite pass; full-source validation follows T773 audit contract merge. --- crates/workdir/src/http.rs | 21 +- crates/workdir/src/lib.rs | 95 ++++ crates/workdir/tests/failure_diagnostics.rs | 31 + crates/workdir/tests/remote_denial_reasons.rs | 103 ++++ crates/worker-runtime/src/http_server.rs | 26 +- .../tests/runtime_workdir_failure_tests.rs | 528 ++++++++++++++++++ .../src/worker_backend/deferred_workdir.rs | 163 +++++- crates/workspace-server/src/hosts.rs | 17 +- .../src/hosts/tests/workspace_auth_tests.rs | 68 +++ crates/workspace-server/src/server.rs | 6 +- .../src/server/tests/api_logging_tests.rs | 56 ++ 11 files changed, 1093 insertions(+), 21 deletions(-) create mode 100644 crates/worker-runtime/src/http_server/tests/runtime_workdir_failure_tests.rs diff --git a/crates/workdir/src/http.rs b/crates/workdir/src/http.rs index f99f804c5..889a7eb93 100644 --- a/crates/workdir/src/http.rs +++ b/crates/workdir/src/http.rs @@ -375,12 +375,14 @@ impl WorkdirTransportError { use WorkdirTransportErrorCode as Code; let error = match self.code { Code::NotFound => WorkdirError::NotFound("".into()), - Code::Conflict => WorkdirError::Conflict(self.message), + Code::Conflict => WorkdirError::Conflict( + "The target file's content or existence changed since it was last observed; read the file again before retrying".into(), + ), Code::OutcomeUnknown => WorkdirError::OutcomeUnknown( "Workdir operation may have effects; inspect before retrying".into(), ), - Code::Unsupported => WorkdirError::UnsupportedOperation(self.message), - Code::InvalidRequest => WorkdirError::InvalidArgument(self.message), + Code::Unsupported => WorkdirError::UnsupportedOperation("Workdir operation is not supported".into()), + Code::InvalidRequest => WorkdirError::InvalidArgument("Workdir operation request is invalid".into()), Code::Denied => WorkdirError::Denied(crate::WorkdirDenial { reason: self.denial_reason, // Provider-authored text is not trusted diagnostic data. @@ -409,12 +411,12 @@ impl WorkdirTransportError { target: "".into(), }, Code::UnknownCommand => WorkdirError::UnknownCommand("".to_string()), - Code::Unavailable => WorkdirError::Unavailable(self.message), + Code::Unavailable => WorkdirError::Unavailable("Workdir session is unavailable".into()), Code::Io => WorkdirError::Io { path: "".into(), - source: std::io::Error::other(self.message), + source: std::io::Error::other("Workdir I/O operation failed"), }, - Code::Transport => WorkdirError::Transport(self.message), + Code::Transport => WorkdirError::Transport("Workdir transport failed".into()), Code::Internal => WorkdirError::OperationFailed, }; let error = if let Some(reason) = self.failure_reason { @@ -549,7 +551,8 @@ mod client { if opened.workdir_id.as_str() != workdir_id.as_str() { return Err(WorkdirError::Unavailable( "Runtime opened a session for a different Workdir".to_string(), - )); + ) + .with_failure_reason(crate::WorkdirFailureReason::SessionIdentityMismatch)); } Ok(Self { client, @@ -608,6 +611,7 @@ mod client { WorkdirError::Unavailable(format!( "Runtime returned a mismatched Workdir operation result; expected {expected}" )) + .with_failure_reason(crate::WorkdirFailureReason::OperationResultMismatch) } } @@ -770,7 +774,8 @@ mod client { } Ok(_) if mutation => Err(WorkdirError::OutcomeUnknown( "mismatched response after checkout mutation".into(), - )), + ) + .with_failure_reason(crate::WorkdirFailureReason::OperationResultMismatch)), Ok(_) => Err(Self::mismatch("checkout_execute")), Err(error) if mutation diff --git a/crates/workdir/src/lib.rs b/crates/workdir/src/lib.rs index 1a515ac16..10ba3bc6c 100644 --- a/crates/workdir/src/lib.rs +++ b/crates/workdir/src/lib.rs @@ -414,6 +414,24 @@ pub enum WorkdirFailureReason { HttpTransfer, HttpResponseInvalid, HttpResponseTooLarge, + OperationResultMismatch, + SessionIdentityMismatch, + SessionOpenTaskFailed, + SessionOpenFailed, + AuthorizationMissing, + AuthorizationStale, + AuthorizationUnavailable, + SigningIdentityUnavailable, + AuthorizationSignFailed, + RuntimeStopped, + RuntimeStateUnavailable, + RuntimeBackendUnavailable, + RuntimeOwnerMismatch, + RuntimeInvalidRequest, + WorkingDirectoryNotFound, + WorkingDirectoryNotActive, + WorkingDirectoryRecordInvalid, + WorkingDirectoryAccessUnavailable, } impl WorkdirFailureReason { @@ -427,6 +445,83 @@ impl WorkdirFailureReason { Self::HttpTransfer => "http_transfer", Self::HttpResponseInvalid => "http_response_invalid", Self::HttpResponseTooLarge => "http_response_too_large", + Self::OperationResultMismatch => "operation_result_mismatch", + Self::SessionIdentityMismatch => "session_identity_mismatch", + Self::SessionOpenTaskFailed => "session_open_task_failed", + Self::SessionOpenFailed => "session_open_failed", + Self::AuthorizationMissing => "authorization_missing", + Self::AuthorizationStale => "authorization_stale", + Self::AuthorizationUnavailable => "authorization_unavailable", + Self::SigningIdentityUnavailable => "signing_identity_unavailable", + Self::AuthorizationSignFailed => "authorization_sign_failed", + Self::RuntimeStopped => "runtime_stopped", + Self::RuntimeStateUnavailable => "runtime_state_unavailable", + Self::RuntimeBackendUnavailable => "runtime_backend_unavailable", + Self::RuntimeOwnerMismatch => "runtime_owner_mismatch", + Self::RuntimeInvalidRequest => "runtime_invalid_request", + Self::WorkingDirectoryNotFound => "working_directory_not_found", + Self::WorkingDirectoryNotActive => "working_directory_not_active", + Self::WorkingDirectoryRecordInvalid => "working_directory_record_invalid", + Self::WorkingDirectoryAccessUnavailable => "working_directory_access_unavailable", + } + } + + /// The observed failing boundary, not a dispatch or effects assertion. + pub const fn stage(self) -> WorkdirFailureStage { + use WorkdirFailureStage as Stage; + match self { + Self::SessionClosed => Stage::Session, + Self::ProviderUnavailable => Stage::Provider, + Self::CommandTerminalMissing => Stage::Command, + Self::HttpTimeout | Self::HttpConnect | Self::HttpTransfer => Stage::Http, + Self::HttpResponseInvalid + | Self::HttpResponseTooLarge + | Self::OperationResultMismatch => Stage::ResponseValidation, + Self::SessionIdentityMismatch + | Self::SessionOpenTaskFailed + | Self::SessionOpenFailed + | Self::WorkingDirectoryNotFound + | Self::WorkingDirectoryNotActive + | Self::WorkingDirectoryRecordInvalid + | Self::WorkingDirectoryAccessUnavailable => Stage::SessionOpen, + Self::AuthorizationMissing + | Self::AuthorizationStale + | Self::AuthorizationUnavailable + | Self::SigningIdentityUnavailable + | Self::AuthorizationSignFailed => Stage::AuthorizationPreparation, + Self::RuntimeStopped + | Self::RuntimeStateUnavailable + | Self::RuntimeBackendUnavailable + | Self::RuntimeOwnerMismatch + | Self::RuntimeInvalidRequest => Stage::Runtime, + } + } +} + +/// Internal operational boundary derived only from a bounded reason. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WorkdirFailureStage { + Session, + Provider, + Command, + Http, + ResponseValidation, + SessionOpen, + AuthorizationPreparation, + Runtime, +} + +impl WorkdirFailureStage { + pub const fn as_str(self) -> &'static str { + match self { + Self::Session => "session", + Self::Provider => "provider", + Self::Command => "command", + Self::Http => "http", + Self::ResponseValidation => "response_validation", + Self::SessionOpen => "session_open", + Self::AuthorizationPreparation => "authorization_preparation", + Self::Runtime => "runtime", } } } diff --git a/crates/workdir/tests/failure_diagnostics.rs b/crates/workdir/tests/failure_diagnostics.rs index a94f4fe9f..b39336f04 100644 --- a/crates/workdir/tests/failure_diagnostics.rs +++ b/crates/workdir/tests/failure_diagnostics.rs @@ -83,3 +83,34 @@ fn provider_cannot_inject_free_text_failure_reason() { .is_err() ); } + +#[test] +fn provider_authored_operational_messages_never_enter_error_display_or_debug() { + for code in [ + Code::Conflict, + Code::OutcomeUnknown, + Code::Unsupported, + Code::InvalidRequest, + Code::Unavailable, + Code::Io, + Code::Transport, + Code::Internal, + ] { + let wire = serde_json::json!({ + "code": code, "message":"/host/private command-secret body-secret proof-secret key-secret Authorization: Bearer bearer-secret", + "failure_reason": "session_open_task_failed" + }); + let decoded: WorkdirTransportError = serde_json::from_value(wire).unwrap(); + let error = decoded.into_workdir_error(); + for text in [ + format!("{error}"), + format!("{error:?}"), + serde_json::to_string(&WorkdirTransportError::from_workdir_error(&error)).unwrap(), + ] { + assert!(!text.contains("secret"), "{text}"); + assert!(!text.contains("/host/private"), "{text}"); + } + assert_eq!(error.failure_reason(), Some(Reason::SessionOpenTaskFailed)); + assert_eq!(WorkdirTransportError::from_workdir_error(&error).code, code); + } +} diff --git a/crates/workdir/tests/remote_denial_reasons.rs b/crates/workdir/tests/remote_denial_reasons.rs index f14acc816..6769fb136 100644 --- a/crates/workdir/tests/remote_denial_reasons.rs +++ b/crates/workdir/tests/remote_denial_reasons.rs @@ -216,3 +216,106 @@ async fn remote_checkout_diagnostics_preserve_mutation_outcome_and_read_failures } } } + +#[tokio::test] +async fn mismatched_http_results_keep_cause_and_mutation_uncertainty() { + use workdir::{ + CheckoutOperation, CheckoutRequest, CommandHandle, CommandOutputRequest, + WorkdirFailureReason as Failure, + }; + for case in ["variant", "bytes", "mutation", "identity"] { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + reply( + &listener, + "200 OK", + if case == "identity" { + r#"{"session_id":"s","workdir_id":"wrong-secret","capabilities":{"bits":63}}"# + } else { + r#"{"session_id":"s","workdir_id":"w","capabilities":{"bits":63}}"# + }, + ); + if case != "identity" { + let result = if case == "bytes" { + serde_json::json!({"operation":"command_output","result":{ + "chunks": [], "next_cursor": 123, "terminal": null + }}) + } else { + serde_json::to_value( + workdir::http::WorkdirSessionOperationResult::CommandCancel, + ) + .unwrap() + }; + reply(&listener, "200 OK", &result.to_string()); + } + }); + let opened = RemoteWorkdirSession::open( + reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(), + format!("http://{address}").parse().unwrap(), + "request-secret", + Workdir::new("w").id().clone(), + OpenWorkdirSessionRequest::default(), + ) + .await; + let error = if case == "identity" { + opened.unwrap_err() + } else { + let remote = opened.unwrap(); + match case { + "variant" => remote + .stat(StatRequest { + path: WorkdirPath::new("file-secret").unwrap(), + }) + .await + .unwrap_err(), + "bytes" => remote + .command_output(CommandOutputRequest { + handle: CommandHandle("command-secret".into()), + cursor: 0, + limit: 16, + wait: false, + }) + .await + .unwrap_err(), + "mutation" => remote + .checkout_execute(CheckoutRequest { + target: WorkdirPath::new("file-secret").unwrap(), + validator: vec![1], + operation: CheckoutOperation::Write { + content: b"body-secret".to_vec(), + expected_hash: [0; 32], + }, + }) + .await + .unwrap_err(), + _ => unreachable!(), + } + }; + server.join().unwrap(); + let expected = if case == "identity" { + Failure::SessionIdentityMismatch + } else { + Failure::OperationResultMismatch + }; + assert_eq!(error.failure_reason(), Some(expected), "{case}: {error:?}"); + let transport = WorkdirTransportError::from_workdir_error(&error); + assert_eq!( + transport.code, + if case == "mutation" { + Code::OutcomeUnknown + } else { + Code::Unavailable + } + ); + let wire = serde_json::to_string(&transport).unwrap(); + assert!(!wire.contains("secret"), "{wire}"); + assert!(!format!("{error:?}").contains("secret")); + let forwarded = transport.into_workdir_error(); + assert_eq!(forwarded.failure_reason(), Some(expected)); + } +} diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index aefc66973..1a6aeefe9 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -2187,7 +2187,25 @@ impl RuntimeHttpWorkdirError { } fn runtime(error: RuntimeError) -> Self { - match error { + use workdir::WorkdirFailureReason as Reason; + let reason = match &error { + RuntimeError::WorkingDirectory(diagnostic) => match diagnostic.code.as_str() { + "working_directory_not_found" => Reason::WorkingDirectoryNotFound, + "working_directory_not_active" => Reason::WorkingDirectoryNotActive, + "working_directory_record_invalid" => Reason::WorkingDirectoryRecordInvalid, + "working_directory_repository_access_unavailable" => { + Reason::WorkingDirectoryAccessUnavailable + } + _ => Reason::SessionOpenFailed, + }, + RuntimeError::RuntimeStopped => Reason::RuntimeStopped, + RuntimeError::StatePoisoned => Reason::RuntimeStateUnavailable, + RuntimeError::ExecutionBackendUnavailable { .. } => Reason::RuntimeBackendUnavailable, + RuntimeError::WorkspaceOwnerMismatch { .. } => Reason::RuntimeOwnerMismatch, + RuntimeError::InvalidRequest(_) => Reason::RuntimeInvalidRequest, + _ => Reason::SessionOpenFailed, + }; + let mut response = match error { RuntimeError::WorkingDirectory(diagnostic) if diagnostic.code == "working_directory_not_found" => { @@ -2200,7 +2218,9 @@ impl RuntimeHttpWorkdirError { WorkdirTransportErrorCode::Unavailable, "Workdir session is unavailable", ), - } + }; + response.payload.failure_reason = Some(reason); + response } } @@ -2222,6 +2242,7 @@ impl IntoResponse for RuntimeHttpWorkdirError { status = self.status.as_u16(), code = self.payload.code.as_str(), failure_reason = reason.as_str(), + failure_stage = reason.stage().as_str(), "Workdir provider operation failed" ); } @@ -2434,6 +2455,7 @@ pub enum RuntimeHttpServerError { #[cfg(test)] mod tests { mod command_transport_tests; + mod runtime_workdir_failure_tests; #[cfg(feature = "fs-store")] mod session_material_tests; diff --git a/crates/worker-runtime/src/http_server/tests/runtime_workdir_failure_tests.rs b/crates/worker-runtime/src/http_server/tests/runtime_workdir_failure_tests.rs new file mode 100644 index 000000000..55873420b --- /dev/null +++ b/crates/worker-runtime/src/http_server/tests/runtime_workdir_failure_tests.rs @@ -0,0 +1,528 @@ +//! Session-open integration through the production signed router, Runtime owner/binding +//! checks, HTTP DTO and tracing boundary. Only the execution backend is replaced +//! for deterministic lookup/open faults and inert owner-binding Worker fixtures; +//! no handler, middleware or grant is replaced. Runs in the normal http-server lib +//! suite with isolated loopback sockets. + +use super::*; +use crate::working_directory::WorkingDirectoryDiagnostic; +use std::io::{self, Write}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +const OPEN_PATH: &str = "/v1/working-directories/private-workdir-marker/sessions"; +const PRIVATE_DIAGNOSTIC: &str = concat!( + "raw-runtime-error-marker /private/host-path-marker ", + "command=private-command-marker contents=private-file-marker ", + "proof=private-proof-marker Authorization: Bearer private-header-marker ", + "working_directory_not_found authorization_stale runtime_stopped", +); +const PRIVATE_MARKERS: &[&str] = &[ + "raw-runtime-error-marker", + "/private/host-path-marker", + "private-command-marker", + "private-file-marker", + "private-proof-marker", + "private-header-marker", + "private-workdir-marker", + "private-owner-marker", +]; + +#[derive(Clone, Default)] +struct LogCapture(Arc>>); + +impl Write for LogCapture { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl LogCapture { + fn subscriber(&self) -> impl tracing::Subscriber + Send + Sync + 'static { + let writer = self.clone(); + tracing_subscriber::fmt() + .with_ansi(false) + .without_time() + .with_max_level(tracing::Level::WARN) + .with_writer(move || writer.clone()) + .finish() + } + + fn take(&self) -> String { + String::from_utf8(std::mem::take(&mut *self.0.lock().unwrap())).unwrap() + } +} + +fn assert_private_absent(text: &str) { + for marker in PRIVATE_MARKERS { + assert!( + !text.contains(marker), + "private input escaped error boundary" + ); + } +} + +fn assert_failure_log(log: &str, status: StatusCode, code: &str, reason: &str, stage: &str) { + let events: Vec<_> = log + .lines() + .filter(|line| line.contains("workdir_provider_failure")) + .collect(); + assert_eq!(events.len(), 1, "one diagnostic per failed request: {log}"); + let event = events[0]; + for field in [ + "WARN".to_string(), + "worker_runtime::workdir".to_string(), + format!("status={}", status.as_u16()), + format!("code=\"{code}\""), + format!("failure_reason=\"{reason}\""), + format!("failure_stage=\"{stage}\""), + ] { + assert!(event.contains(&field), "missing {field}: {event}"); + } + assert_private_absent(log); +} + +fn assert_failure_body(body: &str, code: &str, message: &str, reason: &str) { + assert_private_absent(body); + // Exact safe transport vocabulary; raw diagnostics and stage are not wire fields. + assert_eq!( + serde_json::from_str::(body).unwrap(), + serde_json::json!({"code": code, "message": message, "failure_reason": reason}), + ); +} + +struct Harness { + server: tokio::task::JoinHandle<()>, + client: reqwest::Client, + base: String, + identity: RuntimeIdentityMaterial, + trust: WorkspaceIssuerTrustRecord, +} + +impl Drop for Harness { + fn drop(&mut self) { + self.server.abort(); + } +} + +impl Harness { + async fn new(runtime: Runtime) -> Self { + let identity = RuntimeIdentityMaterial::generate("runtime-failure-test-key").unwrap(); + let trust = workspace_trust(&identity); + let app = runtime_http_router_with_workspace_auth( + runtime, + None, + workspace_test_auth(vec![trust.clone()]), + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + Self { + server, + client: reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(), + base, + identity, + trust, + } + } + + async fn send( + &self, + claims: &WorkspaceCapabilityClaims, + body: Vec, + workspace: &str, + ) -> (StatusCode, String, String) { + let token = issue_workspace_capability_token(&self.identity.signing_key().unwrap(), claims) + .unwrap(); + let response = self + .client + .post(format!("{}{OPEN_PATH}", self.base)) + .bearer_auth(&token) + .header(header::CONTENT_TYPE, "application/json") + .header(RUNTIME_WORKSPACE_SCOPE_HEADER, workspace) + .header("x-private-test-header", "private-header-marker") + .body(body) + .send() + .await + .unwrap(); + let status = StatusCode::from_u16(response.status().as_u16()).unwrap(); + let headers = format!("{:?}", response.headers()); + let body = response.text().await.unwrap(); + assert_private_absent(&headers); + for output in [&body, &headers] { + assert!(!output.contains(&token), "signed bearer token leaked"); + assert!( + !output.contains(&self.identity.private_key), + "signing key leaked" + ); + } + (status, body, token) + } + + async fn open(&self) -> (StatusCode, String, String) { + let body = serde_json::to_vec(&OpenWorkdirSessionRequest::default()).unwrap(); + let claims = signed_claims(&self.trust, "POST", OPEN_PATH, &body); + self.send(&claims, body, "workspace-a").await + } +} + +#[derive(Clone, Copy)] +enum FailureAt { + Lookup, + Open, +} + +struct FailingBackend { + at: FailureAt, + code: &'static str, + lookups: AtomicUsize, + opens: AtomicUsize, +} + +impl FailingBackend { + fn runtime(at: FailureAt, code: &'static str) -> (Runtime, Arc) { + let backend = Arc::new(Self { + at, + code, + lookups: AtomicUsize::new(0), + opens: AtomicUsize::new(0), + }); + let runtime = + Runtime::with_execution_backend(RuntimeOptions::default(), backend.clone()).unwrap(); + (runtime, backend) + } + + fn failure(&self) -> WorkingDirectoryDiagnostic { + WorkingDirectoryDiagnostic::rejected(self.code, PRIVATE_DIAGNOSTIC) + } +} + +impl WorkerExecutionBackend for FailingBackend { + fn backend_id(&self) -> &str { + "runtime-failure-test" + } + + fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { + AcceptingBackend.spawn_worker(request) + } + + fn dispatch_input(&self, worker: &WorkerRef, input: WorkerInput) -> WorkerExecutionResult { + AcceptingBackend.dispatch_input(worker, input) + } + + fn stop_worker(&self, worker: &WorkerRef) -> WorkerExecutionResult { + AcceptingBackend.stop_worker(worker) + } + + fn working_directory( + &self, + id: &str, + ) -> Result { + self.lookups.fetch_add(1, Ordering::SeqCst); + match self.at { + FailureAt::Lookup => Err(self.failure()), + FailureAt::Open => Ok(WorkingDirectoryStatus { + summary: crate::catalog::WorkingDirectorySummary { + working_directory_id: id.into(), + display_name: None, + repository_id: "repository-test".into(), + creation_selector: None, + creation_ref: None, + creation_tree: None, + current_selector: None, + current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, + materializer_kind: crate::catalog::MaterializerKind::RuntimeGitClone, + cleanup_target: None, + status: crate::catalog::WorkingDirectoryStatusKind::Active, + cleanliness: None, + occupied_by: None, + }, + }), + } + } + + fn open_workdir_session( + &self, + _: &str, + ) -> Result { + self.opens.fetch_add(1, Ordering::SeqCst); + Err(self.failure()) + } +} + +// Current-thread execution keeps the server tasks inside the thread-local tracing +// capture across awaits, without replacing the process-global subscriber. +#[tokio::test(flavor = "current_thread")] +async fn signed_http_session_open_preserves_runtime_availability_and_owner_failures() { + let capture = LogCapture::default(); + let _guard = tracing::subscriber::set_default(capture.subscriber()); + tracing::callsite::rebuild_interest_cache(); + tokio::time::timeout(Duration::from_secs(15), async { + let stopped = Runtime::new_memory(); + stopped.stop_runtime().unwrap(); + let (owned, backend) = FailingBackend::runtime(FailureAt::Lookup, "must_not_reach_backend"); + let mut request = task_request("bind owner"); + request.config_bundle = None; + request.memory_settings.as_mut().unwrap().workspace_id = "workspace-a".into(); + owned + .create_worker_scoped( + &RuntimeWorkspaceScope::new("workspace-a", "private-owner-marker"), + request, + ) + .unwrap(); + for (runtime, status, code, message, reason) in [ + ( + stopped, + StatusCode::SERVICE_UNAVAILABLE, + "unavailable", + "Workdir session is unavailable", + "runtime_stopped", + ), + ( + Runtime::new_memory(), + StatusCode::SERVICE_UNAVAILABLE, + "unavailable", + "Workdir session is unavailable", + "runtime_backend_unavailable", + ), + ( + owned, + StatusCode::NOT_FOUND, + "not_found", + "Workdir session was not found", + "runtime_owner_mismatch", + ), + ] { + let harness = Harness::new(runtime).await; + capture.take(); + let (actual, body, token) = harness.open().await; + assert_eq!(actual, status); + assert_failure_body(&body, code, message, reason); + let log = capture.take(); + assert_failure_log(&log, status, code, reason, "runtime"); + assert!(!log.contains(&token)); + assert!(!log.contains(&harness.identity.private_key)); + } + assert_eq!(backend.lookups.load(Ordering::SeqCst), 0); + assert_eq!(backend.opens.load(Ordering::SeqCst), 0); + }) + .await + .expect("bounded Runtime failure HTTP requests"); +} + +#[tokio::test(flavor = "current_thread")] +async fn signed_http_session_open_allowlists_diagnostic_codes_without_guessing_unknown_causes() { + let capture = LogCapture::default(); + let _guard = tracing::subscriber::set_default(capture.subscriber()); + tracing::callsite::rebuild_interest_cache(); + tokio::time::timeout(Duration::from_secs(20), async { + for (diagnostic, reason) in [ + ("working_directory_not_found", "working_directory_not_found"), + ( + "working_directory_not_active", + "working_directory_not_active", + ), + ( + "working_directory_record_invalid", + "working_directory_record_invalid", + ), + ( + "working_directory_repository_access_unavailable", + "working_directory_access_unavailable", + ), + // Plausible provider text/code is not evidence of a typed cause. + ("authorization_stale", "session_open_failed"), + ( + "working_directory_not_found_private-proof-marker", + "session_open_failed", + ), + ] { + for at in [FailureAt::Lookup, FailureAt::Open] { + let (runtime, backend) = FailingBackend::runtime(at, diagnostic); + let harness = Harness::new(runtime).await; + capture.take(); + let (status, body, token) = harness.open().await; + let (expected, code, message) = if diagnostic == "working_directory_not_found" { + ( + StatusCode::NOT_FOUND, + "not_found", + "Workdir session was not found", + ) + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + "unavailable", + "Workdir session is unavailable", + ) + }; + assert_eq!(status, expected, "{diagnostic}"); + assert_failure_body(&body, code, message, reason); + let log = capture.take(); + assert_failure_log(&log, status, code, reason, "session_open"); + assert!(!log.contains(&token)); + assert!(!log.contains(&harness.identity.private_key)); + assert_eq!(backend.lookups.load(Ordering::SeqCst), 1, "no hidden retry"); + assert_eq!( + backend.opens.load(Ordering::SeqCst), + usize::from(matches!(at, FailureAt::Open)), + ); + } + } + }) + .await + .expect("bounded diagnostic fault matrix"); +} + +#[tokio::test(flavor = "current_thread")] +async fn signed_http_session_open_rejects_bad_capability_scope_and_owner_binding_before_backend() { + let capture = LogCapture::default(); + let _guard = tracing::subscriber::set_default(capture.subscriber()); + tracing::callsite::rebuild_interest_cache(); + tokio::time::timeout(Duration::from_secs(15), async { + let (runtime, backend) = + FailingBackend::runtime(FailureAt::Open, "working_directory_record_invalid"); + let mut request = task_request("foreign workspace binding"); + request.config_bundle = None; + request.memory_settings.as_mut().unwrap().workspace_id = "other-workspace".into(); + let foreign_worker = runtime + .create_worker_scoped( + &RuntimeWorkspaceScope::new("other-workspace", "https://backend.test"), + request, + ) + .unwrap() + .worker_ref; + let harness = Harness::new(runtime).await; + for case in ["permission", "body", "workspace", "owner_binding"] { + let body = if case == "owner_binding" { + serde_json::to_vec(&OpenWorkdirSessionRequest { + owner_worker_id: Some(foreign_worker.worker_id.to_string()), + }) + .unwrap() + } else { + b"{}".to_vec() + }; + let mut claims = signed_claims(&harness.trust, "POST", OPEN_PATH, &body); + if case == "permission" { + claims.operation = "workers:read".into(); + } + if case == "workspace" { + claims.issuer_workspace_id = "other-workspace".into(); + } + let sent = if case == "body" { + b"{ }".to_vec() + } else { + body + }; + capture.take(); + let (status, body, token) = harness.send(&claims, sent, "workspace-a").await; + assert_eq!( + status, + if case == "owner_binding" { + StatusCode::NOT_FOUND + } else { + StatusCode::UNAUTHORIZED + }, + "{case}", + ); + assert_private_absent(&body); + let log = capture.take(); + assert_private_absent(&log); + assert!(!log.contains(&token)); + assert!(!log.contains(&harness.identity.private_key)); + if case == "owner_binding" { + assert_failure_body( + &body, + "not_found", + "Workdir session was not found", + "working_directory_not_found", + ); + assert_failure_log( + &log, + status, + "not_found", + "working_directory_not_found", + "session_open", + ); + } else { + let wire: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(wire["error"]["code"], "unauthorized"); + assert!( + !log.contains("workdir_provider_failure"), + "auth rejection is not a provider fault", + ); + } + } + assert_eq!(backend.lookups.load(Ordering::SeqCst), 0); + assert_eq!(backend.opens.load(Ordering::SeqCst), 0); + }) + .await + .expect("bounded authorization rejection requests"); +} + +#[tokio::test(flavor = "current_thread")] +async fn runtime_error_response_keeps_raw_details_private_and_unclassified_causes_generic() { + let capture = LogCapture::default(); + let _guard = tracing::subscriber::set_default(capture.subscriber()); + tracing::callsite::rebuild_interest_cache(); + for (error, status, code, message, reason, stage) in [ + ( + RuntimeError::ExecutionBackendUnavailable { + message: PRIVATE_DIAGNOSTIC.into(), + }, + StatusCode::SERVICE_UNAVAILABLE, + "unavailable", + "Workdir session is unavailable", + "runtime_backend_unavailable", + "runtime", + ), + ( + RuntimeError::StatePoisoned, + StatusCode::SERVICE_UNAVAILABLE, + "unavailable", + "Workdir session is unavailable", + "runtime_state_unavailable", + "runtime", + ), + ( + RuntimeError::InvalidRequest(PRIVATE_DIAGNOSTIC.into()), + StatusCode::BAD_REQUEST, + "invalid_request", + "Workdir operation request is invalid", + "runtime_invalid_request", + "runtime", + ), + ( + RuntimeError::StoreCorrupt { + operation: "session_open", + path: PathBuf::from("/private/host-path-marker"), + message: PRIVATE_DIAGNOSTIC.into(), + }, + StatusCode::SERVICE_UNAVAILABLE, + "unavailable", + "Workdir session is unavailable", + "session_open_failed", + "session_open", + ), + ] { + capture.take(); + let response = RuntimeHttpWorkdirError::runtime(error).into_response(); + assert_eq!(response.status(), status); + let bytes = to_bytes(response.into_body(), 16384).await.unwrap(); + let body = std::str::from_utf8(&bytes).unwrap(); + assert_failure_body(body, code, message, reason); + assert_failure_log(&capture.take(), status, code, reason, stage); + } +} diff --git a/crates/worker-runtime/src/worker_backend/deferred_workdir.rs b/crates/worker-runtime/src/worker_backend/deferred_workdir.rs index eebedde57..9bcb3962d 100644 --- a/crates/worker-runtime/src/worker_backend/deferred_workdir.rs +++ b/crates/worker-runtime/src/worker_backend/deferred_workdir.rs @@ -66,13 +66,32 @@ impl DeferredWorkdirSession { let open = self.open.clone(); let session = tokio::task::spawn_blocking(move || open()) .await - .map_err(|_| WorkdirError::Unavailable("Workdir session opener failed".into()))? - .map_err(WorkdirError::Unavailable)?; + .map_err(|_| { + WorkdirError::Unavailable("Workdir session opener failed".into()) + .with_failure_reason(WorkdirFailureReason::SessionOpenTaskFailed) + })? + .map_err(|_| { + // The opener's string contract cannot prove a more specific cause. + WorkdirError::Unavailable("Workdir session could not be opened".into()) + .with_failure_reason(WorkdirFailureReason::SessionOpenFailed) + })?; if session.workdir().id() != self.workdir.id() { - session.close().await?; - return Err(WorkdirError::Unavailable( - "Workdir session identity mismatch".into(), - )); + if let Err(error) = session.close().await { + // Retain a failed-close owner for explicit close; never use it. + state.closed = true; + state.session = Some(session); + tracing::warn!( + target: "worker_runtime::workdir", + event = "workdir_session_identity_cleanup_failed", + failure_reason = error.failure_reason().map(WorkdirFailureReason::as_str), + denial_reason = error.denial_reason().map(WorkdirDenialReason::as_str), + "Mismatched Workdir session cleanup failed" + ); + } + return Err( + WorkdirError::Unavailable("Workdir session identity mismatch".into()) + .with_failure_reason(WorkdirFailureReason::SessionIdentityMismatch), + ); } if let Some(mut source) = session.subscribe_command_events() { let events = self.events.clone(); @@ -224,10 +243,16 @@ mod tests { }, ); assert_eq!(attempts.load(Ordering::SeqCst), 0); + let failed = session.read(read_request()).await.unwrap_err(); assert!(matches!( - session.read(read_request()).await, - Err(WorkdirError::Unavailable(_)) + failed.classification_source(), + WorkdirError::Unavailable(_) )); + assert_eq!( + failed.failure_reason(), + Some(WorkdirFailureReason::SessionOpenFailed) + ); + assert_eq!(attempts.load(Ordering::SeqCst), 1); assert_eq!( session.read(read_request()).await.unwrap().bytes, b"restored" @@ -253,4 +278,126 @@ mod tests { ); session.close().await.unwrap(); } + + #[tokio::test] + async fn opener_task_and_open_errors_keep_distinct_safe_causes() { + for panic in [false, true] { + let attempts = Arc::new(AtomicUsize::new(0)); + let observed = attempts.clone(); + let session = DeferredWorkdirSession::handle( + "saved", + WorkdirSessionCapabilities::READ_ONLY, + move || { + observed.fetch_add(1, Ordering::SeqCst); + if panic { + panic!( + "/host/private bearer-secret command-secret proof-secret key-secret" + ); + } + Err("/host/private bearer-secret command-secret proof-secret key-secret".into()) + }, + ); + let error = session.read(read_request()).await.unwrap_err(); + assert_eq!(attempts.load(Ordering::SeqCst), 1); + let reason = if panic { + WorkdirFailureReason::SessionOpenTaskFailed + } else { + WorkdirFailureReason::SessionOpenFailed + }; + assert_eq!(error.failure_reason(), Some(reason)); + assert_eq!(reason.stage().as_str(), "session_open"); + let transport = workdir::http::WorkdirTransportError::from_workdir_error(&error); + assert_eq!(transport.code.http_status(), 503); + assert!( + !serde_json::to_string(&transport) + .unwrap() + .contains("secret") + ); + assert!(!format!("{error:?}").contains("secret")); + session.close().await.unwrap(); + assert_eq!(attempts.load(Ordering::SeqCst), 1); + } + } + + #[derive(Debug)] + struct MismatchedSession { + workdir: Workdir, + closes: Arc, + fail_close: bool, + } + + macro_rules! mismatched_session { + ($( $method:ident($request:ident: $request_type:ty) -> $result:ty; )*) => { + #[async_trait] + impl WorkdirSession for MismatchedSession { + fn workdir(&self) -> &Workdir { &self.workdir } + fn capabilities(&self) -> WorkdirSessionCapabilities { WorkdirSessionCapabilities::READ_ONLY } + $(async fn $method(&self, _: $request_type) -> Result<$result, WorkdirError> { + panic!("mismatched session must never be used"); + })* + async fn close(&self) -> Result<(), WorkdirError> { + let attempt = self.closes.fetch_add(1, Ordering::SeqCst); + if self.fail_close && attempt == 0 { + Err(WorkdirError::Transport("/host/private key-secret".into()) + .with_failure_reason(WorkdirFailureReason::HttpTransfer)) + } else { Ok(()) } + } + } + }; + } + mismatched_session! { + stat(request: StatRequest) -> StatResult; + read(request: ReadRequest) -> ReadResult; + write(request: WriteRequest) -> WriteResult; + edit(request: EditRequest) -> EditResult; + list(request: ListRequest) -> ListResult; + glob(request: GlobRequest) -> GlobResult; + grep(request: GrepRequest) -> GrepResult; + start_command(request: CommandRequest) -> CommandHandle; + command_status(handle: CommandHandle) -> CommandStatus; + command_output(request: CommandOutputRequest) -> CommandOutput; + cancel_command(handle: CommandHandle) -> (); + } + + #[tokio::test] + async fn identity_mismatch_preserves_primary_cause_and_failed_close_owner() { + for fail_close in [false, true] { + let closes = Arc::new(AtomicUsize::new(0)); + let raw: WorkdirSessionHandle = Arc::new(MismatchedSession { + workdir: Workdir::new("wrong-secret"), + closes: closes.clone(), + fail_close, + }); + let attempts = Arc::new(AtomicUsize::new(0)); + let observed = attempts.clone(); + let session = DeferredWorkdirSession::handle( + "saved", + WorkdirSessionCapabilities::READ_ONLY, + move || { + observed.fetch_add(1, Ordering::SeqCst); + Ok(raw.clone()) + }, + ); + let error = session.read(read_request()).await.unwrap_err(); + assert_eq!( + error.failure_reason(), + Some(WorkdirFailureReason::SessionIdentityMismatch) + ); + assert!(!format!("{error:?}").contains("secret")); + assert_eq!(closes.load(Ordering::SeqCst), 1); + if fail_close { + assert!(matches!( + session.read(read_request()).await, + Err(WorkdirError::SessionClosed) + )); + assert_eq!(attempts.load(Ordering::SeqCst), 1); + } + session.close().await.unwrap(); + assert_eq!( + closes.load(Ordering::SeqCst), + if fail_close { 2 } else { 1 } + ); + assert_eq!(attempts.load(Ordering::SeqCst), 1); + } + } } diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 4027b221a..7fbb8bf25 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -3995,12 +3995,27 @@ impl WorkdirHttpAuthorization for RemoteWorkdirAuthorization { return authorizer .issue_async(method, path_and_query, operation, None, body) .await - .map_err(|error| WorkdirError::Unavailable(error.message)); + .map_err(|error| { + use workdir::WorkdirFailureReason as Reason; + let reason = match error.code.as_str() { + "workspace_runtime_authorization_stale" => Reason::AuthorizationStale, + "workspace_signing_identity_unavailable" => { + Reason::SigningIdentityUnavailable + } + "workspace_runtime_authorization_sign_failed" => { + Reason::AuthorizationSignFailed + } + _ => Reason::AuthorizationUnavailable, + }; + WorkdirError::Unavailable("Workdir authorization is unavailable".into()) + .with_failure_reason(reason) + }); } authorization.fallback_bearer_token.ok_or_else(|| { WorkdirError::Unavailable( "remote Runtime does not have bearer authorization configured".to_string(), ) + .with_failure_reason(workdir::WorkdirFailureReason::AuthorizationMissing) }) }) } diff --git a/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs b/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs index 29c0cb52c..f6b8f1c73 100644 --- a/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs +++ b/crates/workspace-server/src/hosts/tests/workspace_auth_tests.rs @@ -315,3 +315,71 @@ async fn capability_claims_must_match_current_workspace_identity_before_signing( .unwrap(); verify(&fixture.verifier(), &token).unwrap(); } + +#[tokio::test] +async fn remote_workdir_authorization_failures_keep_safe_reason_without_dispatch_or_fallback() { + use workdir::WorkdirFailureReason as Reason; + use workdir::http::{OpenWorkdirSessionRequest, RemoteWorkdirSession, WorkdirTransportError}; + for case in ["stale", "missing_material", "missing_authorization"] { + let fixture = Fixture::new().await; + if case == "stale" { + fixture + .store + .revoke_workspace_runtime_binding(WORKSPACE, RUNTIME, "revoked") + .unwrap(); + } else if case == "missing_material" { + fixture + .materials + .delete(&fixture.identity.private_material_ref) + .unwrap(); + } + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let authorization = RemoteWorkdirAuthorization { + workspace_authorization: (case != "missing_authorization") + .then(|| fixture.authorizer.clone()), + fallback_bearer_token: (case != "missing_authorization") + .then(|| "bearer-secret".into()), + }; + let error = RemoteWorkdirSession::open_with_authorization( + reqwest::Client::builder().no_proxy().build().unwrap(), + format!("http://{}", listener.local_addr().unwrap()) + .parse() + .unwrap(), + Arc::new(authorization), + workdir::Workdir::new("workdir-secret").id().clone(), + OpenWorkdirSessionRequest::default(), + ) + .await + .unwrap_err(); + let reason = match case { + "stale" => Reason::AuthorizationStale, + "missing_material" => Reason::AuthorizationUnavailable, + _ => Reason::AuthorizationMissing, + }; + assert_eq!(error.failure_reason(), Some(reason)); + assert_eq!(reason.stage().as_str(), "authorization_preparation"); + let transport = WorkdirTransportError::from_workdir_error(&error); + assert_eq!(transport.code.http_status(), 503); + assert_eq!(transport.message, "Workdir session is unavailable"); + assert!( + !serde_json::to_string(&transport) + .unwrap() + .contains("secret") + ); + let debug = format!("{error:?}"); + for secret in [ + "secret", + "workspace-signing/", + "private material", + "Authorization:", + "Bearer ", + ] { + assert!(!debug.contains(secret), "{case}: {debug}"); + } + assert!( + matches!(listener.accept(), Err(error) if error.kind() == std::io::ErrorKind::WouldBlock), + "authorization failure sent HTTP or fell back" + ); + } +} diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index a1b36997f..33bed6818 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -13506,6 +13506,7 @@ async fn log_failed_api_response(request: Request, next: Next) -> Response { request_token_id_hash = context.and(request_token_id_hash.as_deref()), denial_reason = error.and_then(|error| error.denial_reason).map(|reason| reason.as_str()), failure_reason = error.and_then(|error| error.failure_reason).map(|reason| reason.as_str()), + failure_stage = error.and_then(|error| error.failure_reason).map(|reason| reason.stage().as_str()), status = event.status, kind = event.kind.unwrap_or("unknown"), message = event.message.unwrap_or(""), @@ -36942,8 +36943,9 @@ impl ApiErrorLog { log.denial_reason = error.denial_reason; log.failure_reason = error.failure_reason; if log.workdir_log.is_some() { - // Existing public API details are unchanged; do not copy request-derived - // alias/handle text or session-open errors into operational diagnostics. + // Public classification and arbitrary diagnostic text are not safe + // operational causes. Keep only the separately typed reason/stage; + // never infer a reason by parsing request- or provider-authored text. log.kind = format!("workdir_session_operation_api_{}", status.as_u16()); // This can also fail after a command was dispatched. Do not claim // a pre-dispatch refusal or absence of effects in the safe message. diff --git a/crates/workspace-server/src/server/tests/api_logging_tests.rs b/crates/workspace-server/src/server/tests/api_logging_tests.rs index fc883168b..604296ac5 100644 --- a/crates/workspace-server/src/server/tests/api_logging_tests.rs +++ b/crates/workspace-server/src/server/tests/api_logging_tests.rs @@ -605,6 +605,10 @@ async fn generated_workdir_provider_errors_preserve_response_and_log_details() { workdir::WorkdirFailureReason::SessionClosed, workdir::WorkdirFailureReason::CommandTerminalMissing, workdir::WorkdirFailureReason::HttpTimeout, + workdir::WorkdirFailureReason::OperationResultMismatch, + workdir::WorkdirFailureReason::SessionOpenTaskFailed, + workdir::WorkdirFailureReason::AuthorizationStale, + workdir::WorkdirFailureReason::RuntimeStopped, ] { let request = runtime_source_request( &identity, @@ -653,6 +657,7 @@ async fn generated_workdir_provider_errors_preserve_response_and_log_details() { ); assert_eq!(events.len(), 1, "{events:?}"); assert_eq!(events[0]["failure_reason"], reason.as_str()); + assert_eq!(events[0]["failure_stage"], reason.stage().as_str()); assert_eq!(events[0]["operation"], "command_output"); assert_eq!(events[0]["workdir_stage"], "provider_dispatch"); let log = serde_json::to_string(&events).unwrap(); @@ -1249,3 +1254,54 @@ async fn non_api_workspace_paths_are_not_logged_as_ticket_api_errors() { assert_eq!(status, StatusCode::BAD_REQUEST); assert!(events.is_empty()); } + +#[tokio::test] +async fn session_open_failure_causes_survive_registry_api_conversion_without_raw_diagnostics() { + use workdir::WorkdirFailureReason as Reason; + for reason in [ + Reason::SessionOpenFailed, + Reason::SessionOpenTaskFailed, + Reason::SessionIdentityMismatch, + Reason::AuthorizationStale, + Reason::AuthorizationUnavailable, + Reason::RuntimeStopped, + Reason::WorkingDirectoryRecordInvalid, + Reason::OperationResultMismatch, + ] { + let mut fixture = manual_worker_assignment_fixture().await; + fixture.runtime.workdir_session_factory = Some(Arc::new(move |_| { + Err(workdir::WorkdirError::Unavailable( + "/host/private body-secret proof-secret key-secret credential-secret".into(), + ) + .with_failure_reason(reason)) + })); + fixture + .api + .runtime + .register_or_replace(fixture.runtime.clone()); + let identity = + worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id) + .unwrap(); + configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id); + let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-session/operations"); + let request = runtime_source_request(&identity, Some(&fixture.worker.worker_id), "POST", &path, serde_json::to_vec(&json!({ + "target_workdir": "checkout", + "operation": {"operation": "read", "request": {"path":"README.md", "offset":0, "limit":10, "max_bytes":1024}} + })).unwrap()); + let (status, body, events) = + request_with_logs(build_router(fixture.api.clone()), request).await; + assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}"); + assert_eq!( + body["message"], + "workdir_session_open_failed: Workdir session is unavailable" + ); + assert!(body.get("failure_reason").is_none(), "{body}"); + assert!(!body.to_string().contains("secret")); + assert_eq!(events.len(), 1, "{events:?}"); + assert_eq!(events[0]["workdir_stage"], "session_open"); + assert_eq!(events[0]["failure_reason"], reason.as_str()); + assert_eq!(events[0]["failure_stage"], reason.stage().as_str()); + assert_eq!(events[0]["message"], "Workdir operation failed"); + assert_eq!(events[0]["diagnostics"], "[]"); + } +} From 60eabea19adaf65888428b922f71179cb1eec4fe Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 06:13:16 +0900 Subject: [PATCH 10/46] fix: assert retained remote session close diagnostics --- crates/workspace-server/src/hosts.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 4027b221a..18fe40e22 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -8866,7 +8866,14 @@ mod tests { }) .await .expect_err("closed session must reject local operation without another request"); - assert!(matches!(error, WorkdirError::Unavailable(_))); + assert!(matches!( + error.classification_source(), + WorkdirError::Unavailable(_) + )); + assert_eq!( + error.failure_reason(), + Some(workdir::WorkdirFailureReason::SessionClosed) + ); server.join().expect("mock remote server finished"); } From 6181426abd675f4313d6300c6a01ddeb2ccdd6df Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 06:27:54 +0900 Subject: [PATCH 11/46] fix: assert retained checkout outcome diagnostics --- crates/workspace-server/src/server.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index a1b36997f..9bd590949 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -37791,10 +37791,15 @@ mod tests { ))) .unwrap(); } + let error = write.await.unwrap().unwrap_err(); assert!(matches!( - write.await.unwrap(), - Err(workdir::WorkdirError::OutcomeUnknown(_)) + error.classification_source(), + workdir::WorkdirError::OutcomeUnknown(_) )); + assert_eq!( + error.failure_reason(), + lose_response.then_some(workdir::WorkdirFailureReason::ProviderUnavailable) + ); } } From b1361d86e9f8f6ce733f22fc4bba8c341a5a36a9 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 06:39:54 +0900 Subject: [PATCH 12/46] fix: distinguish external Workdir admission dispatch and response failures Integrate the published T773 audit contract without overwriting operation outcomes. Preserve bounded admission, send refusal, response loss/timeout and invalid result reasons independently of audit failures and request stages. Signed HTTP/SQLite/log fault tests pass (10), API logging 14 and relevant focused assertions pass. Root check, docs, generated artifacts, fmt/diff and state-contract naming checks pass. Full six-package normal suite ran with no-fail-fast but is not green: long Unix socket fixture paths and existing shared close assertion plus environment-isolation failures are recorded in target/t774 logs. Isolated focused checks resolved the latter; socket validation requires a granted shorter test root. This is a partial checkpoint, not a reviewed completion. --- crates/workdir/src/lib.rs | 22 +++ crates/workspace-server/src/hosts.rs | 11 +- crates/workspace-server/src/server.rs | 32 +++- .../workspace_admission_tests/audit_tests.rs | 161 +++++++++++++++++- 4 files changed, 212 insertions(+), 14 deletions(-) diff --git a/crates/workdir/src/lib.rs b/crates/workdir/src/lib.rs index 7aecce776..c3c3d97e8 100644 --- a/crates/workdir/src/lib.rs +++ b/crates/workdir/src/lib.rs @@ -408,6 +408,11 @@ impl From<&str> for WorkdirDenial { pub enum WorkdirFailureReason { SessionClosed, ProviderUnavailable, + ProviderGrantExpired, + ProviderAdmissionClosed, + ProviderDispatchRejected, + ProviderResponseLost, + ProviderOperationTimeout, CommandTerminalMissing, AuditStoreUnavailable, HttpTimeout, @@ -440,6 +445,11 @@ impl WorkdirFailureReason { match self { Self::SessionClosed => "session_closed", Self::ProviderUnavailable => "provider_unavailable", + Self::ProviderGrantExpired => "provider_grant_expired", + Self::ProviderAdmissionClosed => "provider_admission_closed", + Self::ProviderDispatchRejected => "provider_dispatch_rejected", + Self::ProviderResponseLost => "provider_response_lost", + Self::ProviderOperationTimeout => "provider_operation_timeout", Self::CommandTerminalMissing => "command_terminal_missing", Self::AuditStoreUnavailable => "audit_store_unavailable", Self::HttpTimeout => "http_timeout", @@ -474,6 +484,10 @@ impl WorkdirFailureReason { match self { Self::SessionClosed => Stage::Session, Self::ProviderUnavailable => Stage::Provider, + Self::ProviderGrantExpired | Self::ProviderAdmissionClosed => Stage::ProviderAdmission, + Self::ProviderDispatchRejected => Stage::ProviderDispatch, + Self::ProviderResponseLost | Self::ProviderOperationTimeout => Stage::ProviderResponse, + Self::AuditStoreUnavailable => Stage::Audit, Self::CommandTerminalMissing => Stage::Command, Self::HttpTimeout | Self::HttpConnect | Self::HttpTransfer => Stage::Http, Self::HttpResponseInvalid @@ -505,6 +519,10 @@ impl WorkdirFailureReason { pub enum WorkdirFailureStage { Session, Provider, + ProviderAdmission, + ProviderDispatch, + ProviderResponse, + Audit, Command, Http, ResponseValidation, @@ -518,6 +536,10 @@ impl WorkdirFailureStage { match self { Self::Session => "session", Self::Provider => "provider", + Self::ProviderAdmission => "provider_admission", + Self::ProviderDispatch => "provider_dispatch", + Self::ProviderResponse => "provider_response", + Self::Audit => "audit", Self::Command => "command", Self::Http => "http", Self::ResponseValidation => "response_validation", diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 7fbb8bf25..da0b02671 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -6854,9 +6854,16 @@ mod tests { .await .unwrap_err(); assert!(matches!( - error, - WorkdirError::Unavailable(message) if message == Error::DbCapacityExceeded.to_string() + error.classification_source(), + WorkdirError::Unavailable(_) )); + assert_eq!( + error.failure_reason(), + Some(workdir::WorkdirFailureReason::AuthorizationUnavailable) + ); + let transport = workdir::http::WorkdirTransportError::from_workdir_error(&error); + assert_eq!(transport.message, "Workdir session is unavailable"); + assert!(!format!("{error:?}").contains(&Error::DbCapacityExceeded.to_string())); drop(release); executor.drain().await.unwrap(); } diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 753526c71..11a33bbc6 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1062,7 +1062,8 @@ impl ExternalProviderWorkdirSession { { return Err(workdir::WorkdirError::Unavailable( "External Workdir grant has expired".to_string(), - )); + ) + .with_failure_reason(workdir::WorkdirFailureReason::ProviderGrantExpired)); } // Cancellation must reach the provider even when waiting output readers // occupy every data-operation slot. The socket separately bounds control requests. @@ -1079,6 +1080,7 @@ impl ExternalProviderWorkdirSession { workdir::WorkdirError::Unavailable( "External Workdir provider admission is closed".to_string(), ) + .with_failure_reason(workdir::WorkdirFailureReason::ProviderAdmissionClosed) })?, ) }; @@ -1089,7 +1091,8 @@ impl ExternalProviderWorkdirSession { { return Err(workdir::WorkdirError::Unavailable( "External Workdir grant has expired".to_string(), - )); + ) + .with_failure_reason(workdir::WorkdirFailureReason::ProviderGrantExpired)); } let operation_id = ExternalWorkdirOperationId::new(format!("op-{}", Uuid::now_v7())) .map_err(workdir::WorkdirError::InvalidArgument)?; @@ -1134,7 +1137,7 @@ impl ExternalProviderWorkdirSession { ) .await; return Err(workdir::WorkdirError::NotDispatched - .with_failure_reason(workdir::WorkdirFailureReason::ProviderUnavailable)); + .with_failure_reason(workdir::WorkdirFailureReason::ProviderDispatchRejected)); } let (result, stage) = match tokio::time::timeout(operation_timeout, receive).await { Ok(result) => { @@ -1148,7 +1151,8 @@ impl ExternalProviderWorkdirSession { Err(workdir::WorkdirError::OutcomeUnknown( "External Workdir provider response lost; inspect before retrying" .to_string(), - )), + ) + .with_failure_reason(workdir::WorkdirFailureReason::ProviderResponseLost)), Stage::ResponseUnknown, ), } @@ -1159,7 +1163,8 @@ impl ExternalProviderWorkdirSession { Err(workdir::WorkdirError::OutcomeUnknown( "External Workdir provider response timed out; inspect before retrying" .to_string(), - )), + ) + .with_failure_reason(workdir::WorkdirFailureReason::ProviderOperationTimeout)), Stage::ResponseUnknown, ) } @@ -1328,6 +1333,7 @@ impl ExternalProviderWorkdirSession { workdir::WorkdirError::Unavailable(format!( "External Workdir provider returned a mismatched result; expected {expected}" )) + .with_failure_reason(workdir::WorkdirFailureReason::OperationResultMismatch) } } @@ -1457,7 +1463,8 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession { } Ok(_) if mutation => Err(workdir::WorkdirError::OutcomeUnknown( "mismatched checkout mutation response".into(), - )), + ) + .with_failure_reason(workdir::WorkdirFailureReason::OperationResultMismatch)), Ok(_) => Err(Self::mismatch("checkout_execute")), Err(error) if mutation @@ -37863,10 +37870,19 @@ mod tests { ))) .unwrap(); } + let error = write.await.unwrap().unwrap_err(); assert!(matches!( - write.await.unwrap(), - Err(workdir::WorkdirError::OutcomeUnknown(_)) + error.classification_source(), + workdir::WorkdirError::OutcomeUnknown(_) )); + assert_eq!( + error.failure_reason(), + Some(if lose_response { + workdir::WorkdirFailureReason::ProviderResponseLost + } else { + workdir::WorkdirFailureReason::OperationResultMismatch + }) + ); } } diff --git a/crates/workspace-server/src/server/tests/workspace_admission_tests/audit_tests.rs b/crates/workspace-server/src/server/tests/workspace_admission_tests/audit_tests.rs index 212198b6f..a6f76f548 100644 --- a/crates/workspace-server/src/server/tests/workspace_admission_tests/audit_tests.rs +++ b/crates/workspace-server/src/server/tests/workspace_admission_tests/audit_tests.rs @@ -260,6 +260,9 @@ impl AuditHttpFixture { PATH_SECRET, PROVIDER_SECRET, pending.proof.as_str(), + worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER, + self.fixture.identity.private_key.as_str(), + self.fixture.identity.public_key.as_str(), self.fixture._root.path().to_str().unwrap(), ] { assert!(!output.contains(secret), "sensitive data escaped: {output}"); @@ -363,13 +366,33 @@ fn assert_audit_failure( operation_id.to_owned() } -fn assert_failure_reason(events: &[Value], reason: &str) { +fn assert_failure_reason(events: &[Value], reason: &str, stage: &str) { let errors: Vec<&Value> = events .iter() .filter(|event| event["event"] == "api_error") .collect(); assert_eq!(errors.len(), 1, "{events:?}"); assert_eq!(errors[0]["failure_reason"], reason, "{events:?}"); + assert_eq!(errors[0]["failure_stage"], stage, "{events:?}"); +} + +fn assert_operational_failure( + body: &Value, + events: &[Value], + reason: &str, + stage: &str, + code: &str, + message: &str, +) { + assert_failure_reason(events, reason, stage); + assert_eq!(body["code"], code, "{body}"); + assert_eq!(body["message"], message, "{body}"); + for field in ["failure_reason", "failure_stage"] { + assert!( + body.get(field).is_none(), + "internal diagnostic escaped: {body}" + ); + } } #[tokio::test] @@ -436,7 +459,14 @@ async fn lost_write_response_after_audit_failure_stays_unknown_without_repeating // The provider applied the effect, but the server cannot infer that from response loss. drop(reply); let (body, events) = h.finish(pending, StatusCode::INTERNAL_SERVER_ERROR).await; - assert_eq!(body["code"], "outcome_unknown"); + assert_operational_failure( + &body, + &events, + "provider_response_lost", + "provider_response", + "outcome_unknown", + "Workdir operation may have effects; inspect before retrying", + ); let receipt_id = assert_audit_failure( &body, &events, @@ -467,7 +497,7 @@ async fn before_write_audit_failure_is_not_dispatched_and_has_no_provider_effect json!({"kind":"not_dispatched"}), "NotDispatched", ); - assert_failure_reason(&events, "audit_store_unavailable"); + assert_failure_reason(&events, "audit_store_unavailable", "audit"); assert!( body.get("failure_reason").is_none(), "internal reason leaked: {body}" @@ -494,12 +524,135 @@ async fn rejected_write_send_with_audit_failure_is_not_dispatched_not_unknown() json!({"kind":"not_dispatched"}), "NotDispatched", ); - assert_failure_reason(&events, "provider_unavailable"); + assert_operational_failure( + &body, + &events, + "provider_dispatch_rejected", + "provider_dispatch", + "not_dispatched", + "Workdir operation was not dispatched", + ); assert!(!h.effect_path().exists()); provider.assert_dispatches(0); h.assert_audits(&[(&receipt_id, "requested")]); } +#[tokio::test] +async fn expired_connection_grant_is_not_dispatched_and_logs_admission_reason() { + let (h, mut provider) = AuditHttpFixture::new().await; + { + let mut registry = h.fixture.api.external_workdir_providers.lock().unwrap(); + let connection = registry.connections.get_mut("external-1").unwrap(); + // Keep the persisted grant online so the real handler reaches operate's + // connection-expiry fence rather than the separate session-open fence. + Arc::get_mut(connection).unwrap().expires_at = + Some(Utc::now() - chrono::Duration::seconds(1)); + } + h.fail_audit(false); + let (body, events) = h + .finish(h.start_write(), StatusCode::SERVICE_UNAVAILABLE) + .await; + assert_operational_failure( + &body, + &events, + "provider_grant_expired", + "provider_admission", + "unavailable", + "Workdir session is unavailable", + ); + assert!(body.get("audit_failures").is_none(), "{body}"); + assert!( + events + .iter() + .all(|event| event["event"] != "external_workdir_audit_failed") + ); + assert!(!h.effect_path().exists()); + provider.assert_dispatches(0); + h.assert_audits(&[]); +} + +#[tokio::test] +async fn closed_provider_admission_is_not_dispatched_and_logs_admission_reason() { + let (h, mut provider) = AuditHttpFixture::new().await; + h.fixture + .api + .external_workdir_providers + .lock() + .unwrap() + .connection("external-1") + .unwrap() + .admission + .close(); + h.fail_audit(false); + let (body, events) = h + .finish(h.start_write(), StatusCode::SERVICE_UNAVAILABLE) + .await; + assert_operational_failure( + &body, + &events, + "provider_admission_closed", + "provider_admission", + "unavailable", + "Workdir session is unavailable", + ); + assert!(body.get("audit_failures").is_none(), "{body}"); + assert!( + events + .iter() + .all(|event| event["event"] != "external_workdir_audit_failed") + ); + assert!(!h.effect_path().exists()); + provider.assert_dispatches(0); + h.assert_audits(&[]); +} + +#[tokio::test] +async fn timed_out_write_after_audit_failure_stays_unknown_and_logs_timeout_reason() { + let (mut h, mut provider) = AuditHttpFixture::new().await; + // Keep the HTTP client's independent deadline beyond the unchanged 30s + // provider timeout. Pause only after the signed HTTP request was dispatched, + // avoiding automatic virtual-time advancement during socket/SQLite setup. + h.client = reqwest::Client::builder() + .no_proxy() + .timeout(std::time::Duration::from_secs(40)) + .build() + .unwrap(); + h.fail_audit(false); + let mut pending = h.start_write(); + let (dispatched_id, reply) = provider.receive_write(&mut pending).await; + h.apply_provider_write(); + tokio::time::pause(); + tokio::time::advance(std::time::Duration::from_secs(31)).await; + tokio::time::resume(); + let (body, events) = h.finish(pending, StatusCode::INTERNAL_SERVER_ERROR).await; + assert_operational_failure( + &body, + &events, + "provider_operation_timeout", + "provider_response", + "outcome_unknown", + "Workdir operation may have effects; inspect before retrying", + ); + let receipt_id = assert_audit_failure( + &body, + &events, + "response_unknown", + "ResponseUnknown", + json!({"kind":"unknown"}), + "Unknown", + ); + assert_eq!(receipt_id, dispatched_id); + assert!(reply.is_closed(), "timeout retained its response receiver"); + assert!(matches!( + provider.receiver.try_recv().unwrap(), + ExternalProviderCommand::Cancel { operation_id } + if operation_id.as_str() == dispatched_id.as_str() + )); + assert_eq!(std::fs::read(h.effect_path()).unwrap(), WRITE_SECRET); + provider.assert_dispatches(1); + h.assert_audits(&[(&dispatched_id, "requested")]); +} + #[tokio::test] async fn successful_write_persists_requested_and_completed_with_same_operation_id() { let (h, mut provider) = AuditHttpFixture::new().await; From ca6d4b4b728efa9b9fdaf0d592a061a2ccc58252 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 20:39:46 +0900 Subject: [PATCH 13/46] fix: audit validated external workdir operation outcomes --- crates/workspace-server/src/server.rs | 593 ++++++++++-------- .../workspace_admission_tests/audit_tests.rs | 202 +++++- 2 files changed, 537 insertions(+), 258 deletions(-) diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 11a33bbc6..f9377801c 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1046,10 +1046,17 @@ impl ExternalProviderWorkdirSession { }) } - async fn operate( + async fn operate( &self, operation: WorkdirSessionOperation, - ) -> std::result::Result { + validate: F, + ) -> std::result::Result + where + T: Send, + F: FnOnce(std::result::Result) -> Fut + + Send, + Fut: std::future::Future> + Send, + { let operation_kind = external_workdir_operation_kind(&operation); let operation_timeout = external_workdir_operation_timeout(&operation); if self.closed.load(Ordering::Acquire) { @@ -1169,6 +1176,12 @@ impl ExternalProviderWorkdirSession { ) } }; + // Provider dispatch is finished; validation may issue bounded command cleanup + // and must not retain a data-operation slot while awaiting another operation. + drop(_admission); + // A received response is not evidence of operation success until its requested + // type and semantics (including mutation uncertainty) have been checked. + let result = validate(result).await; let outcome = match &result { Ok(_) => Outcome::Succeeded, Err(error) @@ -1275,9 +1288,15 @@ impl ExternalProviderWorkdirSession { // Do not acquire the output lock until cancellation has unblocked any reader. // The reader records its last known cursor before cleanup acknowledges bytes. let _ = self - .operate(WorkdirSessionOperation::CommandCancel( - command.handle.clone(), - )) + .operate( + WorkdirSessionOperation::CommandCancel(command.handle.clone()), + |result| async move { + match result? { + WorkdirSessionOperationResult::CommandCancel => Ok(()), + _ => Err(Self::mismatch("command_cancel")), + } + }, + ) .await; let mut cursor = tokio::time::timeout(std::time::Duration::from_secs(30), command.cursor.lock()) @@ -1292,31 +1311,31 @@ impl ExternalProviderWorkdirSession { } let drain = async { loop { - match self - .operate(WorkdirSessionOperation::CommandOutput( - workdir::CommandOutputRequest { + let output = self + .operate( + WorkdirSessionOperation::CommandOutput(workdir::CommandOutputRequest { handle: command.handle.clone(), cursor: *cursor, limit: workdir::MAX_COMMAND_READ_BYTES, wait: true, + }), + |result| async move { + match result? { + WorkdirSessionOperationResult::CommandOutput(output) => Ok(output), + _ => Err(Self::mismatch("command_output")), + } }, - )) - .await? - { - WorkdirSessionOperationResult::CommandOutput(output) => { - *cursor = output.next_cursor; - if output.terminal.is_some() { - *command.terminal_at.lock().unwrap() = Some(std::time::Instant::now()); - return Ok(()); - } - if output.chunks.is_empty() { - return Err(workdir::WorkdirError::Unavailable( - "External Workdir provider did not confirm command termination" - .to_string(), - )); - } - } - _ => return Err(Self::mismatch("command_output")), + ) + .await?; + *cursor = output.next_cursor; + if output.terminal.is_some() { + *command.terminal_at.lock().unwrap() = Some(std::time::Instant::now()); + return Ok(()); + } + if output.chunks.is_empty() { + return Err(workdir::WorkdirError::Unavailable( + "External Workdir provider did not confirm command termination".to_string(), + )); } } }; @@ -1384,46 +1403,55 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession { request: workdir::WorkdirScopeAuthorizationRequest, ) -> std::result::Result<(), workdir::WorkdirError> { self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; - match self - .operate(WorkdirSessionOperation::AuthorizeScope(request)) - .await? - { - WorkdirSessionOperationResult::AuthorizeScope => Ok(()), - _ => Err(Self::mismatch("authorize_scope")), - } + self.operate( + WorkdirSessionOperation::AuthorizeScope(request), + |result| async move { + match result? { + WorkdirSessionOperationResult::AuthorizeScope => Ok(()), + _ => Err(Self::mismatch("authorize_scope")), + } + }, + ) + .await } async fn scope_rules_overlap( &self, request: workdir::WorkdirScopeOverlapRequest, ) -> std::result::Result { self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; - match self - .operate(WorkdirSessionOperation::ScopeRulesOverlap(request)) - .await? - { - WorkdirSessionOperationResult::ScopeRulesOverlap { overlaps } => Ok(overlaps), - _ => Err(Self::mismatch("scope_rules_overlap")), - } + self.operate( + WorkdirSessionOperation::ScopeRulesOverlap(request), + |result| async move { + match result? { + WorkdirSessionOperationResult::ScopeRulesOverlap { overlaps } => Ok(overlaps), + _ => Err(Self::mismatch("scope_rules_overlap")), + } + }, + ) + .await } async fn checkout_observe( &self, path: workdir::WorkdirPath, ) -> std::result::Result { self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; - match self - .operate(WorkdirSessionOperation::CheckoutObserve(path.clone())) - .await? - { - WorkdirSessionOperationResult::CheckoutObserve(mut value) - if value.path == path - && !value.validator.is_empty() - && value.validator.len() <= 256 => - { - value.capabilities = value.capabilities.intersection(self.capabilities()); - Ok(value) - } - _ => Err(Self::mismatch("checkout_observe")), - } + self.operate( + WorkdirSessionOperation::CheckoutObserve(path.clone()), + |result| async move { + match result? { + WorkdirSessionOperationResult::CheckoutObserve(mut value) + if value.path == path + && !value.validator.is_empty() + && value.validator.len() <= 256 => + { + value.capabilities = value.capabilities.intersection(self.capabilities()); + Ok(value) + } + _ => Err(Self::mismatch("checkout_observe")), + } + }, + ) + .await } async fn checkout_execute( &self, @@ -1433,63 +1461,67 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession { self.ensure_capability(cap)?; let target = request.target.clone(); let mutation = cap != workdir::WorkdirSessionCapability::Read; - match self - .operate(WorkdirSessionOperation::CheckoutExecute(request)) - .await - { - Ok(WorkdirSessionOperationResult::CheckoutExecute(mut value)) - if value.observation.path == target - && !value.observation.validator.is_empty() - && value.observation.validator.len() <= 256 - && matches!( - (&value.output, cap), - ( - workdir::CheckoutOutput::Read(_), - workdir::WorkdirSessionCapability::Read - ) | ( - workdir::CheckoutOutput::Write(_), - workdir::WorkdirSessionCapability::Write - ) | ( - workdir::CheckoutOutput::Edit(_), - workdir::WorkdirSessionCapability::Edit - ) - ) => - { - value.observation.capabilities = value - .observation - .capabilities - .intersection(self.capabilities()); - Ok(value) - } - Ok(_) if mutation => Err(workdir::WorkdirError::OutcomeUnknown( - "mismatched checkout mutation response".into(), - ) - .with_failure_reason(workdir::WorkdirFailureReason::OperationResultMismatch)), - Ok(_) => Err(Self::mismatch("checkout_execute")), - Err(error) - if mutation - && matches!( - error.classification_source(), - workdir::WorkdirError::Unavailable(_) | workdir::WorkdirError::Transport(_) - ) => - { - let outcome = workdir::WorkdirError::OutcomeUnknown( - "checkout provider response lost; inspect effects before retry".into(), - ); - let outcome = match error.failure_reason() { - Some(reason) => outcome.with_failure_reason(reason), - None => outcome, - }; - Err(match error.denial_reason() { - Some(reason) => workdir::WorkdirError::DenialContext { - reason, - source: Box::new(outcome), - }, - None => outcome, - }) - } - Err(error) => Err(error), - } + self.operate( + WorkdirSessionOperation::CheckoutExecute(request), + |result| async move { + match result { + Ok(WorkdirSessionOperationResult::CheckoutExecute(mut value)) + if value.observation.path == target + && !value.observation.validator.is_empty() + && value.observation.validator.len() <= 256 + && matches!( + (&value.output, cap), + ( + workdir::CheckoutOutput::Read(_), + workdir::WorkdirSessionCapability::Read + ) | ( + workdir::CheckoutOutput::Write(_), + workdir::WorkdirSessionCapability::Write + ) | ( + workdir::CheckoutOutput::Edit(_), + workdir::WorkdirSessionCapability::Edit + ) + ) => + { + value.observation.capabilities = value + .observation + .capabilities + .intersection(self.capabilities()); + Ok(value) + } + Ok(_) if mutation => Err(workdir::WorkdirError::OutcomeUnknown( + "mismatched checkout mutation response".into(), + ) + .with_failure_reason(workdir::WorkdirFailureReason::OperationResultMismatch)), + Ok(_) => Err(Self::mismatch("checkout_execute")), + Err(error) + if mutation + && matches!( + error.classification_source(), + workdir::WorkdirError::Unavailable(_) + | workdir::WorkdirError::Transport(_) + ) => + { + let outcome = workdir::WorkdirError::OutcomeUnknown( + "checkout provider response lost; inspect effects before retry".into(), + ); + let outcome = match error.failure_reason() { + Some(reason) => outcome.with_failure_reason(reason), + None => outcome, + }; + Err(match error.denial_reason() { + Some(reason) => workdir::WorkdirError::DenialContext { + reason, + source: Box::new(outcome), + }, + None => outcome, + }) + } + Err(error) => Err(error), + } + }, + ) + .await } async fn checkout_search( &self, @@ -1505,108 +1537,153 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession { Op::Grep(_) => Cap::Grep, }; self.ensure_capability(cap)?; - match self - .operate(WorkdirSessionOperation::CheckoutSearch(request)) - .await? - { - WorkdirSessionOperationResult::CheckoutSearch(value) - if matches!( - (&value, cap), - (Result::List(_), Cap::Read) - | (Result::Glob(_), Cap::Glob) - | (Result::Grep(_), Cap::Grep) - ) => - { - Ok(value) - } - _ => Err(Self::mismatch("checkout_search")), - } + self.operate( + WorkdirSessionOperation::CheckoutSearch(request), + |result| async move { + match result? { + WorkdirSessionOperationResult::CheckoutSearch(value) + if matches!( + (&value, cap), + (Result::List(_), Cap::Read) + | (Result::Glob(_), Cap::Glob) + | (Result::Grep(_), Cap::Grep) + ) => + { + Ok(value) + } + _ => Err(Self::mismatch("checkout_search")), + } + }, + ) + .await } async fn stat( &self, request: workdir::StatRequest, ) -> std::result::Result { self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; - match self.operate(WorkdirSessionOperation::Stat(request)).await? { - WorkdirSessionOperationResult::Stat(value) => Ok(value), - _ => Err(Self::mismatch("stat")), - } + self.operate( + WorkdirSessionOperation::Stat(request), + |result| async move { + match result? { + WorkdirSessionOperationResult::Stat(value) => Ok(value), + _ => Err(Self::mismatch("stat")), + } + }, + ) + .await } async fn read( &self, request: workdir::ReadRequest, ) -> std::result::Result { self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; - match self.operate(WorkdirSessionOperation::Read(request)).await? { - WorkdirSessionOperationResult::Read(value) => Ok(value), - _ => Err(Self::mismatch("read")), - } + self.operate( + WorkdirSessionOperation::Read(request), + |result| async move { + match result? { + WorkdirSessionOperationResult::Read(value) => Ok(value), + _ => Err(Self::mismatch("read")), + } + }, + ) + .await } async fn read_bytes( &self, request: workdir::ReadBytesRequest, ) -> std::result::Result { self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; - match self - .operate(WorkdirSessionOperation::ReadBytes(request)) - .await? - { - WorkdirSessionOperationResult::ReadBytes(value) => Ok(value), - _ => Err(Self::mismatch("read_bytes")), - } + self.operate( + WorkdirSessionOperation::ReadBytes(request), + |result| async move { + match result? { + WorkdirSessionOperationResult::ReadBytes(value) => Ok(value), + _ => Err(Self::mismatch("read_bytes")), + } + }, + ) + .await } async fn write( &self, request: workdir::WriteRequest, ) -> std::result::Result { self.ensure_capability(workdir::WorkdirSessionCapability::Write)?; - match self - .operate(WorkdirSessionOperation::Write(request)) - .await? - { - WorkdirSessionOperationResult::Write(value) => Ok(value), - _ => Err(Self::mismatch("write")), - } + self.operate( + WorkdirSessionOperation::Write(request), + |result| async move { + match result? { + WorkdirSessionOperationResult::Write(value) => Ok(value), + _ => Err(Self::mismatch("write")), + } + }, + ) + .await } async fn edit( &self, request: workdir::EditRequest, ) -> std::result::Result { self.ensure_capability(workdir::WorkdirSessionCapability::Edit)?; - match self.operate(WorkdirSessionOperation::Edit(request)).await? { - WorkdirSessionOperationResult::Edit(value) => Ok(value), - _ => Err(Self::mismatch("edit")), - } + self.operate( + WorkdirSessionOperation::Edit(request), + |result| async move { + match result? { + WorkdirSessionOperationResult::Edit(value) => Ok(value), + _ => Err(Self::mismatch("edit")), + } + }, + ) + .await } async fn list( &self, request: workdir::ListRequest, ) -> std::result::Result { self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; - match self.operate(WorkdirSessionOperation::List(request)).await? { - WorkdirSessionOperationResult::List(value) => Ok(value), - _ => Err(Self::mismatch("list")), - } + self.operate( + WorkdirSessionOperation::List(request), + |result| async move { + match result? { + WorkdirSessionOperationResult::List(value) => Ok(value), + _ => Err(Self::mismatch("list")), + } + }, + ) + .await } async fn glob( &self, request: workdir::GlobRequest, ) -> std::result::Result { self.ensure_capability(workdir::WorkdirSessionCapability::Glob)?; - match self.operate(WorkdirSessionOperation::Glob(request)).await? { - WorkdirSessionOperationResult::Glob(value) => Ok(value), - _ => Err(Self::mismatch("glob")), - } + self.operate( + WorkdirSessionOperation::Glob(request), + |result| async move { + match result? { + WorkdirSessionOperationResult::Glob(value) => Ok(value), + _ => Err(Self::mismatch("glob")), + } + }, + ) + .await } async fn grep( &self, request: workdir::GrepRequest, ) -> std::result::Result { self.ensure_capability(workdir::WorkdirSessionCapability::Grep)?; - match self.operate(WorkdirSessionOperation::Grep(request)).await? { - WorkdirSessionOperationResult::Grep(value) => Ok(value), - _ => Err(Self::mismatch("grep")), - } + self.operate( + WorkdirSessionOperation::Grep(request), + |result| async move { + match result? { + WorkdirSessionOperationResult::Grep(value) => Ok(value), + _ => Err(Self::mismatch("grep")), + } + }, + ) + .await } async fn start_command( &self, @@ -1631,62 +1708,68 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession { "External Workdir active command limit reached".to_string(), )); } - match self - .operate(WorkdirSessionOperation::CommandStart(request)) - .await? - { - WorkdirSessionOperationResult::CommandStart(provider_handle) => { - let duplicate_handles = { - let commands = self.active_commands.lock().await; - commands - .iter() - .filter(|(_, active)| active.handle == provider_handle) - .map(|(external, _)| external.clone()) - .collect::>() - }; - if !duplicate_handles.is_empty() { - // A provider handle must identify exactly one command in one - // generation. Remove every ambiguous route before attempting - // cleanup so no caller can address the wrong process. - let mut commands = self.active_commands.lock().await; - let mut quarantined = None; - for external_handle in duplicate_handles { - if let Some(command) = commands.remove(&external_handle) { - quarantined.get_or_insert(command); + self.operate( + WorkdirSessionOperation::CommandStart(request), + |result| async move { + match result? { + WorkdirSessionOperationResult::CommandStart(provider_handle) => { + let duplicate_handles = { + let commands = self.active_commands.lock().await; + commands + .iter() + .filter(|(_, active)| active.handle == provider_handle) + .map(|(external, _)| external.clone()) + .collect::>() + }; + if !duplicate_handles.is_empty() { + // A provider handle must identify exactly one command in one + // generation. Remove every ambiguous route before attempting + // cleanup so no caller can address the wrong process. + let mut commands = self.active_commands.lock().await; + let mut quarantined = None; + for external_handle in duplicate_handles { + if let Some(command) = commands.remove(&external_handle) { + quarantined.get_or_insert(command); + } + } + let quarantine_handle = workdir::CommandHandle(format!( + "quarantined-command-{}", + Uuid::now_v7() + )); + let command = quarantined.expect("duplicate command route disappeared"); + *command.terminal_at.lock().unwrap() = None; + commands.insert(quarantine_handle.0.clone(), command.clone()); + drop(commands); + if self + .terminate_provider_command(command.clone(), true) + .await + .is_ok() + { + self.remove_active_command_if_matches(&quarantine_handle, &command) + .await; + } else { + // An earlier read of the reused handle cannot establish that + // the newly started, ambiguous process has been cleaned up. + *command.terminal_at.lock().unwrap() = None; + } + return Err(workdir::WorkdirError::Unavailable( + "External Workdir provider reused an active command handle" + .to_string(), + )); } + let external_handle = + workdir::CommandHandle(format!("external-command-{}", Uuid::now_v7())); + self.active_commands.lock().await.insert( + external_handle.0.clone(), + Arc::new(ExternalProviderCommandState::new(provider_handle)), + ); + Ok(external_handle) } - let quarantine_handle = - workdir::CommandHandle(format!("quarantined-command-{}", Uuid::now_v7())); - let command = quarantined.expect("duplicate command route disappeared"); - *command.terminal_at.lock().unwrap() = None; - commands.insert(quarantine_handle.0.clone(), command.clone()); - drop(commands); - if self - .terminate_provider_command(command.clone(), true) - .await - .is_ok() - { - self.remove_active_command_if_matches(&quarantine_handle, &command) - .await; - } else { - // An earlier read of the reused handle cannot establish that - // the newly started, ambiguous process has been cleaned up. - *command.terminal_at.lock().unwrap() = None; - } - return Err(workdir::WorkdirError::Unavailable( - "External Workdir provider reused an active command handle".to_string(), - )); + _ => Err(Self::mismatch("command_start")), } - let external_handle = - workdir::CommandHandle(format!("external-command-{}", Uuid::now_v7())); - self.active_commands.lock().await.insert( - external_handle.0.clone(), - Arc::new(ExternalProviderCommandState::new(provider_handle)), - ); - Ok(external_handle) - } - _ => Err(Self::mismatch("command_start")), - } + }, + ) + .await } async fn command_status( &self, @@ -1694,13 +1777,16 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession { ) -> std::result::Result { self.ensure_capability(workdir::WorkdirSessionCapability::Command)?; let provider_handle = self.provider_command_handle(&external_handle).await?; - match self - .operate(WorkdirSessionOperation::CommandStatus(provider_handle)) - .await? - { - WorkdirSessionOperationResult::CommandStatus(status) => Ok(status), - _ => Err(Self::mismatch("command_status")), - } + self.operate( + WorkdirSessionOperation::CommandStatus(provider_handle), + |result| async move { + match result? { + WorkdirSessionOperationResult::CommandStatus(status) => Ok(status), + _ => Err(Self::mismatch("command_status")), + } + }, + ) + .await } async fn command_output( &self, @@ -1709,29 +1795,33 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession { self.ensure_capability(workdir::WorkdirSessionCapability::Command)?; let external_handle = request.handle.clone(); let command = self.provider_command(&external_handle).await?; + let command = &command; let mut cursor = command.cursor.lock().await; request.handle = command.handle.clone(); // If the response is lost, this request cursor still replays the provider page. - match self - .operate(WorkdirSessionOperation::CommandOutput(request)) - .await - { - Ok(WorkdirSessionOperationResult::CommandOutput(output)) => { - *cursor = output.next_cursor; - if output.terminal.is_some() { - command - .terminal_at - .lock() - .unwrap() - .get_or_insert_with(std::time::Instant::now); + self.operate( + WorkdirSessionOperation::CommandOutput(request), + |result| async move { + match result { + Ok(WorkdirSessionOperationResult::CommandOutput(output)) => { + *cursor = output.next_cursor; + if output.terminal.is_some() { + command + .terminal_at + .lock() + .unwrap() + .get_or_insert_with(std::time::Instant::now); + } + drop(cursor); + self.prune_terminal_commands().await; + Ok(output) + } + Ok(_) => Err(Self::mismatch("command_output")), + Err(error) => Err(error), } - drop(cursor); - self.prune_terminal_commands().await; - Ok(output) - } - Ok(_) => Err(Self::mismatch("command_output")), - Err(error) => Err(error), - } + }, + ) + .await } async fn cancel_command( &self, @@ -1739,13 +1829,16 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession { ) -> std::result::Result<(), workdir::WorkdirError> { self.ensure_capability(workdir::WorkdirSessionCapability::Command)?; let provider_handle = self.provider_command_handle(&external_handle).await?; - match self - .operate(WorkdirSessionOperation::CommandCancel(provider_handle)) - .await? - { - WorkdirSessionOperationResult::CommandCancel => Ok(()), - _ => Err(Self::mismatch("command_cancel")), - } + self.operate( + WorkdirSessionOperation::CommandCancel(provider_handle), + |result| async move { + match result? { + WorkdirSessionOperationResult::CommandCancel => Ok(()), + _ => Err(Self::mismatch("command_cancel")), + } + }, + ) + .await } async fn close(&self) -> std::result::Result<(), workdir::WorkdirError> { let _close_guard = self.close_lock.lock().await; diff --git a/crates/workspace-server/src/server/tests/workspace_admission_tests/audit_tests.rs b/crates/workspace-server/src/server/tests/workspace_admission_tests/audit_tests.rs index a6f76f548..bdf75c837 100644 --- a/crates/workspace-server/src/server/tests/workspace_admission_tests/audit_tests.rs +++ b/crates/workspace-server/src/server/tests/workspace_admission_tests/audit_tests.rs @@ -46,7 +46,10 @@ struct ProviderProbe { } impl ProviderProbe { - async fn receive_write(&mut self, pending: &mut PendingHttpWrite) -> (String, ProviderReply) { + async fn receive_operation( + &mut self, + pending: &mut PendingHttpWrite, + ) -> (String, WorkdirSessionOperation, ProviderReply) { let command = tokio::time::timeout(BOUND, async { tokio::select! { command = self.receiver.recv() => command.expect("provider channel closed"), @@ -66,6 +69,11 @@ impl ProviderProbe { panic!("expected write dispatch, received cancellation") }; self.dispatches += 1; + (operation_id.as_str().to_owned(), operation, response) + } + + async fn receive_write(&mut self, pending: &mut PendingHttpWrite) -> (String, ProviderReply) { + let (operation_id, operation, response) = self.receive_operation(pending).await; let WorkdirSessionOperation::Write(request) = operation else { panic!("expected mutation, got {operation:?}") }; @@ -75,7 +83,7 @@ impl ProviderProbe { ); assert_eq!(request.content, WRITE_SECRET); assert_eq!(request.expected_hash, None); - (operation_id.as_str().to_owned(), response) + (operation_id, response) } fn assert_dispatches(&mut self, expected: usize) { @@ -200,14 +208,18 @@ impl AuditHttpFixture { } fn start_write(&self) -> PendingHttpWrite { + self.start_operation(WorkdirSessionOperation::Write(workdir::WriteRequest { + path: workdir::WorkdirPath::new(PATH_SECRET).unwrap(), + content: WRITE_SECRET.to_vec(), + expected_hash: None, + })) + } + + fn start_operation(&self, operation: WorkdirSessionOperation) -> PendingHttpWrite { let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-session/operations"); let body = serde_json::to_vec(&WorkspaceWorkdirSessionOperationRequest { target_workdir: "attachment".into(), - operation: WorkdirSessionOperation::Write(workdir::WriteRequest { - path: workdir::WorkdirPath::new(PATH_SECRET).unwrap(), - content: WRITE_SECRET.to_vec(), - expected_hash: None, - }), + operation, }) .unwrap(); let request = self.fixture.request( @@ -259,6 +271,7 @@ impl AuditHttpFixture { encoded_content.as_str(), PATH_SECRET, PROVIDER_SECRET, + PROVIDER_SECRET.trim_start_matches('/'), pending.proof.as_str(), worker_runtime::auth::RUNTIME_REQUEST_SOURCE_PROOF_HEADER, self.fixture.identity.private_key.as_str(), @@ -276,6 +289,10 @@ impl AuditHttpFixture { } fn assert_audits(&self, expected: &[(&str, &str)]) { + self.assert_operation_audits(expected, "write"); + } + + fn assert_operation_audits(&self, expected: &[(&str, &str)], operation: &str) { let connection = rusqlite::Connection::open(&self.fixture.api.config.database_path).unwrap(); let mut statement = connection @@ -298,7 +315,10 @@ impl AuditHttpFixture { .collect(); assert_eq!(actual, expected, "persisted audits: {rows:?}"); for (_, _, summary) in rows { - assert!(summary.contains("operation_kind=write"), "{summary}"); + assert!( + summary.contains(&format!("operation_kind={operation}")), + "{summary}" + ); for secret in [ AUDIT_SECRET, PATH_SECRET, @@ -697,3 +717,169 @@ async fn provider_reported_unknown_preserves_received_response_and_unknown_effec provider.assert_dispatches(1); h.assert_audits(&[(&dispatched_id, "requested")]); } + +#[derive(Clone, Copy, Debug)] +enum CheckoutReplyFault { + WrongOperation, + WrongPath, + EmptyValidator, + WrongOutput, + ProviderUnavailable, + ProviderTransport, +} + +async fn assert_checkout_unknown_audit(fault: CheckoutReplyFault, fail_completion: bool) { + let (h, mut provider) = AuditHttpFixture::new().await; + if fail_completion { + h.fail_audit(false); + } + let target = workdir::WorkdirPath::new(PATH_SECRET).unwrap(); + let mut pending = h.start_operation(WorkdirSessionOperation::CheckoutExecute( + workdir::CheckoutRequest { + target: target.clone(), + validator: vec![1], + operation: workdir::CheckoutOperation::Write { + content: WRITE_SECRET.to_vec(), + expected_hash: [0; 32], + }, + }, + )); + let (operation_id, operation, reply) = provider.receive_operation(&mut pending).await; + let WorkdirSessionOperation::CheckoutExecute(request) = operation else { + panic!("expected checkout mutation, got {operation:?}"); + }; + assert_eq!(request.target, target); + assert_eq!(request.validator, vec![1]); + assert!( + matches!(request.operation, workdir::CheckoutOperation::Write { ref content, .. } if content == WRITE_SECRET) + ); + h.apply_provider_write(); + let result = match fault { + CheckoutReplyFault::WrongOperation => Ok(write_result()), + CheckoutReplyFault::ProviderUnavailable => Err(WorkdirTransportError::from_workdir_error( + &workdir::WorkdirError::Unavailable(PROVIDER_SECRET.into()), + )), + CheckoutReplyFault::ProviderTransport => Err(WorkdirTransportError::from_workdir_error( + &workdir::WorkdirError::Transport(PROVIDER_SECRET.into()), + )), + _ => { + let mut result = workdir::CheckoutResult { + observation: workdir::CheckoutObservation { + path: target, + validator: vec![2], + kind: workdir::EntryKind::File, + size: WRITE_SECRET.len() as u64, + capabilities: workdir::WorkdirSessionCapabilities::READ_WRITE, + }, + output: workdir::CheckoutOutput::Write(workdir::WriteResult { + bytes_written: WRITE_SECRET.len(), + created: false, + }), + }; + match fault { + CheckoutReplyFault::WrongPath => { + result.observation.path = + workdir::WorkdirPath::new(PROVIDER_SECRET.trim_start_matches('/')).unwrap() + } + CheckoutReplyFault::EmptyValidator => result.observation.validator.clear(), + CheckoutReplyFault::WrongOutput => { + result.output = workdir::CheckoutOutput::Edit(workdir::EditResult { + replacements: 1, + content_hash: [0; 32], + bytes_written: 1, + }) + } + _ => unreachable!(), + } + Ok(WorkdirSessionOperationResult::CheckoutExecute(result)) + } + }; + reply.send(result).unwrap(); + let (body, events) = h.finish(pending, StatusCode::INTERNAL_SERVER_ERROR).await; + assert_eq!(body["code"], "outcome_unknown", "{fault:?}: {body}"); + assert_eq!( + body["message"], + "Workdir operation may have effects; inspect before retrying" + ); + if !matches!( + fault, + CheckoutReplyFault::ProviderUnavailable | CheckoutReplyFault::ProviderTransport + ) { + assert_failure_reason(&events, "operation_result_mismatch", "response_validation"); + } + if fail_completion { + assert_eq!( + body["audit_failures"], + json!([{ + "operation_id": operation_id, + "stage": "response_received", + "reason": "audit_store_unavailable", + "operation_outcome": {"kind":"unknown"}, + }]), + "{fault:?}: {body}" + ); + let audit_logs: Vec<_> = events + .iter() + .filter(|e| e["event"] == "external_workdir_audit_failed") + .collect(); + assert_eq!(audit_logs.len(), 1, "{events:?}"); + let event = audit_logs[0]; + assert_eq!(event["operation_id"], operation_id); + assert_eq!(event["operation"], "checkout_execute"); + assert_eq!(event["audit_stage"], "ResponseReceived"); + assert_eq!(event["operation_outcome"], "Unknown"); + h.assert_operation_audits(&[(&operation_id, "requested")], "checkout_execute"); + } else { + assert!(body.get("audit_failures").is_none(), "{body}"); + assert!( + events + .iter() + .all(|e| e["event"] != "external_workdir_audit_failed") + ); + h.assert_operation_audits( + &[ + (&operation_id, "requested"), + (&operation_id, "outcome_unknown"), + ], + "checkout_execute", + ); + } + assert_eq!(std::fs::read(h.effect_path()).unwrap(), WRITE_SECRET); + provider.assert_dispatches(1); +} + +#[tokio::test] +async fn checkout_invalid_reply_records_unknown_in_healthy_completion_audit() { + for fault in [ + CheckoutReplyFault::WrongOperation, + CheckoutReplyFault::WrongPath, + CheckoutReplyFault::EmptyValidator, + CheckoutReplyFault::WrongOutput, + ] { + assert_checkout_unknown_audit(fault, false).await; + } +} + +#[tokio::test] +async fn checkout_invalid_reply_records_unknown_in_failed_completion_receipt_and_log() { + for fault in [ + CheckoutReplyFault::WrongOperation, + CheckoutReplyFault::WrongPath, + CheckoutReplyFault::EmptyValidator, + CheckoutReplyFault::WrongOutput, + ] { + assert_checkout_unknown_audit(fault, true).await; + } +} + +#[tokio::test] +async fn checkout_uncertain_reply_normalization_matches_audit_outcome() { + for fault in [ + CheckoutReplyFault::ProviderUnavailable, + CheckoutReplyFault::ProviderTransport, + ] { + for fail_completion in [false, true] { + assert_checkout_unknown_audit(fault, fail_completion).await; + } + } +} From 5af9dc8e4aab4b8beca5a08e63b83ee08a131aa5 Mon Sep 17 00:00:00 2001 From: Hare Date: Wed, 21 Oct 2026 23:57:01 +0900 Subject: [PATCH 14/46] test: focus checkout hardlink coverage on alias and symlink contracts --- crates/workdir/src/checkout.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/workdir/src/checkout.rs b/crates/workdir/src/checkout.rs index 2b7c243fd..52c8a91ae 100644 --- a/crates/workdir/src/checkout.rs +++ b/crates/workdir/src/checkout.rs @@ -556,15 +556,13 @@ mod tests { )); } #[tokio::test] - async fn checkout_hardlink_save_isolates_alias_and_rejects_symlinks_special_files() { + async fn checkout_hardlink_save_isolates_alias_and_rejects_symlinks() { let root = tempfile::tempdir().unwrap(); std::fs::write(root.path().join("a"), "original").unwrap(); std::fs::hard_link(root.path().join("a"), root.path().join("alias")).unwrap(); std::os::unix::fs::symlink("a", root.path().join("link")).unwrap(); - let _socket = std::os::unix::net::UnixListener::bind(root.path().join("socket")).unwrap(); let session = session(root.path()); assert!(session.checkout_observe(path("link")).await.is_err()); - assert!(session.checkout_observe(path("socket")).await.is_err()); let read = checked_read(&session, session.checkout_observe(path("a")).await.unwrap()).await; session .checkout_execute(CheckoutRequest { From 533477c2e724b98d13a25ec0c7ad5ef5975272a1 Mon Sep 17 00:00:00 2001 From: Hare Date: Thu, 22 Oct 2026 01:29:39 +0900 Subject: [PATCH 15/46] feat: unify headless execution under the Standalone Host Expose an optional private same-uid Unix operator adapter and JSONL serve/attach CLI using the production Standalone Profile, store, restore lease and shutdown lifecycle. Remove direct Worker subprocess/discovery/readiness/controller sockets and parent socket callbacks; retain metadata-only peer registration and legacy safety readers. Validated real Host/client input, events, provenance, disconnect, admitted shutdown and shared production restore/duplicate exclusion; normal standalone/worker/manifest/protocol/client/yoi/tui/server tests, root check, rustdoc, fmt and state-contract audit. Runtime lib: 476 passed, damaged_checkout_does_not_block_restore_but_is_rejected_when_used fails at missing_git diagnostic; unchanged restore logic, not relaxed or retried. Runtime bins/docs pass. --- Cargo.lock | 3 +- crates/manifest/src/paths.rs | 39 +- crates/protocol/src/lib.rs | 27 +- crates/standalone/Cargo.toml | 2 +- crates/standalone/README.md | 3 +- crates/standalone/src/host.rs | 49 +- crates/standalone/src/lib.rs | 38 + .../journal/tests/connected_compaction.rs | 1 - crates/standalone/src/unix.rs | 296 +++ crates/standalone/tests/host.rs | 229 ++ crates/worker-runtime/src/worker_backend.rs | 37 +- crates/worker/Cargo.toml | 1 - crates/worker/src/bootstrap.rs | 35 +- crates/worker/src/controller.rs | 738 +------ .../src/controller_subject_release_tests.rs | 1 - crates/worker/src/discovery.rs | 1861 ----------------- crates/worker/src/entrypoint.rs | 1099 ---------- crates/worker/src/ipc/event.rs | 68 +- crates/worker/src/ipc/mod.rs | 1 - crates/worker/src/ipc/server.rs | 224 -- crates/worker/src/lib.rs | 8 +- crates/worker/src/peer.rs | 186 ++ crates/worker/src/profile_manifest.rs | 183 ++ crates/worker/src/runtime/dir.rs | 310 +-- .../src/runtime/worker_allocation/table.rs | 48 +- crates/worker/src/runtime_command.rs | 165 -- crates/worker/src/spawn/comm_tools.rs | 181 -- crates/worker/src/spawn/mod.rs | 1 - crates/worker/src/subjektiv.rs | 8 +- crates/worker/src/worker.rs | 176 +- crates/worker/tests/controller_test.rs | 489 ++--- crates/worker/tests/worker_events_test.rs | 40 +- crates/workspace-server/src/server.rs | 1 - .../src/server_assignment_attention_tests.rs | 1 - .../src/server_assignment_delivery_tests.rs | 1 - crates/yoi/Cargo.toml | 2 +- crates/yoi/src/cli_connection.rs | 8 +- crates/yoi/src/local_cli.rs | 273 +++ crates/yoi/src/main.rs | 353 ++-- crates/yoi/src/worker_cleanup_cli.rs | 7 +- 40 files changed, 1830 insertions(+), 5363 deletions(-) create mode 100644 crates/standalone/src/unix.rs delete mode 100644 crates/worker/src/discovery.rs delete mode 100644 crates/worker/src/entrypoint.rs delete mode 100644 crates/worker/src/ipc/server.rs create mode 100644 crates/worker/src/peer.rs create mode 100644 crates/worker/src/profile_manifest.rs delete mode 100644 crates/worker/src/runtime_command.rs delete mode 100644 crates/worker/src/spawn/comm_tools.rs create mode 100644 crates/yoi/src/local_cli.rs diff --git a/Cargo.lock b/Cargo.lock index fa56fd6d7..fb65b50ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6521,7 +6521,6 @@ dependencies = [ "axum", "base64 0.22.1", "chrono", - "clap", "client", "config-source", "dotenv", @@ -6687,6 +6686,7 @@ dependencies = [ "futures", "manifest", "project-record", + "protocol", "reqwest", "serde", "serde_json", @@ -6702,7 +6702,6 @@ dependencies = [ "tui", "uuid", "workdir", - "worker", ] [[package]] diff --git a/crates/manifest/src/paths.rs b/crates/manifest/src/paths.rs index abd79e343..e0c8cd736 100644 --- a/crates/manifest/src/paths.rs +++ b/crates/manifest/src/paths.rs @@ -100,21 +100,6 @@ pub fn worker_allocation_path() -> Option { worker_allocation_path_from_runtime_dir(runtime_dir()) } -/// `//` — per-Worker runtime directory. -pub fn worker_runtime_dir(worker_name: &str) -> Option { - worker_runtime_dir_from_runtime_dir(runtime_dir(), worker_name) -} - -/// `//sock` — Worker's Unix socket path. -/// -/// The `worker` crate's `RuntimeDir::socket_path()` creates the socket inside the -/// Worker process. This function **predicts** the path for external callers that -/// know the Worker name, such as the TUI attach flow. Both should return the same -/// path. -pub fn worker_socket_path(worker_name: &str) -> Option { - worker_socket_path_from_runtime_dir(runtime_dir(), worker_name) -} - // ---- internals -------------------------------------------------------------- fn resolve_config_dir_from_parts( @@ -204,20 +189,6 @@ fn worker_allocation_path_from_runtime_dir(runtime_dir: Option) -> Opti Some(runtime_dir?.join("workers.json")) } -fn worker_runtime_dir_from_runtime_dir( - runtime_dir: Option, - worker_name: &str, -) -> Option { - Some(runtime_dir?.join(worker_name)) -} - -fn worker_socket_path_from_runtime_dir( - runtime_dir: Option, - worker_name: &str, -) -> Option { - Some(worker_runtime_dir_from_runtime_dir(runtime_dir, worker_name)?.join("sock")) -} - /// Treats empty environment values as unset. `std::env::var` distinguishes /// `Ok("")` from `Err(NotPresent)`, but path resolution treats both as unset /// for intuitive behavior. @@ -455,16 +426,8 @@ mod tests { PathBuf::from("/sand/sessions") ); assert_eq!( - worker_allocation_path_from_runtime_dir(runtime_dir.clone()).unwrap(), + worker_allocation_path_from_runtime_dir(runtime_dir).unwrap(), PathBuf::from("/sand/run/workers.json") ); - assert_eq!( - worker_runtime_dir_from_runtime_dir(runtime_dir.clone(), "foo").unwrap(), - PathBuf::from("/sand/run/foo") - ); - assert_eq!( - worker_socket_path_from_runtime_dir(runtime_dir, "foo").unwrap(), - PathBuf::from("/sand/run/foo/sock") - ); } } diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs index 8be8c3d9a..ae54c455e 100644 --- a/crates/protocol/src/lib.rs +++ b/crates/protocol/src/lib.rs @@ -52,7 +52,7 @@ pub enum AuthenticatedInputSource { /// Assigned whenever a serialized tracked method crosses an untrusted /// protocol boundary. Receivers must handle it exactly like public input. UntrustedWire, - /// Stamped only by the operator-owned standalone in-process transport. + /// Stamped only by the operator-owned Standalone Host after local admission. /// This is not a Backend Account or authority accepted from wire fields. LocalOperator, Account { @@ -399,15 +399,12 @@ pub enum Method { /// Typed lifecycle events sent from a child Worker to its parent. /// -/// Delivered as `Method::WorkerEvent` over the parent's Unix socket. The -/// parent Controller always applies variant-specific side effects -/// (registry / worker-allocation updates). Agent-visible variants are also -/// queued into the notification buffer; control-plane-only variants are -/// not injected into the parent's LLM context. +/// Delivered in-process to the parent Controller's protocol intake. Agent-visible variants +/// are queued into the notification buffer. Reports are advisory: receivers must check the +/// actual child session before treating its work as complete. /// -/// Transport is fire-and-forget; receivers must tolerate out-of-order -/// delivery (e.g. `TurnEnded` arriving after `ShutDown` for the same -/// child Worker). +/// Retained legacy control-only variants can be decoded from saved history, but do not +/// register live children, delegate scope, or establish a socket callback. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] @@ -429,16 +426,8 @@ pub enum WorkerEvent { /// Child has stopped (controller loop is exiting). ShutDown { worker_name: String }, - /// Child SubWorker sub-delegated scope to a grandchild SubWorker via `SubWorkerSpawn`. - /// - /// Control-plane only: receivers apply registry side effects and - /// propagate upward, but do not expose this as an agent notification. - /// - /// The parent uses this to add the grandchild to its own - /// `spawned_workers.json` so it can manage the grandchild directly - /// even if the intermediate child dies. The parent then re-fires - /// this event upward (if it has a parent of its own) to maintain - /// the chain to root. + /// Legacy process-child scope report retained for saved Session decoding only. + /// Live Controllers ignore it; only the Host-owned Internal registry grants child scope. ScopeSubDelegated { /// Sub-delegating Worker (= the sender itself). parent_worker: String, diff --git a/crates/standalone/Cargo.toml b/crates/standalone/Cargo.toml index bf05d8375..599d4a565 100644 --- a/crates/standalone/Cargo.toml +++ b/crates/standalone/Cargo.toml @@ -23,7 +23,7 @@ serde.workspace = true serde_json.workspace = true session-store.workspace = true thiserror.workspace = true -tokio = { workspace = true, features = ["rt", "sync", "time"] } +tokio = { workspace = true, features = ["rt", "sync", "time", "net", "io-util", "io-std", "signal"] } uuid = { workspace = true, features = ["v7"] } worker.workspace = true diff --git a/crates/standalone/README.md b/crates/standalone/README.md index 2425ec0fe..fdf71d85c 100644 --- a/crates/standalone/README.md +++ b/crates/standalone/README.md @@ -4,7 +4,8 @@ A Host that starts existing Workers in the same process. Assembles interactive W ## Reference index -- [`host`] — bootstrap and confirmed shutdown. +- [`host`] — bootstrap and confirmed shutdown, with or without a TUI. +- [`unix`] — optional operator transport; disconnect does not release execution ownership. - [`launch`] — Profile, cwd, and Subject validation before launch. - [`store`] — Worker records and leases. - [`jobs`] — Job execution and independent result and cleanup lifetimes. diff --git a/crates/standalone/src/host.rs b/crates/standalone/src/host.rs index 6d55edb67..070a3e214 100644 --- a/crates/standalone/src/host.rs +++ b/crates/standalone/src/host.rs @@ -24,7 +24,6 @@ use session_store::{ }; use thiserror::Error; use worker::bootstrap::{WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout}; -use worker::controller::WorkerControllerTransport; use worker::ipc::protocol_session::{ WorkerProtocolSessionStreams, dispatch_worker_protocol_method, live_log_entry_event, subscribe_worker_protocol_session, @@ -222,7 +221,6 @@ impl StandaloneHost { workspace_context, filesystem_authority, WorkerBootstrapLayout::Direct { runtime_base }, - WorkerControllerTransport::InProcess, ); if let Some(subject) = &subject { bootstrap = bootstrap.with_subjektiv_host(subject.host.clone()); @@ -430,7 +428,6 @@ impl StandaloneHost { workspace_context, filesystem_authority, WorkerBootstrapLayout::Direct { runtime_base }, - WorkerControllerTransport::InProcess, ); if let Some(subject) = &subject { bootstrap = bootstrap.with_subjektiv_host(subject.host.clone()); @@ -520,10 +517,7 @@ impl StandaloneHost { /// Working events, committed session entries, alert snapshots, and the /// initial history snapshot are merged behind the client boundary. pub fn connect(&self) -> Client { - let streams = subscribe_worker_protocol_session(&self.handle); - let (socket, peer) = InProcessSocket::pair(); - tokio::spawn(run_protocol_session(self.handle.clone(), streams, peer)); - Client::new(socket) + Client::new(connect_protocol(self.handle.clone())) } /// Logical Subject disconnection, not Worker shutdown or completion of its represented work. @@ -565,6 +559,35 @@ impl StandaloneHost { } } + /// Run without a TUI until an admitted protocol Shutdown, controller exit, or process signal. + /// Client EOF only disconnects that client. All exits confirm the same production shutdown + /// and lease release as the in-process UI; an unconfirmed shutdown retains the lease. + pub async fn serve_unix( + self, + endpoint: crate::unix::UnixEndpoint, + ) -> Result<(), crate::HeadlessError> { + let mut adapter = crate::unix::UnixAdapter::start(endpoint, self.handle.clone()); + let signal = async { + let mut terminate = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + tokio::select! { + result = tokio::signal::ctrl_c() => result, + _ = terminate.recv() => Ok(()), + } + }; + let result = tokio::select! { + _ = self.handle.protocol_closed() => Ok(()), + result = signal => result, + result = adapter.ended() => result, + }; + // Reclaim the public entry and every connection before releasing execution ownership. + let transport = adapter.shutdown().await; + self.shutdown().await?; + result?; + transport?; + Ok(()) + } + pub fn with_shutdown_timeout(mut self, shutdown_timeout: Duration) -> Self { self.shutdown_timeout = shutdown_timeout; self @@ -647,9 +670,8 @@ impl Drop for StandaloneHost { } } -// The in-process channel is owned by the local operator's Host, not an -// arbitrary socket or Backend account. decode_method discards wire provenance; -// only this boundary can stamp accepted local human input. +// Both in-process views and credential-checked Unix clients are admitted by this operator's +// Host. decode_method discards wire provenance; only this boundary stamps local human input. fn authorize_local_input(method: protocol::Method) -> protocol::Method { use protocol::{AuthenticatedInputSource, Method}; match method { @@ -692,6 +714,13 @@ fn authorize_local_input(method: protocol::Method) -> protocol::Method { } } +pub(crate) fn connect_protocol(handle: worker::WorkerHandle) -> InProcessSocket { + let streams = subscribe_worker_protocol_session(&handle); + let (socket, peer) = InProcessSocket::pair(); + tokio::spawn(run_protocol_session(handle, streams, peer)); + socket +} + async fn run_protocol_session( handle: worker::WorkerHandle, streams: WorkerProtocolSessionStreams, diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index f58e536a4..92e0e539f 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -7,6 +7,44 @@ pub mod launch; pub mod store; pub mod subjektiv; mod subjektiv_sessions; +pub mod unix; + +pub use unix::{UnixEndpoint, relay_unix_client, run_unix_client}; + +/// Restore reads the saved cwd and Profile, not a newly resolved launch recipe. +pub enum HeadlessLaunch { + Fresh(ResolvedStandaloneLaunch), + Restore { + state_dir: std::path::PathBuf, + worker_id: WorkerId, + }, +} + +#[derive(Debug, thiserror::Error)] +pub enum HeadlessError { + #[error(transparent)] + Transport(#[from] std::io::Error), + #[error(transparent)] + Startup(#[from] StandaloneStartupError), + #[error(transparent)] + Shutdown(#[from] StandaloneShutdownError), +} + +/// Reserve the optional public transport before creating or restoring execution state. +pub async fn run_headless( + launch: HeadlessLaunch, + endpoint: std::path::PathBuf, +) -> Result<(), HeadlessError> { + let endpoint = UnixEndpoint::bind(endpoint)?; + let host = match launch { + HeadlessLaunch::Fresh(launch) => StandaloneHost::start(launch).await?, + HeadlessLaunch::Restore { + state_dir, + worker_id, + } => StandaloneHost::restore(state_dir, worker_id).await?, + }; + host.serve_unix(endpoint).await +} pub use host::{ StandaloneHost, StandaloneShutdownError, StandaloneStartupError, diff --git a/crates/standalone/src/subjektiv/journal/tests/connected_compaction.rs b/crates/standalone/src/subjektiv/journal/tests/connected_compaction.rs index b3295945b..50ad7cfbc 100644 --- a/crates/standalone/src/subjektiv/journal/tests/connected_compaction.rs +++ b/crates/standalone/src/subjektiv/journal/tests/connected_compaction.rs @@ -127,7 +127,6 @@ async fn local_host_compaction_freezes_pending_journal_through_request_and_durab worker::WorkerWorkspaceContext::no_workspace(), worker::WorkerFilesystemAuthority::None, worker::WorkerBootstrapLayout::Direct { runtime_base: fixture._temp.path().join("runtime") }, - worker::WorkerControllerTransport::InProcess, ).with_model_client(model.clone()).with_subjektiv_host(fixture.host().clone()).prepare_restored("journal-test").await.unwrap(); assert_eq!(prepared.worker().session_id().to_string(), fixture.context.session_id); let source_segment = prepared.worker().segment_id(); diff --git a/crates/standalone/src/unix.rs b/crates/standalone/src/unix.rs new file mode 100644 index 000000000..ae0881eed --- /dev/null +++ b/crates/standalone/src/unix.rs @@ -0,0 +1,296 @@ +//! Optional operator transport owned by the Standalone Host, never by a Worker. +//! +//! The endpoint's existing parent must be private to the current Unix user. Binding never +//! removes an existing entry, including stale sockets. Recovery is an explicit operator action. +//! Peers must have that same uid; admitted methods still cross the normal local protocol +//! authorization boundary. Disconnect only drops a view, not the Host or its execution lease. + +use std::io; +use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; +use std::path::{Path, PathBuf}; + +use client::transport::Socket; +use protocol::stream::{JsonLineWriter, decode_event, decode_method}; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::net::{UnixListener, UnixStream}; +use tokio::sync::oneshot; +use tokio::task::{JoinHandle, JoinSet}; + +/// A bound endpoint reservation. Drop removes only the exact socket created by this bind. +/// It can be reserved before starting a Host, so bind failures do not allocate Workers. +pub struct UnixEndpoint { + listener: UnixListener, + path: PathBuf, + device: u64, + inode: u64, +} + +impl UnixEndpoint { + pub fn bind(path: impl AsRef) -> io::Result { + let path = path.as_ref(); + let name = path.file_name().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "endpoint requires a filename") + })?; + let parent = std::fs::canonicalize( + path.parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")), + )?; + let metadata = std::fs::metadata(&parent)?; + // SAFETY: geteuid has no preconditions and does not mutate process identity. + let uid = unsafe { libc::geteuid() }; + if !metadata.is_dir() || metadata.uid() != uid || metadata.mode() & 0o077 != 0 { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "endpoint parent must be a current-user-owned private directory (mode 0700)", + )); + } + let path = parent.join(name); + let listener = UnixListener::bind(&path)?; + let metadata = std::fs::symlink_metadata(&path)?; + let endpoint = Self { + listener, + path, + device: metadata.dev(), + inode: metadata.ino(), + }; + std::fs::set_permissions(&endpoint.path, std::fs::Permissions::from_mode(0o600))?; + Ok(endpoint) + } + + pub fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for UnixEndpoint { + fn drop(&mut self) { + if let Ok(current) = std::fs::symlink_metadata(&self.path) + && current.file_type().is_socket() + && current.dev() == self.device + && current.ino() == self.inode + { + let _ = std::fs::remove_file(&self.path); + } + } +} + +pub(crate) struct UnixAdapter { + task: Option>>, + stop: Option>, +} + +impl UnixAdapter { + pub(crate) fn start(endpoint: UnixEndpoint, handle: worker::WorkerHandle) -> Self { + let (stop, mut stopped) = oneshot::channel(); + let task = tokio::spawn(async move { + let mut connections = JoinSet::new(); + let result = loop { + tokio::select! { + _ = &mut stopped => break Ok(()), + accepted = endpoint.listener.accept(), if connections.len() < 64 => { + match accepted { + Ok((stream, _)) => { + // The filesystem gate protects connection establishment; credentials + // also fence open connections to the Host's operator identity. + let allowed = stream.peer_cred().is_ok_and(|cred| { + // SAFETY: geteuid has no preconditions. + cred.uid() == unsafe { libc::geteuid() } + }); + if allowed { + let socket = crate::host::connect_protocol(handle.clone()); + connections.spawn(bridge(stream, socket)); + } + } + Err(error) => break Err(error), + } + } + _ = connections.join_next(), if !connections.is_empty() => {} + } + }; + connections.shutdown().await; + // Keep the bound endpoint alive until every connection has been reclaimed. + drop(endpoint); + result + }); + Self { + task: Some(task), + stop: Some(stop), + } + } + + pub(crate) async fn ended(&mut self) -> io::Result<()> { + let result = self.task.as_mut().expect("adapter task").await; + self.task.take(); + result.map_err(io::Error::other)? + } + + pub(crate) async fn shutdown(mut self) -> io::Result<()> { + if let Some(stop) = self.stop.take() { + let _ = stop.send(()); + } + if self.task.is_some() { + self.ended().await?; + } + Ok(()) + } +} + +impl Drop for UnixAdapter { + fn drop(&mut self) { + if let Some(stop) = self.stop.take() { + let _ = stop.send(()); + } + // Sending stop lets the task join its connection set and drop its endpoint. + // A Host dropped without confirmed shutdown keeps the production execution lease. + } +} + +async fn bridge( + stream: UnixStream, + mut socket: client::transport::in_process::Socket, +) -> io::Result<()> { + let (reader, writer) = stream.into_split(); + let mut reader = BufReader::new(reader).lines(); + let mut writer = JsonLineWriter::new(writer); + loop { + tokio::select! { + line = reader.next_line() => { + let Some(message) = line? else { return Ok(()); }; + if message.trim().is_empty() { continue; } + socket.send(message).await.map_err(io::Error::other)?; + } + event = socket.next() => { + let Some(message) = event.map_err(io::Error::other)? else { return Ok(()); }; + let event = decode_event(&message).map_err(io::Error::other)?; + writer.write_event(&event).await?; + } + } + } +} + +/// Connect a JSONL operator client. EOF disconnects without issuing Shutdown. +/// Input is decoded as Worker Methods and output contains Worker Events only. +pub async fn run_unix_client(endpoint: PathBuf) -> io::Result<()> { + relay_unix_client(endpoint, tokio::io::stdin(), tokio::io::stdout()).await +} + +/// The CLI's stream relay, also usable by embedders that own their input/output streams. +pub async fn relay_unix_client(endpoint: PathBuf, input: R, output: W) -> io::Result<()> +where + R: tokio::io::AsyncRead + Unpin, + W: tokio::io::AsyncWrite + Unpin, +{ + let stream = UnixStream::connect(endpoint).await?; + let (reader, writer) = stream.into_split(); + let mut input = BufReader::new(input).lines(); + let mut to_host = JsonLineWriter::new(writer); + let mut from_host = BufReader::new(reader).lines(); + let mut output = JsonLineWriter::new(output); + loop { + tokio::select! { + line = input.next_line() => { + let Some(line) = line? else { return Ok(()); }; + if line.trim().is_empty() { continue; } + let method = decode_method(&line).map_err(io::Error::other)?; + to_host.write(&method).await?; + } + line = from_host.next_line() => { + let Some(line) = line? else { return Ok(()); }; + let event = decode_event(&line).map_err(io::Error::other)?; + output.write_event(&event).await?; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn endpoint_requires_private_parent_and_is_owner_only() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("host.sock"); + std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let error = UnixEndpoint::bind(&path) + .err() + .expect("public parent rejected"); + assert_eq!(error.kind(), io::ErrorKind::PermissionDenied); + assert!(!path.exists()); + std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let endpoint = UnixEndpoint::bind(&path).unwrap(); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let client = UnixStream::connect(&path).await.unwrap(); + let (server, _) = endpoint.listener.accept().await.unwrap(); + assert_eq!( + client.peer_cred().unwrap().uid(), + server.peer_cred().unwrap().uid() + ); + drop(endpoint); + assert!(!path.exists()); + } + + #[tokio::test] + async fn bind_failure_precedes_worker_state_and_startup_failure_releases_only_its_endpoint() { + let directory = tempfile::tempdir().unwrap(); + std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let path = directory.path().join("host.sock"); + let owner = UnixEndpoint::bind(&path).unwrap(); + let state_dir = directory.path().join("must-not-be-opened"); + let error = crate::run_headless( + crate::HeadlessLaunch::Restore { + state_dir: state_dir.clone(), + worker_id: protocol::WorkerId::now_v7(), + }, + path.clone(), + ) + .await + .unwrap_err(); + assert!(matches!(error, crate::HeadlessError::Transport(_))); + assert!(!state_dir.exists()); + assert!(path.exists()); + drop(owner); + let error = crate::run_headless( + crate::HeadlessLaunch::Restore { + state_dir, + worker_id: protocol::WorkerId::now_v7(), + }, + path.clone(), + ) + .await + .unwrap_err(); + assert!(matches!(error, crate::HeadlessError::Startup(_))); + assert!( + !path.exists(), + "a failed Host startup must release its own bind" + ); + } + + #[tokio::test] + async fn bind_failure_and_old_endpoint_drop_preserve_another_owner() { + let directory = tempfile::tempdir().unwrap(); + std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let path = directory.path().join("host.sock"); + let first = UnixEndpoint::bind(&path).unwrap(); + let first_inode = std::fs::metadata(&path).unwrap().ino(); + assert_eq!( + UnixEndpoint::bind(&path).err().unwrap().kind(), + io::ErrorKind::AddrInUse + ); + assert_eq!(std::fs::metadata(&path).unwrap().ino(), first_inode); + std::fs::rename(&path, directory.path().join("retained.sock")).unwrap(); + let second = UnixEndpoint::bind(&path).unwrap(); + let second_inode = std::fs::metadata(&path).unwrap().ino(); + assert_ne!(first_inode, second_inode); + drop(first); + assert_eq!(std::fs::metadata(&path).unwrap().ino(), second_inode); + let _client = UnixStream::connect(&path).await.unwrap(); + let _connection = second.listener.accept().await.unwrap(); + drop(second); + assert!(!path.exists()); + } +} diff --git a/crates/standalone/tests/host.rs b/crates/standalone/tests/host.rs index a5a632a5e..4670410eb 100644 --- a/crates/standalone/tests/host.rs +++ b/crates/standalone/tests/host.rs @@ -608,6 +608,235 @@ async fn standalone_metadata_fails_closed_on_incomplete_or_newer_records() -> Te Ok(()) } +#[tokio::test] +async fn headless_unix_client_uses_production_history_lease_and_local_authority() -> TestResult { + use protocol::stream::{JsonLineReader, encode_method}; + use std::os::unix::fs::PermissionsExt; + use tokio::io::AsyncWriteExt; + + let temp = tempfile::tempdir()?; + // Only the transport endpoint needs a short private pathname. Persistence remains the + // same production store exercised by the in-process Host tests above. + std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o700))?; + let endpoint_path = temp.path().join("operator.sock"); + let endpoint = standalone::UnixEndpoint::bind(&endpoint_path)?; + let state_dir = temp.path().join("state"); + let launch = StandaloneLaunchConfig::new( + temp.path(), + &state_dir, + manifest::ProfileSelector::Default, + "headless-operator", + ) + .resolve()?; + let model = ScriptedClient::new(vec![vec![ + LlmEvent::text_block_start(0), + LlmEvent::text_delta(0, "headless production response"), + LlmEvent::text_block_stop(0, Some(StopReason::EndTurn)), + ]]); + let inspection = model.clone(); + let host = StandaloneHost::start_with_model_client(launch, model).await?; + let worker_id = host.worker_id(); + let session_id = host.record().active_session_id; + let profile = host.record().manifest.clone(); + let serve = tokio::spawn(host.serve_unix(endpoint)); + + // Exercise the real external client's stdin/stdout relay over the actual Unix adapter. + let (mut input, relay_input) = tokio::io::duplex(64 * 1024); + let (relay_output, output) = tokio::io::duplex(64 * 1024); + let relay = tokio::spawn(standalone::relay_unix_client( + endpoint_path.clone(), + relay_input, + relay_output, + )); + let mut events = JsonLineReader::new(output); + let snapshot = tokio::time::timeout(Duration::from_secs(10), events.next::()) + .await?? + .expect("snapshot"); + assert!(matches!(snapshot, Event::Snapshot { .. })); + let request = Method::SubmitTracked { + submission_request_id: protocol::new_submission_request_id(), + input: vec![protocol::Segment::text("headless production input")], + source: protocol::AuthenticatedInputSource::Backend { + operation_id: "forged-backend-source".into(), + }, + }; + let wire = encode_method(&request)?; + println!("headless input: {wire}"); + input.write_all(wire.as_bytes()).await?; + input.write_all(b"\n").await?; + let mut saw_user = false; + let mut saw_response = false; + let mut saw_state = false; + tokio::time::timeout(Duration::from_secs(20), async { + loop { + let event = events.next::().await?.expect("live event"); + match &event { + Event::UserMessage { segments, .. } => { + saw_user |= format!("{segments:?}").contains("headless production input"); + } + Event::TextDelta { text } => { + saw_response |= text == "headless production response"; + } + Event::WorkerState { .. } => saw_state = true, + Event::RunEnd { .. } => break, + _ => {} + } + println!( + "headless output: {}", + protocol::stream::encode_event(&event)? + ); + } + Ok::<_, Box>(()) + }) + .await??; + assert!(saw_user && saw_response && saw_state); + assert_eq!(inspection.requests().len(), 1); + assert!(matches!( + StandaloneHost::restore_with_model_client( + state_dir.clone(), + worker_id, + ScriptedClient::new(vec![]) + ) + .await, + Err(StandaloneStartupError::WorkerActive) + )); + + // EOF is a disconnect, not a shutdown; another view can reconnect to the same history. + drop(input); + tokio::time::timeout(Duration::from_secs(10), relay).await???; + assert!(!serve.is_finished()); + let mut malformed = tokio::net::UnixStream::connect(&endpoint_path).await?; + malformed.write_all(b"not-json\n").await?; + let mut discarded = Vec::new(); + use tokio::io::AsyncReadExt; + tokio::time::timeout( + Duration::from_secs(10), + malformed.read_to_end(&mut discarded), + ) + .await??; + assert!( + !serve.is_finished(), + "invalid client input must not stop the Host" + ); + let mut socket_client = + Client::new(client::transport::unix_socket::Socket::connect(&endpoint_path).await?); + let snapshot = tokio::time::timeout(Duration::from_secs(10), async { + loop { + let event = socket_client.next_event().await?.expect("reconnect event"); + if matches!(event, Event::Snapshot { .. }) { + break Ok::<_, Box>(event); + } + } + }) + .await??; + let snapshot_text = format!("{snapshot:?}"); + assert!( + snapshot_text.contains("headless production input"), + "{snapshot_text}" + ); + assert!( + snapshot_text.contains("headless production response"), + "{snapshot_text}" + ); + socket_client + .send(&Method::Shutdown { + command: protocol::WorkerCommandEnvelope::new(0), + }) + .await?; + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Some(Event::CommandAcknowledged { acknowledgement }) = + socket_client.next_event().await? + { + assert_eq!( + acknowledgement.disposition, + protocol::WorkerCommandDisposition::StaleCommandId + ); + break; + } + } + Ok::<_, Box>(()) + }) + .await??; + assert!( + !serve.is_finished(), + "stale Shutdown must not bypass Controller admission" + ); + socket_client + .send(&Method::Shutdown { + command: protocol::WorkerCommandEnvelope::new(100), + }) + .await?; + tokio::time::timeout(Duration::from_secs(15), serve).await???; + assert!(!endpoint_path.exists()); + let store = StandaloneWorkerStore::open(&state_dir)?; + let saved = store.load(worker_id)?; + assert_eq!(saved.status, StandaloneWorkerStatus::Stopped); + use session_store::Store; + let history = session_store::FsStore::new(store.sessions_dir(worker_id))?.read_all( + saved.active_session_id, + saved.active_segment_id.expect("active segment"), + )?; + let origins = history + .iter() + .filter_map(|entry| match entry { + session_store::LogEntry::AnnotatedUserInput { history, .. } => Some(history), + _ => None, + }) + .flatten() + .map(|entry| &entry.metadata.origin) + .collect::>(); + assert!(origins.contains(&&session_store::LoggedSessionHistoryOrigin::LocalHumanInput)); + assert!(!format!("{origins:?}").contains("forged-backend-source")); + + // This is the same restoration path called by the TUI, not a headless catalog/import. + let restored = StandaloneHost::restore_with_model_client( + state_dir.clone(), + worker_id, + ScriptedClient::new(vec![]), + ) + .await?; + assert_eq!(restored.worker_id(), worker_id); + assert_eq!(restored.record().active_session_id, session_id); + assert_eq!( + serde_json::to_value(&restored.record().manifest)?, + serde_json::to_value(&profile)? + ); + assert_eq!( + restored.record().cwd.canonical_path, + temp.path().canonicalize()? + ); + let mut in_process = restored.connect(); + let snapshot = tokio::time::timeout(Duration::from_secs(10), async { + loop { + let event = in_process + .next_event() + .await? + .expect("in-process restored event"); + if matches!(event, Event::Snapshot { .. }) { + break Ok::<_, Box>(event); + } + } + }) + .await??; + assert!(format!("{snapshot:?}").contains("headless production response")); + assert!( + !endpoint_path.exists(), + "in-process restore must not recreate the adapter" + ); + let runtime = state_dir + .join(worker_id.to_string()) + .join("runtime") + .join(format!("standalone-{worker_id}")); + assert!(!runtime.join("sock").exists()); + assert!(!runtime.join("pid").exists()); + restored.shutdown().await?; + println!( + "verdict: input/state/response/disconnect/explicit shutdown, shared production restore and concurrent lease exclusion passed for {worker_id}" + ); + Ok(()) +} + async fn wait_for_run_end(client: &mut Client) -> TestResult { tokio::time::timeout(Duration::from_secs(10), async { loop { diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index a482902c6..9b603ff29 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -146,9 +146,9 @@ use worker::feature::builtin::{ use worker::ipc::protocol_session::{WorkerProtocolSessionStreams, live_log_entry_event}; use worker::{ PreparedWorker, PromptCatalogSource, SegmentLogSink, SubjektivSessionAttributionLifecycle, - Worker, WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout, - WorkerControllerTransport, WorkerError, WorkerFilesystemAuthority, WorkerHandle, - WorkerSharedState, WorkerWorkspaceContext, WorkspaceClient, WorkspaceId, + Worker, WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout, WorkerError, + WorkerFilesystemAuthority, WorkerHandle, WorkerSharedState, WorkerWorkspaceContext, + WorkspaceClient, WorkspaceId, }; #[cfg(feature = "ws-server")] @@ -508,7 +508,6 @@ pub struct ProfileRuntimeWorkerFactory { worker_mutation_identity: Option, workspace_request_clients: Arc>, embedded_worker_mutation_dispatcher: Option>, - controller_transport: WorkerControllerTransport, failed_restore_sessions: Arc< Mutex< HashMap< @@ -536,7 +535,6 @@ impl ProfileRuntimeWorkerFactory { worker_mutation_identity: None, workspace_request_clients: Arc::new(HashMap::new()), embedded_worker_mutation_dispatcher: None, - controller_transport: WorkerControllerTransport::InProcess, failed_restore_sessions: Arc::new(Mutex::new(HashMap::new())), } } @@ -585,14 +583,6 @@ impl ProfileRuntimeWorkerFactory { self } - pub fn with_controller_transport( - mut self, - controller_transport: WorkerControllerTransport, - ) -> Self { - self.controller_transport = controller_transport; - self - } - pub fn with_runtime_store_dir(mut self, runtime_store_dir: impl Into) -> Self { self.worker_aggregate_root = Some(runtime_store_dir.into().join("workers")); self @@ -1598,13 +1588,13 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .resolve_profile(selector, &worker_root, &worker_name) .map_err(|err| format!("failed to resolve profile source archive: {err}"))?; if has_local_filesystem { - worker::entrypoint::resolve_runtime_profile_manifest_from_manifest( + worker::profile_manifest::resolve_runtime_profile_manifest_from_manifest( manifest, &worker_root, &worker_name, )? } else { - worker::entrypoint::resolve_runtime_profile_manifest_from_manifest_without_filesystem( + worker::profile_manifest::resolve_runtime_profile_manifest_from_manifest_without_filesystem( manifest, &worker_root, &worker_name, @@ -1651,7 +1641,6 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { WorkerBootstrapLayout::RuntimeManagedRun { run_dir: run_dir.clone(), }, - self.controller_transport, ); let bootstrap = match &self.spawn_model_client_factory { Some(factory) => bootstrap.with_model_client(factory()), @@ -2004,7 +1993,6 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { WorkerBootstrapLayout::RuntimeManagedRun { run_dir: run_dir.clone(), }, - self.controller_transport, ) .start() .await @@ -6515,8 +6503,7 @@ mod tests { let backend = WorkerRuntimeExecutionBackend::new( ProfileRuntimeWorkerFactory::new(root.path().join("missing-profiles")) .with_runtime_store_dir(&store) - .with_runtime_id("runtime-subjektiv-test") - .with_controller_transport(WorkerControllerTransport::InProcess), + .with_runtime_id("runtime-subjektiv-test"), ) .unwrap(); backend.preflight_restore(&request).unwrap(); @@ -9132,17 +9119,6 @@ mod tests { assert!(error.contains("replacement Worker is required"), "{error}"); } - #[test] - fn profile_runtime_factory_defaults_to_in_process_controller_transport() { - let root = tempfile::tempdir().unwrap(); - let factory = ProfileRuntimeWorkerFactory::new(root.path()); - - assert!(matches!( - factory.controller_transport, - WorkerControllerTransport::InProcess - )); - } - #[tokio::test] #[serial_test::serial(worker_allocation)] async fn in_process_restore_does_not_bind_unix_socket_under_overlong_store_path() { @@ -9205,7 +9181,6 @@ mod tests { let controller = ProfileRuntimeWorkerFactory::new(root.path()) .with_runtime_store_dir(&runtime_store_dir) - .with_controller_transport(WorkerControllerTransport::InProcess) .restore_controller(WorkerExecutionRestoreRequest { operation_id: crate::execution::WorkerLifecycleOperationId::new(), worker_ref: worker_ref.clone(), diff --git a/crates/worker/Cargo.toml b/crates/worker/Cargo.toml index a96239ba3..368ec7662 100644 --- a/crates/worker/Cargo.toml +++ b/crates/worker/Cargo.toml @@ -11,7 +11,6 @@ default = [] [dependencies] async-trait = { workspace = true } -clap = { version = "4.6.0", features = ["derive"] } agen = { workspace = true, features = ["codex"] } session-store = { workspace = true } secrets = { workspace = true } diff --git a/crates/worker/src/bootstrap.rs b/crates/worker/src/bootstrap.rs index d7871660b..8bbc9ed52 100644 --- a/crates/worker/src/bootstrap.rs +++ b/crates/worker/src/bootstrap.rs @@ -6,9 +6,7 @@ use thiserror::Error; use workdir::WorkdirSessionHandle; use crate::PromptCatalogSource; -use crate::controller::{ - ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle, -}; +use crate::controller::{ShutdownReceiver, WorkerController, WorkerHandle}; use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext}; use manifest::WorkerManifest; @@ -30,7 +28,6 @@ pub struct WorkerBootstrap { workspace_context: WorkerWorkspaceContext, filesystem_authority: WorkerFilesystemAuthority, layout: WorkerBootstrapLayout, - transport: WorkerControllerTransport, model_client: Option>, workdir_session: Option, subjektiv_host: Option>, @@ -41,7 +38,6 @@ pub struct WorkerBootstrap { pub struct PreparedWorker { worker: Worker, layout: WorkerBootstrapLayout, - transport: WorkerControllerTransport, } /// Live controller returned only after Worker construction and feature @@ -68,7 +64,6 @@ impl WorkerBootstrap where St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { - #[allow(clippy::too_many_arguments)] pub fn new( manifest: WorkerManifest, store: St, @@ -76,7 +71,6 @@ where workspace_context: WorkerWorkspaceContext, filesystem_authority: WorkerFilesystemAuthority, layout: WorkerBootstrapLayout, - transport: WorkerControllerTransport, ) -> Self { Self { manifest, @@ -85,7 +79,6 @@ where workspace_context, filesystem_authority, layout, - transport, model_client: None, workdir_session: None, subjektiv_host: None, @@ -165,7 +158,7 @@ where ) .map_err(WorkerBootstrapError::Worker)?; } - Ok(PreparedWorker::new(worker, self.layout, self.transport)) + Ok(PreparedWorker::new(worker, self.layout)) } pub async fn prepare_restored( @@ -206,7 +199,7 @@ where ) .map_err(WorkerBootstrapError::Worker)?; } - Ok(PreparedWorker::new(worker, self.layout, self.transport)) + Ok(PreparedWorker::new(worker, self.layout)) } pub async fn start(self) -> Result { @@ -221,16 +214,8 @@ where { /// Wrap a restored Worker in the same pre-exposure lifecycle used by fresh /// bootstraps. - pub fn new( - worker: Worker, - layout: WorkerBootstrapLayout, - transport: WorkerControllerTransport, - ) -> Self { - Self { - worker, - layout, - transport, - } + pub fn new(worker: Worker, layout: WorkerBootstrapLayout) -> Self { + Self { worker, layout } } pub fn worker(&self) -> &Worker { @@ -242,7 +227,7 @@ where } pub async fn start(self) -> Result { - start_worker_controller(self.worker, self.layout, self.transport).await + start_worker_controller(self.worker, self.layout).await } } @@ -252,7 +237,6 @@ where pub async fn start_worker_controller( worker: Worker, layout: WorkerBootstrapLayout, - transport: WorkerControllerTransport, ) -> Result where C: LlmClient + Clone + 'static, @@ -261,13 +245,10 @@ where let cleanup_sessions = worker.workdir_sessions(); let controller = match layout { WorkerBootstrapLayout::Direct { runtime_base } => { - WorkerController::spawn_with_transport_owned(worker, &runtime_base, transport).await + WorkerController::spawn_owned(worker, &runtime_base).await } WorkerBootstrapLayout::RuntimeManagedRun { run_dir } => { - WorkerController::spawn_runtime_managed_run_with_transport_owned( - worker, &run_dir, transport, - ) - .await + WorkerController::spawn_runtime_managed_run_owned(worker, &run_dir).await } }; diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index df6a0475d..b96359f8d 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -10,12 +10,10 @@ use session_store::WorkerMetadataStore; use session_store::{LogEntry, Store}; use tokio::sync::{broadcast, mpsc, oneshot}; -use crate::discovery::WorkerDiscovery; use crate::feature::FeatureRegistryBuilder; use crate::in_flight::{InFlightEvents, snapshot_from_guard}; use crate::ipc::alerter::Alerter; use crate::ipc::notify_buffer::NotifyBuffer; -use crate::ipc::server::SocketServer; use crate::runtime::dir::RuntimeDir; use crate::segment_log_sink::SegmentLogSink; use crate::shared_state::{WorkerCommandAdmission, WorkerSharedState}; @@ -317,7 +315,7 @@ impl WorkerHandle { } } - /// Broadcast an event to all listeners (including socket clients). + /// Broadcast an event to all listeners. pub fn send_event(&self, event: Event) -> Result> { self.working_event_tx.send(event) } @@ -436,12 +434,10 @@ fn reject_invalid_command_state( async fn set_controller_state( shared_state: &Arc, - runtime_dir: &RuntimeDir, working_event_tx: &broadcast::Sender, state: WorkerState, ) -> protocol::WorkerStateSnapshot { let snapshot = shared_state.transition(state); - let _ = runtime_dir.write_status(shared_state).await; let _ = working_event_tx.send(Event::WorkerState { snapshot: snapshot.clone(), }); @@ -461,7 +457,6 @@ fn publish_context_usage( async fn set_controller_status( shared_state: &Arc, - runtime_dir: &RuntimeDir, working_event_tx: &broadcast::Sender, status: WorkerStatus, ) { @@ -470,13 +465,12 @@ async fn set_controller_status( WorkerStatus::Running => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)), WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)), }; - set_controller_state(shared_state, runtime_dir, working_event_tx, state).await; + set_controller_state(shared_state, working_event_tx, state).await; } async fn finish_controller_run( worker: &mut Worker, shared_state: &Arc, - runtime_dir: &RuntimeDir, working_event_tx: &broadcast::Sender, new_status: WorkerStatus, finished_submission_request_id: Option, @@ -509,10 +503,9 @@ async fn finish_controller_run( WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)), }; let snapshot = shared_state.finish_submission_run(state, submission_request_id); - let _ = runtime_dir.write_status(shared_state).await; let _ = working_event_tx.send(Event::WorkerState { snapshot }); } else { - set_controller_status(shared_state, runtime_dir, working_event_tx, new_status).await; + set_controller_status(shared_state, working_event_tx, new_status).await; } } @@ -653,39 +646,12 @@ fn prepare_pending_run( }) } -impl PendingRun { - /// Whether this turn was kicked off by the parent (via `Method::Submit` - /// or `Method::Resume`). Used by [`drive_turn`] to gate upward - /// `WorkerEvent::TurnEnded` / `WorkerEvent::Errored` reports so the parent - /// only sees completion signals for work it actually delegated. - /// `RunForNotification` covers self-initiated turns kicked from the - /// notify buffer (Notify / inbound WorkerEvent) and stays silent. - fn is_parent_originated(&self) -> bool { - match self { - PendingRun::Submit(_) | PendingRun::Resume => true, - PendingRun::RunForNotification { .. } => false, - } - } -} - // --------------------------------------------------------------------------- // WorkerController — actor that owns a Worker // --------------------------------------------------------------------------- pub type ShutdownReceiver = oneshot::Receiver<()>; -/// Client transport exposed by a Worker controller. -/// -/// Process-hosted Workers use a Unix socket for external attach clients. Runtimes -/// that retain the returned [`WorkerHandle`] in the same process can disable that -/// redundant listener and drive the controller directly through its channels. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum WorkerControllerTransport { - #[default] - UnixSocket, - InProcess, -} - pub struct WorkerController; impl WorkerController { @@ -697,50 +663,24 @@ impl WorkerController { C: LlmClient + Clone + 'static, St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { - let (handle, shutdown, _task) = Self::spawn_inner( - worker, - runtime_base, - false, - None, - WorkerControllerTransport::UnixSocket, - ) - .await?; + let (handle, shutdown, _task) = Self::spawn_inner(worker, runtime_base, None).await?; Ok((handle, shutdown)) } - /// Spawn a direct Worker while letting an in-process host select the - /// controller transport explicitly. - pub async fn spawn_with_transport( + pub(crate) async fn spawn_owned( worker: Worker, runtime_base: &Path, - - transport: WorkerControllerTransport, - ) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error> - where - C: LlmClient + Clone + 'static, - St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, - { - let (handle, shutdown, _task) = - Self::spawn_with_transport_owned(worker, runtime_base, transport).await?; - Ok((handle, shutdown)) - } - - pub(crate) async fn spawn_with_transport_owned( - worker: Worker, - runtime_base: &Path, - - transport: WorkerControllerTransport, ) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error> where C: LlmClient + Clone + 'static, St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { - Self::spawn_inner(worker, runtime_base, false, None, transport).await + Self::spawn_inner(worker, runtime_base, None).await } /// Spawn a Worker owned by `worker-runtime`. /// - /// The controller uses an ephemeral directory for Unix sockets. Bash output is + /// The controller uses an ephemeral directory for run artifacts. Bash output is /// owned by its live Feature instance, independently of filesystem layout. /// Runtime-managed Workers do not write legacy pid/status/manifest liveness /// projections. @@ -752,14 +692,7 @@ impl WorkerController { C: LlmClient + Clone + 'static, St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { - let (handle, shutdown, _task) = Self::spawn_inner( - worker, - runtime_base, - true, - None, - WorkerControllerTransport::UnixSocket, - ) - .await?; + let (handle, shutdown, _task) = Self::spawn_inner(worker, runtime_base, None).await?; Ok((handle, shutdown)) } @@ -768,41 +701,18 @@ impl WorkerController { worker: Worker, run_dir: &Path, ) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error> - where - C: LlmClient + Clone + 'static, - St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, - { - Self::spawn_runtime_managed_run_with_transport( - worker, - run_dir, - WorkerControllerTransport::UnixSocket, - ) - .await - } - - /// Spawn into an exact persistent `runs/` directory using the - /// requested client transport. - pub async fn spawn_runtime_managed_run_with_transport( - worker: Worker, - run_dir: &Path, - - transport: WorkerControllerTransport, - ) -> Result<(WorkerHandle, ShutdownReceiver), std::io::Error> where C: LlmClient + Clone + 'static, St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { let (handle, shutdown, _task) = - Self::spawn_runtime_managed_run_with_transport_owned(worker, run_dir, transport) - .await?; + Self::spawn_runtime_managed_run_owned(worker, run_dir).await?; Ok((handle, shutdown)) } - pub(crate) async fn spawn_runtime_managed_run_with_transport_owned( + pub(crate) async fn spawn_runtime_managed_run_owned( worker: Worker, run_dir: &Path, - - transport: WorkerControllerTransport, ) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error> where C: LlmClient + Clone + 'static, @@ -811,30 +721,20 @@ impl WorkerController { let parent = run_dir .parent() .ok_or_else(|| std::io::Error::other("run path has no parent"))?; - Self::spawn_inner(worker, parent, true, Some(run_dir), transport).await + Self::spawn_inner(worker, parent, Some(run_dir)).await } async fn spawn_inner( worker: Worker, runtime_base: &Path, - - runtime_managed: bool, runtime_run: Option<&Path>, - transport: WorkerControllerTransport, ) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error> where C: LlmClient + Clone + 'static, St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { let sessions = worker.workdir_sessions(); - let result = Self::spawn_initialized( - worker, - runtime_base, - runtime_managed, - runtime_run, - transport, - ) - .await; + let result = Self::spawn_initialized(worker, runtime_base, runtime_run).await; if result.is_err() && let Err(error) = sessions.close_all().await { @@ -846,10 +746,7 @@ impl WorkerController { async fn spawn_initialized( mut worker: Worker, runtime_base: &Path, - - runtime_managed: bool, runtime_run: Option<&Path>, - transport: WorkerControllerTransport, ) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error> where C: LlmClient + Clone + 'static, @@ -868,19 +765,14 @@ impl WorkerController { let in_flight = InFlightEvents::new(working_event_tx.clone()); worker.attach_in_flight_events(in_flight.clone()); - // Runtime directory is created before tool registration and before - // initial status/history writes consume - // the greeting we build after registration is complete. + // Retain run artifacts before registering the Worker tools. let runtime_dir = Arc::new(if let Some(run_dir) = runtime_run { RuntimeDir::create_worker_run(run_dir).await? - } else if runtime_managed { - RuntimeDir::create_transient(runtime_base, &worker.manifest().worker.name).await? } else { RuntimeDir::create(runtime_base, &worker.manifest().worker.name).await? }); let spawner_name = worker.manifest().worker.name.clone(); - let self_parent_socket = worker.callback_socket().cloned(); let loaded_registry = SpawnedWorkerRegistry::load_from_worker_state_with_reclaim( runtime_dir.clone(), worker.store().clone(), @@ -959,8 +851,7 @@ impl WorkerController { .map_err(|error| std::io::Error::other(error.to_string()))?; } - // === 4. Initial runtime files + WorkerSharedState + WorkerHandle + - // SocketServer === + // === 4. WorkerSharedState + WorkerHandle === let manifest_toml = toml::to_string_pretty(worker.manifest()).unwrap_or_default(); worker .recover_unfinished_compaction() @@ -970,7 +861,7 @@ impl WorkerController { let shared_state = Arc::new(WorkerSharedState::new( worker.manifest().worker.name.clone(), worker.segment_id(), - manifest_toml.clone(), + manifest_toml, greeting, )); shared_state.set_feature_invocations(worker.feature_invocations()); @@ -987,8 +878,6 @@ impl WorkerController { if let Some(fs_for_view) = fs_for_view.as_ref() { shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view.clone())); } - runtime_dir.write_manifest(&manifest_toml).await?; - runtime_dir.write_status(&shared_state).await?; let artifact_store: Arc = Arc::new(worker.store().clone()); let session_id = worker.session_id(); @@ -1014,26 +903,10 @@ impl WorkerController { pending_activations, }; - // Do not start an observer before fallible head/recovery/runtime-file - // initialization. The only remaining fallible step is socket startup, - // whose error path must abort AND join this task before returning. + // Start the observer only after fallible head/recovery initialization. let command_observer = fs_for_view .as_ref() .and_then(|session| wire_workdir_command_events(session, &in_flight)); - let socket_server = match transport { - WorkerControllerTransport::UnixSocket => match SocketServer::start(&handle).await { - Ok(server) => Some(server), - Err(error) => { - if let Some(observer) = command_observer { - observer.abort(); - let _ = observer.await; - } - return Err(error); - } - }, - WorkerControllerTransport::InProcess => None, - }; - // === 5. controller_loop === // Clone cancel sender and notification buffer before moving worker // into the controller task so the in-flight turn can be reached @@ -1051,11 +924,8 @@ impl WorkerController { cancel_tx, pause_tx, notify_buffer, - self_parent_socket, - spawner_name, spawned_registry, shutdown_tx, - socket_server, shutdown_after_idle, command_observer, )); @@ -2290,15 +2160,12 @@ async fn controller_loop( mut method_rx: mpsc::Receiver, working_event_tx: broadcast::Sender, shared_state: Arc, - runtime_dir: Arc, + _runtime_dir: Arc, cancel_tx: mpsc::Sender<()>, pause_tx: mpsc::Sender<()>, notify_buffer: NotifyBuffer, - self_parent_socket: Option, - spawner_name: String, spawned_registry: Arc, shutdown_tx: oneshot::Sender<()>, - socket_server: Option, shutdown_after_idle: ShutdownAfterIdleRequest, mut command_observer: Option>, ) where @@ -2311,25 +2178,6 @@ async fn controller_loop( .expect("Controller release authority installed") .clone(); let _release_lifetime = crate::subject_release::ControllerLifetime(release_control.clone()); - // Hold an optional external attach server alive for the controller lifetime. - // In-process runtimes retain and drive the WorkerHandle directly. - let _socket_server = socket_server; - - let discovery_runtime_base = runtime_dir - .path() - .parent() - .map(PathBuf::from) - .unwrap_or_else(|| runtime_dir.path().to_path_buf()); - let discovery_cwd = worker - .local_working_directory() - .map(|local| local.cwd.clone()); - let discovery = WorkerDiscovery::new( - worker.store().clone(), - spawner_name.clone(), - discovery_runtime_base, - discovery_cwd, - spawned_registry.clone(), - ); let notification_coalesce_delay = worker.notification_coalesce_delay(); let pending_submissions = worker.pending_submission_handle(); let mut pending = match prepare_pending_run(&pending_submissions, ¬ify_buffer, None, false) { @@ -2368,20 +2216,14 @@ async fn controller_loop( // clear at run start prevents stale partial output left by an older // interrupted/error turn from being carried into the next snapshot. worker.clear_in_flight_events(); - let parent_originated = run.is_parent_originated(); let finished_submission_request_id = match &run { PendingRun::Submit(submission) => Some(submission.submission_request_id.clone()), PendingRun::RunForNotification { .. } | PendingRun::Resume => None, }; let user_input_submit = matches!(&run, PendingRun::Submit(_)); if !user_input_submit { - set_controller_status( - &shared_state, - &runtime_dir, - &working_event_tx, - WorkerStatus::Running, - ) - .await; + set_controller_status(&shared_state, &working_event_tx, WorkerStatus::Running) + .await; } let (mut new_status, shutdown, may_drain_pending) = match run { PendingRun::Submit(submission) => { @@ -2404,14 +2246,9 @@ async fn controller_loop( &cancel_tx, &pause_tx, &shared_state, - &runtime_dir, Some((input_commit_rx, committed_submission)), ¬ify_buffer, &pending_submissions, - self_parent_socket.as_ref(), - &spawner_name, - &spawned_registry, - parent_originated, ) .await } @@ -2423,14 +2260,9 @@ async fn controller_loop( &cancel_tx, &pause_tx, &shared_state, - &runtime_dir, None, ¬ify_buffer, &pending_submissions, - self_parent_socket.as_ref(), - &spawner_name, - &spawned_registry, - parent_originated, ) .await } @@ -2442,14 +2274,9 @@ async fn controller_loop( &cancel_tx, &pause_tx, &shared_state, - &runtime_dir, None, ¬ify_buffer, &pending_submissions, - self_parent_socket.as_ref(), - &spawner_name, - &spawned_registry, - parent_originated, ) .await } @@ -2475,7 +2302,6 @@ async fn controller_loop( finish_controller_run( &mut worker, &shared_state, - &runtime_dir, &working_event_tx, new_status, finished_submission_request_id, @@ -2607,13 +2433,7 @@ async fn controller_loop( if worker.has_pending_compaction_cleanup() { match worker.retry_pending_compaction_cleanup().await { Ok(()) => { - set_controller_state( - &shared_state, - &runtime_dir, - &working_event_tx, - WorkerState::Idle, - ) - .await; + set_controller_state(&shared_state, &working_event_tx, WorkerState::Idle).await; } Err(error) => { if matches!(&method, Method::Shutdown { .. }) { @@ -2847,7 +2667,6 @@ async fn controller_loop( } set_controller_state( &shared_state, - &runtime_dir, &working_event_tx, WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)), ) @@ -2889,7 +2708,6 @@ async fn controller_loop( } set_controller_state( &shared_state, - &runtime_dir, &working_event_tx, WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)), ) @@ -2904,18 +2722,12 @@ async fn controller_loop( match worker.cancel_paused_turn() { Ok(()) => { worker.clear_in_flight_events(); - set_controller_state( - &shared_state, - &runtime_dir, - &working_event_tx, - WorkerState::Idle, - ) - .await; + set_controller_state(&shared_state, &working_event_tx, WorkerState::Idle) + .await; } Err(error) => { set_controller_state( &shared_state, - &runtime_dir, &working_event_tx, WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)), ) @@ -2973,7 +2785,6 @@ async fn controller_loop( } set_controller_state( &shared_state, - &runtime_dir, &working_event_tx, WorkerState::Busy(WorkerBusyState::Maintenance( WorkerMaintenanceState::Compacting, @@ -3076,13 +2887,7 @@ async fn controller_loop( | Err(WorkerError::SegmentActivationIncomplete { .. }) | Err(WorkerError::CompactionCleanupPending { .. }) ) { - set_controller_state( - &shared_state, - &runtime_dir, - &working_event_tx, - WorkerState::Idle, - ) - .await; + set_controller_state(&shared_state, &working_event_tx, WorkerState::Idle).await; } if let Err(error) = result && !matches!(&error, WorkerError::CompactCancelled) @@ -3164,80 +2969,45 @@ async fn controller_loop( break; } - Method::ListWorkers => match discovery.list_visible().await { - Ok(workers) => match serde_json::to_value(workers) { - Ok(workers) => { - let _ = working_event_tx.send(Event::WorkersListed { workers }); - } + Method::ListWorkers | Method::RestoreWorker { .. } => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: "Worker discovery and restore are owned by the Host or Runtime".into(), + }); + } + + Method::RegisterPeer { name } => { + match crate::peer::register_peer( + worker.store(), + &worker.manifest().worker.name, + &name, + ) { + Ok(result) => match serde_json::to_value(result) { + Ok(result) => { + let _ = working_event_tx.send(Event::PeerRegistered { result }); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: format!("serialize peer registration result: {error}"), + }); + } + }, Err(error) => { let _ = working_event_tx.send(Event::Error { - code: ErrorCode::Internal, - message: format!("serialize visible workers: {error}"), + code: ErrorCode::InvalidRequest, + message: error.to_string(), }); } - }, - Err(error) => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::InvalidRequest, - message: error.to_string(), - }); } - }, + } - Method::RestoreWorker { name } => match discovery.restore(&name).await { - Ok(result) => match serde_json::to_value(result) { - Ok(result) => { - let _ = working_event_tx.send(Event::WorkerRestored { result }); - } - Err(error) => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::Internal, - message: format!("serialize worker restore result: {error}"), - }); - } - }, - Err(error) => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::InvalidRequest, - message: error.to_string(), - }); - } - }, - - Method::RegisterPeer { name } => match discovery.register_peer(&name) { - Ok(result) => match serde_json::to_value(result) { - Ok(result) => { - let _ = working_event_tx.send(Event::PeerRegistered { result }); - } - Err(error) => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::Internal, - message: format!("serialize peer registration result: {error}"), - }); - } - }, - Err(error) => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::InvalidRequest, - message: error.to_string(), - }); - } - }, - - // ListCompletions is handled at the socket layer (direct + // ListCompletions is handled by the Host protocol adapter (direct // response). If it reaches the controller, ignore it. Method::ListCompletions { .. } => {} Method::WorkerEvent(event) => { - if handle_inbound_worker_event( - event, - &spawned_registry, - &spawner_name, - self_parent_socket.as_ref(), - ¬ify_buffer, - ) - .await - { + if handle_inbound_worker_event(event, ¬ify_buffer) { // Auto-kick a turn if the Worker is idle so the // notification is not stranded. Matches the // `Method::Notify` idle path. @@ -3256,20 +3026,6 @@ async fn controller_loop( // senders then fail instead of accepting work that this execution can no // longer process. drop(method_rx); - let had_socket_server = _socket_server.is_some(); - if let Some(socket_server) = _socket_server { - socket_server.shutdown().await; - } - while had_socket_server { - match runtime_dir.close_socket().await { - Ok(()) => break, - Err(error) => { - tracing::warn!(%error, "Worker runtime socket cleanup failed; retrying"); - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - } - } - // Feature callbacks and tasks share the Worker scope. Stop them before // Memory/Workdir teardown so they cannot observe a partially closed Worker. worker.stop_feature_runtime("controller shutdown").await; @@ -3302,49 +3058,11 @@ async fn controller_loop( let _ = observer.await; } - // Report upward that this Worker is stopping before the controller - // task exits. Awaited (not fire-and-forget): after `shutdown_tx.send` - // the process may exit quickly, and a spawned task would be killed - // mid-send. The `connect_and_send` helper enforces a 5 s timeout so - // a stuck parent cannot block process exit indefinitely. - if let Some(parent) = self_parent_socket.as_ref() { - if let Err(e) = crate::ipc::event::send_worker_event( - parent, - protocol::WorkerEvent::ShutDown { - worker_name: spawner_name.clone(), - }, - ) - .await - { - tracing::warn!(error = %e, "ShutDown WorkerEvent send failed"); - } - } - let _ = shutdown_tx.send(()); } -/// Apply an inbound child `WorkerEvent` exactly once. -/// -/// Side effects are control-plane state updates and upward propagation; they -/// run for every event. Only agent-visible events are staged on the notify -/// buffer. The caller owns lifecycle-dependent follow-up such as idle -/// `RunForNotification` auto-kick. -async fn handle_inbound_worker_event( - event: protocol::WorkerEvent, - spawned_registry: &Arc, - self_name: &str, - parent_socket: Option<&PathBuf>, - notify_buffer: &NotifyBuffer, -) -> bool { - let self_parent_socket = parent_socket.cloned(); - crate::ipc::event::apply_event_side_effects( - &event, - spawned_registry, - self_name, - &self_parent_socket, - ) - .await; - +/// Stage an agent-visible Internal SubWorker lifecycle event. +fn handle_inbound_worker_event(event: protocol::WorkerEvent, notify_buffer: &NotifyBuffer) -> bool { let notify_agent = event.should_notify_agent(); if notify_agent { notify_buffer.push_worker_event(event); @@ -3354,19 +3072,7 @@ async fn handle_inbound_worker_event( /// Drives a Worker future (one in-flight turn) while concurrently /// processing incoming methods through an inner select! arm. Returns -/// `(final_status, shutdown_requested)`. -/// -/// `parent_socket` / `self_name` drive upward `WorkerEvent` reports -/// (`TurnEnded` on a clean Finished, `Errored` on a worker failure). -/// `None` parent skips the send (top-level Worker). Transient method -/// rejections such as `AlreadyRunning` are intentionally NOT reported -/// as `Errored` — only the worker-execution `Err` branch below fires. -/// -/// `parent_originated` further restricts both upward reports to turns -/// the parent actually delegated (`Method::Submit` / `Method::Resume`). -/// `Method::Notify` / inbound `WorkerEvent` auto-kicks complete silently -/// so the parent's history does not get flooded with child-internal -/// turn boundaries. +/// `(final_status, shutdown_requested, may_drain_pending)`. #[allow(clippy::too_many_arguments)] async fn drive_turn( worker_future: F, @@ -3375,14 +3081,9 @@ async fn drive_turn( cancel_tx: &mpsc::Sender<()>, pause_tx: &mpsc::Sender<()>, shared_state: &Arc, - runtime_dir: &RuntimeDir, mut input_commit: Option<(oneshot::Receiver<()>, crate::worker::PendingSubmission)>, notify_buffer: &NotifyBuffer, pending_submissions: &crate::worker::PendingSubmissionHandle, - parent_socket: Option<&PathBuf>, - self_name: &str, - spawned_registry: &Arc, - parent_originated: bool, ) -> (WorkerStatus, bool, bool) where F: std::future::Future>, @@ -3418,7 +3119,6 @@ where } set_controller_status( shared_state, - runtime_dir, working_event_tx, WorkerStatus::Running, ) @@ -3468,35 +3168,14 @@ where code, message: message.clone(), }); - if parent_originated { - crate::ipc::event::fire_and_forget( - parent_socket.cloned(), - protocol::WorkerEvent::Errored { - worker_name: self_name.to_string(), - message, - }, - ); - } return (WorkerStatus::Idle, shutdown_requested, false); } }; let _ = working_event_tx.send(Event::RunEnd { result: run_result }); - if parent_originated && matches!(run_result, RunResult::Finished) { - crate::ipc::event::fire_and_forget( - parent_socket.cloned(), - protocol::WorkerEvent::TurnEnded { - worker_name: self_name.to_string(), - }, - ); - } (status, shutdown_requested, may_drain_pending) } Err(WorkerError::Engine(EngineError::Cancelled)) if pause_requested => { - // User-initiated Pause. Report the transition to - // clients as a normal Paused run-end, and - // intentionally skip `WorkerEvent::Errored` upward: - // that channel is reserved for worker runtime - // failures, not deliberate interruptions. + // User-initiated Pause is a normal Paused run-end. let _ = working_event_tx.send(Event::RunEnd { result: RunResult::Paused }); (WorkerStatus::Paused, shutdown_requested, false) } @@ -3515,17 +3194,8 @@ where let message = e.to_string(); let _ = working_event_tx.send(Event::Error { code, - message: message.clone(), + message, }); - if parent_originated { - crate::ipc::event::fire_and_forget( - parent_socket.cloned(), - protocol::WorkerEvent::Errored { - worker_name: self_name.to_string(), - message, - }, - ); - } } (status, shutdown_requested, false) } @@ -3560,7 +3230,6 @@ where } set_controller_state( shared_state, - runtime_dir, working_event_tx, WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)), ) @@ -3602,7 +3271,6 @@ where pause_requested = true; set_controller_state( shared_state, - runtime_dir, working_event_tx, WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Pausing)), ) @@ -3630,7 +3298,6 @@ where shutdown_requested = true; set_controller_state( shared_state, - runtime_dir, working_event_tx, WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)), ) @@ -3852,12 +3519,8 @@ where // from the notify buffer on its next history append. handle_inbound_worker_event( event, - spawned_registry, - self_name, - parent_socket, notify_buffer, - ) - .await; + ); } None => { let _ = cancel_tx.try_send(()); @@ -4029,10 +3692,8 @@ mod subject_release_tests; mod tests { use super::*; use protocol::WorkerEvent; - use protocol::stream::{JsonLineReader, JsonLineWriter}; use std::time::Duration; use tempfile::TempDir; - use tokio::net::UnixListener; mod engine_alerts { use super::*; @@ -4418,19 +4079,8 @@ mod tests { assert!(materialize < exposure); let observer = startup.find("let command_observer = fs_for_view").unwrap(); let recover = startup.find(".recover_unfinished_compaction()").unwrap(); - let files = startup - .find("runtime_dir.write_status(&shared_state).await?") - .unwrap(); let loop_start = startup.find("tokio::spawn(controller_loop(").unwrap(); - assert!(recover < observer && files < observer && observer < loop_start); - let socket_startup = &startup[observer..loop_start]; - let abort = socket_startup.find("observer.abort()").unwrap(); - let join = socket_startup.find("observer.await").unwrap(); - let failure = socket_startup.find("return Err(error)").unwrap(); - assert!( - abort < join && join < failure, - "startup cannot detach its observer on error" - ); + assert!(recover < observer && observer < loop_start); assert!( startup[materialize_gate..materialize] .contains("needs_initial_session_head_materialization") @@ -4643,17 +4293,6 @@ mod tests { assert!(!model_supports_image_attachments(&anthropic)); } - #[test] - fn pending_run_parent_origin_table() { - assert!(PendingRun::Resume.is_parent_originated()); - assert!( - !PendingRun::RunForNotification { - invoke_kind: protocol::InvokeKind::Notify, - } - .is_parent_originated() - ); - } - struct DriveTurnEnv { // Held to keep the channel alive; without this `method_rx.recv()` // would observe channel-closed and confuse the select! arm. @@ -4667,19 +4306,11 @@ mod tests { shared_state: Arc, notify_buffer: NotifyBuffer, pending_submissions: crate::worker::PendingSubmissionHandle, - spawned_registry: Arc, - parent_socket_path: PathBuf, - runtime_dir: Arc, _temp: TempDir, } async fn make_env() -> DriveTurnEnv { let temp = tempfile::tempdir().expect("tempdir"); - let runtime_dir = Arc::new( - RuntimeDir::create(temp.path(), "child-worker") - .await - .expect("runtime dir create"), - ); let (method_tx, method_rx) = mpsc::channel::(16); let (working_event_tx, _) = broadcast::channel::(16); let (cancel_tx, cancel_rx) = mpsc::channel::<()>(1); @@ -4704,8 +4335,6 @@ mod tests { let notify_buffer = NotifyBuffer::new(); let pending_submissions = crate::worker::PendingSubmissionHandle::for_test(&temp.path().join("pending-sessions")); - let spawned_registry = SpawnedWorkerRegistry::new(runtime_dir.clone()); - let parent_socket_path = temp.path().join("parent.sock"); DriveTurnEnv { _method_tx: method_tx, @@ -4718,54 +4347,10 @@ mod tests { shared_state, notify_buffer, pending_submissions, - spawned_registry, - parent_socket_path, - runtime_dir, _temp: temp, } } - /// Listen on a bound UnixListener for one inbound connection and - /// return the first `Method::WorkerEvent` read from it. Returns `None` - /// on timeout / EOF / non-WorkerEvent. - async fn recv_worker_event(listener: UnixListener, timeout: Duration) -> Option { - let accept = async { - let (stream, _) = listener.accept().await.ok()?; - let (r, w) = stream.into_split(); - let mut writer = JsonLineWriter::new(w); - writer - .write(&Event::Snapshot { - session: protocol::SessionSnapshot { - pending_submissions: protocol::PendingSubmissionsSnapshot::default(), - entries: Vec::new(), - }, - greeting: protocol::Greeting { - worker_name: "parent".into(), - cwd: "/tmp".into(), - provider: "test".into(), - model: "test".into(), - scope_summary: String::new(), - tools: Vec::new(), - context_window: 200_000, - context_tokens: 0, - reasoning: None, - context_usage: None, - }, - state: WorkerStatus::Idle.into(), - in_flight: Default::default(), - internal_workers: Vec::new(), - }) - .await - .ok()?; - let mut reader = JsonLineReader::new(r); - match reader.next::().await { - Ok(Some(Method::WorkerEvent(e))) => Some(e), - _ => None, - } - }; - tokio::time::timeout(timeout, accept).await.ok().flatten() - } - #[tokio::test] async fn idle_only_input_is_rejected_in_busy_maintenance_admission() { let env = make_env().await; @@ -4796,43 +4381,6 @@ mod tests { ); } - #[tokio::test] - async fn parent_originated_finished_fires_turn_ended() { - let mut env = make_env().await; - let listener = UnixListener::bind(&env.parent_socket_path).expect("bind listener"); - let recv = tokio::spawn(recv_worker_event(listener, Duration::from_secs(2))); - - let worker_future = async { Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, shutdown, _) = drive_turn( - worker_future, - &mut env.method_rx, - &env.working_event_tx, - &env.cancel_tx, - &env.pause_tx, - &env.shared_state, - &env.runtime_dir, - None, - &env.notify_buffer, - &env.pending_submissions, - Some(&env.parent_socket_path), - "child-worker", - &env.spawned_registry, - true, - ) - .await; - assert_eq!(status, WorkerStatus::Idle); - assert!(!shutdown); - - let event = recv - .await - .expect("recv task") - .expect("WorkerEvent received"); - match event { - WorkerEvent::TurnEnded { worker_name } => assert_eq!(worker_name, "child-worker"), - other => panic!("expected TurnEnded, got {other:?}"), - } - } - #[test] fn busy_submit_is_durable_and_idempotent_before_shutdown() { let dir = tempfile::tempdir().unwrap(); @@ -4957,14 +4505,9 @@ mod tests { &env.cancel_tx, &env.pause_tx, &env.shared_state, - &env.runtime_dir, None, &env.notify_buffer, &env.pending_submissions, - None, - "worker", - &env.spawned_registry, - false, ) .await; @@ -5006,14 +4549,9 @@ mod tests { &env.cancel_tx, &env.pause_tx, &env.shared_state, - &env.runtime_dir, None, &env.notify_buffer, &env.pending_submissions, - None, - "worker", - &env.spawned_registry, - false, ) .await; @@ -5051,14 +4589,9 @@ mod tests { &env.cancel_tx, &env.pause_tx, &env.shared_state, - &env.runtime_dir, None, &env.notify_buffer, &env.pending_submissions, - None, - "child-worker", - &env.spawned_registry, - true, ) .await; @@ -5069,90 +4602,9 @@ mod tests { assert!(env._cancel_rx.try_recv().is_err()); } - #[tokio::test] - async fn non_parent_originated_finished_stays_silent() { - let mut env = make_env().await; - let listener = UnixListener::bind(&env.parent_socket_path).expect("bind listener"); - - let worker_future = async { Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, _, _) = drive_turn( - worker_future, - &mut env.method_rx, - &env.working_event_tx, - &env.cancel_tx, - &env.pause_tx, - &env.shared_state, - &env.runtime_dir, - None, - &env.notify_buffer, - &env.pending_submissions, - Some(&env.parent_socket_path), - "child-worker", - &env.spawned_registry, - false, - ) - .await; - assert_eq!(status, WorkerStatus::Idle); - - // Wait long enough for any (incorrect) fire-and-forget send to - // land; expect the accept to time out. - let accept = tokio::time::timeout(Duration::from_millis(200), listener.accept()).await; - assert!( - accept.is_err(), - "expected no WorkerEvent for non-parent-originated turn" - ); - } - - #[tokio::test] - async fn parent_originated_worker_error_fires_errored() { - let mut env = make_env().await; - let listener = UnixListener::bind(&env.parent_socket_path).expect("bind listener"); - let recv = tokio::spawn(recv_worker_event(listener, Duration::from_secs(2))); - - let worker_future = async { - Err::(WorkerError::Engine(EngineError::Aborted( - "boom from test".into(), - ))) - }; - let (status, _, _) = drive_turn( - worker_future, - &mut env.method_rx, - &env.working_event_tx, - &env.cancel_tx, - &env.pause_tx, - &env.shared_state, - &env.runtime_dir, - None, - &env.notify_buffer, - &env.pending_submissions, - Some(&env.parent_socket_path), - "child-worker", - &env.spawned_registry, - true, - ) - .await; - assert_eq!(status, WorkerStatus::Idle); - - let event = recv - .await - .expect("recv task") - .expect("WorkerEvent received"); - match event { - WorkerEvent::Errored { - worker_name, - message, - } => { - assert_eq!(worker_name, "child-worker"); - assert!(message.contains("boom from test"), "got message: {message}"); - } - other => panic!("expected Errored, got {other:?}"), - } - } - #[tokio::test] async fn delivered_run_failure_is_not_reported_again_by_controller() { let mut env = make_env().await; - let listener = UnixListener::bind(&env.parent_socket_path).expect("bind listener"); let mut events = env.working_event_tx.subscribe(); let worker_future = async { Err::(WorkerError::RunFailureAlreadyDelivered { @@ -5167,26 +4619,15 @@ mod tests { &env.cancel_tx, &env.pause_tx, &env.shared_state, - &env.runtime_dir, None, &env.notify_buffer, &env.pending_submissions, - Some(&env.parent_socket_path), - "child-worker", - &env.spawned_registry, - true, ) .await; assert_eq!(status, WorkerStatus::Idle); assert!(!may_drain_pending, "failed runs must not become successful"); assert!(events.try_recv().is_err(), "must not emit a generic Error"); - assert!( - tokio::time::timeout(Duration::from_millis(200), listener.accept()) - .await - .is_err(), - "must not send a second parent Errored event" - ); } #[tokio::test] @@ -5209,14 +4650,9 @@ mod tests { &env.cancel_tx, &env.pause_tx, &env.shared_state, - &env.runtime_dir, None, &env.notify_buffer, &env.pending_submissions, - None, - "child-worker", - &env.spawned_registry, - false, ) .await; @@ -5230,42 +4666,6 @@ mod tests { assert!(events.try_recv().is_err(), "must report once"); } - #[tokio::test] - async fn non_parent_originated_worker_error_stays_silent() { - let mut env = make_env().await; - let listener = UnixListener::bind(&env.parent_socket_path).expect("bind listener"); - - let worker_future = async { - Err::(WorkerError::Engine(EngineError::Aborted( - "boom from notify".into(), - ))) - }; - let (status, _, _) = drive_turn( - worker_future, - &mut env.method_rx, - &env.working_event_tx, - &env.cancel_tx, - &env.pause_tx, - &env.shared_state, - &env.runtime_dir, - None, - &env.notify_buffer, - &env.pending_submissions, - Some(&env.parent_socket_path), - "child-worker", - &env.spawned_registry, - false, - ) - .await; - assert_eq!(status, WorkerStatus::Idle); - - let accept = tokio::time::timeout(Duration::from_millis(200), listener.accept()).await; - assert!( - accept.is_err(), - "expected no WorkerEvent for notification-originated worker error" - ); - } - #[tokio::test] async fn running_legacy_scope_callback_has_no_registry_authority_or_notify() { let mut env = make_env().await; @@ -5290,14 +4690,9 @@ mod tests { &env.cancel_tx, &env.pause_tx, &env.shared_state, - &env.runtime_dir, None, &env.notify_buffer, &env.pending_submissions, - Some(&env.parent_socket_path), - "parent", - &env.spawned_registry, - false, ) .await; @@ -5330,14 +4725,9 @@ mod tests { &env.cancel_tx, &env.pause_tx, &env.shared_state, - &env.runtime_dir, None, &env.notify_buffer, &env.pending_submissions, - Some(&env.parent_socket_path), - "parent", - &env.spawned_registry, - false, ) .await; @@ -5368,14 +4758,9 @@ mod tests { &env.cancel_tx, &env.pause_tx, &env.shared_state, - &env.runtime_dir, None, &env.notify_buffer, &env.pending_submissions, - Some(&env.parent_socket_path), - "parent", - &env.spawned_registry, - false, ) .await; @@ -5410,14 +4795,9 @@ mod tests { &env.cancel_tx, &env.pause_tx, &env.shared_state, - &env.runtime_dir, None, &env.notify_buffer, &env.pending_submissions, - Some(&env.parent_socket_path), - "child-worker", - &env.spawned_registry, - false, ) .await; assert_eq!(status, WorkerStatus::Idle); diff --git a/crates/worker/src/controller_subject_release_tests.rs b/crates/worker/src/controller_subject_release_tests.rs index d57dc4530..5e843bb2e 100644 --- a/crates/worker/src/controller_subject_release_tests.rs +++ b/crates/worker/src/controller_subject_release_tests.rs @@ -65,7 +65,6 @@ async fn prepared( WorkerBootstrapLayout::Direct { runtime_base: temp.path().join("runtime"), }, - WorkerControllerTransport::InProcess, ) .with_model_client(client) .prepare() diff --git a/crates/worker/src/discovery.rs b/crates/worker/src/discovery.rs deleted file mode 100644 index dd7ecb8e8..000000000 --- a/crates/worker/src/discovery.rs +++ /dev/null @@ -1,1861 +0,0 @@ -//! Worker-state-backed discovery and restore tools. -//! -//! This surface deliberately does not enumerate every Worker on the host. The -//! listing path starts from the caller's visibility set (the caller itself and -//! Workers it spawned according to durable Worker state) and only then reads each -//! Worker's own state. Name-targeted operations distinguish missing state from -//! state that exists but is outside that visibility set. - -use std::collections::BTreeMap; -use std::fmt; -use std::io; -use std::path::{Path, PathBuf}; -use std::process::Stdio; -use std::sync::Arc; -use std::time::Duration; - -use crate::runtime_command::WorkerRuntimeCommand; -use agen::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput}; -use async_trait::async_trait; -use manifest::{Permission, ScopeRule}; -use protocol::stream::JsonLineReader; -use protocol::{Event, Method, WorkerStatus}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use session_store::{SegmentId, SessionId}; -use session_store::{ - WorkerActiveSegmentRef, WorkerMetadata, WorkerMetadataStore, validate_worker_name, -}; -use tokio::net::UnixStream; -use tokio::process::Command; - -use crate::runtime::worker_allocation; -use crate::spawn::comm_tools::connect_and_send; -use crate::spawn::registry::SpawnedWorkerRegistry; - -const PROBE_TIMEOUT: Duration = Duration::from_millis(500); -const RESTORE_START_TIMEOUT: Duration = Duration::from_secs(20); - -#[derive(Clone)] -pub struct WorkerDiscovery { - store: St, - self_worker_name: String, - runtime_base: PathBuf, - cwd: Option, - store_dir: Option, -} - -impl WorkerDiscovery -where - St: WorkerMetadataStore + Clone + Send + Sync + 'static, -{ - pub fn new( - store: St, - self_worker_name: String, - runtime_base: PathBuf, - cwd: Option, - _spawned_registry: Arc, - ) -> Self { - let store_dir = store.root_dir(); - Self { - store, - self_worker_name, - runtime_base, - cwd, - store_dir, - } - } - - pub async fn list_visible(&self) -> Result, WorkerDiscoveryError> { - let visibility = self.visibility().await?; - let mut items = Vec::with_capacity(visibility.visible.len()); - for worker_name in visibility.visible.keys() { - items.push( - self.build_item_for_visible_name(worker_name, &visibility) - .await, - ); - } - Ok(items) - } - - pub async fn inspect(&self, worker_name: &str) -> Result { - let visibility = self.visibility().await?; - let known_names = self.store.list_names()?; - let state_exists = known_names.iter().any(|n| n == worker_name); - if !state_exists { - return Err(WorkerDiscoveryError::StateMissing { - worker_name: worker_name.to_string(), - }); - } - if !visibility.visible.contains_key(worker_name) { - return Err(WorkerDiscoveryError::NotVisible { - worker_name: worker_name.to_string(), - }); - } - - match self.store.read_by_name(worker_name)? { - Some(metadata) => Ok(self.detail_from_metadata(metadata, &visibility).await), - None => Err(WorkerDiscoveryError::StateMissing { - worker_name: worker_name.to_string(), - }), - } - } - - pub async fn restore(&self, worker_name: &str) -> Result { - match self.plan_restore(worker_name).await? { - RestorePlan::AlreadyLive { - worker_name, - socket_path, - status, - } => Ok(RestoreResult::AlreadyLive { - worker_name, - socket_path, - status, - }), - RestorePlan::Restore { - worker_name, - socket_path, - } => { - self.spawn_restore_process(&worker_name, &socket_path) - .await?; - Ok(RestoreResult::Restored { - worker_name, - socket_path, - }) - } - } - } - - pub async fn plan_restore( - &self, - worker_name: &str, - ) -> Result { - let detail = self.inspect(worker_name).await?; - if detail.live.reachable { - return Ok(RestorePlan::AlreadyLive { - worker_name: worker_name.to_string(), - socket_path: detail.live.socket_path, - status: detail.live.status, - }); - } - - let active = detail - .active - .ok_or_else(|| WorkerDiscoveryError::NotRestorable { - worker_name: worker_name.to_string(), - reason: "worker state has no active session".into(), - })?; - let segment_id = active - .segment_id - .ok_or_else(|| WorkerDiscoveryError::NotRestorable { - worker_name: worker_name.to_string(), - reason: "worker state has an active session but no active segment yet".into(), - })?; - - if let Some(lock) = lookup_segment_lock(segment_id)? { - let lock_live = probe_socket(&lock.socket).await; - return if lock_live.reachable { - Ok(RestorePlan::AlreadyLive { - worker_name: lock.worker_name, - socket_path: lock.socket, - status: lock_live.status, - }) - } else { - Err(WorkerDiscoveryError::LockConflict { - worker_name: worker_name.to_string(), - segment_id, - owner_worker: lock.worker_name, - socket_path: lock.socket, - pid: lock.pid, - }) - }; - } - - Ok(RestorePlan::Restore { - worker_name: worker_name.to_string(), - socket_path: self.default_socket_path(worker_name), - }) - } - - pub fn register_peer( - &self, - peer_name: &str, - ) -> Result { - self.ensure_existing_peer(peer_name)? - .ok_or_else(|| WorkerDiscoveryError::MissingWorker { - worker_name: peer_name.to_string(), - }) - } - - pub fn ensure_existing_peer( - &self, - peer_name: &str, - ) -> Result, WorkerDiscoveryError> { - validate_worker_name(peer_name)?; - if peer_name == self.self_worker_name { - return Err(WorkerDiscoveryError::SelfPeer { - worker_name: peer_name.to_string(), - }); - } - let self_metadata = self - .store - .read_by_name(&self.self_worker_name)? - .ok_or_else(|| WorkerDiscoveryError::StateMissing { - worker_name: self.self_worker_name.clone(), - })?; - let prior_self_peers = self_metadata.peers.clone(); - if self.store.read_by_name(peer_name)?.is_none() { - return Ok(None); - } - - self.store.add_peer(&self.self_worker_name, peer_name)?; - if let Err(error) = self.store.add_peer(peer_name, &self.self_worker_name) { - let _ = self - .store - .set_peers(&self.self_worker_name, prior_self_peers); - return Err(WorkerDiscoveryError::WorkerStore(error)); - } - - Ok(Some(PeerRegistrationResult { - source: self.self_worker_name.clone(), - peer: peer_name.to_string(), - })) - } - - async fn visibility(&self) -> Result { - let mut visible = BTreeMap::new(); - let mut child_sockets = BTreeMap::new(); - let mut comm_registry = BTreeMap::new(); - visible.insert(self.self_worker_name.clone(), VisibilityReason::SelfWorker); - - // Durable parent -> child state is the primary visibility source. - if let Some(metadata) = self.store.read_by_name(&self.self_worker_name)? { - for child in metadata.spawned_children { - visible - .entry(child.worker_name.clone()) - .or_insert(VisibilityReason::SpawnedChild); - child_sockets.insert(child.worker_name.clone(), child.socket_path.clone()); - comm_registry.insert( - child.worker_name.clone(), - comm_info_from_spawned_child(&child), - ); - } - for peer in metadata.peers { - visible - .entry(peer.worker_name) - .or_insert(VisibilityReason::Peer); - } - } - - Ok(VisibilitySet { - visible, - child_sockets, - comm_registry, - }) - } - - async fn build_item_for_visible_name( - &self, - worker_name: &str, - visibility: &VisibilitySet, - ) -> VisibleWorkerItem { - let visibility_reason = visibility.reason_for(worker_name); - match self.store.read_by_name(worker_name) { - Ok(Some(metadata)) => { - let detail = self.detail_from_metadata(metadata, visibility).await; - VisibleWorkerItem { - worker_name: worker_name.to_string(), - visibility: visibility_reason, - state: WorkerStateStatus::Readable, - active: detail.active, - live: detail.live, - restore: detail.restore, - comm_registry: detail.comm_registry, - spawned_children: detail.spawned_children, - error: None, - } - } - Ok(None) => VisibleWorkerItem { - worker_name: worker_name.to_string(), - visibility: visibility_reason, - state: WorkerStateStatus::Missing, - active: None, - live: self.live_for_name(worker_name, None).await, - restore: RestoreInfo::not_possible("worker state missing"), - comm_registry: visibility.comm_info_for(worker_name), - spawned_children: SpawnedChildrenSummary::default(), - error: None, - }, - Err(error) => VisibleWorkerItem { - worker_name: worker_name.to_string(), - visibility: visibility_reason, - state: WorkerStateStatus::Corrupt, - active: None, - live: self.live_for_name(worker_name, None).await, - restore: RestoreInfo::not_possible("worker state is unreadable"), - comm_registry: visibility.comm_info_for(worker_name), - spawned_children: SpawnedChildrenSummary::default(), - error: Some(error.to_string()), - }, - } - } - - async fn detail_from_metadata( - &self, - metadata: WorkerMetadata, - visibility: &VisibilitySet, - ) -> WorkerDetail { - let child_socket = visibility.child_socket_for(&metadata.worker_name); - let live = self - .live_for_name(&metadata.worker_name, child_socket.as_deref()) - .await; - let restore = self - .restore_info(&metadata.worker_name, metadata.active.as_ref()) - .await; - let spawned_children = summarize_spawned_children(&metadata.spawned_children).await; - WorkerDetail { - worker_name: metadata.worker_name.clone(), - visibility: visibility.reason_for(&metadata.worker_name), - active: metadata.active.map(ActivePointer::from), - live, - restore, - comm_registry: visibility.comm_info_for(&metadata.worker_name), - spawned_children, - } - } - - async fn restore_info( - &self, - worker_name: &str, - active: Option<&WorkerActiveSegmentRef>, - ) -> RestoreInfo { - let Some(active) = active else { - return RestoreInfo::not_possible("worker state has no active session"); - }; - let Some(segment_id) = active.segment_id else { - return RestoreInfo::not_possible("active segment is not known yet"); - }; - match lookup_segment_lock(segment_id) { - Ok(Some(lock)) => RestoreInfo { - possible: false, - restore_name: Some(worker_name.to_string()), - reason: Some(format!( - "segment is currently locked by `{}` (pid {})", - lock.worker_name, lock.pid - )), - }, - Ok(None) => RestoreInfo { - possible: true, - restore_name: Some(worker_name.to_string()), - reason: None, - }, - Err(error) => RestoreInfo { - possible: false, - restore_name: Some(worker_name.to_string()), - reason: Some(format!("lock lookup failed: {error}")), - }, - } - } - - pub async fn send_weak_notify_to_live_peer( - &self, - peer_name: &str, - message: String, - ) -> WeakNotifyDelivery { - let detail = match self.inspect(peer_name).await { - Ok(detail) => detail, - Err( - WorkerDiscoveryError::StateMissing { .. } - | WorkerDiscoveryError::MissingWorker { .. }, - ) => { - return WeakNotifyDelivery::SkippedMissing; - } - Err(WorkerDiscoveryError::NotVisible { .. }) => { - return WeakNotifyDelivery::SkippedNotVisible; - } - Err(error) => { - return WeakNotifyDelivery::SendFailed { - error: error.to_string(), - }; - } - }; - if detail.visibility != VisibilityReason::Peer { - return WeakNotifyDelivery::SkippedNotPeer { - visibility: detail.visibility, - }; - } - if !detail.live.reachable { - return WeakNotifyDelivery::SkippedNotLive { - reason: detail.live.error, - }; - } - match send_notify(&detail.live.socket_path, message).await { - Ok(()) => WeakNotifyDelivery::Delivered, - Err(error) => WeakNotifyDelivery::SendFailed { - error: error.to_string(), - }, - } - } - - async fn live_for_name(&self, worker_name: &str, socket_override: Option<&Path>) -> LiveInfo { - let socket_path = socket_override - .map(Path::to_path_buf) - .unwrap_or_else(|| self.default_socket_path(worker_name)); - probe_socket(&socket_path).await - } - - fn default_socket_path(&self, worker_name: &str) -> PathBuf { - self.runtime_base.join(worker_name).join("sock") - } - - async fn spawn_restore_process( - &self, - worker_name: &str, - socket_path: &Path, - ) -> Result<(), WorkerDiscoveryError> { - let runtime_command = - WorkerRuntimeCommand::resolve().map_err(WorkerDiscoveryError::RestoreSpawn)?; - let Some(cwd) = &self.cwd else { - return Err(WorkerDiscoveryError::NotRestorable { - worker_name: worker_name.to_string(), - reason: "restore requires local Worker filesystem authority".into(), - }); - }; - let mut command = Command::new(runtime_command.program()); - command - .args(runtime_command.prefix_args()) - .arg("--worker") - .arg(worker_name) - .arg("--require-worker-state") - .current_dir(cwd) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .process_group(0); - if let Some(store_dir) = &self.store_dir { - command.arg("--store").arg(store_dir); - } - - let mut child = - command - .spawn() - .map_err(|source| WorkerDiscoveryError::RestoreLaunchFailed { - command: runtime_command.clone(), - source, - })?; - let deadline = tokio::time::Instant::now() + RESTORE_START_TIMEOUT; - loop { - if probe_socket(socket_path).await.reachable { - tokio::spawn(async move { - let _ = child.wait().await; - }); - return Ok(()); - } - if let Some(status) = child - .try_wait() - .map_err(WorkerDiscoveryError::RestoreSpawn)? - { - return Err(WorkerDiscoveryError::RestoreExited { status }); - } - if tokio::time::Instant::now() >= deadline { - let _ = child.start_kill(); - let _ = child.wait().await; - return Err(WorkerDiscoveryError::RestoreTimeout); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum VisibilityReason { - SelfWorker, - SpawnedChild, - Peer, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum WorkerStateStatus { - Readable, - Missing, - Corrupt, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct ActivePointer { - pub session_id: SessionId, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub segment_id: Option, -} - -impl From for ActivePointer { - fn from(value: WorkerActiveSegmentRef) -> Self { - Self { - session_id: value.session_id, - segment_id: value.segment_id, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct LiveInfo { - pub socket_path: PathBuf, - pub reachable: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub status: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct RestoreInfo { - pub possible: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub restore_name: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub reason: Option, -} - -impl RestoreInfo { - fn not_possible(reason: impl Into) -> Self { - Self { - possible: false, - restore_name: None, - reason: Some(reason.into()), - } - } -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] -pub struct SpawnedChildrenSummary { - pub count: usize, - pub reachable: usize, - pub unreachable: usize, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct CommRegistryInfo { - pub registered: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub socket_path: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub scope_delegated: Vec, -} - -impl CommRegistryInfo { - fn missing() -> Self { - Self { - registered: false, - socket_path: None, - scope_delegated: Vec::new(), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct VisibleWorkerItem { - pub worker_name: String, - pub visibility: VisibilityReason, - pub state: WorkerStateStatus, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub active: Option, - pub live: LiveInfo, - pub restore: RestoreInfo, - pub comm_registry: CommRegistryInfo, - pub spawned_children: SpawnedChildrenSummary, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct WorkerDetail { - pub worker_name: String, - pub visibility: VisibilityReason, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub active: Option, - pub live: LiveInfo, - pub restore: RestoreInfo, - pub comm_registry: CommRegistryInfo, - pub spawned_children: SpawnedChildrenSummary, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "action", rename_all = "snake_case")] -pub enum RestorePlan { - AlreadyLive { - worker_name: String, - socket_path: PathBuf, - #[serde(default, skip_serializing_if = "Option::is_none")] - status: Option, - }, - Restore { - worker_name: String, - socket_path: PathBuf, - }, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(tag = "action", rename_all = "snake_case")] -pub enum RestoreResult { - AlreadyLive { - worker_name: String, - socket_path: PathBuf, - #[serde(default, skip_serializing_if = "Option::is_none")] - status: Option, - }, - Restored { - worker_name: String, - socket_path: PathBuf, - }, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct PeerRegistrationResult { - pub source: String, - pub peer: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum WeakNotifyDelivery { - Delivered, - SkippedMissing, - SkippedNotVisible, - SkippedNotPeer { visibility: VisibilityReason }, - SkippedNotLive { reason: Option }, - SendFailed { error: String }, -} - -impl WeakNotifyDelivery { - pub fn delivered(&self) -> bool { - matches!(self, WeakNotifyDelivery::Delivered) - } -} - -impl fmt::Display for WeakNotifyDelivery { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - WeakNotifyDelivery::Delivered => write!(f, "delivered"), - WeakNotifyDelivery::SkippedMissing => { - write!(f, "skipped: target worker metadata is missing") - } - WeakNotifyDelivery::SkippedNotVisible => { - write!(f, "skipped: target worker is not visible") - } - WeakNotifyDelivery::SkippedNotPeer { visibility } => { - write!( - f, - "skipped: target worker is visible as {visibility:?}, not peer" - ) - } - WeakNotifyDelivery::SkippedNotLive { reason } => { - if let Some(reason) = reason { - write!(f, "skipped: target peer is not live/reachable ({reason})") - } else { - write!(f, "skipped: target peer is not live/reachable") - } - } - WeakNotifyDelivery::SendFailed { error } => write!(f, "send failed: {error}"), - } - } -} - -#[derive(Debug, thiserror::Error)] -pub enum WorkerDiscoveryError { - #[error("worker state missing for `{worker_name}`")] - StateMissing { worker_name: String }, - #[error("worker `{worker_name}` is not visible to this Worker")] - NotVisible { worker_name: String }, - #[error("worker `{worker_name}` is not restorable: {reason}")] - NotRestorable { worker_name: String, reason: String }, - #[error("worker `{worker_name}` cannot be registered as a peer of itself")] - SelfPeer { worker_name: String }, - #[error("worker `{worker_name}` does not exist")] - MissingWorker { worker_name: String }, - #[error( - "worker `{worker_name}` segment {segment_id} is locked by `{owner_worker}` pid {pid} at {socket_path}" - )] - LockConflict { - worker_name: String, - segment_id: SegmentId, - owner_worker: String, - socket_path: PathBuf, - pid: u32, - }, - #[error("session store error: {0}")] - Store(#[from] session_store::StoreError), - #[error("worker store error: {0}")] - WorkerStore(#[from] session_store::WorkerStoreError), - #[error("scope lock error: {0}")] - ScopeLock(#[from] worker_allocation::ScopeLockError), - #[error("failed to launch restore process: {0}")] - RestoreSpawn(io::Error), - #[error("failed to launch restore runtime command `{command}`: {source}")] - RestoreLaunchFailed { - command: WorkerRuntimeCommand, - #[source] - source: io::Error, - }, - #[error("restore process exited before socket became reachable: {status}")] - RestoreExited { status: std::process::ExitStatus }, - #[error("restore process did not become reachable before timeout")] - RestoreTimeout, -} - -struct VisibilitySet { - visible: BTreeMap, - child_sockets: BTreeMap, - comm_registry: BTreeMap, -} - -impl VisibilitySet { - fn reason_for(&self, worker_name: &str) -> VisibilityReason { - self.visible - .get(worker_name) - .cloned() - .unwrap_or(VisibilityReason::SpawnedChild) - } - - fn child_socket_for(&self, worker_name: &str) -> Option { - self.child_sockets.get(worker_name).cloned() - } - - fn comm_info_for(&self, worker_name: &str) -> CommRegistryInfo { - self.comm_registry - .get(worker_name) - .cloned() - .unwrap_or_else(CommRegistryInfo::missing) - } -} - -fn comm_info_from_spawned_child(child: &session_store::WorkerSpawnedChild) -> CommRegistryInfo { - let scope_delegated = child - .scope_delegated - .iter() - .filter_map(|rule| { - let permission = match rule.permission.as_str() { - "read" => Permission::Read, - "write" => Permission::Write, - _ => return None, - }; - Some(ScopeRule { - target: rule.target.clone(), - permission, - recursive: rule.recursive, - symlink_policy: rule.symlink_policy, - }) - }) - .collect(); - CommRegistryInfo { - registered: true, - socket_path: Some(child.socket_path.clone()), - scope_delegated, - } -} - -async fn summarize_spawned_children( - children: &[session_store::WorkerSpawnedChild], -) -> SpawnedChildrenSummary { - let mut summary = SpawnedChildrenSummary { - count: children.len(), - ..Default::default() - }; - for child in children { - if probe_socket(&child.socket_path).await.reachable { - summary.reachable += 1; - } else { - summary.unreachable += 1; - } - } - summary -} - -async fn probe_socket(socket_path: &Path) -> LiveInfo { - match tokio::time::timeout(PROBE_TIMEOUT, UnixStream::connect(socket_path)).await { - Ok(Ok(stream)) => { - let (r, _w) = stream.into_split(); - let mut reader = JsonLineReader::new(r); - let mut status = None; - loop { - match tokio::time::timeout(PROBE_TIMEOUT, reader.next::()).await { - Ok(Ok(Some(Event::Snapshot { - state: snapshot_state, - .. - }))) => { - status = Some(snapshot_state.catalog_status()); - break; - } - Ok(Ok(Some(Event::Alert(_)))) => continue, - Ok(Ok(Some(_))) | Ok(Ok(None)) | Ok(Err(_)) | Err(_) => break, - } - } - LiveInfo { - socket_path: socket_path.to_path_buf(), - reachable: true, - status, - error: None, - } - } - Ok(Err(error)) => LiveInfo { - socket_path: socket_path.to_path_buf(), - reachable: false, - status: None, - error: Some(error.to_string()), - }, - Err(_) => LiveInfo { - socket_path: socket_path.to_path_buf(), - reachable: false, - status: None, - error: Some("connect timed out".into()), - }, - } -} - -fn lookup_segment_lock( - segment_id: SegmentId, -) -> Result, worker_allocation::ScopeLockError> { - worker_allocation::lookup_segment(segment_id) -} - -#[derive(Debug, Deserialize, JsonSchema)] -struct WorkerNameInput { - /// Worker name to restore. - name: String, -} - -#[derive(Debug, Deserialize, JsonSchema)] -struct SendToPeerWorkerInput { - /// Target peer Worker name. - name: String, - /// Text delivered to the peer as a peer notification. - message: String, -} - -struct ListWorkersTool { - discovery: WorkerDiscovery, -} - -#[async_trait] -impl Tool for ListWorkersTool -where - St: WorkerMetadataStore + Clone + Send + Sync + 'static, -{ - async fn execute( - &self, - _input_json: &str, - _ctx: agen::tool::ToolExecutionContext, - ) -> Result { - let items = self - .discovery - .list_visible() - .await - .map_err(discovery_error_to_tool_error)?; - let summary = format!("{} visible worker(s)", items.len()); - Ok(ToolOutput { - summary, - content: Some(json_content(&items)?), - attachments: Vec::new(), - }) - } -} - -struct RestoreWorkerTool { - discovery: WorkerDiscovery, -} - -#[async_trait] -impl Tool for RestoreWorkerTool -where - St: WorkerMetadataStore + Clone + Send + Sync + 'static, -{ - async fn execute( - &self, - input_json: &str, - _ctx: agen::tool::ToolExecutionContext, - ) -> Result { - let input: WorkerNameInput = serde_json::from_str(input_json) - .map_err(|e| ToolError::InvalidArgument(format!("invalid RestoreWorker input: {e}")))?; - let result = self - .discovery - .restore(&input.name) - .await - .map_err(discovery_error_to_tool_error)?; - let summary = match &result { - RestoreResult::AlreadyLive { worker_name, .. } => { - format!("worker `{worker_name}` is already live") - } - RestoreResult::Restored { worker_name, .. } => { - format!("worker `{worker_name}` restored from worker state") - } - }; - Ok(ToolOutput { - summary, - content: Some(json_content(&result)?), - - attachments: Vec::new(), - }) - } -} - -pub fn list_workers_tool(discovery: WorkerDiscovery) -> ToolDefinition -where - St: WorkerMetadataStore + Clone + Send + Sync + 'static, -{ - Arc::new(move || { - let meta = ToolMeta::new("ListWorkers") - .description( - "List Workers visible to this Worker from durable Worker state, peer metadata, and the spawned-child registry. This does not expose the host-wide Worker universe.", - ) - .input_schema(serde_json::json!({ - "type": "object", - "properties": {}, - "additionalProperties": false, - })); - let tool: Arc = Arc::new(ListWorkersTool { - discovery: discovery.clone(), - }); - (meta, tool) - }) -} - -pub fn restore_worker_tool(discovery: WorkerDiscovery) -> ToolDefinition -where - St: WorkerMetadataStore + Clone + Send + Sync + 'static, -{ - Arc::new(move || { - let meta = ToolMeta::new("RestoreWorker") - .description( - "Restore a visible stopped/restorable Worker, or report that a visible Worker is already live. Missing state is an error.", - ) - .input_schema(serde_json::to_value(schemars::schema_for!(WorkerNameInput)).unwrap()); - let tool: Arc = Arc::new(RestoreWorkerTool { - discovery: discovery.clone(), - }); - (meta, tool) - }) -} - -const SEND_TO_PEER_POD_DESCRIPTION: &str = "Send a text message to a peer Worker made visible by explicit reciprocal peer metadata. The message is delivered as a peer notification through the target Worker's durable notification/history path. This does not grant delegated scope, create a spawned-child output cursor, imply parent ownership, or produce child completion notifications. Fails clearly if the target is not a visible live peer; it does not auto-restore stopped peers."; - -struct SendToPeerWorkerTool { - discovery: WorkerDiscovery, -} - -#[async_trait] -impl Tool for SendToPeerWorkerTool -where - St: WorkerMetadataStore + Clone + Send + Sync + 'static, -{ - async fn execute( - &self, - input_json: &str, - _ctx: agen::tool::ToolExecutionContext, - ) -> Result { - let input: SendToPeerWorkerInput = serde_json::from_str(input_json).map_err(|e| { - ToolError::InvalidArgument(format!("invalid SendToPeerWorker input: {e}")) - })?; - let detail = self - .discovery - .inspect(&input.name) - .await - .map_err(discovery_error_to_tool_error)?; - if detail.visibility != VisibilityReason::Peer { - return Err(ToolError::InvalidArgument(format!( - "worker `{}` is visible as {:?}, not as a peer", - input.name, detail.visibility - ))); - } - if !detail.live.reachable { - return Err(ToolError::ExecutionFailed(format!( - "peer worker `{}` is not live/reachable; restore it before sending", - input.name - ))); - } - - let message = format!( - "[Peer message from `{}`]\n{}", - self.discovery.self_worker_name, input.message - ); - send_peer_notify(&detail.live.socket_path, message) - .await - .map_err(|error| { - ToolError::ExecutionFailed(format!("send to peer `{}`: {error}", input.name)) - })?; - - Ok(ToolOutput { - summary: format!("sent peer message to `{}`", input.name), - content: None, - attachments: Vec::new(), - }) - } -} - -pub fn send_to_peer_worker_tool(discovery: WorkerDiscovery) -> ToolDefinition -where - St: WorkerMetadataStore + Clone + Send + Sync + 'static, -{ - Arc::new(move || { - let meta = ToolMeta::new("SendToPeerWorker") - .description(SEND_TO_PEER_POD_DESCRIPTION) - .input_schema( - serde_json::to_value(schemars::schema_for!(SendToPeerWorkerInput)).unwrap(), - ); - let tool: Arc = Arc::new(SendToPeerWorkerTool { - discovery: discovery.clone(), - }); - (meta, tool) - }) -} - -async fn send_peer_notify(socket_path: &Path, message: String) -> io::Result<()> { - send_notify(socket_path, message).await -} - -async fn send_notify(socket_path: &Path, message: String) -> io::Result<()> { - let notification_request_id = protocol::new_submission_request_id(); - connect_and_send( - socket_path, - &Method::NotifyTracked { - notification_request_id: notification_request_id.clone(), - message, - source: protocol::AuthenticatedInputSource::Backend { - operation_id: notification_request_id, - }, - }, - ) - .await -} - -fn json_content(value: &T) -> Result { - serde_json::to_string_pretty(value) - .map_err(|e| ToolError::Internal(format!("serialize worker discovery output: {e}"))) -} - -fn discovery_error_to_tool_error(error: WorkerDiscoveryError) -> ToolError { - match error { - WorkerDiscoveryError::StateMissing { .. } - | WorkerDiscoveryError::NotVisible { .. } - | WorkerDiscoveryError::NotRestorable { .. } - | WorkerDiscoveryError::SelfPeer { .. } - | WorkerDiscoveryError::MissingWorker { .. } => { - ToolError::InvalidArgument(error.to_string()) - } - WorkerDiscoveryError::LockConflict { .. } - | WorkerDiscoveryError::Store(_) - | WorkerDiscoveryError::WorkerStore(_) - | WorkerDiscoveryError::ScopeLock(_) - | WorkerDiscoveryError::RestoreSpawn(_) - | WorkerDiscoveryError::RestoreLaunchFailed { .. } - | WorkerDiscoveryError::RestoreExited { .. } - | WorkerDiscoveryError::RestoreTimeout => ToolError::ExecutionFailed(error.to_string()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use manifest::{Permission, ScopeRule}; - use protocol::stream::JsonLineWriter; - use protocol::{Alert, AlertLevel, AlertSource}; - use session_store::{ - FsWorkerStore, WorkerSpawnedChild, WorkerSpawnedScopeRule, WorkerStoreError, - }; - use session_store::{new_segment_id, new_session_id}; - use tempfile::TempDir; - use tokio::net::UnixListener; - - use crate::runtime::dir::RuntimeDir; - use crate::runtime::worker_allocation::test_util::RuntimeDirSandbox; - - #[derive(Clone)] - struct FailTargetPeerStore { - inner: FsWorkerStore, - } - - impl WorkerMetadataStore for FailTargetPeerStore { - fn write(&self, metadata: &WorkerMetadata) -> Result<(), WorkerStoreError> { - if metadata.worker_name == "target" - && metadata - .peers - .iter() - .any(|peer| peer.worker_name == "source") - { - return Err(WorkerStoreError::Io(io::Error::other( - "injected target-side peer write failure", - ))); - } - self.inner.write(metadata) - } - - fn read_by_name( - &self, - worker_name: &str, - ) -> Result, WorkerStoreError> { - self.inner.read_by_name(worker_name) - } - - fn list_names(&self) -> Result, WorkerStoreError> { - self.inner.list_names() - } - - fn root_dir(&self) -> Option { - self.inner.root_dir() - } - - fn delete_by_name(&self, worker_name: &str) -> Result<(), WorkerStoreError> { - self.inner.delete_by_name(worker_name) - } - } - - #[tokio::test(flavor = "current_thread")] - async fn state_backed_visibility_and_restore_planning() { - let root = TempDir::new().unwrap(); - let store_dir = root.path().join("store"); - let runtime_base = root.path().join("runtime"); - std::fs::create_dir_all(&runtime_base).unwrap(); - let _runtime_sandbox = RuntimeDirSandbox::new(&runtime_base); - - let store = FsWorkerStore::new(&store_dir).unwrap(); - let session_id = new_session_id(); - let active_child_segment = new_segment_id(); - let pending_session_id = new_session_id(); - let live_socket = runtime_base.join("child-live").join("sock"); - std::fs::create_dir_all(live_socket.parent().unwrap()).unwrap(); - let live_listener = spawn_snapshot_socket(&live_socket).await; - - let stale_socket = runtime_base.join("child-stale").join("sock"); - let pending_socket = runtime_base.join("child-pending").join("sock"); - let parent = WorkerMetadata { - worker_name: "parent".into(), - active: None, - workspace_root: None, - workspace_id: None, - spawned_children: vec![ - child("child-live", &live_socket), - child("child-stale", &stale_socket), - child("child-pending", &pending_socket), - ], - reclaimed_children: Vec::new(), - peers: vec![session_store::WorkerPeer { - worker_name: "peer".into(), - }], - resolved_manifest_snapshot: None, - subjektiv_session_attribution: None, - }; - store.write(&parent).unwrap(); - store - .write(&WorkerMetadata { - worker_name: "child-live".into(), - active: Some(WorkerActiveSegmentRef::active_segment( - session_id, - active_child_segment, - )), - workspace_root: None, - workspace_id: None, - spawned_children: Vec::new(), - reclaimed_children: Vec::new(), - peers: Vec::new(), - resolved_manifest_snapshot: None, - subjektiv_session_attribution: None, - }) - .unwrap(); - store - .write(&WorkerMetadata { - worker_name: "child-stale".into(), - active: Some(WorkerActiveSegmentRef::active_segment( - session_id, - active_child_segment, - )), - workspace_root: None, - workspace_id: None, - spawned_children: Vec::new(), - reclaimed_children: Vec::new(), - peers: Vec::new(), - resolved_manifest_snapshot: None, - subjektiv_session_attribution: None, - }) - .unwrap(); - store - .write(&WorkerMetadata { - worker_name: "child-pending".into(), - active: Some(WorkerActiveSegmentRef::pending_segment(pending_session_id)), - workspace_root: None, - workspace_id: None, - spawned_children: Vec::new(), - reclaimed_children: Vec::new(), - peers: Vec::new(), - resolved_manifest_snapshot: None, - subjektiv_session_attribution: None, - }) - .unwrap(); - store - .write(&WorkerMetadata { - worker_name: "hidden".into(), - active: Some(WorkerActiveSegmentRef::active_segment( - session_id, - new_segment_id(), - )), - workspace_root: None, - workspace_id: None, - spawned_children: Vec::new(), - reclaimed_children: Vec::new(), - peers: Vec::new(), - resolved_manifest_snapshot: None, - subjektiv_session_attribution: None, - }) - .unwrap(); - store - .write(&WorkerMetadata { - worker_name: "peer".into(), - active: None, - workspace_root: None, - workspace_id: None, - spawned_children: Vec::new(), - reclaimed_children: Vec::new(), - peers: vec![session_store::WorkerPeer { - worker_name: "parent".into(), - }], - resolved_manifest_snapshot: None, - subjektiv_session_attribution: None, - }) - .unwrap(); - - // RuntimeDir creates parent runtime files; discovery must still use - // Worker state when spawned_workers.json is absent. - let runtime_dir = Arc::new(RuntimeDir::create(&runtime_base, "parent").await.unwrap()); - let runtime_file = runtime_dir.path().join("spawned_workers.json"); - assert!(!runtime_file.exists()); - let registry = SpawnedWorkerRegistry::new(runtime_dir); - let discovery = WorkerDiscovery::new( - store.clone(), - "parent".into(), - runtime_base.clone(), - Some(root.path().to_path_buf()), - registry, - ); - - let list_tool_def = list_workers_tool(discovery.clone()); - let (list_meta, _) = list_tool_def(); - assert_eq!(list_meta.name, "ListWorkers"); - let restore_tool_def = restore_worker_tool(discovery.clone()); - let (restore_meta, _) = restore_tool_def(); - assert_eq!(restore_meta.name, "RestoreWorker"); - let send_peer_tool_def = send_to_peer_worker_tool(discovery.clone()); - let (send_peer_meta, _) = send_peer_tool_def(); - assert_eq!(send_peer_meta.name, "SendToPeerWorker"); - - let list = discovery.list_visible().await.unwrap(); - let names: Vec<_> = list.iter().map(|p| p.worker_name.as_str()).collect(); - assert_eq!( - names, - vec![ - "child-live", - "child-pending", - "child-stale", - "parent", - "peer" - ] - ); - assert!(!names.contains(&"hidden")); - assert_eq!( - list.iter() - .find(|p| p.worker_name == "peer") - .unwrap() - .visibility, - VisibilityReason::Peer - ); - assert_eq!( - list.iter() - .find(|p| p.worker_name == "child-live") - .unwrap() - .visibility, - VisibilityReason::SpawnedChild - ); - assert!( - list.iter() - .find(|p| p.worker_name == "child-live") - .unwrap() - .live - .reachable - ); - assert!( - !list - .iter() - .find(|p| p.worker_name == "child-stale") - .unwrap() - .live - .reachable - ); - - let pending = list - .iter() - .find(|p| p.worker_name == "child-pending") - .unwrap(); - assert_eq!(pending.state, WorkerStateStatus::Readable); - assert_eq!( - pending.active.as_ref().unwrap().session_id, - pending_session_id - ); - assert_eq!(pending.active.as_ref().unwrap().segment_id, None); - assert!(!pending.restore.possible); - - let hidden_err = discovery.inspect("hidden").await.unwrap_err(); - assert!(matches!( - hidden_err, - WorkerDiscoveryError::NotVisible { .. } - )); - let missing_err = discovery.inspect("missing").await.unwrap_err(); - assert!(matches!( - missing_err, - WorkerDiscoveryError::StateMissing { .. } - )); - let hidden_restore_err = discovery.plan_restore("hidden").await.unwrap_err(); - assert!(matches!( - hidden_restore_err, - WorkerDiscoveryError::NotVisible { .. } - )); - - let live_plan = discovery.plan_restore("child-live").await.unwrap(); - assert!(matches!(live_plan, RestorePlan::AlreadyLive { .. })); - let restore_plan = discovery.plan_restore("child-stale").await.unwrap(); - assert!(matches!(restore_plan, RestorePlan::Restore { .. })); - - let lock_socket = runtime_base.join("lock-owner.sock"); - let _guard = worker_allocation::install_top_level( - "lock-owner".into(), - std::process::id(), - lock_socket.clone(), - vec![ScopeRule { - target: root.path().to_path_buf(), - permission: Permission::Read, - recursive: true, - symlink_policy: Default::default(), - }], - active_child_segment, - ) - .unwrap(); - let locked_err = discovery.plan_restore("child-stale").await.unwrap_err(); - assert!(matches!( - locked_err, - WorkerDiscoveryError::LockConflict { .. } - )); - - live_listener.abort(); - } - - #[tokio::test(flavor = "current_thread")] - async fn register_peer_persists_reciprocal_metadata() { - let root = TempDir::new().unwrap(); - let store_dir = root.path().join("store"); - let runtime_base = root.path().join("runtime"); - std::fs::create_dir_all(&runtime_base).unwrap(); - let store = FsWorkerStore::new(&store_dir).unwrap(); - store.write(&WorkerMetadata::new("source", None)).unwrap(); - store.write(&WorkerMetadata::new("target", None)).unwrap(); - let runtime_dir = Arc::new(RuntimeDir::create(&runtime_base, "source").await.unwrap()); - - let discovery = WorkerDiscovery::new( - store.clone(), - "source".into(), - runtime_base.clone(), - Some(root.path().to_path_buf()), - SpawnedWorkerRegistry::new(runtime_dir), - ); - let result = discovery.register_peer("target").unwrap(); - assert_eq!(result.source, "source"); - assert_eq!(result.peer, "target"); - - let source = store.read_by_name("source").unwrap().unwrap(); - let target = store.read_by_name("target").unwrap().unwrap(); - assert_eq!(source.peers[0].worker_name, "target"); - assert_eq!(target.peers[0].worker_name, "source"); - - let list = discovery.list_visible().await.unwrap(); - assert_eq!( - list.iter() - .find(|item| item.worker_name == "target") - .unwrap() - .visibility, - VisibilityReason::Peer - ); - } - - #[tokio::test(flavor = "current_thread")] - async fn register_peer_rejects_self_and_missing_target() { - let root = TempDir::new().unwrap(); - let store_dir = root.path().join("store"); - let runtime_base = root.path().join("runtime"); - std::fs::create_dir_all(&runtime_base).unwrap(); - let store = FsWorkerStore::new(&store_dir).unwrap(); - store.write(&WorkerMetadata::new("source", None)).unwrap(); - let runtime_dir = Arc::new(RuntimeDir::create(&runtime_base, "source").await.unwrap()); - let discovery = WorkerDiscovery::new( - store, - "source".into(), - runtime_base, - Some(root.path().to_path_buf()), - SpawnedWorkerRegistry::new(runtime_dir), - ); - - let self_err = discovery.register_peer("source").unwrap_err(); - assert!(matches!(self_err, WorkerDiscoveryError::SelfPeer { .. })); - let missing_err = discovery.register_peer("missing").unwrap_err(); - assert!(matches!( - missing_err, - WorkerDiscoveryError::MissingWorker { .. } - )); - } - - #[tokio::test(flavor = "current_thread")] - async fn register_peer_target_failure_preserves_existing_source_peer() { - let root = TempDir::new().unwrap(); - let store_dir = root.path().join("store"); - let runtime_base = root.path().join("runtime"); - std::fs::create_dir_all(&runtime_base).unwrap(); - let inner = FsWorkerStore::new(&store_dir).unwrap(); - inner - .write(&WorkerMetadata { - worker_name: "source".into(), - active: None, - workspace_root: None, - workspace_id: None, - spawned_children: Vec::new(), - reclaimed_children: Vec::new(), - peers: vec![session_store::WorkerPeer { - worker_name: "target".into(), - }], - resolved_manifest_snapshot: None, - subjektiv_session_attribution: None, - }) - .unwrap(); - inner.write(&WorkerMetadata::new("target", None)).unwrap(); - let store = FailTargetPeerStore { inner }; - let runtime_dir = Arc::new(RuntimeDir::create(&runtime_base, "source").await.unwrap()); - let discovery = WorkerDiscovery::new( - store.clone(), - "source".into(), - runtime_base, - Some(root.path().to_path_buf()), - SpawnedWorkerRegistry::new(runtime_dir), - ); - - let err = discovery.register_peer("target").unwrap_err(); - assert!(matches!(err, WorkerDiscoveryError::WorkerStore(_))); - let source = store.read_by_name("source").unwrap().unwrap(); - assert_eq!(source.peers.len(), 1); - assert_eq!(source.peers[0].worker_name, "target"); - let target = store.read_by_name("target").unwrap().unwrap(); - assert!(target.peers.is_empty()); - } - - #[tokio::test(flavor = "current_thread")] - async fn send_to_peer_worker_delivers_notify_without_child_registry() { - let root = TempDir::new().unwrap(); - let store_dir = root.path().join("store"); - let runtime_base = root.path().join("runtime"); - std::fs::create_dir_all(runtime_base.join("target")).unwrap(); - let store = FsWorkerStore::new(&store_dir).unwrap(); - store - .write(&WorkerMetadata { - worker_name: "source".into(), - active: None, - workspace_root: None, - workspace_id: None, - spawned_children: Vec::new(), - reclaimed_children: Vec::new(), - peers: vec![session_store::WorkerPeer { - worker_name: "target".into(), - }], - resolved_manifest_snapshot: None, - subjektiv_session_attribution: None, - }) - .unwrap(); - store - .write(&WorkerMetadata { - worker_name: "target".into(), - active: None, - workspace_root: None, - workspace_id: None, - spawned_children: Vec::new(), - reclaimed_children: Vec::new(), - peers: vec![session_store::WorkerPeer { - worker_name: "source".into(), - }], - resolved_manifest_snapshot: None, - subjektiv_session_attribution: None, - }) - .unwrap(); - let runtime_dir = Arc::new(RuntimeDir::create(&runtime_base, "source").await.unwrap()); - let discovery = WorkerDiscovery::new( - store, - "source".into(), - runtime_base.clone(), - Some(root.path().to_path_buf()), - SpawnedWorkerRegistry::new(runtime_dir), - ); - - let socket = runtime_base.join("target").join("sock"); - let listener = UnixListener::bind(&socket).unwrap(); - let (tx, mut rx) = tokio::sync::mpsc::channel(1); - let target = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); - let mut writer = JsonLineWriter::new(stream); - writer - .write(&Event::Snapshot { - session: protocol::SessionSnapshot { - pending_submissions: protocol::PendingSubmissionsSnapshot::default(), - entries: Vec::new(), - }, - greeting: protocol::Greeting { - worker_name: "target".into(), - cwd: "/tmp".into(), - provider: "test".into(), - model: "test".into(), - scope_summary: String::new(), - tools: Vec::new(), - context_window: 0, - context_tokens: 0, - reasoning: None, - context_usage: None, - }, - state: WorkerStatus::Idle.into(), - in_flight: Default::default(), - internal_workers: Vec::new(), - }) - .await - .unwrap(); - - let (stream, _) = listener.accept().await.unwrap(); - let (reader_half, writer_half) = stream.into_split(); - let mut reader = JsonLineReader::new(reader_half); - let mut writer = JsonLineWriter::new(writer_half); - writer - .write(&Event::Alert(Alert { - id: None, - position: None, - level: AlertLevel::Warn, - source: AlertSource::Worker, - message: "connect-time alert".into(), - timestamp_ms: 0, - })) - .await - .unwrap(); - writer - .write(&Event::Snapshot { - session: protocol::SessionSnapshot { - pending_submissions: protocol::PendingSubmissionsSnapshot::default(), - entries: Vec::new(), - }, - greeting: protocol::Greeting { - worker_name: "target".into(), - cwd: "/tmp".into(), - provider: "test".into(), - model: "test".into(), - scope_summary: String::new(), - tools: Vec::new(), - context_window: 0, - context_tokens: 0, - reasoning: None, - context_usage: None, - }, - state: WorkerStatus::Idle.into(), - in_flight: Default::default(), - internal_workers: Vec::new(), - }) - .await - .unwrap(); - let method = reader.next::().await.unwrap().unwrap(); - if let Method::NotifyTracked { message, .. } = method { - tx.send(message).await.unwrap(); - } else { - panic!("expected Notify, got {method:?}"); - } - }); - - let (_, tool) = send_to_peer_worker_tool(discovery)(); - let output = tool - .execute(r#"{"name":"target","message":"hello"}"#, Default::default()) - .await - .unwrap(); - assert_eq!(output.summary, "sent peer message to `target`"); - let message = rx.recv().await.unwrap(); - assert_eq!(message, "[Peer message from `source`]\nhello"); - target.await.unwrap(); - } - - #[tokio::test(flavor = "current_thread")] - async fn weak_notify_to_live_peer_uses_realtime_notify_and_noops_when_missing() { - let root = TempDir::new().unwrap(); - let store_dir = root.path().join("store"); - let runtime_base = root.path().join("runtime"); - std::fs::create_dir_all(runtime_base.join("target")).unwrap(); - let store = FsWorkerStore::new(&store_dir).unwrap(); - store - .write(&WorkerMetadata { - worker_name: "source".into(), - active: None, - workspace_root: None, - workspace_id: None, - spawned_children: Vec::new(), - reclaimed_children: Vec::new(), - peers: vec![session_store::WorkerPeer { - worker_name: "target".into(), - }], - resolved_manifest_snapshot: None, - subjektiv_session_attribution: None, - }) - .unwrap(); - store - .write(&WorkerMetadata { - worker_name: "target".into(), - active: None, - workspace_root: None, - workspace_id: None, - spawned_children: Vec::new(), - reclaimed_children: Vec::new(), - peers: vec![session_store::WorkerPeer { - worker_name: "source".into(), - }], - resolved_manifest_snapshot: None, - subjektiv_session_attribution: None, - }) - .unwrap(); - let runtime_dir = Arc::new(RuntimeDir::create(&runtime_base, "source").await.unwrap()); - let discovery = WorkerDiscovery::new( - store, - "source".into(), - runtime_base.clone(), - Some(root.path().to_path_buf()), - SpawnedWorkerRegistry::new(runtime_dir), - ); - - let socket = runtime_base.join("target").join("sock"); - let listener = UnixListener::bind(&socket).unwrap(); - let (tx, mut rx) = tokio::sync::mpsc::channel(1); - let target = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); - let mut writer = JsonLineWriter::new(stream); - writer - .write(&Event::Snapshot { - session: protocol::SessionSnapshot { - pending_submissions: protocol::PendingSubmissionsSnapshot::default(), - entries: Vec::new(), - }, - greeting: protocol::Greeting { - worker_name: "target".into(), - cwd: "/tmp".into(), - provider: "test".into(), - model: "test".into(), - scope_summary: String::new(), - tools: Vec::new(), - context_window: 0, - context_tokens: 0, - reasoning: None, - context_usage: None, - }, - state: WorkerStatus::Idle.into(), - in_flight: Default::default(), - internal_workers: Vec::new(), - }) - .await - .unwrap(); - - let (stream, _) = listener.accept().await.unwrap(); - let (reader_half, writer_half) = stream.into_split(); - let mut reader = JsonLineReader::new(reader_half); - let mut writer = JsonLineWriter::new(writer_half); - writer - .write(&Event::Snapshot { - session: protocol::SessionSnapshot { - pending_submissions: protocol::PendingSubmissionsSnapshot::default(), - entries: Vec::new(), - }, - greeting: protocol::Greeting { - worker_name: "target".into(), - cwd: "/tmp".into(), - provider: "test".into(), - model: "test".into(), - scope_summary: String::new(), - tools: Vec::new(), - context_window: 0, - context_tokens: 0, - reasoning: None, - context_usage: None, - }, - state: WorkerStatus::Idle.into(), - in_flight: Default::default(), - internal_workers: Vec::new(), - }) - .await - .unwrap(); - let method = reader.next::().await.unwrap().unwrap(); - if let Method::NotifyTracked { message, .. } = method { - tx.send(message).await.unwrap(); - } else { - panic!("expected Notify, got {method:?}"); - } - }); - - assert_eq!( - discovery - .send_weak_notify_to_live_peer("target", "weak event".into()) - .await, - WeakNotifyDelivery::Delivered - ); - assert_eq!(rx.recv().await.unwrap(), "weak event"); - target.await.unwrap(); - - assert_eq!( - discovery - .send_weak_notify_to_live_peer("missing", "no-op".into()) - .await, - WeakNotifyDelivery::SkippedMissing - ); - } - - #[tokio::test(flavor = "current_thread")] - async fn weak_notify_does_not_send_to_spawned_child_visibility() { - let root = TempDir::new().unwrap(); - let store_dir = root.path().join("store"); - let runtime_base = root.path().join("runtime"); - std::fs::create_dir_all(runtime_base.join("target")).unwrap(); - let store = FsWorkerStore::new(&store_dir).unwrap(); - let socket = runtime_base.join("target").join("sock"); - store - .write(&WorkerMetadata { - worker_name: "source".into(), - active: None, - workspace_root: None, - workspace_id: None, - spawned_children: vec![child("target", &socket)], - reclaimed_children: Vec::new(), - peers: Vec::new(), - resolved_manifest_snapshot: None, - subjektiv_session_attribution: None, - }) - .unwrap(); - store.write(&WorkerMetadata::new("target", None)).unwrap(); - let runtime_dir = Arc::new(RuntimeDir::create(&runtime_base, "source").await.unwrap()); - let discovery = WorkerDiscovery::new( - store, - "source".into(), - runtime_base, - Some(root.path().to_path_buf()), - SpawnedWorkerRegistry::new(runtime_dir), - ); - - assert_eq!( - discovery - .send_weak_notify_to_live_peer("target", "must not send".into()) - .await, - WeakNotifyDelivery::SkippedNotPeer { - visibility: VisibilityReason::SpawnedChild - } - ); - } - - #[tokio::test(flavor = "current_thread")] - async fn probe_socket_reads_status_after_replayed_alert() { - let root = TempDir::new().unwrap(); - let socket = root.path().join("worker.sock"); - let listener = UnixListener::bind(&socket).unwrap(); - let handle = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); - let mut writer = JsonLineWriter::new(stream); - writer - .write(&Event::Alert(Alert { - id: None, - position: None, - level: AlertLevel::Warn, - source: AlertSource::Worker, - message: "replayed alert".into(), - timestamp_ms: 0, - })) - .await - .unwrap(); - writer - .write(&Event::Snapshot { - session: protocol::SessionSnapshot { - pending_submissions: protocol::PendingSubmissionsSnapshot::default(), - entries: Vec::new(), - }, - greeting: protocol::Greeting { - worker_name: "alerted".into(), - cwd: "/tmp".into(), - provider: "test".into(), - model: "test".into(), - scope_summary: String::new(), - tools: Vec::new(), - context_window: 0, - context_tokens: 0, - reasoning: None, - context_usage: None, - }, - state: WorkerStatus::Paused.into(), - in_flight: Default::default(), - internal_workers: Vec::new(), - }) - .await - .unwrap(); - }); - - let info = probe_socket(&socket).await; - assert!(info.reachable); - assert!(matches!(info.status, Some(WorkerStatus::Paused))); - handle.await.unwrap(); - } - - fn child(name: &str, socket_path: &Path) -> WorkerSpawnedChild { - WorkerSpawnedChild { - worker_name: name.to_string(), - socket_path: socket_path.to_path_buf(), - scope_delegated: vec![WorkerSpawnedScopeRule { - target: PathBuf::from("/tmp"), - permission: "read".into(), - recursive: true, - symlink_policy: Default::default(), - }], - callback_address: PathBuf::from("/tmp/parent.sock"), - } - } - - async fn spawn_snapshot_socket(socket_path: &Path) -> tokio::task::JoinHandle<()> { - let _ = std::fs::remove_file(socket_path); - let listener = UnixListener::bind(socket_path).unwrap(); - tokio::spawn(async move { - loop { - let Ok((stream, _)) = listener.accept().await else { - break; - }; - tokio::spawn(async move { - let mut writer = JsonLineWriter::new(stream); - let _ = writer - .write(&Event::Snapshot { - session: protocol::SessionSnapshot { - pending_submissions: protocol::PendingSubmissionsSnapshot::default( - ), - entries: Vec::new(), - }, - greeting: protocol::Greeting { - worker_name: "child-live".into(), - cwd: "/tmp".into(), - provider: "test".into(), - model: "test".into(), - scope_summary: String::new(), - tools: Vec::new(), - context_window: 0, - context_tokens: 0, - reasoning: None, - context_usage: None, - }, - state: WorkerStatus::Idle.into(), - in_flight: Default::default(), - internal_workers: Vec::new(), - }) - .await; - }); - } - }) - } -} diff --git a/crates/worker/src/entrypoint.rs b/crates/worker/src/entrypoint.rs deleted file mode 100644 index 15f15407b..000000000 --- a/crates/worker/src/entrypoint.rs +++ /dev/null @@ -1,1099 +0,0 @@ -use std::ffi::OsString; -use std::path::{Path, PathBuf}; -use std::process::ExitCode; - -use crate::{ - PromptCatalogSource, Worker, WorkerBootstrapLayout, WorkerControllerTransport, - WorkerFilesystemAuthority, WorkerWorkspaceContext, start_worker_controller, -}; -use clap::{CommandFactory, FromArgMatches, Parser}; -use manifest::{Permission, ScopeConfig, ScopeRule, WorkerManifest, WorkerManifestConfig, paths}; -use session_store::{CombinedStore, FsWorkerStore, WorkerMetadataStore}; -use session_store::{FsStore, SegmentId, Store}; -use ticket::config::TicketRole; - -#[derive(Debug, Parser)] -#[command(about = "Spawn a Worker process from a manifest or builtin defaults")] -struct Cli { - /// Legacy profile selector. Direct worker startup no longer resolves - /// workspace-local profiles; pass a Backend-resolved manifest instead. - #[arg( - long, - value_name = "PROFILE", - conflicts_with_all = ["manifest", "project", "session", "adopt"] - )] - profile: Option, - - /// Process root/cwd context for direct worker startup. This is not Workspace - /// authority and does not enable `.yoi` discovery. Defaults to current dir. - #[arg(long, value_name = "PATH")] - workspace: Option, - - /// Manifest TOML to use directly as a one-file compatibility/debug input. - /// This bypasses profile discovery but still applies builtin defaults and - /// the same required-field validation boundary. - #[arg(long, value_name = "PATH", conflicts_with_all = ["project"])] - manifest: Option, - - /// Deprecated manifest-cascade project root flag. Ambient project/user - /// manifest discovery has been removed; configure/select a profile instead. - #[arg(long, value_name = "PATH")] - project: Option, - - /// Internal resolved manifest config for delegated child Worker spawning. - #[arg( - long, - value_name = "JSON", - requires = "adopt", - conflicts_with_all = ["profile", "manifest", "project", "worker", "session"], - hide = true - )] - spawn_config_json: Option, - - /// Directory for session persistence. Defaults to - /// `/sessions/` (see `manifest::paths`). - #[arg(short, long)] - store: Option, - - /// Claim a scope allocation pre-registered by a spawning Worker, rather - /// than installing a new top-level allocation. Used only when this - /// process is launched by `SubWorkerSpawn`; end users should never pass it. - #[arg(long)] - adopt: bool, - - /// Socket path of the spawning Worker, for delivering `Method::Notify` - /// callbacks upward. Required alongside `--adopt`. - #[arg(long, value_name = "PATH", requires = "adopt")] - callback: Option, - - /// Process-local Ticket role marker supplied by the Ticket role launcher. - #[arg(long, hide = true)] - ticket_role: Option, - - /// Resume or create a Worker by name. If name-keyed Worker state exists, - /// the active session/segment recorded there is restored; otherwise a - /// fresh top-level Worker is created with this name. - #[arg(long, value_name = "NAME", conflicts_with_all = ["adopt"])] - worker: Option, - - /// Require `--worker` to restore existing Worker state instead of creating a - /// fresh Worker when no state exists. Used by Worker discovery restore flows. - #[arg(long, requires = "worker")] - require_worker_state: bool, - - /// Restore a Worker from an existing session. The Worker re-uses the - /// given session id and appends new turns to the same jsonl; - /// concurrent writers are prevented by the worker-allocation. - /// Mutually exclusive with `--adopt` (spawned children always start - /// fresh). - #[arg(long, value_name = "UUID", conflicts_with_all = ["adopt"])] - session: Option, -} - -fn runtime_workspace_root(cli: &Cli) -> Result { - let raw = cli.workspace.as_deref().unwrap_or_else(|| Path::new(".")); - if raw.is_absolute() { - Ok(raw.to_path_buf()) - } else { - std::env::current_dir() - .map_err(|e| format!("failed to resolve current directory for workspace: {e}")) - .map(|cwd| cwd.join(raw)) - } -} - -fn runtime_workspace_context() -> WorkerWorkspaceContext { - WorkerWorkspaceContext::no_workspace() -} - -fn runtime_worker_name(cli: &Cli, workspace_root: &Path) -> String { - cli.worker - .as_deref() - .map(str::to_string) - .unwrap_or_else(|| default_worker_name_for_workspace(workspace_root)) -} - -fn default_worker_name_for_workspace(workspace_root: &Path) -> String { - let raw = workspace_root - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("workspace"); - sanitise_worker_name(raw) -} - -fn sanitise_worker_name(raw: &str) -> String { - let name: String = raw - .chars() - .map(|c| { - if c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.') { - c - } else { - '-' - } - }) - .collect(); - if name.chars().any(|c| c.is_ascii_alphanumeric()) { - name - } else { - "workspace".to_string() - } -} - -fn resolve_manifest(cli: &Cli) -> Result<(WorkerManifest, PromptCatalogSource), String> { - let process_root = runtime_workspace_root(cli)?; - let runtime_worker_name = runtime_worker_name(cli, &process_root); - let ((mut manifest, loader), apply_direct_launch_policy) = if let Some(config_json) = - cli.spawn_config_json.as_deref() - { - (load_spawn_config_json(config_json)?, false) - } else if let Some(profile) = &cli.profile { - return Err(format!( - "--profile {profile} requires Backend-resolved manifest authority; direct worker startup does not discover workspace-local profiles" - )); - } else if let Some(path) = &cli.manifest { - ( - load_single_manifest(path, cli.worker.as_deref(), &runtime_worker_name)?, - false, - ) - } else { - if cli.project.is_some() { - return Err( - "--project is no longer supported; direct worker startup uses --manifest, \ - --spawn-config-json, or builtin defaults only" - .to_string(), - ); - } - (load_builtin_default_manifest(&runtime_worker_name)?, true) - }; - - if apply_direct_launch_policy { - apply_profile_launch_policy(&mut manifest, &process_root, cli.ticket_role.as_deref())?; - } - apply_session_restore_overrides(&mut manifest, cli)?; - Ok((manifest, loader)) -} - -fn apply_session_restore_overrides(manifest: &mut WorkerManifest, cli: &Cli) -> Result<(), String> { - if let Some(worker_name) = cli.worker.as_deref() { - manifest.worker.name = worker_name.to_string(); - } - Ok(()) -} - -fn load_spawn_config_json( - config_json: &str, -) -> Result<(WorkerManifest, PromptCatalogSource), String> { - let config = serde_json::from_str::(config_json) - .map_err(|e| format!("failed to parse --spawn-config-json: {e}"))?; - let manifest = - WorkerManifest::try_from(WorkerManifestConfig::resolution_defaults().merge(config)) - .map_err(|e| format!("failed to resolve --spawn-config-json: {e}"))?; - Ok((manifest, PromptCatalogSource::builtins_only())) -} - -fn load_builtin_default_manifest( - worker_name: &str, -) -> Result<(WorkerManifest, PromptCatalogSource), String> { - let mut config = WorkerManifestConfig::resolution_defaults(); - config.worker.name = Some(worker_name.to_string()); - let manifest = WorkerManifest::try_from(config) - .map_err(|e| format!("failed to resolve builtin worker defaults: {e}"))?; - Ok((manifest, PromptCatalogSource::builtins_only())) -} - -pub fn resolve_runtime_profile_manifest( - _profile: Option<&str>, - _workspace_root: &Path, - _worker_name: &str, -) -> Result<(WorkerManifest, PromptCatalogSource), String> { - Err( - "runtime profile resolution requires a pre-resolved manifest/profile archive from Backend authority" - .to_string(), - ) -} - -pub fn resolve_runtime_profile_manifest_from_manifest( - mut manifest: WorkerManifest, - workspace_root: &Path, - worker_name: &str, -) -> Result<(WorkerManifest, PromptCatalogSource), String> { - if manifest.worker.name.is_empty() { - manifest.worker.name = worker_name.to_string(); - } - apply_profile_launch_policy(&mut manifest, workspace_root, None)?; - // Do not run plugin discovery here: runtime-created Workers receive their - // resolved manifest/profile archive from Backend authority, not by scanning - // materialized workdir-local plugin stores. - Ok((manifest, PromptCatalogSource::builtins_only())) -} - -pub fn resolve_runtime_profile_manifest_from_manifest_without_filesystem( - mut manifest: WorkerManifest, - _workspace_root: &Path, - worker_name: &str, -) -> Result<(WorkerManifest, PromptCatalogSource), String> { - if manifest.worker.name.is_empty() { - manifest.worker.name = worker_name.to_string(); - } - manifest.scope = ScopeConfig::default(); - manifest.delegation_scope = ScopeConfig::default(); - // Same as the filesystem-capable runtime path: no local discovery. - Ok((manifest, PromptCatalogSource::builtins_only())) -} - -fn load_single_manifest( - path: &Path, - explicit_worker_name: Option<&str>, - default_worker_name: &str, -) -> Result<(WorkerManifest, PromptCatalogSource), String> { - let toml = std::fs::read_to_string(path) - .map_err(|e| format!("failed to read manifest {}: {e}", path.display()))?; - let absolute_path = if path.is_absolute() { - path.to_path_buf() - } else { - std::env::current_dir() - .map_err(|e| format!("failed to resolve current directory: {e}"))? - .join(path) - }; - let base_dir = absolute_path.parent().ok_or_else(|| { - format!( - "manifest path {} has no parent directory", - absolute_path.display() - ) - })?; - let mut config = WorkerManifestConfig::resolution_defaults().merge( - WorkerManifestConfig::from_toml(&toml) - .map_err(|e| format!("failed to parse manifest {}: {e}", path.display()))? - .resolve_paths(base_dir), - ); - if let Some(worker_name) = explicit_worker_name { - config.worker.name = Some(worker_name.to_string()); - } else if config.worker.name.is_none() { - config.worker.name = Some(default_worker_name.to_string()); - } - let manifest = WorkerManifest::try_from(config) - .map_err(|e| format!("failed to resolve manifest {}: {e}", path.display()))?; - if manifest.scope.allow.is_empty() { - return Err(format!( - "manifest {} must declare scope.allow; profile launches receive concrete scope from launch policy", - path.display() - )); - } - Ok((manifest, PromptCatalogSource::builtins_only())) -} - -fn read_rule(target: PathBuf) -> ScopeRule { - ScopeRule { - target, - permission: Permission::Read, - recursive: true, - symlink_policy: Default::default(), - } -} - -fn write_rule(target: PathBuf) -> ScopeRule { - ScopeRule { - target, - permission: Permission::Write, - recursive: true, - symlink_policy: Default::default(), - } -} - -fn workspace_scope( - workspace_root: &Path, - permission: Permission, - deny_write: &[PathBuf], -) -> ScopeConfig { - let allow_rule = ScopeRule { - target: workspace_root.to_path_buf(), - permission, - recursive: true, - symlink_policy: Default::default(), - }; - let deny = deny_write - .iter() - .cloned() - .map(write_rule) - .collect::>(); - ScopeConfig { - allow: vec![allow_rule], - deny, - } -} - -fn workspace_worktree_delegation(workspace_root: &Path) -> ScopeConfig { - ScopeConfig { - allow: vec![ - read_rule(workspace_root.to_path_buf()), - write_rule(workspace_root.join(".worktree")), - ], - deny: Vec::new(), - } -} - -fn append_missing_rules(target: &mut Vec, defaults: Vec) { - for rule in defaults { - if !target.contains(&rule) { - target.push(rule); - } - } -} - -fn apply_scope_launch_defaults(scope: &mut ScopeConfig, defaults: ScopeConfig) { - // Profile resolution has already applied explicit profile/workspace override scope rules. - // Launch policy contributes runtime defaults on top rather than replacing those grants. - append_missing_rules(&mut scope.allow, defaults.allow); - append_missing_rules(&mut scope.deny, defaults.deny); -} - -fn apply_profile_launch_policy( - manifest: &mut WorkerManifest, - workspace_root: &Path, - ticket_role: Option<&str>, -) -> Result<(), String> { - let role = match ticket_role { - Some(raw) => { - Some(TicketRole::parse(raw).ok_or_else(|| format!("invalid ticket role `{raw}`"))?) - } - None => None, - }; - match role { - Some(TicketRole::Orchestrator) => { - let default_scope = workspace_scope(workspace_root, Permission::Read, &[]); - apply_scope_launch_defaults(&mut manifest.scope, default_scope); - manifest.delegation_scope = workspace_worktree_delegation(workspace_root); - } - Some(TicketRole::Intake) | Some(TicketRole::Reviewer) => { - let default_scope = workspace_scope(workspace_root, Permission::Read, &[]); - apply_scope_launch_defaults(&mut manifest.scope, default_scope); - manifest.delegation_scope = ScopeConfig::default(); - } - // Work assignment is not a filesystem or delegation grant. The selected - // Profile and explicit resource bindings remain the authority. - Some(TicketRole::Worker) => {} - None => { - let worktree_root = workspace_root.join(".worktree"); - let default_scope = workspace_scope( - workspace_root, - Permission::Write, - std::slice::from_ref(&worktree_root), - ); - apply_scope_launch_defaults(&mut manifest.scope, default_scope); - manifest.delegation_scope = workspace_worktree_delegation(workspace_root); - } - } - Ok(()) -} - -pub async fn run_cli() -> ExitCode { - run_cli_from("yoi worker", std::env::args_os().skip(1)).await -} - -pub async fn run_cli_from(bin_name: &'static str, args: I) -> ExitCode -where - I: IntoIterator, - T: Into, -{ - let cli = match parse_cli_from(bin_name, args) { - Ok(cli) => cli, - Err(err) => { - let code = err.exit_code(); - if let Err(print_err) = err.print() { - eprintln!("error: failed to write CLI error: {print_err}"); - } - return exit_code_from_i32(code); - } - }; - - run_cli_inner(cli).await -} - -fn parse_cli_from(bin_name: &'static str, args: I) -> Result -where - I: IntoIterator, - T: Into, -{ - let argv = std::iter::once(OsString::from(bin_name)) - .chain(args.into_iter().map(Into::into)) - .collect::>(); - let matches = Cli::command() - .name(bin_name) - .bin_name(bin_name) - .try_get_matches_from(argv)?; - Cli::from_arg_matches(&matches) -} - -fn exit_code_from_i32(code: i32) -> ExitCode { - match code { - 0 => ExitCode::SUCCESS, - 1 => ExitCode::FAILURE, - code => ExitCode::from(code.clamp(0, u8::MAX as i32) as u8), - } -} - -async fn run_cli_inner(cli: Cli) -> ExitCode { - let cwd = match std::env::current_dir() { - Ok(path) => path, - Err(e) => { - eprintln!("error: failed to resolve current directory: {e}"); - return ExitCode::FAILURE; - } - }; - let workspace_root = match runtime_workspace_root(&cli) { - Ok(root) => root, - Err(e) => { - eprintln!("error: {e}"); - return ExitCode::FAILURE; - } - }; - let (mut manifest, loader) = match resolve_manifest(&cli) { - Ok(pair) => pair, - Err(e) => { - eprintln!("error: {e}"); - return ExitCode::FAILURE; - } - }; - // Initialize persistent store. `paths::sessions_dir()` only - // returns None when none of YOI_DATA_DIR / YOI_HOME / - // XDG_DATA_HOME / HOME is set — surface that as a hard error to match the - // runtime-dir resolution below, rather than silently writing to a - // relative path under cwd. - let store_dir = match cli.store.clone() { - Some(p) => p, - None => match paths::sessions_dir() { - Some(d) => d, - None => { - eprintln!( - "error: could not resolve sessions directory \ - (set --store, YOI_DATA_DIR, YOI_HOME, XDG_DATA_HOME, or HOME)" - ); - return ExitCode::FAILURE; - } - }, - }; - let session_store = match FsStore::new(&store_dir) { - Ok(s) => s, - Err(e) => { - eprintln!("error: failed to initialize session store at {store_dir:?}: {e}"); - return ExitCode::FAILURE; - } - }; - let worker_metadata_dir = match paths::data_dir() { - Some(data_dir) => data_dir.join("workers"), - None => store_dir - .parent() - .map(|parent| parent.join("workers")) - .unwrap_or_else(|| PathBuf::from("workers")), - }; - let worker_metadata_store = match FsWorkerStore::new(&worker_metadata_dir) { - Ok(s) => s, - Err(e) => { - eprintln!("error: failed to initialize worker store at {worker_metadata_dir:?}: {e}"); - return ExitCode::FAILURE; - } - }; - let store = CombinedStore::new(session_store, worker_metadata_store); - let filesystem_authority = - WorkerFilesystemAuthority::local(workspace_root.clone(), cwd.clone()); - let workspace_context = runtime_workspace_context(); - - let mut worker = if cli.adopt { - let callback = match cli.callback.clone() { - Some(p) => p, - None => { - eprintln!("error: --adopt requires --callback"); - return ExitCode::FAILURE; - } - }; - match Worker::from_manifest_spawned_with_context( - manifest, - store, - loader, - callback, - workspace_context.clone(), - filesystem_authority.clone(), - ) - .await - { - Ok(p) => p, - Err(e) => { - eprintln!("error: failed to create spawned worker: {e}"); - return ExitCode::FAILURE; - } - } - } else if let Some(source_segment_id) = cli.session { - let source_session_id = match store.lookup_session_of(source_segment_id) { - Ok(Some(sid)) => sid, - Ok(None) => { - eprintln!( - "error: --session {source_segment_id}: segment is not registered to any session" - ); - return ExitCode::FAILURE; - } - Err(e) => { - eprintln!("error: lookup_session_of failed: {e}"); - return ExitCode::FAILURE; - } - }; - match Worker::restore_from_manifest_with_context( - source_session_id, - source_segment_id, - manifest, - store, - loader, - workspace_context.clone(), - filesystem_authority.clone(), - ) - .await - { - Ok(p) => p, - Err(e) => { - eprintln!("error: failed to restore worker: {e}"); - return ExitCode::FAILURE; - } - } - } else if let Some(worker_name) = cli.worker.as_deref() { - manifest.worker.name = worker_name.to_string(); - match store.read_by_name(worker_name) { - Ok(Some(_)) => { - match Worker::restore_from_worker_metadata_with_context( - worker_name, - manifest, - store, - loader, - workspace_context.clone(), - filesystem_authority.clone(), - ) - .await - { - Ok(p) => p, - Err(e) => { - eprintln!("error: failed to restore worker {worker_name}: {e}"); - return ExitCode::FAILURE; - } - } - } - Ok(None) if cli.require_worker_state => { - eprintln!("error: worker state missing for {worker_name}"); - return ExitCode::FAILURE; - } - Ok(None) => { - match Worker::from_manifest_with_context( - manifest, - store, - loader, - workspace_context.clone(), - filesystem_authority.clone(), - ) - .await - { - Ok(p) => p, - Err(e) => { - eprintln!("error: failed to create worker {worker_name}: {e}"); - return ExitCode::FAILURE; - } - } - } - Err(e) => { - eprintln!("error: failed to read worker state for {worker_name}: {e}"); - return ExitCode::FAILURE; - } - } - } else { - match Worker::from_manifest_with_context( - manifest, - store, - loader, - workspace_context.clone(), - filesystem_authority.clone(), - ) - .await - { - Ok(p) => p, - Err(e) => { - eprintln!("error: failed to create worker: {e}"); - return ExitCode::FAILURE; - } - } - }; - if let Some(role) = cli.ticket_role.clone() { - if TicketRole::parse(&role).is_none() { - eprintln!("error: invalid --ticket-role {role:?}"); - return ExitCode::FAILURE; - } - worker.set_runtime_ticket_role(Some(role)); - } - let worker_name = worker.manifest().worker.name.clone(); - // Spawn the controller (starts socket server) - let runtime_base = match paths::runtime_dir() { - Some(d) => d, - None => { - eprintln!( - "error: could not resolve runtime directory \ - (set YOI_HOME, YOI_RUNTIME_DIR, XDG_RUNTIME_DIR, or HOME)" - ); - return ExitCode::FAILURE; - } - }; - - let started = match start_worker_controller( - worker, - WorkerBootstrapLayout::Direct { - runtime_base: runtime_base.clone(), - }, - WorkerControllerTransport::UnixSocket, - ) - .await - { - Ok(started) => started, - Err(e) => { - eprintln!("error: failed to start worker controller: {e}"); - return ExitCode::FAILURE; - } - }; - let (handle, shutdown_rx) = (started.handle, started.shutdown); - - let socket_path = handle.runtime_dir.socket_path(); - // Machine-readable ready line for parents that spawned this Worker - // (e.g. the TUI's interactive `spawn` flow). Tab-separated so a - // worker name with spaces still parses cleanly. Emit before the - // human line so a stderr-watching parent sees it first. - eprintln!("YOI-READY\t{worker_name}\t{}", socket_path.display()); - eprintln!("worker: {worker_name} listening on {:?}", socket_path); - - tokio::select! { - _ = tokio::signal::ctrl_c() => { - eprintln!("worker: {worker_name} shutting down (signal)"); - } - _ = shutdown_rx => { - eprintln!("worker: {worker_name} shutting down (client request)"); - } - } - - drop(handle); - ExitCode::SUCCESS -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::TempDir; - - fn write(path: &Path, contents: &str) { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).unwrap(); - } - std::fs::write(path, contents).unwrap(); - } - - fn manifest_toml(name: &str, scope: &Path) -> String { - format!( - r#" -[worker] -name = "{name}" - -[model] -scheme = "anthropic" -model_id = "test-model" - -[engine] - -[[scope.allow]] -target = "{scope}" -permission = "write" -"#, - scope = scope.display() - ) - } - - fn scope_rule(target: &Path, permission: Permission) -> ScopeRule { - ScopeRule { - target: target.to_path_buf(), - permission, - recursive: true, - symlink_policy: Default::default(), - } - } - - fn assert_scope_contains(rules: &[ScopeRule], target: &Path, permission: Permission) { - let expected = scope_rule(target, permission); - assert!( - rules.contains(&expected), - "expected scope rules to contain {expected:?}; got {rules:?}" - ); - } - - #[test] - fn user_manifest_flag_is_not_accepted() { - let err = - Cli::try_parse_from(["yoi worker", "--user-manifest", "manifest.toml"]).unwrap_err(); - assert_eq!(err.kind(), clap::error::ErrorKind::UnknownArgument); - } - - #[test] - fn subcommand_help_uses_yoi_worker_invocation() { - let err = parse_cli_from("yoi worker", ["--help"]).unwrap_err(); - assert_eq!(err.kind(), clap::error::ErrorKind::DisplayHelp); - let help = err.to_string(); - assert!(help.contains("Usage: yoi worker"), "{help}"); - assert!(help.contains("--worker "), "{help}"); - } - - #[test] - fn manifest_conflicts_with_project() { - let project_err = Cli::try_parse_from([ - "yoi worker", - "--manifest", - "manifest.toml", - "--project", - ".", - ]) - .unwrap_err(); - assert_eq!(project_err.kind(), clap::error::ErrorKind::ArgumentConflict); - } - - #[test] - fn overlay_flag_is_not_accepted() { - let err = - Cli::try_parse_from(["yoi worker", "--overlay", "worker.name = 'x'"]).unwrap_err(); - assert_eq!(err.kind(), clap::error::ErrorKind::UnknownArgument); - } - - #[test] - fn manifest_loads_single_file_without_user_or_workspace_prompt_loader() { - let tmp = TempDir::new().unwrap(); - let manifest = tmp.path().join("manifest.toml"); - write(&manifest, &manifest_toml("single", tmp.path())); - let cli = - Cli::try_parse_from(["yoi worker", "--manifest", manifest.to_str().unwrap()]).unwrap(); - - let (manifest, _loader) = resolve_manifest(&cli).unwrap(); - - assert_eq!(manifest.worker.name, "single"); - } - - #[test] - fn manifest_mode_does_not_apply_workspace_local_override() { - let tmp = TempDir::new().unwrap(); - let yoi_dir = tmp.path().join(".yoi"); - std::fs::create_dir_all(&yoi_dir).unwrap(); - write( - &yoi_dir.join("ignored-local-file.toml"), - r#" -[worker] -name = "from-local-override" -[engine] -language = "override" -"#, - ); - let manifest_path = tmp.path().join("manifest.toml"); - write( - &manifest_path, - &format!( - r#" -[worker] -name = "from-single-file" - -[model] -scheme = "anthropic" -model_id = "test-model" - -[engine] -language = "manifest" - -[[scope.allow]] -target = "{}" -permission = "write" -"#, - tmp.path().display() - ), - ); - - let cli = - Cli::try_parse_from(["yoi worker", "--manifest", manifest_path.to_str().unwrap()]) - .unwrap(); - let (manifest, _loader) = resolve_manifest(&cli).unwrap(); - - assert_eq!(manifest.worker.name, "from-single-file"); - assert_eq!(manifest.engine.language, "manifest"); - } - - #[test] - fn profile_resolution_is_rejected_for_direct_worker_startup() { - let cli = Cli::try_parse_from(["yoi worker", "--profile", "project:coder"]).unwrap(); - - let err = resolve_manifest(&cli).unwrap_err(); - - assert!( - err.contains("--profile project:coder requires Backend-resolved manifest authority") - ); - } - - #[test] - fn clean_workspace_manifest_resolution_does_not_create_repository_local_yoi() { - let tmp = TempDir::new().unwrap(); - let workspace = tmp.path().join("runtime-workspace"); - std::fs::create_dir(&workspace).unwrap(); - let cli = Cli::try_parse_from(["yoi worker", "--workspace", workspace.to_str().unwrap()]) - .unwrap(); - - let (manifest, _loader) = resolve_manifest(&cli).unwrap(); - - assert_eq!(manifest.worker.name, "runtime-workspace"); - assert!(!workspace.join(".yoi").exists()); - } - - #[test] - fn normal_startup_uses_builtin_defaults_without_local_profile_discovery() { - let tmp = TempDir::new().unwrap(); - let workspace = tmp.path().join("runtime-workspace"); - let yoi_dir = workspace.join(".yoi"); - std::fs::create_dir_all(&yoi_dir).unwrap(); - write( - &yoi_dir.join("ignored-local-file.toml"), - r#" -[worker] -name = "from-local-override" - -[engine] -language = "override" -"#, - ); - let cli = Cli::try_parse_from(["yoi worker", "--workspace", workspace.to_str().unwrap()]) - .unwrap(); - - let (manifest, _loader) = resolve_manifest(&cli).unwrap(); - - assert_eq!(manifest.worker.name, "runtime-workspace"); - assert_ne!(manifest.engine.language, "override"); - assert_scope_contains(&manifest.scope.allow, &workspace, Permission::Write); - } - - #[test] - fn ticket_worker_role_preserves_profile_scope_and_delegation_without_adding_grants() { - let tmp = TempDir::new().unwrap(); - let workspace = tmp.path().join("workspace"); - let explicit_scope = ScopeConfig { - allow: vec![scope_rule(&workspace.join("documents"), Permission::Write)], - deny: vec![scope_rule( - &workspace.join("documents/private"), - Permission::Write, - )], - }; - let explicit_delegation = ScopeConfig { - allow: vec![scope_rule(&workspace.join("reference"), Permission::Read)], - deny: vec![scope_rule( - &workspace.join("reference/private"), - Permission::Read, - )], - }; - for (scope, delegation) in [ - (ScopeConfig::default(), ScopeConfig::default()), - (explicit_scope, explicit_delegation), - ] { - let (mut manifest, _) = load_builtin_default_manifest("ticket-worker").unwrap(); - manifest.scope = scope.clone(); - manifest.delegation_scope = delegation.clone(); - - apply_profile_launch_policy(&mut manifest, &workspace, Some("worker")).unwrap(); - - assert_eq!(manifest.scope.allow, scope.allow); - assert_eq!(manifest.scope.deny, scope.deny); - assert_eq!(manifest.delegation_scope.allow, delegation.allow); - assert_eq!(manifest.delegation_scope.deny, delegation.deny); - } - } - - #[test] - fn orchestrator_direct_startup_gets_read_root_and_worktree_delegation_from_launch_policy() { - let tmp = TempDir::new().unwrap(); - let workspace = tmp.path().join("original-workspace"); - std::fs::create_dir(&workspace).unwrap(); - let cli = Cli::try_parse_from([ - "yoi worker", - "--workspace", - workspace.to_str().unwrap(), - "--ticket-role", - "orchestrator", - ]) - .unwrap(); - - let (manifest, _loader) = resolve_manifest(&cli).unwrap(); - - assert_scope_contains(&manifest.scope.allow, &workspace, Permission::Read); - assert!(manifest.scope.deny.is_empty()); - assert_eq!(manifest.delegation_scope.allow.len(), 2); - assert_eq!(manifest.delegation_scope.allow[0].target, workspace); - assert_eq!( - manifest.delegation_scope.allow[0].permission, - Permission::Read - ); - assert_eq!( - manifest.delegation_scope.allow[1].target, - tmp.path().join("original-workspace/.worktree") - ); - assert_eq!( - manifest.delegation_scope.allow[1].permission, - Permission::Write - ); - } - - #[test] - fn project_flag_no_longer_enables_ambient_manifest_cascade() { - let cli = Cli::try_parse_from(["yoi worker", "--project", "."]).unwrap(); - let err = resolve_manifest(&cli).unwrap_err(); - assert!(err.contains("--project is no longer supported")); - } - - #[test] - fn worker_flag_is_runtime_identity_for_session_restore() { - let tmp = TempDir::new().unwrap(); - let workspace = tmp.path().join("explicit-workspace"); - let store = tmp.path().join("sessions"); - std::fs::create_dir(&workspace).unwrap(); - let segment_id = session_store::new_segment_id().to_string(); - let cli = Cli::try_parse_from([ - "yoi worker", - "--workspace", - workspace.to_str().unwrap(), - "--session", - &segment_id, - "--worker", - "explicit-name", - "--store", - store.to_str().unwrap(), - ]) - .unwrap(); - - assert_eq!(cli.session.unwrap().to_string(), segment_id); - assert_eq!(cli.worker.as_deref(), Some("explicit-name")); - assert_eq!(runtime_worker_name(&cli, &workspace), "explicit-name"); - } - - #[test] - fn worker_flag_sets_requested_name_after_manifest_resolution() { - let tmp = TempDir::new().unwrap(); - let manifest = tmp.path().join("manifest.toml"); - write(&manifest, &manifest_toml("from-file", tmp.path())); - let cli = Cli::try_parse_from([ - "yoi worker", - "--manifest", - manifest.to_str().unwrap(), - "--worker", - "from-flag", - ]) - .unwrap(); - - let (manifest, _loader) = resolve_manifest(&cli).unwrap(); - - assert_eq!(manifest.worker.name, "from-flag"); - } - - #[test] - fn worker_flag_supplies_missing_name_for_single_manifest() { - let tmp = TempDir::new().unwrap(); - let manifest = tmp.path().join("manifest.toml"); - write( - &manifest, - r#" -[engine] - -[model] -scheme = "anthropic" -model_id = "test-model" - -[[scope.allow]] -target = "." -permission = "write" -"#, - ); - let cli = Cli::try_parse_from([ - "yoi worker", - "--manifest", - manifest.to_str().unwrap(), - "--worker", - "from-flag", - ]) - .unwrap(); - - let (manifest, _loader) = resolve_manifest(&cli).unwrap(); - - assert_eq!(manifest.worker.name, "from-flag"); - assert_eq!(manifest.scope.allow[0].target, tmp.path()); - } - - #[test] - fn worker_flag_with_no_manifest_creates_from_builtin_defaults_with_typed_name() { - let cli = Cli::try_parse_from(["yoi worker", "--worker", "agent"]).unwrap(); - - let (manifest, _loader) = resolve_manifest(&cli).unwrap(); - - assert_eq!(manifest.worker.name, "agent"); - } - - #[test] - fn profile_conflicts_with_manifest_and_restore_modes() { - let segment_id = session_store::new_segment_id().to_string(); - for args in [ - vec!["yoi worker", "--profile", "p.toml", "--manifest", "m.toml"], - vec![ - "yoi worker", - "--profile", - "p.toml", - "--session", - &segment_id, - ], - ] { - let err = Cli::try_parse_from(args).unwrap_err(); - assert_eq!(err.kind(), clap::error::ErrorKind::ArgumentConflict); - } - } - - #[test] - fn profile_and_worker_are_independent_startup_inputs() { - let cli = Cli::try_parse_from(["yoi worker", "--profile", "p.toml", "--worker", "agent"]) - .unwrap(); - assert_eq!(cli.profile.as_deref(), Some("p.toml")); - assert_eq!(cli.worker.as_deref(), Some("agent")); - } - - #[test] - fn old_session_worker_name_identity_alias_is_rejected() { - let segment_id = session_store::new_segment_id().to_string(); - let err = Cli::try_parse_from([ - "yoi worker", - "--session", - &segment_id, - "--session-worker-name", - "agent", - ]) - .unwrap_err(); - assert_eq!(err.kind(), clap::error::ErrorKind::UnknownArgument); - } - - #[test] - fn removed_profile_worker_name_alias_is_rejected() { - let err = - Cli::try_parse_from(["yoi worker", "--profile-worker-name", "agent"]).unwrap_err(); - assert_eq!(err.kind(), clap::error::ErrorKind::UnknownArgument); - } - - #[test] - fn manifest_mode_loads_single_file_with_minimal_prompt_loader() { - let tmp = TempDir::new().unwrap(); - let single_manifest = tmp.path().join("single.toml"); - write(&single_manifest, &manifest_toml("single-file", tmp.path())); - std::fs::create_dir_all(tmp.path().join("prompts")).unwrap(); - std::fs::create_dir_all(tmp.path().join(".yoi").join("prompts")).unwrap(); - let cli = Cli::try_parse_from([ - "yoi worker", - "--manifest", - single_manifest.to_str().unwrap(), - ]) - .unwrap(); - - let (manifest, _loader) = resolve_manifest(&cli).unwrap(); - - assert_eq!(manifest.worker.name, "single-file"); - } -} diff --git a/crates/worker/src/ipc/event.rs b/crates/worker/src/ipc/event.rs index 18c9da908..8fb6090f5 100644 --- a/crates/worker/src/ipc/event.rs +++ b/crates/worker/src/ipc/event.rs @@ -1,57 +1,6 @@ -//! `WorkerEvent` send / receive helpers. -//! -//! This module owns the parent-facing lifecycle-event primitive -//! (`WorkerEvent`) that children fire upward on turn-end / error / -//! shutdown / scope-sub-delegation. Three responsibilities live here: -//! -//! - **Send** a `Method::WorkerEvent` to the parent socket, fire-and-forget, -//! logging failures without blocking the child. -//! - **Render** agent-visible variants into human-readable strings for the -//! parent's notification buffer. Control-plane-only variants may still have -//! a renderer for diagnostics, but receive-side classification keeps them -//! out of LLM history/context. -//! - **Apply side effects** on the parent (registry / worker-allocation -//! updates) so that the receive path is idempotent and tolerant of -//! out-of-order delivery. -//! -//! Transport is fire-and-forget — the ticket's decision is that -//! callbacks are an optimisation and `ListWorkers` + `reclaim_stale` are -//! the real fallback. This module is allowed to drop events on the -//! floor (with a warn log) rather than retry. -//! -//! `apply_event_side_effects` takes its dependencies (registry, scope -//! lock path, self identity) by reference so the caller owns lifetime -//! and locking concerns. +//! Agent-visible rendering of typed Internal SubWorker lifecycle events. -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use protocol::{Method, WorkerEvent}; - -use crate::spawn::comm_tools::connect_and_send; -use crate::spawn::registry::SpawnedWorkerRegistry; - -/// Connect to `socket`, send a single `Method::WorkerEvent(event)`, and -/// return. Used by children to report up to their parent. -/// -/// This is a synchronous helper — callers that want fire-and-forget -/// semantics should wrap the call in `tokio::spawn` themselves. -pub async fn send_worker_event(socket: &Path, event: WorkerEvent) -> std::io::Result<()> { - connect_and_send(socket, &Method::WorkerEvent(event)).await -} - -/// Spawn a fire-and-forget task that sends `event` to `socket`. If -/// `socket` is `None`, no send happens (top-level Workers have no parent). -/// Any send failure is logged at warn level but otherwise ignored — -/// the parent is treated as best-effort. -pub fn fire_and_forget(socket: Option, event: WorkerEvent) { - let Some(socket) = socket else { return }; - tokio::spawn(async move { - if let Err(e) = send_worker_event(&socket, event).await { - tracing::warn!(error = %e, socket = %socket.display(), "WorkerEvent send failed"); - } - }); -} +use protocol::WorkerEvent; /// Render a variant into a one-line human-readable string. /// @@ -83,16 +32,3 @@ pub fn render_event(event: &WorkerEvent) -> String { } } } - -/// Legacy process callback events have no SubWorker registry authority. -/// -/// Internal SubWorker lifecycle is applied directly through typed session handles. A callback from -/// an externally adopted Worker may still be rendered for diagnostics, but it cannot add/remove -/// Internal children or transfer filesystem authority. -pub async fn apply_event_side_effects( - _event: &WorkerEvent, - _registry: &Arc, - _self_name: &str, - _self_parent_socket: &Option, -) { -} diff --git a/crates/worker/src/ipc/mod.rs b/crates/worker/src/ipc/mod.rs index 053bbe355..412657b55 100644 --- a/crates/worker/src/ipc/mod.rs +++ b/crates/worker/src/ipc/mod.rs @@ -1,7 +1,6 @@ pub mod alerter; pub mod event; pub mod protocol_session; -pub mod server; pub(crate) mod interceptor; pub(crate) mod notify_buffer; diff --git a/crates/worker/src/ipc/server.rs b/crates/worker/src/ipc/server.rs deleted file mode 100644 index 03f0c52a8..000000000 --- a/crates/worker/src/ipc/server.rs +++ /dev/null @@ -1,224 +0,0 @@ -use std::io; -use std::io::ErrorKind; -use std::path::PathBuf; - -use protocol::stream::{JsonLineReader, JsonLineWriter}; -use tokio::net::UnixListener; -use tokio::sync::oneshot; -use tokio::task::{JoinHandle, JoinSet}; - -use crate::controller::WorkerHandle; -use crate::ipc::protocol_session::{ - dispatch_worker_protocol_method, live_log_entry_event, subscribe_worker_protocol_session, -}; -use protocol::{ErrorCode, Event}; - -/// Unix socket server for Worker Protocol. -/// -/// Listens on the Worker's runtime directory socket path. -/// Each client connection gets bidirectional JSONL: -/// - Client writes Method lines → forwarded to WorkerController -/// - Worker events → written as Event lines to all connected clients -pub struct SocketServer { - accept_task: Option>, - shutdown: Option>, - path: PathBuf, -} - -impl SocketServer { - /// Start listening on the WorkerHandle's socket path. - pub async fn start(handle: &WorkerHandle) -> Result { - let path = handle.runtime_dir.socket_path(); - - // Remove stale socket file if it exists - let _ = tokio::fs::remove_file(&path).await; - - let listener = UnixListener::bind(&path)?; - let handle = handle.clone(); - let (shutdown_tx, mut shutdown_rx) = oneshot::channel(); - - let accept_task = tokio::spawn(async move { - let mut connections = JoinSet::new(); - loop { - tokio::select! { - _ = &mut shutdown_rx => break, - accepted = listener.accept() => match accepted { - Ok((stream, _)) => { - let handle = handle.clone(); - connections.spawn(handle_connection(stream, handle)); - } - Err(_) => break, - }, - completed = connections.join_next(), if !connections.is_empty() => { - let _ = completed; - } - } - } - connections.shutdown().await; - }); - - Ok(Self { - accept_task: Some(accept_task), - shutdown: Some(shutdown_tx), - path, - }) - } - - /// Stop accepting connections and join the server task. Dropping its - /// `JoinSet` cancels every active connection task before this returns. - pub async fn shutdown(mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); - } - if let Some(task) = self.accept_task.take() { - let _ = task.await; - } - let _ = tokio::fs::remove_file(&self.path).await; - } - - /// The socket file path. - pub fn path(&self) -> &std::path::Path { - &self.path - } -} - -impl Drop for SocketServer { - fn drop(&mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); - } - let _ = self.accept_task.take(); - let _ = std::fs::remove_file(&self.path); - } -} - -fn is_peer_disconnect_read_error(error: &io::Error) -> bool { - matches!( - error.kind(), - ErrorKind::ConnectionReset - | ErrorKind::ConnectionAborted - | ErrorKind::BrokenPipe - | ErrorKind::UnexpectedEof - ) -} - -async fn handle_connection(stream: tokio::net::UnixStream, handle: WorkerHandle) { - let (reader, writer) = stream.into_split(); - let mut reader = JsonLineReader::new(reader); - let mut writer = JsonLineWriter::new(writer); - - // Hold the in-flight stream lock while taking the session-log mirror - // snapshot. `LogEntry::AnnotatedAssistantItem` is mirror-only for live clients, - // so a finalized assistant block must be observed either as an already - // committed entry or as the still-present in-flight block. This lock - // order matches `append_entry` (in-flight clear before sink publish) and - // keeps the snapshot/live boundary gap-free. - let mut streams = subscribe_worker_protocol_session(&handle); - for alert in streams.alert_snapshot { - if writer.write_event(&Event::Alert(alert)).await.is_err() { - return; - } - } - - // Send the typed snapshot up front so late attachers can - // reconstruct view state without an extra round trip. - if writer.write_event(&streams.snapshot_event).await.is_err() { - return; - } - - loop { - tokio::select! { - // Live session-log entries → dispatched as the role-specific - // wire events. `SegmentLogSink` only broadcasts committed log - // entries with live UI meaning; `UserInput` travels this lane so - // the visible user line is ordered with `SegmentStart` rotation. - entry = streams.log_entries.recv() => { - match entry { - Ok(entry) => { - if let Some(event) = live_log_entry_event(entry) { - if writer.write_event(&event).await.is_err() { - break; - } - } - } - Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { - // Slow client fell behind the broadcast buffer. - // Drop the connection so the next reconnect - // re-seeds the prefix via subscribe_with_snapshot. - break; - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => break, - } - } - // Broadcast events → this client - event = streams.events.recv() => { - match event { - Ok(event) => { - let Ok(event) = handle.alerter.freshen_event(event) else { - // The queued Alert was evicted; reconnect must rebuild - // the bounded prefix rather than forward a stale anchor. - break; - }; - if writer.write_event(&event).await.is_err() { - break; - } - } - Err(_) => break, - } - } - // Client methods → handle or forward to controller - method = reader.next_method() => { - match method { - Ok(Some(method)) => { - if let Some(response) = dispatch_worker_protocol_method(&handle, method).await - && writer.write_event(&response).await.is_err() - { - break; - } - } - Ok(None) => break, - Err(e) if is_peer_disconnect_read_error(&e) => break, - Err(e) => { - if writer - .write(&Event::Error { - code: ErrorCode::InvalidRequest, - message: format!("invalid method: {e}"), - }) - .await - .is_err() - { - break; - } - } - } - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn peer_disconnect_read_errors_are_connection_close() { - for kind in [ - ErrorKind::ConnectionReset, - ErrorKind::ConnectionAborted, - ErrorKind::BrokenPipe, - ErrorKind::UnexpectedEof, - ] { - let error = io::Error::new(kind, "peer disconnected"); - assert!( - is_peer_disconnect_read_error(&error), - "{kind:?} should be treated as a normal peer disconnect" - ); - } - } - - #[test] - fn invalid_data_is_not_peer_disconnect() { - let error = io::Error::new(ErrorKind::InvalidData, "malformed method"); - assert!(!is_peer_disconnect_read_error(&error)); - } -} diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index 0037fa8cb..8c096a7d8 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -7,8 +7,6 @@ pub mod bootstrap; pub mod checkout; pub mod compact; pub mod controller; -pub mod discovery; -pub mod entrypoint; pub mod feature; mod file_operation; pub mod fs_view; @@ -19,9 +17,9 @@ pub mod job; mod metadata_lifecycle; pub mod model_client; mod paste_artifact_tool; +pub mod profile_manifest; pub mod prompt; pub mod runtime; -pub mod runtime_command; pub mod segment_log_sink; mod session_capture; mod session_history; @@ -37,6 +35,7 @@ pub mod wip; mod internal_worker; mod interrupt_prep; +mod peer; mod permission; mod worker; @@ -45,10 +44,9 @@ pub use bootstrap::{ WorkerBootstrapLayout, start_worker_controller, }; pub use compact::token_counter::{EstimateSource, SplitPoint, TokenEstimate}; -pub use controller::{ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle}; +pub use controller::{ShutdownReceiver, WorkerController, WorkerHandle}; pub use hook::{Hook, HookEventKind, HookRegistryBuilder}; pub use ipc::alerter::Alerter; -pub use ipc::server::SocketServer; pub use manifest::{ AuthRef, ModelManifest, SchemeKind, Scope, WorkerManifest, WorkerManifestConfig, WorkerMetaConfig, diff --git a/crates/worker/src/peer.rs b/crates/worker/src/peer.rs new file mode 100644 index 000000000..6120ebc09 --- /dev/null +++ b/crates/worker/src/peer.rs @@ -0,0 +1,186 @@ +//! Reciprocal peer metadata registration, independent of discovery and transport. +//! +//! Registering a peer does not start a Worker or grant filesystem, observation, or +//! messaging authority. The two metadata writes are not a transaction: a failed +//! reciprocal write triggers best-effort restoration of the source's prior peers. + +use serde::Serialize; +use session_store::{WorkerMetadataStore, WorkerStoreError, validate_worker_name}; + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub(crate) struct PeerRegistrationResult { + pub source: String, + pub peer: String, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum PeerRegistrationError { + #[error("worker state missing for `{worker_name}`")] + StateMissing { worker_name: String }, + #[error("worker `{worker_name}` cannot be registered as a peer of itself")] + SelfPeer { worker_name: String }, + #[error("worker `{worker_name}` does not exist")] + MissingWorker { worker_name: String }, + #[error("worker store error: {0}")] + WorkerStore(#[from] WorkerStoreError), +} + +pub(crate) fn register_peer( + store: &impl WorkerMetadataStore, + source: &str, + peer: &str, +) -> Result { + ensure_existing_peer(store, source, peer)?.ok_or_else(|| PeerRegistrationError::MissingWorker { + worker_name: peer.to_string(), + }) +} + +fn ensure_existing_peer( + store: &impl WorkerMetadataStore, + source: &str, + peer: &str, +) -> Result, PeerRegistrationError> { + validate_worker_name(peer)?; + if peer == source { + return Err(PeerRegistrationError::SelfPeer { + worker_name: peer.to_string(), + }); + } + let prior_source_peers = store + .read_by_name(source)? + .ok_or_else(|| PeerRegistrationError::StateMissing { + worker_name: source.to_string(), + })? + .peers; + if store.read_by_name(peer)?.is_none() { + return Ok(None); + } + + store.add_peer(source, peer)?; + if let Err(error) = store.add_peer(peer, source) { + let _ = store.set_peers(source, prior_source_peers); + return Err(PeerRegistrationError::WorkerStore(error)); + } + Ok(Some(PeerRegistrationResult { + source: source.to_string(), + peer: peer.to_string(), + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use session_store::{FsWorkerStore, WorkerMetadata}; + + #[test] + fn register_peer_persists_reciprocal_metadata_without_duplicates() { + let root = tempfile::tempdir().unwrap(); + let store = FsWorkerStore::new(root.path()).unwrap(); + store.write(&WorkerMetadata::new("source", None)).unwrap(); + store.write(&WorkerMetadata::new("target", None)).unwrap(); + for _ in 0..2 { + let result = register_peer(&store, "source", "target").unwrap(); + assert_eq!( + serde_json::to_value(result).unwrap(), + serde_json::json!({ + "source": "source", "peer": "target", + }) + ); + } + // Reopen the real store rather than asserting only the write calls. + let persisted = FsWorkerStore::new(root.path()).unwrap(); + let source = persisted.read_by_name("source").unwrap().unwrap(); + let target = persisted.read_by_name("target").unwrap().unwrap(); + assert_eq!(source.peers.len(), 1); + assert_eq!(source.peers[0].worker_name, "target"); + assert_eq!(target.peers.len(), 1); + assert_eq!(target.peers[0].worker_name, "source"); + } + + #[test] + fn invalid_self_or_missing_peer_leaves_metadata_unchanged() { + let root = tempfile::tempdir().unwrap(); + let store = FsWorkerStore::new(root.path()).unwrap(); + store.write(&WorkerMetadata::new("source", None)).unwrap(); + assert!(matches!( + register_peer(&store, "source", "source"), + Err(PeerRegistrationError::SelfPeer { .. }) + )); + assert!(matches!( + register_peer(&store, "source", "missing"), + Err(PeerRegistrationError::MissingWorker { .. }) + )); + assert!(matches!( + register_peer(&store, "source", "../escape"), + Err(PeerRegistrationError::WorkerStore(_)) + )); + assert!(matches!( + register_peer(&store, "missing-source", "source"), + Err(PeerRegistrationError::StateMissing { .. }) + )); + assert!( + store + .read_by_name("source") + .unwrap() + .unwrap() + .peers + .is_empty() + ); + assert_eq!(store.list_names().unwrap(), vec!["source"]); + } + + struct FailTargetPeerStore(FsWorkerStore); + + impl WorkerMetadataStore for FailTargetPeerStore { + fn write(&self, metadata: &WorkerMetadata) -> Result<(), WorkerStoreError> { + if metadata.worker_name == "target" && !metadata.peers.is_empty() { + return Err(WorkerStoreError::Io(std::io::Error::other( + "target write failed", + ))); + } + self.0.write(metadata) + } + + fn read_by_name(&self, name: &str) -> Result, WorkerStoreError> { + self.0.read_by_name(name) + } + + fn list_names(&self) -> Result, WorkerStoreError> { + self.0.list_names() + } + + fn delete_by_name(&self, name: &str) -> Result<(), WorkerStoreError> { + self.0.delete_by_name(name) + } + } + + #[test] + fn reciprocal_write_failure_restores_exact_prior_source_peers() { + let root = tempfile::tempdir().unwrap(); + let inner = FsWorkerStore::new(root.path()).unwrap(); + for name in ["source", "target", "existing"] { + inner.write(&WorkerMetadata::new(name, None)).unwrap(); + } + inner.add_peer("source", "existing").unwrap(); + for had_target in [false, true] { + if had_target { + inner.add_peer("source", "target").unwrap(); + } + let before = inner.read_by_name("source").unwrap().unwrap().peers; + let store = FailTargetPeerStore(inner.clone()); + assert!(matches!( + register_peer(&store, "source", "target"), + Err(PeerRegistrationError::WorkerStore(_)) + )); + assert_eq!(inner.read_by_name("source").unwrap().unwrap().peers, before); + assert!( + inner + .read_by_name("target") + .unwrap() + .unwrap() + .peers + .is_empty() + ); + } + } +} diff --git a/crates/worker/src/profile_manifest.rs b/crates/worker/src/profile_manifest.rs new file mode 100644 index 000000000..f029cf420 --- /dev/null +++ b/crates/worker/src/profile_manifest.rs @@ -0,0 +1,183 @@ +//! Runtime manifest preparation from Backend-resolved profiles. +//! +//! This module has no process entrypoint or local profile discovery authority. + +use std::path::{Path, PathBuf}; + +use crate::PromptCatalogSource; +use manifest::{Permission, ScopeConfig, ScopeRule, WorkerManifest}; + +pub fn resolve_runtime_profile_manifest_from_manifest( + mut manifest: WorkerManifest, + workspace_root: &Path, + worker_name: &str, +) -> Result<(WorkerManifest, PromptCatalogSource), String> { + if manifest.worker.name.is_empty() { + manifest.worker.name = worker_name.to_string(); + } + apply_profile_launch_policy(&mut manifest, workspace_root); + // Do not run plugin discovery here: runtime-created Workers receive their + // resolved manifest/profile archive from Backend authority, not by scanning + // materialized workdir-local plugin stores. + Ok((manifest, PromptCatalogSource::builtins_only())) +} + +pub fn resolve_runtime_profile_manifest_from_manifest_without_filesystem( + mut manifest: WorkerManifest, + _workspace_root: &Path, + worker_name: &str, +) -> Result<(WorkerManifest, PromptCatalogSource), String> { + if manifest.worker.name.is_empty() { + manifest.worker.name = worker_name.to_string(); + } + manifest.scope = ScopeConfig::default(); + manifest.delegation_scope = ScopeConfig::default(); + // Same as the filesystem-capable runtime path: no local discovery. + Ok((manifest, PromptCatalogSource::builtins_only())) +} + +fn read_rule(target: PathBuf) -> ScopeRule { + ScopeRule { + target, + permission: Permission::Read, + recursive: true, + symlink_policy: Default::default(), + } +} + +fn write_rule(target: PathBuf) -> ScopeRule { + ScopeRule { + target, + permission: Permission::Write, + recursive: true, + symlink_policy: Default::default(), + } +} + +fn workspace_scope( + workspace_root: &Path, + permission: Permission, + deny_write: &[PathBuf], +) -> ScopeConfig { + let allow_rule = ScopeRule { + target: workspace_root.to_path_buf(), + permission, + recursive: true, + symlink_policy: Default::default(), + }; + let deny = deny_write + .iter() + .cloned() + .map(write_rule) + .collect::>(); + ScopeConfig { + allow: vec![allow_rule], + deny, + } +} + +fn workspace_worktree_delegation(workspace_root: &Path) -> ScopeConfig { + ScopeConfig { + allow: vec![ + read_rule(workspace_root.to_path_buf()), + write_rule(workspace_root.join(".worktree")), + ], + deny: Vec::new(), + } +} + +fn append_missing_rules(target: &mut Vec, defaults: Vec) { + for rule in defaults { + if !target.contains(&rule) { + target.push(rule); + } + } +} + +fn apply_scope_launch_defaults(scope: &mut ScopeConfig, defaults: ScopeConfig) { + // Profile resolution has already applied explicit profile/workspace override scope rules. + // Launch policy contributes runtime defaults on top rather than replacing those grants. + append_missing_rules(&mut scope.allow, defaults.allow); + append_missing_rules(&mut scope.deny, defaults.deny); +} + +fn apply_profile_launch_policy(manifest: &mut WorkerManifest, workspace_root: &Path) { + let worktree_root = workspace_root.join(".worktree"); + let default_scope = workspace_scope( + workspace_root, + Permission::Write, + std::slice::from_ref(&worktree_root), + ); + apply_scope_launch_defaults(&mut manifest.scope, default_scope); + manifest.delegation_scope = workspace_worktree_delegation(workspace_root); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn manifest() -> WorkerManifest { + let mut config = manifest::WorkerManifestConfig::resolution_defaults(); + config.worker.name = Some("resolved".into()); + WorkerManifest::try_from(config).unwrap() + } + + #[test] + fn runtime_launch_preserves_explicit_scope_without_repository_discovery() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir(root.path().join(".yoi")).unwrap(); + std::fs::write(root.path().join(".yoi/manifest.toml"), "invalid TOML").unwrap(); + let mut original = manifest(); + let explicit = read_rule(root.path().join("reference")); + original.scope.allow.push(explicit.clone()); + let (resolved, _) = + resolve_runtime_profile_manifest_from_manifest(original, root.path(), "fallback") + .unwrap(); + assert_eq!(resolved.worker.name, "resolved"); + assert!(resolved.scope.allow.contains(&explicit)); + assert!( + resolved + .scope + .allow + .contains(&write_rule(root.path().to_path_buf())) + ); + assert!( + resolved + .scope + .deny + .contains(&write_rule(root.path().join(".worktree"))) + ); + assert_eq!( + resolved.delegation_scope.allow, + vec![ + read_rule(root.path().to_path_buf()), + write_rule(root.path().join(".worktree")), + ] + ); + } + + #[test] + fn runtime_without_filesystem_clears_scope_and_delegation_grants() { + let mut original = manifest(); + original.worker.name.clear(); + original + .scope + .allow + .push(write_rule(PathBuf::from("/work"))); + original + .delegation_scope + .allow + .push(write_rule(PathBuf::from("/work"))); + let (resolved, _) = resolve_runtime_profile_manifest_from_manifest_without_filesystem( + original, + Path::new("/unused"), + "runtime-worker", + ) + .unwrap(); + assert_eq!(resolved.worker.name, "runtime-worker"); + assert!(resolved.scope.allow.is_empty()); + assert!(resolved.scope.deny.is_empty()); + assert!(resolved.delegation_scope.allow.is_empty()); + assert!(resolved.delegation_scope.deny.is_empty()); + } +} diff --git a/crates/worker/src/runtime/dir.rs b/crates/worker/src/runtime/dir.rs index ec09099d4..95606b320 100644 --- a/crates/worker/src/runtime/dir.rs +++ b/crates/worker/src/runtime/dir.rs @@ -1,17 +1,12 @@ use std::io; use std::path::{Path, PathBuf}; -use manifest::{ScopeRule, paths}; +use manifest::ScopeRule; use serde::{Deserialize, Serialize}; use tokio::fs; -use crate::shared_state::WorkerSharedState; - -/// One spawned-child record mirrored to `spawned_workers.json`. -/// -/// Written by the spawner after registry changes so runtime-local tools -/// have a materialised snapshot. Durable restore uses Worker state metadata; -/// this file is not the authoritative source. +/// Legacy process-child projection used only by persisted scope reclamation. +/// Socket fields retain the stored format; they are never connected to. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SpawnedWorkerRecord { /// Spawned Worker's identity. @@ -25,56 +20,33 @@ pub struct SpawnedWorkerRecord { pub callback_address: PathBuf, } -/// Manages the Worker's runtime directory on tmpfs. +/// Artifact directory retained by an in-process Controller. /// -/// ```text -/// /{worker_name}/ -/// ├── pid -/// ├── status.json -/// ├── manifest.toml -/// ├── history.json -/// └── sock (created by socket listener, not by RuntimeDir) -/// ``` -/// -/// `` is resolved via [`manifest::paths::runtime_dir`]. -/// Files are written atomically (write tmp → rename). -/// The directory is removed on drop. +/// Worker liveness and configuration belong to the Host, not filesystem projections. +/// Persistent run directories and pre-existing data are never removed on drop. pub struct RuntimeDir { path: PathBuf, - write_legacy_snapshots: bool, preserve_on_drop: bool, - socket_file_name: &'static str, } impl RuntimeDir { - /// Create the runtime directory and write the PID file. + /// Create an ephemeral directory without PID, status, manifest or socket files. pub async fn create(base: &Path, worker_name: &str) -> Result { - let path = base.join(worker_name); - fs::create_dir_all(&path).await?; - - let pid = std::process::id().to_string(); - fs::write(path.join("pid"), pid.as_bytes()).await?; - - Ok(Self { - path, - write_legacy_snapshots: true, - preserve_on_drop: false, - socket_file_name: "sock", - }) + Self::create_transient(base, worker_name).await } - /// Create an ephemeral Runtime-owned artifact directory. - /// - /// Runtime-managed Workers keep their status in the owning Runtime and do - /// not materialize legacy pid/status/manifest projections. + /// Preserve pre-existing directories rather than claiming their contents for cleanup. pub async fn create_transient(base: &Path, worker_name: &str) -> Result { + fs::create_dir_all(base).await?; let path = base.join(worker_name); - fs::create_dir_all(&path).await?; + let preserve_on_drop = match fs::create_dir(&path).await { + Ok(()) => false, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists && path.is_dir() => true, + Err(error) => return Err(error), + }; Ok(Self { path, - write_legacy_snapshots: false, - preserve_on_drop: false, - socket_file_name: "sock", + preserve_on_drop, }) } @@ -100,39 +72,11 @@ impl RuntimeDir { std::fs::File::open(parent)?.sync_all()?; Ok(Self { path: path.to_path_buf(), - write_legacy_snapshots: false, preserve_on_drop: true, - socket_file_name: "worker.sock", }) } - /// Create in the default base directory resolved via - /// [`manifest::paths::runtime_dir`]. - pub async fn create_default(worker_name: &str) -> Result { - let base = default_base()?; - Self::create(&base, worker_name).await - } - - /// Write status.json atomically. - pub async fn write_status(&self, state: &WorkerSharedState) -> Result<(), io::Error> { - if !self.write_legacy_snapshots { - return Ok(()); - } - let content = state.status_json(); - atomic_write(&self.path.join("status.json"), content.as_bytes()).await - } - - /// Write manifest.toml (typically once at startup). - pub async fn write_manifest(&self, toml: &str) -> Result<(), io::Error> { - if !self.write_legacy_snapshots { - return Ok(()); - } - atomic_write(&self.path.join("manifest.toml"), toml.as_bytes()).await - } - - /// Write `spawned_workers.json` atomically. The entries are the full - /// set of spawned children known to this Worker — callers pass the - /// replacement list, no incremental merge. + /// Rewrite the legacy projection during persisted process-child reclamation. pub async fn write_spawned_workers( &self, records: &[SpawnedWorkerRecord], @@ -145,26 +89,10 @@ impl RuntimeDir { pub fn path(&self) -> &Path { &self.path } - - /// Path where the Unix socket should be created. External callers - /// that only know the worker name (e.g. the TUI's attach flow) - /// predict the same path via [`manifest::paths::worker_socket_path`]. - pub fn socket_path(&self) -> PathBuf { - self.path.join(self.socket_file_name) - } - - pub async fn close_socket(&self) -> Result<(), io::Error> { - match fs::remove_file(self.socket_path()).await { - Ok(()) => Ok(()), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error), - } - } } impl Drop for RuntimeDir { fn drop(&mut self) { - let _ = std::fs::remove_file(self.socket_path()); if !self.preserve_on_drop { let _ = std::fs::remove_dir_all(&self.path); } @@ -179,156 +107,72 @@ async fn atomic_write(target: &Path, content: &[u8]) -> Result<(), io::Error> { Ok(()) } -/// Resolve the default base directory for runtime data. -/// -/// Thin wrapper over [`manifest::paths::runtime_dir`] that converts a -/// missing-env situation into an `io::Error`. -pub fn default_base() -> Result { - paths::runtime_dir().ok_or_else(|| { - io::Error::new( - io::ErrorKind::NotFound, - "could not resolve runtime directory (no YOI_HOME / \ - YOI_RUNTIME_DIR / XDG_RUNTIME_DIR / HOME)", - ) - }) -} - #[cfg(test)] mod tests { use super::*; - use crate::shared_state::WorkerSharedState; - - fn test_state() -> WorkerSharedState { - WorkerSharedState::new( - "test-worker".into(), - session_store::new_segment_id(), - "[engine]\nname = \"test-worker\"".into(), - protocol::Greeting { - worker_name: "test-worker".into(), - cwd: "/tmp".into(), - provider: "anthropic".into(), - model: "claude".into(), - scope_summary: String::new(), - tools: Vec::new(), - context_window: 200_000, - context_tokens: 0, - reasoning: None, - context_usage: None, - }, - ) - } #[tokio::test] - async fn creates_directory_and_pid() { - let tmp = tempfile::tempdir().unwrap(); - let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap(); - - assert!(rt.path().join("pid").exists()); - let pid = std::fs::read_to_string(rt.path().join("pid")).unwrap(); - assert_eq!(pid, std::process::id().to_string()); - } - - #[tokio::test] - async fn write_status_creates_file() { - let tmp = tempfile::tempdir().unwrap(); - let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap(); - let state = test_state(); - - rt.write_status(&state).await.unwrap(); - - let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap(); - let parsed: serde_json::Value = serde_json::from_str(&content).unwrap(); - assert_eq!(parsed["state"], "idle"); - assert_eq!(parsed["worker_name"], "test-worker"); - } - - #[tokio::test] - async fn write_status_reflects_changes() { - let tmp = tempfile::tempdir().unwrap(); - let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap(); - let state = test_state(); - - state.transition(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( - protocol::WorkerRunState::Running, - ))); - rt.write_status(&state).await.unwrap(); - - let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap(); - let parsed: serde_json::Value = serde_json::from_str(&content).unwrap(); - assert_eq!(parsed["state"], "running"); - } - - #[tokio::test] - async fn write_manifest_creates_file() { - let tmp = tempfile::tempdir().unwrap(); - let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap(); - - rt.write_manifest("[engine]\nname = \"test\"") - .await - .unwrap(); - - let content = std::fs::read_to_string(rt.path().join("manifest.toml")).unwrap(); - assert_eq!(content, "[engine]\nname = \"test\""); - } - - #[tokio::test] - async fn transient_directory_does_not_write_liveness_snapshots() { - let tmp = tempfile::tempdir().unwrap(); - let rt = RuntimeDir::create_transient(tmp.path(), "runtime-worker") - .await - .unwrap(); - - rt.write_status(&test_state()).await.unwrap(); - rt.write_manifest("[worker]\nname = \"runtime-worker\"") - .await - .unwrap(); - - assert!(!rt.path().join("pid").exists()); - assert!(!rt.path().join("status.json").exists()); - assert!(!rt.path().join("manifest.toml").exists()); - } - - #[tokio::test] - async fn write_spawned_workers_creates_file() { - use manifest::{Permission, ScopeRule}; - let tmp = tempfile::tempdir().unwrap(); - let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap(); - - let records = vec![SpawnedWorkerRecord { - worker_name: "child".into(), - socket_path: "/run/yoi/child/sock".into(), - scope_delegated: vec![ScopeRule { - target: "/tmp/work".into(), - permission: Permission::Write, - recursive: true, - symlink_policy: Default::default(), - }], - callback_address: "/run/yoi/my-worker/sock".into(), - }]; - rt.write_spawned_workers(&records).await.unwrap(); - - let content = std::fs::read_to_string(rt.path().join("spawned_workers.json")).unwrap(); - let parsed: Vec = serde_json::from_str(&content).unwrap(); - assert_eq!(parsed.len(), 1); - assert_eq!(parsed[0].worker_name, "child"); - } - - #[tokio::test] - async fn socket_path() { - let tmp = tempfile::tempdir().unwrap(); - let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap(); - assert_eq!(rt.socket_path(), rt.path().join("sock")); - } - - #[tokio::test] - async fn drop_removes_directory() { - let tmp = tempfile::tempdir().unwrap(); - let dir_path; + async fn transient_controller_directory_has_no_liveness_projections_and_is_released() { + let root = tempfile::tempdir().unwrap(); + let path; { - let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap(); - dir_path = rt.path().to_owned(); - assert!(dir_path.exists()); + let runtime = RuntimeDir::create(root.path(), "worker").await.unwrap(); + path = runtime.path().to_owned(); + assert_eq!(std::fs::read_dir(&path).unwrap().count(), 0); } - assert!(!dir_path.exists()); + assert!(!path.exists()); + } + + #[tokio::test] + async fn preexisting_data_is_not_overwritten_or_removed_on_drop() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("worker"); + std::fs::create_dir(&path).unwrap(); + std::fs::write(path.join("pid"), "retained legacy data").unwrap(); + { + let _runtime = RuntimeDir::create(root.path(), "worker").await.unwrap(); + } + assert_eq!( + std::fs::read_to_string(path.join("pid")).unwrap(), + "retained legacy data" + ); + } + + #[tokio::test] + async fn persistent_run_preserves_logs_and_rejects_directory_reuse() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("runs/1"); + { + let _runtime = RuntimeDir::create_worker_run(&path).await.unwrap(); + for name in ["artifacts", "spawned", "worker.out.log", "worker.err.log"] { + assert!(path.join(name).exists(), "missing {name}"); + } + } + assert!(path.exists()); + let error = RuntimeDir::create_worker_run(&path).await.err().unwrap(); + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + } + + #[tokio::test] + async fn legacy_reclamation_projection_retains_its_stored_format() { + let root = tempfile::tempdir().unwrap(); + let runtime = RuntimeDir::create(root.path(), "worker").await.unwrap(); + let record: SpawnedWorkerRecord = serde_json::from_value(serde_json::json!({ + "worker_name": "legacy-child", "socket_path": "/old/child/sock", + "scope_delegated": [], "callback_address": "/old/parent/sock", + })) + .unwrap(); + runtime.write_spawned_workers(&[record]).await.unwrap(); + let value: serde_json::Value = serde_json::from_slice( + &std::fs::read(runtime.path().join("spawned_workers.json")).unwrap(), + ) + .unwrap(); + assert_eq!(value[0]["socket_path"], "/old/child/sock"); + assert_eq!(value[0]["callback_address"], "/old/parent/sock"); + runtime.write_spawned_workers(&[]).await.unwrap(); + assert_eq!( + std::fs::read_to_string(runtime.path().join("spawned_workers.json")).unwrap(), + "[]" + ); } } diff --git a/crates/worker/src/runtime/worker_allocation/table.rs b/crates/worker/src/runtime/worker_allocation/table.rs index b9d0a16ae..571a36fb1 100644 --- a/crates/worker/src/runtime/worker_allocation/table.rs +++ b/crates/worker/src/runtime/worker_allocation/table.rs @@ -72,7 +72,8 @@ pub struct Allocation { pub worker_name: String, /// Owning process. Checked with `kill(pid, 0)` for stale detection. pub pid: u32, - /// Worker's Unix socket path. + /// Legacy stored endpoint hint. Empty means the Host owns connectivity. + /// Allocation admission and stale reclamation use identity/PID, never this path. pub socket: PathBuf, /// Allow rules granted to this Worker (write + read). pub scope_allow: Vec, @@ -338,6 +339,51 @@ mod tests { assert_eq!(guard.data().allocations[0].worker_name, "a"); } + #[test] + fn empty_legacy_socket_roundtrips_without_path_resolution_or_cleanup() { + let root = TempDir::new().unwrap(); + let path = root.path().join("workers.json"); + let retained = root.path().join("retained-data"); + fs::write(&retained, "do not touch").unwrap(); + let segment = sid(); + { + let mut guard = open_empty(&path); + register_worker( + &mut guard, + "in-process".into(), + std::process::id(), + PathBuf::new(), + Vec::new(), + segment, + ) + .unwrap(); + } + let stored: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(stored["allocations"][0]["socket"], ""); + let mut guard = LockFileGuard::open(&path).unwrap(); + assert!(guard.data().allocations[0].socket.as_os_str().is_empty()); + super::super::reclaim_stale_with(&mut guard, |pid| { + assert_eq!(pid, std::process::id()); + true + }); + assert_eq!( + guard.data().allocations.len(), + 1, + "missing endpoint is not stale ownership" + ); + assert!(guard.data().find_by_segment(segment).is_some()); + super::super::reclaim_stale_with(&mut guard, |_| false); + drop(guard); + assert!( + LockFileGuard::open(&path) + .unwrap() + .data() + .allocations + .is_empty() + ); + assert_eq!(fs::read_to_string(retained).unwrap(), "do not touch"); + } + #[test] fn atomic_save_reopens_complete_table_at_both_rename_boundaries() { let dir = TempDir::new().unwrap(); diff --git a/crates/worker/src/runtime_command.rs b/crates/worker/src/runtime_command.rs deleted file mode 100644 index d594d0723..000000000 --- a/crates/worker/src/runtime_command.rs +++ /dev/null @@ -1,165 +0,0 @@ -use std::ffi::OsString; -use std::fmt; -use std::io; -use std::path::{Path, PathBuf}; - -const POD_RUNTIME_COMMAND_ENV: &str = "YOI_POD_RUNTIME_COMMAND"; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct WorkerRuntimeCommand { - pub program: PathBuf, - pub prefix_args: Vec, -} - -impl WorkerRuntimeCommand { - pub fn new(program: impl Into, prefix_args: Vec) -> Self { - Self { - program: program.into(), - prefix_args, - } - } - - pub fn for_current_exe() -> io::Result { - Ok(Self::for_executable(std::env::current_exe()?)) - } - - pub fn for_executable(program: impl Into) -> Self { - Self::new(program, vec![OsString::from("worker")]) - } - - /// Resolve the Worker runtime command used for subprocess launches. - /// - /// The default launch path is always the current `yoi` executable plus - /// the unified `worker` prefix argument. During development, a non-empty - /// `YOI_POD_RUNTIME_COMMAND` value replaces only the executable path; - /// the `worker` prefix is still added here and the env value is not parsed as a - /// shell command. - pub fn resolve() -> io::Result { - Self::resolve_from_env_value( - std::env::var_os(POD_RUNTIME_COMMAND_ENV), - std::env::current_exe, - ) - } - - fn resolve_from_env_value( - override_program: Option, - current_exe: F, - ) -> io::Result - where - F: FnOnce() -> io::Result, - { - if let Some(program) = override_program.filter(|program| !program.as_os_str().is_empty()) { - return Ok(Self::for_executable(program)); - } - - Ok(Self::for_executable(current_exe()?)) - } - - pub fn program(&self) -> &Path { - &self.program - } - - pub fn prefix_args(&self) -> &[OsString] { - &self.prefix_args - } - - pub fn argv_with(&self, args: I) -> Vec - where - I: IntoIterator, - S: Into, - { - let mut argv = self.prefix_args.clone(); - argv.extend(args.into_iter().map(Into::into)); - argv - } -} - -impl fmt::Display for WorkerRuntimeCommand { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.program.display())?; - for arg in &self.prefix_args { - write!(f, " {}", arg.to_string_lossy())?; - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn yoi_binary_defaults_to_worker_prefix() { - let command = WorkerRuntimeCommand::for_executable("/opt/yoi/bin/yoi"); - - assert_eq!(command.program(), Path::new("/opt/yoi/bin/yoi")); - assert_eq!(command.prefix_args(), [OsString::from("worker")]); - assert_eq!( - command.argv_with(["--worker", "agent"]), - vec!["worker", "--worker", "agent"] - .into_iter() - .map(OsString::from) - .collect::>() - ); - } - - #[test] - fn any_runtime_executable_gets_worker_prefix() { - let command = WorkerRuntimeCommand::for_executable("/opt/yoi/bin/custom-runtime"); - - assert_eq!(command.program(), Path::new("/opt/yoi/bin/custom-runtime")); - assert_eq!(command.prefix_args(), [OsString::from("worker")]); - assert_eq!( - command.argv_with(["--worker", "agent"]), - vec!["worker", "--worker", "agent"] - .into_iter() - .map(OsString::from) - .collect::>() - ); - } - - #[test] - fn resolve_uses_current_exe_when_override_is_unset() { - let command = WorkerRuntimeCommand::resolve_from_env_value(None, || { - Ok(PathBuf::from("/opt/yoi/bin/yoi")) - }) - .unwrap(); - - assert_eq!( - command, - WorkerRuntimeCommand::for_executable("/opt/yoi/bin/yoi") - ); - } - - #[test] - fn resolve_uses_current_exe_when_override_is_empty() { - let command = WorkerRuntimeCommand::resolve_from_env_value(Some(OsString::new()), || { - Ok(PathBuf::from("/opt/yoi/bin/yoi")) - }) - .unwrap(); - - assert_eq!( - command, - WorkerRuntimeCommand::for_executable("/opt/yoi/bin/yoi") - ); - } - - #[test] - fn resolve_override_replaces_only_program_and_keeps_worker_prefix() { - let command = WorkerRuntimeCommand::resolve_from_env_value( - Some(OsString::from("/tmp/rebuilt yoi")), - || panic!("override must not inspect current_exe"), - ) - .unwrap(); - - assert_eq!(command.program(), Path::new("/tmp/rebuilt yoi")); - assert_eq!(command.prefix_args(), [OsString::from("worker")]); - assert_eq!( - command.argv_with(["--worker", "agent"]), - vec!["worker", "--worker", "agent"] - .into_iter() - .map(OsString::from) - .collect::>() - ); - } -} diff --git a/crates/worker/src/spawn/comm_tools.rs b/crates/worker/src/spawn/comm_tools.rs deleted file mode 100644 index 59ca3c167..000000000 --- a/crates/worker/src/spawn/comm_tools.rs +++ /dev/null @@ -1,181 +0,0 @@ -//! Socket communication retained for the legacy top-level Worker callback protocol. -//! -//! Direct Internal SubWorker lifecycle is exposed through `worker.control` and the -//! canonical Worker tools, not a second SubWorker-specific tool family. - -use std::path::Path; -use std::time::Duration; - -use protocol::stream::{JsonLineReader, JsonLineWriter}; -use protocol::{Event, Method}; -use tokio::net::UnixStream; - -/// Timeout applied to each socket-level operation — connect, write, -/// read. Kept short so a stuck child doesn't block the spawner's turn. -const SOCKET_OP_TIMEOUT: Duration = Duration::from_secs(5); - -/// Connect with a timeout, drain the server's connect-time snapshot, -/// write one `Method` line, flush, and close. -/// -/// The Worker socket protocol sends replayed alerts and an initial -/// `Event::Snapshot` before it starts reading client methods. Send-only -/// callers must consume that prefix; otherwise a large snapshot can block -/// the server's writer before it reaches the method-read branch. Any -/// socket error maps to an `io::Error`; the caller decides whether to -/// surface it to the LLM or treat it as "worker stopped". -pub(crate) async fn connect_and_send(socket: &Path, method: &Method) -> std::io::Result<()> { - let stream = tokio::time::timeout(SOCKET_OP_TIMEOUT, UnixStream::connect(socket)) - .await - .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "connect timed out"))??; - let (r, w) = stream.into_split(); - let mut reader = JsonLineReader::new(r); - let mut writer = JsonLineWriter::new(w); - - drain_initial_snapshot(&mut reader).await?; - - tokio::time::timeout(SOCKET_OP_TIMEOUT, writer.write(method)) - .await - .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "write timed out"))??; - Ok(()) -} - -async fn drain_initial_snapshot(reader: &mut JsonLineReader) -> std::io::Result<()> -where - R: tokio::io::AsyncBufRead + Unpin, -{ - loop { - let event = tokio::time::timeout(SOCKET_OP_TIMEOUT, reader.next::()) - .await - .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timed out"))??; - match event { - Some(Event::Snapshot { .. }) => return Ok(()), - Some(_) => continue, - None => { - return Err(std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - "worker closed connection before Snapshot event", - )); - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use protocol::{Alert, AlertLevel, AlertSource, Greeting, WorkerEvent, WorkerStatus}; - use tempfile::TempDir; - use tokio::net::UnixListener; - use tokio::task::JoinHandle; - - fn snapshot(entries: Vec) -> Event { - Event::Snapshot { - session: protocol::SessionSnapshot { - pending_submissions: protocol::PendingSubmissionsSnapshot::default(), - entries: entries - .into_iter() - .enumerate() - .map(|(index, value)| protocol::SessionSnapshotEntry { - use_policy: None, - entry_id: format!("test-{index}"), - timestamp: index as u64, - provenance: protocol::SessionEntryProvenance::LegacyUnknown, - derived_from: Vec::new(), - data: protocol::SessionSnapshotEntryData::RunError { - message: value.to_string(), - failure: None, - }, - }) - .collect(), - }, - greeting: Greeting { - worker_name: "server".into(), - cwd: "/tmp".into(), - provider: "test".into(), - model: "test".into(), - scope_summary: String::new(), - tools: Vec::new(), - context_window: 200_000, - context_tokens: 0, - reasoning: None, - context_usage: None, - }, - state: WorkerStatus::Idle.into(), - in_flight: Default::default(), - internal_workers: Vec::new(), - } - } - - fn serve_initial_events_then_method( - listener: UnixListener, - events: Vec, - ) -> JoinHandle> { - tokio::spawn(async move { - let (stream, _) = listener.accept().await.ok()?; - let (r, w) = stream.into_split(); - let mut reader = JsonLineReader::new(r); - let mut writer = JsonLineWriter::new(w); - for event in events { - writer.write(&event).await.ok()?; - } - reader.next::().await.ok().flatten() - }) - } - - #[tokio::test] - async fn connect_and_send_drains_initial_alert_and_snapshot_before_method() { - let tmp = TempDir::new().unwrap(); - let socket = tmp.path().join("worker.sock"); - let listener = UnixListener::bind(&socket).unwrap(); - let received = serve_initial_events_then_method( - listener, - vec![ - Event::Alert(Alert { - id: None, - position: None, - level: AlertLevel::Warn, - source: AlertSource::Worker, - message: "replayed alert".into(), - timestamp_ms: 0, - }), - snapshot(Vec::new()), - ], - ); - - let method = Method::Shutdown { - command: protocol::WorkerCommandEnvelope::new(1), - }; - connect_and_send(&socket, &method).await.unwrap(); - - let method = received.await.unwrap().expect("expected method"); - assert!(matches!(method, Method::Shutdown { .. })); - } - - #[tokio::test] - async fn connect_and_send_delivers_method_after_large_initial_snapshot() { - let tmp = TempDir::new().unwrap(); - let socket = tmp.path().join("worker.sock"); - let listener = UnixListener::bind(&socket).unwrap(); - let large_payload = "x".repeat(2 * 1024 * 1024); - let received = serve_initial_events_then_method( - listener, - vec![snapshot(vec![ - serde_json::json!({ "payload": large_payload }), - ])], - ); - let expected = Method::WorkerEvent(WorkerEvent::TurnEnded { - worker_name: "child".into(), - }); - - connect_and_send(&socket, &expected).await.unwrap(); - - let method = received.await.unwrap().expect("expected method"); - match method { - Method::WorkerEvent(WorkerEvent::TurnEnded { worker_name }) => { - assert_eq!(worker_name, "child") - } - other => panic!("expected TurnEnded WorkerEvent, got {other:?}"), - } - } -} diff --git a/crates/worker/src/spawn/mod.rs b/crates/worker/src/spawn/mod.rs index 23cb51ad3..ec3a77e8c 100644 --- a/crates/worker/src/spawn/mod.rs +++ b/crates/worker/src/spawn/mod.rs @@ -1,4 +1,3 @@ -pub mod comm_tools; mod published; pub mod registry; pub mod tool; diff --git a/crates/worker/src/subjektiv.rs b/crates/worker/src/subjektiv.rs index 3d5666d5e..3f3cfcc2a 100644 --- a/crates/worker/src/subjektiv.rs +++ b/crates/worker/src/subjektiv.rs @@ -678,8 +678,8 @@ pub(crate) fn test_connection(client: Arc) -> SubjektivHost mod tests { use super::*; use crate::{ - Method, WorkerBootstrap, WorkerBootstrapLayout, WorkerControllerTransport, - WorkerFilesystemAuthority, WorkerManifest, WorkerWorkspaceContext, + Method, WorkerBootstrap, WorkerBootstrapLayout, WorkerFilesystemAuthority, WorkerManifest, + WorkerWorkspaceContext, }; use agen::llm_client::event::{Event as LlmEvent, ResponseStatus, StatusEvent}; use agen::llm_client::{ClientError, LlmClient, Request}; @@ -944,7 +944,6 @@ enabled = true WorkerBootstrapLayout::Direct { runtime_base: temp.path().join("runtime"), }, - WorkerControllerTransport::InProcess, ) .with_model_client(TaskScriptClient::default()) .with_subjektiv_host(host) @@ -1137,7 +1136,6 @@ enabled = true WorkerBootstrapLayout::Direct { runtime_base: temp.path().join("runtime"), }, - WorkerControllerTransport::InProcess, ) .with_model_client(ScriptedClient(tx)) .with_subjektiv_host(host.clone()) @@ -1318,7 +1316,6 @@ enabled = true WorkerBootstrapLayout::Direct { runtime_base: temp.path().join("runtime"), }, - WorkerControllerTransport::InProcess, ) .with_model_client(ScriptedClient(tx)) .with_subjektiv_host(host.clone()) @@ -1377,7 +1374,6 @@ enabled = true WorkerBootstrapLayout::Direct { runtime_base: temp.path().join("runtime"), }, - WorkerControllerTransport::InProcess, ) .with_model_client(ScriptedClient(tx)); let bootstrap = if attach { diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index ac9d44d6e..2f609b735 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -531,7 +531,6 @@ use crate::prompt::agents_md::read_agents_md; use crate::prompt::catalog::{CatalogError, PromptCatalog, WorkspacePromptProjection}; use crate::prompt::source::PromptCatalogSource; use crate::prompt::system::{SystemPromptContext, SystemPromptError, SystemPromptTemplate}; -use crate::runtime::dir; use crate::runtime::worker_allocation::{ self, ScopeAllocationGuard, ScopeLockError, SegmentActivationGuard, }; @@ -3456,18 +3455,13 @@ pub struct Worker { /// authority before the outer allocation guard reacquires the same lock. failed_activation_authority: Mutex>, /// Scope allocation in the machine-wide lock file. `Some` for - /// Workers built via `from_manifest` / `from_manifest_spawned` / + /// Workers built via `from_manifest` / /// `restore_from_manifest` (production paths); `None` for the /// low-level `Worker::new` constructor used in tests, which bypasses /// the registry. Kept purely for its `Drop` impl, which releases /// the allocation when the Worker is dropped. #[allow(dead_code)] scope_allocation: Option, - /// Socket path of the spawning Worker. `Some` only for Workers built via - /// `from_manifest_spawned`. Consumed by the controller to fire - /// `Method::WorkerEvent` reports upward (turn end, error, shutdown, - /// scope sub-delegation). - callback_socket: Option, /// Transient launch role for Ticket role sessions. This is process-local /// runtime identity used by controller policy; it is not model-visible and /// is not persisted into Ticket claim/session records. @@ -3751,7 +3745,6 @@ impl Worker { pending_committed_history: Arc::new(Mutex::new(std::collections::VecDeque::new())), failed_activation_authority: Mutex::new(None), scope_allocation: None, - callback_socket: None, runtime_ticket_role: None, prompts, inject_resident_summary: true, @@ -3916,8 +3909,8 @@ impl Worker { self.runtime_ticket_role.as_deref() } - /// Set the process-local Ticket role marker. Intended for entrypoint - /// launch metadata, not for model-visible prompts or durable claims. + /// Set the process-local Ticket role marker supplied by the Host at launch, + /// not by model-visible prompts or durable claims. pub fn set_runtime_ticket_role(&mut self, role: Option) { self.runtime_ticket_role = role; } @@ -4788,15 +4781,6 @@ impl Worker { self.pending_notifies.clone() } - /// Parent callback socket set by `from_manifest_spawned`. - /// - /// Consumed by the Controller to fire `Method::WorkerEvent` upward on - /// lifecycle transitions. `None` for top-level Workers, in which case - /// the Controller silently skips the send. - pub fn callback_socket(&self) -> Option<&PathBuf> { - self.callback_socket.as_ref() - } - pub(crate) fn install_compaction_budget( &mut self, budget: Option, @@ -8304,14 +8288,10 @@ where // Register this Worker in the machine-wide worker-allocation // before building anything else, so a spawn that conflicts on // scope fails fast. - let socket_path = dir::default_base() - .map_err(ScopeLockError::from)? - .join(&manifest.worker.name) - .join("sock"); let scope_allocation = worker_allocation::install_top_level( manifest.worker.name.clone(), std::process::id(), - socket_path, + PathBuf::new(), // Reserved legacy allocation field; no Worker endpoint exists. common.scope.allow_rules(), segment_id, )?; @@ -8372,7 +8352,6 @@ where pending_committed_history: Arc::new(Mutex::new(std::collections::VecDeque::new())), failed_activation_authority: Mutex::new(None), scope_allocation: Some(scope_allocation), - callback_socket: None, runtime_ticket_role: None, prompts: common.prompts, inject_resident_summary: true, @@ -8773,7 +8752,6 @@ where pending_committed_history: Arc::new(Mutex::new(std::collections::VecDeque::new())), failed_activation_authority: Mutex::new(None), scope_allocation: None, - callback_socket: None, runtime_ticket_role: None, prompts: common.prompts, inject_resident_summary: true, @@ -8796,141 +8774,6 @@ where Ok(worker) } - /// Build a Worker spawned by another Worker (sibling process). - /// - /// Behaves like [`Worker::from_manifest`] but claims the scope - /// allocation that the spawner pre-registered via - /// [`worker_allocation::delegate_scope`], rather than installing a new - /// top-level entry. `callback_socket` carries the spawner's - /// Unix-socket path so the spawned Worker can send `Method::Notify` - /// back to the spawner. - pub async fn from_manifest_spawned( - manifest: WorkerManifest, - store: St, - loader: PromptCatalogSource, - callback_socket: PathBuf, - ) -> Result { - let cwd = current_cwd()?; - let authority = WorkerFilesystemAuthority::local(cwd.clone(), cwd.clone()); - let workspace_context = WorkerWorkspaceContext::local_filesystem(None); - Self::from_manifest_spawned_with_context( - manifest, - store, - loader, - callback_socket, - workspace_context, - authority, - ) - .await - } - - pub async fn from_manifest_spawned_with_context( - manifest: WorkerManifest, - store: St, - loader: PromptCatalogSource, - callback_socket: PathBuf, - workspace_context: WorkerWorkspaceContext, - filesystem_authority: WorkerFilesystemAuthority, - ) -> Result { - let common = prepare_worker_common_with_context( - &manifest, - &loader, - /* parse_template */ true, - workspace_context, - filesystem_authority, - manifest.scope.clone(), - )?; - - // A spawned child starts its own conversation, so it mints a - // fresh Session rather than joining the spawner's. - let session_id = session_store::new_session_id(); - let segment_id = session_store::new_segment_id(); - let scope_allocation = worker_allocation::adopt_allocation( - manifest.worker.name.clone(), - std::process::id(), - segment_id, - )?; - - let mut worker = - Engine::, Mutable, SessionHistoryMetadata>::new_annotated( - common.client, - ); - apply_worker_manifest(&mut worker, &manifest.engine); - worker.set_cache_key(Some(segment_id.to_string())); - let worker_metadata_writer = Some(worker_metadata_writer_for_store(&store)); - let worker_metadata_segment_cas = Some(worker_metadata_segment_cas_for_store(&store)); - let scope = SharedScope::new(common.scope); - let workdir_sessions = - workdir_sessions_from_authority(&common.filesystem_authority, &scope); - - let mut worker = Self { - backend_job: std::sync::OnceLock::new(), - manifest, - engine: Some(worker), - session: WorkerSession::new(session_id, Vec::new()), - last_run_interrupted: false, - store, - worker_metadata_writer, - worker_metadata_segment_cas, - segment_state: SegmentState::new(session_id, segment_id, 0), - filesystem_authority: common.filesystem_authority, - workdir_sessions, - workspace_context: common.workspace_context, - flow_runtime_state: Arc::new(Mutex::new(None)), - flow_feature_enabled: false, - scope, - hook_builder: HookRegistryBuilder::new(), - hook_registry: None, - feature_background_tasks: FeatureBackgroundTaskRegistry::default(), - feature_invocations: FeatureInvocationRegistry::default(), - manifest_lifecycle_features_enabled: true, - interceptor_installed: false, - compaction_service_budget: None, - compact_state: None, - usage_tracker: Arc::new(UsageTracker::new()), - metrics_tracker: Arc::new(crate::compact::metrics_tracker::MetricsTracker::new()), - usage_history: Arc::new(Mutex::new(Vec::new())), - tracker: None, - task_feature: TaskFeature::new(), - worker_observation_provider: None, - system_prompt_template: common.system_prompt_template, - feature_instructions: common.feature_instructions, - alerter: None, - working_event_tx: None, - internal_worker_registry: None, - pending_compaction_cleanup: Mutex::new(None), - in_flight: None, - ai_activity_counter: Arc::new(AtomicUsize::new(0)), - pending_notifies: NotifyBuffer::new(), - notification_coalesce_delay: DEFAULT_NOTIFICATION_COALESCE_DELAY, - pending_attachments: Arc::new(Mutex::new(Vec::::new())), - pending_committed_history: Arc::new(Mutex::new(std::collections::VecDeque::new())), - failed_activation_authority: Mutex::new(None), - scope_allocation: Some(scope_allocation), - callback_socket: Some(callback_socket), - runtime_ticket_role: None, - prompts: common.prompts, - inject_resident_summary: true, - feature_resident_summary_source: None, - session_context_source: None, - prepared_initial_seed: None, - prepared_initial_prompt_completion: None, - subjektiv_host: None, - feature_installation: crate::feature::FeatureInstallation::Activate, - restore_feature_lifecycle_pending: false, - feature_system_prompt_override: None, - user_segments: Vec::new(), - sink: SegmentLogSink::new(), - history_persistence_wired: false, - log_writer: None, - notification_committer: None, - }; - worker.apply_permissions_from_manifest(); - worker.apply_prune_from_manifest(); - worker.write_worker_metadata_pending()?; - Ok(worker) - } - /// Restore a Worker by resolving its name-keyed metadata to an active /// `(SessionId, SegmentId)` and then using the normal session-log restore /// path. The metadata stores only the active pointer; lineage and origin @@ -9215,14 +9058,10 @@ where // the lookup + install inside a single `LockFileGuard` is what // makes "no two live Workers write to the same session log" // actually structural rather than a hopeful pre-check. - let socket_path = dir::default_base() - .map_err(ScopeLockError::from)? - .join(&manifest.worker.name) - .join("sock"); let scope_allocation = worker_allocation::install_top_level_with_deny( manifest.worker.name.clone(), std::process::id(), - socket_path, + PathBuf::new(), // Reserved legacy allocation field; no Worker endpoint exists. common.scope.allow_rules(), common.scope.deny_rules(), segment_id, @@ -9320,7 +9159,6 @@ where pending_committed_history: Arc::new(Mutex::new(std::collections::VecDeque::new())), failed_activation_authority: Mutex::new(None), scope_allocation: Some(scope_allocation), - callback_socket: None, runtime_ticket_role: None, prompts: common.prompts, inject_resident_summary: true, @@ -10416,6 +10254,8 @@ struct WorkerCommon { feature_instructions: Vec, } +/// Migration-only safety probe for retained process-child metadata. +/// New Internal SubWorkers never publish socket metadata or use this path. async fn restored_child_reachable(child: &WorkerSpawnedChild) -> bool { tokio::time::timeout( RESTORE_RECONCILIATION_REACHABILITY_TIMEOUT, @@ -17962,7 +17802,6 @@ permission = "write" WorkerWorkspaceContext::no_workspace(), WorkerFilesystemAuthority::None, layout, - crate::WorkerControllerTransport::InProcess, ) .with_model_client(NoopClient) .with_subjektiv_host(host) @@ -18282,7 +18121,6 @@ permission = "write" .join("runtime/runs") .join(uuid::Uuid::now_v7().to_string()), }, - crate::WorkerControllerTransport::InProcess, ) .with_model_client(NoopClient) } diff --git a/crates/worker/tests/controller_test.rs b/crates/worker/tests/controller_test.rs index 52ebdc62c..afc2450d3 100644 --- a/crates/worker/tests/controller_test.rs +++ b/crates/worker/tests/controller_test.rs @@ -1738,42 +1738,24 @@ async fn snapshot_includes_user_input_for_in_flight_turn() { // The Running event is the in-flight visibility fence: the committed // annotated input must already be available to an immediately attaching // subscriber rather than racing behind this status transition. - let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path()) - .await - .unwrap(); - let (reader, _writer) = stream.into_split(); - let mut reader = protocol::stream::JsonLineReader::new(reader); - - loop { - let event = reader.next::().await.unwrap().unwrap(); - match event { - Event::Snapshot { session, .. } => { - let found = session.entries.iter().any(|entry| match &entry.data { - protocol::SessionSnapshotEntryData::UserInput { segments } => { - protocol::Segment::flatten_to_text(segments) == "hello in-flight" - } - protocol::SessionSnapshotEntryData::Message { - role: protocol::SessionMessageRole::User, - content, - } => content.iter().any(|part| { - matches!( - part, - protocol::SessionContentPart::Text { text } - if text == "hello in-flight" - ) - }), - _ => false, - }); - assert!( - found, - "snapshot must carry the in-flight UserInput entry: {session:?}" - ); - return; + let Event::Snapshot { session, .. } = handle.snapshot_event() else { + panic!("expected snapshot"); + }; + assert!( + session.entries.iter().any(|entry| match &entry.data { + protocol::SessionSnapshotEntryData::UserInput { segments } => { + protocol::Segment::flatten_to_text(segments) == "hello in-flight" } - Event::Alert(_) => continue, - other => panic!("expected Snapshot first, got {other:?}"), - } - } + protocol::SessionSnapshotEntryData::Message { + role: protocol::SessionMessageRole::User, + content, + } => content.iter().any(|part| matches!(part, + protocol::SessionContentPart::Text { text } if text == "hello in-flight" + )), + _ => false, + }), + "snapshot must carry the in-flight UserInput entry: {session:?}" + ); } #[tokio::test] @@ -1855,24 +1837,10 @@ async fn attach_snapshot_includes_current_status() { .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; - let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path()) - .await - .unwrap(); - let (reader, _writer) = stream.into_split(); - let mut reader = protocol::stream::JsonLineReader::new(reader); - - // First event after connect is the snapshot — it carries the current status. - loop { - let event = reader.next::().await.unwrap().unwrap(); - match event { - Event::Snapshot { state, .. } => { - assert_eq!(state.catalog_status(), WorkerStatus::Running); - return; - } - Event::Alert(_) => continue, - other => panic!("expected Snapshot, got {other:?}"), - } - } + let Event::Snapshot { state, .. } = handle.snapshot_event() else { + panic!("expected snapshot"); + }; + assert_eq!(state.catalog_status(), WorkerStatus::Running); } #[tokio::test] @@ -2928,7 +2896,7 @@ async fn status_json_reflects_worker_name() { } // --------------------------------------------------------------------------- -// Socket transport tests +// Controller lifecycle tests // --------------------------------------------------------------------------- #[tokio::test] @@ -2966,283 +2934,148 @@ async fn shutdown_closes_method_admission_before_terminal_confirmation() { shutdown_rx.await.unwrap(); } +/// Uses the real Controller and metadata store; only the model client is replaced. #[tokio::test] -async fn shutdown_joins_socket_server_with_active_connection() { - use tokio::net::UnixStream; - +async fn register_peer_returns_existing_response_and_persists_reciprocal_metadata() { + use session_store::{WorkerMetadata, WorkerMetadataStore}; let worker = make_worker(MockClient::new(simple_text_events())).await; - let runtime_base = tempfile::tempdir().unwrap(); - let (handle, shutdown_rx) = WorkerController::spawn(worker, runtime_base.path()) - .await + let store = worker.store().clone(); + store + .write(&WorkerMetadata::new("test-worker", None)) .unwrap(); - let socket_path = handle.runtime_dir.socket_path(); - let _connection = UnixStream::connect(&socket_path).await.unwrap(); - + store + .write(&WorkerMetadata::new("peer-worker", None)) + .unwrap(); + let handle = spawn_controller(worker).await; + let mut events = handle.subscribe(); handle - .send(Method::Shutdown { - command: worker_command(&handle), + .send(Method::RegisterPeer { + name: "peer-worker".into(), }) .await .unwrap(); - tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_rx) + assert!( + drain_until( + &mut events, + std::time::Duration::from_secs(2), + |event| matches!( + event, + Event::PeerRegistered { result } if result == &serde_json::json!({ + "source": "test-worker", "peer": "peer-worker", + }) + ) + ) .await - .expect("controller should join its socket tasks") - .expect("controller shutdown signal should remain open"); - assert!(!socket_path.exists()); + ); + assert_eq!( + store.read_by_name("test-worker").unwrap().unwrap().peers[0].worker_name, + "peer-worker" + ); + assert_eq!( + store.read_by_name("peer-worker").unwrap().unwrap().peers[0].worker_name, + "test-worker" + ); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); } #[tokio::test] -async fn socket_run_receives_events() { - use protocol::stream::{JsonLineReader, JsonLineWriter}; - use tokio::net::UnixStream; - - let client = MockClient::new(simple_text_events()); - let worker = make_worker(client).await; +async fn register_peer_missing_target_is_rejected_without_starting_a_worker() { + use session_store::{WorkerMetadata, WorkerMetadataStore}; + let worker = make_worker(MockClient::new(simple_text_events())).await; + let store = worker.store().clone(); + store + .write(&WorkerMetadata::new("test-worker", None)) + .unwrap(); let handle = spawn_controller(worker).await; - - // Give the socket server a moment to bind - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - let sock_path = handle.runtime_dir.socket_path(); - let stream = UnixStream::connect(&sock_path).await.unwrap(); - let (reader, writer) = stream.into_split(); - let mut reader = JsonLineReader::new(reader); - let mut writer = JsonLineWriter::new(writer); - - // Send run method via socket - writer - .write(&Method::submit_text( - protocol::new_submission_request_id(), - "Hello", - )) + let mut events = handle.subscribe(); + handle + .send(Method::RegisterPeer { + name: "missing-peer".into(), + }) .await .unwrap(); - - // Collect events - let mut saw_turn_start = false; - let mut saw_text_delta = false; - let mut saw_turn_end = false; - - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); - loop { - tokio::select! { - event = reader.next::() => { - match event { - Ok(Some(Event::TurnStart { .. })) => saw_turn_start = true, - Ok(Some(Event::TextDelta { .. })) => saw_text_delta = true, - Ok(Some(Event::TurnEnd { .. })) => { - saw_turn_end = true; - break; - } - Ok(None) | Err(_) => break, - _ => {} - } - } - _ = tokio::time::sleep_until(deadline) => break, - } - } - - assert!(saw_turn_start, "should see turn_start via socket"); - assert!(saw_text_delta, "should see text_delta via socket"); - assert!(saw_turn_end, "should see turn_end via socket"); -} - -#[tokio::test] -async fn socket_worker_event_turn_ended_while_idle_auto_starts_turn() { - use protocol::stream::{JsonLineReader, JsonLineWriter}; - use tokio::net::UnixStream; - - let client = MockClient::new(simple_text_events()); - let worker = make_worker(client).await; - let handle = spawn_controller(worker).await; - - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - let sock_path = handle.runtime_dir.socket_path(); - let stream = UnixStream::connect(&sock_path).await.unwrap(); - let (reader, writer) = stream.into_split(); - let mut reader = JsonLineReader::new(reader); - let mut writer = JsonLineWriter::new(writer); - - writer - .write(&Method::WorkerEvent(protocol::WorkerEvent::TurnEnded { - worker_name: "child".into(), - })) + assert!( + drain_until( + &mut events, + std::time::Duration::from_secs(2), + |event| matches!( + event, + Event::Error { code: worker::ErrorCode::InvalidRequest, message } + if message.contains("missing-peer") && message.contains("does not exist") + ) + ) .await + ); + assert!(store.read_by_name("missing-peer").unwrap().is_none()); + assert!( + store + .read_by_name("test-worker") + .unwrap() + .unwrap() + .peers + .is_empty() + ); +} + +#[tokio::test] +async fn legacy_process_discovery_methods_require_host_or_runtime_authority() { + let worker = make_worker(MockClient::new(simple_text_events())).await; + let handle = spawn_controller(worker).await; + let mut events = handle.subscribe(); + for method in [ + Method::ListWorkers, + Method::RestoreWorker { + name: "old-worker".into(), + }, + ] { + handle.send(method).await.unwrap(); + assert!( + drain_until( + &mut events, + std::time::Duration::from_secs(2), + |event| matches!( + event, + Event::Error { code: worker::ErrorCode::InvalidRequest, message } + if message.contains("Host or Runtime") + ) + ) + .await + ); + } + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); +} + +#[tokio::test] +async fn all_controller_launch_modes_leave_external_transport_to_the_host() { + let root = tempfile::tempdir().unwrap(); + for mode in ["direct", "runtime", "run"] { + let worker = make_worker(MockClient::new(simple_text_events())).await; + let base = root.path().join(mode); + let (handle, shutdown) = match mode { + "direct" => WorkerController::spawn(worker, &base).await, + "runtime" => WorkerController::spawn_runtime_managed(worker, &base).await, + "run" => { + WorkerController::spawn_runtime_managed_run(worker, &base.join("runs/1")).await + } + _ => unreachable!(), + } .unwrap(); - - let mut saw_worker_event_echo = false; - let mut saw_turn_start = false; - let mut saw_turn_end = false; - - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); - // The SystemItem and TurnEnd events arrive through independent - // broadcast lanes (sink fan-out vs `working_event_tx`), so their relative - // order on the wire is non-deterministic. Keep reading until both - // are observed (or the deadline trips), rather than breaking on - // the first TurnEnd. - loop { - if saw_worker_event_echo && saw_turn_end { - break; - } - tokio::select! { - event = reader.next::() => { - match event { - Ok(Some(Event::SystemItem { ref item, .. })) - if item.get("kind").and_then(|k| k.as_str()) == Some("worker_event") - && item - .pointer("/presentation/event/worker_name") - .and_then(|v| v.as_str()) == Some("child") => - { - saw_worker_event_echo = true; - } - Ok(Some(Event::TurnStart { .. })) => saw_turn_start = true, - Ok(Some(Event::TurnEnd { .. })) => { - saw_turn_end = true; - } - Ok(None) | Err(_) => break, - _ => {} - } - } - _ = tokio::time::sleep_until(deadline) => break, - } - } - - assert!( - saw_worker_event_echo, - "WorkerEvent::TurnEnded via socket should be echoed as Event::SystemItem(WorkerEvent)" - ); - assert!( - saw_turn_start, - "WorkerEvent::TurnEnded via socket should auto-start a turn" - ); - assert!( - saw_turn_end, - "auto-triggered turn should reach turn_end via socket" - ); -} - -async fn socket_error_after_method_line( - handle: &WorkerHandle, - line: &[u8], -) -> (worker::ErrorCode, String) { - use protocol::stream::JsonLineReader; - use tokio::io::AsyncWriteExt; - use tokio::net::UnixStream; - - let sock_path = handle.runtime_dir.socket_path(); - let stream = UnixStream::connect(&sock_path).await.unwrap(); - let (reader, mut writer) = stream.into_split(); - let mut reader = JsonLineReader::new(reader); - - writer.write_all(line).await.unwrap(); - - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1); - loop { - tokio::select! { - event = reader.next::() => { - match event { - Ok(Some(Event::Error { code, message })) => return (code, message), - Ok(Some(_)) => {} - Ok(None) => panic!("socket closed before invalid-method error"), - Err(e) => panic!("socket read failed before invalid-method error: {e}"), - } - } - _ = tokio::time::sleep_until(deadline) => { - panic!("timed out waiting for invalid-method error") - } - } - } -} - -#[tokio::test] -async fn socket_schema_invalid_method_returns_error() { - let client = MockClient::new(simple_text_events()); - let worker = make_worker(client).await; - let handle = spawn_controller(worker).await; - - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - let (code, message) = socket_error_after_method_line(&handle, b"{\"bad\":\"json\"}\n").await; - - assert_eq!(code, worker::ErrorCode::InvalidRequest); - assert!( - message.contains("invalid method"), - "expected invalid-method diagnostic, got: {message}" - ); -} - -#[tokio::test] -async fn socket_malformed_method_returns_error() { - let client = MockClient::new(simple_text_events()); - let worker = make_worker(client).await; - let handle = spawn_controller(worker).await; - - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - let (code, message) = socket_error_after_method_line(&handle, b"{not-json}\n").await; - - assert_eq!(code, worker::ErrorCode::InvalidRequest); - assert!( - message.contains("invalid method"), - "expected invalid-method diagnostic, got: {message}" - ); -} - -#[tokio::test] -async fn socket_peer_close_without_method_does_not_broadcast_error() { - use protocol::stream::JsonLineReader; - use tokio::net::UnixStream; - - let client = MockClient::new(simple_text_events()); - let worker = make_worker(client).await; - let handle = spawn_controller(worker).await; - - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - let mut broadcast_rx = handle.subscribe(); - let sock_path = handle.runtime_dir.socket_path(); - let stream = UnixStream::connect(&sock_path).await.unwrap(); - let (reader, writer) = stream.into_split(); - let mut reader = JsonLineReader::new(reader); - - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1); - loop { - tokio::select! { - event = reader.next::() => { - match event { - Ok(Some(Event::Snapshot { .. })) => break, - Ok(Some(_)) => {} - Ok(None) => panic!("socket closed before connect-time snapshot"), - Err(e) => panic!("socket read failed before connect-time snapshot: {e}"), - } - } - _ = tokio::time::sleep_until(deadline) => { - panic!("timed out waiting for connect-time snapshot") - } - } - } - - drop(writer); - drop(reader); - - let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(200); - loop { - tokio::select! { - event = broadcast_rx.recv() => { - match event { - Ok(Event::Error { code, message }) => { - panic!("peer close without Method broadcast error {code:?}: {message}") - } - Ok(_) => {} - Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { - panic!("broadcast receiver lagged while checking peer close: {n}") - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => break, - } - } - _ = tokio::time::sleep_until(deadline) => break, + for legacy in ["pid", "status.json", "manifest.toml", "sock", "worker.sock"] { + assert!( + !handle.runtime_dir.path().join(legacy).exists(), + "{mode} created {legacy}" + ); } + handle + .send(Method::Shutdown { + command: worker_command(&handle), + }) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), shutdown) + .await + .unwrap() + .unwrap(); } } @@ -4006,27 +3839,15 @@ fn item_text_contains(item: &Item, needle: &str) -> bool { } async fn snapshot_contains_user_input(handle: &WorkerHandle, needle: &str) -> bool { - let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path()) - .await - .unwrap(); - let (reader, _writer) = stream.into_split(); - let mut reader = protocol::stream::JsonLineReader::new(reader); - - loop { - let event = reader.next::().await.unwrap().unwrap(); - match event { - Event::Snapshot { session, .. } => { - return session.entries.into_iter().any(|entry| match entry.data { - protocol::SessionSnapshotEntryData::UserInput { segments } => { - protocol::Segment::flatten_to_text(&segments).contains(needle) - } - _ => false, - }); - } - Event::Alert(_) => continue, - other => panic!("expected Snapshot first, got {other:?}"), + let Event::Snapshot { session, .. } = handle.snapshot_event() else { + panic!("expected snapshot"); + }; + session.entries.into_iter().any(|entry| match entry.data { + protocol::SessionSnapshotEntryData::UserInput { segments } => { + protocol::Segment::flatten_to_text(&segments).contains(needle) } - } + _ => false, + }) } #[tokio::test] diff --git a/crates/worker/tests/worker_events_test.rs b/crates/worker/tests/worker_events_test.rs index 8e6e7c734..cce0893f2 100644 --- a/crates/worker/tests/worker_events_test.rs +++ b/crates/worker/tests/worker_events_test.rs @@ -1,15 +1,10 @@ -//! Legacy process callback events are diagnostics only after Internal SubWorker migration. +//! Agent-visible Internal SubWorker event summaries. -use std::sync::Arc; - -use protocol::{Permission, ScopeRule, WorkerEvent}; -use tempfile::TempDir; -use worker::ipc::event::{apply_event_side_effects, render_event}; -use worker::runtime::dir::RuntimeDir; -use worker::spawn::registry::SpawnedWorkerRegistry; +use protocol::WorkerEvent; +use worker::ipc::event::render_event; #[test] -fn render_event_keeps_bounded_legacy_diagnostics() { +fn render_event_identifies_child_and_failure() { let rendered = render_event(&WorkerEvent::Errored { worker_name: "legacy-child".into(), message: "boom".into(), @@ -17,30 +12,3 @@ fn render_event_keeps_bounded_legacy_diagnostics() { assert!(rendered.contains("legacy-child")); assert!(rendered.contains("boom")); } - -#[tokio::test] -async fn legacy_callback_cannot_register_process_subworker_authority() { - let runtime_base = TempDir::new().unwrap(); - let runtime_dir = Arc::new( - RuntimeDir::create(runtime_base.path(), "parent") - .await - .unwrap(), - ); - let registry = SpawnedWorkerRegistry::new(runtime_dir.clone()); - let scope_root = TempDir::new().unwrap(); - let event = WorkerEvent::ScopeSubDelegated { - parent_worker: "legacy-parent".into(), - sub_worker: "legacy-child".into(), - sub_socket: "/tmp/legacy-child.sock".into(), - scope: vec![ScopeRule { - target: scope_root.path().to_path_buf(), - permission: Permission::Write, - recursive: true, - symlink_policy: Default::default(), - }], - }; - - apply_event_side_effects(&event, ®istry, "parent", &None).await; - - assert!(!runtime_dir.path().join("spawned_workers.json").exists()); -} diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index f9377801c..9461b46ea 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -4169,7 +4169,6 @@ impl WorkspaceApi { .with_runtime_request_source(&embedded_identity, embedded_request_audience), ) .with_runtime_store_dir(config.embedded_runtime_store_root.clone()) - .with_controller_transport(worker::WorkerControllerTransport::InProcess) .with_resource_client(Arc::new(resource_broker.clone())), ) .map_err(|err| { diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs index ae622fe8b..101afdd5e 100644 --- a/crates/workspace-server/src/server_assignment_attention_tests.rs +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -467,7 +467,6 @@ impl AttentionWorkerFactory { worker::WorkerBootstrapLayout::RuntimeManagedRun { run_dir: root.join(uuid::Uuid::now_v7().to_string()), }, - worker::WorkerControllerTransport::InProcess, ) .start() .await diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs index af1948942..4cdf7c351 100644 --- a/crates/workspace-server/src/server_assignment_delivery_tests.rs +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -800,7 +800,6 @@ impl DeliveryWorkerFactory { worker::WorkerBootstrapLayout::RuntimeManagedRun { run_dir: root.join(uuid::Uuid::now_v7().to_string()), }, - worker::WorkerControllerTransport::InProcess, ) .with_model_client(self.client.clone()); let prepared = if restore { diff --git a/crates/yoi/Cargo.toml b/crates/yoi/Cargo.toml index b951864da..37349b4af 100644 --- a/crates/yoi/Cargo.toml +++ b/crates/yoi/Cargo.toml @@ -15,7 +15,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock"] } client = { workspace = true } futures = { workspace = true } manifest = { workspace = true } -worker = { workspace = true } +protocol = { workspace = true } server-api = { workspace = true } workdir = { workspace = true } tokio-tungstenite = { workspace = true } diff --git a/crates/yoi/src/cli_connection.rs b/crates/yoi/src/cli_connection.rs index 573eeafd9..109dd8aa2 100644 --- a/crates/yoi/src/cli_connection.rs +++ b/crates/yoi/src/cli_connection.rs @@ -25,7 +25,7 @@ pub(crate) enum CliCommand { Panel, Keys, SetupModel, - WorkerRuntime, + Serve, WorkerCleanup, Ticket, Objective, @@ -45,7 +45,7 @@ impl CliCommand { CliCommand::Panel => "yoi panel", CliCommand::Keys => "yoi keys", CliCommand::SetupModel => "yoi setup-model", - CliCommand::WorkerRuntime => "yoi worker", + CliCommand::Serve => "yoi --local serve", CliCommand::WorkerCleanup => "yoi worker management", CliCommand::Ticket => "yoi ticket", CliCommand::Objective => "yoi objective", @@ -68,7 +68,7 @@ impl CliCommand { CliCommand::Login => CliConnectionRequirement::BackendOnly, CliCommand::Keys | CliCommand::SetupModel - | CliCommand::WorkerRuntime + | CliCommand::Serve | CliCommand::WorkerCleanup | CliCommand::Plugin | CliCommand::Mcp @@ -325,7 +325,7 @@ mod tests { for command in [ CliCommand::Keys, CliCommand::SetupModel, - CliCommand::WorkerRuntime, + CliCommand::Serve, CliCommand::WorkerCleanup, CliCommand::Plugin, CliCommand::Mcp, diff --git a/crates/yoi/src/local_cli.rs b/crates/yoi/src/local_cli.rs new file mode 100644 index 000000000..b4af7a53a --- /dev/null +++ b/crates/yoi/src/local_cli.rs @@ -0,0 +1,273 @@ +//! Explicit Standalone socket commands; launch and transport remain Host-owned. + +use std::path::PathBuf; + +use super::{ParseError, required_option_value}; + +pub(crate) const SERVE_HELP: &str = r#"yoi --local serve + +Usage: + yoi --local serve --socket [--profile ] [--workspace ] + yoi --local serve --socket --worker-id + +Run a foreground Standalone Host without a TUI. New Workers use the same Profile +resolution and saved Worker store as `yoi --local`. --worker-id restores a saved +Standalone Worker with its saved Profile and working directory; it is not a Runtime id. + +Options: + --socket Host-owned Unix endpoint (required) + --profile Profile for a new Worker; conflicts with --worker-id + --workspace Working directory for a new Worker (defaults to cwd) + --worker-id Restore a saved Standalone Worker + -h, --help Print help + +Connect with `yoi --local attach --socket `. The endpoint parent must already +be a private directory owned by this Unix user (mode 0700). Existing endpoints are +never replaced or auto-removed as stale; only same-uid clients are admitted. +Client EOF/disconnect leaves this Host running. Send protocol Method::Shutdown to +explicitly stop the Host, or use SIGINT/SIGTERM for confirmed Host shutdown. +"#; + +pub(crate) const ATTACH_HELP: &str = r#"yoi --local attach + +Usage: + yoi --local attach --socket + +Connect stdin/stdout to an existing Standalone Host: one protocol Method JSON +object per stdin line, one protocol Event JSON object per stdout line (JSONL). +EOF disconnects this client without stopping the Host. Method::Shutdown explicitly +stops the Host. No TUI, Worker spawn, or Backend/Runtime selection is performed. + +Options: + --socket Host-owned Unix endpoint (required) + -h, --help Print help +"#; + +#[derive(Debug)] +pub(crate) enum LocalCli { + Help(&'static str), + Serve(ServeOptions), + Attach(PathBuf), +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct ServeOptions { + pub endpoint: PathBuf, + pub workspace: Option, + pub profile: Option, + pub restore: Option, +} + +pub(crate) fn parse(command: &str, args: &[String]) -> Result { + let serve = command == "serve"; + let mut endpoint = None; + let mut workspace = None; + let mut profile = None; + let mut restore = None; + let mut i = 0; + while i < args.len() { + let arg = &args[i]; + if matches!(arg.as_str(), "--help" | "-h") { + return Ok(LocalCli::Help(if serve { SERVE_HELP } else { ATTACH_HELP })); + } + let (option, inline) = arg + .split_once('=') + .map_or((arg.as_str(), None), |(key, value)| (key, Some(value))); + if !matches!(option, "--socket") + && !(serve && matches!(option, "--workspace" | "--profile" | "--worker-id")) + { + return Err(ParseError(format!("unknown yoi {command} option `{arg}`"))); + } + let value = if let Some(value) = inline { + if value.is_empty() { + return Err(ParseError(format!("{option} requires a value"))); + } + i += 1; + value + } else { + let value = required_option_value(args, i, option)?; + i += 2; + value + }; + match option { + "--socket" => set_once(&mut endpoint, PathBuf::from(value), option)?, + "--workspace" => set_once(&mut workspace, PathBuf::from(value), option)?, + "--profile" => set_once(&mut profile, value.to_owned(), option)?, + "--worker-id" => { + let id = value.parse().map_err(|_| { + ParseError("--worker-id requires a Standalone Worker UUID".to_owned()) + })?; + set_once(&mut restore, id, option)?; + } + _ => unreachable!("validated option"), + } + } + let endpoint = endpoint + .ok_or_else(|| ParseError(format!("yoi --local {command} requires --socket ")))?; + if !serve { + return Ok(LocalCli::Attach(endpoint)); + } + if restore.is_some() && (profile.is_some() || workspace.is_some()) { + return Err(ParseError( + "--worker-id restores the saved Profile and cwd; it cannot be combined with --profile or --workspace" + .to_owned(), + )); + } + Ok(LocalCli::Serve(ServeOptions { + endpoint, + workspace, + profile, + restore, + })) +} + +fn set_once(slot: &mut Option, value: T, option: &str) -> Result<(), ParseError> { + if slot.is_some() { + return Err(ParseError(format!("{option} may only be provided once"))); + } + *slot = Some(value); + Ok(()) +} + +pub(crate) async fn run_serve( + options: ServeOptions, + state_dir: PathBuf, +) -> Result<(), Box> { + let launch = if let Some(worker_id) = options.restore { + standalone::HeadlessLaunch::Restore { + state_dir, + worker_id, + } + } else { + let workspace = options.workspace.map_or_else(std::env::current_dir, Ok)?; + let profile = options + .profile + .map_or(manifest::ProfileSelector::Default, |value| { + manifest::ProfileSelector::parse_cli(&value) + }); + let launch = + standalone::StandaloneLaunchConfig::new(workspace, state_dir, profile, "local") + .resolve()?; + standalone::HeadlessLaunch::Fresh(launch) + }; + standalone::run_headless(launch, options.endpoint).await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse_strings(command: &str, args: &[&str]) -> Result { + parse( + command, + &args.iter().map(|arg| (*arg).to_owned()).collect::>(), + ) + } + + #[test] + fn serve_parses_fresh_launch_and_saved_worker_restore() { + let LocalCli::Serve(options) = parse_strings( + "serve", + &[ + "--socket", + "/tmp/host.sock", + "--profile=builtin:standalone", + "--workspace", + "/work", + ], + ) + .unwrap() else { + panic!("expected serve") + }; + assert_eq!(options.endpoint, PathBuf::from("/tmp/host.sock")); + assert_eq!(options.workspace, Some(PathBuf::from("/work"))); + assert_eq!(options.profile.as_deref(), Some("builtin:standalone")); + assert_eq!(options.restore, None); + + let id = "01900000-0000-7000-8000-000000000001"; + let LocalCli::Serve(options) = + parse_strings("serve", &["--socket=/tmp/host.sock", "--worker-id", id]).unwrap() + else { + panic!("expected serve") + }; + assert_eq!(options.restore.unwrap().to_string(), id); + } + + #[test] + fn serve_rejects_invalid_restore_or_launch_overrides() { + for args in [ + vec!["--worker-id", "not-a-uuid"], + vec![ + "--worker-id", + "01900000-0000-7000-8000-000000000001", + "--profile", + "standalone", + ], + vec![ + "--worker-id", + "01900000-0000-7000-8000-000000000001", + "--workspace", + "/work", + ], + vec!["--runtime-id", "runtime"], + vec!["--backend=http://backend.example"], + vec!["--subject", "subject"], + ] { + let input = [vec!["--socket", "/tmp/host.sock"], args].concat(); + assert!(parse_strings("serve", &input).is_err(), "{input:?}"); + } + } + + #[test] + fn attach_accepts_only_endpoint_options() { + for args in [ + vec!["--socket", "/tmp/host.sock"], + vec!["--socket=/tmp/host.sock"], + ] { + let LocalCli::Attach(endpoint) = parse_strings("attach", &args).unwrap() else { + panic!("expected attach") + }; + assert_eq!(endpoint, PathBuf::from("/tmp/host.sock")); + } + for args in [ + vec!["--profile", "standalone"], + vec!["--workspace", "/work"], + vec!["--worker-id", "01900000-0000-7000-8000-000000000001"], + vec!["--backend", "http://backend.example"], + ] { + let input = [vec!["--socket", "/tmp/host.sock"], args].concat(); + assert!(parse_strings("attach", &input).is_err(), "{input:?}"); + } + } + + #[test] + fn socket_is_required_nonempty_and_unique() { + for command in ["serve", "attach"] { + for args in [ + vec![], + vec!["--socket"], + vec!["--socket="], + vec!["--socket", ""], + vec!["--socket", "--profile"], + vec!["--socket=a", "--socket", "b"], + ] { + assert!(parse_strings(command, &args).is_err(), "{command} {args:?}"); + } + } + } + + #[test] + fn help_explains_protocol_disconnect_and_explicit_shutdown() { + for (command, expected) in [("serve", SERVE_HELP), ("attach", ATTACH_HELP)] { + let LocalCli::Help(help) = parse_strings(command, &["--help"]).unwrap() else { + panic!("expected help") + }; + assert_eq!(help, expected); + assert!(help.contains("Method::Shutdown")); + assert!(help.contains("--socket ")); + } + assert!(ATTACH_HELP.contains("JSONL")); + assert!(ATTACH_HELP.contains("EOF disconnects")); + } +} diff --git a/crates/yoi/src/main.rs b/crates/yoi/src/main.rs index 5ec7513ea..12871d0ec 100644 --- a/crates/yoi/src/main.rs +++ b/crates/yoi/src/main.rs @@ -4,6 +4,7 @@ //! execution in lower-level crates. mod cli_connection; +mod local_cli; mod mcp_cli; mod objective_cli; mod plugin_cli; @@ -66,7 +67,12 @@ enum Mode { WorkspaceCreate(WorkspaceCreateOptions), WorkdirHelp, WorkdirShare(workdir_share::WorkdirShareOptions), - WorkerRuntime(Vec), + LocalHelp(&'static str), + LocalServe { + options: local_cli::ServeOptions, + state_dir: PathBuf, + }, + LocalAttach(PathBuf), Keys, SetupModel, Tui { @@ -260,7 +266,26 @@ async fn run(mode: Mode) -> ExitCode { } } } - Mode::WorkerRuntime(args) => worker::entrypoint::run_cli_from("yoi worker", args).await, + Mode::LocalHelp(help) => { + print!("{help}"); + ExitCode::SUCCESS + } + Mode::LocalServe { options, state_dir } => { + match local_cli::run_serve(options, state_dir).await { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("yoi --local serve: {error}"); + ExitCode::FAILURE + } + } + } + Mode::LocalAttach(endpoint) => match standalone::run_unix_client(endpoint).await { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("yoi --local attach: {error}"); + ExitCode::FAILURE + } + }, Mode::Keys => tui::keys::launch().await, Mode::SetupModel => tui::setup_model::launch().await, Mode::Tui { @@ -467,6 +492,33 @@ fn parse_args_slice_with_connection_resolver( "workdir" => { return parse_workdir_args(&args[1..], &target_selection); } + "serve" | "attach" => { + if target_selection.backend_url.is_some() || target_selection.workspace_id.is_some() { + return Err(ParseError(format!( + "yoi {} is local-only and does not accept Backend selectors", + args[0] + ))); + } + if !target_selection.explicit_local { + return Err(ParseError(format!( + "yoi {} requires explicit --local selection", + args[0] + ))); + } + return match local_cli::parse(&args[0], &args[1..])? { + local_cli::LocalCli::Help(help) => Ok(Mode::LocalHelp(help)), + local_cli::LocalCli::Attach(endpoint) => Ok(Mode::LocalAttach(endpoint)), + local_cli::LocalCli::Serve(options) => { + let target = + resolve_local_cli_connection(connection_resolver, CliCommand::Serve)?; + let state_dir = target + .spawn_worker() + .map_err(|error| ParseError(error.to_string()))? + .state_dir; + Ok(Mode::LocalServe { options, state_dir }) + } + }; + } "worker" => { if let Some(cli) = worker_cleanup_cli::parse_worker_management_args(&args[1..]) .map_err(|e| ParseError(e.to_string()))? @@ -475,9 +527,9 @@ fn parse_args_slice_with_connection_resolver( resolve_local_cli_connection(connection_resolver, CliCommand::WorkerCleanup)?; return Ok(Mode::WorkerCleanup(cli)); } - let _target = - resolve_local_cli_connection(connection_resolver, CliCommand::WorkerRuntime)?; - return Ok(Mode::WorkerRuntime(args[1..].to_vec())); + return Err(ParseError( + "direct Worker execution was removed; use `yoi --local serve --socket ` (legacy metadata cleanup: `yoi worker --help`)".to_owned(), + )); } "objective" => { let cli = objective_cli::parse_objective_args(&args[1..]) @@ -973,10 +1025,8 @@ fn parse_console_options( ) -> Result { let mut workspace_root = current_dir()?; let mut worker_name = None; - let mut session = None; let mut profile = None; let mut subject_id = None; - let mut socket_override = None; let mut runtime_id = None; let mut worker_id = None; let mut i = 0; @@ -1003,26 +1053,6 @@ fn parse_console_options( workspace_root = PathBuf::from(value); i += 2; } - "--session" => { - let value = args - .get(i + 1) - .ok_or_else(|| ParseError("--session requires a path".to_string()))?; - if value.starts_with('-') || value.is_empty() { - return Err(ParseError("--session requires a path".to_string())); - } - session = Some(PathBuf::from(value)); - i += 2; - } - "--socket" => { - let value = args - .get(i + 1) - .ok_or_else(|| ParseError("--socket requires a path".to_string()))?; - if value.starts_with('-') || value.is_empty() { - return Err(ParseError("--socket requires a path".to_string())); - } - socket_override = Some(PathBuf::from(value)); - i += 2; - } "--subject" => { let value = required_option_value(args, i, "--subject")?; set_subject_selector(&mut subject_id, value)?; @@ -1078,22 +1108,6 @@ fn parse_console_options( workspace_root = PathBuf::from(value); i += 1; } - arg if arg.starts_with("--session=") => { - let value = arg.trim_start_matches("--session="); - if value.is_empty() { - return Err(ParseError("--session requires a path".to_string())); - } - session = Some(PathBuf::from(value)); - i += 1; - } - arg if arg.starts_with("--socket=") => { - let value = arg.trim_start_matches("--socket="); - if value.is_empty() { - return Err(ParseError("--socket requires a path".to_string())); - } - socket_override = Some(PathBuf::from(value)); - i += 1; - } arg if arg.starts_with("--profile=") => { let value = arg.trim_start_matches("--profile="); if value.is_empty() { @@ -1138,11 +1152,7 @@ fn parse_console_options( } if subject_id.is_some() { - if session.is_some() - || socket_override.is_some() - || runtime_id.is_some() - || worker_id.is_some() - { + if runtime_id.is_some() || worker_id.is_some() { return Err(ParseError( "--subject can only be used for fresh local spawn, not attach or restore" .to_owned(), @@ -1160,29 +1170,12 @@ fn parse_console_options( "--worker-id requires --runtime-id for Runtime API attach".to_string(), )); } - if (runtime_id.is_some() || worker_id.is_some()) - && (session.is_some() - || worker_name.is_some() - || socket_override.is_some() - || profile.is_some()) + if (runtime_id.is_some() || worker_id.is_some()) && (worker_name.is_some() || profile.is_some()) { return Err(ParseError( - "Runtime API target cannot be combined with --worker, --socket, --session, or --profile".to_string(), + "Runtime API target cannot be combined with --worker or --profile".to_string(), )); } - if profile.is_some() && (session.is_some() || socket_override.is_some()) { - return Err(ParseError( - "--profile can only be used for fresh spawn".to_string(), - )); - } - if session.is_some() && socket_override.is_some() { - return Err(ParseError( - "--session cannot be combined with --socket".to_string(), - )); - } - if socket_override.is_some() && worker_name.is_none() { - return Err(ParseError("--socket requires --worker".to_string())); - } let target = if subject_id.is_some() { // A local Subject selector must never trigger Backend Workspace discovery. @@ -1215,18 +1208,6 @@ fn parse_console_options( .to_string(), )); } - if session.is_some() { - return Err(ParseError( - "--local does not accept legacy --session; use `yoi --local resume` for Standalone Worker restore" - .to_string(), - )); - } - if socket_override.is_some() { - return Err(ParseError( - "--local uses an in-process Standalone connection and does not accept --socket" - .to_string(), - )); - } } if let (Some(runtime_id), Some(worker_id)) = (runtime_id.clone(), worker_id) { @@ -1253,13 +1234,6 @@ fn parse_console_options( }); } - if target.kind() == TargetKind::Standalone && (session.is_some() || socket_override.is_some()) { - return Err(ParseError( - "Standalone does not accept legacy Worker session or socket selectors; use `yoi --local resume` for the standalone Worker store" - .to_string(), - )); - } - let mode = if target.kind() == TargetKind::Standalone { LaunchMode::Spawn { worker_name, @@ -1267,14 +1241,9 @@ fn parse_console_options( subject_id, } } else { - if worker_name.is_some() - || profile.is_some() - || session.is_some() - || socket_override.is_some() - { + if worker_name.is_some() || profile.is_some() { return Err(ParseError( - "Backend target does not accept host-local Worker, profile, session, or socket selectors" - .to_string(), + "Backend target does not accept host-local Worker or profile selectors".to_string(), )); } LaunchMode::BackendSpawn @@ -2109,7 +2078,9 @@ Host commands: yoi workdir share Interactively share a client-hosted External Workdir. keys Manage local model/API keys setup-model Configure a local model provider - worker [WORKER_OPTIONS] Run the direct Worker process entrypoint + --local serve --socket Run a headless Standalone Host (serve --help for restore) + --local attach --socket Connect a JSONL protocol client; EOF leaves Host running + worker delete/prune Safely clean up legacy Worker metadata only ticket Manage Tickets through a Backend target objective Manage Objectives through a Backend target plugin Author/check/pack explicit Plugin packages @@ -2939,39 +2910,23 @@ backend = "shared" } #[test] - fn parser_standalone_rejects_legacy_session_and_socket_inputs() { - let resolver = DefaultBackendCliConnectionResolver { - backend_url: "http://default-backend.example", - }; - let session_args = ["--local", "--session", "session-1"] - .into_iter() - .map(String::from) - .collect::>(); - let err = parse_args_slice_with_connection_resolver(&session_args, &resolver).unwrap_err(); - assert_eq!( - err.0, - "--local does not accept legacy --session; use `yoi --local resume` for Standalone Worker restore" - ); - - let socket_args = [ - "--local", - "--worker", - "worker-a", - "--socket", - "/tmp/worker.sock", - ] - .into_iter() - .map(String::from) - .collect::>(); - let err = parse_args_slice_with_connection_resolver(&socket_args, &resolver).unwrap_err(); - assert_eq!( - err.0, - "--local uses an in-process Standalone connection and does not accept --socket" - ); + fn console_rejects_removed_session_and_socket_options() { + for prefix in [vec!["--local"], vec!["--backend", "http://backend.example"]] { + for option in [ + "--session", + "--session=log", + "--socket", + "--socket=endpoint", + ] { + let args = [prefix.clone(), vec![option]].concat(); + let error = parse_args_from(args).unwrap_err(); + assert_eq!(error.to_string(), format!("unknown argument: {option}")); + } + } } #[test] - fn parse_backend_runtime_target_rejects_legacy_socket_mix() { + fn backend_runtime_target_rejects_local_launch_options() { let err = parse_args_from([ "--backend", "http://127.0.0.1:8787", @@ -2985,7 +2940,7 @@ backend = "shared" .unwrap_err(); assert_eq!( err.to_string(), - "Runtime API target cannot be combined with --worker, --socket, --session, or --profile" + "Runtime API target cannot be combined with --worker or --profile" ); } @@ -3124,15 +3079,121 @@ backend = "shared" } #[test] - fn parse_worker_subcommand_uses_runtime_mode() { - match parse_args_from(["worker", "--worker", "agent", "--profile", "default"]).unwrap() { - Mode::WorkerRuntime(args) => { - assert_eq!(args, ["--worker", "agent", "--profile", "default"]) - } - _ => panic!("expected WorkerRuntime mode"), + fn direct_worker_execution_is_rejected_without_compatibility_fallback() { + for args in [ + vec!["worker"], + vec!["worker", "--worker", "agent", "--profile", "default"], + ] { + let error = parse_args_from(args).unwrap_err(); + assert!( + error + .to_string() + .contains("direct Worker execution was removed") + ); + assert!(error.to_string().contains("yoi --local serve --socket")); } } + #[test] + fn socket_commands_require_explicit_local_and_reject_backend_selection() { + let resolver = DefaultBackendCliConnectionResolver { + backend_url: "http://backend.example", + }; + for command in ["serve", "attach"] { + for args in [ + vec![command, "--socket", "/tmp/host.sock"], + vec![ + "--backend=http://backend.example", + command, + "--socket=/tmp/host.sock", + ], + vec![ + "--local", + "--workspace-id=workspace", + command, + "--socket=/tmp/host.sock", + ], + ] { + let input = args.iter().map(|arg| (*arg).to_owned()).collect::>(); + assert!( + parse_args_slice_with_connection_resolver(&input, &resolver).is_err(), + "{input:?}" + ); + } + } + } + + #[test] + fn socket_commands_dispatch_without_backend_default_or_workspace_discovery() { + let resolver = DefaultBackendCliConnectionResolver { + backend_url: "http://backend.example", + }; + let args = [ + "--local", + "serve", + "--socket=/tmp/host.sock", + "--profile=builtin:standalone", + ] + .into_iter() + .map(String::from) + .collect::>(); + let Mode::LocalServe { options, state_dir } = + parse_args_slice_with_connection_resolver(&args, &resolver).unwrap() + else { + panic!("expected headless serve") + }; + assert_eq!(options.profile.as_deref(), Some("builtin:standalone")); + assert!(state_dir.is_absolute()); + + let args = ["--local", "attach", "--socket=/tmp/host.sock"] + .into_iter() + .map(String::from) + .collect::>(); + let Mode::LocalAttach(endpoint) = + parse_args_slice_with_connection_resolver(&args, &resolver).unwrap() + else { + panic!("expected attach") + }; + assert_eq!(endpoint, PathBuf::from("/tmp/host.sock")); + } + + #[test] + fn socket_attach_and_help_do_not_require_worker_state_resolution() { + struct UnavailableWorkerState; + impl CliConnectionResolver for UnavailableWorkerState { + fn resolve_connection( + &self, + _command: CliCommand, + _input: CliConnectionInput<'_>, + ) -> Result, ParseError> { + Err(ParseError( + "Standalone state directory is unavailable".to_owned(), + )) + } + } + let resolver = UnavailableWorkerState; + for command in ["serve", "attach"] { + let args = ["--local", command, "--help"] + .into_iter() + .map(String::from) + .collect::>(); + assert!(matches!( + parse_args_slice_with_connection_resolver(&args, &resolver).unwrap(), + Mode::LocalHelp(_) + )); + } + let args = ["--local", "attach", "--socket=/tmp/host.sock"] + .into_iter() + .map(String::from) + .collect::>(); + assert!(matches!( + parse_args_slice_with_connection_resolver(&args, &resolver).unwrap(), + Mode::LocalAttach(_) + )); + assert!(TOP_LEVEL_HELP.contains("--local serve --socket ")); + assert!(TOP_LEVEL_HELP.contains("--local attach --socket ")); + } + #[test] fn parse_worker_delete_uses_cleanup_mode() { match parse_args_from(["worker", "delete", "agent", "--dry-run"]).unwrap() { @@ -3880,36 +3941,6 @@ backend = "shared" } } - #[test] - fn parse_profile_rejects_resume_attach_modes() { - let segment_id = session_store::new_segment_id().to_string(); - let cases = [ - ( - vec![ - "--profile".to_string(), - "p.toml".to_string(), - "--session".to_string(), - segment_id, - ], - "--profile can only be used for fresh spawn", - ), - ( - vec![ - "--profile".to_string(), - "p.toml".to_string(), - "--socket".to_string(), - "/tmp/yoi/sock".to_string(), - ], - "--profile can only be used for fresh spawn", - ), - ]; - - for (args, message) in cases { - let err = parse_args_from(args).unwrap_err(); - assert_eq!(err.to_string(), message); - } - } - #[test] fn parse_dashboard_word_is_not_an_alias_or_worker_name() { let err = parse_args_from(["dashboard"]).unwrap_err(); diff --git a/crates/yoi/src/worker_cleanup_cli.rs b/crates/yoi/src/worker_cleanup_cli.rs index 54d1eddbb..69bcd3766 100644 --- a/crates/yoi/src/worker_cleanup_cli.rs +++ b/crates/yoi/src/worker_cleanup_cli.rs @@ -1,3 +1,8 @@ +//! Cleanup of legacy name-keyed Worker metadata, not Standalone execution state. +//! +//! Historical socket probes only refuse unsafe deletion; they never discover or +//! attach Workers for execution. Session logs and shared history remain untouched. + use std::fmt; use std::io; use std::path::{Path, PathBuf}; @@ -484,7 +489,7 @@ fn to_error(error: E) -> WorkerCleanupCliError { } pub fn help_text() -> &'static str { - "yoi worker\n\nUsage:\n yoi worker delete [--force] [--dry-run]\n yoi worker prune --older-than [--force] [--dry-run]\n yoi worker [WORKER_OPTIONS]\n\nDescription:\n delete/prune are safe Worker metadata cleanup commands. `worker delete` removes only name-keyed Worker metadata and never removes session logs/history. Live or uncertain Worker liveness is refused. Without --force the command reports only.\n\nDuration units: s, m, h, d, w\n\nOptions:\n --force Perform deletion after safety checks\n --dry-run Report only, even with --force\n --older-than Required explicit age threshold for prune\n -h, --help Print help\n" + "yoi worker\n\nUsage:\n yoi worker delete [--force] [--dry-run]\n yoi worker prune --older-than [--force] [--dry-run]\n\nDescription:\n delete/prune clean up legacy name-keyed Worker metadata only; they do not start or restore Workers. Use `yoi --local serve --socket ` for a headless Standalone Host. `worker delete` removes only name-keyed Worker metadata and never removes session logs/history. Live or uncertain Worker liveness is refused. Without --force the command reports only.\n\nDuration units: s, m, h, d, w\n\nOptions:\n --force Perform deletion after safety checks\n --dry-run Report only, even with --force\n --older-than Required explicit age threshold for prune\n -h, --help Print help\n" } fn delete_help_text() -> &'static str { From b860c67ea23cf141c0d232d6614db869d1f5d1fb Mon Sep 17 00:00:00 2001 From: Hare Date: Thu, 22 Oct 2026 03:46:46 +0900 Subject: [PATCH 16/46] fix: align Runtime restore diagnostics and socket test fixtures --- .../src/retention/runtime_v5_tests.rs | 6 +++-- crates/worker-runtime/src/worker_backend.rs | 26 +++++++++++++++--- .../worker-runtime/src/working_directory.rs | 27 ++++++++++--------- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/crates/worker-runtime/src/retention/runtime_v5_tests.rs b/crates/worker-runtime/src/retention/runtime_v5_tests.rs index d9a3bccdd..518a4c6fc 100644 --- a/crates/worker-runtime/src/retention/runtime_v5_tests.rs +++ b/crates/worker-runtime/src/retention/runtime_v5_tests.rs @@ -682,8 +682,10 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() { #[cfg(unix)] #[test] fn current_runtime_session_migration_still_rejects_live_worker_sockets() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().join("runtime"); + // Relative socket names keep this fixture independent of the checkout/TMPDIR + // length. The TempDir still owns the same tree; no process cwd is changed. + let temp = tempfile::tempdir_in(".").unwrap(); + let root = Path::new(temp.path().file_name().unwrap()).join("runtime"); let (worker, _, _) = runtime_fixture(&root, 12); let before = tree_bytes(&root); let runs = root.join("workers").join(worker.to_string()).join("runs"); diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 9b603ff29..9947ce5b0 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -9889,12 +9889,25 @@ mod tests { }) .await; session.close().await.unwrap(); - Ok(result.unwrap_err().to_string()) + Ok(result.unwrap_err()) }) .unwrap(); + // The String-returning opener cannot safely classify its raw cause. The + // deferred boundary must preserve its typed stage without exposing that text. + assert_eq!( + error.failure_reason(), + Some(workdir::WorkdirFailureReason::SessionOpenFailed), + "{damage}: {error:?}" + ); assert!( - error.contains("working_directory_cleanup_"), - "{damage}: {error}" + matches!(error.classification_source(), workdir::WorkdirError::Unavailable(message) + if message == "Workdir session could not be opened"), + "{damage}: {error:?}" + ); + assert_eq!( + error.denial_reason(), + None, + "{damage}: no guessed denial cause" ); assert_eq!( observed.lock().unwrap().len(), @@ -9907,6 +9920,13 @@ mod tests { "damaged checkout must be retained" ); assert!(backend.stop_worker(&request.worker_ref).is_accepted()); + println!( + "damage={damage}; reason={:?}; classification={:?}; cwd_bind_count={}; retained_files={}; retained_data_unchanged=true; stopped=true", + error.failure_reason(), + error.classification_source(), + observed.lock().unwrap().len(), + before.len(), + ); } } diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index a541ca2e1..e61fa0fee 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -3677,7 +3677,11 @@ mod tests { #[test] fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() { let repo = create_clean_repo(); - let runtime_root = tempfile::tempdir().unwrap(); + // ssh-agent and the broker receive relative socket paths in this owned + // fixture tree, so the test does not depend on checkout/TMPDIR length. + // Neither this process nor its peers change cwd. + let runtime_root = tempfile::tempdir_in(".").unwrap(); + let runtime_path = Path::new(runtime_root.path().file_name().unwrap()); let key_root = tempfile::tempdir().unwrap(); let key_path = key_root.path().join("id_ed25519"); let fallback_key_path = key_root.path().join("id_ed25519_fallback"); @@ -3689,7 +3693,7 @@ mod tests { .unwrap(); assert!(status.success()); } - let materializer = RuntimeGitMaterializer::new(runtime_root.path()); + let materializer = RuntimeGitMaterializer::new(runtime_path); let mut request = request(repo.path()); request.materialization = Some(crate::catalog::RepositoryMaterializationContext { workspace_id: "workspace-1".to_string(), @@ -3731,7 +3735,7 @@ mod tests { kind: server_api::RepositorySourceKind::Ssh, uri: "ssh://git@example.test/repo.git".to_string(), }; - let command_access = RepositoryCommandAccess::prepare(runtime_root.path(), &ssh_operation) + let command_access = RepositoryCommandAccess::prepare(runtime_path, &ssh_operation) .unwrap() .unwrap(); assert!(!command_access.root.join("identity").exists()); @@ -3873,18 +3877,17 @@ mod tests { .unwrap(); // Restore preflight must validate without consuming this one-shot SSH // delivery or creating agent/session resources needed by actual bind. - let record_before = - fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(); + let record_before = fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap(); materializer .preflight_bind_working_directory(&id, None) .unwrap(); assert_eq!( - fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(), + fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap(), record_before ); assert_eq!(materializer.list_working_directories().unwrap().len(), 1); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 0 @@ -3898,7 +3901,7 @@ mod tests { assert!(ssh_policy.contains("__repository-ssh")); assert!(!ssh_policy.contains("SSH_AUTH_SOCK")); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 1 @@ -3916,7 +3919,7 @@ mod tests { "consumed delivery expiry must not revoke a live Workdir session" ); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 1, @@ -3925,7 +3928,7 @@ mod tests { drop(binding); assert!(!ssh_command.exists()); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 0 @@ -3971,7 +3974,7 @@ mod tests { ], &read_only_command_policy, )); - let broker_socket = fs::read_dir(runtime_root.path().join(REPOSITORY_ACCESS_DIR)) + let broker_socket = fs::read_dir(runtime_path.join(REPOSITORY_ACCESS_DIR)) .unwrap() .filter_map(Result::ok) .map(|entry| entry.path().join("b.sock")) @@ -4104,7 +4107,7 @@ mod tests { .code, "working_directory_repository_access_expired" ); - let restored = RuntimeGitMaterializer::new(runtime_root.path()); + let restored = RuntimeGitMaterializer::new(runtime_path); assert_eq!( restored.bind_working_directory(&id, None).unwrap_err().code, "working_directory_remote_repository_access_required" From 8b6e3070dd4219082525b5869dabfee5764295c1 Mon Sep 17 00:00:00 2001 From: Hare Date: Thu, 22 Oct 2026 20:46:10 +0900 Subject: [PATCH 17/46] fix: align Workdir session cwd and model environment Project alias, cwd, scope and command ceilings from actual sessions. Bind initial attachment cwd once in the shared broker/router while keeping root-relative delegation and detach generations. Preserve caller coordinates across reads, searches, checked checkout and commands; keep provider authorization and symlink boundaries. Connect real SubWorkers, all filesystem Tools, scoped aliases, HTTP providers and limited Host grants in permanent regressions. Validated related normal suites (workdir http-client, tools, worker, worker-runtime, server-api typescript, fs-operation), workspace-server normal suite (1272 passed, 3 existing ignores), root check, product binary builds, rustdoc/doctests, generated contract checks, format and state-contract names. Test roots are checkout-local; Git parent discovery requires GIT_CEILING_DIRECTORIES at TMPDIR, confirmed against exact b860 baseline. --- crates/server-api/src/lib.rs | 4 + crates/tools/src/bash.rs | 4 +- crates/tools/src/edit.rs | 2 +- crates/tools/src/glob.rs | 4 +- crates/tools/src/grep.rs | 4 +- crates/tools/src/read.rs | 4 +- crates/tools/src/view_image.rs | 4 +- crates/tools/src/write.rs | 2 +- crates/tools/tests/integration.rs | 586 ++++++++++++++++ crates/workdir/src/checkout.rs | 5 +- crates/workdir/src/environment.rs | 68 ++ crates/workdir/src/environment_tests.rs | 175 +++++ crates/workdir/src/http.rs | 12 + crates/workdir/src/lib.rs | 14 + crates/workdir/src/local.rs | 46 ++ .../src/local/command_transport/tests/http.rs | 3 + crates/workdir/src/router.rs | 267 +++++++- crates/workdir/src/scope.rs | 93 ++- crates/workdir/tests/remote_denial_reasons.rs | 32 +- crates/worker-runtime/src/http_server.rs | 1 + .../tests/command_transport_tests.rs | 3 + .../src/runtime/workdir_session.rs | 4 + crates/worker-runtime/src/worker_backend.rs | 65 +- .../src/worker_backend/deferred_workdir.rs | 12 + crates/worker/src/authorized_workdir.rs | 72 +- .../authorized_workdir/tests/environment.rs | 198 ++++++ crates/worker/src/checkout_http_tests.rs | 37 ++ crates/worker/src/checkout_race_tests.rs | 13 +- .../src/feature/builtin/manage_workdir.rs | 91 ++- .../src/feature/builtin/manage_workdir/wip.rs | 2 +- crates/worker/src/job.rs | 3 +- crates/worker/src/prompt/system.rs | 95 ++- crates/worker/src/spawn/tool.rs | 2 + .../src/spawn/tool/tests/environment.rs | 629 ++++++++++++++++++ crates/worker/src/worker.rs | 46 +- .../tests/system_prompt_template_test.rs | 6 +- crates/workspace-server/src/hosts.rs | 3 + crates/workspace-server/src/server.rs | 110 ++- .../server/tests/workdir_delegation_tests.rs | 1 + 39 files changed, 2562 insertions(+), 160 deletions(-) create mode 100644 crates/workdir/src/environment.rs create mode 100644 crates/workdir/src/environment_tests.rs create mode 100644 crates/worker/src/authorized_workdir/tests/environment.rs create mode 100644 crates/worker/src/spawn/tool/tests/environment.rs diff --git a/crates/server-api/src/lib.rs b/crates/server-api/src/lib.rs index 951a21566..5efd0b0d5 100644 --- a/crates/server-api/src/lib.rs +++ b/crates/server-api/src/lib.rs @@ -7514,6 +7514,9 @@ pub struct CurrentWorkerWorkdirAttachmentResponse { pub alias: String, pub working_directory_id: String, pub capabilities: workdir::WorkdirSessionCapabilities, + /// Present for attachment responses, absent after detach. + #[serde(skip_serializing_if = "Option::is_none")] + pub environment: Option, pub attached: bool, } @@ -13286,6 +13289,7 @@ mod tests { alias: "checkout".to_string(), working_directory_id: "workdir".to_string(), capabilities: workdir::WorkdirSessionCapabilities::EMPTY, + environment: None, attached: false, }; let value = serde_json::to_value(&response).unwrap(); diff --git a/crates/tools/src/bash.rs b/crates/tools/src/bash.rs index abb9700dc..6d42494b1 100644 --- a/crates/tools/src/bash.rs +++ b/crates/tools/src/bash.rs @@ -30,7 +30,9 @@ struct BashParams { /// Worker-local alias of the Workdir attachment to use. #[serde(default)] target_workdir: Option, - /// Workdir-relative directory in which to execute the command. + /// Directory in which to execute the command. + // Keep provider path-coordinate documentation out of the model-facing schema. + #[schemars(with = "String")] cwd: WorkdirPath, command: String, #[serde(default)] diff --git a/crates/tools/src/edit.rs b/crates/tools/src/edit.rs index 8f3e8e947..edd015acd 100644 --- a/crates/tools/src/edit.rs +++ b/crates/tools/src/edit.rs @@ -21,7 +21,7 @@ pub(crate) struct EditParams { /// Worker-local alias of the Workdir attachment to use. #[serde(default)] pub target_workdir: Option, - /// Logical path relative to the bound Workdir root. + /// Existing file in which to replace text. pub file_path: String, #[serde(flatten)] pub edit: fs_operation::text::EditArgs, diff --git a/crates/tools/src/glob.rs b/crates/tools/src/glob.rs index 509d5f3ea..440492698 100644 --- a/crates/tools/src/glob.rs +++ b/crates/tools/src/glob.rs @@ -17,7 +17,7 @@ pub(crate) struct GlobParams { pub target_workdir: Option, /// Glob pattern, for example `**/*.rs` or `src/**/test_*.py`. pub pattern: String, - /// Logical Workdir-relative directory. Defaults to the Workdir root. + /// Directory to search. Defaults to `.`. #[serde(default)] pub path: Option, } @@ -106,7 +106,7 @@ pub(crate) fn routed_glob_tool(router: Arc) -> ToolDefinit Arc::new(move || { let schema = schemars::schema_for!(GlobParams); let meta = ToolMeta::new("Glob") - .description("Find files matching a glob pattern inside the selected Workdir. Results are sorted and capped at 1000 entries. Paths are Workdir-relative.") + .description("Find files matching a glob pattern inside the selected Workdir. Results are sorted and capped at 1000 entries.") .input_schema(serde_json::to_value(schema).expect("Glob schema serialization")); let tool: Arc = Arc::new(GlobTool { router: router.clone(), diff --git a/crates/tools/src/grep.rs b/crates/tools/src/grep.rs index 1fa48343a..813419fc7 100644 --- a/crates/tools/src/grep.rs +++ b/crates/tools/src/grep.rs @@ -27,7 +27,7 @@ pub(crate) struct GrepParams { #[serde(default)] target_workdir: Option, pattern: String, - /// Workdir-relative path, or an absolute path covered by readable scope. Defaults to the Workdir root. + /// File or directory to search. Defaults to `.`. #[serde(default)] pub path: Option, #[serde(default)] @@ -160,7 +160,7 @@ pub(crate) fn routed_grep_tool(router: Arc) -> ToolDefinit Arc::new(move || { let schema = schemars::schema_for!(GrepParams); let meta = ToolMeta::new("Grep") - .description("Search a file or directory in the selected Workdir with a regex. Content results group lines by file; `>` marks matching lines and unmarked lines are context. Directory traversal executes inside the WorkdirSession provider. Results are bounded and Workdir-relative.") + .description("Search a file or directory in the selected Workdir with a regex. Content results group lines by file; `>` marks matching lines and unmarked lines are context. Directory traversal executes inside the WorkdirSession provider. Results are bounded.") .input_schema(serde_json::to_value(schema).expect("Grep schema serialization")); let tool: Arc = Arc::new(GrepTool { router: router.clone(), diff --git a/crates/tools/src/read.rs b/crates/tools/src/read.rs index e32d2389a..b669dd3ab 100644 --- a/crates/tools/src/read.rs +++ b/crates/tools/src/read.rs @@ -13,7 +13,7 @@ use workdir::{WorkdirPath, WorkdirSessionHandle, WorkdirSessionRouter}; const DESCRIPTION: &str = "Read a text file from a Workdir attachment selected by its Worker-local alias. \ Supports offset/limit for large files. Returns line-numbered output (1-based). \ Directories cannot be read. The file must be read before Write or Edit can \ -modify it. Paths are Workdir-relative unless an absolute path is explicitly readable. \ +modify it. \ When exactly one Workdir is attached, target_workdir may be omitted."; const DEFAULT_LIMIT: usize = 2000; @@ -24,7 +24,7 @@ pub(crate) struct ReadParams { /// Worker-local alias of the Workdir attachment to use. #[serde(default)] pub target_workdir: Option, - /// Workdir-relative path, or an absolute path covered by readable scope. + /// Text file to read; directories are not supported. pub file_path: String, #[serde(flatten)] pub read: fs_operation::text::LineReadArgs, diff --git a/crates/tools/src/view_image.rs b/crates/tools/src/view_image.rs index 4d2ff2214..bb775027d 100644 --- a/crates/tools/src/view_image.rs +++ b/crates/tools/src/view_image.rs @@ -20,7 +20,7 @@ pub const MAX_IMAGE_BYTES: usize = 10 * 1024 * 1024; const IMAGE_READ_CHUNK_BYTES: usize = 1024 * 1024; const DESCRIPTION: &str = "Attach an image from the selected Workdir attachment to the next model request. \ -The path must be logical and Workdir-relative. Supported formats: PNG, JPEG, GIF, and WebP. \ +Supported formats: PNG, JPEG, GIF, and WebP. \ When exactly one Workdir is attached, target_workdir may be omitted."; #[derive(Debug, Deserialize, schemars::JsonSchema)] @@ -28,7 +28,7 @@ struct ViewImageParams { /// Worker-local alias of the Workdir attachment to use. #[serde(default)] target_workdir: Option, - /// Logical path relative to the bound Workdir root. + /// Image file to attach to the next model request. path: String, } diff --git a/crates/tools/src/write.rs b/crates/tools/src/write.rs index fefa660a6..4b5e59e59 100644 --- a/crates/tools/src/write.rs +++ b/crates/tools/src/write.rs @@ -20,7 +20,7 @@ pub(crate) struct WriteParams { /// Worker-local alias of the Workdir attachment to use. #[serde(default)] pub target_workdir: Option, - /// Logical path relative to the bound Workdir root. + /// File to create or overwrite. pub file_path: String, #[serde(flatten)] pub write: fs_operation::text::WriteArgs, diff --git a/crates/tools/tests/integration.rs b/crates/tools/tests/integration.rs index 50ddfd32d..5ba767655 100644 --- a/crates/tools/tests/integration.rs +++ b/crates/tools/tests/integration.rs @@ -3,6 +3,11 @@ //! `ToolServerHandle::register_tool` / `flush_pending` are `pub(crate)` in //! agen, so from here we exercise the factories directly — the same //! code path that `flush_pending()` runs at production time. +//! +//! Scoped routing cases connect real LocalWorkdirSession, WorkdirToolBroker::scope, +//! attachment router and Tool factories without replacing their permission or path checks. +//! They use RAII temporary roots and existing image fixtures; command cases require the +//! same local shell as the Bash suite and run in the normal `tools --test integration` target. use std::path::Path; use std::sync::Arc; @@ -80,6 +85,101 @@ fn setup_routed() -> (TempDir, TempDir, Arc, Registry) { (left, right, router, Registry::new(definitions)) } +const READ_SAME_FILE: &str = "IFS= read -r line < same.txt && printf '%s\\n' \"$line\""; + +struct ScopedRegistry { + roots: Vec, + directories: Vec, + brokers: Vec, + leases: Vec, + router: Arc, + reg: Registry, +} + +impl ScopedRegistry { + // Cwd and scope targets here are delegation declarations in provider coordinates. + async fn new(attachments: &[(&str, &str, bool)]) -> Self { + let router = Arc::new(WorkdirSessionRouter::new()); + let mut roots = Vec::new(); + let mut directories = Vec::new(); + let mut brokers = Vec::new(); + let mut leases = Vec::new(); + for &(alias, cwd, command) in attachments { + let root = TempDir::new().unwrap(); + let directory = root.path().join(cwd); + std::fs::create_dir_all(directory.join("subdir")).unwrap(); + std::fs::write(directory.join("same.txt"), format!("{alias} original\n")).unwrap(); + std::fs::write( + directory.join("subdir/same.txt"), + format!("{alias} nested needle\n"), + ) + .unwrap(); + std::fs::write( + directory.join("image.png"), + include_bytes!("fixtures/view-image/valid.png"), + ) + .unwrap(); + if cwd != "." { + // Decoys make accidental root-based resolution observable. + std::fs::write(root.path().join("same.txt"), "root decoy\n").unwrap(); + std::fs::write(root.path().join("image.png"), "not an image").unwrap(); + } + let source: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::new( + Scope::writable(root.path()).unwrap(), + root.path().to_path_buf(), + )); + let broker = WorkdirToolBroker::new(source); + let lease = broker + .scope(WorkdirToolScope { + rules: vec![WorkdirToolScopeRule { + target: WorkdirPath::new(cwd).unwrap(), + permission: WorkdirToolScopePermission::Write, + recursive: true, + symlink_policy: Default::default(), + }], + cwd: WorkdirPath::new(cwd).unwrap(), + command, + }) + .await + .unwrap(); + router + .attach( + WorkdirAttachmentAlias::new(alias).unwrap(), + lease.tool_session(), + ) + .unwrap(); + roots.push(root); + directories.push(directory); + brokers.push(broker); + leases.push(lease); + } + let mut definitions = routed_builtin_tools( + router.clone(), + Tracker::new(), + tools::bash_output::BashOutputView::new(), + ); + definitions.push(routed_view_image_tool(router.clone())); + Self { + roots, + directories, + brokers, + leases, + router, + reg: Registry::new(definitions), + } + } + + async fn close(self) { + self.router.close_all().await.unwrap(); + for lease in &self.leases { + lease.close().await.unwrap(); + } + for broker in &self.brokers { + broker.close().await.unwrap(); + } + } +} + fn valid_gif_with_len(target_len: usize) -> Vec { let fixture = include_bytes!("fixtures/view-image/valid.gif"); assert_eq!( @@ -156,6 +256,71 @@ fn routed_tool_schemas_share_optional_target_workdir() { } } +#[test] +fn routed_tool_help_does_not_override_session_path_coordinates() { + fn assert_no_root_coordinates(tool: &str, description: &str) { + let normalized = description + .split_whitespace() + .collect::>() + .join(" ") + .to_ascii_lowercase(); + for obsolete in ["workdir-relative", "workdir root", "root-relative"] { + assert!( + !normalized.contains(obsolete), + "{tool} help overrides session path coordinates: {description}" + ); + } + } + + fn check_schema_descriptions(tool: &str, schema: &serde_json::Value) { + match schema { + serde_json::Value::Object(object) => { + if let Some(description) = object.get("description").and_then(|v| v.as_str()) { + assert_no_root_coordinates(tool, description); + } + for value in object.values() { + check_schema_descriptions(tool, value); + } + } + serde_json::Value::Array(array) => { + for value in array { + check_schema_descriptions(tool, value); + } + } + _ => {} + } + } + + let router = Arc::new(WorkdirSessionRouter::new()); + let mut definitions = routed_builtin_tools( + router.clone(), + Tracker::new(), + tools::bash_output::BashOutputView::new(), + ); + definitions.push(routed_view_image_tool(router)); + for definition in definitions { + let (meta, _) = definition(); + assert_no_root_coordinates(&meta.name, &meta.description); + // Referenced type definitions are model-visible too, not just properties. + check_schema_descriptions(&meta.name, &meta.input_schema); + + let path_parameter = match meta.name.as_str() { + "Read" | "Write" | "Edit" => "file_path", + "Glob" | "Grep" | "ViewImage" => "path", + "Bash" => "cwd", + other => panic!("missing path parameter contract for {other}"), + }; + let help = meta.input_schema["properties"][path_parameter]["description"] + .as_str() + .unwrap_or_default(); + assert!( + !help.trim().is_empty(), + "{}.{path_parameter} must retain argument-specific help", + meta.name + ); + } +} + #[tokio::test] async fn routed_tools_require_alias_only_when_attachment_set_is_ambiguous() { let (_left, _right, router, reg) = setup_routed(); @@ -284,6 +449,427 @@ async fn target_alias_routes_each_tool_and_isolates_read_before_edit() { router.close_all().await.unwrap(); } +#[tokio::test] +async fn scoped_cwd_file_search_and_image_tools_share_reusable_paths() { + for cwd in [".", "work"] { + let fixture = ScopedRegistry::new(&[("checkout", cwd, false)]).await; + let read = fixture.reg.get("Read"); + let original = call(&read, json!({ "file_path": "same.txt" })).await; + assert!(original.content.unwrap().contains("checkout original")); + call( + &fixture.reg.get("Write"), + json!({ "file_path": "same.txt", "content": "written marker\n" }), + ) + .await; + call( + &fixture.reg.get("Edit"), + json!({ "file_path": "same.txt", "old_string": "written", "new_string": "edited" }), + ) + .await; + let edited = call(&read, json!({ "file_path": "same.txt" })).await; + assert!(edited.content.unwrap().contains("edited marker")); + assert_eq!( + std::fs::read_to_string(fixture.directories[0].join("same.txt")).unwrap(), + "edited marker\n" + ); + + // Omitted search paths start at cwd, not at the underlying provider root. + for (name, input) in [ + ("Glob", json!({ "pattern": "same.txt" })), + ( + "Grep", + json!({ "pattern": "edited marker", "output_mode": "files_with_matches" }), + ), + ] { + let result = call(&fixture.reg.get(name), input).await; + assert_eq!( + result.content.unwrap().trim(), + "same.txt", + "{name}, cwd={cwd}" + ); + } + + // A search rooted below cwd still returns cwd-relative reusable paths. + for (name, input) in [ + ("Glob", json!({ "pattern": "*.txt", "path": "subdir" })), + ( + "Grep", + json!({ "pattern": "needle", "path": "subdir", "output_mode": "files_with_matches" }), + ), + ] { + let result = call(&fixture.reg.get(name), input).await; + let body = result.content.unwrap(); + let paths: Vec<_> = body.lines().collect(); + assert_eq!(paths, ["subdir/same.txt"], "{name}, cwd={cwd}"); + let path = paths[0]; + let found = call(&read, json!({ "file_path": path })).await; + assert!(found.content.unwrap().contains("checkout nested needle")); + call( + &fixture.reg.get("Edit"), + json!({ "file_path": path, "old_string": "needle", "new_string": format!("needle-{name}") }), + ) + .await; + } + assert_eq!( + std::fs::read_to_string(fixture.directories[0].join("subdir/same.txt")).unwrap(), + "checkout nested needle-Grep-Glob\n" + ); + let image = call( + &fixture.reg.get("ViewImage"), + json!({ "path": "image.png" }), + ) + .await; + assert_eq!(image.attachments.len(), 1); + let agen::tool::Attachment::Image(image) = &image.attachments[0]; + assert_eq!(image.mime_type(), "image/png"); + assert_eq!( + image.data(), + include_bytes!("fixtures/view-image/valid.png") + ); + if cwd != "." { + assert_eq!( + std::fs::read_to_string(fixture.roots[0].path().join("same.txt")).unwrap(), + "root decoy\n" + ); + } + fixture.close().await; + } +} + +#[tokio::test] +async fn scoped_bash_relative_cwd_and_cd_do_not_change_later_tool_cwd() { + for cwd in [".", "work"] { + let fixture = ScopedRegistry::new(&[("checkout", cwd, true)]).await; + for (command_cwd, command, expected) in [ + (".", READ_SAME_FILE, "checkout original\n"), + ("subdir", READ_SAME_FILE, "checkout nested needle\n"), + ("./subdir//.", READ_SAME_FILE, "checkout nested needle\n"), + ( + ".", + "cd subdir && IFS= read -r line < same.txt && printf '%s\\n' \"$line\"", + "checkout nested needle\n", + ), + (".", READ_SAME_FILE, "checkout original\n"), + ] { + let output = call( + &fixture.reg.get("Bash"), + json!({ "cwd": command_cwd, "command": command, "timeout": 5 }), + ) + .await; + let body: serde_json::Value = serde_json::from_str(&output.content.unwrap()).unwrap(); + assert_eq!(body["output"]["terminal"]["exit_code"], 0, "{body}"); + assert_eq!(body["stdout_preview"], expected, "cwd={cwd}, {command}"); + let subsequent = + call(&fixture.reg.get("Read"), json!({ "file_path": "same.txt" })).await; + assert!(subsequent.content.unwrap().contains("checkout original")); + } + fixture.close().await; + } +} + +#[tokio::test] +async fn scoped_aliases_with_different_cwds_keep_paths_and_observations_separate() { + let fixture = + ScopedRegistry::new(&[("left", "work", true), ("right", "other/deep", true)]).await; + let read = fixture.reg.get("Read"); + let edit = fixture.reg.get("Edit"); + call( + &read, + json!({ "target_workdir": "left", "file_path": "same.txt" }), + ) + .await; + let unread = call_err( + &edit, + json!({ "target_workdir": "right", "file_path": "same.txt", "old_string": "original", "new_string": "wrong" }), + ) + .await; + assert!(unread.to_string().contains("has not been read"), "{unread}"); + call( + &read, + json!({ "target_workdir": "right", "file_path": "same.txt" }), + ) + .await; + call( + &fixture.reg.get("Write"), + json!({ "target_workdir": "right", "file_path": "same.txt", "content": "right written\n" }), + ) + .await; + call( + &edit, + json!({ "target_workdir": "right", "file_path": "same.txt", "old_string": "written", "new_string": "edited" }), + ) + .await; + assert_eq!( + std::fs::read_to_string(fixture.directories[0].join("same.txt")).unwrap(), + "left original\n" + ); + assert_eq!( + std::fs::read_to_string(fixture.directories[1].join("same.txt")).unwrap(), + "right edited\n" + ); + + // Different valid bytes make cross-alias image reads observable too. + let png: &[u8] = include_bytes!("fixtures/view-image/valid.png"); + let gif: &[u8] = include_bytes!("fixtures/view-image/valid.gif"); + std::fs::write(fixture.directories[1].join("image.png"), gif).unwrap(); + for (alias, expected, image_bytes, mime) in [ + ("left", "left original", png, "image/png"), + ("right", "right edited", gif, "image/gif"), + ] { + let output = call( + &read, + json!({ "target_workdir": alias, "file_path": "same.txt" }), + ) + .await; + assert!(output.content.unwrap().contains(expected)); + for (name, input) in [ + ( + "Glob", + json!({ "target_workdir": alias, "pattern": "*.txt", "path": "subdir" }), + ), + ( + "Grep", + json!({ "target_workdir": alias, "pattern": alias, "path": "subdir", "output_mode": "files_with_matches" }), + ), + ] { + let output = call(&fixture.reg.get(name), input).await; + let path = output.content.unwrap(); + let path = path.trim(); + assert_eq!(path, "subdir/same.txt", "{alias}: {name}"); + let found = call(&read, json!({ "target_workdir": alias, "file_path": path })).await; + assert!( + found + .content + .unwrap() + .contains(&format!("{alias} nested needle")) + ); + } + let image = call( + &fixture.reg.get("ViewImage"), + json!({ "target_workdir": alias, "path": "image.png" }), + ) + .await; + let agen::tool::Attachment::Image(image) = &image.attachments[0]; + assert_eq!(image.mime_type(), mime); + assert_eq!(image.data(), image_bytes); + let output = call( + &fixture.reg.get("Bash"), + json!({ "target_workdir": alias, "cwd": ".", "command": READ_SAME_FILE, "timeout": 5 }), + ) + .await; + let body: serde_json::Value = serde_json::from_str(&output.content.unwrap()).unwrap(); + assert_eq!(body["output"]["terminal"]["exit_code"], 0, "{body}"); + assert_eq!(body["stdout_preview"], format!("{expected}\n")); + } + fixture.close().await; +} + +#[tokio::test] +async fn scoped_command_false_rejects_bash_without_fallback_to_another_alias() { + let fixture = + ScopedRegistry::new(&[("enabled", "work", true), ("disabled", "other", false)]).await; + for cwd in [".", "subdir"] { + let error = call_err( + &fixture.reg.get("Bash"), + json!({ "target_workdir": "disabled", "cwd": cwd, "command": "printf forbidden > blocked.txt", "timeout": 5 }), + ) + .await; + assert!( + error.to_string().contains("does not support Command"), + "{error}" + ); + } + // Write permission remains usable, but does not imply command permission. + call( + &fixture.reg.get("Write"), + json!({ "target_workdir": "disabled", "file_path": "created.txt", "content": "allowed write\n" }), + ) + .await; + assert_eq!( + std::fs::read_to_string(fixture.directories[1].join("created.txt")).unwrap(), + "allowed write\n" + ); + for (root, directory) in fixture.roots.iter().zip(&fixture.directories) { + for path in [ + root.path().to_path_buf(), + directory.clone(), + directory.join("subdir"), + ] { + assert!(!path.join("blocked.txt").exists()); + } + } + fixture.close().await; +} + +#[tokio::test] +async fn scoped_normalized_paths_share_read_before_mutation_history() { + let fixture = ScopedRegistry::new(&[("checkout", "work", false)]).await; + let read = call( + &fixture.reg.get("Read"), + json!({ "file_path": "./subdir//./same.txt" }), + ) + .await; + assert!(read.content.unwrap().contains("checkout nested needle")); + call( + &fixture.reg.get("Write"), + json!({ "file_path": "subdir/same.txt", "content": "normalized write\n" }), + ) + .await; + call( + &fixture.reg.get("Edit"), + json!({ "file_path": "./subdir/same.txt", "old_string": "write", "new_string": "edit" }), + ) + .await; + assert_eq!( + std::fs::read_to_string(fixture.directories[0].join("subdir/same.txt")).unwrap(), + "normalized edit\n" + ); + for (name, input) in [ + ("Glob", json!({ "pattern": "*.txt", "path": "./subdir//." })), + ( + "Grep", + json!({ "pattern": "normalized edit", "path": "./subdir//.", "output_mode": "files_with_matches" }), + ), + ] { + let result = call(&fixture.reg.get(name), input).await; + assert_eq!(result.content.unwrap().trim(), "subdir/same.txt", "{name}"); + } + let image = call( + &fixture.reg.get("ViewImage"), + json!({ "path": "././image.png" }), + ) + .await; + let agen::tool::Attachment::Image(image) = &image.attachments[0]; + assert_eq!( + image.data(), + include_bytes!("fixtures/view-image/valid.png") + ); + fixture.close().await; +} + +#[tokio::test] +async fn scoped_tools_reject_parent_traversal_and_paths_outside_the_grant() { + let fixture = ScopedRegistry::new(&[("checkout", "work", true)]).await; + let outside = fixture.roots[0].path().join("outside"); + std::fs::create_dir(&outside).unwrap(); + std::fs::write(outside.join("secret.txt"), "private needle\n").unwrap(); + for invalid in [ + "../outside/secret.txt", + "subdir/../../outside/secret.txt", + "subdir\\same.txt", + ] { + for (name, input) in [ + ("Read", json!({ "file_path": invalid })), + ( + "Write", + json!({ "file_path": invalid, "content": "forbidden" }), + ), + ( + "Edit", + json!({ "file_path": invalid, "old_string": "needle", "new_string": "forbidden" }), + ), + ("Glob", json!({ "path": invalid, "pattern": "*" })), + ("Grep", json!({ "path": invalid, "pattern": "needle" })), + ("ViewImage", json!({ "path": invalid })), + ( + "Bash", + json!({ "cwd": invalid, "command": "printf forbidden > blocked.txt", "timeout": 5 }), + ), + ] { + let error = call_err(&fixture.reg.get(name), input).await; + assert!( + error.to_string().contains("path"), + "{name}: {invalid}: {error}" + ); + } + } + // These paths are inside the provider's grant, but outside the child's grant. + // Absolute readable paths must not bypass the delegated scope after cwd rebasing. + for (name, input) in [ + ("Read", json!({ "file_path": outside.join("secret.txt") })), + ("Grep", json!({ "path": outside, "pattern": "needle" })), + ( + "Bash", + json!({ "cwd": outside, "command": "printf forbidden > blocked.txt", "timeout": 5 }), + ), + ] { + let error = call_err(&fixture.reg.get(name), input).await; + let message = error.to_string(); + assert!( + matches!(error, ToolError::InvalidArgument(_)), + "{name}: {message}" + ); + assert!( + message.contains("scope") || message.contains("path"), + "{name}: {message}" + ); + } + assert_eq!( + std::fs::read_to_string(outside.join("secret.txt")).unwrap(), + "private needle\n" + ); + assert!(!outside.join("blocked.txt").exists()); + assert!(!fixture.directories[0].join("blocked.txt").exists()); + assert!(!fixture.roots[0].path().join("blocked.txt").exists()); + fixture.close().await; +} + +#[cfg(unix)] +#[tokio::test] +async fn scoped_cwd_tools_reject_resolved_symlink_escape_after_a_read() { + use std::os::unix::fs::symlink; + + let fixture = ScopedRegistry::new(&[("checkout", "work", true)]).await; + let outside = fixture.roots[0].path().join("outside"); + std::fs::create_dir(&outside).unwrap(); + // Equal bytes avoid disguising an authority failure as a stale-hash rejection. + std::fs::write(outside.join("secret.txt"), "checkout original\n").unwrap(); + let png = include_bytes!("fixtures/view-image/valid.png"); + std::fs::write(outside.join("image.png"), png).unwrap(); + call(&fixture.reg.get("Read"), json!({ "file_path": "same.txt" })).await; + std::fs::remove_file(fixture.directories[0].join("same.txt")).unwrap(); + symlink( + "../outside/secret.txt", + fixture.directories[0].join("same.txt"), + ) + .unwrap(); + symlink("../outside", fixture.directories[0].join("escape")).unwrap(); + + for (name, input) in [ + ("Read", json!({ "file_path": "same.txt" })), + ( + "Write", + json!({ "file_path": "escape/created.txt", "content": "forbidden" }), + ), + ( + "Edit", + json!({ "file_path": "same.txt", "old_string": "original", "new_string": "forbidden" }), + ), + ("Glob", json!({ "path": "escape", "pattern": "*" })), + ("Grep", json!({ "path": "escape", "pattern": "original" })), + ("ViewImage", json!({ "path": "escape/image.png" })), + ( + "Bash", + json!({ "cwd": "escape", "command": "printf forbidden > blocked.txt", "timeout": 5 }), + ), + ] { + let error = call_err(&fixture.reg.get(name), input).await; + let message = error.to_string(); + assert!( + message.contains("scope") || message.contains("symlink"), + "{name}: {message}" + ); + } + assert_eq!( + std::fs::read_to_string(outside.join("secret.txt")).unwrap(), + "checkout original\n" + ); + assert_eq!(std::fs::read(outside.join("image.png")).unwrap(), png); + assert!(!outside.join("created.txt").exists()); + assert!(!outside.join("blocked.txt").exists()); + fixture.close().await; +} + #[tokio::test] async fn selected_attachment_capability_is_checked_without_fallback() { let left = TempDir::new().unwrap(); diff --git a/crates/workdir/src/checkout.rs b/crates/workdir/src/checkout.rs index 52c8a91ae..d140f7511 100644 --- a/crates/workdir/src/checkout.rs +++ b/crates/workdir/src/checkout.rs @@ -6,7 +6,7 @@ //! hashes alone do not accept same-content inode replacements. Callers own Read line numbering //! and per-attachment read history. //! -//! Create paths are checkout-root-relative and must be strict descendants of the observed target +//! Create paths use the target's session coordinates and must be strict descendants of the observed target //! directory. Existing destinations are never overwritten; results return the updated observation //! of the retained target directory, not the created file. Read results also return the validator //! of the state actually read, never a replacement from a later independent stat. @@ -18,7 +18,8 @@ //! //! Search delegates to shared List/Glob/Grep engines. Scope rules form unions within layers and //! intersections across layers, provider authority, and output roots. Wrappers transform coordinates -//! once; returned paths and Grep content are relative to the final session's logical root. +//! once; returned paths and Grep content are relative to the composed output root (the caller's +//! cwd by default). //! Paths are not reverse-parsed from rendered filenames; non-UTF-8 names are not lossily aliased. //! Unsupported providers reject search rather than fall back to broader trait searches. Search is //! live traversal, not an atomic snapshot of all descendants or exclusion of external editors. diff --git a/crates/workdir/src/environment.rs b/crates/workdir/src/environment.rs new file mode 100644 index 000000000..24fc259db --- /dev/null +++ b/crates/workdir/src/environment.rs @@ -0,0 +1,68 @@ +//! Model-facing session coordinates, separate from provider paths and authority. + +use crate::{WorkdirPath, WorkdirSessionCapabilities, WorkdirToolScopeRule}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// A snapshot of the environment actually used by a session's operations. +/// +/// Cwd is a logical Workdir-root coordinate. Scope targets use that root or +/// explicit absolute provider paths. Scope is an additional allow ceiling, not +/// a replacement for provider checks. Providers +/// may retain further restrictions (including deny rules); this projection +/// never authorizes an operation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct WorkdirSessionEnvironment { + #[serde(deserialize_with = "deserialize_cwd")] + pub cwd: WorkdirPath, + pub scope: Option>, + pub capabilities: WorkdirSessionCapabilities, + pub restrictions: Vec, +} + +fn deserialize_cwd<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result { + let path = String::deserialize(deserializer)?; + WorkdirPath::new(path).map_err(serde::de::Error::custom) +} + +impl WorkdirSessionEnvironment { + pub fn root(capabilities: WorkdirSessionCapabilities) -> Self { + Self { + cwd: WorkdirPath::root(), + scope: None, + capabilities, + restrictions: Vec::new(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wire_cwd_is_normalized_and_cannot_claim_an_absolute_or_parent_base() { + let mut wire = serde_json::to_value(WorkdirSessionEnvironment::root( + WorkdirSessionCapabilities::READ_ONLY, + )) + .unwrap(); + for invalid in [ + "/absolute", + "../parent", + "nested/../parent", + "nested\\other", + ] { + wire["cwd"] = invalid.into(); + assert!( + serde_json::from_value::(wire.clone()).is_err(), + "{invalid}" + ); + } + wire["cwd"] = "./nested//child".into(); + let decoded: WorkdirSessionEnvironment = serde_json::from_value(wire).unwrap(); + assert_eq!(decoded.cwd.as_str(), "nested/child"); + } +} diff --git a/crates/workdir/src/environment_tests.rs b/crates/workdir/src/environment_tests.rs new file mode 100644 index 000000000..fedba6031 --- /dev/null +++ b/crates/workdir/src/environment_tests.rs @@ -0,0 +1,175 @@ +//! Real Local/scoped/read-only/router sessions across loopback HTTP. Only the +//! HTTP authentication boundary is a fixture; coordinate and scope checks are +//! production implementations. Runs in the normal http-client library suite. +#![cfg(all(feature = "http-client", target_os = "linux"))] + +use crate::http::*; +use crate::*; +use axum::{ + Json, Router, + extract::State, + routing::{delete, post}, +}; +use std::sync::Arc; + +struct Server(tokio::task::JoinHandle<()>); +impl Drop for Server { + fn drop(&mut self) { + self.0.abort(); + } +} + +async fn open(State(session): State) -> Json { + Json(OpenWorkdirSessionResponse { + session_id: WorkdirSessionId::new("environment-session").unwrap(), + workdir_id: session.workdir().id().clone(), + capabilities: session.capabilities(), + environment: session.environment(), + }) +} +async fn operation( + State(session): State, + Json(request): Json, +) -> Result< + Json, + (axum::http::StatusCode, Json), +> { + dispatch_workdir_session_operation(session.as_ref(), request.operation) + .await + .map(Json) + .map_err(|error| { + ( + axum::http::StatusCode::FORBIDDEN, + Json(WorkdirTransportError::from_workdir_error(&error)), + ) + }) +} + +#[tokio::test] +async fn remote_environment_matches_scoped_readonly_operations_without_rebasing_twice() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir(root.path().join("nested")).unwrap(); + std::fs::write(root.path().join("nested/file.txt"), "nested marker").unwrap(); + std::fs::write(root.path().join("file.txt"), "root decoy").unwrap(); + let broker = WorkdirToolBroker::new(Arc::new(LocalWorkdirSession::new( + manifest::Scope::writable(root.path()).unwrap(), + root.path().to_path_buf(), + ))); + let lease = broker + .scope(WorkdirToolScope { + cwd: WorkdirPath::new("nested").unwrap(), + rules: vec![WorkdirToolScopeRule { + target: WorkdirPath::new("nested").unwrap(), + permission: WorkdirToolScopePermission::Write, + recursive: true, + symlink_policy: Default::default(), + }], + command: true, + }) + .await + .unwrap(); + let session: WorkdirSessionHandle = Arc::new(ReadOnlyWorkdirSession::new(lease.tool_session())); + let expected = session.environment(); + let app = Router::new() + .route("/v1/working-directories/{id}/sessions", post(open)) + .route("/v1/workdir-sessions/{id}/operations", post(operation)) + .route( + "/v1/workdir-sessions/{id}", + delete(|| async { axum::http::StatusCode::NO_CONTENT }), + ) + .with_state(session.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let _server = Server(tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + })); + let remote: WorkdirSessionHandle = Arc::new( + RemoteWorkdirSession::open( + reqwest::Client::builder().no_proxy().build().unwrap(), + format!("http://{address}").parse().unwrap(), + "fixture", + session.workdir().id().clone(), + OpenWorkdirSessionRequest::default(), + ) + .await + .unwrap(), + ); + let router = WorkdirSessionRouter::new(); + router + .attach(WorkdirAttachmentAlias::new("remote").unwrap(), remote) + .unwrap(); + let selected = router.resolve(Some("remote")).unwrap().session; + assert_eq!(selected.environment(), expected); + assert_eq!(expected.cwd, WorkdirPath::new("nested").unwrap()); + assert!( + !expected + .capabilities + .supports(WorkdirSessionCapability::Command) + ); + assert!( + expected + .scope + .unwrap() + .iter() + .all(|rule| rule.permission == WorkdirToolScopePermission::Read) + ); + let found = selected + .glob(GlobRequest { + path: WorkdirPath::root(), + pattern: "*.txt".into(), + limit: 10, + }) + .await + .unwrap(); + assert_eq!(found.paths, vec![WorkdirPath::new("file.txt").unwrap()]); + let path = found.paths[0].clone(); + let read = selected + .read(ReadRequest { + path: path.clone(), + offset: 0, + limit: 10, + max_bytes: 100, + }) + .await + .unwrap(); + assert_eq!(read.path, path); + assert_eq!(read.bytes, b"nested marker"); + assert!( + selected + .write(WriteRequest { + path, + content: b"denied".to_vec(), + expected_hash: Some(read.content_hash) + }) + .await + .is_err() + ); + assert!( + selected + .read(ReadRequest { + path: WorkdirPath::new_scoped(root.path().join("file.txt").to_str().unwrap()) + .unwrap(), + offset: 0, + limit: 10, + max_bytes: 100 + }) + .await + .is_err() + ); + assert!( + selected + .start_command(CommandRequest { + command: "touch escaped".into(), + cwd: WorkdirPath::root(), + timeout_secs: 5, + transport_byte_budget: 100, + tool_call_id: None + }) + .await + .is_err() + ); + assert!(!root.path().join("nested/escaped").exists()); + router.close_all().await.unwrap(); + lease.close().await.unwrap(); + broker.close().await.unwrap(); +} diff --git a/crates/workdir/src/http.rs b/crates/workdir/src/http.rs index 7c2f66ba4..20a57a696 100644 --- a/crates/workdir/src/http.rs +++ b/crates/workdir/src/http.rs @@ -54,6 +54,7 @@ pub struct OpenWorkdirSessionResponse { pub session_id: WorkdirSessionId, pub workdir_id: WorkdirId, pub capabilities: WorkdirSessionCapabilities, + pub environment: crate::WorkdirSessionEnvironment, } /// One provider-side Workdir operation. @@ -553,6 +554,7 @@ mod client { workdir: Workdir, session_id: WorkdirSessionId, capabilities: WorkdirSessionCapabilities, + environment: crate::WorkdirSessionEnvironment, closed: AtomicBool, } @@ -612,6 +614,7 @@ mod client { workdir: Workdir::new(opened.workdir_id.as_str()), session_id: opened.session_id, capabilities: opened.capabilities, + environment: opened.environment, closed: AtomicBool::new(false), }) } @@ -676,6 +679,12 @@ mod client { self.capabilities } + fn environment(&self) -> crate::WorkdirSessionEnvironment { + let mut environment = self.environment.clone(); + environment.capabilities = self.capabilities(); + environment + } + async fn authorize_scope_path( &self, request: WorkdirScopeAuthorizationRequest, @@ -1209,6 +1218,9 @@ mod client { workdir, session_id: WorkdirSessionId::new("s").unwrap(), capabilities: WorkdirSessionCapabilities::EMPTY, + environment: crate::WorkdirSessionEnvironment::root( + WorkdirSessionCapabilities::EMPTY, + ), closed: AtomicBool::new(false), }; let stat = StatRequest { diff --git a/crates/workdir/src/lib.rs b/crates/workdir/src/lib.rs index c3c3d97e8..bfb9f6af2 100644 --- a/crates/workdir/src/lib.rs +++ b/crates/workdir/src/lib.rs @@ -5,6 +5,8 @@ pub use checkout::{ CheckoutObservation, CheckoutOperation, CheckoutOutput, CheckoutRequest, CheckoutResult, CheckoutSearchOperation, CheckoutSearchRequest, CheckoutSearchResult, }; +mod environment; +pub use environment::WorkdirSessionEnvironment; pub mod external; pub mod http; mod local; @@ -16,6 +18,8 @@ pub mod workspace; #[cfg(test)] mod checkout_search_tests; #[cfg(test)] +mod environment_tests; +#[cfg(test)] mod external_local_tests; use std::path::{Path, PathBuf}; @@ -185,6 +189,9 @@ pub type WriteOutcome = WriteResult; /// Worker-bound operation session for a persistent [`Workdir`]. /// /// Filesystem search and commands run on the host owning materialization. +/// Relative operation paths and reusable search results use the session's cwd; +/// root-coordinate providers are adapted once at attachment, not by each Tool. +/// Scope declarations retain their own root-coordinate authority contract. /// Session closure is terminal but does not delete the Workdir or files. /// Unsupported checkout/search implementations explicitly reject requests, with no ambient /// filesystem fallback. Providers own identity validators; callers own read hash history @@ -194,6 +201,13 @@ pub trait WorkdirSession: std::fmt::Debug + Send + Sync { fn workdir(&self) -> &Workdir; fn capabilities(&self) -> WorkdirSessionCapabilities; + /// Describe the coordinates used by this session, not a caller's Profile + /// or filesystem authority. Root-coordinate providers use the default; + /// coordinate and authority wrappers must project their effective view. + fn environment(&self) -> WorkdirSessionEnvironment { + WorkdirSessionEnvironment::root(self.capabilities()) + } + /// Validate an attenuated filesystem rule at the provider boundary without /// exposing the resolved host path. Providers that cannot resolve symbolic /// links must reject resolved-policy checks rather than downgrade them. diff --git a/crates/workdir/src/local.rs b/crates/workdir/src/local.rs index 344cab6a8..f4e8cf972 100644 --- a/crates/workdir/src/local.rs +++ b/crates/workdir/src/local.rs @@ -1340,6 +1340,52 @@ impl WorkdirSession for LocalWorkdirSession { self.inner.capabilities } + fn environment(&self) -> crate::WorkdirSessionEnvironment { + let scope = self.inner.scope.snapshot(); + let project = |rule: manifest::ScopeRule| -> Option { + let target = if self.inner.root.starts_with(&rule.target) && rule.recursive { + WorkdirPath::root() + } else { + let path = rule + .target + .strip_prefix(&self.inner.root) + .unwrap_or(&rule.target); + WorkdirPath::new_scoped(path.to_str()?).ok()? + }; + Some(crate::WorkdirToolScopeRule { + target, + permission: match rule.permission { + Permission::Read => WorkdirToolScopePermission::Read, + Permission::Write => WorkdirToolScopePermission::Write, + }, + recursive: rule.recursive, + symlink_policy: rule.symlink_policy, + }) + }; + crate::WorkdirSessionEnvironment { + cwd: WorkdirPath::root(), + scope: Some( + scope + .allow_rules() + .into_iter() + .filter_map(project) + .collect(), + ), + capabilities: self.capabilities(), + restrictions: scope + .deny_rules() + .into_iter() + .map(|rule| { + project(rule) + .map(|rule| format!("Deny: {rule:?}")) + .unwrap_or_else(|| { + "Unrepresentable provider deny rule remains enforced".into() + }) + }) + .collect(), + } + } + async fn authorize_scope_path( &self, request: WorkdirScopeAuthorizationRequest, diff --git a/crates/workdir/src/local/command_transport/tests/http.rs b/crates/workdir/src/local/command_transport/tests/http.rs index 0bed3d4dc..594b5df3f 100644 --- a/crates/workdir/src/local/command_transport/tests/http.rs +++ b/crates/workdir/src/local/command_transport/tests/http.rs @@ -134,6 +134,9 @@ async fn pipe_and_wait_faults_cross_real_http_with_prefix_and_terminal_replay() session_id: WorkdirSessionId::new("fault-session").unwrap(), workdir_id: crate::Workdir::new("fault-workdir").id().clone(), capabilities: crate::WorkdirSessionCapabilities::ALL, + environment: crate::WorkdirSessionEnvironment::root( + crate::WorkdirSessionCapabilities::ALL, + ), }) }), ) diff --git a/crates/workdir/src/router.rs b/crates/workdir/src/router.rs index 8810b4435..38e73def8 100644 --- a/crates/workdir/src/router.rs +++ b/crates/workdir/src/router.rs @@ -225,16 +225,27 @@ impl Drop for ActiveOperation<'_> { #[derive(Debug, Clone)] pub struct RoutedWorkdirSession { state: Arc, + view: Option, +} + +impl RoutedWorkdirSession { + fn inner(&self) -> &WorkdirSessionHandle { + self.view.as_ref().unwrap_or(&self.state.session) + } } #[async_trait] impl WorkdirSession for RoutedWorkdirSession { fn workdir(&self) -> &Workdir { - self.state.session.workdir() + self.inner().workdir() } fn capabilities(&self) -> WorkdirSessionCapabilities { - self.state.session.capabilities() + self.inner().capabilities() + } + + fn environment(&self) -> crate::WorkdirSessionEnvironment { + self.inner().environment() } async fn authorize_scope_path( @@ -242,7 +253,7 @@ impl WorkdirSession for RoutedWorkdirSession { request: WorkdirScopeAuthorizationRequest, ) -> Result<(), WorkdirError> { let _active = self.state.enter()?; - self.state.session.authorize_scope_path(request).await + self.inner().authorize_scope_path(request).await } async fn scope_rules_overlap( @@ -250,7 +261,7 @@ impl WorkdirSession for RoutedWorkdirSession { request: WorkdirScopeOverlapRequest, ) -> Result { let _active = self.state.enter()?; - self.state.session.scope_rules_overlap(request).await + self.inner().scope_rules_overlap(request).await } async fn checkout_search( @@ -258,7 +269,7 @@ impl WorkdirSession for RoutedWorkdirSession { request: crate::CheckoutSearchRequest, ) -> Result { let _active = self.state.enter()?; - self.state.session.checkout_search(request).await + self.inner().checkout_search(request).await } async fn checkout_observe( @@ -266,59 +277,59 @@ impl WorkdirSession for RoutedWorkdirSession { path: crate::WorkdirPath, ) -> Result { let _active = self.state.enter()?; - self.state.session.checkout_observe(path).await + self.inner().checkout_observe(path).await } async fn checkout_execute( &self, request: crate::CheckoutRequest, ) -> Result { let _active = self.state.enter()?; - self.state.session.checkout_execute(request).await + self.inner().checkout_execute(request).await } async fn stat(&self, request: StatRequest) -> Result { let _active = self.state.enter()?; - self.state.session.stat(request).await + self.inner().stat(request).await } async fn read(&self, request: ReadRequest) -> Result { let _active = self.state.enter()?; - self.state.session.read(request).await + self.inner().read(request).await } async fn read_bytes(&self, request: ReadBytesRequest) -> Result { let _active = self.state.enter()?; - self.state.session.read_bytes(request).await + self.inner().read_bytes(request).await } async fn write(&self, request: WriteRequest) -> Result { let _active = self.state.enter()?; - self.state.session.write(request).await + self.inner().write(request).await } async fn edit(&self, request: EditRequest) -> Result { let _active = self.state.enter()?; - self.state.session.edit(request).await + self.inner().edit(request).await } async fn list(&self, request: ListRequest) -> Result { let _active = self.state.enter()?; - self.state.session.list(request).await + self.inner().list(request).await } async fn glob(&self, request: GlobRequest) -> Result { let _active = self.state.enter()?; - self.state.session.glob(request).await + self.inner().glob(request).await } async fn grep(&self, request: GrepRequest) -> Result { let _active = self.state.enter()?; - self.state.session.grep(request).await + self.inner().grep(request).await } async fn start_command(&self, request: CommandRequest) -> Result { let _active = self.state.enter()?; - let handle = self.state.session.start_command(request).await?; + let handle = self.inner().start_command(request).await?; self.state .active_commands .lock() @@ -329,7 +340,7 @@ impl WorkdirSession for RoutedWorkdirSession { async fn command_status(&self, handle: CommandHandle) -> Result { let _active = self.state.enter()?; - let status = self.state.session.command_status(handle.clone()).await?; + let status = self.inner().command_status(handle.clone()).await?; self.state.mark_terminal(&handle, status); Ok(status) } @@ -340,7 +351,7 @@ impl WorkdirSession for RoutedWorkdirSession { ) -> Result { let _active = self.state.enter()?; let handle = request.handle.clone(); - let output = self.state.session.command_output(request).await?; + let output = self.inner().command_output(request).await?; if let Some(terminal) = &output.terminal { self.state.mark_terminal(&handle, terminal.status); } @@ -349,7 +360,7 @@ impl WorkdirSession for RoutedWorkdirSession { async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError> { let _active = self.state.enter()?; - self.state.session.cancel_command(handle.clone()).await?; + self.inner().cancel_command(handle.clone()).await?; self.state .active_commands .lock() @@ -359,11 +370,11 @@ impl WorkdirSession for RoutedWorkdirSession { } fn subscribe_command_events(&self) -> Option> { - self.state.session.subscribe_command_events() + self.inner().subscribe_command_events() } fn command_snapshot(&self) -> Vec { - let snapshots = self.state.session.command_snapshot(); + let snapshots = self.inner().command_snapshot(); self.state.reconcile_terminal_commands(&snapshots); snapshots } @@ -374,7 +385,7 @@ impl WorkdirSession for RoutedWorkdirSession { "Workdir attachment has active operations or commands".to_string(), )); } - self.state.session.close().await + self.inner().close().await } } @@ -387,6 +398,7 @@ impl WorkdirSession for RoutedWorkdirSession { pub struct WorkdirSessionRouter { sessions: RwLock>>, next_generation: AtomicU64, + coordinate_brokers: RwLock>, } impl WorkdirSessionRouter { @@ -473,9 +485,9 @@ impl WorkdirSessionRouter { } }; Ok(ResolvedWorkdirSession { + session: self.routed(&selected.0, selected.1.clone()), alias: selected.0, generation: selected.1.generation, - session: Arc::new(RoutedWorkdirSession { state: selected.1 }), }) } @@ -483,13 +495,22 @@ impl WorkdirSessionRouter { &self, alias: WorkdirAttachmentAlias, session: WorkdirSessionHandle, + ) -> Result { + self.attach_with_cwd(alias, session, None) + } + + fn attach_with_cwd( + &self, + alias: WorkdirAttachmentAlias, + session: WorkdirSessionHandle, + cwd: Option, ) -> Result { let mut sessions = self.sessions.write().map_err(|_| { - WorkdirError::Unavailable("Workdir attachment router is poisoned".to_string()) + WorkdirError::Unavailable("Workdir attachment router is poisoned".into()) })?; if sessions.contains_key(&alias) { return Err(WorkdirError::Conflict(format!( - "Workdir attachment alias `{alias}` already exists" + "Workdir attachment alias \u{0060}{alias}\u{0060} already exists" ))); } let state = Arc::new(RoutedSessionState { @@ -499,8 +520,61 @@ impl WorkdirSessionRouter { active_operations: AtomicUsize::new(0), active_commands: Mutex::new(HashSet::new()), }); + let view = if let Some(cwd) = cwd { + let source = Arc::new(RoutedWorkdirSession { + state: state.clone(), + view: None, + }); + let broker = crate::WorkdirToolBroker::at_cwd(source, cwd)?; + let view = broker.tool_session(); + self.coordinate_brokers + .write() + .map_err(|_| { + WorkdirError::Unavailable("Workdir coordinate broker map is poisoned".into()) + })? + .insert(alias.clone(), (state.generation, broker)); + Some(view) + } else { + None + }; sessions.insert(alias, state.clone()); - Ok(Arc::new(RoutedWorkdirSession { state })) + Ok(Arc::new(RoutedWorkdirSession { state, view })) + } + + fn routed( + &self, + alias: &WorkdirAttachmentAlias, + state: Arc, + ) -> Arc { + let view = self + .coordinate_broker(alias, state.generation) + .map(|broker| broker.tool_session()); + Arc::new(RoutedWorkdirSession { state, view }) + } + + pub(crate) fn coordinate_broker( + &self, + alias: &WorkdirAttachmentAlias, + generation: u64, + ) -> Option { + self.coordinate_brokers + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(alias) + .filter(|(current, _)| *current == generation) + .map(|(_, broker)| broker.clone()) + } + + /// Bind tool coordinates once while retaining root-coordinate delegation + /// and provider authorization. Both views share the route's detach fence. + pub fn attach_at_cwd( + &self, + alias: WorkdirAttachmentAlias, + source: WorkdirSessionHandle, + cwd: crate::WorkdirPath, + ) -> Result { + let cwd = crate::WorkdirPath::new(cwd.as_str())?; + self.attach_with_cwd(alias, source, Some(cwd)) } pub fn session(&self, alias: &WorkdirAttachmentAlias) -> Option { @@ -509,7 +583,7 @@ impl WorkdirSessionRouter { .ok()? .get(alias) .cloned() - .map(|state| Arc::new(RoutedWorkdirSession { state }) as WorkdirSessionHandle) + .map(|state| self.routed(alias, state) as WorkdirSessionHandle) } /// Compatibility projection used only while filesystem tool schemas have no @@ -521,10 +595,8 @@ impl WorkdirSessionRouter { return None; } sessions - .values() - .next() - .cloned() - .map(|state| Arc::new(RoutedWorkdirSession { state }) as WorkdirSessionHandle) + .first_key_value() + .map(|(alias, state)| self.routed(alias, state.clone()) as WorkdirSessionHandle) } pub fn can_detach(&self, alias: &WorkdirAttachmentAlias) -> Result<(), WorkdirError> { @@ -620,6 +692,10 @@ impl WorkdirSessionRouter { .is_some_and(|current| Arc::ptr_eq(current, &state)) { sessions.remove(alias); + self.coordinate_brokers + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(alias); } Ok(state.session.clone()) } @@ -661,6 +737,10 @@ impl WorkdirSessionRouter { .is_some_and(|current| Arc::ptr_eq(current, &state)) { sessions.remove(&alias); + self.coordinate_brokers + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&alias); } } Ok(()) @@ -684,6 +764,131 @@ mod tests { )) } + #[tokio::test] + async fn cwd_views_share_root_delegation_leases_and_attachment_fences() { + use crate::{ + WorkdirPath as P, WorkdirToolBrokerRouter, WorkdirToolScope, + WorkdirToolScopePermission as Permission, WorkdirToolScopeRule, + }; + let root = TempDir::new().unwrap(); + for directory in ["work", "other", "other/inside"] { + std::fs::create_dir_all(root.path().join(directory)).unwrap(); + std::fs::write(root.path().join(directory).join("marker"), directory).unwrap(); + } + let router = Arc::new(WorkdirSessionRouter::new()); + let alias = WorkdirAttachmentAlias::new("source").unwrap(); + let parent = router + .attach_at_cwd(alias.clone(), session("wd", &root), P::new("work").unwrap()) + .unwrap(); + assert_eq!(parent.environment().cwd, P::new("work").unwrap()); + let request = |cwd: &str| WorkdirToolScope { + rules: vec![WorkdirToolScopeRule { + target: P::new(cwd).unwrap(), + permission: Permission::Write, + recursive: true, + symlink_policy: Default::default(), + }], + cwd: P::new(cwd).unwrap(), + command: false, + }; + let brokers = WorkdirToolBrokerRouter::new(router.clone()); + // Declarations stay root-relative, not work/other. + let child = brokers + .scope(vec![(alias.clone(), request("other"))]) + .await + .unwrap(); + let grandchild = child + .broker_router() + .scope(vec![(alias.clone(), request("other/inside"))]) + .await + .unwrap(); + for (session, cwd) in [ + (parent.clone(), "work"), + ( + child.session_router().resolve(None).unwrap().session, + "other", + ), + ( + grandchild.session_router().resolve(None).unwrap().session, + "other/inside", + ), + ] { + assert_eq!(session.environment().cwd, P::new(cwd).unwrap()); + let result = session + .read(ReadRequest { + path: P::new("marker").unwrap(), + offset: 0, + limit: 10, + max_bytes: 100, + }) + .await + .unwrap(); + assert_eq!(result.bytes, cwd.as_bytes()); + assert_eq!(result.path, P::new("marker").unwrap()); + assert_eq!( + session + .stat(StatRequest { + path: P::new("marker").unwrap() + }) + .await + .unwrap() + .path, + P::new("marker").unwrap() + ); + } + // The model's tool view and delegation view share write-lease exclusion. + let absolute = P::new_scoped(root.path().join("other/marker").to_str().unwrap()).unwrap(); + let denied = parent + .write(WriteRequest { + path: absolute, + content: b"forbidden".to_vec(), + expected_hash: None, + }) + .await + .unwrap_err(); + assert_eq!( + denied.denial_reason(), + Some(crate::WorkdirDenialReason::ChildWriteLeaseConflict) + ); + let old_child = child.session_router().resolve(None).unwrap().session; + router.begin_detach(&alias).await.unwrap(); + assert!( + old_child + .read(ReadRequest { + path: P::new("marker").unwrap(), + offset: 0, + limit: 10, + max_bytes: 100 + }) + .await + .is_err() + ); + router.cancel_detach(&alias); + grandchild.close().await.unwrap(); + child.close().await.unwrap(); + router.detach(&alias).await.unwrap(); + router + .attach_at_cwd( + alias, + session("replacement", &root), + P::new("other").unwrap(), + ) + .unwrap(); + assert_eq!( + router.resolve(None).unwrap().session.environment().cwd, + P::new("other").unwrap() + ); + assert!( + old_child + .stat(StatRequest { + path: P::new("marker").unwrap() + }) + .await + .is_err() + ); + router.close_all().await.unwrap(); + } + #[tokio::test] async fn checkout_routes_use_active_operation_detach_fence_and_old_handles_stay_closed() { let root = TempDir::new().unwrap(); diff --git a/crates/workdir/src/scope.rs b/crates/workdir/src/scope.rs index 9a54e818c..993102076 100644 --- a/crates/workdir/src/scope.rs +++ b/crates/workdir/src/scope.rs @@ -108,17 +108,34 @@ impl WorkdirToolBroker { Self::with_capabilities(source, capabilities) } + /// Set the tool coordinate base without changing provider authority. + /// The input is relative to the source session's coordinates; delegation + /// declarations retain their existing coordinate contract. + pub fn at_cwd(source: WorkdirSessionHandle, cwd: FsPath) -> Result { + let cwd = FsPath::new(cwd.as_str())?; + let capabilities = source.capabilities(); + Ok(Self::with_capabilities_and_cwd(source, capabilities, cwd)) + } + /// Own a session behind an additional Backend-authored capability ceiling. /// The ceiling can only attenuate provider capabilities; it can never add one. pub fn with_capabilities( source: WorkdirSessionHandle, capabilities: WorkdirSessionCapabilities, + ) -> Self { + Self::with_capabilities_and_cwd(source, capabilities, FsPath::root()) + } + + fn with_capabilities_and_cwd( + source: WorkdirSessionHandle, + capabilities: WorkdirSessionCapabilities, + cwd: FsPath, ) -> Self { let capabilities = source.capabilities().intersection(capabilities); let (command_events, _) = broadcast::channel(64); let authority = Arc::new(ScopedWorkdirSession { source, - cwd: FsPath::new("").expect("empty Workdir path is valid"), + cwd, scope: None, capabilities, validity: SessionValidity::root(), @@ -416,6 +433,9 @@ impl WorkdirToolBrokerRouter { .sessions .resolve(Some(alias.as_str())) .map_err(|error| WorkdirError::InvalidArgument(error.to_string()))?; + if let Some(broker) = self.sessions.coordinate_broker(alias, selected.generation) { + return Ok(broker); + } let mut brokers = self.brokers.lock().map_err(|_| { WorkdirError::Unavailable("Workdir tool broker router is poisoned".to_string()) })?; @@ -684,7 +704,7 @@ impl ScopedWorkdirSession { } fn resolve_path(&self, path: &FsPath) -> Result { - if self.cwd.as_str().is_empty() { + if self.cwd.as_str().is_empty() || Path::new(path.as_str()).is_absolute() { return Ok(path.clone()); } let joined = Path::new(self.cwd.as_str()).join(path.as_str()); @@ -1085,6 +1105,43 @@ impl WorkdirSession for ScopedWorkdirSession { self.capabilities } + fn environment(&self) -> crate::WorkdirSessionEnvironment { + let mut environment = self.source.environment(); + let source_cwd = environment.cwd.clone(); + environment.cwd = FsPath::new( + Path::new(source_cwd.as_str()) + .join(self.cwd.as_str()) + .to_str() + .expect("validated UTF-8 paths"), + ) + .expect("validated session cwd"); + if let Some(scope) = &self.scope { + if let Some(inherited) = environment.scope.take() { + environment + .restrictions + .push(format!("Provider allow ceiling: {inherited:?}")); + } + environment.scope = Some( + scope + .iter() + .cloned() + .map(|mut rule| { + rule.target = FsPath::new( + Path::new(source_cwd.as_str()) + .join(rule.target.as_str()) + .to_str() + .expect("validated UTF-8 paths"), + ) + .expect("validated scope target"); + rule + }) + .collect(), + ); + } + environment.capabilities = self.capabilities(); + environment + } + async fn authorize_scope_path( &self, mut request: WorkdirScopeAuthorizationRequest, @@ -1273,8 +1330,11 @@ impl WorkdirSession for ScopedWorkdirSession { .resolve_operation_path(&request.path, WorkdirToolScopePermission::Read) .await?; self.ensure_read(&path, WorkdirSessionCapability::Read)?; + let caller_path = request.path.clone(); request.path = path; - self.source.stat(request).await + let mut result = self.source.stat(request).await?; + result.path = caller_path; + Ok(result) } async fn read(&self, mut request: ReadRequest) -> Result { @@ -1282,8 +1342,11 @@ impl WorkdirSession for ScopedWorkdirSession { .resolve_operation_path(&request.path, WorkdirToolScopePermission::Read) .await?; self.ensure_read(&path, WorkdirSessionCapability::Read)?; + let caller_path = request.path.clone(); request.path = path; - self.source.read(request).await + let mut result = self.source.read(request).await?; + result.path = caller_path; + Ok(result) } async fn read_bytes( @@ -1422,9 +1485,10 @@ impl WorkdirSession for ScopedWorkdirSession { ))); } let tool_call_id = request.tool_call_id.clone(); - if self.scope.is_some() { - request.cwd = self.resolve_path(&request.cwd)?; - } + request.cwd = self + .resolve_operation_path(&request.cwd, WorkdirToolScopePermission::Read) + .await?; + self.ensure_path(&request.cwd, WorkdirToolScopePermission::Read)?; if let Some(tool_call_id) = &tool_call_id { self.starting_tool_calls .lock() @@ -1601,7 +1665,20 @@ impl WorkdirSession for ReadOnlyWorkdirSession { } fn capabilities(&self) -> WorkdirSessionCapabilities { - WorkdirSessionCapabilities::READ_ONLY + self.inner + .capabilities() + .intersection(WorkdirSessionCapabilities::READ_ONLY) + } + + fn environment(&self) -> crate::WorkdirSessionEnvironment { + let mut environment = self.inner.environment(); + environment.capabilities = self.capabilities(); + if let Some(scope) = &mut environment.scope { + for rule in scope { + rule.permission = WorkdirToolScopePermission::Read; + } + } + environment } async fn authorize_scope_path( diff --git a/crates/workdir/tests/remote_denial_reasons.rs b/crates/workdir/tests/remote_denial_reasons.rs index 6769fb136..2dceead9c 100644 --- a/crates/workdir/tests/remote_denial_reasons.rs +++ b/crates/workdir/tests/remote_denial_reasons.rs @@ -14,6 +14,18 @@ use workdir::{ }, }; +fn opened(workdir_id: &str) -> String { + serde_json::to_string(&workdir::http::OpenWorkdirSessionResponse { + session_id: workdir::http::WorkdirSessionId::new("s").unwrap(), + workdir_id: Workdir::new(workdir_id).id().clone(), + capabilities: workdir::WorkdirSessionCapabilities::ALL, + environment: workdir::WorkdirSessionEnvironment::root( + workdir::WorkdirSessionCapabilities::ALL, + ), + }) + .unwrap() +} + fn reply(listener: &TcpListener, status: &str, body: &str) { listener.set_nonblocking(true).unwrap(); let deadline = Instant::now() + Duration::from_secs(5); @@ -73,11 +85,7 @@ async fn remote_http_denials_forward_only_typed_reasons_and_old_missing_reason() error["denial_reason"] = serde_json::to_value(reason).unwrap(); } let server = thread::spawn(move || { - reply( - &listener, - "200 OK", - r#"{"session_id":"s","workdir_id":"w","capabilities":{"bits":63}}"#, - ); + reply(&listener, "200 OK", &opened("w")); reply(&listener, "403 Forbidden", &error.to_string()); }); let remote = RemoteWorkdirSession::open( @@ -163,11 +171,7 @@ async fn remote_checkout_diagnostics_preserve_mutation_outcome_and_read_failures let address = listener.local_addr().unwrap(); let error = serde_json::json!({"code":code, "message":"/secret/host provider-secret", "denial_reason":reason}); let server = thread::spawn(move || { - reply( - &listener, - "200 OK", - r#"{"session_id":"s","workdir_id":"w","capabilities":{"bits":63}}"#, - ); + reply(&listener, "200 OK", &opened("w")); reply( &listener, &format!("{} Fixture failure", code.http_status()), @@ -230,11 +234,11 @@ async fn mismatched_http_results_keep_cause_and_mutation_uncertainty() { reply( &listener, "200 OK", - if case == "identity" { - r#"{"session_id":"s","workdir_id":"wrong-secret","capabilities":{"bits":63}}"# + &opened(if case == "identity" { + "wrong-secret" } else { - r#"{"session_id":"s","workdir_id":"w","capabilities":{"bits":63}}"# - }, + "w" + }), ); if case != "identity" { let result = if case == "bytes" { diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 1a6aeefe9..5eb0024bb 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -869,6 +869,7 @@ async fn open_workdir_session( session_id: session_id.clone(), workdir_id: session.workdir().id().clone(), capabilities: session.capabilities(), + environment: session.environment(), }; state .workdir_sessions diff --git a/crates/worker-runtime/src/http_server/tests/command_transport_tests.rs b/crates/worker-runtime/src/http_server/tests/command_transport_tests.rs index fb4c3ffd7..cdaa8ce40 100644 --- a/crates/worker-runtime/src/http_server/tests/command_transport_tests.rs +++ b/crates/worker-runtime/src/http_server/tests/command_transport_tests.rs @@ -111,6 +111,9 @@ impl Harness { session_id: WorkdirSessionId::new("session-transport").unwrap(), workdir_id: Workdir::new("wd-transport").id().clone(), capabilities: WorkdirSessionCapabilities::ALL, + environment: workdir::WorkdirSessionEnvironment::root( + WorkdirSessionCapabilities::ALL, + ), }) }), ) diff --git a/crates/worker-runtime/src/runtime/workdir_session.rs b/crates/worker-runtime/src/runtime/workdir_session.rs index 5a3249c0b..b792557e5 100644 --- a/crates/worker-runtime/src/runtime/workdir_session.rs +++ b/crates/worker-runtime/src/runtime/workdir_session.rs @@ -55,6 +55,10 @@ impl WorkdirSession for RetainedWorkdirSession { self.inner.capabilities() } + fn environment(&self) -> workdir::WorkdirSessionEnvironment { + self.inner.environment() + } + async fn authorize_scope_path( &self, request: WorkdirScopeAuthorizationRequest, diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 9947ce5b0..a67fd9d6a 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -1127,8 +1127,17 @@ fn runtime_local_workdir_router( ), None => source, }; + let cwd = binding + .cwd() + .strip_prefix(binding.root()) + .map_err(|_| format!("Workdir attachment `{alias}` cwd is outside its logical root"))?; + let cwd = workdir::WorkdirPath::new( + cwd.to_str() + .ok_or_else(|| format!("Workdir attachment `{alias}` cwd is not UTF-8"))?, + ) + .map_err(|error| error.to_string())?; router - .attach(alias.clone(), source) + .attach_at_cwd(alias.clone(), source, cwd) .map_err(|error| format!("bind Workdir attachment `{alias}`: {error}"))?; } Ok(router) @@ -10550,6 +10559,60 @@ mod tests { assert!(!binding.working_directory_root().exists()); } + #[cfg(target_os = "linux")] + #[tokio::test] + async fn local_attachment_router_keeps_each_binding_cwd_and_capability_ceiling() { + let runtime_base = tempfile::tempdir().unwrap(); + let repo = create_clean_repo(); + let materializer = RuntimeGitMaterializer::new(runtime_base.path()); + let root_binding = materializer + .create(&working_directory_request(repo.path())) + .unwrap(); + std::fs::create_dir(root_binding.root().join("nested")).unwrap(); + std::fs::write(root_binding.root().join("marker"), "root").unwrap(); + std::fs::write(root_binding.root().join("nested/marker"), "nested").unwrap(); + let mut nested_binding = root_binding.clone(); + nested_binding.cwd = root_binding.root().join("nested"); + let root_alias = WorkdirAttachmentAlias::new("root").unwrap(); + let nested_alias = WorkdirAttachmentAlias::new("nested").unwrap(); + let attachments = BTreeMap::from([ + (root_alias.clone(), root_binding), + (nested_alias.clone(), nested_binding), + ]); + let ceilings = BTreeMap::from([ + (root_alias, WorkdirSessionCapabilities::ALL), + (nested_alias, WorkdirSessionCapabilities::READ_ONLY), + ]); + let router = runtime_local_workdir_router(&attachments, &ceilings, None).unwrap(); + assert!(router.resolve(None).is_err()); + for (alias, cwd, content, command) in [ + ("root", "", "root", true), + ("nested", "nested", "nested", false), + ] { + let selected = router.resolve(Some(alias)).unwrap().session; + let environment = selected.environment(); + assert_eq!(environment.cwd.as_str(), cwd); + assert_eq!( + environment + .capabilities + .supports(workdir::WorkdirSessionCapability::Command), + command + ); + let read = selected + .read(workdir::ReadRequest { + path: workdir::WorkdirPath::new("marker").unwrap(), + offset: 0, + limit: 10, + max_bytes: 100, + }) + .await + .unwrap(); + assert_eq!(read.bytes, content.as_bytes()); + assert_eq!(read.path.as_str(), "marker"); + } + router.close_all().await.unwrap(); + } + #[test] fn spawn_failure_with_existing_working_directory_preserves_workdir() { let runtime_base = tempfile::tempdir().unwrap(); diff --git a/crates/worker-runtime/src/worker_backend/deferred_workdir.rs b/crates/worker-runtime/src/worker_backend/deferred_workdir.rs index 9bcb3962d..02b104efd 100644 --- a/crates/worker-runtime/src/worker_backend/deferred_workdir.rs +++ b/crates/worker-runtime/src/worker_backend/deferred_workdir.rs @@ -124,6 +124,18 @@ macro_rules! impl_deferred_session { self.capabilities } + fn environment(&self) -> WorkdirSessionEnvironment { + let mut environment = self.state.try_lock().ok().and_then(|state| { + state.session.as_ref().map(|session| session.environment()) + }).unwrap_or_else(|| { + let mut environment = WorkdirSessionEnvironment::root(self.capabilities); + environment.restrictions.push("Deferred provider: scope is checked on connection and every operation".into()); + environment + }); + environment.capabilities = self.capabilities; + environment + } + $( async fn $method(&self, $request: $request_type) -> Result<$result, WorkdirError> { self.session().await?.$method($request).await diff --git a/crates/worker/src/authorized_workdir.rs b/crates/worker/src/authorized_workdir.rs index 6967f0db0..27fa85c1d 100644 --- a/crates/worker/src/authorized_workdir.rs +++ b/crates/worker/src/authorized_workdir.rs @@ -206,26 +206,36 @@ impl AuthorizedWorkdirSession { .map(CheckoutSearchResult::Grep), }; }; - scoped - .authorize_scope_path(WorkdirScopeAuthorizationRequest { - rules: vec![WorkdirToolScopeRule { - target: WorkdirPath::root(), - ..rule.clone() - }], - path: relative_path(&root, operation.path())?, - permission: WorkdirToolScopePermission::Read, - }) - .await?; - // Lease cwd rebases paths. Use the same provider scoped-traversal contract - // with attachment output_root instead, retaining typed paths, cursors and - // Grep's rendered filenames without parsing or post-filtering output. - let mut request = CheckoutSearchRequest::new(operation); - request.scope_layers.push(vec![rule]); - self.session.checkout_search(request).await + self.search_with_scope(scoped, root, rule, CheckoutSearchRequest::new(operation)) + .await }) .await } + async fn search_with_scope( + &self, + scoped: WorkdirSessionHandle, + root: WorkdirPath, + rule: WorkdirToolScopeRule, + mut request: CheckoutSearchRequest, + ) -> Result { + scoped + .authorize_scope_path(WorkdirScopeAuthorizationRequest { + rules: vec![WorkdirToolScopeRule { + target: WorkdirPath::root(), + ..rule.clone() + }], + path: relative_path(&root, request.operation.path())?, + permission: WorkdirToolScopePermission::Read, + }) + .await?; + // Preserve the caller's coordinate root, cursors and all existing + // attenuation layers. The provider applies the Host ceiling during + // traversal, not by filtering already-rendered filenames afterward. + request.scope_layers.push(vec![rule]); + self.session.checkout_search(request).await + } + fn authorize_command(&self) -> Result<(), WorkdirError> { // cwd is not a sandbox: arbitrary shell can access the entire attachment. self.authorize(COMMAND, &WorkdirPath::root(), true) @@ -261,6 +271,12 @@ impl WorkdirSession for AuthorizedWorkdirSession { self.session.capabilities() } + fn environment(&self) -> workdir::WorkdirSessionEnvironment { + let mut environment = self.session.environment(); + environment.capabilities = environment.capabilities.intersection(self.capabilities()); + environment + } + async fn authorize_scope_path( &self, mut request: WorkdirScopeAuthorizationRequest, @@ -296,8 +312,14 @@ impl WorkdirSession for AuthorizedWorkdirSession { &self, request: CheckoutSearchRequest, ) -> Result { - self.require_legacy(READ)?; - self.session.checkout_search(request).await + let path = request.operation.path().clone(); + self.with_authorized(READ, &path, true, |scoped, root, rule| async move { + match rule { + Some(rule) => self.search_with_scope(scoped, root, rule, request).await, + None => self.session.checkout_search(request).await, + } + }) + .await } async fn checkout_observe( @@ -490,6 +512,8 @@ mod tests { //! Component tests connect this adapter to the real LocalWorkdirSession. //! Only the Host authorization transport/policy is replaced; these are not E2E tests. + mod environment; + use super::*; use crate::{WorkspaceClientError, WorkspaceRequest, WorkspaceResponse}; use server_api::authorization::ScopedPermission; @@ -842,18 +866,6 @@ mod tests { .await, Err(WorkdirError::Denied(_)) )); - assert!(matches!( - session - .checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::List( - ListRequest { - path: WorkdirPath::root(), - after: None, - limit: 10, - } - ))) - .await, - Err(WorkdirError::Denied(_)) - )); let rule = WorkdirToolScopeRule { target: WorkdirPath::root(), permission: WorkdirToolScopePermission::Read, diff --git a/crates/worker/src/authorized_workdir/tests/environment.rs b/crates/worker/src/authorized_workdir/tests/environment.rs new file mode 100644 index 000000000..95ad94f92 --- /dev/null +++ b/crates/worker/src/authorized_workdir/tests/environment.rs @@ -0,0 +1,198 @@ +//! Normal lib tests connecting live Host-grant checks to real provider traversal +//! and router cwd projection. Only the Host transport is replaced by the parent +//! module's grant fixture; no commands, network or ambient Workspace state are used. + +use super::*; + +#[tokio::test] +async fn authorization_wrapper_preserves_inner_environment_and_capability_ceiling() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir(root.path().join("src")).unwrap(); + let provider = Arc::new(LocalWorkdirSession::materialized( + root.path().to_path_buf(), + root.path().to_path_buf(), + manifest::SharedScope::new(manifest::Scope::writable(root.path()).unwrap()), + WorkdirSessionCapabilities::READ_ONLY, + )); + let broker = WorkdirToolBroker::at_cwd(provider, WorkdirPath::new("src").unwrap()).unwrap(); + let inner = broker.tool_session(); + let expected = inner.environment(); + let client = AuthorizationClient::new(vec![permission(READ, "src", true)]); + let wrapped = authorized_workdir_session(inner, client.clone(), "attachment-id".into()); + assert_eq!(wrapped.environment(), expected); + assert_eq!(wrapped.environment().cwd.as_str(), "src"); + assert_eq!( + wrapped.environment().capabilities, + WorkdirSessionCapabilities::READ_ONLY + ); + assert!( + client.requests.lock().unwrap().is_empty(), + "environment is metadata, not a cached grant decision" + ); + wrapped.close().await.unwrap(); +} + +#[tokio::test] +async fn nonroot_routed_search_retains_host_grants_output_coordinates_and_cursors() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir(root.path().join("src")).unwrap(); + std::fs::create_dir(root.path().join("private")).unwrap(); + for name in ["a.txt", "b.txt"] { + std::fs::write(root.path().join("src").join(name), "needle public marker").unwrap(); + } + std::fs::write( + root.path().join("private/secret.txt"), + "needle private marker", + ) + .unwrap(); + let client = AuthorizationClient::new(vec![permission(READ, "src", true)]); + let authorized = session(&root, client.clone()); + let router = workdir::WorkdirSessionRouter::new(); + let alias = workdir::WorkdirAttachmentAlias::new("checkout").unwrap(); + router + .attach_at_cwd(alias.clone(), authorized, WorkdirPath::new("src").unwrap()) + .unwrap(); + let routed = router.resolve(Some("checkout")).unwrap().session; + assert_eq!(routed.environment().cwd.as_str(), "src"); + let paths = routed + .glob(GlobRequest { + path: WorkdirPath::root(), + pattern: "**/*.txt".into(), + limit: 100, + }) + .await + .unwrap() + .paths; + assert_eq!( + paths, + vec![ + WorkdirPath::new("a.txt").unwrap(), + WorkdirPath::new("b.txt").unwrap() + ] + ); + // Search output is reusable as a path without adding src/ at the caller. + assert_eq!( + routed.read(read(paths[0].as_str())).await.unwrap().bytes, + b"needle public marker" + ); + for output_mode in [ + workdir::GrepOutputMode::Content, + workdir::GrepOutputMode::FilesWithMatches, + workdir::GrepOutputMode::Count, + ] { + let result = routed + .grep(GrepRequest { + path: WorkdirPath::root(), + pattern: "needle".into(), + glob: None, + file_type: None, + case_insensitive: false, + before_context: 0, + after_context: 0, + multiline: false, + output_mode, + limit: 100, + offset: 0, + }) + .await + .unwrap(); + let mut grep_paths = result.paths; + grep_paths.sort_by(|left, right| left.as_str().cmp(right.as_str())); + assert_eq!(grep_paths, paths); + assert!(result.output.contains("a.txt")); + assert!(!result.output.contains("src/")); + assert!(!result.output.contains("secret")); + } + let first = routed + .list(ListRequest { + path: WorkdirPath::root(), + after: None, + limit: 1, + }) + .await + .unwrap(); + assert_eq!(first.entries[0].path.as_str(), "a.txt"); + assert_eq!(first.next_after.as_ref().unwrap().path.as_str(), "a.txt"); + let second = routed + .list(ListRequest { + path: WorkdirPath::root(), + after: first.next_after, + limit: 1, + }) + .await + .unwrap(); + assert_eq!(second.entries[0].path.as_str(), "b.txt"); + assert!(second.next_after.is_none()); + + // An existing caller ceiling must survive the extra Host layer. An empty + // layer is invalid and cannot silently turn into the broader Host grant. + let mut denied = CheckoutSearchRequest::new(CheckoutSearchOperation::Glob(GlobRequest { + path: WorkdirPath::root(), + pattern: "**/*.txt".into(), + limit: 100, + })); + denied.scope_layers.push(Vec::new()); + assert!(routed.checkout_search(denied).await.is_err()); + assert!(client.requests.lock().unwrap().iter().all(|request| { + matches!(&request.resource, PermissionResource::Workdir { path, .. } if path == "src" || path == "src/a.txt") + })); + client.revoked.store(true, Ordering::SeqCst); + assert!( + routed + .glob(GlobRequest { + path: WorkdirPath::root(), + pattern: "**/*.txt".into(), + limit: 100, + }) + .await + .is_err() + ); + router.detach(&alias).await.unwrap(); +} + +#[cfg(target_os = "linux")] +#[tokio::test] +async fn scoped_checkout_search_preserves_caller_layer_and_rejects_resolved_escape() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(root.path().join("src/nested")).unwrap(); + std::fs::create_dir(root.path().join("private")).unwrap(); + std::fs::write(root.path().join("src/public.txt"), "needle public").unwrap(); + std::fs::write(root.path().join("src/nested/hidden.txt"), "needle hidden").unwrap(); + std::fs::write(root.path().join("private/secret.txt"), "needle secret").unwrap(); + std::os::unix::fs::symlink("../private", root.path().join("src/escape")).unwrap(); + std::os::unix::fs::symlink("../private/secret.txt", root.path().join("src/alias.txt")).unwrap(); + let client = AuthorizationClient::new(vec![permission(READ, "src", true)]); + let authorized = session(&root, client); + let request = CheckoutSearchRequest { + operation: CheckoutSearchOperation::Glob(GlobRequest { + path: WorkdirPath::new("src").unwrap(), + pattern: "**/*.txt".into(), + limit: 100, + }), + output_root: WorkdirPath::new("src").unwrap(), + scope_layers: vec![vec![WorkdirToolScopeRule { + target: WorkdirPath::new("src").unwrap(), + recursive: false, + permission: WorkdirToolScopePermission::Read, + symlink_policy: manifest::SymlinkPolicy::Resolved, + }]], + }; + let result = authorized.checkout_search(request).await.unwrap(); + let CheckoutSearchResult::Glob(result) = result else { + panic!("expected Glob result") + }; + assert_eq!(result.paths, vec![WorkdirPath::new("public.txt").unwrap()]); + assert!( + authorized + .checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::Glob( + GlobRequest { + path: WorkdirPath::new("private").unwrap(), + pattern: "**/*.txt".into(), + limit: 100, + } + ))) + .await + .is_err() + ); + authorized.close().await.unwrap(); +} diff --git a/crates/worker/src/checkout_http_tests.rs b/crates/worker/src/checkout_http_tests.rs index 827a5b623..b1402b299 100644 --- a/crates/worker/src/checkout_http_tests.rs +++ b/crates/worker/src/checkout_http_tests.rs @@ -121,6 +121,7 @@ async fn open_session( session_id: WorkdirSessionId::new(SESSION).unwrap(), workdir_id: state.session.workdir().id().clone(), capabilities: state.session.capabilities(), + environment: state.session.environment(), }) .into_response() } @@ -251,6 +252,10 @@ impl Drop for HttpFixture { impl HttpFixture { async fn new(capabilities: WorkdirSessionCapabilities) -> Self { + Self::at_cwd(capabilities, "").await + } + + async fn at_cwd(capabilities: WorkdirSessionCapabilities, cwd: &str) -> Self { let dir = TempDir::new().unwrap(); std::fs::create_dir_all(dir.path().join("src/deep")).unwrap(); std::fs::write( @@ -264,6 +269,13 @@ impl HttpFixture { manifest::SharedScope::new(manifest::Scope::writable(dir.path()).unwrap()), capabilities, )); + let session: WorkdirSessionHandle = if cwd.is_empty() { + session + } else { + workdir::WorkdirToolBroker::at_cwd(session, WorkdirPath::new(cwd).unwrap()) + .unwrap() + .tool_session() + }; let state = Arc::new(ProviderState { session, requests: Mutex::new(Vec::new()), @@ -349,6 +361,31 @@ impl HttpFixture { } } +#[tokio::test] +async fn remote_open_keeps_provider_environment_and_nonroot_operation_coordinates() { + let fixture = HttpFixture::at_cwd(WorkdirSessionCapabilities::READ_ONLY, "src/deep").await; + let remote = fixture.open(TOKEN).await.unwrap(); + assert_eq!(remote.environment(), fixture.state.session.environment()); + assert_eq!(remote.environment().cwd.as_str(), "src/deep"); + assert_eq!( + remote.environment().capabilities, + WorkdirSessionCapabilities::READ_ONLY + ); + let result = remote + .read(workdir::ReadRequest { + path: WorkdirPath::new("a.txt").unwrap(), + offset: 0, + limit: 10, + max_bytes: 4096, + }) + .await + .unwrap(); + assert_eq!(result.bytes, b"first\nneedle needle\nlast\n"); + fixture.public_output(&serde_json::to_string(&remote.environment()).unwrap()); + remote.close().await.unwrap(); + fixture.assert_endpoints(); +} + fn native_runtime(remote: Arc) -> (WipRuntime, Arc) { let router = Arc::new(WorkdirSessionRouter::new()); router diff --git a/crates/worker/src/checkout_race_tests.rs b/crates/worker/src/checkout_race_tests.rs index 3e585db69..def2a253d 100644 --- a/crates/worker/src/checkout_race_tests.rs +++ b/crates/worker/src/checkout_race_tests.rs @@ -78,6 +78,12 @@ impl WorkdirSession for HookedSession { self.local.capabilities() } + fn environment(&self) -> workdir::WorkdirSessionEnvironment { + let mut environment = self.local.environment(); + environment.capabilities = environment.capabilities.intersection(self.capabilities()); + environment + } + async fn authorize_scope_path( &self, request: WorkdirScopeAuthorizationRequest, @@ -201,11 +207,10 @@ async fn checkout_wip_file_operations_share_core_and_keep_prior_read_boundary() let dir = TempDir::new().unwrap(); std::fs::write(dir.path().join("text.txt"), "旧 旧\n").unwrap(); let router = Arc::new(WorkdirSessionRouter::new()); + let hooked = HookedSession::new(&dir); + assert_eq!(hooked.environment(), hooked.local.environment()); router - .attach( - WorkdirAttachmentAlias::new("main").unwrap(), - HookedSession::new(&dir), - ) + .attach(WorkdirAttachmentAlias::new("main").unwrap(), hooked) .unwrap(); let provider = provider(router); let route = format!("{}/text.txt", checkout_root("main")); diff --git a/crates/worker/src/feature/builtin/manage_workdir.rs b/crates/worker/src/feature/builtin/manage_workdir.rs index d8f2302f2..9158821de 100644 --- a/crates/worker/src/feature/builtin/manage_workdir.rs +++ b/crates/worker/src/feature/builtin/manage_workdir.rs @@ -226,6 +226,7 @@ pub struct WorkspaceAttachedWorkdirSession { target_workdir: String, workdir: Workdir, capabilities: WorkdirSessionCapabilities, + environment: workdir::WorkdirSessionEnvironment, } impl WorkspaceAttachedWorkdirSession { @@ -249,17 +250,38 @@ impl WorkspaceAttachedWorkdirSession { ) } + /// Bind a known root-coordinate attachment. Opened provider sessions must + /// instead pass their metadata to [`Self::handle_for_workdir_with_environment`]. pub fn handle_for_workdir_with_capabilities( client: Arc, target_workdir: impl Into, workdir_id: impl Into, capabilities: WorkdirSessionCapabilities, + ) -> WorkdirSessionHandle { + Self::handle_for_workdir_with_environment( + client, + target_workdir, + workdir_id, + capabilities, + workdir::WorkdirSessionEnvironment::root(capabilities), + ) + } + + /// Bind provider-authored coordinates without allowing metadata to expand + /// the Backend attachment capability ceiling. + pub fn handle_for_workdir_with_environment( + client: Arc, + target_workdir: impl Into, + workdir_id: impl Into, + capabilities: WorkdirSessionCapabilities, + environment: workdir::WorkdirSessionEnvironment, ) -> WorkdirSessionHandle { Arc::new(Self { client, target_workdir: target_workdir.into(), workdir: Workdir::new(workdir_id), - capabilities, + capabilities: capabilities.intersection(environment.capabilities), + environment, }) } @@ -362,6 +384,12 @@ impl WorkdirSession for WorkspaceAttachedWorkdirSession { self.capabilities } + fn environment(&self) -> workdir::WorkdirSessionEnvironment { + let mut environment = self.environment.clone(); + environment.capabilities = environment.capabilities.intersection(self.capabilities()); + environment + } + async fn authorize_scope_path( &self, request: WorkdirScopeAuthorizationRequest, @@ -684,11 +712,12 @@ impl WorkspaceHttpWorkdirBackend { let response = self.attach_response(alias, working_directory_id)?; if let Err(error) = self.session_router.attach( alias_key.clone(), - WorkspaceAttachedWorkdirSession::handle_for_workdir_with_capabilities( + WorkspaceAttachedWorkdirSession::handle_for_workdir_with_environment( self.client.clone(), alias, working_directory_id, - response.capabilities, + response.attachment.capabilities, + response.environment.clone(), ), ) { let _ = self.detach(alias); @@ -706,9 +735,9 @@ impl WorkspaceHttpWorkdirBackend { &self, alias: &str, working_directory_id: &str, - ) -> Result { + ) -> Result { let workspace_id = encode_path_segment(self.workspace_id()?); - self.execute_json::(WorkspaceRequest::json( + self.execute_json::(WorkspaceRequest::json( WorkspaceRequestMethod::Post, format!("/api/w/{workspace_id}/workers/self/workdir-attachments"), serde_json::to_string(&WorkdirAttachRequest { @@ -1192,6 +1221,13 @@ struct WorkdirAttachmentResponse { attached: bool, } +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +struct WorkdirSessionAttachmentResponse { + #[serde(flatten)] + attachment: WorkdirAttachmentResponse, + environment: workdir::WorkdirSessionEnvironment, +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct WorkdirDeleteInput { @@ -1502,6 +1538,7 @@ mod tests { "alias": "checkout", "working_directory_id": "wd-created", "capabilities": {"bits": 63}, + "environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL), "attached": true })), response(json!({ @@ -2004,6 +2041,7 @@ mod tests { "alias": "checkout", "working_directory_id": "wd-attached", "capabilities": {"bits": 63}, + "environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL), "attached": true })), response(json!({ @@ -2063,16 +2101,29 @@ mod tests { ); } - #[test] - fn attach_propagates_provider_read_only_capabilities_to_the_alias_session() { + #[tokio::test] + async fn attach_projects_provider_environment_without_expanding_capability_ceiling() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("src")).unwrap(); + let provider = Arc::new(workdir::LocalWorkdirSession::materialized( + dir.path().to_path_buf(), + dir.path().to_path_buf(), + manifest::SharedScope::new(manifest::Scope::writable(dir.path()).unwrap()), + WorkdirSessionCapabilities::ALL, + )); + let broker = + workdir::WorkdirToolBroker::at_cwd(provider, workdir::WorkdirPath::new("src").unwrap()) + .unwrap(); + let mut environment = broker.tool_session().environment(); let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({ "workspace_id": "workspace/test", "alias": "external", "working_directory_id": "external-workdir", "capabilities": {"bits": 25}, + "environment": environment, "attached": true }))])); - let backend = WorkspaceHttpWorkdirBackend::new(client); + let backend = WorkspaceHttpWorkdirBackend::new(client.clone()); backend .attach(WorkdirAttachInput { alias: "external".to_string(), @@ -2084,10 +2135,33 @@ mod tests { .session_router .session(&workdir::WorkdirAttachmentAlias::new("external").unwrap()) .unwrap(); + environment.capabilities = WorkdirSessionCapabilities::READ_ONLY; + assert_eq!(session.environment(), environment); assert_eq!( session.capabilities(), WorkdirSessionCapabilities::READ_ONLY ); + assert!(matches!( + session + .checkout_execute(workdir::CheckoutRequest { + target: workdir::WorkdirPath::root(), + validator: vec![1], + operation: workdir::CheckoutOperation::Create { + path: workdir::WorkdirPath::new("new.txt").unwrap(), + content: b"denied".to_vec(), + }, + }) + .await, + Err(WorkdirError::Unsupported( + workdir::WorkdirSessionCapability::Write + )) + )); + assert_eq!( + client.requests().len(), + 1, + "denied mutation must not reach the Backend" + ); + broker.tool_session().close().await.unwrap(); } #[tokio::test] @@ -2097,6 +2171,7 @@ mod tests { "alias": "checkout", "working_directory_id": "wd-attached", "capabilities": {"bits": 63}, + "environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL), "attached": true }))])); let reopen_calls = Arc::new(AtomicUsize::new(0)); diff --git a/crates/worker/src/feature/builtin/manage_workdir/wip.rs b/crates/worker/src/feature/builtin/manage_workdir/wip.rs index 30cf06dc2..b76941f27 100644 --- a/crates/worker/src/feature/builtin/manage_workdir/wip.rs +++ b/crates/worker/src/feature/builtin/manage_workdir/wip.rs @@ -1253,7 +1253,7 @@ pub(crate) mod tests { state.generation += 1; let raw = json!({"alias":body["alias"], "working_directory_id":body["working_directory_id"], "connection_id":format!("connection-{}", state.generation), "capabilities":{"bits":63}}); state.attachments.push(raw); - json!({"workspace_id":"test-workspace", "alias":body["alias"], "working_directory_id":body["working_directory_id"], "capabilities":{"bits":63}, "attached":true}) + json!({"workspace_id":"test-workspace", "alias":body["alias"], "working_directory_id":body["working_directory_id"], "capabilities":{"bits":63}, "environment":workdir::WorkdirSessionEnvironment::root(workdir::WorkdirSessionCapabilities::ALL), "attached":true}) } (WorkspaceRequestMethod::Delete, p) if p.starts_with("workers/self/workdir-attachments/") => diff --git a/crates/worker/src/job.rs b/crates/worker/src/job.rs index 0837c51a6..d94454141 100644 --- a/crates/worker/src/job.rs +++ b/crates/worker/src/job.rs @@ -13,7 +13,6 @@ mod result; pub use result::{JobResultFeature, JobResultFinalizer, JobResultSink, SUBMIT_JOB_RESULT_TOOL}; -use std::borrow::Cow; use std::path::Path; use std::sync::Arc; @@ -154,7 +153,7 @@ fn prepare_job_from_registry_with_grant( )?; let mut system_prompt = template.render(&SystemPromptContext { now: chrono::Utc::now(), - cwd: Cow::Borrowed("(no filesystem)"), + workdir_sessions: &workdir::WorkdirSessionRouter::new(), language: &manifest.engine.language, scope: &scope, tool_names, diff --git a/crates/worker/src/prompt/system.rs b/crates/worker/src/prompt/system.rs index 17415fbc6..d5e31c2ee 100644 --- a/crates/worker/src/prompt/system.rs +++ b/crates/worker/src/prompt/system.rs @@ -7,13 +7,12 @@ //! eagerly syntax-checks it at Worker construction. The final system //! prompt is materialised exactly once just before the first LLM turn: //! the rendered body is appended with a fixed trailing section carrying -//! the Worker's `Scope` summary, (if present) the project's `AGENTS.md` -//! contents, resident memory sections, and conditional Worker-orchestration -//! guidance, then the whole string is handed to the Engine via -//! `set_system_prompt`. Subsequent turns and compactions reuse that +//! the Worker's `Scope` summary, actual Workdir session environments, (if +//! present) the project's `AGENTS.md` contents, resident memory sections, and +//! conditional Worker-orchestration guidance, then the whole string is handed +//! to the Engine via `set_system_prompt`. Subsequent turns and compactions reuse that //! materialised string verbatim. -use std::borrow::Cow; use std::collections::BTreeMap; use std::sync::Arc; @@ -105,7 +104,8 @@ impl std::fmt::Debug for SystemPromptTemplate { /// templates cannot drop them on the floor. pub struct SystemPromptContext<'a> { pub now: DateTime, - pub cwd: Cow<'a, str>, + /// The same session router used by the Worker's filesystem tools. + pub workdir_sessions: &'a workdir::WorkdirSessionRouter, /// Language policy exposed to instruction templates as `{{ language }}`. pub language: &'a str, pub scope: &'a Scope, @@ -140,7 +140,14 @@ impl<'a> SystemPromptContext<'a> { "datetime".into(), Value::from(self.now.to_rfc3339_opts(SecondsFormat::Secs, true)), ); - root.insert("cwd".into(), Value::from(self.cwd.as_ref())); + // A singular template cwd is meaningful only with one attachment. Never + // substitute the local Profile/Memory discovery directory for a session cwd. + let cwd = self + .workdir_sessions + .only_session() + .map(|session| display_workdir_path(&session.environment().cwd).to_owned()) + .unwrap_or_else(|| "no single Workdir session cwd; see execution environment".into()); + root.insert("cwd".into(), Value::from(cwd)); root.insert("language".into(), Value::from(self.language)); root.insert( "tools".into(), @@ -255,6 +262,70 @@ fn exact_prompt_name(reference: &str) -> Option { } } +fn display_workdir_path(path: &workdir::WorkdirPath) -> &str { + if path.as_str().is_empty() { + "." + } else { + path.as_str() + } +} + +/// Project provider-owned session metadata, rather than reconstructing authority +/// from a manifest Scope or the directory used to discover Profiles and Memory. +pub(crate) fn workdir_environment_section( + router: &workdir::WorkdirSessionRouter, +) -> Result { + use workdir::WorkdirSessionCapability::{Command, Edit, Glob, Grep, Read, Write}; + + let mut environments = Vec::new(); + for alias in router.aliases() { + let resolved = router + .resolve(Some(alias.as_str())) + .map_err(|error| SystemPromptError::Render(error.to_string()))?; + let environment = resolved.session.environment(); + let capabilities: Vec<_> = [ + (Read, "read"), + (Write, "write"), + (Edit, "edit"), + (Glob, "glob"), + (Grep, "grep"), + (Command, "command"), + ] + .into_iter() + .filter_map(|(capability, name)| { + environment + .capabilities + .supports(capability) + .then_some(name) + }) + .collect(); + environments.push(serde_json::json!({ + "alias": alias.as_str(), + "cwd": display_workdir_path(&environment.cwd), + "scope": environment.scope, + "restrictions": environment.restrictions, + "capabilities": capabilities, + "command": environment.capabilities.supports(Command), + })); + } + let json = serde_json::to_string_pretty(&environments) + .map_err(|error| SystemPromptError::Render(error.to_string()))?; + Ok(format!( + "## Workdir execution environment\n\n\ + Select a Workdir by its alias (`target_workdir`); omission is allowed only with one attachment. \ + Relative filesystem operation paths are resolved from that selected session's cwd, not its Workdir root. \ + Paths returned by Glob/Grep use the same cwd-relative coordinates and can be passed unchanged to other filesystem tools. \ + Bash starts at the session cwd; an explicit relative cwd is resolved from it, and shell cd does not change the session cwd.\n\n\ + In the metadata below, cwd and relative scope targets are Workdir-root-relative (`.` or an empty path denotes the root); explicit absolute provider scope targets stay absolute. \ + A null scope means the provider's full available scope, not unrestricted host access; an empty scope grants no paths. \ + Restrictions retain additional provider or ancestor ceilings, including deny rules. \ + Capabilities and command availability are session ceilings, not new grants. \ + Resolved targets remain subject to scope, symlink and command authorization. \ + SubWorker scope declarations remain Workdir-root-relative; other Scope declarations and Profile/Memory roots retain their own meaning and are not rebased by session cwd.\n\n\ + ```json\n{json}\n```" + )) +} + /// Build the final system prompt by appending the fixed trailing /// section to `body`. The Rust side owns the layout (blank-line /// separators, trailing-whitespace trim); each section's header + body @@ -276,8 +347,12 @@ fn append_trailing_section( } out.push('\n'); - let boundaries = prompts.working_boundaries_section(&scope.summary())?; - out.push_str(boundaries.trim_end_matches(&['\n', ' '][..])); + if ctx.workdir_sessions.is_empty() { + let boundaries = prompts.working_boundaries_section(&scope.summary())?; + out.push_str(boundaries.trim_end_matches(&['\n', ' '][..])); + out.push('\n'); + } + out.push_str(&workdir_environment_section(ctx.workdir_sessions)?); out.push('\n'); if let Some(agents) = agents_md { out.push('\n'); @@ -392,7 +467,7 @@ mod tests { now: DateTime::parse_from_rfc3339("2026-10-08T00:00:00Z") .unwrap() .with_timezone(&Utc), - cwd: Cow::Borrowed("."), + workdir_sessions: &workdir::WorkdirSessionRouter::new(), language: "en", scope: &scope, tool_names: metas.iter().map(|meta| meta.name.clone()).collect(), diff --git a/crates/worker/src/spawn/tool.rs b/crates/worker/src/spawn/tool.rs index b88952ea1..9ea143520 100644 --- a/crates/worker/src/spawn/tool.rs +++ b/crates/worker/src/spawn/tool.rs @@ -1107,6 +1107,8 @@ fn sub_worker_spawn_tool_impl( #[cfg(test)] mod tests { + mod environment; + use super::*; use manifest::{DelegationScope, Permission, Scope, SharedScope}; use std::pin::Pin; diff --git a/crates/worker/src/spawn/tool/tests/environment.rs b/crates/worker/src/spawn/tool/tests/environment.rs new file mode 100644 index 000000000..2af0a9788 --- /dev/null +++ b/crates/worker/src/spawn/tool/tests/environment.rs @@ -0,0 +1,629 @@ +//! Connection tests for model-visible environments and actual routed file operations. +//! Runs the production SubWorker spawn/broker/Worker/prompt/tool path with temporary +//! LocalWorkdirSession providers; only the external LLM is scripted. Selected by +//! the normal worker lib suite, without network credentials or live Workspace data. + +use super::*; +use agen::Item; +use serde_json::{Value, json}; +use workdir::{WorkdirSessionCapability, WorkdirSessionRouter}; + +const TASK: &str = "Read marker.txt from each attached Workdir."; + +fn prompt_environments(prompt: &str) -> Vec { + let section = prompt + .split_once("## Workdir execution environment\n") + .expect("Worker must supply the execution environment independently of task text") + .1; + let json = section.split_once("```json\n").unwrap().1; + serde_json::from_str(json.split_once("\n```").unwrap().0).unwrap() +} + +#[derive(Clone, Default)] +struct EnvironmentClient { + requests: Arc>>, + read_issued: Arc, +} + +#[async_trait] +impl LlmClient for EnvironmentClient { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + + async fn stream( + &self, + request: Request, + ) -> Result> + Send>>, ClientError> + { + let environments = prompt_environments(request.system_prompt.as_deref().unwrap()); + self.requests.lock().unwrap().push(request); + let mut events = Vec::new(); + if !self.read_issued.swap(true, Ordering::SeqCst) { + // Use only the model-visible aliases and the same relative path in + // every session. No test-side cwd prefix is passed to the Read tool. + for (index, environment) in environments.iter().enumerate() { + let alias = environment["alias"].as_str().unwrap(); + events.extend([ + LlmEvent::tool_use_start(index, format!("read-{alias}"), "Read"), + LlmEvent::tool_input_delta( + index, + json!({"target_workdir": alias, "file_path": "marker.txt"}).to_string(), + ), + LlmEvent::tool_use_stop(index), + ]); + } + } else { + events.extend([ + LlmEvent::text_block_start(0), + LlmEvent::text_delta(0, "read complete"), + LlmEvent::text_block_stop(0, None), + ]); + } + events.push(LlmEvent::Status(StatusEvent { + status: ResponseStatus::Completed, + })); + Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok)))) + } +} + +struct EnvironmentFixture { + runtime: TempDir, + root: PathBuf, + manifest: WorkerManifest, + profiles: AvailableProfiles, + registry: Arc, + sessions: Arc, +} + +impl EnvironmentFixture { + fn new(aliases: &[(&str, &str)]) -> Self { + let runtime = TempDir::new().unwrap(); + let root = runtime.path().join("profile-root"); + // Profile discovery stays at this root, deliberately outside every + // attached Workdir. Session cwd must never redirect it into work/nested. + let profiles = write_project_profile_registry(&root, None, &[]); + let mut manifest = parent_manifest(&root, None); + manifest.feature.sub_worker.enabled = true; + let registry = SpawnedWorkerRegistry::new_internal( + "parent".into(), + SharedScope::new(Scope::from_config(&manifest.scope).unwrap()), + ); + let sessions = Arc::new(WorkdirSessionRouter::new()); + for (alias, cwd) in aliases { + let checkout = runtime.path().join(alias); + for path in ["", "work", "work/nested", "work/nested/deep", "other"] { + std::fs::create_dir_all(checkout.join(path)).unwrap(); + std::fs::write(checkout.join(path).join("marker.txt"), marker(alias, path)) + .unwrap(); + } + let scope = SharedScope::new(Scope::writable(&checkout).unwrap()); + let session = Arc::new(workdir::LocalWorkdirSession::materialized_bound( + workdir::Workdir::new(format!("fixture-{alias}")), + checkout.clone(), + checkout.join(cwd), + scope, + workdir::WorkdirSessionCapabilities::ALL, + )); + sessions + .attach_at_cwd( + workdir::WorkdirAttachmentAlias::new(*alias).unwrap(), + session, + workdir::WorkdirPath::new(cwd).unwrap(), + ) + .unwrap(); + } + Self { + runtime, + root, + manifest, + profiles, + registry, + sessions, + } + } + + async fn spawn( + &self, + spawner: &str, + registry: Arc, + broker: workdir::WorkdirToolBrokerRouter, + input: Value, + ) -> ( + crate::spawn::registry::InternalSpawnedWorkerRecord, + EnvironmentClient, + ) { + assert_eq!(input["task"], TASK); + let client = EnvironmentClient::default(); + let mut manifest = self.manifest.clone(); + manifest.worker.name = spawner.into(); + let tool = SubWorkerSpawnTool::new( + spawner.into(), + crate::worker::WorkerWorkspaceContext::local_filesystem(None), + ParentNotificationTarget::Durable(Arc::new(|_| {})), + self.runtime.path().to_path_buf(), + self.root.clone(), + Some(broker), + registry.clone(), + manifest, + PromptCatalogSource::builtins_only(), + self.profiles.clone(), + ) + .with_internal_client(Box::new(client.clone())); + tool.execute( + &input.to_string(), + agen::tool::ToolExecutionContext::direct(), + ) + .await + .expect("spawn through production delegation and Worker construction"); + let record = registry + .get_internal(input["name"].as_str().unwrap()) + .unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), record.session.wait_until_idle()) + .await + .expect("scripted read turn must terminate"), + InternalWorkerSessionStatus::Idle, + ); + (record, client) + } +} + +fn marker(alias: &str, cwd: &str) -> String { + format!( + "{alias}:{}:session-marker", + if cwd.is_empty() { "." } else { cwd } + ) +} + +fn assert_environment_and_reads( + router: &WorkdirSessionRouter, + client: &EnvironmentClient, + expected: &[(&str, &str, bool)], +) -> String { + let requests = client.requests.lock().unwrap(); + assert!( + requests.len() >= 2, + "Read must execute and return to the model" + ); + let prompt = requests[0].system_prompt.as_deref().unwrap(); + assert!( + !prompt.contains("## Working boundaries"), + "legacy Scope must not override the connected session environment" + ); + let environments = prompt_environments(prompt); + assert_eq!(environments.len(), expected.len()); + assert_eq!(router.len(), expected.len()); + for (alias, cwd, command) in expected { + let environment = router.resolve(Some(alias)).unwrap().session.environment(); + assert_eq!(environment.cwd.as_str(), *cwd); + assert_eq!( + environment + .capabilities + .supports(WorkdirSessionCapability::Command), + *command + ); + let projected = environments + .iter() + .find(|env| env["alias"] == *alias) + .unwrap(); + assert_eq!(projected["cwd"], if cwd.is_empty() { "." } else { *cwd }); + assert_eq!( + projected["scope"], + serde_json::to_value(&environment.scope).unwrap() + ); + assert_eq!( + projected["restrictions"], + serde_json::to_value(&environment.restrictions).unwrap() + ); + assert_eq!(projected["command"], *command); + for (capability, name) in [ + (WorkdirSessionCapability::Read, "read"), + (WorkdirSessionCapability::Write, "write"), + (WorkdirSessionCapability::Edit, "edit"), + (WorkdirSessionCapability::Glob, "glob"), + (WorkdirSessionCapability::Grep, "grep"), + (WorkdirSessionCapability::Command, "command"), + ] { + assert_eq!( + projected["capabilities"] + .as_array() + .unwrap() + .iter() + .any(|value| value == name), + environment.capabilities.supports(capability), + ); + } + let expected_marker = marker(alias, cwd); + let observed_read = requests + .iter() + .flat_map(|request| &request.items) + .any(|item| { + matches!(item, Item::ToolResult { call_id, content: Some(content), .. } + if call_id == &format!("read-{alias}") && content.contains(&expected_marker)) + }); + assert!( + observed_read, + "{alias}: relative Read did not return {expected_marker}" + ); + } + prompt.to_owned() +} + +fn assert_retained_environment( + record: &crate::spawn::registry::InternalSpawnedWorkerRecord, + prompt: &str, +) { + let retained = record.session.entries().iter().any(|entry| { + matches!(entry, session_store::LogEntry::AnnotatedSegmentStart { system_prompt: Some(saved), .. } + if saved == prompt) + }); + assert!( + retained, + "the model-visible environment must also be retained in session history" + ); +} + +#[tokio::test] +async fn spawn_root_and_nonroot_cwd_are_projected_without_task_annotations() { + for (name, cwd, command, permission) in [ + ("root-child", "", false, "read"), + ("cwd-child", "work", true, "write"), + ] { + let fixture = EnvironmentFixture::new(&[("checkout", "")]); + let (record, client) = fixture + .spawn( + "parent", + fixture.registry.clone(), + workdir::WorkdirToolBrokerRouter::new(fixture.sessions.clone()), + json!({ + "name": name, "task": TASK, "cwd": if cwd.is_empty() { "." } else { cwd }, + "scope": [{"target": ".", "permission": permission}], + "command": command, + }), + ) + .await; + let router = record.workdir_tool_scope.session_router(); + let prompt = assert_environment_and_reads(&router, &client, &[("checkout", cwd, command)]); + assert_retained_environment(&record, &prompt); + assert_eq!( + record.installed_tools.iter().any(|tool| tool == "Bash"), + command + ); + fixture.registry.remove_internal(name).await.unwrap(); + } +} + +#[tokio::test] +async fn spawn_multiple_aliases_and_nested_child_keep_real_session_environment() { + let fixture = EnvironmentFixture::new(&[("alpha", ""), ("beta", "")]); + let (child, child_client) = fixture + .spawn( + "parent", + fixture.registry.clone(), + workdir::WorkdirToolBrokerRouter::new(fixture.sessions.clone()), + json!({ + "name": "child", "task": TASK, "cwd": "work", "command": true, + "scope": [ + {"target_workdir": "alpha", "target": ".", "permission": "write"}, + {"target_workdir": "beta", "target": ".", "permission": "write"}, + ], + }), + ) + .await; + let child_router = child.workdir_tool_scope.session_router(); + let child_prompt = assert_environment_and_reads( + &child_router, + &child_client, + &[("alpha", "work", true), ("beta", "work", true)], + ); + assert_retained_environment(&child, &child_prompt); + assert!( + child + .installed_tools + .iter() + .any(|tool| tool == "SubWorkerSpawn") + ); + + // Reuse the actual child's registry and attenuated broker. The nested tool + // receives the scripted LLM at the same boundary as the first spawn; session + // construction, routing, delegation and prompt materialization remain real. + let (nested, nested_client) = fixture + .spawn( + "child", + child.child_registry.clone(), + child.workdir_tool_scope.broker_router(), + json!({ + "name": "nested", "task": TASK, "cwd": "work/nested", + "scope": [ + {"target_workdir": "alpha", "target": "work/nested", "permission": "read"}, + {"target_workdir": "beta", "target": "work/nested", "permission": "read"}, + ], + }), + ) + .await; + let nested_router = nested.workdir_tool_scope.session_router(); + let nested_prompt = assert_environment_and_reads( + &nested_router, + &nested_client, + &[ + ("alpha", "work/nested", false), + ("beta", "work/nested", false), + ], + ); + assert_retained_environment(&nested, &nested_prompt); + assert!(!nested.installed_tools.iter().any(|tool| tool == "Bash")); + for alias in ["alpha", "beta"] { + let session = nested_router.resolve(Some(alias)).unwrap().session; + let environment = session.environment(); + let scope = environment + .scope + .expect("nested session must remain scoped"); + assert_eq!(scope.len(), 1); + assert_eq!(scope[0].target.as_str(), "work/nested"); + assert_eq!( + scope[0].permission, + workdir::WorkdirToolScopePermission::Read + ); + assert!( + session + .read(workdir::ReadRequest { + path: workdir::WorkdirPath::new_scoped( + fixture + .runtime + .path() + .join(alias) + .join("marker.txt") + .display() + .to_string(), + ) + .unwrap(), + offset: 0, + limit: 10, + max_bytes: 1024, + }) + .await + .is_err() + ); + } + // Grandchild cwd and reduced command authority must not rewrite its parent. + assert_environment_and_reads( + &child_router, + &child_client, + &[("alpha", "work", true), ("beta", "work", true)], + ); + child + .child_registry + .remove_internal("nested") + .await + .unwrap(); + fixture.registry.remove_internal("child").await.unwrap(); +} + +#[tokio::test] +async fn spawn_multiple_aliases_keep_root_scope_independent_of_parent_session_cwds() { + let fixture = EnvironmentFixture::new(&[("alpha", "work"), ("beta", "other")]); + let (record, client) = fixture + .spawn( + "parent", + fixture.registry.clone(), + workdir::WorkdirToolBrokerRouter::new(fixture.sessions.clone()), + json!({ + "name": "child", "task": TASK, "cwd": "work/nested", + "scope": [ + {"target_workdir": "alpha", "target": "work/nested", "permission": "read"}, + {"target_workdir": "beta", "target": "work/nested", "permission": "read"}, + ], + }), + ) + .await; + let prompt = assert_environment_and_reads( + &record.workdir_tool_scope.session_router(), + &client, + &[ + ("alpha", "work/nested", false), + ("beta", "work/nested", false), + ], + ); + assert_retained_environment(&record, &prompt); + for (alias, cwd) in [("alpha", "work"), ("beta", "other")] { + assert_eq!( + fixture + .sessions + .resolve(Some(alias)) + .unwrap() + .session + .environment() + .cwd + .as_str(), + cwd + ); + } + fixture.registry.remove_internal("child").await.unwrap(); +} + +#[tokio::test] +async fn nonroot_startup_then_spawn_and_nested_child_keep_root_relative_scope() { + let fixture = EnvironmentFixture::new(&[("workdir", "")]); + let checkout = fixture.runtime.path().join("workdir"); + let client = EnvironmentClient::default(); + let mut manifest = fixture.manifest.clone(); + manifest.scope.allow = vec![abs_rule(&checkout, Permission::Write)]; + let mut worker = + Worker::, EphemeralSessionStore>::from_internal_manifest_with_context( + manifest, + EphemeralSessionStore::default(), + PromptCatalogSource::builtins_only(), + crate::worker::WorkerWorkspaceContext::local_filesystem(None), + WorkerFilesystemAuthority::local(checkout.clone(), checkout.join("work")), + Some(Box::new(client.clone())), + ) + .await + .unwrap(); + // Do not bind a test router: exercise workdir_sessions_from_authority via + // the real constructor and the normal controller's broker selection. + let startup = worker.workdir_sessions(); + register_worker_tools( + &mut worker, + fixture.runtime.path().to_path_buf(), + fixture.registry.clone(), + None, + None, + ) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), worker.run_text(TASK)) + .await + .expect("startup read turn must terminate") + .unwrap(); + assert_environment_and_reads(&startup, &client, &[("workdir", "work", true)]); + + let (child, child_client) = fixture + .spawn( + "parent", + fixture.registry.clone(), + workdir::WorkdirToolBrokerRouter::new(startup.clone()), + json!({ + "name": "child", "task": TASK, "cwd": "work/nested", + "scope": [{"target": "work/nested", "permission": "read"}], + }), + ) + .await; + let child_router = child.workdir_tool_scope.session_router(); + let child_prompt = assert_environment_and_reads( + &child_router, + &child_client, + &[("workdir", "work/nested", false)], + ); + assert_retained_environment(&child, &child_prompt); + let (nested, nested_client) = fixture + .spawn( + "child", + child.child_registry.clone(), + child.workdir_tool_scope.broker_router(), + json!({ + "name": "nested", "task": TASK, "cwd": "work/nested/deep", + "scope": [{"target": "work/nested/deep", "permission": "read"}], + }), + ) + .await; + let nested_router = nested.workdir_tool_scope.session_router(); + let nested_prompt = assert_environment_and_reads( + &nested_router, + &nested_client, + &[("workdir", "work/nested/deep", false)], + ); + assert_retained_environment(&nested, &nested_prompt); + for (router, target) in [ + (&child_router, "work/nested"), + (&nested_router, "work/nested/deep"), + ] { + let environment = router.resolve(None).unwrap().session.environment(); + let rules = environment.scope.unwrap(); + assert_eq!(rules.len(), 1); + assert_eq!(rules[0].target.as_str(), target); + } + assert_eq!( + startup + .resolve(None) + .unwrap() + .session + .environment() + .cwd + .as_str(), + "work" + ); + let alias = workdir::WorkdirAttachmentAlias::new("workdir").unwrap(); + startup.begin_detach(&alias).await.unwrap(); + // Both the parent tool view and descendant root-coordinate broker source + // must share this fence, rather than bypassing it through a cached broker. + for router in [&startup, &child_router, &nested_router] { + let session = router.resolve(None).unwrap().session; + assert!( + session + .read(workdir::ReadRequest { + path: workdir::WorkdirPath::new("marker.txt").unwrap(), + offset: 0, + limit: 10, + max_bytes: 1024, + }) + .await + .is_err() + ); + } + startup.cancel_detach(&alias); + child + .child_registry + .remove_internal("nested") + .await + .unwrap(); + fixture.registry.remove_internal("child").await.unwrap(); +} + +#[tokio::test] +async fn startup_rejects_cwd_outside_workdir_root_without_running_the_model() { + let fixture = EnvironmentFixture::new(&[("workdir", "")]); + let client = EnvironmentClient::default(); + let mut manifest = fixture.manifest.clone(); + manifest.scope.allow = vec![abs_rule(fixture.runtime.path(), Permission::Write)]; + let error = + Worker::, EphemeralSessionStore>::from_internal_manifest_with_context( + manifest, + EphemeralSessionStore::default(), + PromptCatalogSource::builtins_only(), + crate::worker::WorkerWorkspaceContext::local_filesystem(None), + WorkerFilesystemAuthority::local( + fixture.runtime.path().join("workdir"), + fixture.root.clone(), + ), + Some(Box::new(client.clone())), + ) + .await + .err() + .expect("a session cwd must be representable within its Workdir root"); + assert!(matches!(error, crate::WorkerError::InvalidCwd { .. })); + assert!(client.requests.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn worker_multiple_aliases_project_distinct_session_cwds_not_discovery_root() { + for prompt_override in [None, Some("Feature-owned instruction body")] { + let fixture = EnvironmentFixture::new(&[("alpha", "work"), ("beta", "other")]); + let client = EnvironmentClient::default(); + let mut worker = Worker::, EphemeralSessionStore>::from_internal_manifest_with_context( + fixture.manifest.clone(), + EphemeralSessionStore::default(), + PromptCatalogSource::builtins_only(), + crate::worker::WorkerWorkspaceContext::local_filesystem(None), + WorkerFilesystemAuthority::None, + Some(Box::new(client.clone())), + ) + .await + .unwrap(); + worker.bind_workdir_sessions(fixture.sessions.clone()); + register_worker_tools( + &mut worker, + fixture.runtime.path().to_path_buf(), + fixture.registry.clone(), + None, + Some(workdir::WorkdirToolBrokerRouter::new( + fixture.sessions.clone(), + )), + ) + .await + .unwrap(); + worker.install_system_prompt_contribution(None, prompt_override.map(str::to_owned)); + tokio::time::timeout(Duration::from_secs(5), worker.run_text(TASK)) + .await + .expect("scripted worker turn must terminate") + .unwrap(); + let prompt = assert_environment_and_reads( + &fixture.sessions, + &client, + &[("alpha", "work", true), ("beta", "other", true)], + ); + if let Some(body) = prompt_override { + assert!(prompt.starts_with(body)); + } + } +} diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 2f609b735..f46be3d4a 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -3700,7 +3700,7 @@ impl Worker { let prompts = Arc::new(ArcSwap::from(PromptCatalog::builtins_only()?)); DelegationScope::from_config(&manifest.delegation_scope).map_err(WorkerError::Scope)?; let scope = SharedScope::new(scope); - let workdir_sessions = workdir_sessions_from_authority(&filesystem_authority, &scope); + let workdir_sessions = workdir_sessions_from_authority(&filesystem_authority, &scope)?; let mut worker = Self { backend_job: std::sync::OnceLock::new(), manifest, @@ -4980,11 +4980,14 @@ impl Worker { let Some(template) = self.system_prompt_template.as_ref() else { return Ok(()); }; - if let Some(rendered) = self.feature_system_prompt_override.as_ref() { + if let Some(body) = self.feature_system_prompt_override.as_ref() { + let environment = + crate::prompt::system::workdir_environment_section(&self.workdir_sessions) + .map_err(|source| WorkerError::SystemPromptRender { source })?; self.engine .as_mut() .expect("worker present") - .set_system_prompt(rendered.clone()); + .set_system_prompt(format!("{body}\n\n{environment}")); return Ok(()); } let alerter = self.alerter.clone(); @@ -5029,14 +5032,10 @@ impl Worker { }; let worker_language = worker_language(&self.manifest.engine); let scope_snapshot = self.scope.snapshot(); - let cwd_for_prompt = self - .local_working_directory() - .map(|local| local.cwd.display().to_string()) - .unwrap_or_else(|| "no local working directory".to_string()); let prompt_catalog = self.prompts.load_full(); let ctx = SystemPromptContext { now: chrono::Utc::now(), - cwd: cwd_for_prompt.into(), + workdir_sessions: &self.workdir_sessions, language: worker_language, scope: &scope_snapshot, tool_names, @@ -8306,7 +8305,7 @@ where let worker_metadata_segment_cas = Some(worker_metadata_segment_cas_for_store(&store)); let scope = SharedScope::new(common.scope); let workdir_sessions = - workdir_sessions_from_authority(&common.filesystem_authority, &scope); + workdir_sessions_from_authority(&common.filesystem_authority, &scope)?; let mut worker = Self { backend_job: std::sync::OnceLock::new(), @@ -8707,7 +8706,7 @@ where engine.set_cache_key(Some(segment_id.to_string())); let scope = SharedScope::new(common.scope); let workdir_sessions = - workdir_sessions_from_authority(&common.filesystem_authority, &scope); + workdir_sessions_from_authority(&common.filesystem_authority, &scope)?; let mut worker = Self { backend_job: std::sync::OnceLock::new(), manifest, @@ -9109,7 +9108,7 @@ where let worker_metadata_segment_cas = Some(worker_metadata_segment_cas_for_store(&store)); let scope = SharedScope::new(common.scope); let workdir_sessions = - workdir_sessions_from_authority(&common.filesystem_authority, &scope); + workdir_sessions_from_authority(&common.filesystem_authority, &scope)?; let mut worker = Self { backend_job: std::sync::OnceLock::new(), @@ -10220,9 +10219,25 @@ pub enum WorkerError { fn workdir_sessions_from_authority( authority: &WorkerFilesystemAuthority, scope: &SharedScope, -) -> Arc { +) -> Result, WorkerError> { let router = Arc::new(WorkdirSessionRouter::new()); if let Some(local) = authority.as_local() { + let invalid_cwd = |error: String| WorkerError::InvalidCwd { + cwd: local.cwd.clone(), + source: std::io::Error::new(std::io::ErrorKind::InvalidInput, error), + }; + let relative_cwd = local.cwd.strip_prefix(&local.root).map_err(|error| { + invalid_cwd(format!( + "session cwd must be within its Workdir root: {error}" + )) + })?; + let relative_cwd = relative_cwd + .to_str() + .ok_or_else(|| invalid_cwd("session cwd must be UTF-8".into()))?; + let relative_cwd = workdir::WorkdirPath::new(relative_cwd) + .map_err(|error| invalid_cwd(error.to_string()))?; + // The provider keeps root-coordinate operations. Only the routed tool + // view uses cwd; delegation reuses the same root-coordinate broker. let session = Arc::new(LocalWorkdirSession::materialized( local.root.clone(), local.cwd.clone(), @@ -10230,13 +10245,14 @@ fn workdir_sessions_from_authority( WorkdirSessionCapabilities::ALL, )) as WorkdirSessionHandle; router - .attach( + .attach_at_cwd( WorkdirAttachmentAlias::new("workdir").expect("static alias is valid"), session, + relative_cwd, ) - .expect("fresh Workdir router accepts its only attachment"); + .map_err(|error| invalid_cwd(error.to_string()))?; } - router + Ok(router) } /// Bundle of resources that every high-level Worker constructor needs: diff --git a/crates/worker/tests/system_prompt_template_test.rs b/crates/worker/tests/system_prompt_template_test.rs index 727990c6e..4c351336d 100644 --- a/crates/worker/tests/system_prompt_template_test.rs +++ b/crates/worker/tests/system_prompt_template_test.rs @@ -187,7 +187,8 @@ async fn materialise_on_first_turn_populates_worker() { .expect("system prompt materialised") .to_string(); assert!(rendered.contains("date=")); - assert!(rendered.contains(&pwd.display().to_string())); + assert!(rendered.contains("## Workdir execution environment")); + assert!(!rendered.contains(&pwd.display().to_string())); assert!(rendered.starts_with("date=")); } @@ -206,7 +207,8 @@ async fn session_start_state_captures_rendered_prompt() { LogEntry::AnnotatedSegmentStart { system_prompt, .. } => { let sp = system_prompt.as_deref().expect("system prompt set"); assert!(sp.starts_with("hello")); - assert!(sp.contains(&pwd.display().to_string())); + assert!(sp.contains("## Workdir execution environment")); + assert!(!sp.contains(&pwd.display().to_string())); } other => panic!("expected AnnotatedSegmentStart as first entry, got {other:?}"), } diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index d2fe3ee54..0f9cd24a1 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -8827,6 +8827,9 @@ mod tests { session_id: workdir::http::WorkdirSessionId::new("session-1").unwrap(), workdir_id: Workdir::new("wd-1").id().clone(), capabilities: workdir::WorkdirSessionCapabilities::ALL, + environment: workdir::WorkdirSessionEnvironment::root( + workdir::WorkdirSessionCapabilities::ALL, + ), }; let stat = workdir::http::WorkdirSessionOperationResult::Stat(StatResult { path: WorkdirPath::new("hello.txt").unwrap(), diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 9461b46ea..809f7d203 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -20656,6 +20656,55 @@ async fn import_current_worker_workdir_observation( Ok(()) } +// The Backend proxy has root-coordinate operations even before its provider is +// opened. Multiple attachments must remain attachable without opening every +// provider: a pending/offline provider is checked when an operation selects it. +async fn current_worker_attachment_environment_locked( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + link: &WorkerWorkdirLinkRecord, + capabilities: workdir::WorkdirSessionCapabilities, + open_if_single: bool, + session_guard: &Arc>, +) -> Result { + if open_if_single { + let stage_api = api.clone(); + let stage_worker = worker.clone(); + let count = api + .store + .db_executor() + .run(move || { + active_filesystem_workdir_links(&stage_api, &stage_worker).map(|links| links.len()) + }) + .await?; + if count == 1 { + return Ok(open_current_worker_workdir_session_locked( + api, + worker, + link, + session_guard, + ) + .await? + .environment()); + } + } + if let Some(mut environment) = api + .workdir_sessions + .lock() + .expect("Workdir session registry lock poisoned") + .attachments + .get(worker) + .filter(|source| source.workdir().id().as_str() == link.workdir_id) + .map(|source| source.environment()) + { + environment.capabilities = environment.capabilities.intersection(capabilities); + return Ok(environment); + } + let mut environment = workdir::WorkdirSessionEnvironment::root(capabilities); + environment.restrictions.push("Backend proxy: provider scope and availability are checked when each operation selects this attachment".into()); + Ok(environment) +} + async fn scoped_attach_current_worker_workdir( State(api): State, AxumPath(path): AxumPath, @@ -20782,17 +20831,17 @@ async fn scoped_attach_current_worker_workdir( } let effective_capabilities = existing.capabilities.intersection(capabilities); if effective_capabilities != existing.capabilities { - let mut attenuated = existing; + let mut attenuated = existing.clone(); attenuated.capabilities = effective_capabilities; api.store.attach_worker_workdir(&attenuated)?; } - Ok(Some(effective_capabilities)) + Ok(Some((existing, effective_capabilities))) } else { Ok(None) } }) .await?; - if let Some(effective_capabilities) = existing { + if let Some((link, effective_capabilities)) = existing { sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await?; api.worker_projection .refresh_async(&worker) @@ -20803,6 +20852,17 @@ async fn scoped_attach_current_worker_workdir( alias: alias.to_string(), working_directory_id: workdir_id.to_string(), capabilities: effective_capabilities, + environment: Some( + current_worker_attachment_environment_locked( + &api, + &worker, + &link, + effective_capabilities, + false, + &_session_guard, + ) + .await?, + ), attached: true, })); } @@ -20828,25 +20888,25 @@ async fn scoped_attach_current_worker_workdir( detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await; return Err(error); } - let stage_api = api.clone(); - let stage_worker = worker.clone(); - let active_count = api - .store - .db_executor() - .run(move || { - active_filesystem_workdir_links(&stage_api, &stage_worker).map(|links| links.len()) - }) - .await?; - if active_count == 1 - && let Err(error) = - open_current_worker_workdir_session_locked(&api, &worker, &link, &_session_guard).await + let environment = match current_worker_attachment_environment_locked( + &api, + &worker, + &link, + capabilities, + true, + &_session_guard, + ) + .await { - let _ = - detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard).await; - let _ = - sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard).await; - return Err(error.into()); - } + Ok(environment) => environment, + Err(error) => { + let _ = detach_worker_workdir_stage_locked(&api, &worker, workdir_id, &_session_guard) + .await; + let _ = sync_runtime_worker_workdir_attachments_locked(&api, &worker, &_session_guard) + .await; + return Err(error.into()); + } + }; if let Err(error) = api .worker_projection .refresh_async(&worker) @@ -20863,6 +20923,7 @@ async fn scoped_attach_current_worker_workdir( alias: alias.to_string(), working_directory_id: workdir_id.to_string(), capabilities, + environment: Some(environment), attached: true, })) } @@ -21105,6 +21166,7 @@ async fn detach_current_worker_workdir( alias: alias.to_string(), working_directory_id: link.workdir_id, capabilities: workdir::WorkdirSessionCapabilities::EMPTY, + environment: None, attached: false, }) } @@ -60835,6 +60897,12 @@ mod tests { serde_json::from_slice(&attached_body).unwrap(); assert!(attached.attached); assert_eq!(attached.working_directory_id, fixture.docs_workdir_id); + let environment = attached + .environment + .expect("attached proxy has an execution environment"); + assert!(environment.cwd.is_root()); + assert_eq!(environment.capabilities, attached.capabilities); + assert!(!environment.restrictions.is_empty()); assert_eq!( api.store .get_workdir_registry(TEST_WORKSPACE_ID, &fixture.docs_workdir_id) diff --git a/crates/workspace-server/src/server/tests/workdir_delegation_tests.rs b/crates/workspace-server/src/server/tests/workdir_delegation_tests.rs index 17240665a..b421c1b43 100644 --- a/crates/workspace-server/src/server/tests/workdir_delegation_tests.rs +++ b/crates/workspace-server/src/server/tests/workdir_delegation_tests.rs @@ -200,6 +200,7 @@ async fn backend_delegation_reaches_remote_repository_resolution() { session_id: WorkdirSessionId::new("delegation-session").unwrap(), workdir_id, capabilities: Caps::ALL, + environment: workdir::WorkdirSessionEnvironment::root(Caps::ALL), }) } }), From 994e75233be390633c179e763cb4c8930815cb82 Mon Sep 17 00:00:00 2001 From: Hare Date: Thu, 22 Oct 2026 22:25:47 +0900 Subject: [PATCH 18/46] feat(session): share immutable captures and bounded body transport --- crates/session-store/README.md | 2 + crates/session-store/src/capture.rs | 948 +++++++++++++++ crates/session-store/src/capture/body.rs | 934 +++++++++++++++ crates/session-store/src/lib.rs | 1 + .../session-store/tests/capture_contract.rs | 594 ++++++++++ .../src/feature/builtin/flow_transition.rs | 13 +- .../src/feature/builtin/memory_lifecycle.rs | 40 +- .../feature/builtin/memory_staging_output.rs | 13 +- .../src/feature/builtin/session_explore.rs | 184 ++- .../src/feature/builtin/subjektiv_memory.rs | 91 +- .../src/feature/builtin/worker_observation.rs | 14 +- crates/worker/src/session_capture.rs | 1038 +++-------------- crates/worker/src/worker.rs | 6 +- 13 files changed, 2923 insertions(+), 955 deletions(-) create mode 100644 crates/session-store/src/capture.rs create mode 100644 crates/session-store/src/capture/body.rs create mode 100644 crates/session-store/tests/capture_contract.rs diff --git a/crates/session-store/README.md b/crates/session-store/README.md index 1aecdad4a..4a3b9bbd3 100644 --- a/crates/session-store/README.md +++ b/crates/session-store/README.md @@ -11,5 +11,7 @@ checks or Runtime catalog lifecycle management. - [`worker_session_store`]: storage boundary for a Session bound to a Worker - [`worker_metadata`]: current Worker state and active Segment switching - [`system_item`]: records of system input presented to the model +- [`capture`]: Worker-independent immutable public values and bounded body transport; + authorization, source I/O, and delivery lifecycle stay with the Host - [`public_snapshot`] / [`public_index`]: public projections of stored history - [`event_trace`]: diagnostic records independent of history replay diff --git a/crates/session-store/src/capture.rs b/crates/session-store/src/capture.rs new file mode 100644 index 000000000..4c9b4d3bc --- /dev/null +++ b/crates/session-store/src/capture.rs @@ -0,0 +1,948 @@ +//! Exploration domain for immutable Session captures, with no Workspace/Memory mutation capability. +//! +//! A capture is an ordered projection of committed entries; exploration excludes reasoning and +//! system prompts, including Host compaction summaries and other derived context. Full inherited +//! summary input belongs to compaction, not public observation or Memory evidence. Overview, +//! search, bounded reads, and evidence conversion share `SessionEntryRef`. +//! `SessionEntryRef` is a contextual UI reference. `ScopedSessionEntryRef` additionally binds +//! the Session and exact immutable capture, and is required for portable checked operations. +//! Appends preserve existing entry identities. Sparse real user/assistant anchors and intervening +//! entry counts index long histories. Search with an empty filter also provides bounded range lists. +//! +//! Host-fixed snapshot exploration, Memory candidate evidence validation, and current Worker +//! observation share only this domain; they do not enable or modify each other's Features. +//! The provider owns authorization to recapture a current Worker. A capture itself proves neither +//! completion nor approval and grants no business-mutation authority. + +use std::sync::Arc; + +use crate::{ + LoggedSessionHistoryMetadata as SessionHistoryMetadata, + LoggedSessionHistoryOrigin as HistoryOrigin, +}; +mod body; +use agen::{HistoryEntry, Item, Role}; +pub use body::*; +use protocol::{ + SessionContentPart, SessionEntryProvenance, SessionMessageRole, SessionSnapshot, + SessionSnapshotEntryData, +}; +use serde::{Deserialize, Serialize}; + +const DEFAULT_SEARCH_LIMIT: usize = 20; +const MAX_SEARCH_LIMIT: usize = 50; +const DEFAULT_READ_MAX_ITEMS: usize = 40; +const MAX_READ_MAX_ITEMS: usize = 80; +const DEFAULT_READ_MAX_BYTES: usize = 32 * 1024; +const OVERVIEW_ANCHOR_STRIDE: usize = 8; + +/// Append-stable contextual UI reference derived from a logical entry identity. +/// Portable consumers must bind it with `SessionCapture::scoped_ref` before use. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct SessionEntryRef(String); + +impl SessionEntryRef { + pub fn from_history_entry_id(entry_id: &crate::LoggedSessionHistoryEntryId) -> Self { + Self(format!("E{}", entry_id.0)) + } + + pub fn parse(value: &str) -> Option { + let suffix = value.strip_prefix('E')?; + if suffix.is_empty() + || suffix.len() > 64 + || !suffix + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + { + return None; + } + Some(Self(value.to_string())) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for SessionEntryRef { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReferenceKind { + User, + Assistant, + System, + Tool, +} + +impl ReferenceKind { + pub fn as_str(self) -> &'static str { + match self { + Self::User => "user", + Self::Assistant => "assistant", + Self::System => "system", + Self::Tool => "tool", + } + } + + pub fn parse(value: &str) -> Option { + match value { + "user" => Some(Self::User), + "assistant" | "agent" => Some(Self::Assistant), + "system" => Some(Self::System), + "tool" => Some(Self::Tool), + _ => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ToolPart { + Input, + Output, + Both, +} + +impl ToolPart { + pub fn parse(value: &str) -> Option { + match value { + "input" => Some(Self::Input), + "output" => Some(Self::Output), + "both" => Some(Self::Both), + _ => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Input => "input", + Self::Output => "output", + Self::Both => "both", + } + } + + pub fn matches(self, actual: ToolPart) -> bool { + matches!(self, ToolPart::Both) || self == actual + } +} + +#[derive(Debug, Clone)] +pub struct OverviewItem { + pub id: SessionEntryRef, + pub origin: SessionEntryProvenance, + pub entry_range: [u64; 2], + pub kind: ReferenceKind, + pub label: String, + pub text: String, + pub intervening_entries: usize, +} + +#[derive(Debug, Clone)] +pub struct ReferenceEntry { + pub attachment_count: usize, + use_policy: Option, + pub id: SessionEntryRef, + pub origin: SessionEntryProvenance, + pub entry_range: [u64; 2], + pub kind: ReferenceKind, + pub tool_part: Option, + pub tool_name: Option, + pub label: String, + pub summary: String, + search_text: String, +} + +#[derive(Debug, Clone, Default)] +pub struct SearchOptions { + pub query: String, + pub kind: Option, + pub tool_part: Option, + pub tool_name: Option, + pub limit: Option, + pub min_entry_index: Option, + pub from: Option, + pub through: Option, + pub offset: usize, +} + +#[derive(Debug, Clone)] +pub struct SearchHit { + pub id: SessionEntryRef, + pub origin: SessionEntryProvenance, + pub kind: ReferenceKind, + pub tool_part: Option, + pub tool_name: Option, + pub entry_range: [u64; 2], + pub label: String, + pub summary: String, +} + +#[derive(Debug, Clone, Copy)] +pub enum ReadSelector<'a> { + Id(&'a str), + EntryRange([u64; 2]), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReadDetail { + Compact, + Full, +} + +#[derive(Debug, Clone)] +pub struct ReadOptions { + pub include_tools: bool, + pub tool_part: ToolPart, + pub detail: ReadDetail, + pub max_items: usize, + pub max_bytes: usize, +} + +impl Default for ReadOptions { + fn default() -> Self { + Self { + include_tools: true, + tool_part: ToolPart::Both, + detail: ReadDetail::Compact, + max_items: DEFAULT_READ_MAX_ITEMS, + max_bytes: DEFAULT_READ_MAX_BYTES, + } + } +} + +#[derive(Debug, Clone)] +pub struct ReadEntry { + pub id: SessionEntryRef, + pub origin: SessionEntryProvenance, + pub kind: ReferenceKind, + pub tool_part: Option, + pub tool_name: Option, + pub entry_range: [u64; 2], + pub label: String, + pub text: String, +} + +#[derive(Debug, Clone)] +pub struct ReadResult { + pub entries: Vec, + pub truncated: bool, +} + +#[derive(Debug, Clone)] +pub struct SessionEntryEvidence { + pub segment_id: String, + pub entry_ref: SessionEntryRef, + pub origin: SessionEntryProvenance, + pub entry_range: [u64; 2], + pub kind: ReferenceKind, + pub tool_part: Option, + pub label: String, + pub summary: String, + pub excerpt: String, +} + +#[derive(Debug, Clone)] +struct CapturedHistoryEntry { + use_policy: Option, + item: Item, + entry_id: crate::LoggedSessionHistoryEntryId, + origin: SessionEntryProvenance, + dto: CaptureEntryDto, +} + +#[derive(Debug, Clone)] +pub struct SessionCapture { + session_id: String, + segment_id: String, + capture_id: String, + entries: Arc>, + overview: Vec, + index: Vec, +} + +impl SessionCapture { + pub fn from_session_snapshot( + session_id: impl Into, + segment_id: impl Into, + snapshot: SessionSnapshot, + ) -> Self { + let segment_id = segment_id.into(); + let entries = snapshot + .entries + .into_iter() + .enumerate() + .filter_map(|(index, entry)| { + let policy = CaptureUsePolicy { + recording_material: false, + memory_evidence: entry.use_policy.is_none_or(|p| p.memory_evidence), + observation_readable: entry.use_policy.is_none_or(|p| p.observation_readable), + }; + let explicit = entry.use_policy.is_some(); + let item = match entry.data { + SessionSnapshotEntryData::UserInput { segments } => { + Item::user_message(protocol::Segment::flatten_to_text(&segments)) + } + SessionSnapshotEntryData::Message { role, content } => Item::Message { + id: None, + status: None, + role: match role { + SessionMessageRole::User => Role::User, + SessionMessageRole::Assistant => Role::Assistant, + }, + content: content + .into_iter() + .map(|part| match part { + SessionContentPart::Text { text } => { + agen::ContentPart::Text { text } + } + SessionContentPart::Refusal { refusal } => { + agen::ContentPart::Refusal { refusal } + } + }) + .collect(), + }, + SessionSnapshotEntryData::ToolCall { + call_id, + name, + arguments, + } => Item::tool_call(call_id, name, arguments), + SessionSnapshotEntryData::ToolResult { + call_id, + summary, + content, + is_error, + .. + } => Item::tool_result_item(call_id, summary, content, is_error), + SessionSnapshotEntryData::SystemItem { content, .. } if explicit => { + Item::system_message(content) + } + _ => return None, + }; + let body = CaptureBody::project(&item, &entry.provenance, explicit)?; + if !policy.any() { + return None; + } + Some(CaptureEntryDto { + entry_id: entry.entry_id, + order: index as u64, + source: CaptureSource { + segment_id: segment_id.clone(), + entry_index: index as u64, + provenance: entry.provenance, + history_origin: None, + derived_from: entry.derived_from, + }, + policy, + body, + }) + }) + .collect(); + Self::from_projected(session_id.into(), segment_id, entries) + } + + pub fn new( + session_id: impl Into, + segment_id: impl Into, + items: Vec, + ) -> Self { + let segment_id = segment_id.into(); + let entries = items + .into_iter() + .enumerate() + .filter_map(|(index, item)| { + let origin = SessionEntryProvenance::LegacyUnknown; + let body = CaptureBody::project(&item, &origin, false)?; + Some(CaptureEntryDto { + entry_id: format!("{index:08}"), + order: index as u64, + source: CaptureSource { + segment_id: segment_id.clone(), + entry_index: index as u64, + provenance: origin, + history_origin: None, + derived_from: Vec::new(), + }, + policy: CaptureUsePolicy::PUBLIC, + body, + }) + }) + .collect(); + Self::from_projected(session_id.into(), segment_id, entries) + } + + pub fn from_history_entries( + session_id: impl Into, + segment_id: impl Into, + entries: Vec>, + ) -> Self { + let segment_id = segment_id.into(); + let entries = entries + .into_iter() + .enumerate() + .filter_map(|(index, entry)| { + let origin = public_provenance(&entry.annotation.origin); + let mut item = entry.item; + let mut explicit = false; + let policy = if let Some(input) = entry.annotation.input_metadata { + explicit = true; + if matches!( + item, + Item::Message { + role: Role::System, + .. + } + ) { + // Only the explicit public projection crosses this boundary. + let public = input.public_item()?; + item = Item::system_message(public.body.clone()); + } else if input.feature_payload().is_some() || input.public_item().is_some() { + return None; + } + CaptureUsePolicy::from(input.usage()) + } else { + if matches!(entry.annotation.origin, HistoryOrigin::Provided { .. }) { + return None; + } + CaptureUsePolicy::PUBLIC + }; + if !policy.any() { + return None; + } + let body = CaptureBody::project(&item, &origin, explicit)?; + Some(CaptureEntryDto { + entry_id: entry.annotation.entry_id.0, + order: index as u64, + source: CaptureSource { + segment_id: segment_id.clone(), + entry_index: index as u64, + provenance: origin, + history_origin: Some(entry.annotation.origin), + derived_from: entry + .annotation + .derivation + .map(|d| d.sources.into_iter().map(|id| id.0).collect()) + .unwrap_or_default(), + }, + policy, + body, + }) + }) + .collect(); + Self::from_projected(session_id.into(), segment_id, entries) + } + + fn from_projected(session_id: String, segment_id: String, dtos: Vec) -> Self { + let dto = CaptureDto { + version: CAPTURE_VERSION, + session_id: session_id.clone(), + segment_id: segment_id.clone(), + entries: dtos, + }; + let capture_id = dto.fingerprint(); + let entries: Vec<_> = dto + .entries + .into_iter() + .map(|dto| CapturedHistoryEntry { + use_policy: Some(protocol::SessionEntryUsePolicy { + observation_readable: dto.policy.observation_readable, + memory_evidence: dto.policy.memory_evidence, + }), + item: dto.body.to_item(), + entry_id: crate::LoggedSessionHistoryEntryId(dto.entry_id.clone()), + origin: dto.source.provenance.clone(), + dto, + }) + .collect(); + let mut overview = Vec::new(); + let mut index = Vec::new(); + + for entry in &entries { + let item = &entry.item; + let entry_range = [entry.dto.order; 2]; + let attachment_count = match entry.dto.body { + CaptureBody::ToolResult { + attachment_count, .. + } => attachment_count, + _ => 0, + }; + match item { + Item::Message { role, content, .. } => { + let Some(kind) = message_reference_kind(&entry.origin, role, entry.use_policy) + else { + continue; + }; + let text = content + .iter() + .map(|p| p.as_text()) + .collect::>() + .join(""); + let label = format!("{} message", kind.as_str()); + let summary = truncate_chars(&text, 240); + let id = SessionEntryRef::from_history_entry_id(&entry.entry_id); + index.push(ReferenceEntry { + use_policy: entry.use_policy, + attachment_count, + id: id.clone(), + origin: entry.origin.clone(), + entry_range, + kind, + tool_part: None, + tool_name: None, + label: label.clone(), + summary: summary.clone(), + search_text: text.clone(), + }); + if entry.use_policy.is_none_or(|p| p.observation_readable) + && matches!(kind, ReferenceKind::User | ReferenceKind::Assistant) + { + overview.push(OverviewItem { + id: id.clone(), + origin: entry.origin.clone(), + entry_range, + kind, + label, + text, + intervening_entries: 0, + }); + } + } + Item::ToolCall { + name, arguments, .. + } => { + let text = format!("{name}\n{arguments}"); + index.push(ReferenceEntry { + use_policy: entry.use_policy, + attachment_count, + id: SessionEntryRef::from_history_entry_id(&entry.entry_id), + origin: entry.origin.clone(), + entry_range, + kind: ReferenceKind::Tool, + tool_part: Some(ToolPart::Input), + tool_name: Some(name.clone()), + label: format!("tool input: {name}"), + summary: format!("Tool call {name}"), + search_text: text, + }); + } + Item::ToolResult { + summary, + content, + attachments: _, + .. + } => { + let attachment_marker = if attachment_count == 0 { + String::new() + } else { + format!("\n[{} image attachment(s)]", attachment_count) + }; + let text = format!( + "{summary}\n{}{attachment_marker}", + content.as_deref().unwrap_or_default(), + ); + index.push(ReferenceEntry { + use_policy: entry.use_policy, + attachment_count, + id: SessionEntryRef::from_history_entry_id(&entry.entry_id), + origin: entry.origin.clone(), + entry_range, + kind: ReferenceKind::Tool, + tool_part: Some(ToolPart::Output), + tool_name: None, + label: "tool output".to_string(), + summary: truncate_chars(summary, 240), + search_text: text, + }); + } + Item::AssistantResponseBoundary { .. } | Item::Reasoning { .. } => {} + } + } + + if overview.len() > 2 { + let last = overview.len() - 1; + overview = overview + .into_iter() + .enumerate() + .filter_map(|(index, entry)| { + (index == 0 || index == last || index % OVERVIEW_ANCHOR_STRIDE == 0) + .then_some(entry) + }) + .collect(); + } + + for overview_index in 0..overview.len().saturating_sub(1) { + let current_entry = overview[overview_index].entry_range[0]; + let next_entry = overview[overview_index + 1].entry_range[0]; + overview[overview_index].intervening_entries = index + .iter() + .filter(|entry| { + let entry_index = entry.entry_range[0]; + entry_index > current_entry && entry_index < next_entry + }) + .count(); + } + + Self { + session_id, + segment_id, + capture_id, + entries: Arc::new(entries), + overview, + index, + } + } + + /// Public source entries are immutable; callers retain their own business decisions. + pub fn entries(&self) -> impl ExactSizeIterator { + self.entries.iter().map(|entry| &entry.dto) + } + + pub fn overview(&self) -> &[OverviewItem] { + &self.overview + } + + pub fn source_index_for_ref(&self, reference: &SessionEntryRef) -> Option { + self.index + .iter() + .find(|entry| entry.id == *reference) + .map(|entry| entry.entry_range[0]) + } + + pub fn search(&self, options: &SearchOptions) -> Vec { + if self.validate_search(options).is_err() { + return Vec::new(); + } + let query = options.query.trim().to_lowercase(); + let limit = options + .limit + .unwrap_or(DEFAULT_SEARCH_LIMIT) + .clamp(1, MAX_SEARCH_LIMIT); + let tool_name = options.tool_name.as_deref(); + let min_entry_index = options + .from + .as_ref() + .and_then(|reference| self.source_index_for_ref(reference)) + .unwrap_or_else(|| options.min_entry_index.unwrap_or(0)); + let max_entry_index = options + .through + .as_ref() + .and_then(|reference| self.source_index_for_ref(reference)) + .unwrap_or(u64::MAX); + let mut skipped = 0usize; + let mut hits = Vec::new(); + + for entry in &self.index { + if entry.use_policy.is_some_and(|p| !p.observation_readable) { + continue; + } + if entry.entry_range[0] < min_entry_index || entry.entry_range[0] > max_entry_index { + continue; + } + if let Some(kind) = options.kind { + if entry.kind != kind { + continue; + } + } + if let Some(part) = options.tool_part { + if entry.kind != ReferenceKind::Tool { + continue; + } + let Some(actual) = entry.tool_part else { + continue; + }; + if !part.matches(actual) { + continue; + } + } + if let Some(name) = tool_name { + if entry.tool_name.as_deref() != Some(name) { + continue; + } + } + if !query.is_empty() && !entry.search_text.to_lowercase().contains(&query) { + continue; + } + if skipped < options.offset { + skipped += 1; + continue; + } + hits.push(SearchHit { + id: entry.id.clone(), + origin: entry.origin.clone(), + kind: entry.kind, + tool_part: entry.tool_part, + tool_name: entry.tool_name.clone(), + entry_range: entry.entry_range, + label: entry.label.clone(), + summary: entry.summary.clone(), + }); + if hits.len() >= limit { + break; + } + } + + hits + } + + pub fn read(&self, selector: ReadSelector<'_>, options: ReadOptions) -> ReadResult { + let max_items = options.max_items.clamp(1, MAX_READ_MAX_ITEMS); + let max_bytes = options.max_bytes.max(1); + let mut entries = Vec::new(); + let mut bytes = 0usize; + let mut truncated = false; + + let selected: Vec<&ReferenceEntry> = match selector { + ReadSelector::Id(id) => self + .index + .iter() + .filter(|entry| entry.id.as_str() == id) + .collect(), + ReadSelector::EntryRange([start, end]) => self + .index + .iter() + .filter(|entry| entry.entry_range[0] >= start && entry.entry_range[0] <= end) + .collect(), + }; + + for entry in selected { + if entry.use_policy.is_some_and(|p| !p.observation_readable) { + continue; + } + if entries.len() >= max_items { + truncated = true; + break; + } + if entry.kind == ReferenceKind::Tool { + if !options.include_tools { + continue; + } + if let Some(actual) = entry.tool_part { + if !options.tool_part.matches(actual) { + continue; + } + } + } + let Some(item) = self + .entries + .iter() + .find(|captured| captured.dto.order == entry.entry_range[0]) + .map(|entry| &entry.item) + else { + continue; + }; + let remaining = max_bytes.saturating_sub(bytes); + if remaining == 0 { + truncated = true; + break; + } + let rendered = render_item(item, entry, options.detail, usize::MAX); + truncated |= rendered.len() > remaining; + let text = truncate_bytes(&rendered, remaining); + bytes = bytes.saturating_add(text.len()); + entries.push(ReadEntry { + id: entry.id.clone(), + origin: entry.origin.clone(), + kind: entry.kind, + tool_part: entry.tool_part, + tool_name: entry.tool_name.clone(), + entry_range: entry.entry_range, + label: entry.label.clone(), + text, + }); + if truncated { + break; + } + } + + ReadResult { entries, truncated } + } + + /// Resolves the committed tool-call input for a provider call id. This is + /// used by explicit-memory receipts only after the containing run is durable, + /// so a model-authored claim can cite its own committed tool invocation + /// without inventing a SessionEntryRef. + pub fn evidence_for_tool_call(&self, call_id: &str) -> Option { + let captured = self.entries.iter().find(|entry| { + matches!( + &entry.item, + Item::ToolCall { + call_id: stored_call_id, + .. + } if stored_call_id == call_id + ) + })?; + let reference = self.index.iter().find(|entry| { + entry.entry_range[0] == captured.dto.order + && entry.kind == ReferenceKind::Tool + && entry.tool_part == Some(ToolPart::Input) + })?; + self.evidence_for(reference.id.as_str()) + } + + pub fn evidence_for(&self, id: &str) -> Option { + let entry = self.index.iter().find(|entry| entry.id.as_str() == id)?; + if entry.use_policy.is_some_and(|p| !p.memory_evidence) { + return None; + } + let item = &self + .entries + .iter() + .find(|captured| captured.dto.order == entry.entry_range[0])? + .item; + let excerpt = render_item(item, entry, ReadDetail::Compact, 2 * 1024); + Some(SessionEntryEvidence { + segment_id: self + .entries + .iter() + .find(|captured| captured.dto.order == entry.entry_range[0])? + .dto + .source + .segment_id + .clone(), + entry_ref: entry.id.clone(), + origin: entry.origin.clone(), + entry_range: entry.entry_range, + kind: entry.kind, + tool_part: entry.tool_part, + label: entry.label.clone(), + summary: entry.summary.clone(), + excerpt, + }) + } +} + +fn public_provenance(origin: &HistoryOrigin) -> SessionEntryProvenance { + match origin { + HistoryOrigin::HumanInput { .. } | HistoryOrigin::LocalHumanInput => { + SessionEntryProvenance::HumanInput + } + HistoryOrigin::WorkerInput { .. } => SessionEntryProvenance::WorkerInput, + HistoryOrigin::FlowInstruction { .. } => SessionEntryProvenance::FlowInstruction, + HistoryOrigin::BackendInstruction { .. } | HistoryOrigin::Provided { .. } => { + SessionEntryProvenance::BackendInstruction + } + HistoryOrigin::ModelOutput { .. } => SessionEntryProvenance::ModelOutput, + HistoryOrigin::ToolOutput { .. } => SessionEntryProvenance::ToolOutput, + HistoryOrigin::CompactionSummary + | HistoryOrigin::DerivedSummary + | HistoryOrigin::JournalContext => SessionEntryProvenance::DerivedSummary, + HistoryOrigin::LegacyUnknown => SessionEntryProvenance::LegacyUnknown, + } +} + +fn message_reference_kind( + origin: &SessionEntryProvenance, + provider_role: &Role, + use_policy: Option, +) -> Option { + if use_policy.is_some() { + return match provider_role { + Role::System => Some(ReferenceKind::System), + Role::User => Some(ReferenceKind::User), + Role::Assistant => Some(ReferenceKind::Assistant), + }; + } + match origin { + SessionEntryProvenance::HumanInput | SessionEntryProvenance::WorkerInput => { + Some(ReferenceKind::User) + } + SessionEntryProvenance::ModelOutput => Some(ReferenceKind::Assistant), + SessionEntryProvenance::ToolOutput => Some(ReferenceKind::Tool), + SessionEntryProvenance::LegacyUnknown => match provider_role { + Role::User => Some(ReferenceKind::User), + Role::Assistant => Some(ReferenceKind::Assistant), + Role::System => None, + }, + // Flow/backend/system content remains out of the observation surface + // even when represented with a provider user/system role. + SessionEntryProvenance::FlowInstruction + | SessionEntryProvenance::BackendInstruction + | SessionEntryProvenance::DerivedSummary => None, + } +} + +fn render_item( + item: &Item, + entry: &ReferenceEntry, + detail: ReadDetail, + max_bytes: usize, +) -> String { + let text = match item { + Item::Message { role, content, .. } => { + let text = content + .iter() + .map(|p| p.as_text()) + .collect::>() + .join(""); + format!("[{} {:?}] {text}", entry.id, role) + } + Item::ToolCall { + name, arguments, .. + } => match detail { + ReadDetail::Compact => format!("[{} ToolInput {name}] (arguments omitted)", entry.id), + ReadDetail::Full => format!("[{} ToolInput {name}]\narguments: {arguments}", entry.id), + }, + Item::ToolResult { + summary, + content, + attachments: _, + is_error, + .. + } => { + let attachment_line = if entry.attachment_count == 0 { + String::new() + } else { + format!("\nattachments: {} image(s)", entry.attachment_count) + }; + match detail { + ReadDetail::Compact => format!( + "[{} ToolOutput{}]\nsummary: {summary}\ncontent: (omitted){attachment_line}", + entry.id, + if *is_error { " error" } else { "" }, + ), + ReadDetail::Full => format!( + "[{} ToolOutput{}]\nsummary: {summary}\ncontent: {}{attachment_line}", + entry.id, + if *is_error { " error" } else { "" }, + content.as_deref().unwrap_or_default(), + ), + } + } + Item::AssistantResponseBoundary { .. } => { + format!("[{} Assistant response boundary omitted]", entry.id) + } + Item::Reasoning { .. } => format!("[{} Reasoning omitted]", entry.id), + }; + truncate_bytes(&text, max_bytes) +} + +fn truncate_chars(text: &str, max_chars: usize) -> String { + if max_chars == 0 { + return "… [truncated]".to_string(); + } + if text.chars().count() <= max_chars { + return text.to_string(); + } + let mut out = text.chars().take(max_chars).collect::(); + out.push_str("… [truncated]"); + out +} + +fn truncate_bytes(text: &str, max_bytes: usize) -> String { + if text.len() <= max_bytes { + return text.to_owned(); + } + let marker = "… [truncated]"; + let mut end = max_bytes.saturating_sub(marker.len()); + while !text.is_char_boundary(end) { + end -= 1; + } + let mut out = text[..end].to_owned(); + if max_bytes >= marker.len() { + out.push_str(marker); + } + out +} diff --git a/crates/session-store/src/capture/body.rs b/crates/session-store/src/capture/body.rs new file mode 100644 index 000000000..747586889 --- /dev/null +++ b/crates/session-store/src/capture/body.rs @@ -0,0 +1,934 @@ +//! Portable public capture values. DTOs are transport data, not authorization handles. +//! Checked construction validates an explicitly public projection; adapters must never put +//! a provider System body or private Feature payload in `PublicSystem`. + +use super::*; +use serde::de::Error as _; +use sha2::{Digest, Sha256}; +use std::collections::HashSet; +use std::io::Write; + +pub const CAPTURE_VERSION: u32 = 1; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CaptureUsePolicy { + pub recording_material: bool, + pub memory_evidence: bool, + pub observation_readable: bool, +} + +impl CaptureUsePolicy { + pub const PUBLIC: Self = Self { + recording_material: true, + memory_evidence: true, + observation_readable: true, + }; + pub fn any(self) -> bool { + self.recording_material || self.memory_evidence || self.observation_readable + } + pub fn allows(self, purpose: CapturePurpose) -> bool { + match purpose { + CapturePurpose::Recording => self.recording_material, + CapturePurpose::MemoryEvidence => self.memory_evidence, + CapturePurpose::Observation => self.observation_readable, + } + } +} + +impl From for CaptureUsePolicy { + fn from(policy: crate::HistoryInputUsePolicy) -> Self { + Self { + recording_material: policy.recording_material == crate::InputEvidenceUse::Include, + memory_evidence: policy.memory_evidence == crate::InputEvidenceUse::Include, + observation_readable: policy.observation == crate::InputObservationUse::Readable, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CapturePurpose { + Recording, + MemoryEvidence, + Observation, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CaptureRole { + User, + Assistant, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum CaptureContentPart { + Text { text: String }, + Refusal { refusal: String }, +} + +impl CaptureContentPart { + fn text(&self) -> &str { + match self { + Self::Text { text } => text, + Self::Refusal { refusal } => refusal, + } + } +} + +/// The type has no reasoning, raw System prompt, Feature payload, or image bytes variant. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum CaptureBody { + Message { + role: CaptureRole, + content: Vec, + }, + PublicSystem { + text: String, + }, + ToolCall { + call_id: String, + name: String, + arguments: String, + }, + ToolResult { + call_id: String, + summary: String, + content: Option, + is_error: bool, + attachment_count: usize, + }, +} + +impl CaptureBody { + pub(super) fn project( + item: &Item, + origin: &SessionEntryProvenance, + explicit: bool, + ) -> Option { + match item { + Item::Message { role, content, .. } => { + message_reference_kind( + origin, + role, + explicit.then_some(protocol::SessionEntryUsePolicy { + observation_readable: true, + memory_evidence: true, + }), + )?; + if *role == Role::System { + if !explicit { + return None; + } + return Some(Self::PublicSystem { + text: content.iter().map(|p| p.as_text()).collect::(), + }); + } + Some(Self::Message { + role: if *role == Role::User { + CaptureRole::User + } else { + CaptureRole::Assistant + }, + content: content + .iter() + .map(|p| match p { + agen::ContentPart::Text { text } => { + CaptureContentPart::Text { text: text.clone() } + } + agen::ContentPart::Refusal { refusal } => CaptureContentPart::Refusal { + refusal: refusal.clone(), + }, + }) + .collect(), + }) + } + Item::ToolCall { + call_id, + name, + arguments, + .. + } => Some(Self::ToolCall { + call_id: call_id.clone(), + name: name.clone(), + arguments: arguments.clone(), + }), + Item::ToolResult { + call_id, + summary, + content, + is_error, + attachments, + .. + } => Some(Self::ToolResult { + call_id: call_id.clone(), + summary: summary.clone(), + content: content.clone(), + is_error: *is_error, + attachment_count: attachments.len(), + }), + Item::Reasoning { .. } | Item::AssistantResponseBoundary { .. } => None, + } + } + + pub(super) fn to_item(&self) -> Item { + match self { + Self::Message { role, content } => Item::Message { + id: None, + status: None, + role: match role { + CaptureRole::User => Role::User, + CaptureRole::Assistant => Role::Assistant, + }, + content: content + .iter() + .map(|p| match p { + CaptureContentPart::Text { text } => { + agen::ContentPart::Text { text: text.clone() } + } + CaptureContentPart::Refusal { refusal } => agen::ContentPart::Refusal { + refusal: refusal.clone(), + }, + }) + .collect(), + }, + Self::PublicSystem { text } => Item::system_message(text.clone()), + Self::ToolCall { + call_id, + name, + arguments, + } => Item::tool_call(call_id.clone(), name.clone(), arguments.clone()), + Self::ToolResult { + call_id, + summary, + content, + is_error, + .. + } => { + Item::tool_result_item(call_id.clone(), summary.clone(), content.clone(), *is_error) + } + } + } + + fn fields(&self) -> Vec<(RawField, &str)> { + match self { + Self::Message { content, .. } => content + .iter() + .enumerate() + .map(|(i, p)| (RawField::MessagePart(i), p.text())) + .collect(), + Self::PublicSystem { text } => vec![(RawField::PublicSystem, text)], + Self::ToolCall { arguments, .. } => vec![(RawField::ToolArguments, arguments)], + Self::ToolResult { + summary, content, .. + } => { + let mut fields = vec![(RawField::ToolSummary, summary.as_str())]; + if let Some(content) = content { + fields.push((RawField::ToolContent, content)); + } + fields + } + } + } + + fn metadata(&self, field: RawField) -> CaptureBodyMetadata { + match self { + Self::Message { role, content } => CaptureBodyMetadata::Message { + role: *role, + refusal: matches!(field, RawField::MessagePart(i) if matches!(content.get(i), Some(CaptureContentPart::Refusal { .. }))), + }, + Self::PublicSystem { .. } => CaptureBodyMetadata::PublicSystem, + Self::ToolCall { call_id, name, .. } => CaptureBodyMetadata::ToolCall { + call_id: call_id.clone(), + name: name.clone(), + }, + Self::ToolResult { + call_id, + content, + is_error, + attachment_count, + .. + } => CaptureBodyMetadata::ToolResult { + call_id: call_id.clone(), + has_content: content.is_some(), + is_error: *is_error, + attachment_count: *attachment_count, + }, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CaptureSource { + pub segment_id: String, + pub entry_index: u64, + pub provenance: SessionEntryProvenance, + /// Full original origin when supplied by the committed-history adapter. + #[serde(deserialize_with = "strict_origin")] + pub history_origin: Option, + pub derived_from: Vec, +} + +// Existing history origin types predate strict DTOs. Comparing their canonical value +// rejects unknown nested keys instead of silently accepting/losing them at this boundary. +fn strict_origin<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + let value = Option::::deserialize(deserializer)?; + value + .map(|value| { + let origin: crate::LoggedSessionHistoryOrigin = + serde_json::from_value(value.clone()).map_err(D::Error::custom)?; + if serde_json::to_value(&origin).map_err(D::Error::custom)? != value { + return Err(D::Error::custom("noncanonical or unknown origin fields")); + } + Ok(origin) + }) + .transpose() +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CaptureEntryDto { + pub entry_id: String, + /// Position in this captured view, independent of the original segment position. + pub order: u64, + pub source: CaptureSource, + pub policy: CaptureUsePolicy, + pub body: CaptureBody, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CaptureDto { + pub version: u32, + pub session_id: String, + pub segment_id: String, + pub entries: Vec, +} + +#[derive(Debug, Clone, Copy)] +pub struct CaptureLimits { + pub max_bytes: usize, + pub max_entries: usize, + pub max_text_bytes: usize, + pub max_identity_bytes: usize, + pub max_derivation_sources: usize, +} + +impl Default for CaptureLimits { + fn default() -> Self { + Self { + max_bytes: 16 * 1024 * 1024, + max_entries: 100_000, + max_text_bytes: 8 * 1024 * 1024, + max_identity_bytes: 4096, + max_derivation_sources: 1024, + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum CaptureError { + #[error("unsupported capture representation version")] + UnsupportedVersion, + #[error("capture exceeds caller resource limits")] + ResourceLimit, + #[error("invalid capture: {0}")] + Invalid(&'static str), + #[error("reference or cursor belongs to another capture/session/purpose")] + ForeignReference, + #[error("unknown entry reference")] + UnknownReference, + #[error("entry is not eligible for this purpose")] + PolicyDenied, + #[error("invalid entry range or UTF-8 byte offset")] + InvalidRange, + #[error("budget cannot fit the next item and continuation metadata")] + BudgetTooSmall, + #[error("invalid capture JSON: {0}")] + Json(#[from] serde_json::Error), +} + +struct ByteCounter { + bytes: usize, + limit: usize, +} +impl Write for ByteCounter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.bytes = self + .bytes + .checked_add(buf.len()) + .ok_or_else(|| std::io::Error::other("capture byte limit"))?; + if self.bytes > self.limit { + return Err(std::io::Error::other("capture byte limit")); + } + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +fn json_size(value: &T, limit: usize) -> Result { + let mut counter = ByteCounter { bytes: 0, limit }; + serde_json::to_writer(&mut counter, value).map_err(|_| CaptureError::ResourceLimit)?; + Ok(counter.bytes) +} + +fn identity(value: &str, limits: CaptureLimits) -> Result<(), CaptureError> { + if value.is_empty() + || value.len() > limits.max_identity_bytes + || value.chars().any(char::is_control) + { + return Err(CaptureError::Invalid("identity")); + } + Ok(()) +} + +impl CaptureDto { + pub fn validate(&self, limits: CaptureLimits) -> Result<(), CaptureError> { + if self.version != CAPTURE_VERSION { + return Err(CaptureError::UnsupportedVersion); + } + if self.entries.len() > limits.max_entries { + return Err(CaptureError::ResourceLimit); + } + identity(&self.session_id, limits)?; + identity(&self.segment_id, limits)?; + let mut ids = HashSet::new(); + let mut sources = HashSet::new(); + let mut previous = None; + for entry in &self.entries { + identity(&entry.entry_id, limits)?; + if SessionEntryRef::parse(&format!("E{}", entry.entry_id)).is_none() + || !ids.insert(&entry.entry_id) + { + return Err(CaptureError::Invalid("entry identity")); + } + if previous.is_some_and(|order| order >= entry.order) { + return Err(CaptureError::Invalid("entry order")); + } + previous = Some(entry.order); + identity(&entry.source.segment_id, limits)?; + if !sources.insert((&entry.source.segment_id, entry.source.entry_index)) { + return Err(CaptureError::Invalid("duplicate source position")); + } + if !entry.policy.any() { + return Err(CaptureError::Invalid("unusable retained entry")); + } + if entry.source.derived_from.len() > limits.max_derivation_sources { + return Err(CaptureError::ResourceLimit); + } + for source in &entry.source.derived_from { + identity(source, limits)?; + } + if let Some(origin) = &entry.source.history_origin { + if public_provenance(origin) != entry.source.provenance { + return Err(CaptureError::Invalid("origin mismatch")); + } + json_size(origin, limits.max_text_bytes)?; + } + for (_, text) in entry.body.fields() { + if text.len() > limits.max_text_bytes { + return Err(CaptureError::ResourceLimit); + } + } + match &entry.body { + CaptureBody::Message { content, .. } if content.is_empty() => { + return Err(CaptureError::Invalid("empty message parts")); + } + CaptureBody::ToolCall { call_id, name, .. } => { + identity(call_id, limits)?; + identity(name, limits)?; + } + CaptureBody::ToolResult { call_id, .. } => identity(call_id, limits)?, + _ => {} + } + } + json_size(self, limits.max_bytes)?; + Ok(()) + } + + pub fn encode(&self, limits: CaptureLimits) -> Result, CaptureError> { + self.validate(limits)?; + Ok(serde_json::to_vec(self)?) + } + + pub fn decode(bytes: &[u8], limits: CaptureLimits) -> Result { + if bytes.len() > limits.max_bytes { + return Err(CaptureError::ResourceLimit); + } + let dto: Self = serde_json::from_slice(bytes)?; + dto.validate(limits)?; + Ok(dto) + } + + pub(super) fn fingerprint(&self) -> String { + // These DTOs have only finite, infallibly serializable JSON primitives. + let bytes = serde_json::to_vec(self).expect("capture DTO serialization"); + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScopedSessionEntryRef { + pub session_id: String, + pub capture_id: String, + pub entry_ref: SessionEntryRef, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RawField { + MessagePart(usize), + PublicSystem, + ToolArguments, + ToolSummary, + ToolContent, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RawRead { + pub reference: ScopedSessionEntryRef, + pub purpose: CapturePurpose, + pub field: RawField, + pub offset: usize, + pub text: String, + pub next_offset: Option, + pub total_bytes: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum CaptureBodyMetadata { + Message { + role: CaptureRole, + refusal: bool, + }, + PublicSystem, + ToolCall { + call_id: String, + name: String, + }, + ToolResult { + call_id: String, + has_content: bool, + is_error: bool, + attachment_count: usize, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CaptureCursor { + pub session_id: String, + pub capture_id: String, + pub purpose: CapturePurpose, + pub entry_position: usize, + pub field_index: usize, + pub byte_offset: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CaptureSliceItem { + pub reference: ScopedSessionEntryRef, + pub order: u64, + pub source: CaptureSource, + pub policy: CaptureUsePolicy, + pub metadata: CaptureBodyMetadata, + pub field: RawField, + pub offset: usize, + pub total_bytes: usize, + /// Raw source bytes, not a rendered preview or ellipsis-bearing truncation. + pub text: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CaptureSlice { + pub version: u32, + pub session_id: String, + pub capture_id: String, + pub purpose: CapturePurpose, + pub items: Vec, + pub next_cursor: Option, +} + +/// Bytes and value remain paired and immutable so the size proof cannot go stale. +#[derive(Debug, Clone)] +pub struct BudgetedCaptureSlice { + slice: CaptureSlice, + json: Vec, +} +impl BudgetedCaptureSlice { + pub fn slice(&self) -> &CaptureSlice { + &self.slice + } + pub fn as_json(&self) -> &[u8] { + &self.json + } + pub fn encoded_len(&self) -> usize { + self.json.len() + } +} + +impl SessionCapture { + pub fn session_id(&self) -> &str { + &self.session_id + } + pub fn segment_id(&self) -> &str { + &self.segment_id + } + pub fn capture_id(&self) -> &str { + &self.capture_id + } + + pub fn to_dto(&self) -> CaptureDto { + CaptureDto { + version: CAPTURE_VERSION, + session_id: self.session_id.clone(), + segment_id: self.segment_id.clone(), + entries: self.entries.iter().map(|e| e.dto.clone()).collect(), + } + } + + pub fn try_from_dto(dto: CaptureDto, limits: CaptureLimits) -> Result { + dto.validate(limits)?; + Ok(Self::from_projected( + dto.session_id, + dto.segment_id, + dto.entries, + )) + } + + pub fn encode(&self, limits: CaptureLimits) -> Result, CaptureError> { + self.to_dto().encode(limits) + } + pub fn decode(bytes: &[u8], limits: CaptureLimits) -> Result { + Self::try_from_dto(CaptureDto::decode(bytes, limits)?, limits) + } + + pub fn scoped_ref( + &self, + reference: &SessionEntryRef, + ) -> Result { + if self.source_index_for_ref(reference).is_none() { + return Err(CaptureError::UnknownReference); + } + Ok(ScopedSessionEntryRef { + session_id: self.session_id.clone(), + capture_id: self.capture_id.clone(), + entry_ref: reference.clone(), + }) + } + + pub fn resolve_ref( + &self, + reference: &ScopedSessionEntryRef, + purpose: CapturePurpose, + ) -> Result<&CaptureEntryDto, CaptureError> { + if reference.session_id != self.session_id || reference.capture_id != self.capture_id { + return Err(CaptureError::ForeignReference); + } + let entry = self + .entries + .iter() + .find(|entry| { + SessionEntryRef::from_history_entry_id(&entry.entry_id) == reference.entry_ref + }) + .ok_or(CaptureError::UnknownReference)?; + if !entry.dto.policy.allows(purpose) { + return Err(CaptureError::PolicyDenied); + } + Ok(&entry.dto) + } + + pub fn checked_range( + &self, + from: &ScopedSessionEntryRef, + through: &ScopedSessionEntryRef, + purpose: CapturePurpose, + ) -> Result, CaptureError> { + let start = self.resolve_ref(from, purpose)?.order; + let end = self.resolve_ref(through, purpose)?.order; + if start > end { + return Err(CaptureError::InvalidRange); + } + Ok(self + .entries + .iter() + .map(|entry| &entry.dto) + .filter(|entry| { + entry.order >= start && entry.order <= end && entry.policy.allows(purpose) + }) + .collect()) + } + + pub(super) fn validate_search(&self, options: &SearchOptions) -> Result<(), CaptureError> { + let from = options + .from + .as_ref() + .map(|r| { + self.source_index_for_ref(r) + .ok_or(CaptureError::UnknownReference) + }) + .transpose()?; + let through = options + .through + .as_ref() + .map(|r| { + self.source_index_for_ref(r) + .ok_or(CaptureError::UnknownReference) + }) + .transpose()?; + if from.unwrap_or(options.min_entry_index.unwrap_or(0)) > through.unwrap_or(u64::MAX) { + return Err(CaptureError::InvalidRange); + } + Ok(()) + } + + pub fn search_checked(&self, options: &SearchOptions) -> Result, CaptureError> { + self.validate_search(options)?; + Ok(self.search(options)) + } + + pub fn read_checked( + &self, + selector: ReadSelector<'_>, + options: ReadOptions, + ) -> Result { + match selector { + ReadSelector::Id(id) => { + let reference = SessionEntryRef::parse(id).ok_or(CaptureError::UnknownReference)?; + self.source_index_for_ref(&reference) + .ok_or(CaptureError::UnknownReference)?; + } + ReadSelector::EntryRange([start, end]) => { + if start > end + || !self.index.iter().any(|entry| entry.entry_range[0] == start) + || !self.index.iter().any(|entry| entry.entry_range[0] == end) + { + return Err(CaptureError::InvalidRange); + } + } + } + Ok(self.read(selector, options)) + } + + pub fn read_raw( + &self, + reference: &ScopedSessionEntryRef, + purpose: CapturePurpose, + field: RawField, + offset: usize, + max_bytes: usize, + ) -> Result { + let entry = self.resolve_ref(reference, purpose)?; + let text = entry + .body + .fields() + .into_iter() + .find(|(f, _)| *f == field) + .map(|(_, text)| text) + .ok_or(CaptureError::InvalidRange)?; + if offset > text.len() || !text.is_char_boundary(offset) { + return Err(CaptureError::InvalidRange); + } + let mut end = offset.saturating_add(max_bytes).min(text.len()); + while !text.is_char_boundary(end) { + end -= 1; + } + if end == offset && offset < text.len() { + return Err(CaptureError::BudgetTooSmall); + } + Ok(RawRead { + reference: reference.clone(), + purpose, + field, + offset, + text: text[offset..end].to_owned(), + next_offset: (end < text.len()).then_some(end), + total_bytes: text.len(), + }) + } + + fn cursor( + &self, + purpose: CapturePurpose, + entry_position: usize, + field_index: usize, + byte_offset: usize, + ) -> CaptureCursor { + CaptureCursor { + session_id: self.session_id.clone(), + capture_id: self.capture_id.clone(), + purpose, + entry_position, + field_index, + byte_offset, + } + } + + fn normalize_cursor(&self, mut cursor: CaptureCursor) -> Option { + while let Some(entry) = self.entries.get(cursor.entry_position) { + if entry.dto.policy.allows(cursor.purpose) + && cursor.field_index < entry.dto.body.fields().len() + { + return Some(cursor); + } + cursor.entry_position += 1; + cursor.field_index = 0; + cursor.byte_offset = 0; + } + None + } + + /// The returned JSON includes every item's source/policy, escaped body, and continuation. + /// Both the item count and the exact encoded length are bounded, with no Job-specific limit. + pub fn slice( + &self, + purpose: CapturePurpose, + cursor: Option<&CaptureCursor>, + max_items: usize, + max_bytes: usize, + ) -> Result { + if max_items == 0 { + return Err(CaptureError::BudgetTooSmall); + } + let start = if let Some(cursor) = cursor { + if cursor.session_id != self.session_id + || cursor.capture_id != self.capture_id + || cursor.purpose != purpose + { + return Err(CaptureError::ForeignReference); + } + let entry = self + .entries + .get(cursor.entry_position) + .ok_or(CaptureError::InvalidRange)?; + if !entry.dto.policy.allows(purpose) { + return Err(CaptureError::PolicyDenied); + } + let fields = entry.dto.body.fields(); + let text = fields + .get(cursor.field_index) + .ok_or(CaptureError::InvalidRange)? + .1; + if cursor.byte_offset > text.len() || !text.is_char_boundary(cursor.byte_offset) { + return Err(CaptureError::InvalidRange); + } + Some(cursor.clone()) + } else { + self.normalize_cursor(self.cursor(purpose, 0, 0, 0)) + }; + let mut slice = CaptureSlice { + version: CAPTURE_VERSION, + session_id: self.session_id.clone(), + capture_id: self.capture_id.clone(), + purpose, + items: Vec::new(), + next_cursor: start, + }; + if json_size(&slice, max_bytes).is_err() { + return Err(CaptureError::BudgetTooSmall); + } + while slice.items.len() < max_items { + let Some(current) = slice.next_cursor.clone() else { + break; + }; + let dto = &self.entries[current.entry_position].dto; + let fields = dto.body.fields(); + let (field, text) = fields[current.field_index]; + let reference = ScopedSessionEntryRef { + session_id: self.session_id.clone(), + capture_id: self.capture_id.clone(), + entry_ref: SessionEntryRef(format!("E{}", dto.entry_id)), + }; + let item = CaptureSliceItem { + reference, + order: dto.order, + source: dto.source.clone(), + policy: dto.policy, + metadata: dto.body.metadata(field), + field, + offset: current.byte_offset, + total_bytes: text.len(), + text: String::new(), + }; + slice.items.push(item); + let item_index = slice.items.len() - 1; + // Search only the partial interval: its cursor stays in the same field, + // making encoded size monotonic even when completing a field drops metadata. + let remaining = &text[current.byte_offset..]; + let mut candidate_bytes = remaining.len().min(max_bytes); + while !remaining.is_char_boundary(candidate_bytes) { + candidate_bytes -= 1; + } + let mut boundaries: Vec = remaining[..candidate_bytes] + .char_indices() + .map(|(i, _)| i) + .collect(); + boundaries.push(candidate_bytes); + let mut accepted = None; + let mut low = 0; + let mut high = boundaries.len(); + while low < high { + let mid = low + (high - low) / 2; + let end = current.byte_offset + boundaries[mid]; + slice.items[item_index].text = text[current.byte_offset..end].to_owned(); + slice.next_cursor = + Some(self.cursor(purpose, current.entry_position, current.field_index, end)); + if json_size(&slice, max_bytes).is_ok() { + accepted = Some(end); + low = mid + 1; + } else { + high = mid; + } + } + // A completed field can fit even when its partial continuation does not. + let completed = self.normalize_cursor(self.cursor( + purpose, + current.entry_position, + current.field_index + 1, + 0, + )); + if remaining.len() <= max_bytes { + slice.items[item_index].text = remaining.to_owned(); + slice.next_cursor = completed.clone(); + if json_size(&slice, max_bytes).is_ok() { + accepted = Some(text.len()); + } + } + let Some(end) = accepted.filter(|end| *end > current.byte_offset || text.is_empty()) + else { + slice.items.pop(); + slice.next_cursor = Some(current); + if slice.items.is_empty() { + return Err(CaptureError::BudgetTooSmall); + } + break; + }; + slice.items[item_index].text = text[current.byte_offset..end].to_owned(); + slice.next_cursor = if end == text.len() { + completed + } else { + Some(self.cursor(purpose, current.entry_position, current.field_index, end)) + }; + } + let json = serde_json::to_vec(&slice)?; + if json.len() > max_bytes { + return Err(CaptureError::BudgetTooSmall); + } + Ok(BudgetedCaptureSlice { slice, json }) + } +} diff --git a/crates/session-store/src/lib.rs b/crates/session-store/src/lib.rs index d767defca..15abc32dd 100644 --- a/crates/session-store/src/lib.rs +++ b/crates/session-store/src/lib.rs @@ -38,6 +38,7 @@ pub(crate) fn read_without_atime_bounded(path: &Path, max_bytes: u64) -> io::Res } } +pub mod capture; pub mod event_trace; pub mod fs_store; pub mod history; diff --git a/crates/session-store/tests/capture_contract.rs b/crates/session-store/tests/capture_contract.rs new file mode 100644 index 000000000..a605d6d6e --- /dev/null +++ b/crates/session-store/tests/capture_contract.rs @@ -0,0 +1,594 @@ +//! Public consumer boundary: real capture adapters, checked DTO codec and pure readers. +//! No Worker, persistence, live Session provider, or substituted capture implementation. + +use agen::{HistoryEntry, Item}; +use protocol::{ + SessionEntryProvenance, SessionSnapshot, SessionSnapshotEntry, SessionSnapshotEntryData, +}; +use session_store::capture::*; +use session_store::{ + FeatureInputPayload, HistoryInputMetadata, HistoryInputUsePolicy, InputCompactionUse, + InputEvidenceUse, InputObservationUse, LoggedHistoryDerivation, LoggedSessionHistoryEntryId, + LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, PublicSystemItem, + PublicSystemPresentation, +}; + +fn history( + id: &str, + item: Item, + input_metadata: Option, +) -> HistoryEntry { + HistoryEntry::new( + item, + LoggedSessionHistoryMetadata { + entry_id: LoggedSessionHistoryEntryId(id.into()), + origin: LoggedSessionHistoryOrigin::LocalHumanInput, + derivation: Some(LoggedHistoryDerivation { + sources: vec![LoggedSessionHistoryEntryId("earlier-entry".into())], + }), + input_metadata, + }, + ) +} + +fn usage(recording: bool, memory: bool, observation: bool) -> HistoryInputUsePolicy { + HistoryInputUsePolicy { + recording_material: if recording { + InputEvidenceUse::Include + } else { + InputEvidenceUse::Exclude + }, + memory_evidence: if memory { + InputEvidenceUse::Include + } else { + InputEvidenceUse::Exclude + }, + observation: if observation { + InputObservationUse::Readable + } else { + InputObservationUse::Hidden + }, + compaction: InputCompactionUse::Retain, + } +} + +fn reference(capture: &SessionCapture, id: &str) -> ScopedSessionEntryRef { + capture + .scoped_ref(&SessionEntryRef::parse(id).unwrap()) + .unwrap() +} + +#[test] +fn public_consumer_roundtrip_retains_body_original_source_order_origin_and_independent_policy() { + let entries = vec![ + history( + "recording", + Item::user_message("record-only"), + Some(HistoryInputMetadata::new(usage(true, false, false), None, None).unwrap()), + ), + history( + "memory", + Item::user_message("memory-only"), + Some(HistoryInputMetadata::new(usage(false, true, false), None, None).unwrap()), + ), + history( + "observation", + Item::user_message("observation-only"), + Some(HistoryInputMetadata::new(usage(false, false, true), None, None).unwrap()), + ), + ]; + let capture = SessionCapture::from_history_entries("synthetic-session", "selected", entries); + let mut dto = capture.to_dto(); + // The checked boundary also accepts entries retained from different original segments. + dto.entries[0].source.segment_id = "original-a".into(); + dto.entries[0].source.entry_index = 80; + dto.entries[1].source.segment_id = "original-b".into(); + dto.entries[1].source.entry_index = 3; + let capture = SessionCapture::try_from_dto(dto.clone(), CaptureLimits::default()).unwrap(); + let encoded = capture.encode(CaptureLimits::default()).unwrap(); + let restored = SessionCapture::decode(&encoded, CaptureLimits::default()).unwrap(); + assert_eq!(restored.to_dto(), dto); + assert_eq!(restored.capture_id(), capture.capture_id()); + for (id, allowed) in [ + ("Erecording", CapturePurpose::Recording), + ("Ememory", CapturePurpose::MemoryEvidence), + ("Eobservation", CapturePurpose::Observation), + ] { + let reference = reference(&capture, id); + assert_eq!( + restored.resolve_ref(&reference, allowed).unwrap(), + capture.resolve_ref(&reference, allowed).unwrap() + ); + for purpose in [ + CapturePurpose::Recording, + CapturePurpose::MemoryEvidence, + CapturePurpose::Observation, + ] { + assert_eq!( + restored.resolve_ref(&reference, purpose).is_ok(), + purpose == allowed + ); + } + } + assert_eq!(restored.overview().len(), 1); + assert_eq!(restored.search(&SearchOptions::default()).len(), 1); + assert!(restored.evidence_for("Erecording").is_none()); + let evidence = restored.evidence_for("Ememory").unwrap(); + assert_eq!(evidence.segment_id, "original-b"); + assert!(evidence.excerpt.contains("memory-only")); + let recorded = restored + .slice(CapturePurpose::Recording, None, 10, 16_000) + .unwrap(); + assert_eq!(recorded.slice().items.len(), 1); + assert_eq!(recorded.slice().items[0].text, "record-only"); + for (purpose, expected) in [ + (CapturePurpose::MemoryEvidence, "memory-only"), + (CapturePurpose::Observation, "observation-only"), + ] { + let slice = restored.slice(purpose, None, 10, 16_000).unwrap(); + assert_eq!(slice.slice().items.len(), 1); + assert_eq!(slice.slice().items[0].text, expected); + assert!( + restored + .read_raw( + &reference(&restored, "Erecording"), + purpose, + RawField::MessagePart(0), + 0, + 100 + ) + .is_err() + ); + } +} + +#[test] +fn common_data_drops_reasoning_raw_system_and_private_payload_even_when_unindexed() { + let payload = FeatureInputPayload::encode( + "capture-test", + 1, + &serde_json::json!({"secret": "PRIVATE-PAYLOAD"}), + ) + .unwrap(); + let public = PublicSystemItem { + kind: "notice".into(), + body: "public notice".into(), + presentation: PublicSystemPresentation::Ignored, + }; + let system = + HistoryInputMetadata::new(usage(true, true, true), Some(payload.clone()), Some(public)) + .unwrap(); + let private = HistoryInputMetadata::new(usage(true, true, true), Some(payload), None).unwrap(); + let capture = SessionCapture::from_history_entries( + "s", + "segment", + vec![ + history("raw", Item::system_message("RAW-SYSTEM-PROMPT"), None), + history("reasoning", Item::reasoning("HIDDEN-REASONING"), None), + history( + "private", + Item::system_message("PRIVATE-SYSTEM-BODY"), + Some(private), + ), + history( + "public", + Item::system_message("MODEL-SYSTEM-BODY"), + Some(system), + ), + history("ordinary", Item::user_message("ordinary"), None), + ], + ); + let encoded = String::from_utf8(capture.encode(CaptureLimits::default()).unwrap()).unwrap(); + let retained_debug = format!("{capture:?}"); + for secret in [ + "RAW-SYSTEM-PROMPT", + "HIDDEN-REASONING", + "PRIVATE-SYSTEM-BODY", + "MODEL-SYSTEM-BODY", + "PRIVATE-PAYLOAD", + ] { + assert!(!encoded.contains(secret)); + assert!(!retained_debug.contains(secret)); + } + assert_eq!(capture.to_dto().entries.len(), 2); + assert!(encoded.contains("public notice")); + // Filtered entries do not renumber the public source positions. + assert_eq!(capture.to_dto().entries[0].order, 3); + assert_eq!( + capture.source_index_for_ref(&SessionEntryRef::parse("Eordinary").unwrap()), + Some(4) + ); +} + +#[test] +fn protocol_projection_does_not_invent_recording_permission() { + let capture = SessionCapture::from_session_snapshot( + "s", + "seg", + SessionSnapshot { + pending_submissions: Default::default(), + entries: vec![SessionSnapshotEntry { + entry_id: "projected".into(), + timestamp: 0, + provenance: SessionEntryProvenance::HumanInput, + derived_from: vec!["source".into()], + use_policy: Some(protocol::SessionEntryUsePolicy { + observation_readable: true, + memory_evidence: false, + }), + data: SessionSnapshotEntryData::Message { + role: protocol::SessionMessageRole::User, + content: vec![protocol::SessionContentPart::Text { + text: "projected".into(), + }], + }, + }], + }, + ); + let reference = reference(&capture, "Eprojected"); + assert!(matches!( + capture.resolve_ref(&reference, CapturePurpose::Recording), + Err(CaptureError::PolicyDenied) + )); + assert!( + capture + .resolve_ref(&reference, CapturePurpose::Observation) + .is_ok() + ); + assert!(capture.evidence_for("Eprojected").is_none()); +} + +#[test] +fn malformed_dtos_and_oversized_representations_are_rejected() { + let capture = SessionCapture::new("s", "seg", vec![Item::user_message("body")]); + let original = serde_json::to_value(capture.to_dto()).unwrap(); + let mut invalids = Vec::new(); + let mut value = original.clone(); + value["version"] = 999.into(); + invalids.push(value); + let mut value = original.clone(); + value["unexpected"] = true.into(); + invalids.push(value); + let mut value = original.clone(); + value["entries"][0]["policy"]["recording_material"] = "include".into(); + invalids.push(value); + let mut value = original.clone(); + value["entries"][0]["policy"] + .as_object_mut() + .unwrap() + .remove("memory_evidence"); + invalids.push(value); + let mut value = original.clone(); + value["entries"][0]["body"] = serde_json::json!({"kind":"reasoning","text":"secret"}); + invalids.push(value); + let mut value = original.clone(); + value["entries"][0]["body"]["feature_payload"] = serde_json::json!({"private":true}); + invalids.push(value); + let mut value = original.clone(); + value["entries"][0]["body"]["role"] = "system".into(); + invalids.push(value); + let mut value = original.clone(); + value["entries"][0]["source"]["unknown"] = true.into(); + invalids.push(value); + let mut value = original.clone(); + value["entries"][0]["source"]["history_origin"] = + serde_json::json!({"kind":"legacy_unknown","unknown":true}); + invalids.push(value); + let mut value = original.clone(); + value["entries"][0]["entry_id"] = "bad/id".into(); + invalids.push(value); + let mut value = original.clone(); + value["entries"] + .as_array_mut() + .unwrap() + .push(original["entries"][0].clone()); + invalids.push(value); + for invalid in invalids { + let bytes = serde_json::to_vec(&invalid).unwrap(); + assert!( + SessionCapture::decode(&bytes, CaptureLimits::default()).is_err(), + "accepted {invalid}" + ); + } + let bytes = capture.encode(CaptureLimits::default()).unwrap(); + assert!( + SessionCapture::decode( + &bytes, + CaptureLimits { + max_bytes: bytes.len() - 1, + ..CaptureLimits::default() + } + ) + .is_err() + ); + assert!( + capture + .encode(CaptureLimits { + max_entries: 0, + ..CaptureLimits::default() + }) + .is_err() + ); + assert!( + capture + .encode(CaptureLimits { + max_text_bytes: 3, + ..CaptureLimits::default() + }) + .is_err() + ); + let mut reversed = capture.to_dto(); + let mut second = reversed.entries[0].clone(); + second.entry_id = "second".into(); + second.source.entry_index = 4; + second.order = 0; + reversed.entries[0].order = 1; + reversed.entries.push(second); + assert!(SessionCapture::try_from_dto(reversed, CaptureLimits::default()).is_err()); +} + +#[test] +fn raw_utf8_tool_input_and_result_are_lossless_without_preview_markers() { + let arguments = "\"quoted\"\\path\n日本語🙂".repeat(300); + let result = "result\n\t\\\"λ🙂".repeat(400); + let capture = SessionCapture::new( + "s", + "seg", + vec![ + Item::tool_call("call", "tool", arguments.clone()), + Item::tool_result_item("call", "summary", Some(result.clone()), false), + ], + ); + for (id, field, expected) in [ + ("E00000000", RawField::ToolArguments, arguments), + ("E00000001", RawField::ToolContent, result), + ] { + let reference = reference(&capture, id); + let mut offset = 0; + let mut actual = String::new(); + loop { + let part = capture + .read_raw(&reference, CapturePurpose::Observation, field, offset, 17) + .unwrap(); + assert!(part.text.len() <= 17); + assert_eq!(part.offset, offset); + actual.push_str(&part.text); + match part.next_offset { + Some(next) => { + assert!(next > offset); + offset = next; + } + None => break, + } + } + assert_eq!(actual, expected); + } + let unicode = SessionCapture::new("s", "seg", vec![Item::tool_call("call", "tool", "🙂")]); + let reference = reference(&unicode, "E00000000"); + assert!(matches!( + unicode.read_raw( + &reference, + CapturePurpose::Observation, + RawField::ToolArguments, + 1, + 8 + ), + Err(CaptureError::InvalidRange) + )); + assert!(matches!( + unicode.read_raw( + &reference, + CapturePurpose::Observation, + RawField::ToolArguments, + 0, + 3 + ), + Err(CaptureError::BudgetTooSmall) + )); + assert!(matches!( + unicode.read_raw( + &reference, + CapturePurpose::Observation, + RawField::ToolArguments, + 5, + 8 + ), + Err(CaptureError::InvalidRange) + )); +} + +#[test] +fn escaped_json_slice_budget_includes_metadata_and_bound_continuation() { + let body = "\"\\\n\t🙂日本語".repeat(500); + let capture = SessionCapture::new( + "synthetic-session", + "segment", + vec![Item::tool_call("call", "tool", body.clone())], + ); + let other_session = SessionCapture::new( + "other-session", + "segment", + vec![Item::tool_call("call", "tool", body.clone())], + ); + let changed = SessionCapture::new( + "synthetic-session", + "segment", + vec![Item::tool_call("call", "tool", "changed")], + ); + let mut cursor = None; + let mut actual = String::new(); + let budget = 1400; + loop { + let part = capture + .slice(CapturePurpose::Recording, cursor.as_ref(), 1, budget) + .unwrap(); + assert!(part.encoded_len() <= budget); + assert_eq!(part.encoded_len(), part.as_json().len()); + assert_eq!(serde_json::to_vec(part.slice()).unwrap(), part.as_json()); + assert_eq!(part.slice().items.len(), 1); + let item = &part.slice().items[0]; + assert_eq!(item.offset, actual.len()); + actual.push_str(&item.text); + cursor = part.slice().next_cursor.clone(); + if let Some(cursor) = &cursor { + assert!(matches!( + other_session.slice(CapturePurpose::Recording, Some(cursor), 1, budget), + Err(CaptureError::ForeignReference) + )); + assert!(matches!( + changed.slice(CapturePurpose::Recording, Some(cursor), 1, budget), + Err(CaptureError::ForeignReference) + )); + assert!(matches!( + capture.slice(CapturePurpose::Observation, Some(cursor), 1, budget), + Err(CaptureError::ForeignReference) + )); + } else { + break; + } + } + assert_eq!(actual, body); + assert!(matches!( + capture.slice(CapturePurpose::Recording, None, 1, 8), + Err(CaptureError::BudgetTooSmall) + )); + assert!(matches!( + capture.slice(CapturePurpose::Recording, None, 0, budget), + Err(CaptureError::BudgetTooSmall) + )); +} + +#[test] +fn snapshot_immutability_and_scoped_operations_reject_foreign_or_unknown_refs_and_bad_ranges() { + let mut source = vec![ + Item::user_message("before"), + Item::assistant_message("answer"), + ]; + let capture = SessionCapture::new("s", "seg", source.clone()); + let fixed_bytes = capture.encode(CaptureLimits::default()).unwrap(); + let first = reference(&capture, "E00000000"); + let second = reference(&capture, "E00000001"); + source.push(Item::user_message("appended")); + source[0] = Item::user_message("compacted"); + source.clear(); + assert_eq!( + capture.encode(CaptureLimits::default()).unwrap(), + fixed_bytes + ); + assert_eq!( + capture.clone().encode(CaptureLimits::default()).unwrap(), + fixed_bytes + ); + let foreign = SessionCapture::new( + "different", + "seg", + vec![ + Item::user_message("before"), + Item::assistant_message("answer"), + ], + ); + assert!(matches!( + foreign.resolve_ref(&first, CapturePurpose::Observation), + Err(CaptureError::ForeignReference) + )); + assert!(matches!( + capture.checked_range(&second, &first, CapturePurpose::Observation), + Err(CaptureError::InvalidRange) + )); + assert_eq!( + capture + .checked_range(&first, &second, CapturePurpose::Observation) + .unwrap() + .len(), + 2 + ); + let unknown = SessionEntryRef::parse("E99999999").unwrap(); + assert_eq!(capture.source_index_for_ref(&unknown), None); + assert!(matches!( + capture.scoped_ref(&unknown), + Err(CaptureError::UnknownReference) + )); + assert!( + capture + .search_checked(&SearchOptions { + from: Some(unknown.clone()), + ..Default::default() + }) + .is_err() + ); + assert!( + capture + .search(&SearchOptions { + through: Some(unknown), + ..Default::default() + }) + .is_empty() + ); + assert!( + capture + .read_checked(ReadSelector::EntryRange([1, 0]), ReadOptions::default()) + .is_err() + ); + assert!( + capture + .read_checked(ReadSelector::EntryRange([0, 99]), ReadOptions::default()) + .is_err() + ); +} + +#[test] +fn slices_preserve_empty_and_long_result_fields_under_item_and_metadata_budgets() { + let summary = "summary\n\"🙂".repeat(100); + let content = "content\t\\日本語".repeat(200); + let capture = SessionCapture::new( + "s", + "seg", + vec![ + Item::tool_result_item("first", summary.clone(), Some(content.clone()), true), + Item::tool_result_item("empty", "", Some(String::new()), false), + ], + ); + let mut cursor = None; + let mut fields: Vec<(String, RawField, String)> = Vec::new(); + loop { + let part = capture + .slice(CapturePurpose::Recording, cursor.as_ref(), 2, 1500) + .unwrap(); + assert!(part.slice().items.len() <= 2); + assert!(serde_json::to_vec(part.slice()).unwrap().len() <= 1500); + for item in &part.slice().items { + let id = item.reference.entry_ref.to_string(); + if let Some((last_id, last_field, text)) = fields.last_mut() { + if *last_id == id && *last_field == item.field { + assert_eq!(text.len(), item.offset); + text.push_str(&item.text); + continue; + } + } + assert_eq!(item.offset, 0); + fields.push((id, item.field, item.text.clone())); + } + cursor = part.slice().next_cursor.clone(); + if cursor.is_none() { + break; + } + } + assert_eq!( + fields, + vec![ + ("E00000000".into(), RawField::ToolSummary, summary), + ("E00000000".into(), RawField::ToolContent, content), + ("E00000001".into(), RawField::ToolSummary, String::new()), + ("E00000001".into(), RawField::ToolContent, String::new()), + ] + ); + let metadata_heavy = SessionCapture::new( + "s", + "seg", + vec![Item::tool_call("call", "name".repeat(200), "x")], + ); + assert!(matches!( + metadata_heavy.slice(CapturePurpose::Recording, None, 1, 700), + Err(CaptureError::BudgetTooSmall) + )); +} diff --git a/crates/worker/src/feature/builtin/flow_transition.rs b/crates/worker/src/feature/builtin/flow_transition.rs index e3a9fc196..4c9cd98e3 100644 --- a/crates/worker/src/feature/builtin/flow_transition.rs +++ b/crates/worker/src/feature/builtin/flow_transition.rs @@ -26,7 +26,7 @@ use flow::{ use manifest::{Scope, WorkerManifest}; use schemars::JsonSchema; use serde::Deserialize; -use session_store::{SegmentId, SessionId, Store, collect_state}; +use session_store::{SegmentId, SessionId, Store}; use uuid::Uuid; use crate::feature::builtin::session_explore::{SessionExploreFeature, SessionExploreState}; @@ -469,10 +469,15 @@ where .store .read_all(self.session_id, self.segment_id) .map_err(|error| format!("read committed parent session: {error}"))?; - let restored = collect_state(&entries); - Ok(SessionCapture::new( + let history = crate::session_history::restore_history_entries( + self.session_id, + self.segment_id, + &entries, + )?; + Ok(SessionCapture::from_history_entries( + self.session_id.to_string(), self.segment_id.to_string(), - restored.history, + history, )) } } diff --git a/crates/worker/src/feature/builtin/memory_lifecycle.rs b/crates/worker/src/feature/builtin/memory_lifecycle.rs index 94a2a2481..7db770eb0 100644 --- a/crates/worker/src/feature/builtin/memory_lifecycle.rs +++ b/crates/worker/src/feature/builtin/memory_lifecycle.rs @@ -457,6 +457,7 @@ impl MemoryLifecycleTask { return Ok(()); } let view = SessionCapture::from_history_entries( + capture.session_id.clone(), capture.segment_id.clone(), capture.history[history_start..history_end].to_vec(), ); @@ -1556,11 +1557,14 @@ permission = "write" #[tokio::test] async fn run_committed_background_task_stages_non_empty_extraction_and_commits_pointer() { let source = capture(2, 250); - let entry_ref = - SessionCapture::from_history_entries(source.segment_id.clone(), source.history.clone()) - .overview()[0] - .id - .to_string(); + let entry_ref = SessionCapture::from_history_entries( + source.session_id.clone(), + source.segment_id.clone(), + source.history.clone(), + ) + .overview()[0] + .id + .to_string(); let client = ScriptClient::new(vec![ stage_candidate_events("stage-1", &entry_ref), finish_events("finish-1", 1), @@ -1609,11 +1613,14 @@ permission = "write" #[tokio::test] async fn subjektiv_extraction_stages_once_and_commits_its_independent_pointer() { let source = capture(2, 250); - let entry_ref = - SessionCapture::from_history_entries(source.segment_id.clone(), source.history.clone()) - .overview()[0] - .id - .to_string(); + let entry_ref = SessionCapture::from_history_entries( + source.session_id.clone(), + source.segment_id.clone(), + source.history.clone(), + ) + .overview()[0] + .id + .to_string(); let client = ScriptClient::new(vec![ stage_candidate_events("stage-1", &entry_ref), finish_events("finish-1", 1), @@ -1715,11 +1722,14 @@ permission = "write" #[tokio::test] async fn subjektiv_candidate_remains_staged_when_later_extraction_fails_without_pointer() { let source = capture(2, 250); - let entry_ref = - SessionCapture::from_history_entries(source.segment_id.clone(), source.history.clone()) - .overview()[0] - .id - .to_string(); + let entry_ref = SessionCapture::from_history_entries( + source.session_id.clone(), + source.segment_id.clone(), + source.history.clone(), + ) + .overview()[0] + .id + .to_string(); let client = ScriptClient::new(vec![stage_candidate_events("stage-1", &entry_ref)]); let extension_writes = Arc::new(Mutex::new(Vec::new())); let (event_tx, _) = broadcast::channel(32); diff --git a/crates/worker/src/feature/builtin/memory_staging_output.rs b/crates/worker/src/feature/builtin/memory_staging_output.rs index ad5de09df..dd830349d 100644 --- a/crates/worker/src/feature/builtin/memory_staging_output.rs +++ b/crates/worker/src/feature/builtin/memory_staging_output.rs @@ -510,7 +510,11 @@ mod tests { fn state() -> MemoryStagingOutputState { MemoryStagingOutputState::new( - SessionCapture::new("segment-1", vec![Item::user_message("durable decision")]), + SessionCapture::new( + "session", + "segment-1", + vec![Item::user_message("durable decision")], + ), crate::worker::marker_workspace_client(None, "test-backend"), SourceRef { segment_id: "segment-1".to_string(), @@ -537,6 +541,7 @@ mod tests { #[test] fn render_input_uses_session_entry_refs_and_new_tool_names() { let view = SessionCapture::new( + "session", "segment-1", vec![ Item::user_message("preference"), @@ -586,7 +591,11 @@ mod tests { async fn subjektiv_destination_sends_committed_session_without_subject_scope() { let client = Arc::new(RecordingSubjektivClient::default()); let state = MemoryStagingOutputState::new_subjektiv( - SessionCapture::new("segment-1", vec![Item::user_message("durable decision")]), + SessionCapture::new( + "session", + "segment-1", + vec![Item::user_message("durable decision")], + ), { let mut host = crate::subjektiv::test_connection(client.clone()); host.context.session_id = "session-committed".into(); diff --git a/crates/worker/src/feature/builtin/session_explore.rs b/crates/worker/src/feature/builtin/session_explore.rs index aacc16081..1400f51b4 100644 --- a/crates/worker/src/feature/builtin/session_explore.rs +++ b/crates/worker/src/feature/builtin/session_explore.rs @@ -17,14 +17,14 @@ use crate::feature::{ ToolDeclaration, }; use crate::session_capture::{ - ReadDetail, ReadOptions, ReadSelector, ReferenceKind, SearchOptions, SessionCapture, - SessionEntryRef, ToolPart, + CapturePurpose, RawField, ReadDetail, ReadOptions, ReadSelector, ReferenceKind, SearchOptions, + SessionCapture, SessionEntryRef, ToolPart, }; const SHOW_OVERVIEW_DESCRIPTION: &str = "Show a sparse, bounded index of real user and assistant session entries."; const SEARCH_ENTRIES_DESCRIPTION: &str = "Search or compactly list a bounded range of committed session entries. Reasoning entries are never exposed."; -const READ_ENTRY_DESCRIPTION: &str = "Read one committed session entry by stable SessionEntryRef. Compact mode is the default; full mode includes bounded tool input or output."; +const READ_ENTRY_DESCRIPTION: &str = "Read one committed session entry by stable SessionEntryRef. Compact mode is the default; full mode is a bounded rendering, not raw source. Raw mode requires raw_field and supports UTF-8 byte_offset/next_offset without omission; use capture_id to bind continuation to this exact capture."; const DEFAULT_PAGE_LIMIT: usize = 20; const MAX_PAGE_LIMIT: usize = 100; const DEFAULT_READ_ITEMS: usize = 10; @@ -173,6 +173,38 @@ struct ReadEntryParams { mode: String, #[serde(default)] max_items: Option, + #[serde(default)] + raw_field: Option, + #[serde(default)] + part_index: Option, + #[serde(default)] + byte_offset: Option, + #[serde(default)] + max_bytes: Option, + #[serde(default)] + capture_id: Option, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +enum RawEntryField { + MessagePart, + PublicSystem, + ToolArguments, + ToolSummary, + ToolContent, +} + +impl RawEntryField { + fn into_common(self, part_index: usize) -> RawField { + match self { + Self::MessagePart => RawField::MessagePart(part_index), + Self::PublicSystem => RawField::PublicSystem, + Self::ToolArguments => RawField::ToolArguments, + Self::ToolSummary => RawField::ToolSummary, + Self::ToolContent => RawField::ToolContent, + } + } } fn default_read_mode() -> String { @@ -243,25 +275,20 @@ impl Tool for SearchEntriesTool { let from = params.from.as_deref().map(parse_entry_ref).transpose()?; let through = params.through.as_deref().map(parse_entry_ref).transpose()?; let view = self.state.view(); - if let (Some(from), Some(through)) = (&from, &through) { - if view.source_index_for_ref(from) > view.source_index_for_ref(through) { - return Err(ToolError::InvalidArgument( - "SearchEntries from must not be after through".to_string(), - )); - } - } let limit = bounded_limit(params.limit, DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT); - let hits = view.search(&SearchOptions { - query: params.query, - kind, - tool_part, - tool_name: params.tool_name, - limit: Some(limit), - min_entry_index: None, - from, - through, - offset: params.offset, - }); + let hits = view + .search_checked(&SearchOptions { + query: params.query, + kind, + tool_part, + tool_name: params.tool_name, + limit: Some(limit), + min_entry_index: None, + from, + through, + offset: params.offset, + }) + .map_err(|error| ToolError::InvalidArgument(error.to_string()))?; let entries = hits .iter() .map(|hit| { @@ -272,6 +299,7 @@ impl Tool for SearchEntriesTool { "tool_name": hit.tool_name, "label": hit.label, "text": hit.summary, + "text_kind": "preview", }) }) .collect::>(); @@ -299,12 +327,62 @@ impl Tool for ReadEntryTool { ) -> Result { let params: ReadEntryParams = parse_input("ReadEntry", input_json)?; let entry_ref = parse_entry_ref(¶ms.entry_ref)?; + let view = self.state.view(); + if params + .capture_id + .as_deref() + .is_some_and(|id| id != view.capture_id()) + { + return Err(ToolError::InvalidArgument( + "capture identity does not match".into(), + )); + } + if params.mode == "raw" { + let field = params + .raw_field + .ok_or_else(|| ToolError::InvalidArgument("raw mode requires raw_field".into()))?; + if params.part_index.is_some() && !matches!(field, RawEntryField::MessagePart) { + return Err(ToolError::InvalidArgument( + "part_index is only valid for message_part".into(), + )); + } + let reference = view + .scoped_ref(&entry_ref) + .map_err(|error| ToolError::InvalidArgument(error.to_string()))?; + let raw = view + .read_raw( + &reference, + CapturePurpose::Observation, + field.into_common(params.part_index.unwrap_or(0)), + params.byte_offset.unwrap_or(0), + params + .max_bytes + .unwrap_or(MAX_READ_BYTES) + .clamp(1, MAX_READ_BYTES), + ) + .map_err(|error| ToolError::InvalidArgument(error.to_string()))?; + return json_output( + "Read raw session source bytes.".into(), + serde_json::json!({ + "text_kind": "raw", "raw": raw, + }), + ); + } + if params.raw_field.is_some() + || params.part_index.is_some() + || params.byte_offset.is_some() + || params.max_bytes.is_some() + { + return Err(ToolError::InvalidArgument( + "raw fields require raw mode".into(), + )); + } let detail = match params.mode.as_str() { "compact" => ReadDetail::Compact, "full" => ReadDetail::Full, other => { return Err(ToolError::InvalidArgument(format!( - "invalid ReadEntry mode {other:?}; expected compact or full" + "invalid ReadEntry mode {other:?}; expected compact, full, or raw" ))); } }; @@ -347,6 +425,7 @@ impl Tool for ReadEntryTool { serde_json::json!({ "entries": entries, "truncated": read.truncated, + "text_kind": "rendered", }), ) } @@ -408,6 +487,7 @@ mod tests { fn state() -> SessionExploreState { SessionExploreState::new(SessionCapture::new( + "session", "segment-1", vec![ Item::user_message("first"), @@ -479,4 +559,64 @@ mod tests { .unwrap(); assert!(output.content.unwrap().contains("second")); } + #[tokio::test] + async fn search_rejects_unknown_or_reversed_capture_endpoints() { + let search = search_entries_definition(state())().1; + for input in [ + r#"{"from":"E99999999"}"#, + r#"{"through":"E99999999"}"#, + r#"{"from":"E00000003","through":"E00000000"}"#, + ] { + assert!( + search + .execute(input, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + } + } + + #[tokio::test] + async fn raw_tool_arguments_reassemble_without_preview_or_utf8_loss() { + let arguments = "日本語\n\"\\".repeat(100); + let state = SessionExploreState::new(SessionCapture::new( + "session", + "segment", + vec![Item::tool_call("call", "Read", arguments.clone())], + )); + let capture_id = state.view().capture_id().to_owned(); + let tool = read_entry_definition(state)().1; + let mut offset = 0; + let mut assembled = String::new(); + loop { + let input = serde_json::json!({ + "entry_ref": "E00000000", "mode": "raw", "raw_field": "tool_arguments", + "byte_offset": offset, "max_bytes": 17, "capture_id": capture_id, + }); + let output = tool + .execute( + &input.to_string(), + agen::tool::ToolExecutionContext::direct(), + ) + .await + .unwrap(); + let value: serde_json::Value = serde_json::from_str(&output.content.unwrap()).unwrap(); + assert_eq!(value["text_kind"], "raw"); + let raw = &value["raw"]; + assert_eq!(raw["offset"], offset); + let text = raw["text"].as_str().unwrap(); + assert!(text.len() <= 17); + assembled.push_str(text); + match raw["next_offset"].as_u64() { + Some(next) => { + assert!(next > offset as u64); + offset = next as usize; + } + None => break, + } + } + assert_eq!(assembled, arguments); + assert!(tool.execute(r#"{"entry_ref":"E00000000","mode":"raw","raw_field":"tool_arguments","capture_id":"other"}"#, + agen::tool::ToolExecutionContext::direct()).await.is_err()); + } } diff --git a/crates/worker/src/feature/builtin/subjektiv_memory.rs b/crates/worker/src/feature/builtin/subjektiv_memory.rs index cd82ca390..82ce970f9 100644 --- a/crates/worker/src/feature/builtin/subjektiv_memory.rs +++ b/crates/worker/src/feature/builtin/subjektiv_memory.rs @@ -39,13 +39,66 @@ use crate::hook::{ HookErrorCategory, HookExecutionPolicy, HookFailurePolicy, HookPreRequestAction, PreLlmRequest, PreRequestContext, RunCommitted, RunCommittedContext, }; -use crate::session_capture::{SessionCapture, SessionEntryEvidence}; +use crate::session_capture::{CaptureBody, SessionCapture, SessionEntryEvidence}; use crate::subjektiv::SubjektivHostConnection; #[cfg(test)] use crate::worker::WorkspaceClient; #[cfg(test)] use server_api::SubjektivMemoryBackendRequest; +// Pending-commit receipt interpretation belongs to Memory, not the shared capture. +struct CommittedMemoryCall { + call_id: String, + name: String, + arguments: String, + evidence: SessionEntryEvidence, +} + +fn committed_pending_tool_calls(view: &SessionCapture, names: &[&str]) -> Vec { + let pending: HashSet<_> = view + .entries() + .filter_map(|entry| { + let CaptureBody::ToolResult { + call_id, + content: Some(content), + is_error: false, + .. + } = &entry.body + else { + return None; + }; + (serde_json::from_str::(content) + .ok() + .and_then(|value| value.get("status").cloned()) + .as_ref() + .and_then(serde_json::Value::as_str) + == Some("pending_commit")) + .then_some(call_id.as_str()) + }) + .collect(); + view.entries() + .filter_map(|entry| { + let CaptureBody::ToolCall { + call_id, + name, + arguments, + } = &entry.body + else { + return None; + }; + if !names.contains(&name.as_str()) || !pending.contains(call_id.as_str()) { + return None; + } + Some(CommittedMemoryCall { + call_id: call_id.clone(), + name: name.clone(), + arguments: arguments.clone(), + evidence: view.evidence_for_tool_call(call_id)?, + }) + }) + .collect() +} + const QUERY_TOOL: &str = "SubjektivMemoryQuery"; const READ_TOOL: &str = "SubjektivMemoryRead"; const LIST_CHANGES_TOOL: &str = "SubjektivMemoryListChanges"; @@ -518,14 +571,18 @@ impl SubjektivMemoryState { .capture() .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?; let session_id = capture.session_id.clone(); - let view = SessionCapture::from_history_entries(capture.segment_id, capture.history); - let committed = view - .committed_pending_tool_calls(&[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL]) - .into_iter() - .any(|call| { - receipt_id(&session_id, &call.call_id) - .is_ok_and(|committed_receipt| committed_receipt == expected_receipt_id) - }); + let view = SessionCapture::from_history_entries( + capture.session_id.clone(), + capture.segment_id, + capture.history, + ); + let committed = + committed_pending_tool_calls(&view, &[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL]) + .into_iter() + .any(|call| { + receipt_id(&session_id, &call.call_id) + .is_ok_and(|committed_receipt| committed_receipt == expected_receipt_id) + }); if committed { status.status = SubjektivMemoryReceiptStatus::PendingCommit; } @@ -623,10 +680,13 @@ fn replay_pending_committed_memory_with_policy( .capture .capture() .map_err(|error| HookError::new(HookErrorCategory::Dependency, error.to_string()))?; - let view = - SessionCapture::from_history_entries(capture.segment_id.clone(), capture.history.clone()); + let view = SessionCapture::from_history_entries( + capture.session_id.clone(), + capture.segment_id.clone(), + capture.history.clone(), + ); let mut failures = Vec::new(); - for call in view.committed_pending_tool_calls(&[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL]) { + for call in committed_pending_tool_calls(&view, &[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL]) { let receipt_id = match receipt_id(&capture.session_id, &call.call_id) { Ok(value) => value, Err(error) => { @@ -755,8 +815,11 @@ fn resolve_entry_refs( capture: &crate::feature::session::CommittedSessionCapture, entry_refs: &[String], ) -> Result, ToolError> { - let view = - SessionCapture::from_history_entries(capture.segment_id.clone(), capture.history.clone()); + let view = SessionCapture::from_history_entries( + capture.session_id.clone(), + capture.segment_id.clone(), + capture.history.clone(), + ); let mut seen = HashSet::new(); entry_refs .iter() diff --git a/crates/worker/src/feature/builtin/worker_observation.rs b/crates/worker/src/feature/builtin/worker_observation.rs index 9b6d5488c..d1c4d0cce 100644 --- a/crates/worker/src/feature/builtin/worker_observation.rs +++ b/crates/worker/src/feature/builtin/worker_observation.rs @@ -570,15 +570,8 @@ impl Tool for SearchSessionEntriesTool { let view = latest_view(&*self.provider, ¶ms.subject).await?; let from = params.from.as_deref().map(parse_entry_ref).transpose()?; let through = params.through.as_deref().map(parse_entry_ref).transpose()?; - if let (Some(from), Some(through)) = (&from, &through) { - if view.source_index_for_ref(from) > view.source_index_for_ref(through) { - return Err(ToolError::InvalidArgument( - "SearchSessionEntries from must not be after through".to_string(), - )); - } - } let entries = view - .search(&SearchOptions { + .search_checked(&SearchOptions { query: params.query, kind: params.kind.as_deref().map(parse_kind).transpose()?, tool_part: params @@ -593,6 +586,7 @@ impl Tool for SearchSessionEntriesTool { through, offset: params.offset, }) + .map_err(|error| ToolError::InvalidArgument(error.to_string()))? .into_iter() .map(|entry| { serde_json::json!({ @@ -604,6 +598,7 @@ impl Tool for SearchSessionEntriesTool { "tool_name": entry.tool_name, "label": entry.label, "text": entry.summary, + "text_kind": "preview", }) }) .collect::>(); @@ -687,7 +682,10 @@ async fn latest_view( .capture_worker_session(subject) .await .map_err(tool_error)?; + // Observation providers expose a stable subject scope rather than a raw + // Session handle. This scope is never used to select recording material. Ok(SessionCapture::from_session_snapshot( + format!("observation:{}", capture.segment_id), capture.segment_id, capture.session, )) diff --git a/crates/worker/src/session_capture.rs b/crates/worker/src/session_capture.rs index c255d7dad..979c8d44a 100644 --- a/crates/worker/src/session_capture.rs +++ b/crates/worker/src/session_capture.rs @@ -1,881 +1,14 @@ -//! Exploration domain for immutable Session captures, with no Workspace/Memory mutation capability. -//! -//! A capture is an ordered projection of committed entries; exploration excludes reasoning and -//! system prompts, including Host compaction summaries and other derived context. Full inherited -//! summary input belongs to compaction, not public observation or Memory evidence. Overview, -//! search, bounded reads, and evidence conversion share `SessionEntryRef`. -//! References identify entries in the selected Session and cannot be reused for another subject. -//! Appends preserve existing entry identities. Sparse real user/assistant anchors and intervening -//! entry counts index long histories. Search with an empty filter also provides bounded range lists. -//! -//! Host-fixed snapshot exploration, Memory candidate evidence validation, and current Worker -//! observation share only this domain; they do not enable or modify each other's Features. -//! The provider owns authorization to recapture a current Worker. A capture itself proves neither -//! completion nor approval and grants no business-mutation authority. +//! Host consumers share the pure immutable Session capture implementation. +//! Live authorization, append fences, and recapture remain with the calling Host. -use std::collections::HashSet; -use std::sync::Arc; - -use crate::session_history::{SessionHistoryMetadata, WorkerHistoryProvenance}; -use agen::{HistoryEntry, Item, Role}; -use protocol::{ - SessionContentPart, SessionEntryProvenance, SessionMessageRole, SessionSnapshot, - SessionSnapshotEntryData, -}; -use serde::{Deserialize, Serialize}; - -const DEFAULT_SEARCH_LIMIT: usize = 20; -const MAX_SEARCH_LIMIT: usize = 50; -const DEFAULT_READ_MAX_ITEMS: usize = 40; -const MAX_READ_MAX_ITEMS: usize = 80; -const DEFAULT_READ_MAX_BYTES: usize = 32 * 1024; -const OVERVIEW_ANCHOR_STRIDE: usize = 8; - -/// Append-stable exploration reference derived from an entry identity in the selected Session. -/// Shared by overview, search, read, and evidence conversion; it does not identify other Sessions' entries. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(transparent)] -pub struct SessionEntryRef(String); - -impl SessionEntryRef { - pub fn from_history_entry_id(entry_id: &crate::SessionHistoryEntryId) -> Self { - Self(format!("E{}", entry_id.0)) - } - - pub fn parse(value: &str) -> Option { - let suffix = value.strip_prefix('E')?; - if suffix.is_empty() - || suffix.len() > 64 - || !suffix - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') - { - return None; - } - Some(Self(value.to_string())) - } - - pub fn as_str(&self) -> &str { - &self.0 - } - - pub(crate) fn source_index(&self) -> Option { - self.0.strip_prefix('E')?.parse().ok() - } -} - -impl std::fmt::Display for SessionEntryRef { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(&self.0) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ReferenceKind { - User, - Assistant, - System, - Tool, -} - -impl ReferenceKind { - pub(crate) fn as_str(self) -> &'static str { - match self { - Self::User => "user", - Self::Assistant => "assistant", - Self::System => "system", - Self::Tool => "tool", - } - } - - pub(crate) fn parse(value: &str) -> Option { - match value { - "user" => Some(Self::User), - "assistant" | "agent" => Some(Self::Assistant), - "system" => Some(Self::System), - "tool" => Some(Self::Tool), - _ => None, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ToolPart { - Input, - Output, - Both, -} - -impl ToolPart { - pub(crate) fn parse(value: &str) -> Option { - match value { - "input" => Some(Self::Input), - "output" => Some(Self::Output), - "both" => Some(Self::Both), - _ => None, - } - } - - fn matches(self, actual: ToolPart) -> bool { - matches!(self, ToolPart::Both) || self == actual - } -} - -#[derive(Debug, Clone)] -pub(crate) struct OverviewItem { - pub id: SessionEntryRef, - pub origin: SessionEntryProvenance, - pub entry_range: [u64; 2], - pub kind: ReferenceKind, - pub label: String, - pub text: String, - pub intervening_entries: usize, -} - -#[derive(Debug, Clone)] -pub(crate) struct ReferenceEntry { - use_policy: Option, - pub id: SessionEntryRef, - pub origin: SessionEntryProvenance, - pub entry_range: [u64; 2], - pub kind: ReferenceKind, - pub tool_part: Option, - pub tool_name: Option, - pub label: String, - pub summary: String, - search_text: String, -} - -#[derive(Debug, Clone, Default)] -pub(crate) struct SearchOptions { - pub query: String, - pub kind: Option, - pub tool_part: Option, - pub tool_name: Option, - pub limit: Option, - pub min_entry_index: Option, - pub from: Option, - pub through: Option, - pub offset: usize, -} - -#[derive(Debug, Clone)] -pub(crate) struct SearchHit { - pub id: SessionEntryRef, - pub origin: SessionEntryProvenance, - pub kind: ReferenceKind, - pub tool_part: Option, - pub tool_name: Option, - pub entry_range: [u64; 2], - pub label: String, - pub summary: String, -} - -#[derive(Debug, Clone, Copy)] -pub(crate) enum ReadSelector<'a> { - Id(&'a str), - #[cfg(test)] - EntryRange([u64; 2]), -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ReadDetail { - Compact, - Full, -} - -#[derive(Debug, Clone)] -pub(crate) struct ReadOptions { - pub include_tools: bool, - pub tool_part: ToolPart, - pub detail: ReadDetail, - pub max_items: usize, - pub max_bytes: usize, -} - -impl Default for ReadOptions { - fn default() -> Self { - Self { - include_tools: true, - tool_part: ToolPart::Both, - detail: ReadDetail::Compact, - max_items: DEFAULT_READ_MAX_ITEMS, - max_bytes: DEFAULT_READ_MAX_BYTES, - } - } -} - -#[derive(Debug, Clone)] -pub(crate) struct ReadEntry { - pub id: SessionEntryRef, - pub origin: SessionEntryProvenance, - pub kind: ReferenceKind, - pub tool_part: Option, - pub tool_name: Option, - pub entry_range: [u64; 2], - pub label: String, - pub text: String, -} - -#[derive(Debug, Clone)] -pub(crate) struct ReadResult { - pub entries: Vec, - pub truncated: bool, -} - -#[derive(Debug, Clone)] -pub(crate) struct CommittedToolCall { - pub call_id: String, - pub name: String, - pub arguments: String, - pub evidence: SessionEntryEvidence, -} - -#[derive(Debug, Clone)] -pub(crate) struct SessionEntryEvidence { - pub segment_id: String, - pub entry_ref: SessionEntryRef, - pub origin: SessionEntryProvenance, - pub entry_range: [u64; 2], - pub kind: ReferenceKind, - pub tool_part: Option, - pub label: String, - pub summary: String, - pub excerpt: String, -} - -#[derive(Debug, Clone)] -struct CapturedHistoryEntry { - use_policy: Option, - item: Item, - entry_id: session_store::LoggedSessionHistoryEntryId, - origin: SessionEntryProvenance, -} - -#[derive(Debug, Clone)] -pub(crate) struct SessionCapture { - segment_id: String, - entries: Arc>, - overview: Vec, - index: Vec, -} - -impl SessionCapture { - pub(crate) fn from_session_snapshot( - segment_id: impl Into, - snapshot: SessionSnapshot, - ) -> Self { - let entries = snapshot - .entries - .into_iter() - .filter_map(|entry| { - let item = match entry.data { - SessionSnapshotEntryData::UserInput { segments } => { - Item::user_message(protocol::Segment::flatten_to_text(&segments)) - } - SessionSnapshotEntryData::Message { role, content } => { - let role = match role { - SessionMessageRole::User => Role::User, - SessionMessageRole::Assistant => Role::Assistant, - }; - Item::Message { - id: None, - role, - content: content - .into_iter() - .map(|part| match part { - SessionContentPart::Text { text } => { - agen::ContentPart::Text { text } - } - SessionContentPart::Refusal { refusal } => { - agen::ContentPart::Refusal { refusal } - } - }) - .collect(), - status: None, - } - } - SessionSnapshotEntryData::ToolCall { - call_id, - name, - arguments, - } => Item::tool_call(call_id, name, arguments), - SessionSnapshotEntryData::ToolResult { - call_id, - summary, - content, - is_error, - attachments: _, - } => Item::tool_result_item(call_id, summary, content, is_error), - SessionSnapshotEntryData::SystemItem { content, .. } => { - entry.use_policy?; - Item::system_message(content) - } - // Controller errors are not model-visible evidence. - SessionSnapshotEntryData::Invoke { .. } - | SessionSnapshotEntryData::Usage { .. } - | SessionSnapshotEntryData::RunStats { .. } - | SessionSnapshotEntryData::RunYielded { .. } - | SessionSnapshotEntryData::RunResumed { .. } - | SessionSnapshotEntryData::RunCancelled - | SessionSnapshotEntryData::RunError { .. } => return None, - }; - Some(CapturedHistoryEntry { - use_policy: entry.use_policy, - item, - entry_id: session_store::LoggedSessionHistoryEntryId(entry.entry_id), - origin: entry.provenance, - }) - }) - .collect(); - Self::from_captured_entries(segment_id, entries) - } - - pub(crate) fn new(segment_id: impl Into, items: Vec) -> Self { - let entries = items - .into_iter() - .enumerate() - .map(|(index, item)| CapturedHistoryEntry { - use_policy: None, - item, - entry_id: session_store::LoggedSessionHistoryEntryId(format!("{index:08}")), - origin: SessionEntryProvenance::LegacyUnknown, - }) - .collect(); - Self::from_captured_entries(segment_id, entries) - } - - pub(crate) fn from_history_entries( - segment_id: impl Into, - entries: Vec>, - ) -> Self { - let entries = entries - .into_iter() - .map(|entry| { - let (item, use_policy) = if let Some(input) = &entry.annotation.input_metadata { - let is_system = matches!( - entry.item, - Item::Message { - role: Role::System, - .. - } - ); - if is_system { - ( - Item::system_message(input.public_projection().body), - Some(input.public_system_use_policy()), - ) - } else { - (entry.item, Some(input.public_use_policy())) - } - } else { - (entry.item, None) - }; - CapturedHistoryEntry { - item, - use_policy, - entry_id: entry.annotation.entry_id, - origin: public_provenance(&entry.annotation.origin), - } - }) - .collect(); - Self::from_captured_entries(segment_id, entries) - } - - fn from_captured_entries( - segment_id: impl Into, - entries: Vec, - ) -> Self { - let segment_id = segment_id.into(); - let mut overview = Vec::new(); - let mut index = Vec::new(); - - for (idx, entry) in entries.iter().enumerate() { - let item = &entry.item; - let entry_range = [idx as u64, idx as u64]; - match item { - Item::Message { role, content, .. } => { - let Some(kind) = message_reference_kind(&entry.origin, role, entry.use_policy) - else { - continue; - }; - let text = content - .iter() - .map(|p| p.as_text()) - .collect::>() - .join(""); - let label = format!("{} message", kind.as_str()); - let summary = truncate_chars(&text, 240); - let id = SessionEntryRef::from_history_entry_id(&entry.entry_id); - index.push(ReferenceEntry { - use_policy: entry.use_policy, - id: id.clone(), - origin: entry.origin.clone(), - entry_range, - kind, - tool_part: None, - tool_name: None, - label: label.clone(), - summary: summary.clone(), - search_text: text.clone(), - }); - if entry.use_policy.is_none_or(|p| p.observation_readable) - && matches!(kind, ReferenceKind::User | ReferenceKind::Assistant) - { - overview.push(OverviewItem { - id: id.clone(), - origin: entry.origin.clone(), - entry_range, - kind, - label, - text, - intervening_entries: 0, - }); - } - } - Item::ToolCall { - name, arguments, .. - } => { - let text = format!("{name}\n{arguments}"); - index.push(ReferenceEntry { - use_policy: entry.use_policy, - id: SessionEntryRef::from_history_entry_id(&entry.entry_id), - origin: entry.origin.clone(), - entry_range, - kind: ReferenceKind::Tool, - tool_part: Some(ToolPart::Input), - tool_name: Some(name.clone()), - label: format!("tool input: {name}"), - summary: format!("Tool call {name}"), - search_text: text, - }); - } - Item::ToolResult { - summary, - content, - attachments, - .. - } => { - let attachment_marker = if attachments.is_empty() { - String::new() - } else { - format!("\n[{} image attachment(s)]", attachments.len()) - }; - let text = format!( - "{summary}\n{}{attachment_marker}", - content.as_deref().unwrap_or_default(), - ); - index.push(ReferenceEntry { - use_policy: entry.use_policy, - id: SessionEntryRef::from_history_entry_id(&entry.entry_id), - origin: entry.origin.clone(), - entry_range, - kind: ReferenceKind::Tool, - tool_part: Some(ToolPart::Output), - tool_name: None, - label: "tool output".to_string(), - summary: truncate_chars(summary, 240), - search_text: text, - }); - } - Item::AssistantResponseBoundary { .. } | Item::Reasoning { .. } => {} - } - } - - if overview.len() > 2 { - let last = overview.len() - 1; - overview = overview - .into_iter() - .enumerate() - .filter_map(|(index, entry)| { - (index == 0 || index == last || index % OVERVIEW_ANCHOR_STRIDE == 0) - .then_some(entry) - }) - .collect(); - } - - for overview_index in 0..overview.len().saturating_sub(1) { - let current_entry = overview[overview_index].entry_range[0]; - let next_entry = overview[overview_index + 1].entry_range[0]; - overview[overview_index].intervening_entries = index - .iter() - .filter(|entry| { - let entry_index = entry.entry_range[0]; - entry_index > current_entry && entry_index < next_entry - }) - .count(); - } - - Self { - segment_id, - entries: Arc::new(entries), - overview, - index, - } - } - - pub(crate) fn overview(&self) -> &[OverviewItem] { - &self.overview - } - - pub(crate) fn source_index_for_ref(&self, reference: &SessionEntryRef) -> Option { - self.index - .iter() - .find(|entry| entry.id == *reference) - .map(|entry| entry.entry_range[0]) - .or_else(|| reference.source_index()) - } - - pub(crate) fn search(&self, options: &SearchOptions) -> Vec { - let query = options.query.trim().to_lowercase(); - let limit = options - .limit - .unwrap_or(DEFAULT_SEARCH_LIMIT) - .clamp(1, MAX_SEARCH_LIMIT); - let tool_name = options.tool_name.as_deref(); - let min_entry_index = options - .from - .as_ref() - .and_then(|reference| self.source_index_for_ref(reference)) - .unwrap_or_else(|| options.min_entry_index.unwrap_or(0)); - let max_entry_index = options - .through - .as_ref() - .and_then(|reference| self.source_index_for_ref(reference)) - .unwrap_or(u64::MAX); - let mut skipped = 0usize; - let mut hits = Vec::new(); - - for entry in &self.index { - if entry.use_policy.is_some_and(|p| !p.observation_readable) { - continue; - } - if entry.entry_range[0] < min_entry_index || entry.entry_range[0] > max_entry_index { - continue; - } - if let Some(kind) = options.kind { - if entry.kind != kind { - continue; - } - } - if let Some(part) = options.tool_part { - if entry.kind != ReferenceKind::Tool { - continue; - } - let Some(actual) = entry.tool_part else { - continue; - }; - if !part.matches(actual) { - continue; - } - } - if let Some(name) = tool_name { - if entry.tool_name.as_deref() != Some(name) { - continue; - } - } - if !query.is_empty() && !entry.search_text.to_lowercase().contains(&query) { - continue; - } - if skipped < options.offset { - skipped += 1; - continue; - } - hits.push(SearchHit { - id: entry.id.clone(), - origin: entry.origin.clone(), - kind: entry.kind, - tool_part: entry.tool_part, - tool_name: entry.tool_name.clone(), - entry_range: entry.entry_range, - label: entry.label.clone(), - summary: entry.summary.clone(), - }); - if hits.len() >= limit { - break; - } - } - - hits - } - - pub(crate) fn read(&self, selector: ReadSelector<'_>, options: ReadOptions) -> ReadResult { - let max_items = options.max_items.clamp(1, MAX_READ_MAX_ITEMS); - let max_bytes = options.max_bytes.max(1); - let mut entries = Vec::new(); - let mut bytes = 0usize; - let mut truncated = false; - - let selected: Vec<&ReferenceEntry> = match selector { - ReadSelector::Id(id) => self - .index - .iter() - .filter(|entry| entry.id.as_str() == id) - .collect(), - #[cfg(test)] - ReadSelector::EntryRange([start, end]) => self - .index - .iter() - .filter(|entry| entry.entry_range[0] >= start && entry.entry_range[0] <= end) - .collect(), - }; - - for entry in selected { - if entry.use_policy.is_some_and(|p| !p.observation_readable) { - continue; - } - if entries.len() >= max_items { - truncated = true; - break; - } - if entry.kind == ReferenceKind::Tool { - if !options.include_tools { - continue; - } - if let Some(actual) = entry.tool_part { - if !options.tool_part.matches(actual) { - continue; - } - } - } - let Some(item) = self - .entries - .get(entry.entry_range[0] as usize) - .map(|entry| &entry.item) - else { - continue; - }; - let text = render_item(item, entry, options.detail, max_bytes.saturating_sub(bytes)); - bytes = bytes.saturating_add(text.len()); - entries.push(ReadEntry { - id: entry.id.clone(), - origin: entry.origin.clone(), - kind: entry.kind, - tool_part: entry.tool_part, - tool_name: entry.tool_name.clone(), - entry_range: entry.entry_range, - label: entry.label.clone(), - text, - }); - if bytes >= max_bytes { - truncated = true; - break; - } - } - - ReadResult { entries, truncated } - } - - /// Resolves the committed tool-call input for a provider call id. This is - /// used by explicit-memory receipts only after the containing run is durable, - /// so a model-authored claim can cite its own committed tool invocation - /// without inventing a SessionEntryRef. - pub(crate) fn evidence_for_tool_call(&self, call_id: &str) -> Option { - let entry_index = self.entries.iter().position(|entry| { - matches!( - &entry.item, - Item::ToolCall { - call_id: stored_call_id, - .. - } if stored_call_id == call_id - ) - })?; - let reference = self.index.iter().find(|entry| { - entry.entry_range[0] == entry_index as u64 - && entry.kind == ReferenceKind::Tool - && entry.tool_part == Some(ToolPart::Input) - })?; - self.evidence_for(reference.id.as_str()) - } - - pub(crate) fn committed_pending_tool_calls(&self, names: &[&str]) -> Vec { - let pending_call_ids = self - .entries - .iter() - .filter_map(|entry| { - let Item::ToolResult { - call_id, - content: Some(content), - is_error: false, - .. - } = &entry.item - else { - return None; - }; - (serde_json::from_str::(content) - .ok() - .and_then(|value| value.get("status").cloned()) - .as_ref() - .and_then(serde_json::Value::as_str) - == Some("pending_commit")) - .then_some(call_id.as_str()) - }) - .collect::>(); - self.entries - .iter() - .filter_map(|entry| { - let Item::ToolCall { - call_id, - name, - arguments, - .. - } = &entry.item - else { - return None; - }; - if !names.contains(&name.as_str()) || !pending_call_ids.contains(call_id.as_str()) { - return None; - } - Some(CommittedToolCall { - call_id: call_id.clone(), - name: name.clone(), - arguments: arguments.clone(), - evidence: self.evidence_for_tool_call(call_id)?, - }) - }) - .collect() - } - - pub(crate) fn evidence_for(&self, id: &str) -> Option { - let entry = self.index.iter().find(|entry| entry.id.as_str() == id)?; - if entry.use_policy.is_some_and(|p| !p.memory_evidence) { - return None; - } - let item = &self.entries.get(entry.entry_range[0] as usize)?.item; - let excerpt = render_item(item, entry, ReadDetail::Compact, 2 * 1024); - Some(SessionEntryEvidence { - segment_id: self.segment_id.clone(), - entry_ref: entry.id.clone(), - origin: entry.origin.clone(), - entry_range: entry.entry_range, - kind: entry.kind, - tool_part: entry.tool_part, - label: entry.label.clone(), - summary: entry.summary.clone(), - excerpt, - }) - } -} - -fn public_provenance(origin: &WorkerHistoryProvenance) -> SessionEntryProvenance { - match origin { - WorkerHistoryProvenance::HumanInput { .. } | WorkerHistoryProvenance::LocalHumanInput => { - SessionEntryProvenance::HumanInput - } - WorkerHistoryProvenance::WorkerInput { .. } => SessionEntryProvenance::WorkerInput, - WorkerHistoryProvenance::FlowInstruction { .. } => SessionEntryProvenance::FlowInstruction, - WorkerHistoryProvenance::BackendInstruction { .. } - | WorkerHistoryProvenance::Provided { .. } => SessionEntryProvenance::BackendInstruction, - WorkerHistoryProvenance::ModelOutput { .. } => SessionEntryProvenance::ModelOutput, - WorkerHistoryProvenance::ToolOutput { .. } => SessionEntryProvenance::ToolOutput, - WorkerHistoryProvenance::CompactionSummary - | WorkerHistoryProvenance::DerivedSummary - | WorkerHistoryProvenance::JournalContext => SessionEntryProvenance::DerivedSummary, - WorkerHistoryProvenance::LegacyUnknown => SessionEntryProvenance::LegacyUnknown, - } -} - -fn message_reference_kind( - origin: &SessionEntryProvenance, - provider_role: &Role, - use_policy: Option, -) -> Option { - if use_policy.is_some() { - return match provider_role { - Role::System => Some(ReferenceKind::System), - Role::User => Some(ReferenceKind::User), - Role::Assistant => Some(ReferenceKind::Assistant), - }; - } - match origin { - SessionEntryProvenance::HumanInput | SessionEntryProvenance::WorkerInput => { - Some(ReferenceKind::User) - } - SessionEntryProvenance::ModelOutput => Some(ReferenceKind::Assistant), - SessionEntryProvenance::ToolOutput => Some(ReferenceKind::Tool), - SessionEntryProvenance::LegacyUnknown => match provider_role { - Role::User => Some(ReferenceKind::User), - Role::Assistant => Some(ReferenceKind::Assistant), - Role::System => None, - }, - // Flow/backend/system content remains out of the observation surface - // even when represented with a provider user/system role. - SessionEntryProvenance::FlowInstruction - | SessionEntryProvenance::BackendInstruction - | SessionEntryProvenance::DerivedSummary => None, - } -} - -fn render_item( - item: &Item, - entry: &ReferenceEntry, - detail: ReadDetail, - max_bytes: usize, -) -> String { - let text = match item { - Item::Message { role, content, .. } => { - let text = content - .iter() - .map(|p| p.as_text()) - .collect::>() - .join(""); - format!("[{} {:?}] {text}", entry.id, role) - } - Item::ToolCall { - name, arguments, .. - } => match detail { - ReadDetail::Compact => format!("[{} ToolInput {name}] (arguments omitted)", entry.id), - ReadDetail::Full => format!("[{} ToolInput {name}]\narguments: {arguments}", entry.id), - }, - Item::ToolResult { - summary, - content, - attachments, - is_error, - .. - } => { - let attachment_line = if attachments.is_empty() { - String::new() - } else { - format!("\nattachments: {} image(s)", attachments.len()) - }; - match detail { - ReadDetail::Compact => format!( - "[{} ToolOutput{}]\nsummary: {summary}\ncontent: (omitted){attachment_line}", - entry.id, - if *is_error { " error" } else { "" }, - ), - ReadDetail::Full => format!( - "[{} ToolOutput{}]\nsummary: {summary}\ncontent: {}{attachment_line}", - entry.id, - if *is_error { " error" } else { "" }, - content.as_deref().unwrap_or_default(), - ), - } - } - Item::AssistantResponseBoundary { .. } => { - format!("[{} Assistant response boundary omitted]", entry.id) - } - Item::Reasoning { .. } => format!("[{} Reasoning omitted]", entry.id), - }; - truncate_chars(&text, max_bytes) -} - -fn truncate_chars(text: &str, max_chars: usize) -> String { - if max_chars == 0 { - return "… [truncated]".to_string(); - } - if text.chars().count() <= max_chars { - return text.to_string(); - } - let mut out = text.chars().take(max_chars).collect::(); - out.push_str("… [truncated]"); - out -} +pub use session_store::capture::*; #[cfg(test)] mod tests { use super::*; + use crate::session_history::WorkerHistoryProvenance; + use agen::Item; + use protocol::{SessionEntryProvenance, SessionSnapshot}; #[test] fn flow_user_role_is_excluded_while_explicit_human_origin_remains_evidence() { @@ -897,7 +30,7 @@ mod tests { }, ), ]; - let capture = SessionCapture::from_history_entries("segment", entries); + let capture = SessionCapture::from_history_entries("session", "segment", entries); let overview = capture.overview(); assert_eq!(overview.len(), 1); assert!(matches!( @@ -921,8 +54,8 @@ mod tests { }, ); let expected_ref = SessionEntryRef::from_history_entry_id(&retained.annotation.entry_id); - let before = SessionCapture::from_history_entries("old", vec![retained.clone()]); - let after = SessionCapture::from_history_entries("new", vec![retained]); + let before = SessionCapture::from_history_entries("session", "old", vec![retained.clone()]); + let after = SessionCapture::from_history_entries("session", "new", vec![retained]); assert_eq!(before.overview()[0].id, expected_ref); assert_eq!(after.overview()[0].id, expected_ref); assert_eq!( @@ -963,7 +96,7 @@ mod tests { .iter() .map(|entry| SessionEntryRef::from_history_entry_id(&entry.annotation.entry_id)) .collect::>(); - let capture = SessionCapture::from_history_entries("segment", entries); + let capture = SessionCapture::from_history_entries("session", "segment", entries); assert!(capture.overview().is_empty()); assert!( capture @@ -1102,8 +235,16 @@ mod tests { let snapshot: SessionSnapshot = serde_json::from_value(serde_json::to_value(snapshot).unwrap()).unwrap(); for capture in [ - SessionCapture::from_history_entries(segment.to_string(), restored), - SessionCapture::from_session_snapshot(segment.to_string(), snapshot), + SessionCapture::from_history_entries( + session.to_string(), + segment.to_string(), + restored, + ), + SessionCapture::from_session_snapshot( + session.to_string(), + segment.to_string(), + snapshot, + ), ] { let hits = capture.search(&SearchOptions { kind: Some(ReferenceKind::System), @@ -1228,13 +369,15 @@ mod tests { assert_eq!(entry.item, items[index]); assert_eq!(entry.annotation, history[index].metadata); } - let capture = SessionCapture::from_history_entries(fork.to_string(), restored); - assert!( - matches!(&capture.entries[2].item, Item::ToolCall { call_id, .. } if call_id == "derived") - ); - assert!( - matches!(&capture.entries[3].item, Item::ToolResult { call_id, .. } if call_id == "derived") + let capture = SessionCapture::from_history_entries( + session.to_string(), + fork.to_string(), + restored, ); + // Private derived content must be absent even from the saved + // body representation, not merely hidden by the UI index. + let encoded = serde_json::to_string(&capture.to_dto()).unwrap(); + assert!(!encoded.contains("PRIVATE_SYNTHETIC")); let snapshot = public_snapshot::project_session_snapshot(session, &saved); assert!( !serde_json::to_string(&snapshot) @@ -1243,7 +386,11 @@ mod tests { ); for capture in [ capture, - SessionCapture::from_session_snapshot(fork.to_string(), snapshot), + SessionCapture::from_session_snapshot( + session.to_string(), + fork.to_string(), + snapshot, + ), ] { assert_eq!( capture @@ -1315,6 +462,7 @@ mod tests { #[test] fn overview_contains_user_and_assistant_only() { let view = SessionCapture::new( + "session", "segment-1", vec![ Item::system_message("sys"), @@ -1335,6 +483,7 @@ mod tests { #[test] fn search_filters_tool_input_and_output() { let view = SessionCapture::new( + "session", "segment-1", vec![ Item::tool_call("c1", "Read", "{\"file\":\"Cargo.toml\"}"), @@ -1374,6 +523,7 @@ mod tests { #[test] fn read_by_entry_range_is_bounded_and_can_skip_tools() { let view = SessionCapture::new( + "session", "segment-1", vec![ Item::user_message("one"), @@ -1406,6 +556,7 @@ mod tests { #[test] fn tool_image_projection_exposes_only_bounded_metadata() { let view = SessionCapture::new( + "session", "segment-1", vec![Item::tool_result_item_with_attachments( "c1", @@ -1435,6 +586,7 @@ mod tests { #[test] fn system_prompt_and_reasoning_are_excluded_from_every_projection() { let view = SessionCapture::new( + "session", "segment-1", vec![ Item::system_message("raw secret system prompt"), @@ -1472,7 +624,7 @@ mod tests { let items = (0..20) .map(|index| Item::user_message(format!("message-{index}"))) .collect::>(); - let view = SessionCapture::new("segment-1", items); + let view = SessionCapture::new("session", "segment-1", items); let refs = view .overview() .iter() @@ -1490,6 +642,7 @@ mod tests { #[test] fn append_preserves_existing_session_entry_refs() { let first = SessionCapture::new( + "session", "segment-1", vec![ Item::user_message("first"), @@ -1497,6 +650,7 @@ mod tests { ], ); let appended = SessionCapture::new( + "session", "segment-1", vec![ Item::user_message("first"), @@ -1540,7 +694,7 @@ mod tests { #[test] fn evidence_projection_uses_entry_range_and_session_entry_ref() { - let view = SessionCapture::new("segment-1", vec![Item::user_message("hello")]); + let view = SessionCapture::new("session", "segment-1", vec![Item::user_message("hello")]); let source = view.evidence_for("E00000000").unwrap(); assert_eq!(source.segment_id, "segment-1"); assert_eq!(source.entry_range, [0, 0]); @@ -1550,6 +704,7 @@ mod tests { #[test] fn committed_tool_call_can_be_resolved_by_provider_call_id() { let view = SessionCapture::new( + "session", "segment-1", vec![Item::tool_call( "call-remember", @@ -1562,4 +717,111 @@ mod tests { assert_eq!(evidence.tool_part, Some(ToolPart::Input)); assert!(view.evidence_for_tool_call("missing").is_none()); } + // Real Store append and Segment replacement -> Worker replay -> the common + // fixed value used by Memory and compaction exploration. No live I/O remains + // available after the owned temporary source is removed. + #[test] + fn frozen_memory_and_exploration_survive_append_compaction_and_source_removal() { + use crate::feature::builtin::memory_staging_output::render_extract_input; + use crate::feature::builtin::session_explore::SessionExploreState; + use session_store::{FsStore, LogEntry, SegmentOrigin, Store}; + let root = tempfile::tempdir().unwrap(); + let store = FsStore::new(root.path()).unwrap(); + let session = session_store::new_session_id(); + let first = session_store::new_segment_id(); + let next = session_store::new_segment_id(); + let original = crate::session_history::history_entry( + Item::user_message("fixed 日本語 intent"), + WorkerHistoryProvenance::LocalHumanInput, + ); + let seed = |history, compacted_from| LogEntry::AnnotatedSegmentStart { + ts: 1, + session_id: session, + system_prompt: Some("PRIVATE_SYSTEM".into()), + config: Default::default(), + history, + forked_from: None, + compacted_from, + }; + store + .create_segment( + session, + first, + &[seed( + vec![crate::session_history::to_logged_history_entry(&original)], + None, + )], + ) + .unwrap(); + let log = store.read_all(session, first).unwrap(); + let restored = + crate::session_history::restore_history_entries(session, first, &log).unwrap(); + let fixed = + SessionCapture::from_history_entries(session.to_string(), first.to_string(), restored); + let overview = fixed.overview()[0].clone(); + let evidence = fixed.evidence_for(overview.id.as_str()).unwrap(); + let extract = render_extract_input(&fixed); + let encoded = fixed.encode(CaptureLimits::default()).unwrap(); + let state = SessionExploreState::new(fixed.clone()); + store + .append( + session, + first, + &LogEntry::AnnotatedAssistantItem { + ts: 2, + entry: crate::session_history::to_logged_history_entry( + &crate::session_history::history_entry( + Item::assistant_message("later append"), + WorkerHistoryProvenance::LegacyUnknown, + ), + ), + }, + ) + .unwrap(); + let summary = crate::session_history::history_entry( + Item::system_message("PRIVATE_COMPACTED_SUMMARY"), + WorkerHistoryProvenance::DerivedSummary, + ); + store + .create_segment( + session, + next, + &[seed( + vec![crate::session_history::to_logged_history_entry(&summary)], + Some(SegmentOrigin { + segment_id: first, + at_turn_index: 0, + }), + )], + ) + .unwrap(); + drop(store); + drop(root); + assert_eq!(fixed.encode(CaptureLimits::default()).unwrap(), encoded); + assert_eq!(render_extract_input(state.view()), extract); + assert_eq!(state.view().overview()[0].id, overview.id); + assert_eq!(state.view().overview()[0].text, overview.text); + let retained = state.view().evidence_for(overview.id.as_str()).unwrap(); + assert_eq!(retained.entry_range, evidence.entry_range); + assert_eq!(retained.excerpt, evidence.excerpt); + assert!( + state + .view() + .search(&SearchOptions { + query: "later".into(), + ..Default::default() + }) + .is_empty() + ); + let restored = SessionCapture::decode(&encoded, CaptureLimits::default()).unwrap(); + let scoped = fixed.scoped_ref(&overview.id).unwrap(); + assert_eq!( + restored + .resolve_ref(&scoped, CapturePurpose::MemoryEvidence) + .unwrap() + .entry_id, + original.annotation.entry_id.0 + ); + assert!(!String::from_utf8(encoded).unwrap().contains("PRIVATE_")); + } } diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index f46be3d4a..859784866 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -7501,6 +7501,7 @@ impl Worker { crate::feature::builtin::session_explore::SessionExploreFeature::new( crate::feature::builtin::session_explore::SessionExploreState::new( crate::session_capture::SessionCapture::from_history_entries( + self.session_id().to_string(), self.segment_id().to_string(), entries_to_summarise.clone(), ), @@ -17374,8 +17375,9 @@ permission = "write" producer: "arbitrary-context-provider".into() } ); - let view = - crate::session_capture::SessionCapture::from_history_entries("startup", replay); + let view = crate::session_capture::SessionCapture::from_history_entries( + "session", "startup", replay, + ); let extract = crate::feature::builtin::memory_staging_output::render_extract_input(&view); assert!(!extract.contains("PRIOR_JOURNAL_REFERENCE")); From 73a3e674c9656f419632a7d8be970ff5440a917e Mon Sep 17 00:00:00 2001 From: Hare Date: Thu, 22 Oct 2026 22:58:57 +0900 Subject: [PATCH 19/46] feat(worker): separate restored candidates, repair, and activation --- .../session-store/src/worker_session_store.rs | 15 + crates/worker-runtime/src/fs_store.rs | 21 +- crates/worker-runtime/src/lib.rs | 3 + .../src/retention/runtime_v5_tests.rs | 6 +- crates/worker-runtime/src/runtime.rs | 926 ++++++---- crates/worker-runtime/src/worker_backend.rs | 1141 ++++++++---- .../src/worker_backend/restore_recovery.rs | 173 ++ .../worker_backend_restore_candidate_tests.rs | 898 ++++++++++ .../worker_backend_restore_failure_tests.rs | 340 +++- .../worker-runtime/src/working_directory.rs | 27 +- crates/worker/src/bootstrap.rs | 33 + crates/worker/src/controller.rs | 29 + crates/worker/src/discovery.rs | 11 +- crates/worker/src/lib.rs | 3 + crates/worker/src/worker.rs | 1034 ++++++++++- crates/worker/tests/controller_test.rs | 37 +- crates/workspace-server/src/lib.rs | 3 + .../tests/restored_parent_control_tests.rs | 90 +- .../subjektiv_journal_connected_tests.rs | 1580 ++++++++++++++++- .../src/server_assignment_attention_tests.rs | 60 +- .../src/server_assignment_delivery_tests.rs | 142 +- .../src/server_workspace_config_wip_tests.rs | 89 +- deno.json | 4 +- tests/support/restore_environment.rs | 129 ++ tools/test.test.ts | 369 +++- tools/test.ts | 216 ++- 26 files changed, 6592 insertions(+), 787 deletions(-) create mode 100644 crates/worker-runtime/src/worker_backend/restore_recovery.rs create mode 100644 crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs create mode 100644 tests/support/restore_environment.rs diff --git a/crates/session-store/src/worker_session_store.rs b/crates/session-store/src/worker_session_store.rs index a19f1b2db..3a5dad1e8 100644 --- a/crates/session-store/src/worker_session_store.rs +++ b/crates/session-store/src/worker_session_store.rs @@ -194,6 +194,21 @@ impl WorkerSessionStore { Ok(store) } + /// Open a current retained Session without creating or migrating anything. + /// The live constraint is checked here and again by every later publication; + /// attaching the control store itself does not acquire publication authority. + pub fn open_existing_with_retention( + root: impl Into, + retention: crate::retention::SessionRetentionStore, + ) -> Result { + let mut store = Self::open_read_only(root)?; + if let Some(session) = *store.session_id.lock().unwrap() { + retention.ensure_live(session)?; + } + store.retention = Some(retention); + Ok(store) + } + fn lock_publication( &self, session: SessionId, diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index de799b323..21ecfeca8 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -2301,9 +2301,28 @@ impl WorkerAggregateRecord { }); } if let Some(request) = request { + let mut validation_request = request.clone(); + // Pending admission may supply a missing Workspace connection + // without applying it to the saved execution. Validate that + // exact owner's intent against identity; do not widen the + // Available-record rule or mutate the canonical request. + if validation_request.workspace_api.is_none() + && let PersistedWorkerLifecycleOperation::Restore(restore) = &operation + && let Some(api) = restore_guard + .pending_preparation(restore.operation_id) + .and_then(|preparation| preparation.workspace_api.as_ref()) + { + validation_request.workspace_api = + Some( + serde_json::from_value(serde_json::to_value(api).map_err( + |error| runtime_store_corrupt(path, error.to_string()), + )?) + .map_err(|error| runtime_store_corrupt(path, error.to_string()))?, + ); + } WorkerExecutionRecord { schema_version: SCHEMA_VERSION, - request: request.clone(), + request: validation_request, binding: binding.clone(), restore_intent, } diff --git a/crates/worker-runtime/src/lib.rs b/crates/worker-runtime/src/lib.rs index 1e160a6f5..f4a5b02ff 100644 --- a/crates/worker-runtime/src/lib.rs +++ b/crates/worker-runtime/src/lib.rs @@ -16,6 +16,9 @@ pub mod management; pub mod observation; pub mod profile_archive; pub mod resource; +#[cfg(test)] +#[path = "../../../tests/support/restore_environment.rs"] +mod restore_test_environment; #[cfg(feature = "fs-store")] pub mod retention; mod runtime; diff --git a/crates/worker-runtime/src/retention/runtime_v5_tests.rs b/crates/worker-runtime/src/retention/runtime_v5_tests.rs index d9a3bccdd..47d75c940 100644 --- a/crates/worker-runtime/src/retention/runtime_v5_tests.rs +++ b/crates/worker-runtime/src/retention/runtime_v5_tests.rs @@ -682,8 +682,10 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() { #[cfg(unix)] #[test] fn current_runtime_session_migration_still_rejects_live_worker_sockets() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().join("runtime"); + // The same TempDir owns the relative bind/connect reference. No cwd change + // or alias is needed, and the real live-socket migration refusal stays tested. + let temp = tempfile::tempdir_in(".").unwrap(); + let root = Path::new(temp.path().file_name().unwrap()).join("runtime"); let (worker, _, _) = runtime_fixture(&root, 12); let before = tree_bytes(&root); let runs = root.join("workers").join(worker.to_string()).join("runs"); diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 19c17b053..eaec092dc 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -1886,70 +1886,7 @@ impl Runtime { let previous = state.worker(worker_ref)?.clone(); { let worker = state.worker_mut(worker_ref)?; - if !attachments.is_empty() && worker.workspace_id.is_none() { - return Err(RuntimeError::InvalidRequest( - "logical Workdir attachments require Workspace authority".to_string(), - )); - } - if worker.request.is_none() { - return Err(RuntimeError::WorkerExecutionUnavailable { - worker_id: worker.worker_id, - message: "persisted Worker restore request is unavailable".to_string(), - }); - } - let logical_by_alias = attachments - .iter() - .map(|attachment| { - ( - attachment.alias.clone(), - attachment.working_directory_id.as_str(), - ) - }) - .collect::>(); - for local in &worker.workdir_attachments { - let local_id = local - .working_directory - .summary - .working_directory_id - .as_str(); - if let Some(logical_id) = logical_by_alias.get(&local.alias) - && *logical_id != local_id - { - return Err(RuntimeError::InvalidRequest(format!( - "logical Workdir attachment alias `{}` conflicts with Runtime-local Workdir {}", - local.alias, local_id - ))); - } - } - let retained_local = worker - .workdir_attachments - .iter() - .filter_map(|local| { - let local_id = local - .working_directory - .summary - .working_directory_id - .as_str(); - (logical_by_alias.get(&local.alias).copied() == Some(local_id)) - .then(|| (local.alias.clone(), local_id.to_string())) - }) - .collect::>(); - worker - .workdir_attachments - .retain(|local| retained_local.contains_key(&local.alias)); - let request = worker - .request - .as_mut() - .expect("restore request existence checked above"); - request - .workdir_attachment_requests - .retain(|local| retained_local.contains_key(&local.alias)); - request.workdir_attachments.retain(|local| { - retained_local - .get(&local.alias) - .is_some_and(|workdir_id| workdir_id == &local.working_directory_id) - }); - worker.logical_workdir_attachments = attachments; + worker.apply_logical_workdir_attachments(attachments)?; } if let Err(error) = state.persist_worker(&worker_ref.worker_id) { state.workers.insert(worker_ref.worker_id, previous); @@ -2417,54 +2354,6 @@ impl Runtime { return Ok(result); } } - if !owner.preparation_applied { - let uncertain = |message: String| { - RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::ReconciliationRequired, - "worker_restore_preparation_reconciliation_required", - message, - ) - }; - // Apply reference updates under the original pending_restore ID; no provider is opened. - let mut effects_started = false; - if let Err(error) = self.apply_restore_preparation_under_lock( - worker_ref, - owner.request.preparation, - owner.operation_id, - &mut effects_started, - ) { - if effects_started - || matches!(error, RuntimeError::StoreCommitOutcomeUnknown { .. }) - { - return Ok(uncertain(sanitize_worker_create_failure_message( - &error.to_string(), - ))); - } - let result = RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::Rejected, - "worker_restore_preparation_rejected", - sanitize_worker_create_failure_message(&error.to_string()), - ); - self.finish_restore_receipt(worker_ref, &result)?; - return Ok(result); - } - let mut state = self.lock()?; - let mut candidate = state.worker(worker_ref)?.clone(); - candidate - .restore_guard - .owners - .get_mut(&owner.request.request_id) - .expect("admitted owner") - .preparation_applied = true; - if let Err(error) = state.persist_worker_record(&candidate) { - // Keep the started (not applied) record in memory. Even a - // known failed completion write follows possible effects. - return Ok(uncertain(sanitize_worker_create_failure_message( - &error.to_string(), - ))); - } - state.workers.insert(worker_ref.worker_id, candidate); - } } let result = self.restore_worker_under_lock(worker_ref, mode)?; self.finish_restore_receipt(worker_ref, &result)?; @@ -2481,95 +2370,18 @@ impl Runtime { } let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); - if candidate.restore_guard.active_request_id.is_some() { + if candidate.restore_guard.active_request_id.is_some() + || candidate.pending_restore.is_some() + { candidate.restore_guard.finish(result.clone()); candidate.pending_restore = None; + candidate.restore_candidate_context = None; state.persist_worker_record(&candidate)?; state.workers.insert(worker_ref.worker_id, candidate); - } - Ok(()) - } - - fn apply_restore_preparation_under_lock( - &self, - worker_ref: &WorkerRef, - preparation: Option, - operation_id: WorkerLifecycleOperationId, - effects_started: &mut bool, - ) -> Result<(), RuntimeError> { - let Some(preparation) = preparation else { - return Ok(()); - }; - // Validate all read-only structure before any binding effect. - if let Some(attachments) = &preparation.workdir_attachments { - let attachments: Vec = serde_json::from_value( - serde_json::to_value(attachments) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, - ) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; - validate_logical_workdir_attachments(&attachments)?; - } - // DTO conversion and every Runtime preparation effect occur AFTER admission. - if let Some(api) = preparation.workspace_api { - let api: WorkspaceApiRef = serde_json::from_value( - serde_json::to_value(api) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, - ) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; - if self.lock()?.worker(worker_ref)?.workspace_id.as_deref() - != Some(api.workspace_id.as_str()) - { - return Err(RuntimeError::InvalidRequest( - "Prepared Workspace API does not match Worker Workspace identity".into(), - )); - } - let mut state = self.lock()?; - let mut candidate = state.worker(worker_ref)?.clone(); - if candidate.pending_stop.is_some() - || candidate - .pending_restore - .is_none_or(|pending| pending.operation_id != operation_id) - { - return Err(RuntimeError::RestoreObservationConflict { - worker_id: worker_ref.worker_id, - }); - } - let persisted = candidate.request.as_mut().ok_or_else(|| { - RuntimeError::InvalidRequest("Restore specification is missing".into()) - })?; - if persisted.workspace_api.as_ref().is_some_and(|existing| { - existing.workspace_id != api.workspace_id - || existing.base_url.trim_end_matches('/') != api.base_url.trim_end_matches('/') - }) { - return Err(RuntimeError::InvalidRequest( - "Workspace API preparation cannot change Worker identity or base URL".into(), - )); - } - persisted.workspace_api = Some(api); - *effects_started = true; - match state.persist_worker_record(&candidate) { - Ok(()) => { - state.workers.insert(worker_ref.worker_id, candidate); - } - Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { - state.workers.insert(worker_ref.worker_id, candidate); - return Err(error); - } - Err(error) => return Err(error), + if result.state == WorkerRestoreState::Accepted { + state.publish_worker_upsert(worker_ref.worker_id)?; } } - if let Some(attachments) = preparation.workdir_attachments { - let attachments: Vec = serde_json::from_value( - serde_json::to_value(attachments) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, - ) - .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; - validate_logical_workdir_attachments(&attachments)?; - *effects_started = true; - self.replace_worker_workdir_attachments_under_lock(worker_ref, attachments, true)?; - } - // Retained legacy SSH preparation is not a prerequisite or an effect - // of restoring a Worker. Sessions resolve live access when used. Ok(()) } @@ -2621,6 +2433,28 @@ impl Runtime { { return Ok(RuntimeWorkerRestoreResult::accepted(worker.detail())); } + let preparation = worker + .restore_guard + .active_request_id + .as_ref() + .and_then(|id| worker.restore_guard.owners.get(id)) + .filter(|owner| !owner.preparation_applied) + .and_then(|owner| owner.request.preparation.as_ref()); + let prepared_worker = match worker.prepared_restore_record(preparation) { + Ok(prepared) => prepared, + Err(error) => { + return Ok(RuntimeWorkerRestoreResult::failed( + if reconcile { + WorkerRestoreState::ReconciliationRequired + } else { + WorkerRestoreState::Rejected + }, + "worker_restore_preparation_rejected", + sanitize_worker_create_failure_message(&error.to_string()), + )); + } + }; + let worker = &prepared_worker; let Some(worker_request) = worker.request.clone() else { return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::Rejected, @@ -2645,6 +2479,16 @@ impl Runtime { .restore_candidate_context .clone() .unwrap_or_else(|| self.restore_candidate_execution_context(worker_ref.clone())); + // First-head repair needs the original launch Prompt material. Use + // only the exact cached snapshot, never current Profile resolution; + // ordinary saved-head restore still works without that cache. + let config_bundle = worker_request.config_bundle.as_ref().and_then(|reference| { + state + .config_bundles + .get(&reference.id) + .filter(|bundle| bundle.metadata.digest == reference.digest) + .cloned() + }); let request = WorkerExecutionRestoreRequest { operation_id, worker_ref: worker_ref.clone(), @@ -2655,7 +2499,7 @@ impl Runtime { logical_workdir_attachments: worker.logical_workdir_attachments.clone(), workdir_attachments: BTreeMap::new(), deferred_workdir_sessions: BTreeMap::new(), - config_bundle: None, + config_bundle, }; ( backend, @@ -2746,11 +2590,15 @@ impl Runtime { if let Err(error) = backend.activate_restored_worker(operation_id, worker_ref) { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %error, - "Worker restore committed but Backend observation activation failed" - ); + self.retain_restore_reconciliation_pending( + worker_ref, + candidate_context.clone(), + )?; + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_activation_pending", + sanitize_worker_create_failure_message(&error), + )); } if let Err(error) = self.activate_restore_candidate_context(worker_ref, &candidate_context) @@ -2766,13 +2614,21 @@ impl Runtime { sanitize_worker_create_failure_message(&error.to_string()), )); } - Ok(RuntimeWorkerRestoreResult::accepted(worker)) + let result = RuntimeWorkerRestoreResult::accepted(worker); + if let Err(error) = self.finish_restore_receipt(worker_ref, &result) { + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_receipt_commit_pending", + sanitize_worker_create_failure_message(&error.to_string()), + )); + } + Ok(result) } - Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + Err(error) => { tracing::error!( worker_id = %worker_ref.worker_id, error = %error, - "Worker restore commit outcome is unknown; preserving execution evidence" + "Worker restore activated but commit did not confirm success; preserving operation and execution" ); if let Err(retain_error) = self.retain_restore_execution_evidence( worker_ref, @@ -2783,70 +2639,15 @@ impl Runtime { tracing::error!( worker_id = %worker_ref.worker_id, error = %retain_error, - "failed to persist restore execution evidence after an uncertain commit" + "failed to persist restore execution evidence; original operation remains pending" ); } Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, - "worker_restore_commit_outcome_unknown", - "Worker restore commit outcome is uncertain; reread or retry the Worker restore", + "worker_restore_commit_reconciliation_required", + sanitize_worker_create_failure_message(&error.to_string()), )) } - Err(error) => match self.cleanup_failed_restore(&backend, worker_ref) { - Ok(()) => { - self.discard_restore_candidate_context(worker_ref, &candidate_context)?; - tracing::warn!( - worker_id = %worker_ref.worker_id, - error = %error, - "Worker restore commit failed and was rolled back" - ); - let receipt = RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::RolledBack, - "worker_restore_commit_rolled_back", - "Worker restore commit failed and cleanup completed", - ); - if let Err(settle_error) = - self.settle_restore_rollback(worker_ref, &receipt) - { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %settle_error, - "Worker restore cleanup completed but its durable operation could not be settled" - ); - return Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::ReconciliationRequired, - "worker_restore_rollback_commit_failed", - "Worker restore cleanup completed but durable reconciliation is still required", - )); - } - Ok(receipt) - } - Err(cleanup_error) => { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %error, - cleanup_error = %cleanup_error, - "Worker restore requires reconciliation after commit and cleanup failures" - ); - if let Err(retain_error) = self.retain_restore_execution_evidence( - worker_ref, - worker_state, - workdir_attachments, - candidate_context.clone(), - ) { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %retain_error, - "failed to persist restore execution evidence; in-process reconciliation remains required" - ); - } - Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::ReconciliationRequired, - "worker_restore_reconciliation_required", - "Worker restore result is uncertain; reread or retry the Worker restore", - )) - } - }, } } WorkerExecutionSpawnResult::Rejected(result) => { @@ -2962,13 +2763,11 @@ impl Runtime { fn activate_restore_candidate_context( &self, - worker_ref: &WorkerRef, + _worker_ref: &WorkerRef, context: &crate::execution::WorkerExecutionContext, ) -> Result<(), RuntimeError> { + // Retain the publication owner until terminal receipt persistence succeeds. context.activate_candidate()?; - self.lock()? - .worker_mut(worker_ref)? - .restore_candidate_context = None; Ok(()) } @@ -4581,13 +4380,43 @@ impl Runtime { ) -> Result { let mut state = self.lock()?; state.ensure_worker_ref(worker_ref)?; - let mut candidate = state.worker(worker_ref)?.clone(); + let previous = state.worker(worker_ref)?; + if previous.pending_stop.is_some() { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + let mut candidate = if let Some(id) = &previous.restore_guard.active_request_id { + let owner = previous.restore_guard.owners.get(id).ok_or_else(|| { + RuntimeError::InvalidRequest("Active Restore request has no persisted owner".into()) + })?; + if previous + .pending_restore + .is_none_or(|pending| pending.operation_id != owner.operation_id) + { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + let mut candidate = + previous.prepared_restore_record(owner.request.preparation.as_ref())?; + candidate + .restore_guard + .owners + .get_mut(id) + .expect("admitted owner") + .preparation_applied = true; + candidate + } else { + previous.clone() + }; candidate.restore_guard.rotate(); candidate.execution_metadata_available = true; candidate.execution_bound = true; - candidate.pending_restore = None; + // Keep pending/owner until buffered observations also activate. Otherwise + // a retry could replay Accepted after a failed publication activation. + // Activation follows this pending decision; terminal success is still absent. candidate.pending_stop = None; - // Keep the actual publication owner until buffered observations activate. candidate.status = status; let mut worker_state = worker_state; if worker_state.last_finished_submission_request_id.is_none() { @@ -4600,6 +4429,9 @@ impl Runtime { candidate.last_finished_submission_request_id.clone(); } let _ = candidate.apply_worker_state(&worker_state); + if let Some(pending) = candidate.pending_restore.as_mut() { + pending.last_settled_status = candidate.status; + } candidate.restore_intent = restore_intent_for_status(candidate.status); candidate.workdir_attachments = workdir_attachments; @@ -4607,17 +4439,14 @@ impl Runtime { // subsequent publish uses the same pure projection while the Worker // operation lock keeps this candidate stable. candidate.restore_guard.rotate(); - let accepted = RuntimeWorkerRestoreResult::accepted(candidate.detail()); - candidate.restore_guard.finish(accepted); let _subscription_projection = state.subscription_worker(&candidate)?; - // Persist the complete candidate before making it observable. A failed - // write therefore leaves both the in-memory catalog and subscriptions - // on the pre-restore record, so cleanup can produce a true rollback. + // Persist the execution projection before publishing observations. Terminal + // receipt settlement follows successful observation activation separately. state.persist_worker_record(&candidate)?; let detail = candidate.detail(); state.workers.insert(worker_ref.worker_id, candidate); - state.publish_worker_upsert(worker_ref.worker_id)?; + if let Err(error) = state.persist_runtime_snapshot() { tracing::warn!( worker_id = %worker_ref.worker_id, @@ -4723,27 +4552,6 @@ impl Runtime { Ok(()) } - fn cleanup_failed_restore( - &self, - backend: &WorkerExecutionBackendRef, - worker_ref: &WorkerRef, - ) -> Result<(), RuntimeError> { - let result = backend.stop_worker(worker_ref); - if !result.is_accepted() { - return Err(RuntimeError::WorkerExecutionRejected { - worker_id: worker_ref.worker_id, - operation: result.operation, - outcome: result.outcome, - message: format!( - "restored execution cleanup failed after catalog commit failure: {}", - result.message_or_default() - ), - result, - }); - } - Ok(()) - } - /// Bind the Backend registry identity once. Retention evidence fails closed /// until the Runtime host supplies this trusted configuration. pub fn bind_runtime_identity(&self, runtime_id: &str) -> Result<(), RuntimeError> { @@ -6494,6 +6302,21 @@ impl Default for RestoreGuard { } impl RestoreGuard { + pub(crate) fn pending_preparation( + &self, + operation_id: WorkerLifecycleOperationId, + ) -> Option<&runtime_api::WorkerRestorePreparation> { + let id = self.active_request_id.as_ref()?; + let owner = self.owners.get(id)?; + if owner.operation_id != operation_id + || owner.receipt.is_some() + || owner.request.request_id != *id + { + return None; + } + owner.request.preparation.as_ref() + } + fn rotate(&mut self) { self.observation_token = uuid::Uuid::now_v7().to_string(); } @@ -6535,6 +6358,123 @@ struct WorkerRecord { } impl WorkerRecord { + fn apply_logical_workdir_attachments( + &mut self, + mut attachments: Vec, + ) -> Result<(), RuntimeError> { + validate_logical_workdir_attachments(&attachments)?; + attachments.sort_by(|left, right| left.alias.cmp(&right.alias)); + if !attachments.is_empty() && self.workspace_id.is_none() { + return Err(RuntimeError::InvalidRequest( + "logical Workdir attachments require Workspace authority".to_string(), + )); + } + if self.request.is_none() { + return Err(RuntimeError::WorkerExecutionUnavailable { + worker_id: self.worker_id, + message: "persisted Worker restore request is unavailable".to_string(), + }); + } + let logical_by_alias = attachments + .iter() + .map(|attachment| { + ( + attachment.alias.clone(), + attachment.working_directory_id.as_str(), + ) + }) + .collect::>(); + for local in &self.workdir_attachments { + let local_id = local + .working_directory + .summary + .working_directory_id + .as_str(); + if let Some(logical_id) = logical_by_alias.get(&local.alias) + && *logical_id != local_id + { + return Err(RuntimeError::InvalidRequest(format!( + "logical Workdir attachment alias `{}` conflicts with Runtime-local Workdir {}", + local.alias, local_id + ))); + } + } + let retained_local = self + .workdir_attachments + .iter() + .filter_map(|local| { + let local_id = local + .working_directory + .summary + .working_directory_id + .as_str(); + (logical_by_alias.get(&local.alias).copied() == Some(local_id)) + .then(|| (local.alias.clone(), local_id.to_string())) + }) + .collect::>(); + self.workdir_attachments + .retain(|local| retained_local.contains_key(&local.alias)); + let request = self + .request + .as_mut() + .expect("restore request existence checked above"); + request + .workdir_attachment_requests + .retain(|local| retained_local.contains_key(&local.alias)); + request.workdir_attachments.retain(|local| { + retained_local + .get(&local.alias) + .is_some_and(|workdir_id| workdir_id == &local.working_directory_id) + }); + self.logical_workdir_attachments = attachments; + Ok(()) + } + + // Overlay preparation on an unpublished clone, never on Runtime authority. + // The same validation is repeated at the effect boundary under the Worker lock. + fn prepared_restore_record( + &self, + preparation: Option<&runtime_api::WorkerRestorePreparation>, + ) -> Result { + let mut candidate = self.clone(); + let Some(preparation) = preparation else { + return Ok(candidate); + }; + if let Some(api) = &preparation.workspace_api { + let api: WorkspaceApiRef = serde_json::from_value( + serde_json::to_value(api) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + if candidate.workspace_id.as_deref() != Some(api.workspace_id.as_str()) { + return Err(RuntimeError::InvalidRequest( + "Prepared Workspace API does not match Worker Workspace identity".into(), + )); + } + let persisted = candidate.request.as_mut().ok_or_else(|| { + RuntimeError::InvalidRequest("Restore specification is missing".into()) + })?; + if persisted.workspace_api.as_ref().is_some_and(|existing| { + existing.workspace_id != api.workspace_id + || existing.base_url.trim_end_matches('/') != api.base_url.trim_end_matches('/') + }) { + return Err(RuntimeError::InvalidRequest( + "Workspace API preparation cannot change Worker identity or base URL".into(), + )); + } + persisted.workspace_api = Some(api); + } + if let Some(attachments) = &preparation.workdir_attachments { + let attachments: Vec = serde_json::from_value( + serde_json::to_value(attachments) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + candidate.apply_logical_workdir_attachments(attachments)?; + } + Ok(candidate) + } + fn apply_worker_state(&mut self, incoming: &protocol::WorkerStateSnapshot) { if self.worker_state.as_ref().map(|state| &state.state) != Some(&incoming.state) { self.restore_guard.rotate(); @@ -8201,6 +8141,10 @@ mod tests { restore_gate: Mutex>>, restore_count: Mutex, restore_operation_ids: Mutex>, + activation_result: Mutex>>, + activation_gate: Mutex>>, + activation_operation_ids: Mutex>, + restore_requests: Mutex>, #[cfg(feature = "ws-server")] restore_event: Mutex>, config_bundles: Mutex>>, @@ -8448,6 +8392,7 @@ mod tests { &self, request: WorkerExecutionRestoreRequest, ) -> WorkerExecutionSpawnResult { + self.restore_requests.lock().unwrap().push(request.clone()); self.job_bindings .lock() .unwrap() @@ -8493,6 +8438,54 @@ mod tests { } } + fn reconcile_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + let known = self + .restore_requests + .lock() + .unwrap() + .iter() + .any(|previous| { + previous.operation_id == request.operation_id + && previous.worker_ref == request.worker_ref + }); + if known + && self + .contexts + .lock() + .unwrap() + .contains_key(&request.worker_ref.worker_id) + { + return WorkerExecutionSpawnResult::Connected { + worker_state: protocol::WorkerStatus::Idle.into(), + workdir_attachments: request.previous_workdir_attachments, + }; + } + self.restore_worker(request) + } + + fn activate_restored_worker( + &self, + operation_id: WorkerLifecycleOperationId, + _worker_ref: &WorkerRef, + ) -> Result<(), String> { + self.activation_operation_ids + .lock() + .unwrap() + .push(operation_id); + let gate = self.activation_gate.lock().unwrap().clone(); + if let Some(gate) = gate { + gate.enter_and_wait(); + } + self.activation_result + .lock() + .unwrap() + .clone() + .unwrap_or(Ok(())) + } + fn worker_completions( &self, _worker_ref: &WorkerRef, @@ -9755,7 +9748,7 @@ mod tests { } #[test] - fn synced_config_bundle_is_stored_checked_and_used_for_worker_creation() { + fn synced_config_bundle_is_stored_checked_and_used_for_worker_creation_and_restore() { let backend = Arc::new(TestExecutionBackend::default()); let runtime = Runtime::with_execution_backend(RuntimeOptions::default(), backend.clone()).unwrap(); @@ -9807,7 +9800,7 @@ mod tests { .unwrap(); assert_eq!( backend.config_bundles.lock().unwrap().as_slice(), - &[Some(bundle), None] + &[Some(bundle.clone()), Some(bundle)] ); } @@ -11725,6 +11718,343 @@ mod tests { } } + #[test] + fn restore_activation_failure_never_records_or_replays_accepted() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("activation failure")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let stops = *backend.stop_count.lock().unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + *backend.activation_result.lock().unwrap() = Some(Err("partial activation".into())); + let mut operation_id = None; + for _ in 0..2 { + let result = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(result.state, WorkerRestoreState::ReconciliationRequired); + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + let pending = record.pending_restore.unwrap(); + assert_eq!( + *operation_id.get_or_insert(pending.operation_id), + pending.operation_id + ); + assert!(record.restore_candidate_context.is_some()); + let owner = &record.restore_guard.owners[&request.request_id]; + assert_eq!(owner.operation_id, pending.operation_id); + assert!(owner.receipt.is_none()); + assert_eq!( + record.restore_guard.active_request_id.as_ref(), + Some(&request.request_id) + ); + assert_eq!(*backend.stop_count.lock().unwrap(), stops); + } + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + *backend.activation_result.lock().unwrap() = None; + let accepted = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(accepted.state, WorkerRestoreState::Accepted); + let activations = backend.activation_operation_ids.lock().unwrap().len(); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + accepted + ); + assert_eq!( + backend.activation_operation_ids.lock().unwrap().len(), + activations + ); + assert!( + backend + .activation_operation_ids + .lock() + .unwrap() + .iter() + .all(|id| Some(*id) == operation_id) + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + } + + #[cfg(feature = "fs-store")] + #[test] + fn activated_restore_commit_failure_preserves_pending_owner_without_stopping_execution() { + for fault in [ + crate::fs_store::AtomicWriteFault::BeforeRename, + crate::fs_store::AtomicWriteFault::AfterRename, + ] { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Arc::new( + Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("activation-commit"), + backend.clone(), + ) + .unwrap(), + ); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("activated commit failure")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let stops = *backend.stop_count.lock().unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let gate = Arc::new(RestoreGate::default()); + *backend.activation_gate.lock().unwrap() = Some(gate.clone()); + let restoring_runtime = runtime.clone(); + let restoring_ref = worker.worker_ref.clone(); + let restoring_request = request.clone(); + let restoring = std::thread::spawn(move || { + restoring_runtime.restore_worker_operation(&restoring_ref, restoring_request) + }); + assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + runtime_store(&runtime).fail_next_worker_write(fault); + gate.release(); + assert_eq!( + restoring.join().unwrap().unwrap().state, + WorkerRestoreState::ReconciliationRequired + ); + { + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert_eq!(record.pending_restore.unwrap().operation_id, operation_id); + assert!(record.restore_candidate_context.is_some()); + assert!( + record.restore_guard.owners[&request.request_id] + .receipt + .is_none() + ); + } + let persisted: serde_json::Value = serde_json::from_slice( + &std::fs::read( + dir.path() + .join("runtime/workers") + .join(worker.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap(); + let saved_receipt = + &persisted["restore_guard"]["owners"][&request.request_id]["receipt"]; + if matches!(fault, crate::fs_store::AtomicWriteFault::BeforeRename) { + assert!(saved_receipt.is_null()); + } else { + // Activation was confirmed before this uncertain terminal write. + assert_eq!(saved_receipt["state"], "accepted"); + } + assert_eq!(*backend.stop_count.lock().unwrap(), stops); + *backend.activation_gate.lock().unwrap() = None; + let accepted = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(accepted.state, WorkerRestoreState::Accepted); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + accepted + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!(*backend.stop_count.lock().unwrap(), stops); + } + } + + #[cfg(all(feature = "fs-store", feature = "ws-server"))] + #[test] + fn buffered_restore_state_write_failure_keeps_valid_pending_owner_and_replays_once() { + for fault in [ + crate::fs_store::AtomicWriteFault::BeforeRename, + crate::fs_store::AtomicWriteFault::AfterRename, + ] { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Arc::new( + Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("buffered-state"), + backend.clone(), + ) + .unwrap(), + ); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("buffered live state")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let gate = Arc::new(RestoreGate::default()); + *backend.activation_gate.lock().unwrap() = Some(gate.clone()); + *backend.restore_event.lock().unwrap() = Some(protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Running.into(), + }); + let restoring_runtime = runtime.clone(); + let restoring_ref = worker.worker_ref.clone(); + let restoring_request = request.clone(); + let restoring = std::thread::spawn(move || { + restoring_runtime.restore_worker_operation(&restoring_ref, restoring_request) + }); + assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + // The pending execution projection is committed, but its live + // Running observation is buffered until backend activation returns. + runtime_store(&runtime).fail_next_worker_write(fault); + gate.release(); + assert_eq!( + restoring.join().unwrap().unwrap().state, + WorkerRestoreState::ReconciliationRequired + ); + let persisted = runtime_store(&runtime).load_runtime_state().unwrap(); + let record = &persisted.workers[&worker.worker_id]; + assert_eq!(record.status, WorkerStatus::Idle); + let PersistedWorkerExecutionState::ReconciliationRequired( + PersistedWorkerLifecycleOperation::Restore(pending), + ) = &record.execution_state + else { + panic!("pending owner disappeared") + }; + assert_eq!(pending.last_settled_status, record.status); + assert_eq!(pending.operation_id, operation_id); + assert!( + record.restore_guard.owners[&request.request_id] + .receipt + .is_none() + ); + // Live protocol state and settled catalog state are deliberately + // different during activation. Persistence must remain readable. + assert_eq!( + runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .worker_state + .unwrap() + .catalog_status(), + protocol::WorkerStatus::Running + ); + *backend.activation_gate.lock().unwrap() = None; + let accepted = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(accepted.state, WorkerRestoreState::Accepted); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + accepted + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert!( + backend + .activation_operation_ids + .lock() + .unwrap() + .iter() + .all(|id| *id == operation_id) + ); + assert!(matches!( + runtime_store(&runtime) + .load_runtime_state() + .unwrap() + .workers[&worker.worker_id] + .execution_state, + PersistedWorkerExecutionState::Available(_) + )); + } + } + + #[test] + fn restore_candidate_receives_prepared_bindings_without_mutating_runtime_authority() { + let (runtime, backend) = runtime_and_backend(); + let runtime = Arc::new(runtime); + let workspace_scope = scope("workspace-a", "server-a"); + let mut create = scoped_task_request("read-only candidate bindings", "workspace-a"); + create.workspace_api = None; + let worker = runtime + .create_worker_scoped(&workspace_scope, create) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "workspace-a", "base_url": "https://prepared.example" }, + "workdir_attachments": [], "repository_access": [] + })).unwrap()); + let gate = Arc::new(RestoreGate::default()); + *backend.restore_gate.lock().unwrap() = Some(gate.clone()); + let restoring_runtime = runtime.clone(); + let restoring_ref = worker.worker_ref.clone(); + let restoring_scope = workspace_scope.clone(); + let restoring = std::thread::spawn(move || { + restoring_runtime.restore_worker_operation_scoped( + &restoring_scope, + &restoring_ref, + request, + ) + }); + assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); + { + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert!(record.request.as_ref().unwrap().workspace_api.is_none()); + assert!( + !record + .restore_guard + .owners + .values() + .next() + .unwrap() + .preparation_applied + ); + } + assert_eq!( + backend.restore_requests.lock().unwrap()[0] + .request + .workspace_api + .as_ref() + .unwrap() + .base_url, + "https://prepared.example" + ); + assert!(backend.activation_operation_ids.lock().unwrap().is_empty()); + gate.release(); + assert_eq!( + restoring.join().unwrap().unwrap().state, + WorkerRestoreState::Accepted + ); + let state = runtime.lock().unwrap(); + assert_eq!( + state + .worker(&worker.worker_ref) + .unwrap() + .request + .as_ref() + .unwrap() + .workspace_api + .as_ref() + .unwrap() + .base_url, + "https://prepared.example" + ); + } + #[cfg(feature = "ws-server")] #[test] fn committed_restore_retries_retained_observation_activation() { diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index af4933bc3..b5ace780f 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -16,6 +16,7 @@ //! Manifests, credentials, and handles are not passed to browser/API projections. mod deferred_workdir; +mod restore_recovery; use std::collections::{BTreeMap, HashMap}; use std::future::Future; @@ -176,12 +177,25 @@ pub struct RuntimeWorkerController { pub workspace_client: Arc, } +/// An unpublished restore resource owner. No protocol endpoint is available +/// until activation succeeds; discard settles the actual candidate resources +/// directly, without sending Shutdown to an ordinary Controller. +#[async_trait] +pub trait RuntimeRestoreCandidate: Send { + fn initial_state(&self) -> protocol::WorkerStateSnapshot; + + async fn activate(&mut self) -> Result; + + async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure>; +} + /// Failure facts supplied by the resource owner, never inferred from error text. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RuntimeRestoreStage { Preflight, Metadata, Reconstruction, + Recovery, Binding, Feature, Controller, @@ -228,6 +242,7 @@ pub struct RuntimeRestoreFailure { cleanup: RuntimeRestoreCleanup, io_kind: Option, cleanup_failure: Option, + durable_effects_possible: bool, } impl RuntimeRestoreFailure { @@ -239,6 +254,7 @@ impl RuntimeRestoreFailure { cleanup: RuntimeRestoreCleanup::NotNeeded, io_kind: None, cleanup_failure: None, + durable_effects_possible: false, } } @@ -250,6 +266,7 @@ impl RuntimeRestoreFailure { cleanup: RuntimeRestoreCleanup::Required, io_kind: None, cleanup_failure: None, + durable_effects_possible: false, } } @@ -261,6 +278,7 @@ impl RuntimeRestoreFailure { cleanup: RuntimeRestoreCleanup::Required, io_kind: None, cleanup_failure: None, + durable_effects_possible: false, } } @@ -285,7 +303,8 @@ impl std::fmt::Display for RuntimeRestoreFailure { fn restore_worker_cause(error: &WorkerError) -> &'static str { match error { - WorkerError::ConstructionCleanup { source, .. } => restore_worker_cause(source), + WorkerError::ConstructionCleanup { source, .. } + | WorkerError::RestoreActivationEffects { source } => restore_worker_cause(source), WorkerError::SegmentEmpty { .. } => "saved segment empty", WorkerError::SegmentSystemPromptMissing { .. } => "saved segment system prompt missing", WorkerError::WorkerMetadataPending { .. } => "saved Worker metadata pending", @@ -325,6 +344,52 @@ async fn finish_restore_failure( failure } +async fn prepare_restore_operation( + factory: &F, + request: WorkerExecutionRestoreRequest, +) -> Result, RuntimeRestoreFailure> { + factory.preflight_restore(&request).await.map_err(|_| { + RuntimeRestoreFailure::refused(RuntimeRestoreStage::Preflight, "restore preflight refused") + })?; + let repaired = factory.recover_restore_source(&request).await?; + let candidate = factory + .prepare_restore(request) + .await + .map_err(|mut failure| { + failure.durable_effects_possible |= repaired; + failure + })?; + if repaired { + Ok(Box::new(RepairedRestoreCandidate { candidate })) + } else { + Ok(candidate) + } +} + +// A completed repair is a canonical effect even when later activation fails +// before its own mutation boundary. Carry that evidence with the resource owner. +struct RepairedRestoreCandidate { + candidate: Box, +} + +#[async_trait] +impl RuntimeRestoreCandidate for RepairedRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.candidate.initial_state() + } + + async fn activate(&mut self) -> Result { + self.candidate.activate().await.map_err(|mut failure| { + failure.durable_effects_possible = true; + failure + }) + } + + async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure> { + self.candidate.discard().await + } +} + /// Factory seam used by [`WorkerRuntimeExecutionBackend`] to construct a real /// controller-backed Worker for a Runtime catalog entry. #[async_trait] @@ -382,10 +447,28 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { Ok(()) } + /// Separate idempotent recovery, bound to the accepted operation and its + /// canonical Session. A true result records that preparation's no-mutation + /// guarantee cannot be used to describe this operation as a whole. + async fn recover_restore_source( + &self, + _request: &WorkerExecutionRestoreRequest, + ) -> Result { + Ok(false) + } + + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure>; + async fn restore_controller( &self, request: WorkerExecutionRestoreRequest, - ) -> Result; + ) -> Result { + let mut candidate = prepare_restore_operation(self, request).await?; + candidate.activate().await + } /// Prove cleanup of a restore future that completed without returning a /// controller. An error is not absence evidence; unsupported factories keep @@ -667,6 +750,7 @@ pub struct ProfileRuntimeWorkerFactory { struct FailedRestoreResources { operation_id: crate::execution::WorkerLifecycleOperationId, sessions: Vec>, + candidate_workdir_cleanup: Option, allocation: Option, feature_tasks: Option, } @@ -690,6 +774,10 @@ impl ProfileRuntimeWorkerFactory { request: &WorkerExecutionRestoreRequest, error: &WorkerError, ) { + if let WorkerError::RestoreActivationEffects { source } = error { + self.retain_construction_failure(request, source); + return; + } let mut resources = self .failed_restore_resources .lock() @@ -2032,10 +2120,22 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { Ok(()) } - async fn restore_controller( + async fn recover_restore_source( + &self, + request: &WorkerExecutionRestoreRequest, + ) -> Result { + self.recover_saved_restore_source(request) + .await + .map_err(|mut failure| { + failure.durable_effects_possible = true; + failure + }) + } + + async fn prepare_restore( &self, request: WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result, RuntimeRestoreFailure> { self.preflight_restore(&request).await.map_err(|_| { RuntimeRestoreFailure::refused( RuntimeRestoreStage::Preflight, @@ -2047,10 +2147,9 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .executions .fetch_add(1, Ordering::SeqCst); let mut stage = RuntimeRestoreStage::Metadata; - let mut io_kind = None; - // Reconstruction can write before failing; only the resource owner can - // confirm cleanup, even when no Worker value was returned. - let result: Result = async { + // Reconstruction owns read-only resources; cleanup confirmation must + // come from that owner even when it cannot return a candidate. + let result: Result, &'static str> = async { #[cfg(test)] self.restore_checkpoint(stage) .map_err(|_| "injected restore boundary failure")?; @@ -2070,14 +2169,6 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { &request.request, self.runtime_id.as_deref(), ); - let observation_runtime_id = self.runtime_id.clone(); - let observation_workspace_id = request - .request - .workspace_api - .as_ref() - .map(|api| api.workspace_id.clone()); - let observation_grants = request.request.worker_observation_grants.clone(); - let observation_enabled = request.request.worker_observation_enabled; let workspace_request_client = request .request .workspace_api @@ -2096,14 +2187,14 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .worker_aggregate_dir(&request.worker_ref) .map_err(|_| "Worker aggregate root unavailable")?; let session_dir = worker_aggregate_dir.join("session"); - let session_store = WorkerSessionStore::new_with_retention( + let session_store = WorkerSessionStore::open_existing_with_retention( &session_dir, self.session_retention() .map_err(|_| "Session retention unavailable")?, ) .map_err(|_| "canonical Session store initialization failed")?; let worker_metadata_store = - WorkerAggregateStore::new(&worker_aggregate_dir, worker_name.clone()) + WorkerAggregateStore::open_read_only(&worker_aggregate_dir, worker_name.clone()) .map_err(|_| "canonical metadata store initialization failed")?; let metadata = worker_metadata_store .read_by_name(&worker_name) @@ -2122,6 +2213,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { FailedRestoreResources { operation_id: request.operation_id, sessions: Vec::new(), + candidate_workdir_cleanup: None, allocation: None, feature_tasks: None, }, @@ -2129,70 +2221,29 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { #[cfg(test)] self.restore_checkpoint(stage) .map_err(|_| "injected restore boundary failure")?; - let mut worker = match Worker::restore_from_worker_metadata_with_context( - &worker_name, - manifest.clone(), - store.clone(), - loader.clone(), - workspace_context.clone(), - filesystem_authority.clone(), - ) - .await - { - Ok(worker) => worker, - Err(WorkerError::WorkerMetadataPending { .. }) - if request.request.initial_input.is_none() => - { - let pending_loader = if workspace_context.workspace_id().is_some() { - let bundle = request - .config_bundle - .as_ref() - .ok_or("pending restore launch material missing")?; - let resolution = self - .observe_bundle_prompt_projection( - bundle, - observation_workspace_id.as_deref(), - ) - .map_err(|_| "pending restore Prompt projection invalid")? - .ok_or("pending restore Prompt projection missing")?; - loader - .clone() - .with_effective_catalog(resolution.projection.catalog.clone()) - } else { - loader.clone() - }; - Worker::restore_pending_from_worker_metadata_with_context( - &worker_name, - manifest.clone(), - store, - pending_loader, - workspace_context, - filesystem_authority, - ) - .await - .map_err(|error| { - self.retain_construction_failure(&request, &error); - restore_worker_cause(&error) - })? - } - Err(error) => { + let candidate = + Worker::prepare_restored_from_worker_metadata_with_context_and_model_client( + &worker_name, + manifest, + store, + loader, + workspace_context, + filesystem_authority, + self.spawn_model_client_factory + .as_ref() + .map(|factory| factory()), + ) + .await + .map_err(|error| { self.retain_construction_failure(&request, &error); - return Err(restore_worker_cause(&error)); - } - }; - let feature_tasks = worker.retain_startup_feature_cleanup(); + restore_worker_cause(&error) + })?; self.failed_restore_resources .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert( - request.worker_ref.clone(), - FailedRestoreResources { - operation_id: request.operation_id, - sessions: vec![worker.workdir_sessions()], - allocation: worker.restore_allocation_cleanup_handle(), - feature_tasks: Some(feature_tasks), - }, - ); + .get_mut(&request.worker_ref) + .expect("retained restore owner") + .candidate_workdir_cleanup = Some(candidate.workdir_cleanup()); #[cfg(test)] { let gate = self.restore_faults.factory_gate.lock().unwrap().take(); @@ -2209,126 +2260,16 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { #[cfg(test)] self.restore_checkpoint(stage) .map_err(|_| "injected restore boundary failure")?; - if let Some(binding) = request.request.backend_job.as_ref() { - worker - .bind_backend_job(binding.clone()) - .map_err(|_| "restored Backend Job binding failed")?; - } - validate_worker_memory_settings(worker.manifest(), &request.request) + validate_worker_memory_settings(candidate.manifest(), &request.request) .map_err(|_| "saved memory settings mismatch")?; - worker - .finalize_subjektiv_session_attribution( - request.request.subjektiv_attached, - SubjektivSessionAttributionLifecycle::RestoredSession, - ) - .map_err(|error| restore_worker_cause(&error))?; - let flow_transition_enabled = worker.manifest().feature.flow.enabled; - let workdir_sessions = restored_workdir_router( - &request.workdir_attachments, - &request.logical_workdir_attachments, - &request.deferred_workdir_sessions, - &request.request.workdir_attachments, - worker.workspace_client_handle(), - ) - .map_err(|_| "restored Workdir binding failed")?; - self.failed_restore_resources - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .get_mut(&request.worker_ref) - .expect("retained restore owner") - .sessions - .push(workdir_sessions.clone()); - worker - .workdir_sessions() - .close_all() - .await - .map_err(|_| "initial Workdir close failed")?; - worker.bind_workdir_sessions(workdir_sessions); - if let (Some(runtime_id), Some(workspace_id)) = - (observation_runtime_id, observation_workspace_id.clone()) - && observation_enabled - { - let mut providers: Vec> = vec![Arc::new( - WorkspaceClientWorkerObservationProvider::new(worker.workspace_client_handle()), - )]; - if !observation_grants.is_empty() { - providers.push(Arc::new(RuntimeGrantedWorkerObservationProvider { - runtime_id, - workspace_id, - grants: observation_grants.into_iter().take(100).collect(), - hub: self.observation_hub.clone(), - })); - } - worker.bind_worker_observation_provider(Some(Arc::new( - CompositeWorkerObservationProvider::new(providers), - ))); - } - stage = RuntimeRestoreStage::Feature; - #[cfg(test)] - self.restore_checkpoint(stage) - .map_err(|_| "injected restore boundary failure")?; - if flow_transition_enabled { - let report = worker - .install_runtime_flow_transition_feature() - .map_err(|_| "Flow transition installation failed")?; - if report.reports.iter().any(|report| !report.installed) { - return Err("Flow transition installation refused"); - } - } - - stage = RuntimeRestoreStage::Controller; - #[cfg(test)] - self.restore_checkpoint(stage) - .map_err(|_| "injected restore boundary failure")?; - let workspace_client = worker.workspace_client_handle(); - let run_dir = self - .worker_restore_run_dir(&request.worker_ref, request.operation_id) - .map_err(|_| "restore run directory unavailable")?; - let transport = self.controller_transport; - #[cfg(test)] - let transport = if self.restore_faults.socket_failure.load(Ordering::SeqCst) { - WorkerControllerTransport::UnixSocket - } else { - transport - }; - let started = PreparedWorker::new( - worker, - WorkerBootstrapLayout::RuntimeManagedRun { - run_dir: run_dir.clone(), - }, - transport, - ) - .start() - .await - .map_err(|error| match error { - WorkerBootstrapError::Worker(error) => restore_worker_cause(&error), - WorkerBootstrapError::Controller { source, .. } => { - io_kind = Some(source.kind()); - "restored Worker controller startup failed" - } - })?; - self.failed_restore_resources - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(&request.worker_ref); - let (handle, shutdown_rx, controller_task) = - (started.handle, started.shutdown, started.controller_task); - if flow_transition_enabled { - handle.shared_state.enable_flow_transition(); - } - Ok(RuntimeWorkerController { - handle, - shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), - controller_task, - workspace_client, - }) + Ok(Box::new(ProfileRestoreCandidate { + prepared: Some(candidate), + factory: self.clone(), + request: request.clone(), + }) as Box) } .await; - result.map_err(|cause| { - let mut failure = RuntimeRestoreFailure::started(stage, cause); - failure.io_kind = io_kind; - failure - }) + result.map_err(|cause| RuntimeRestoreFailure::started(stage, cause)) } async fn cleanup_failed_restore( @@ -2364,6 +2305,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .await .map_err(|_| RuntimeRestoreCleanupFailure::Feature)?; } + if let Some(cleanup) = resources.candidate_workdir_cleanup.as_ref() { + cleanup + .close() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; + } for sessions in &resources.sessions { sessions .close_all() @@ -2434,6 +2381,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .await .map_err(|_| "failed restore Feature cleanup".to_string())?; } + if let Some(cleanup) = resources.candidate_workdir_cleanup.as_ref() { + cleanup + .close() + .await + .map_err(|_| "failed candidate Workdir cleanup".to_string())?; + } for sessions in &resources.sessions { sessions .close_all() @@ -2472,6 +2425,216 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { } } +type ProfileRestoreStore = CombinedStore; + +struct ProfileRestoreCandidate { + prepared: Option>, + factory: ProfileRuntimeWorkerFactory, + request: WorkerExecutionRestoreRequest, +} + +#[async_trait] +impl RuntimeRestoreCandidate for ProfileRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + protocol::WorkerStateSnapshot::initial() + } + + async fn activate(&mut self) -> Result { + let prepared = self.prepared.take().ok_or_else(|| { + RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "restore candidate already consumed", + ) + })?; + let worker = prepared.activate().await.map_err(|error| { + self.factory + .retain_construction_failure(&self.request, &error); + let mut failure = RuntimeRestoreFailure::started( + RuntimeRestoreStage::Reconstruction, + restore_worker_cause(&error), + ); + failure.durable_effects_possible = + matches!(error, WorkerError::RestoreActivationEffects { .. }); + failure + })?; + self.factory + .activate_prepared_restore(worker, &self.request) + .await + .map_err(|mut failure| { + failure.durable_effects_possible = true; + failure + }) + } + + async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure> { + if let Some(prepared) = self.prepared.as_mut() { + prepared + .discard() + .await + .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; + } + self.prepared = None; + self.factory.cleanup_failed_restore(&self.request).await + } +} + +impl ProfileRuntimeWorkerFactory { + async fn activate_prepared_restore( + &self, + mut worker: Worker< + Box, + CombinedStore, + >, + request: &WorkerExecutionRestoreRequest, + ) -> Result { + let mut stage = RuntimeRestoreStage::Binding; + let mut io_kind = None; + let result: Result = async { + let feature_tasks = worker.retain_startup_feature_cleanup(); + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: vec![worker.workdir_sessions()], + candidate_workdir_cleanup: None, + allocation: worker.restore_allocation_cleanup_handle(), + feature_tasks: Some(feature_tasks), + }, + ); + let observation_runtime_id = self.runtime_id.clone(); + let observation_workspace_id = request + .request + .workspace_api + .as_ref() + .map(|api| api.workspace_id.clone()); + let observation_grants = request.request.worker_observation_grants.clone(); + let observation_enabled = request.request.worker_observation_enabled; + if let Some(binding) = request.request.backend_job.as_ref() { + worker + .bind_backend_job(binding.clone()) + .map_err(|_| "restored Backend Job binding failed")?; + } + validate_worker_memory_settings(worker.manifest(), &request.request) + .map_err(|_| "saved memory settings mismatch")?; + let workdir_sessions = restored_workdir_router( + &request.workdir_attachments, + &request.logical_workdir_attachments, + &request.deferred_workdir_sessions, + &request.request.workdir_attachments, + worker.workspace_client_handle(), + ) + .map_err(|_| "restored Workdir binding failed")?; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get_mut(&request.worker_ref) + .expect("retained restore owner") + .sessions + .push(workdir_sessions.clone()); + worker + .workdir_sessions() + .close_all() + .await + .map_err(|_| "initial Workdir close failed")?; + worker.bind_workdir_sessions(workdir_sessions); + if let (Some(runtime_id), Some(workspace_id)) = + (observation_runtime_id, observation_workspace_id.clone()) + && observation_enabled + { + let mut providers: Vec> = vec![Arc::new( + WorkspaceClientWorkerObservationProvider::new(worker.workspace_client_handle()), + )]; + if !observation_grants.is_empty() { + providers.push(Arc::new(RuntimeGrantedWorkerObservationProvider { + runtime_id, + workspace_id, + grants: observation_grants.into_iter().take(100).collect(), + hub: self.observation_hub.clone(), + })); + } + worker.bind_worker_observation_provider(Some(Arc::new( + CompositeWorkerObservationProvider::new(providers), + ))); + } + worker + .finalize_subjektiv_session_attribution( + request.request.subjektiv_attached, + SubjektivSessionAttributionLifecycle::RestoredSession, + ) + .map_err(|error| restore_worker_cause(&error))?; + let flow_transition_enabled = worker.manifest().feature.flow.enabled; + stage = RuntimeRestoreStage::Feature; + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + if flow_transition_enabled { + let report = worker + .install_runtime_flow_transition_feature() + .map_err(|_| "Flow transition installation failed")?; + if report.reports.iter().any(|report| !report.installed) { + return Err("Flow transition installation refused"); + } + } + + stage = RuntimeRestoreStage::Controller; + #[cfg(test)] + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; + let workspace_client = worker.workspace_client_handle(); + let run_dir = self + .worker_restore_run_dir(&request.worker_ref, request.operation_id) + .map_err(|_| "restore run directory unavailable")?; + let transport = self.controller_transport; + #[cfg(test)] + let transport = if self.restore_faults.socket_failure.load(Ordering::SeqCst) { + WorkerControllerTransport::UnixSocket + } else { + transport + }; + let started = PreparedWorker::new( + worker, + WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: run_dir.clone(), + }, + transport, + ) + .start() + .await + .map_err(|error| match error { + WorkerBootstrapError::Worker(error) => restore_worker_cause(&error), + WorkerBootstrapError::Controller { source, .. } => { + io_kind = Some(source.kind()); + "restored Worker controller startup failed" + } + })?; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&request.worker_ref); + let (handle, shutdown_rx, controller_task) = + (started.handle, started.shutdown, started.controller_task); + if flow_transition_enabled { + handle.shared_state.enable_flow_transition(); + } + Ok(RuntimeWorkerController { + handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), + controller_task, + workspace_client, + }) + } + .await; + result.map_err(|cause| { + let mut failure = RuntimeRestoreFailure::started(stage, cause); + failure.io_kind = io_kind; + failure + }) + } +} + #[derive(Clone)] struct RuntimeExecutionTaskScope { tasks: Arc>>>, @@ -2788,7 +2951,11 @@ struct RuntimeWorkerExecution { struct PendingRuntimeRestore { request: WorkerExecutionRestoreRequest, task: tokio::sync::Mutex>>, - result: Mutex>>, + result: Mutex, RuntimeRestoreFailure>>>, + activated: Mutex>>, + activation_started: AtomicBool, + discard_started: AtomicBool, + discard_result: Mutex>>, failure: Mutex>, } @@ -3081,6 +3248,7 @@ where fn retry_pending_restore_cleanup( &self, pending: &Arc, + resume_activation: bool, ) -> Result<(), String> { let failure = pending .result @@ -3088,7 +3256,12 @@ where .map_err(|_| "restore result lock is poisoned")? .as_ref() .and_then(|result| result.as_ref().err()) - .filter(|failure| failure.needs_cleanup()) + .filter(|failure| { + failure.needs_cleanup() + || (resume_activation + && (failure.durable_effects_possible + || pending.activation_started.load(Ordering::Acquire))) + }) .cloned(); if let Some(failure) = failure { let owner = Arc::clone(pending); @@ -3101,8 +3274,43 @@ where return Err("restore task has not been joined".into()); } *task = Some(self.spawn_on_adapter_runtime(async move { + if resume_activation + && !failure.needs_cleanup() + && (failure.durable_effects_possible + || owner.activation_started.load(Ordering::Acquire)) + { + let result = + match prepare_restore_operation(factory.as_ref(), owner.request.clone()) + .await + { + Ok(candidate) => Ok(candidate), + Err(failure) => Err(finish_restore_failure( + factory.as_ref(), + &owner.request, + failure, + ) + .await), + }; + owner.activation_started.store(false, Ordering::Release); + *owner + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = None; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(result); + return; + } let failure = finish_restore_failure(factory.as_ref(), &owner.request, failure).await; + if owner.activation_started.load(Ordering::Acquire) { + *owner + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure.clone())); + } *owner .result .lock() @@ -3794,64 +4002,148 @@ where return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } if let Err(message) = self - .retry_pending_restore_cleanup(&pending) + .retry_pending_restore_cleanup(&pending, false) .and_then(|()| self.wait_pending_restore(&pending, Duration::from_secs(5))) { return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } - if self.workers.lock().is_err() { + if let Some(Err(failure)) = pending + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + && failure.needs_cleanup() + { return WorkerExecutionResult::errored( WorkerExecutionOperation::Stop, - "worker adapter registry lock is poisoned", + failure.to_string(), ); } - let mut retained = match pending.result.lock() { - Ok(retained) => retained, - Err(_) => { - return WorkerExecutionResult::errored( - WorkerExecutionOperation::Stop, - "restore result lock is poisoned", - ); - } - }; - match retained.as_ref() { - Some(Ok(_)) => { - let controller = match retained.take().unwrap() { - Ok(controller) => controller, - Err(_) => unreachable!(), - }; - let state = Arc::new(RwLock::new(controller.handle.shared_state.snapshot())); - self.retain_uncertain_unconnected_controller( - worker_ref, - controller.handle, - controller.shutdown, - Arc::new(AtomicBool::new(false)), - RuntimeExecutionTaskScope::new(controller.controller_task), - state, - Some(controller.workspace_client), - Some(pending.request.operation_id), - Vec::new(), - ); - } - Some(Err(failure)) => { - if !matches!( - failure.cleanup, - RuntimeRestoreCleanup::NotNeeded | RuntimeRestoreCleanup::Confirmed - ) { + if pending.activation_started.load(Ordering::Acquire) { + let mut activated = match pending.activated.lock() { + Ok(result) => result, + Err(_) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "restore activation result lock is poisoned", + ); + } + }; + match activated.as_ref() { + Some(Ok(_)) => { + let controller = + activated.take().unwrap().unwrap_or_else(|_| unreachable!()); + let state = + Arc::new(RwLock::new(controller.handle.shared_state.snapshot())); + self.retain_uncertain_unconnected_controller( + worker_ref, + controller.handle, + controller.shutdown, + Arc::new(AtomicBool::new(false)), + RuntimeExecutionTaskScope::new(controller.controller_task), + state, + Some(controller.workspace_client), + Some(pending.request.operation_id), + Vec::new(), + ); + } + Some(Err(failure)) + if matches!( + failure.cleanup, + RuntimeRestoreCleanup::Confirmed | RuntimeRestoreCleanup::NotNeeded + ) => {} + Some(Err(failure)) => { return WorkerExecutionResult::errored( WorkerExecutionOperation::Stop, failure.to_string(), ); } + None => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "restore activation ownership is unproven", + ); + } } - None => { - return WorkerExecutionResult::errored( - WorkerExecutionOperation::Stop, - "restore factory has no completion evidence; cleanup is unproven", - ); + } else { + if !pending.discard_started.swap(true, Ordering::AcqRel) { + let owner = pending.clone(); + let task = match self.spawn_on_adapter_runtime(async move { + let result = owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let cleanup = match result { + Some(Ok(mut candidate)) => { + let result = candidate.discard().await; + if result.is_err() { + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Ok(candidate)); + } + result + } + Some(Err(failure)) => { + let result = if matches!( + failure.cleanup, + RuntimeRestoreCleanup::Confirmed + | RuntimeRestoreCleanup::NotNeeded + ) { + Ok(()) + } else { + Err(RuntimeRestoreCleanupFailure::Ownership) + }; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure)); + result + } + None => Err(RuntimeRestoreCleanupFailure::Ownership), + }; + *owner + .discard_result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(cleanup); + }) { + Ok(task) => task, + Err(message) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + message, + ); + } + }; + *pending.task.try_lock().expect("joined candidate task") = Some(task); + } + if let Err(message) = self.wait_pending_restore(&pending, Duration::from_secs(5)) { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + let cleanup = *pending + .discard_result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match cleanup { + Some(Ok(())) => {} + Some(Err(error)) => { + pending.discard_started.store(false, Ordering::Release); + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + error.to_string(), + ); + } + None => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "candidate discard has no completion evidence", + ); + } } } - drop(retained); if let Err(message) = self.remove_pending_restore(worker_ref) { return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } @@ -4729,9 +5021,10 @@ where } drop(workers); - // An already accepted operation owns its bindings. Re-observing mutable - // Workdir materialization must not turn an unknown operation into a new - // read-only refusal or allocate another candidate. + // The operation retains its resources across timeouts, but its saved + // bindings never override a later authoritative attachment/owner change. + // Compare before substituting the owned request or retrying any effects; + // drift retains the same owner for reconciliation/Stop, not a new intent. let admitted_pending = match self.pending_restore(&request.worker_ref) { Ok(Some(pending)) if pending.request.operation_id == request.operation_id => { Some(pending) @@ -4759,6 +5052,22 @@ where } }; if let Some(pending) = admitted_pending.as_ref() { + if request.request != pending.request.request + || request.workspace_scope != pending.request.workspace_scope + || request.logical_workdir_attachments + != pending.request.logical_workdir_attachments + || request.previous_workdir_attachments + != pending.request.previous_workdir_attachments + { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "current Worker authority differs from the owned restore candidate", + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } request = pending.request.clone(); } else { if let Err(result) = self.preflight_restore_inputs(&request) { @@ -4806,6 +5115,10 @@ where request: request.clone(), task: tokio::sync::Mutex::new(None), result: Mutex::new(None), + activated: Mutex::new(None), + activation_started: AtomicBool::new(false), + discard_started: AtomicBool::new(false), + discard_result: Mutex::new(None), failure: Mutex::new(None), }); let mut resources = match self.pending_restores.lock() { @@ -4823,7 +5136,9 @@ where let factory = Arc::clone(&self.factory); let result_owner = Arc::clone(&pending); let task = self.spawn_on_adapter_runtime(async move { - let result = match factory.restore_controller(request.clone()).await { + let result = match prepare_restore_operation(factory.as_ref(), request.clone()) + .await + { Ok(controller) => Ok(controller), Err(failure) => { *result_owner @@ -4877,7 +5192,7 @@ where } if existing_pending { if let Err(message) = self - .retry_pending_restore_cleanup(&pending) + .retry_pending_restore_cleanup(&pending, true) .and_then(|()| self.wait_pending_restore(&pending, self.spawn_restore_timeout)) { return WorkerExecutionSpawnResult::ReconciliationRequired { @@ -4890,8 +5205,30 @@ where }; } } - let controller = { - let mut retained = match pending.result.lock() { + if pending.activation_started.load(Ordering::Acquire) { + let activated = pending + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(Ok(controller)) = activated.as_ref() { + return WorkerExecutionSpawnResult::Connected { + worker_state: controller.handle.shared_state.snapshot(), + workdir_attachments: pending.request.previous_workdir_attachments.clone(), + }; + } + let message = match activated.as_ref() { + Some(Err(failure)) => failure.to_string(), + _ => "restore activation is owned by the same operation; reconcile activation" + .to_string(), + }; + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message), + worker_state: None, + workdir_attachments: pending.request.previous_workdir_attachments.clone(), + }; + } + let worker_state = { + let retained = match pending.result.lock() { Ok(retained) => retained, Err(_) => { return WorkerExecutionSpawnResult::ReconciliationRequired { @@ -4905,16 +5242,23 @@ where } }; match retained.as_ref() { - Some(Ok(_)) => match retained.take().unwrap() { - Ok(controller) => controller, - Err(_) => unreachable!(), - }, + Some(Ok(candidate)) => candidate.initial_state(), Some(Err(failure)) => { let result = WorkerExecutionResult::errored( WorkerExecutionOperation::Restore, failure.to_string(), ); let cleanup = failure.cleanup; + if failure.durable_effects_possible { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result, + worker_state: None, + workdir_attachments: pending + .request + .previous_workdir_attachments + .clone(), + }; + } if matches!( cleanup, RuntimeRestoreCleanup::NotNeeded | RuntimeRestoreCleanup::Confirmed @@ -4959,25 +5303,10 @@ where } }; - let result = self.connect_handle( - WorkerExecutionOperation::Restore, - worker_ref.clone(), - pending.request.context.clone(), - controller.handle, - controller.shutdown, - controller.controller_task, - Some(operation_id), - pending.request.previous_workdir_attachments.clone(), - Some(controller.workspace_client), - ); - if let Err(message) = self.remove_pending_restore(&worker_ref) { - return WorkerExecutionSpawnResult::ReconciliationRequired { - result: WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message), - worker_state: None, - workdir_attachments: Vec::new(), - }; + WorkerExecutionSpawnResult::Connected { + worker_state, + workdir_attachments: pending.request.previous_workdir_attachments.clone(), } - result } fn activate_restored_worker( @@ -4996,15 +5325,109 @@ where return Err(message); } }; + if let Some(pending) = self.pending_restore(worker_ref)? { + if pending.request.operation_id != operation_id { + return Err("restored Worker operation identity does not match".into()); + } + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + if !pending.activation_started.swap(true, Ordering::AcqRel) { + let owner = pending.clone(); + let factory = self.factory.clone(); + let task = self.spawn_on_adapter_runtime(async move { + let candidate = owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let result = match candidate { + Some(Ok(mut candidate)) => { + let result = candidate.activate().await; + match result { + Ok(controller) => Ok(controller), + Err(mut failure) => { + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure.clone())); + match candidate.discard().await { + Ok(()) => { + failure.cleanup = RuntimeRestoreCleanup::Confirmed + } + Err(error) => { + failure.cleanup_failure.get_or_insert(error); + failure.cleanup = RuntimeRestoreCleanup::Failed(error); + } + } + let failure = finish_restore_failure( + factory.as_ref(), + &owner.request, + failure, + ) + .await; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure.clone())); + Err(failure) + } + } + } + Some(Err(failure)) => Err(failure), + None => Err(RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "restore candidate ownership unavailable", + )), + }; + *owner + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(result); + })?; + *pending + .task + .try_lock() + .map_err(|_| "restore activation task is already owned")? = Some(task); + } + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + let controller = { + let mut result = pending + .activated + .lock() + .map_err(|_| "restore activation result lock is poisoned")?; + match result.as_ref() { + Some(Ok(_)) => result.take().unwrap().map_err(|error| error.to_string())?, + Some(Err(failure)) => return Err(failure.to_string()), + None => return Err("restore activation has no completion evidence".into()), + } + }; + match self.connect_handle( + WorkerExecutionOperation::Restore, + worker_ref.clone(), + pending.request.context.clone(), + controller.handle, + controller.shutdown, + controller.controller_task, + Some(operation_id), + pending.request.previous_workdir_attachments.clone(), + Some(controller.workspace_client), + ) { + WorkerExecutionSpawnResult::Connected { .. } => { + self.remove_pending_restore(worker_ref)? + } + other => return Err(format!("restore activation connection failed: {other:?}")), + } + } let workers = self .workers .lock() - .map_err(|_| "worker adapter registry lock is poisoned".to_string())?; + .map_err(|_| "worker adapter registry lock is poisoned")?; let execution = workers .get(worker_ref) - .ok_or_else(|| "restored Worker execution is not registered".to_string())?; + .ok_or("restored Worker execution is not registered")?; if execution.restore_operation_id != Some(operation_id) { - return Err("restored Worker operation identity does not match".to_string()); + return Err("restored Worker operation identity does not match".into()); } self.factory.activate_restored_controller( worker_ref, @@ -5702,6 +6125,7 @@ mod tests { root: &Path, materialize_head: bool, ) -> WorkerExecutionRestoreRequest { + require_restore_test_environment(); let worker_ref = WorkerRef::new(WorkerId::now_v7()); let worker_name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref); let aggregate_dir = root.join("workers").join(worker_ref.worker_id.to_string()); @@ -5901,10 +6325,17 @@ mod tests { }) .unwrap(); backend.preflight_restore(&request).unwrap(); - let expected = match backend.restore_worker(request.clone()) { + let _prepared = match backend.restore_worker(request.clone()) { WorkerExecutionSpawnResult::Connected { worker_state, .. } => worker_state, other => panic!("real Profile backend restore failed: {other:?}"), }; + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); + let expected = backend.workers.lock().unwrap()[&request.worker_ref] + .handle + .shared_state + .snapshot(); assert!(matches!( backend.worker_session(WorkerSessionObservationRequest { worker_ref: request.worker_ref.clone(), @@ -6035,6 +6466,9 @@ mod tests { backend.restore_worker(request.clone()), WorkerExecutionSpawnResult::Connected { .. } )); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); let mut transport_a = Arc::clone(&backend) .attach_worker_protocol(&request.worker_ref) .unwrap(); @@ -6072,6 +6506,9 @@ mod tests { backend.restore_worker(request.clone()), WorkerExecutionSpawnResult::Connected { .. } )); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); let current_b = backend .workers .lock() @@ -6166,6 +6603,9 @@ mod tests { backend.restore_worker(request.clone()), WorkerExecutionSpawnResult::Connected { .. } )); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); let execution = backend.workers.lock().unwrap()[&request.worker_ref].clone(); let mut transport = Arc::clone(&backend) .attach_worker_protocol(&request.worker_ref) @@ -6227,6 +6667,9 @@ mod tests { backend.restore_worker(request.clone()), WorkerExecutionSpawnResult::Connected { .. } )); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); (backend, request) } @@ -6748,6 +7191,10 @@ mod tests { request: direct_restore_request(worker_ref.clone()), task: tokio::sync::Mutex::new(None), result: Mutex::new(None), + activated: Mutex::new(None), + activation_started: AtomicBool::new(false), + discard_started: AtomicBool::new(false), + discard_result: Mutex::new(None), failure: Mutex::new(Some("task completion unknown".into())), }), ); @@ -6787,6 +7234,10 @@ mod tests { request: request.clone(), task: tokio::sync::Mutex::new(Some(failed)), result: Mutex::new(None), + activated: Mutex::new(None), + activation_started: AtomicBool::new(false), + discard_started: AtomicBool::new(false), + discard_result: Mutex::new(None), failure: Mutex::new(None), }); backend @@ -7072,6 +7523,9 @@ mod tests { matches!(restored, WorkerExecutionSpawnResult::Connected { .. }), "{restored:?}" ); + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .unwrap(); let session = backend.workers.lock().unwrap()[&request.worker_ref] .handle .shared_state @@ -8139,6 +8593,23 @@ mod tests { ); } + struct TestRestoreCandidate(Option); + #[async_trait] + impl RuntimeRestoreCandidate for TestRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.0.as_ref().unwrap().handle.shared_state.snapshot() + } + async fn activate(&mut self) -> Result { + Ok(self.0.take().unwrap()) + } + async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure> { + if let Some(controller) = self.0.take() { + shutdown_profile_controller(controller).await; + } + Ok(()) + } + } + struct PendingCreateFactory { started: mpsc::Sender<()>, dropped: mpsc::Sender<()>, @@ -8154,6 +8625,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for PendingCreateFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, _request: WorkerExecutionSpawnRequest, @@ -8324,6 +8804,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for MockFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, request: WorkerExecutionSpawnRequest, @@ -8738,6 +9227,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for FailingFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, _request: WorkerExecutionSpawnRequest, @@ -8766,6 +9264,15 @@ mod tests { #[cfg(feature = "fs-store")] #[async_trait] impl RuntimeWorkerFactory for RestartArtifactFailureFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, request: WorkerExecutionSpawnRequest, @@ -8807,7 +9314,9 @@ mod tests { #[cfg(feature = "fs-store")] #[test] fn fs_restart_stop_cleans_prior_restore_artifacts_and_retains_journal_on_failure() { - let root = tempfile::tempdir().unwrap(); + require_restore_test_environment(); + let root = tempfile::tempdir_in(".").unwrap(); + let socket_base = Path::new(root.path().file_name().unwrap()).join("controllers"); let runtime_store = root.path().join("runtime"); let options = crate::fs_store::FsRuntimeStoreOptions { root: runtime_store.clone(), @@ -8819,7 +9328,7 @@ mod tests { WorkerRuntimeExecutionBackend::new(RestartArtifactFailureFactory { inner: MockFactory { client: MockClient::sequential(vec![]), - runtime_base: root.path().join("controllers"), + runtime_base: socket_base, cwd: root.path().to_path_buf(), store_dir: root.path().join("sessions"), worker_metadata_dir: root.path().join("metadata"), @@ -8965,6 +9474,7 @@ mod tests { FailedRestoreResources { operation_id: request.operation_id, sessions: vec![Arc::clone(&sessions)], + candidate_workdir_cleanup: None, allocation: None, feature_tasks: None, }, @@ -9063,6 +9573,7 @@ mod tests { #[cfg(feature = "fs-store")] include!("worker_backend_restore_failure_tests.rs"); + include!("worker_backend_restore_candidate_tests.rs"); struct PendingFailureFactory { restore_calls: Arc, @@ -9072,6 +9583,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for PendingFailureFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, _request: WorkerExecutionSpawnRequest, @@ -9199,6 +9719,15 @@ mod tests { #[async_trait] impl RuntimeWorkerFactory for LateControllerFactory { + async fn prepare_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + self.restore_controller(request).await.map(|controller| { + Box::new(TestRestoreCandidate(Some(controller))) as Box + }) + } + async fn spawn_controller( &self, request: WorkerExecutionSpawnRequest, @@ -9812,67 +10341,6 @@ mod tests { assert!(!socket_path.exists()); } - #[test] - fn profile_runtime_factory_uses_shared_worker_bootstrap_seams() { - let source = include_str!("worker_backend.rs"); - let production = source - .split_once("#[cfg(test)]\nmod tests") - .map(|(production, _)| production) - .expect("worker backend test module marker"); - let factory = production - .split_once("impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory") - .map(|(_, factory)| factory) - .expect("profile runtime factory implementation"); - let (fresh, restore) = factory - .split_once("async fn restore_controller") - .expect("fresh and restore factory paths"); - let assert_in_order = |path: &str, markers: &[&str]| { - let mut offset = 0; - for marker in markers { - let relative = path[offset..] - .find(marker) - .unwrap_or_else(|| panic!("missing ordered factory marker {marker}")); - offset += relative + marker.len(); - } - }; - assert_in_order( - fresh, - &[ - "WorkerBootstrap::new(", - ".prepare()", - "finalize_subjektiv_session_attribution(", - "worker.bind_workdir_sessions(", - "worker.bind_worker_observation_provider(", - "install_runtime_flow_transition_feature()", - "prepared.start()", - ], - ); - assert_in_order( - restore, - &[ - "Worker::restore_from_worker_metadata_with_context(", - "finalize_subjektiv_session_attribution(", - "worker.bind_workdir_sessions(", - "worker.bind_worker_observation_provider(", - "install_runtime_flow_transition_feature()", - "PreparedWorker::new(", - ".start()", - ], - ); - assert!( - production.contains("WorkerBootstrap::new("), - "fresh runtime Workers must use the shared construction bootstrap" - ); - assert!( - production.contains("PreparedWorker::new("), - "restored runtime Workers must use the shared pre-exposure lifecycle" - ); - assert!( - !production.contains("WorkerController::spawn_runtime_managed_run_with_transport"), - "runtime factory paths must not bypass the shared controller lifecycle" - ); - } - #[test] #[serial_test::serial(worker_allocation)] fn shared_bootstrap_preserves_in_process_transport_for_fresh_and_restored_runtime_workers() { @@ -10018,6 +10486,9 @@ mod tests { factory_calls_after_restore, "reconciliation must not construct a second controller" ); + backend + .activate_restored_worker(operation_id, &worker.worker_ref) + .unwrap(); let workers = backend.workers.lock().unwrap(); let execution = workers.get(&worker.worker_ref).unwrap(); assert_eq!(execution.restore_operation_id, Some(operation_id)); diff --git a/crates/worker-runtime/src/worker_backend/restore_recovery.rs b/crates/worker-runtime/src/worker_backend/restore_recovery.rs new file mode 100644 index 000000000..8dd47ee15 --- /dev/null +++ b/crates/worker-runtime/src/worker_backend/restore_recovery.rs @@ -0,0 +1,173 @@ +//! Explicit, operation-owned repair of saved Session material. This is not +//! read-only candidate preparation: the canonical Session/attribution is the +//! receipt for replay after a lost response, and effects are reported as such. + +use super::*; + +impl ProfileRuntimeWorkerFactory { + pub(super) async fn recover_saved_restore_source( + &self, + request: &WorkerExecutionRestoreRequest, + ) -> Result { + let cause = |cause| RuntimeRestoreFailure::started(RuntimeRestoreStage::Recovery, cause); + let name = Self::runtime_worker_name_for_ref(&request.worker_ref); + let root = self + .worker_aggregate_dir(&request.worker_ref) + .map_err(|_| cause("restore recovery aggregate unavailable"))?; + let metadata_store = WorkerAggregateStore::open_read_only(&root, &name) + .map_err(|_| cause("restore recovery metadata open failed"))?; + let metadata = metadata_store + .read_by_name(&name) + .map_err(|_| cause("restore recovery metadata read failed"))? + .ok_or_else(|| cause("restore recovery metadata missing"))?; + let active = metadata + .active + .as_ref() + .ok_or_else(|| cause("restore recovery Session missing"))?; + let pending = active.segment_id.is_none(); + let incomplete = request.request.subjektiv_attached + && !matches!( + metadata.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Confirmed { .. }) + ); + if !pending && !incomplete { + return Ok(false); + } + // The accepted Runtime operation already owns this Session. Its stable + // Session identity and sole committed seed, not a new restore request, + // are used to reconcile an interrupted first-head publication. + let session_store = WorkerSessionStore::new_with_retention( + root.join("session"), + self.session_retention() + .map_err(|_| cause("restore recovery retention unavailable"))?, + ) + .map_err(|_| cause("restore recovery Session open failed"))?; + let (manifest, mut loader) = Self::manifest_for_restore(&metadata, &request.request) + .map_err(|_| cause("restore recovery saved manifest invalid"))?; + let workspace = RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + self.runtime_id.as_deref(), + ); + let request_client = request + .request + .workspace_api + .as_ref() + .and_then(|api| self.workspace_request_clients.get(&api.workspace_id)); + let context = workspace.worker_context( + &request.worker_ref, + request.workspace_scope.as_ref(), + self.worker_mutation_identity.as_ref(), + request_client, + self.embedded_worker_mutation_dispatcher.as_ref(), + Some(self.prompt_projection_cache.clone()), + ); + if pending && context.workspace_id().is_some() { + let bundle = request + .config_bundle + .as_ref() + .ok_or_else(|| cause("pending recovery launch material missing"))?; + let projection = self + .observe_bundle_prompt_projection( + bundle, + request + .request + .workspace_api + .as_ref() + .map(|api| api.workspace_id.as_str()), + ) + .map_err(|_| cause("pending recovery Prompt projection invalid"))? + .ok_or_else(|| cause("pending recovery Prompt projection missing"))?; + loader = loader.with_effective_catalog(projection.projection.catalog.clone()); + } + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: Vec::new(), + candidate_workdir_cleanup: None, + allocation: None, + feature_tasks: None, + }, + ); + let store = CombinedStore::new(session_store, metadata_store); + let worker_result = Worker::restore_pending_from_worker_metadata_with_context( + &name, + manifest, + store, + loader, + context, + WorkerFilesystemAuthority::None, + ) + .await; + let mut worker = worker_result.map_err(|error| { + self.retain_construction_failure(request, &error); + cause(restore_worker_cause(&error)) + })?; + let tasks = worker.retain_startup_feature_cleanup(); + let sessions = worker.workdir_sessions(); + let allocation = worker.restore_allocation_cleanup_handle(); + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + FailedRestoreResources { + operation_id: request.operation_id, + sessions: vec![sessions.clone()], + candidate_workdir_cleanup: None, + allocation: allocation.clone(), + feature_tasks: Some(tasks.clone()), + }, + ); + let result = async { + if let Some(binding) = request.request.backend_job.as_ref() { + worker + .bind_backend_job(binding.clone()) + .map_err(|_| cause("recovery Backend Job binding failed"))?; + } + worker + .finalize_subjektiv_session_attribution( + request.request.subjektiv_attached, + SubjektivSessionAttributionLifecycle::RestoredSession, + ) + .map_err(|error| cause(restore_worker_cause(&error)))?; + if pending { + worker::WorkerController::recover_pending_head(&mut worker, &root) + .await + .map_err(|_| cause("pending Session head recovery failed"))?; + } + Ok::<_, RuntimeRestoreFailure>(()) + } + .await; + // Never release allocation while the Worker guard is attached. Retained + // handles in the same operation remain available on every cleanup error. + drop(worker); + let cleanup = async { + tasks + .shutdown() + .await + .map_err(|_| cause("recovery Feature settlement failed"))?; + sessions + .close_all() + .await + .map_err(|_| cause("recovery Workdir close failed"))?; + if let Some(allocation) = allocation { + allocation + .release() + .map_err(|_| cause("recovery allocation release failed"))?; + } + Ok::<_, RuntimeRestoreFailure>(()) + } + .await; + result?; + cleanup?; + self.failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&request.worker_ref); + Ok(true) + } +} diff --git a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs new file mode 100644 index 000000000..a5b0312f9 --- /dev/null +++ b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs @@ -0,0 +1,898 @@ +// Included inside worker_backend::tests, alongside the restore failure suite. +// These integration tests connect the real Profile factory, canonical stores, +// restore owner, and Controller. Only model I/O and the Workspace HTTP transport +// are substituted. The loopback Host is not an authentication/policy test. +// Run in the ordinary worker-runtime lib suite with the default product features +// and the filter `restore_candidate::`. No live service is used. Like the +// failure suite, this requires the existing Yoi path settings to resolve into +// target/restore-test-environment; the common fixture rejects unsafe fallbacks +// before any factory, Git, or allocation-table operation. + +#[cfg(feature = "http-server")] +mod restore_candidate { + use super::*; + + const FIRST: &str = "candidate-saved-first"; + const SECOND: &str = "candidate-saved-second"; + const NOTICE: &str = "candidate-saved-notice"; + const FRESH: &str = "candidate-fresh-input"; + const WAIT: Duration = Duration::from_secs(5); + + struct HostProbe { + calls: Arc>>, + task: Option>, + } + + impl Drop for HostProbe { + fn drop(&mut self) { + if let Some(task) = self.task.take() { + task.abort(); + } + } + } + + // Aborting on unwind prevents a failed setup assertion from leaving its + // hanging model turn alive. Successful setup uses the normal shutdown/join. + struct SeedController(Option); + + impl Drop for SeedController { + fn drop(&mut self) { + if let Some(controller) = self.0.as_ref() { + controller.controller_task.abort(); + } + } + } + + async fn receive_matching( + events: &mut tokio::sync::broadcast::Receiver, + phase: &str, + mut matches: impl FnMut(&Event) -> bool, + ) -> Event { + let mut seen = Vec::new(); + tokio::time::timeout(WAIT, async { + loop { + let event = events.recv().await.expect("Controller event stream"); + seen.push(format!("{event:?}")); + if matches(&event) { + return event; + } + assert!( + !matches!( + event, + Event::Error { .. } + | Event::SubmissionRejected { .. } + | Event::NotificationRejected { .. } + ), + "unexpected Controller rejection: {event:?}" + ); + } + }) + .await + .unwrap_or_else(|_| { + panic!("Controller acknowledgement deadline during {phase}; observed {seen:?}") + }) + } + + async fn submit(handle: &WorkerHandle, id: &str) -> Event { + let mut events = handle.subscribe(); + handle + .send(Method::Submit { + submission_request_id: id.into(), + input: vec![protocol::Segment::Text { content: id.into() }], + }) + .await + .unwrap(); + receive_matching(&mut events, id, |event| { + matches!(event, Event::SubmissionAccepted { submission_request_id, .. } + if submission_request_id == id) + }) + .await + } + + async fn notify(handle: &WorkerHandle) { + let mut events = handle.subscribe(); + handle + .send(Method::Notify { + notification_request_id: NOTICE.into(), + message: NOTICE.into(), + }) + .await + .unwrap(); + receive_matching(&mut events, "notification acceptance", |event| { + matches!(event, Event::NotificationAccepted { notification_request_id } + if notification_request_id == NOTICE) + }) + .await; + } + + async fn pending(handle: &WorkerHandle) -> protocol::PendingSubmissionsSnapshot { + let mut events = handle.subscribe(); + handle.send(Method::ListPendingSubmissions).await.unwrap(); + match receive_matching(&mut events, "pending queue snapshot", |event| { + matches!(event, Event::PendingSubmissionsChanged { .. }) + }) + .await + { + Event::PendingSubmissionsChanged { pending } => pending, + _ => unreachable!(), + } + } + + async fn finished(handle: &WorkerHandle, id: &str) { + let mut events = handle.subscribe(); + let is_finished = |snapshot: &protocol::WorkerStateSnapshot| { + snapshot.last_finished_submission_request_id.as_deref() == Some(id) + && snapshot.state == protocol::WorkerState::Idle + }; + if !is_finished(&handle.shared_state.snapshot()) { + receive_matching( + &mut events, + id, + |event| matches!(event, Event::WorkerState { snapshot } if is_finished(snapshot)), + ) + .await; + } + } + + fn queue_checkpoint(entries: &[LogEntry]) -> serde_json::Value { + entries + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == "worker.pending_activations.v1" => Some(payload.clone()), + _ => None, + }) + .expect("real Controller must have committed its pending activation checkpoint") + } + + struct Fixture { + backend: Arc, + request: WorkerExecutionRestoreRequest, + model: MockClient, + session: WorkerSessionStore, + session_id: session_store::SessionId, + segment_id: session_store::SegmentId, + receipts: Vec, + host: HostProbe, + _root: tempfile::TempDir, + } + + impl Fixture { + fn new() -> Self { + Self::with_transport(WorkerControllerTransport::InProcess) + } + + fn with_transport(transport: WorkerControllerTransport) -> Self { + let root = restore_failure_tempdir(); + let model = MockClient::sequential(vec![ + MockResponse::Complete(vec![ + LlmEvent::tool_use_start(0, "candidate-task-probe", "TaskCreate"), + LlmEvent::tool_input_delta( + 0, + r#"{"subject":"Activated candidate","description":"Execution probe"}"#, + ), + LlmEvent::tool_use_stop(0), + LlmEvent::Status(StatusEvent { + status: ResponseStatus::Completed, + }), + ]), + MockResponse::Complete(simple_text_events()), + MockResponse::Complete(simple_text_events()), + MockResponse::Complete(simple_text_events()), + ]); + let injected = model.clone(); + let mut factory = ProfileRuntimeWorkerFactory::new(root.path()) + .with_runtime_store_dir(root.path()) + .with_runtime_id("candidate-runtime") + .with_spawn_model_client_factory(move || Box::new(injected.clone())); + factory.controller_transport = transport; + let catalog = worker::PromptCatalog::builtins_only() + .unwrap() + .projection() + .clone(); + factory + .observe_workspace_prompt_projection( + worker::WorkspacePromptProjection::new( + "workspace-saved-subjektiv", + "candidate-fixture", + catalog.catalog_digest.clone(), + catalog, + ) + .unwrap(), + ) + .unwrap(); + let backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap()); + let fixture_root = root.path().to_path_buf(); + let (request, session, session_id, segment_id, receipts, host) = backend + .run_on_adapter_runtime(async move { + let calls = Arc::new(Mutex::new(Vec::new())); + let authorization_calls = calls.clone(); + let other_calls = calls.clone(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let authorization_path = + "/api/w/workspace-saved-subjektiv/authorization/check"; + let app = axum::Router::new() + .route( + authorization_path, + axum::routing::post(move |axum::Json(input): axum::Json< + server_api::authorization::OperationAuthorizationRequest, + >| { + let calls = authorization_calls.clone(); + async move { + calls.lock().unwrap().push(( + "POST".into(), authorization_path.into(), + )); + let response = if input.operation == "task" + && input.resource == server_api::authorization::PermissionResource::Feature + { + server_api::authorization::OperationAuthorizationResponse { + scope: Some(server_api::authorization::PermissionResource::Feature), + } + } else { + saved_subjektiv_authorization_fixture(input) + .expect("only the saved fixture Feature grants are available") + }; + axum::Json(response) + } + }), + ) + .fallback(move |request: axum::extract::Request| { + let calls = other_calls.clone(); + async move { + calls.lock().unwrap().push(( + request.method().to_string(), request.uri().path().to_owned(), + )); + axum::http::StatusCode::FORBIDDEN + } + }); + let host = HostProbe { + calls, + task: Some(tokio::spawn(async move { axum::serve(listener, app).await.unwrap() })), + }; + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let (mut manifest, mut create) = + saved_subjektiv_policy(&fixture_root, &worker_ref); + // Normal restore deliberately excludes Subject repair. Keep + // a real Workspace client so accidental Host effects are visible. + manifest.feature.subjektiv = manifest::ResolvedSubjektivFeatureConfig::default(); + manifest.feature.task.enabled = true; + create.subjektiv_attached = false; + create.workspace_api.as_mut().unwrap().base_url = base_url; + let aggregate = fixture_root.join("workers").join(worker_ref.worker_id.to_string()); + let session = WorkerSessionStore::new(aggregate.join("session")).unwrap(); + let metadata = WorkerAggregateStore::new(&aggregate, &manifest.worker.name).unwrap(); + let mut engine = Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated( + MockClient::sequential(vec![MockResponse::Hang(simple_text_events().into_iter().take(2).collect())]), + ); + engine.set_system_prompt("Committed restore candidate fixture prompt"); + let mut worker = Worker::new( + manifest.clone(), + engine, + CombinedStore::new(session.clone(), metadata.clone()), + RuntimeWorkspaceBackendRef::from_worker_request(&create, Some("candidate-runtime")) + .worker_context(&worker_ref, Some(&crate::runtime::RuntimeWorkspaceScope::new( + "workspace-saved-subjektiv", "server-main", + )), None, None, None, None), + WorkerFilesystemAuthority::None, + Scope::empty(), + ).await.unwrap(); + worker.enable_worker_metadata_write_through().unwrap(); + worker.materialize_durable_session_head().await.unwrap(); + let (session_id, segment_id) = (worker.session_id(), worker.segment_id()); + let workspace_client = worker.workspace_client_handle(); + let started = PreparedWorker::new( + worker, + WorkerBootstrapLayout::RuntimeManagedRun { run_dir: fixture_root.join("seed") }, + WorkerControllerTransport::InProcess, + ).start().await.unwrap(); + let mut seed = SeedController(Some(RuntimeWorkerController { + handle: started.handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), + controller_task: started.controller_task, + workspace_client, + })); + let handle = seed.0.as_ref().unwrap().handle.clone(); + let mut started_events = handle.subscribe(); + submit(&handle, "candidate-setup-turn").await; + // Running and streaming observations can arrive in either + // order. Do not discard TextDelta while awaiting WorkerState. + let mut running = handle.shared_state.catalog_status() == WorkerStatus::Running; + let mut streaming = false; + receive_matching(&mut started_events, "seed running and streaming", |event| { + if let Event::WorkerState { snapshot } = event { + running |= snapshot.catalog_status() == protocol::WorkerStatus::Running; + } + streaming |= matches!(event, Event::TextDelta { .. }); + running && streaming + }).await; + let state = Arc::new(RwLock::new(handle.shared_state.snapshot())); + let mut events = handle.subscribe(); + handle.send(Method::Pause { command: next_internal_command(&state).unwrap() }) + .await.unwrap(); + receive_matching(&mut events, "seed pause acknowledgement", |event| { + matches!(event, Event::WorkerState { snapshot } + if snapshot.catalog_status() == protocol::WorkerStatus::Paused) + }).await; + let mut receipts = Vec::new(); + for id in [FIRST, SECOND] { + let receipt = submit(&handle, id).await; + assert!(matches!(receipt, Event::SubmissionAccepted { + disposition: protocol::SubmissionDisposition::Queued, .. + })); + receipts.push(serde_json::to_value(receipt).unwrap()); + if id == FIRST { notify(&handle).await; } + } + assert_eq!(pending(&handle).await.submissions.len(), 2); + shutdown_profile_controller(seed.0.take().unwrap()).await; + // This orphan pin would be removed by the normal activation + // reconciliation. Preparation and discard must leave it alone. + let orphan = session.write_uploaded_file( + session_id, "orphan.txt", "text/plain", b"retained orphan", + session_store::UploadedFileLimits::default(), + ).unwrap(); + session.pin_uploaded_file(session_id, &orphan, "uncommitted-acceptance").unwrap(); + let mut saved = metadata.read_by_name(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref)) + .unwrap().unwrap(); + saved.resolved_manifest_snapshot = Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap()); + saved.workspace_root = Some(fixture_root.join("stale-derived-root")); + metadata.write(&saved).unwrap(); + let mut request = direct_restore_request(worker_ref); + request.request = create; + request.workspace_scope = Some(crate::runtime::RuntimeWorkspaceScope::new( + "workspace-saved-subjektiv", "server-main", + )); + Ok((request, session, session_id, segment_id, receipts, host)) + }) + .unwrap(); + let fixture = Self { + backend, + request, + model, + session, + session_id, + segment_id, + receipts, + host, + _root: root, + }; + let checkpoint = queue_checkpoint(&fixture.entries()); + assert_eq!(checkpoint["pending"].as_array().unwrap().len(), 2); + assert_eq!( + checkpoint["pending_notifications"] + .as_array() + .unwrap() + .len(), + 1 + ); + fixture.assert_quiet(); + fixture + } + + fn entries(&self) -> Vec { + self.session + .read_all(self.session_id, self.segment_id) + .unwrap() + } + + fn canonical_files(&self) -> BTreeMap> { + let aggregate = self + .backend + .factory + .worker_aggregate_dir(&self.request.worker_ref) + .unwrap(); + let mut files = persisted_files(&aggregate); + files.retain(|path, _| !path.components().any(|part| part.as_os_str() == "runs")); + files + } + + fn assert_no_modifying_host_calls(&self) { + assert!( + self.host + .calls + .lock() + .unwrap() + .iter() + .all(|(method, path)| { + method == "GET" + || method == "HEAD" + || (method == "POST" + && path == "/api/w/workspace-saved-subjektiv/authorization/check") + }), + "unexpected modifying Host call" + ); + } + + fn assert_quiet(&self) { + assert_eq!(self.model.call_count.load(Ordering::SeqCst), 0); + assert!(self.model.captured.lock().unwrap().is_empty()); + self.assert_no_modifying_host_calls(); + assert!( + session_store::collect_state(&self.entries()) + .annotated_history + .iter() + .all(|entry| { + !matches!( + entry.item, + LoggedItem::ToolCall { .. } | LoggedItem::ToolResult { .. } + ) + }), + "prepare committed tool execution" + ); + } + + fn prepare(&self) { + assert!(matches!( + self.backend.restore_worker(self.request.clone()), + WorkerExecutionSpawnResult::Connected { .. } + )); + // Connected here is the backend's prepared snapshot, not publication. + assert!( + self.backend + .pending_restore(&self.request.worker_ref) + .unwrap() + .is_some() + ); + assert!(self.backend.workers.lock().unwrap().is_empty()); + assert!( + self.backend + .factory + .observation_hub + .get(&self.request.worker_ref) + .is_none() + ); + assert!( + !self + .backend + .factory + .worker_restore_run_dir(&self.request.worker_ref, self.request.operation_id,) + .unwrap() + .exists() + ); + let table = worker::runtime::worker_allocation::LockFileGuard::open( + &worker::runtime::worker_allocation::default_allocation_path().unwrap(), + ) + .unwrap(); + assert!( + table + .data() + .find(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref( + &self.request.worker_ref, + )) + .is_none() + ); + } + + fn assert_no_allocation(&self) { + let table = worker::runtime::worker_allocation::LockFileGuard::open( + &worker::runtime::worker_allocation::default_allocation_path().unwrap(), + ) + .unwrap(); + assert!( + table + .data() + .find(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref( + &self.request.worker_ref, + )) + .is_none() + ); + } + + fn assert_released(&self) { + self.assert_no_allocation(); + assert!( + self.backend + .pending_restore(&self.request.worker_ref) + .unwrap() + .is_none() + ); + assert!(self.backend.workers.lock().unwrap().is_empty()); + assert!( + self.backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty() + ); + assert!( + self.backend + .factory + .observation_hub + .get(&self.request.worker_ref) + .is_none() + ); + } + } + + impl Drop for Fixture { + fn drop(&mut self) { + // Keep the storage and loopback transport alive until owned tasks stop. + let _ = self.backend.stop_worker(&self.request.worker_ref); + if let Some(task) = self.host.task.take() { + task.abort(); + let _ = self.backend.run_on_adapter_runtime(async move { + let _ = task.await; + Ok(()) + }); + } + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn prepare_saved_inputs_rejects_new_input_and_discard_preserves_authority() { + // Exercise the configured socket transport without ever publishing or + // binding it; activation coverage below uses the normal InProcess path. + let mut fixture = Fixture::with_transport(WorkerControllerTransport::UnixSocket); + let opens = Arc::new(AtomicUsize::new(0)); + let counted = opens.clone(); + let raw = deferred_workdir::DeferredWorkdirSession::handle( + "candidate-discard-resource", + workdir::WorkdirSessionCapabilities::READ_ONLY, + move || { + counted.fetch_add(1, Ordering::SeqCst); + Err("not opened during prepare".into()) + }, + ); + fixture.request.deferred_workdir_sessions.insert( + WorkdirAttachmentAlias::new("checkout").unwrap(), + raw.clone(), + ); + let before = fixture.canonical_files(); + fixture.prepare(); + for input in [ + WorkerInput::user("must-not-accept"), + WorkerInput::notify("must-not-notify"), + ] { + assert!( + !fixture + .backend + .dispatch_input(&fixture.request.worker_ref, input) + .is_accepted() + ); + } + #[cfg(feature = "ws-server")] + assert!( + fixture + .backend + .clone() + .attach_worker_protocol(&fixture.request.worker_ref) + .is_err() + ); + assert!(!matches!( + fixture + .backend + .worker_session(WorkerSessionObservationRequest { + worker_ref: fixture.request.worker_ref.clone(), + }), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before); + assert!( + fixture + .backend + .factory + .failed_restore_resources + .lock() + .unwrap()[&fixture.request.worker_ref] + .candidate_workdir_cleanup + .is_some(), + "retain the real prepared Worker's close-only resource owner" + ); + assert!( + fixture + .backend + .stop_worker(&fixture.request.worker_ref) + .is_accepted() + ); + fixture.assert_released(); + + let factory = fixture.backend.factory.clone(); + let request = fixture.request.clone(); + fixture + .backend + .run_on_adapter_runtime(async move { + // The adapter resolves and owns its own raw handles. Separately + // exercise the factory seam with the exact supplied cleanup handle. + let mut candidate = factory + .prepare_restore(request) + .await + .map_err(|error| error.to_string())?; + candidate + .discard() + .await + .map_err(|error| error.to_string())?; + assert!(matches!( + raw.read(workdir::ReadRequest { + path: workdir::WorkdirPath::new("probe.txt").unwrap(), + offset: 0, + limit: 1, + max_bytes: 32, + }) + .await, + Err(workdir::WorkdirError::SessionClosed) + )); + Ok(()) + }) + .unwrap(); + assert_eq!(opens.load(Ordering::SeqCst), 0); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before); + assert!( + fixture + .backend + .stop_worker(&fixture.request.worker_ref) + .is_accepted() + ); + assert_eq!(fixture.canonical_files(), before); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn prepare_failure_at_each_pre_activation_checkpoint_preserves_saved_authority() { + // Feature and Controller are activation checkpoints, not preparation. + for stage in [ + RuntimeRestoreStage::Metadata, + RuntimeRestoreStage::Reconstruction, + RuntimeRestoreStage::Binding, + ] { + let fixture = Fixture::new(); + let before = fixture.canonical_files(); + *fixture + .backend + .factory + .restore_faults + .fail_at + .lock() + .unwrap() = Some(stage); + let result = fixture.backend.restore_worker(fixture.request.clone()); + assert!( + matches!(result, WorkerExecutionSpawnResult::RolledBack(_)), + "{stage:?}: {result:?}" + ); + fixture.assert_released(); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before, "{stage:?}"); + let completed = fixture.backend.completed_restore_failures.lock().unwrap(); + let (_, failure) = &completed[&fixture.request.worker_ref]; + assert_eq!(failure.stage, stage); + assert_eq!(failure.cleanup, RuntimeRestoreCleanup::Confirmed); + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn activate_saved_inputs_drains_fifo_once_and_replays_durable_receipts() { + let fixture = Fixture::new(); + let before = fixture.canonical_files(); + fixture.prepare(); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before); + fixture + .backend + .activate_restored_worker(fixture.request.operation_id, &fixture.request.worker_ref) + .unwrap(); + let handle = fixture.backend.workers.lock().unwrap()[&fixture.request.worker_ref] + .handle + .clone(); + let model = fixture.model.clone(); + let receipts = fixture.receipts.clone(); + fixture + .backend + .run_on_adapter_runtime(async move { + finished(&handle, SECOND).await; + assert_eq!(model.call_count.load(Ordering::SeqCst), 3); + { + let requests = model.captured.lock().unwrap(); + let first = serde_json::to_string(&requests[0].items).unwrap(); + let second = serde_json::to_string(&requests[2].items).unwrap(); + assert!( + requests[0] + .tools + .iter() + .any(|tool| tool.name == "TaskCreate") + ); + assert!(first.contains(FIRST)); + assert!(!first.contains(SECOND), "second Submit overtook the first"); + assert_eq!(first.matches(NOTICE).count(), 1); + assert!(second.find(FIRST).unwrap() < second.find(SECOND).unwrap()); + assert_eq!(second.matches(NOTICE).count(), 1); + } + let committed = serde_json::to_value(handle.committed_entries()).unwrap(); + for (id, mut expected) in [FIRST, SECOND].into_iter().zip(receipts) { + // Activation advances the durable receipt from Queued to Started; + // replay must preserve its original submission identity. + expected["data"]["disposition"] = serde_json::json!("started"); + assert_eq!( + serde_json::to_value(submit(&handle, id).await).unwrap(), + expected + ); + } + notify(&handle).await; + assert!(pending(&handle).await.submissions.is_empty()); + assert_eq!(model.call_count.load(Ordering::SeqCst), 3); + assert_eq!( + serde_json::to_value(handle.committed_entries()).unwrap(), + committed + ); + Ok(()) + }) + .unwrap(); + let history = session_store::collect_state(&fixture.entries()).annotated_history; + assert!( + history.iter().any(|entry| matches!( + &entry.item, + LoggedItem::ToolResult { call_id, is_error: false, .. } + if call_id == "candidate-task-probe" + )), + "activation must execute the real TaskCreate tool, not just decode a call" + ); + for marker in [FIRST, SECOND, NOTICE] { + let count = history + .iter() + .filter(|entry| { + match &entry.item { + LoggedItem::Message { content, .. } => content.iter().any(|part| { + matches!(part, LoggedContentPart::Text { text } if text.contains(marker)) + }), + _ => false, + } + }) + .count(); + assert_eq!(count, 1, "{marker} must commit exactly once"); + } + let mut fresh = WorkerInput::user(FRESH); + fresh.submission_request_id = Some(FRESH.into()); + assert!( + fixture + .backend + .dispatch_input(&fixture.request.worker_ref, fresh) + .is_accepted() + ); + let handle = fixture.backend.workers.lock().unwrap()[&fixture.request.worker_ref] + .handle + .clone(); + fixture + .backend + .run_on_adapter_runtime(async move { + finished(&handle, FRESH).await; + assert!(pending(&handle).await.submissions.is_empty()); + Ok(()) + }) + .unwrap(); + assert_eq!(fixture.model.call_count.load(Ordering::SeqCst), 4); + assert!( + !fixture.host.calls.lock().unwrap().is_empty(), + "activation must reach the configured Host transport" + ); + fixture.assert_no_modifying_host_calls(); + assert!( + fixture + .backend + .stop_worker(&fixture.request.worker_ref) + .is_accepted() + ); + fixture.assert_released(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn explicit_repair_reconciles_lost_attribution_response_without_reseeding() { + for materialized in [true, false] { + let root = restore_failure_tempdir(); + let model = MockClient::new(Vec::new()); + let injected = model.clone(); + let backend = WorkerRuntimeExecutionBackend::new( + ProfileRuntimeWorkerFactory::new(root.path()) + .with_runtime_store_dir(root.path()) + .with_runtime_id("repair-runtime") + .with_spawn_model_client_factory(move || Box::new(injected.clone())), + ) + .unwrap(); + let factory = backend.factory.clone(); + let fixture_root = root.path().to_path_buf(); + backend.run_on_adapter_runtime(async move { + let mut request = saved_subjektiv_restore_fixture(&fixture_root, materialized).await; + let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&request.worker_ref); + let aggregate = fixture_root.join("workers").join(request.worker_ref.worker_id.to_string()); + let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap(); + let mut saved = metadata.read_by_name(&name).unwrap().unwrap(); + let session_id = saved.active.as_ref().unwrap().session_id; + saved.subjektiv_session_attribution = Some(session_store::SubjektivSessionAttributionState::OutcomeUnknown { session_id, diagnostic: "saved interrupted attribution".into() }); + metadata.write(&saved).unwrap(); + // Explicit repair may acquire artifact locks; compare canonical + // contents separately from that lock-file side effect. + let session_files = || { + let mut files = persisted_files(&aggregate.join("session")); + files.remove(Path::new("artifacts/paste/.aggregate.lock")); + files + }; + let original = session_files(); + let mut bundle = test_bundle(); + bundle.metadata.workspace_id = "workspace-saved-subjektiv".into(); + bundle.prompt_catalog = Some(worker::PromptCatalog::builtins_only().unwrap().projection().clone()); + request.config_bundle = Some(bundle.with_computed_digest()); + + // Substitute only the Host transport. A committed attribution + // with an undecodable response models lost acknowledgement; + // its durable Session-keyed receipt survives the retry. This + // does not test the real Host's signature or Subject policy. + let attribution_path = fixture_root.join("host-attribution.json"); + let stored = attribution_path.clone(); + let calls = Arc::new(AtomicUsize::new(0)); + let record_calls = calls.clone(); + let startup_calls = Arc::new(AtomicUsize::new(0)); + let startups = startup_calls.clone(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + request.request.workspace_api.as_mut().unwrap().base_url = format!("http://{}", listener.local_addr().unwrap()); + let app = axum::Router::new() + .route("/api/w/workspace-saved-subjektiv/authorization/check", axum::routing::post( + |axum::Json(input): axum::Json| async move { + axum::Json(saved_subjektiv_authorization_fixture(input).unwrap()) + })) + .route("/api/w/workspace-saved-subjektiv/subjektiv/sessions", axum::routing::post( + move |axum::Json(input): axum::Json| { + let stored = stored.clone(); + let calls = record_calls.clone(); + async move { + assert_eq!(input.session_id, session_id.to_string()); + let ordinal = calls.fetch_add(1, Ordering::SeqCst); + if !stored.exists() { + assert!(input.create_if_missing); + fs::write(&stored, serde_json::to_vec(&server_api::SubjektivRecordSessionResponse { + session_id: input.session_id, subject_id: "saved-subject".into(), + }).unwrap()).unwrap(); + } + (axum::http::StatusCode::OK, [(axum::http::header::CONTENT_TYPE, "application/json")], + if ordinal == 0 { "{}".to_string() } else { fs::read_to_string(stored).unwrap() }) + } + })) + .route("/api/w/workspace-saved-subjektiv/subjektiv/journal/startup", axum::routing::get(move || { + let startups = startups.clone(); + async move { + startups.fetch_add(1, Ordering::SeqCst); + axum::Json(serde_json::json!({ "subject_id": "saved-subject", "journals": [], "omitted": false })) + } + })); + let host = HostProbe { calls: Arc::new(Mutex::new(Vec::new())), task: Some(tokio::spawn(async move { axum::serve(listener, app).await.unwrap() })) }; + // Same real factory and same operation for both repairs. No + // candidate/controller is constructed by the test itself. + assert!(factory.recover_saved_restore_source(&request).await.is_err()); + assert!(attribution_path.exists(), "Host committed before response loss"); + assert_eq!(session_files(), original); + let failed = metadata.read_by_name(&name).unwrap().unwrap(); + assert!(matches!(failed.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::OutcomeUnknown { session_id: id, .. }) if id == session_id)); + assert_eq!(startup_calls.load(Ordering::SeqCst), 0); + factory.cleanup_failed_restore(&request).await.unwrap(); + assert!(factory.recover_saved_restore_source(&request).await.unwrap()); + let repaired = metadata.read_by_name(&name).unwrap().unwrap(); + assert_eq!(repaired.active.as_ref().unwrap().session_id, session_id); + assert!(matches!(repaired.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Confirmed { session_id: id, .. }) if id == session_id)); + let segments = WorkerSessionStore::new(aggregate.join("session")).unwrap() + .list_segments(session_id).unwrap(); + assert_eq!(segments.len(), 1, "retry must retain exactly one seed"); + if materialized { + assert_eq!(session_files(), original); + assert_eq!(startup_calls.load(Ordering::SeqCst), 0); + } else { + assert_eq!(startup_calls.load(Ordering::SeqCst), 1); + } + let after_repair = persisted_files(&aggregate); + let calls_after_repair = calls.load(Ordering::SeqCst); + assert!(!factory.recover_saved_restore_source(&request).await.unwrap()); + assert_eq!(persisted_files(&aggregate), after_repair); + assert_eq!(calls.load(Ordering::SeqCst), calls_after_repair); + assert!(factory.failed_restore_resources.lock().unwrap().is_empty()); + drop(host); + Ok(()) + }).unwrap(); + assert_eq!(model.call_count.load(Ordering::SeqCst), 0); + } + } +} diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs index 8b870cd62..40de8168c 100644 --- a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -1,8 +1,11 @@ // Integration coverage through EmbeddedRuntime -> real Profile factory -> real -// Worker/bootstrap, local Git Workdir provider, and allocation table. Model I/O -// is not invoked; this is a deterministic ordinary suite, not product E2E. +// Worker/bootstrap, local Git Workdir provider, and allocation table. External +// model I/O is replaced for saved-input coverage; this is not product E2E. + +use crate::restore_test_environment::require_restore_test_environment; fn restore_failure_tempdir() -> tempfile::TempDir { + require_restore_test_environment(); // Explicitly stay inside this checkout, even when TMPDIR points elsewhere. // Keep fixture components short; no symlink/alias to evade socket limits. let target = Path::new(env!("CARGO_MANIFEST_DIR")) @@ -102,11 +105,16 @@ struct ProfileFailureFixture { backend: Arc, worker: crate::catalog::WorkerDetail, provider_opens: Arc, + seed_handle: Option, root: tempfile::TempDir, } impl ProfileFailureFixture { fn new(timeout: Duration) -> Self { + Self::with_saved_inputs(timeout, None) + } + + fn with_saved_inputs(timeout: Duration, model: Option) -> Self { let root = restore_failure_tempdir(); let repo = root.path().join("g"); fs::create_dir(&repo).unwrap(); @@ -118,16 +126,20 @@ impl ProfileFailureFixture { git(&repo, &["commit", "-m", "init"]); let store = root.path().join("s"); let provider_opens = Arc::new(AtomicUsize::new(0)); + let mut factory = + ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store); + let queue_inputs = model.is_some(); + if let Some(model) = model { + factory = factory.with_spawn_model_client_factory(move || Box::new(model.clone())); + } let backend = Arc::new( - WorkerRuntimeExecutionBackend::new( - ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store), - ) - .unwrap() - .with_spawn_restore_timeout(timeout) - .with_working_directory_materializer(RestoreOpenCountingMaterializer { - inner: RuntimeGitMaterializer::new(root.path().join("w")), - opens: provider_opens.clone(), - }), + WorkerRuntimeExecutionBackend::new(factory) + .unwrap() + .with_spawn_restore_timeout(timeout) + .with_working_directory_materializer(RestoreOpenCountingMaterializer { + inner: RuntimeGitMaterializer::new(root.path().join("w")), + opens: provider_opens.clone(), + }), ); let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( crate::fs_store::FsRuntimeStoreOptions { @@ -146,12 +158,41 @@ impl ProfileFailureFixture { working_directory: working_directory_request(&repo), }]; let worker = runtime.create_worker(request).unwrap(); + let seed_handle = queue_inputs.then(|| { + backend.workers.lock().unwrap()[&worker.worker_ref] + .handle + .clone() + }); + if let Some(handle) = seed_handle.clone() { + backend.run_on_adapter_runtime(async move { + async fn event(events: &mut tokio::sync::broadcast::Receiver, predicate: impl Fn(&Event) -> bool) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let next = events.recv().await.unwrap(); + assert!(!matches!(next, Event::Error { .. } | Event::SubmissionRejected { .. } | Event::NotificationRejected { .. }), "{next:?}"); + if predicate(&next) { break; } + } + }).await.expect("saved-input fixture acknowledgement"); + } + let mut events = handle.subscribe(); + handle.send(Method::submit_text("revocation-seed", "hold for saved input setup")).await.unwrap(); + event(&mut events, |next| matches!(next, Event::TextDelta { .. })).await; + handle.send(Method::Pause { command: protocol::WorkerCommandEnvelope::new(handle.shared_state.snapshot().last_command_id + 1) }).await.unwrap(); + event(&mut events, |next| matches!(next, Event::WorkerState { snapshot } if snapshot.catalog_status() == protocol::WorkerStatus::Paused)).await; + handle.send(Method::submit_text("revocation-saved-submit", "must stay queued")).await.unwrap(); + event(&mut events, |next| matches!(next, Event::SubmissionAccepted { submission_request_id, disposition: protocol::SubmissionDisposition::Queued, .. } if submission_request_id == "revocation-saved-submit")).await; + handle.send(Method::Notify { notification_request_id: "revocation-saved-notice".into(), message: "must also stay queued".into() }).await.unwrap(); + event(&mut events, |next| matches!(next, Event::NotificationAccepted { notification_request_id } if notification_request_id == "revocation-saved-notice")).await; + Ok(()) + }).unwrap(); + } runtime.stop_worker(&worker.worker_ref, None).unwrap(); // Count restore provider access, not the initial create/stop setup. provider_opens.store(0, Ordering::SeqCst); Self { root, provider_opens, + seed_handle, runtime, backend, worker, @@ -241,6 +282,68 @@ impl ProfileFailureFixture { assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0); } + fn resume_same_operation( + &self, + request: runtime_api::WorkerRestoreRequest, + operation_id: crate::execution::WorkerLifecycleOperationId, + ) { + let result = self + .runtime + .restore_worker_operation(&self.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::Accepted, + "{result:?}" + ); + assert_eq!( + self.backend.workers.lock().unwrap()[&self.worker.worker_ref].restore_operation_id, + Some(operation_id) + ); + let calls = self + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst); + let replay = self + .runtime + .restore_worker_operation(&self.worker.worker_ref, request) + .unwrap(); + assert_eq!( + serde_json::to_value(replay).unwrap(), + serde_json::to_value(result).unwrap() + ); + assert_eq!( + self.backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + calls + ); + self.runtime + .stop_worker(&self.worker.worker_ref, None) + .unwrap(); + assert!( + self.backend + .factory + .observation_hub + .get(&self.worker.worker_ref) + .is_none() + ); + assert!(self.backend.pending_restores.lock().unwrap().is_empty()); + assert!(self.backend.workers.lock().unwrap().is_empty()); + assert!( + self.backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty() + ); + } + fn fresh_restore_and_old_replay( &self, request: runtime_api::WorkerRestoreRequest, @@ -319,6 +422,171 @@ impl ProfileFailureFixture { } } +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_timeout_attachment_revocation_cannot_reactivate_old_candidate() { + let model = MockClient::sequential(vec![MockResponse::Hang( + simple_text_events().into_iter().take(2).collect(), + )]); + let fixture = + ProfileFailureFixture::with_saved_inputs(Duration::from_secs(2), Some(model.clone())); + let body_calls = model.call_count.load(Ordering::SeqCst); + assert_eq!(body_calls, 1); + let aggregate = fixture + .backend + .factory + .worker_aggregate_dir(&fixture.worker.worker_ref) + .unwrap(); + let session_before = persisted_files(&aggregate.join("session")); + let metadata = WorkerAggregateStore::open_read_only( + &aggregate, + ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&fixture.worker.worker_ref), + ) + .unwrap(); + let saved = metadata + .read_by_name(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref( + &fixture.worker.worker_ref, + )) + .unwrap() + .unwrap(); + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .factory_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .factory_entered + .lock() + .unwrap() = Some(entered_tx); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let first = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + entered.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + assert!(!pending.request.logical_workdir_attachments.is_empty()); + let updated = fixture + .runtime + .replace_worker_workdir_attachments(&fixture.worker.worker_ref, Vec::new()) + .unwrap(); + assert!(updated.workdir_attachments.is_empty()); + let authority = || -> serde_json::Value { + serde_json::from_slice( + &fs::read( + fixture + .root + .path() + .join("s/workers") + .join(fixture.worker.worker_ref.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap() + }; + assert!(authority().get("logical_workdir_attachments").is_none()); + release.notify_one(); + let replay = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "current attachment revocation must fence the old candidate: {replay:?}" + ); + assert!( + replay + .message + .as_deref() + .unwrap_or_default() + .contains("current Worker authority differs"), + "{replay:?}" + ); + let record = authority(); + let owner = &record["restore_guard"]["owners"][&request.request_id]; + assert_eq!( + owner["operation_id"], + serde_json::to_value(pending.request.operation_id).unwrap() + ); + assert!(owner["receipt"].is_null()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!(Arc::ptr_eq( + &pending, + &fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap() + )); + assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0); + assert!(!pending.activation_started.load(Ordering::SeqCst)); + assert_eq!(model.call_count.load(Ordering::SeqCst), body_calls); + assert_eq!(persisted_files(&aggregate.join("session")), session_before); + assert_eq!( + metadata.read_by_name(&saved.worker_name).unwrap().unwrap(), + saved + ); + let old = fixture.seed_handle.clone().unwrap(); + fixture + .backend + .run_on_adapter_runtime(async move { + assert!(old.protocol_is_closed()); + let old_command = Method::Resume { + command: protocol::WorkerCommandEnvelope::new( + old.shared_state.snapshot().last_command_id + 1, + ), + }; + assert!( + old.send(old_command).await.is_err(), + "old execution command must not enter the restore candidate" + ); + Ok(()) + }) + .unwrap(); + assert!(authority().get("logical_workdir_attachments").is_none()); + let input = fixture.runtime.send_input( + &fixture.worker.worker_ref, + WorkerInput::user("must-not-enter-revoked-candidate"), + ); + assert!( + input.is_err(), + "pending candidate must not admit fresh input: {input:?}" + ); + // Stop owns cleanup of the original operation, never a newly rebound alias. + fixture + .runtime + .stop_worker(&fixture.worker.worker_ref, None) + .unwrap(); + fixture.assert_closed_deferred_sessions(&pending); + assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0); + assert!(fixture.backend.pending_restores.lock().unwrap().is_empty()); + let old = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request) + .unwrap(); + assert_ne!(old.state, server_api::WorkerRestoreState::Accepted); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); +} + #[test] #[serial_test::serial(worker_allocation)] fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() { @@ -327,8 +595,6 @@ fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() { for stage in [ RuntimeRestoreStage::Reconstruction, RuntimeRestoreStage::Binding, - RuntimeRestoreStage::Feature, - RuntimeRestoreStage::Controller, ] { let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); fixture.fail(stage); @@ -474,7 +740,7 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer .unwrap(); assert_eq!( result.state, - server_api::WorkerRestoreState::RolledBack, + server_api::WorkerRestoreState::ReconciliationRequired, "{result:?}" ); assert!( @@ -488,12 +754,11 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer .iter() .all(|sessions| sessions.aliases().is_empty()) ); - fixture.assert_released(); fixture.assert_closed_deferred_sessions(&pending); eprintln!( - "restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=RolledBack" + "restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=ReconciliationRequired" ); - fixture.fresh_restore_and_old_replay(request, result); + fixture.resume_same_operation(request, pending.request.operation_id); } #[test] @@ -585,6 +850,7 @@ async fn failed_restore_raw_cleanup_is_operation_bound_and_does_not_follow_reuse FailedRestoreResources { operation_id: current.operation_id, sessions: Vec::new(), + candidate_workdir_cleanup: None, allocation: None, feature_tasks: None, }, @@ -715,14 +981,14 @@ fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconcile .unwrap(); assert_eq!( result.state, - server_api::WorkerRestoreState::RolledBack, + server_api::WorkerRestoreState::Accepted, "{result:?}" ); - fixture.assert_released(); + assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); eprintln!( "restore-fault cleanup=timeout retained-task=true late-completion=joined executions=1" ); - fixture.fresh_restore_and_old_replay(request, result); + fixture.resume_same_operation(request, pending.request.operation_id); } #[cfg(unix)] @@ -745,7 +1011,7 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( .unwrap(); assert_eq!( result.state, - server_api::WorkerRestoreState::RolledBack, + server_api::WorkerRestoreState::ReconciliationRequired, "{result:?}" ); let diagnostic = serde_json::to_string(&result).unwrap(); @@ -757,8 +1023,12 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( // This case deliberately exercises the real Unix socket path-limit error; // positive restore cases retain the factory's normal InProcess transport. // Short fixture roots must not accidentally turn it into a successful bind. - let operation_id = - fixture.backend.completed_restore_failures.lock().unwrap()[&fixture.worker.worker_ref].0; + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + let operation_id = pending.request.operation_id; let socket = fixture .backend .factory @@ -767,7 +1037,25 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( .join("worker.sock"); assert!(socket.as_os_str().as_encoded_bytes().len() >= 108); assert!(!diagnostic.contains(&fixture.root.path().to_string_lossy().to_string())); - fixture.assert_released(); + assert!(!socket.exists()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!( + fixture + .backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty() + ); + assert!( + fixture + .backend + .completed_restore_failures + .lock() + .unwrap() + .is_empty() + ); fixture .backend .factory @@ -777,7 +1065,7 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( eprintln!( "restore-fault stage=socket-start real-bootstrap=true partial-run=removed allocation=released" ); - fixture.fresh_restore_and_old_replay(request, result); + fixture.resume_same_operation(request, operation_id); } #[test] @@ -1178,6 +1466,8 @@ fn profile_restore_missing_committed_system_prompt_rolls_back_and_replays_classi "#, ) .unwrap(); + let mut manifest = manifest; + manifest.worker.name = worker_name.clone(); metadata.resolved_manifest_snapshot = Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap()); metadata_store.write(&metadata).unwrap(); diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index a541ca2e1..97cdff43a 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -3676,8 +3676,12 @@ mod tests { #[test] fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() { + crate::restore_test_environment::require_restore_test_environment(); let repo = create_clean_repo(); - let runtime_root = tempfile::tempdir().unwrap(); + // Both ends use a short reference into this TempDir-owned tree. Keep + // real agent/broker sockets and the existing bind/cleanup assertions. + let runtime_root = tempfile::tempdir_in(".").unwrap(); + let runtime_path = Path::new(runtime_root.path().file_name().unwrap()); let key_root = tempfile::tempdir().unwrap(); let key_path = key_root.path().join("id_ed25519"); let fallback_key_path = key_root.path().join("id_ed25519_fallback"); @@ -3689,7 +3693,7 @@ mod tests { .unwrap(); assert!(status.success()); } - let materializer = RuntimeGitMaterializer::new(runtime_root.path()); + let materializer = RuntimeGitMaterializer::new(runtime_path); let mut request = request(repo.path()); request.materialization = Some(crate::catalog::RepositoryMaterializationContext { workspace_id: "workspace-1".to_string(), @@ -3731,7 +3735,7 @@ mod tests { kind: server_api::RepositorySourceKind::Ssh, uri: "ssh://git@example.test/repo.git".to_string(), }; - let command_access = RepositoryCommandAccess::prepare(runtime_root.path(), &ssh_operation) + let command_access = RepositoryCommandAccess::prepare(runtime_path, &ssh_operation) .unwrap() .unwrap(); assert!(!command_access.root.join("identity").exists()); @@ -3873,18 +3877,17 @@ mod tests { .unwrap(); // Restore preflight must validate without consuming this one-shot SSH // delivery or creating agent/session resources needed by actual bind. - let record_before = - fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(); + let record_before = fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap(); materializer .preflight_bind_working_directory(&id, None) .unwrap(); assert_eq!( - fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(), + fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap(), record_before ); assert_eq!(materializer.list_working_directories().unwrap().len(), 1); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 0 @@ -3898,7 +3901,7 @@ mod tests { assert!(ssh_policy.contains("__repository-ssh")); assert!(!ssh_policy.contains("SSH_AUTH_SOCK")); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 1 @@ -3916,7 +3919,7 @@ mod tests { "consumed delivery expiry must not revoke a live Workdir session" ); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 1, @@ -3925,7 +3928,7 @@ mod tests { drop(binding); assert!(!ssh_command.exists()); assert_eq!( - fs::read_dir(runtime_root.path().join(".repository-agents")) + fs::read_dir(runtime_path.join(".repository-agents")) .map(|entries| entries.count()) .unwrap_or_default(), 0 @@ -3971,7 +3974,7 @@ mod tests { ], &read_only_command_policy, )); - let broker_socket = fs::read_dir(runtime_root.path().join(REPOSITORY_ACCESS_DIR)) + let broker_socket = fs::read_dir(runtime_path.join(REPOSITORY_ACCESS_DIR)) .unwrap() .filter_map(Result::ok) .map(|entry| entry.path().join("b.sock")) @@ -4104,7 +4107,7 @@ mod tests { .code, "working_directory_repository_access_expired" ); - let restored = RuntimeGitMaterializer::new(runtime_root.path()); + let restored = RuntimeGitMaterializer::new(runtime_path); assert_eq!( restored.bind_working_directory(&id, None).unwrap_err().code, "working_directory_remote_repository_access_required" diff --git a/crates/worker/src/bootstrap.rs b/crates/worker/src/bootstrap.rs index daf86ee25..00e6c5ddf 100644 --- a/crates/worker/src/bootstrap.rs +++ b/crates/worker/src/bootstrap.rs @@ -9,6 +9,7 @@ use crate::PromptCatalogSource; use crate::controller::{ ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle, }; +pub use crate::worker::{PreparedRestoredWorker, RestoreCandidateWorkdirCleanup}; use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext}; use manifest::WorkerManifest; @@ -168,6 +169,38 @@ where Ok(PreparedWorker::new(worker, self.layout, self.transport)) } + /// Reconstruct a saved materialized Worker without allocation, attribution + /// repair, Feature installation, or controller exposure. The caller owns the + /// later controller layout/transport and must activate before starting it. + pub async fn prepare_restored_candidate( + self, + worker_name: &str, + ) -> Result, WorkerBootstrapError> { + let mut candidate = + Worker::prepare_restored_from_worker_metadata_with_context_and_model_client( + worker_name, + self.manifest, + self.store, + self.prompt_catalog, + self.workspace_context, + self.filesystem_authority, + self.model_client, + ) + .await + .map_err(WorkerBootstrapError::Worker)?; + if let Some(workdir_session) = self.workdir_session { + candidate + .worker_mut() + .map_err(WorkerBootstrapError::Worker)? + .bind_single_workdir_session(Some(workdir_session)); + } + candidate + .worker_mut() + .map_err(WorkerBootstrapError::Worker)? + .bind_subjektiv_host(self.subjektiv_host); + Ok(candidate) + } + pub async fn prepare_restored( self, worker_name: &str, diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index 3ff170dd9..6721753d6 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -689,6 +689,35 @@ pub enum WorkerControllerTransport { pub struct WorkerController; impl WorkerController { + /// Explicit recovery of an unmaterialized Session head. This installs the + /// saved Feature composition needed to produce the seed, but creates no + /// Controller, input channel, socket, observer, or queue consumer. The caller + /// owns the same operation's Feature/session/allocation cleanup on every exit. + pub async fn recover_pending_head( + worker: &mut Worker, + runtime_base: &Path, + ) -> std::io::Result<()> + where + C: LlmClient + Clone + 'static, + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + if worker.needs_initial_session_head_materialization() { + worker + .validate_subjektiv_body_publication() + .map_err(std::io::Error::other)?; + let registry = SpawnedWorkerRegistry::new_internal( + worker.manifest().worker.name.clone(), + worker.scope().clone(), + ); + register_worker_tools(worker, runtime_base.to_path_buf(), registry, None, None).await?; + worker + .materialize_durable_session_head() + .await + .map_err(std::io::Error::other)?; + } + Ok(()) + } + pub async fn spawn( worker: Worker, runtime_base: &Path, diff --git a/crates/worker/src/discovery.rs b/crates/worker/src/discovery.rs index dd7ecb8e8..e8354a499 100644 --- a/crates/worker/src/discovery.rs +++ b/crates/worker/src/discovery.rs @@ -1066,7 +1066,8 @@ mod tests { use tokio::net::UnixListener; use crate::runtime::dir::RuntimeDir; - use crate::runtime::worker_allocation::test_util::RuntimeDirSandbox; + + use crate::restore_test_environment::require_restore_test_environment; #[derive(Clone)] struct FailTargetPeerStore { @@ -1110,11 +1111,13 @@ mod tests { #[tokio::test(flavor = "current_thread")] async fn state_backed_visibility_and_restore_planning() { - let root = TempDir::new().unwrap(); + require_restore_test_environment(); + let root = tempfile::tempdir_in(".").unwrap(); let store_dir = root.path().join("store"); - let runtime_base = root.path().join("runtime"); + // Bind, durable child metadata, and discovery probes all use this same + // short reference into the tree still owned by root. No cwd mutation. + let runtime_base = Path::new(root.path().file_name().unwrap()).join("runtime"); std::fs::create_dir_all(&runtime_base).unwrap(); - let _runtime_sandbox = RuntimeDirSandbox::new(&runtime_base); let store = FsWorkerStore::new(&store_dir).unwrap(); let session_id = new_session_id(); diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index e04ec36a4..2f3df7f9b 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -20,6 +20,9 @@ mod metadata_lifecycle; pub mod model_client; mod paste_artifact_tool; pub mod prompt; +#[cfg(test)] +#[path = "../../../tests/support/restore_environment.rs"] +mod restore_test_environment; pub mod runtime; pub mod runtime_command; pub mod segment_log_sink; diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 1ef93cd0e..59eb439a7 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -3386,6 +3386,208 @@ impl WorkerStartupFeatureCleanup { } } +/// A close-only candidate resource owner, not filesystem execution authority. +#[derive(Clone)] +pub struct RestoreCandidateWorkdirCleanup { + sessions: Arc, +} + +impl RestoreCandidateWorkdirCleanup { + pub async fn close(&self) -> Result<(), WorkerError> { + self.sessions + .close_all() + .await + .map_err(|error| WorkerError::FeatureLifecycle(error.to_string())) + } +} + +/// An unpublished, read-only reconstruction of an existing materialized Worker. +/// +/// The host must hold its per-Worker operation lock through activation. Snapshot +/// checks reject drift; they do not replace that lock or current host authorization. +/// No controller, allocation, Feature installation, or repair runs during prepare. +/// The public candidate exposes no Worker execution or mutation handles. Binding +/// inside this crate is limited to the open candidate; once discard begins, only +/// cleanup retry remains valid, even if its previous attempt failed or was cancelled. +/// Use `discard` for confirmed cleanup; Drop only schedules best-effort cleanup on +/// the current Tokio runtime and cannot attest that asynchronous cleanup completed. +pub struct PreparedRestoredWorker { + worker: Option, St>>, + closing: bool, + metadata: Option, + entries: serde_json::Value, + manifest: serde_json::Value, +} + +impl PreparedRestoredWorker +where + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, +{ + pub fn manifest(&self) -> &WorkerManifest { + &self.worker().manifest + } + + /// Retain the same Workdir resource owner for explicit cleanup, including + /// after discard fails. This does not grant Worker input or execution access. + pub fn workdir_cleanup(&self) -> RestoreCandidateWorkdirCleanup { + RestoreCandidateWorkdirCleanup { + sessions: self.worker().workdir_sessions.clone(), + } + } + + #[cfg(test)] + fn workdir_sessions(&self) -> Arc { + self.worker().workdir_sessions.clone() + } + + fn worker(&self) -> &Worker, St> { + self.worker + .as_ref() + .expect("restore candidate already discarded") + } + + /// Bootstrap may bind host services before returning an open candidate. + /// Never reopen a candidate whose cleanup has begun. + pub(crate) fn worker_mut( + &mut self, + ) -> Result<&mut Worker, St>, WorkerError> { + if self.closing { + return Err(WorkerError::RestoreCandidateClosing); + } + self.worker + .as_mut() + .ok_or(WorkerError::RestoreCandidateClosing) + } + + fn revalidate(&self, worker: &Worker, St>) -> Result<(), WorkerError> { + let changed = |component| WorkerError::RestoreCandidateChanged { + worker_name: worker.manifest.worker.name.clone(), + component, + }; + if worker.store.read_by_name(&worker.manifest.worker.name)? != self.metadata { + return Err(changed("metadata")); + } + if restore_snapshot_value(&worker.manifest)? != self.manifest { + return Err(changed("manifest")); + } + let entries = worker + .store + .read_all(worker.session_id(), worker.segment_id())?; + if restore_snapshot_value(&entries)? != self.entries { + return Err(changed("segment")); + } + Ok(()) + } + + /// Acquire writer allocation only after exact source validation, validate + /// again under that allocation, then apply the existing restore mutations. + /// An error may retain partial durable effects; ConstructionCleanup carries + /// resource ownership and is not evidence of rollback. + pub async fn activate(self) -> Result, St>, WorkerError> { + self.activate_with_allocation(|worker| { + let socket_path = dir::default_base() + .map_err(ScopeLockError::from)? + .join(&worker.manifest.worker.name) + .join("sock"); + Ok(worker_allocation::install_top_level_with_deny( + worker.manifest.worker.name.clone(), + std::process::id(), + socket_path, + worker.scope.snapshot().allow_rules(), + worker.scope.snapshot().deny_rules(), + worker.segment_id(), + )?) + }) + .await + } + + async fn activate_with_allocation( + mut self, + allocate: impl FnOnce( + &Worker, St>, + ) -> Result, + ) -> Result, St>, WorkerError> { + let mut worker = self + .worker + .take() + .ok_or(WorkerError::RestoreCandidateClosing)?; + if self.closing { + // Preserve the same cleanup owner without validating or acquiring + // allocation for a partially closed candidate. + return Err(worker.construction_failure(WorkerError::RestoreCandidateClosing)); + } + self.revalidate(&worker) + .map_err(|error| worker.construction_failure(error))?; + worker.scope_allocation = + Some(allocate(&worker).map_err(|error| worker.construction_failure(error))?); + self.revalidate(&worker) + .map_err(|error| worker.construction_failure(error))?; + worker + .finish_restored_construction() + .await + .map_err(|source| WorkerError::RestoreActivationEffects { + source: Box::new(source), + }) + } + + /// Directly settle candidate resources, without Controller Shutdown. Failed + /// cleanup retains the candidate so the same owner can retry it. + pub async fn discard(&mut self) -> Result<(), WorkerError> { + self.closing = true; + if let Some(worker) = self.worker.as_mut() { + worker + .retain_startup_feature_cleanup() + .shutdown() + .await + .map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?; + worker + .workdir_sessions + .close_all() + .await + .map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?; + // An unpublished candidate has no allocation guard. Activation moves + // the Worker out and uses ConstructionCleanup on failure; never try + // releasing a cleanup handle while its guard is still attached. + debug_assert!(worker.scope_allocation.is_none()); + } + self.worker = None; + Ok(()) + } +} + +impl Drop for PreparedRestoredWorker { + fn drop(&mut self) { + let Some(worker) = self.worker.as_mut() else { + return; + }; + let sessions = worker.workdir_sessions.clone(); + let features = worker.retain_startup_feature_cleanup(); + // Candidates do not acquire allocation until consuming activation. + debug_assert!(worker.scope_allocation.is_none()); + match tokio::runtime::Handle::try_current() { + Ok(runtime) => { + runtime.spawn(async move { + if let Err(error) = features.shutdown().await { + tracing::warn!(%error, "restore candidate Feature cleanup unconfirmed"); + return; + } + if let Err(error) = sessions.close_all().await { + tracing::warn!(%error, "restore candidate Workdir cleanup unconfirmed"); + } + }); + } + Err(error) => { + tracing::warn!(%error, "restore candidate dropped without async cleanup runtime") + } + } + } +} + +fn restore_snapshot_value(value: &impl serde::Serialize) -> Result { + serde_json::to_value(value) + .map_err(|error| WorkerError::InvalidState(format!("encode restore snapshot: {error}"))) +} + /// An independent agent execution unit. /// /// Holds a [`Engine`] directly and persists session state via @@ -3405,6 +3607,7 @@ pub struct Worker { /// constructors install this from the same FsStore that owns the session /// logs; low-level `Worker::new` tests leave it absent. worker_metadata_writer: Option, + persist_restored_manifest: bool, worker_metadata_segment_cas: Option, /// Shared session pointer. Source of truth for the Worker's current /// `segment_id` and append tally. `self.segment_id()` is a thin @@ -3820,6 +4023,7 @@ impl Worker { last_run_interrupted: false, store, worker_metadata_writer: None, + persist_restored_manifest: false, worker_metadata_segment_cas: None, segment_state: SegmentState::new(session_id, segment_id, 0), filesystem_authority, @@ -4534,14 +4738,21 @@ impl Worker { } fn worker_metadata(&self, active: Option) -> WorkerMetadata { - worker_metadata_for_manifest( + let mut metadata = worker_metadata_for_manifest( &self.manifest, self.workspace_id(), self.filesystem_authority .as_local() .map(|local| local.root.as_path()), active, - ) + ); + // A repaired/restored Session must not lose the saved execution policy + // merely because its current composition also permits legacy local use. + if self.persist_restored_manifest { + metadata.resolved_manifest_snapshot = + manifest::write_persisted_worker_manifest_snapshot(&self.manifest).ok(); + } + metadata } fn write_worker_metadata_pending(&self) -> Result<(), WorkerError> { @@ -8445,6 +8656,7 @@ where last_run_interrupted: false, store, worker_metadata_writer, + persist_restored_manifest: existing_session_id.is_some(), worker_metadata_segment_cas, segment_state: SegmentState::new(session_id, segment_id, 0), filesystem_authority: common.filesystem_authority, @@ -8852,6 +9064,7 @@ where last_run_interrupted: false, store, worker_metadata_writer: None, + persist_restored_manifest: false, worker_metadata_segment_cas: None, segment_state: SegmentState::new(session_id, segment_id, 0), filesystem_authority: common.filesystem_authority, @@ -8987,6 +9200,7 @@ where last_run_interrupted: false, store, worker_metadata_writer, + persist_restored_manifest: true, worker_metadata_segment_cas, segment_state: SegmentState::new(session_id, segment_id, 0), filesystem_authority: common.filesystem_authority, @@ -9080,6 +9294,33 @@ where workspace_context: WorkerWorkspaceContext, filesystem_authority: WorkerFilesystemAuthority, ) -> Result { + Self::prepare_restored_from_worker_metadata_with_context_and_model_client( + worker_name, + manifest, + store, + loader, + workspace_context, + filesystem_authority, + None, + ) + .await? + .activate() + .await + } + + /// Prepare only a materialized saved Worker. Pending Session creation and + /// Subject attribution repair must be performed explicitly by the host + /// before entering this read-only path. + #[allow(clippy::too_many_arguments)] + pub async fn prepare_restored_from_worker_metadata_with_context_and_model_client( + worker_name: &str, + fallback: WorkerManifest, + store: St, + loader: PromptCatalogSource, + workspace_context: WorkerWorkspaceContext, + filesystem_authority: WorkerFilesystemAuthority, + model_client: Option>, + ) -> Result, WorkerError> { let metadata = store .read_by_name(worker_name)? @@ -9093,11 +9334,13 @@ where worker_name: worker_name.to_string(), }); } - let active = metadata - .active - .ok_or_else(|| WorkerError::WorkerMetadataInactive { - worker_name: worker_name.to_string(), - })?; + let active = + metadata + .active + .as_ref() + .ok_or_else(|| WorkerError::WorkerMetadataInactive { + worker_name: worker_name.to_string(), + })?; let segment_id = active .segment_id .ok_or_else(|| WorkerError::WorkerMetadataPending { @@ -9106,10 +9349,15 @@ where })?; let manifest = restore_manifest_from_worker_metadata_snapshot( worker_name, - metadata.resolved_manifest_snapshot, - manifest, + metadata.resolved_manifest_snapshot.clone(), + fallback, )?; - Self::restore_from_manifest_with_context( + if manifest.worker.name != worker_name { + return Err(WorkerError::InvalidState( + "restore manifest Worker name mismatch".into(), + )); + } + Self::prepare_restored_from_manifest_snapshot( active.session_id, segment_id, manifest, @@ -9117,6 +9365,8 @@ where loader, workspace_context, filesystem_authority, + model_client, + Some(metadata), ) .await } @@ -9303,10 +9553,41 @@ where filesystem_authority: WorkerFilesystemAuthority, model_client: Option>, ) -> Result { + let metadata = store.read_by_name(&manifest.worker.name)?; + Self::prepare_restored_from_manifest_snapshot( + session_id, + segment_id, + manifest, + store, + loader, + workspace_context, + filesystem_authority, + model_client, + metadata, + ) + .await? + .activate() + .await + } + + #[allow(clippy::too_many_arguments)] + async fn prepare_restored_from_manifest_snapshot( + session_id: SessionId, + segment_id: SegmentId, + manifest: WorkerManifest, + store: St, + loader: PromptCatalogSource, + workspace_context: WorkerWorkspaceContext, + filesystem_authority: WorkerFilesystemAuthority, + model_client: Option>, + metadata: Option, + ) -> Result, WorkerError> { // Read raw entries once so we can both reconstruct state and // seed the broadcast sink's mirror with the same prefix that // sits on disk. let raw_entries = store.read_all(session_id, segment_id)?; + let entries_snapshot = restore_snapshot_value(&raw_entries)?; + let manifest_snapshot = restore_snapshot_value(&manifest)?; let state = session_store::collect_state(&raw_entries); if state.entries_count == 0 { return Err(WorkerError::SegmentEmpty { segment_id }); @@ -9332,24 +9613,6 @@ where model_client, )?; - // Atomic: register_worker inside install_top_level rejects when - // another live allocation already holds `segment_id`. Wrapping - // the lookup + install inside a single `LockFileGuard` is what - // makes "no two live Workers write to the same session log" - // actually structural rather than a hopeful pre-check. - let socket_path = dir::default_base() - .map_err(ScopeLockError::from)? - .join(&manifest.worker.name) - .join("sock"); - let scope_allocation = worker_allocation::install_top_level_with_deny( - manifest.worker.name.clone(), - std::process::id(), - socket_path, - common.scope.allow_rules(), - common.scope.deny_rules(), - segment_id, - )?; - // Build the worker and apply the manifest defaults first, then // overwrite the pieces the session log is authoritative for. let mut worker = @@ -9398,6 +9661,7 @@ where last_run_interrupted: state.last_run_interrupted, store, worker_metadata_writer, + persist_restored_manifest: true, worker_metadata_segment_cas, segment_state: SegmentState::new(session_id, segment_id, state.entries_count), filesystem_authority: common.filesystem_authority, @@ -9436,7 +9700,7 @@ where pending_attachments: Arc::new(Mutex::new(Vec::::new())), pending_committed_history: Arc::new(Mutex::new(std::collections::VecDeque::new())), failed_activation_authority: Mutex::new(None), - scope_allocation: Some(scope_allocation), + scope_allocation: None, callback_socket: None, runtime_ticket_role: None, prompts: common.prompts, @@ -9461,7 +9725,20 @@ where worker .session .restore_pending_activations(&state.extensions); - worker.finish_restored_construction().await + worker.apply_permissions_from_manifest(); + worker.apply_prune_from_manifest(); + let mut candidate = PreparedRestoredWorker { + worker: Some(worker), + closing: false, + metadata, + entries: entries_snapshot, + manifest: manifest_snapshot, + }; + if let Err(error) = candidate.revalidate(candidate.worker()) { + let worker = candidate.worker.take().expect("candidate just constructed"); + return Err(worker.construction_failure(error)); + } + Ok(candidate) } async fn finish_restored_construction(mut self) -> Result { @@ -10329,6 +10606,23 @@ pub enum WorkerError { sessions: Arc, }, + /// Activation entered canonical pin/metadata/delegation updates. Cleanup + /// ownership in the source does not establish rollback of those effects. + #[error("{source}")] + RestoreActivationEffects { + #[source] + source: Box, + }, + + #[error("restore candidate is closing; only cleanup retry is allowed")] + RestoreCandidateClosing, + + #[error("restore candidate source changed for Worker {worker_name}: {component}")] + RestoreCandidateChanged { + worker_name: String, + component: &'static str, + }, + #[error("invalid durable Worker state: {0}")] InvalidState(String), @@ -12998,6 +13292,686 @@ mod build_summary_prompt_tests { assert!(table.data().find("test-worker").is_none()); } + // Real Session/metadata stores, allocation and Workdir providers; only the + // model transport is substituted. Profile/Runtime publication belongs to the + // factory integration suite, not this reconstruction component suite. + mod restored_candidate_tests { + use super::*; + + type CandidateStore = + session_store::CombinedStore; + + #[derive(Clone)] + struct CountedClient(Arc); + + #[async_trait] + impl LlmClient for CountedClient { + async fn stream( + &self, + request: agen::llm_client::Request, + ) -> Result< + std::pin::Pin< + Box< + dyn futures::Stream< + Item = Result< + agen::llm_client::event::Event, + agen::llm_client::ClientError, + >, + > + Send, + >, + >, + agen::llm_client::ClientError, + > { + self.0.fetch_add(1, Ordering::SeqCst); + NoopClient.stream(request).await + } + + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + } + + #[derive(Debug)] + struct FailCloseOnce { + inner: workdir::WorkdirSessionHandle, + attempts: AtomicUsize, + } + + #[async_trait] + impl workdir::WorkdirSession for FailCloseOnce { + fn workdir(&self) -> &workdir::Workdir { + self.inner.workdir() + } + fn capabilities(&self) -> workdir::WorkdirSessionCapabilities { + self.inner.capabilities() + } + async fn stat( + &self, + request: workdir::StatRequest, + ) -> Result { + self.inner.stat(request).await + } + async fn read( + &self, + request: workdir::ReadRequest, + ) -> Result { + self.inner.read(request).await + } + async fn write( + &self, + request: workdir::WriteRequest, + ) -> Result { + self.inner.write(request).await + } + async fn edit( + &self, + request: workdir::EditRequest, + ) -> Result { + self.inner.edit(request).await + } + async fn list( + &self, + request: workdir::ListRequest, + ) -> Result { + self.inner.list(request).await + } + async fn glob( + &self, + request: workdir::GlobRequest, + ) -> Result { + self.inner.glob(request).await + } + async fn grep( + &self, + request: workdir::GrepRequest, + ) -> Result { + self.inner.grep(request).await + } + async fn start_command( + &self, + request: workdir::CommandRequest, + ) -> Result { + self.inner.start_command(request).await + } + async fn command_status( + &self, + handle: workdir::CommandHandle, + ) -> Result { + self.inner.command_status(handle).await + } + async fn command_output( + &self, + request: workdir::CommandOutputRequest, + ) -> Result { + self.inner.command_output(request).await + } + async fn cancel_command( + &self, + handle: workdir::CommandHandle, + ) -> Result<(), workdir::WorkdirError> { + self.inner.cancel_command(handle).await + } + async fn close(&self) -> Result<(), workdir::WorkdirError> { + self.inner.close().await?; + if self.attempts.fetch_add(1, Ordering::SeqCst) == 0 { + // The real provider has closed, but its first response is + // uncertain: the candidate must not become activatable again. + Err(workdir::WorkdirError::Unavailable( + "close response lost".into(), + )) + } else { + Ok(()) + } + } + } + + struct Fixture { + temp: tempfile::TempDir, + store: CandidateStore, + manifest: WorkerManifest, + session_id: SessionId, + segment_id: SegmentId, + orphan: protocol::UploadedFileRef, + model_calls: Arc, + } + + impl Fixture { + async fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let store = session_store::CombinedStore::new( + session_store::FsStore::new(temp.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(temp.path().join("workers")).unwrap(), + ); + let mut manifest = minimal_manifest(); + manifest.scope = ScopeConfig { + allow: vec![ScopeRule { + target: temp.path().canonicalize().unwrap(), + permission: Permission::Write, + recursive: true, + symlink_policy: Default::default(), + }], + deny: Vec::new(), + }; + let model_calls = Arc::new(AtomicUsize::new(0)); + let mut worker = Worker::new( + manifest.clone(), + Engine::, Mutable, SessionHistoryMetadata>::new_annotated( + Box::new(CountedClient(model_calls.clone())), + ), + store.clone(), + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + Scope::empty(), + ) + .await + .unwrap(); + worker + .engine_mut() + .set_system_prompt("saved candidate prompt"); + worker.ensure_segment_head().await.unwrap(); + let session_id = worker.session_id(); + let segment_id = worker.segment_id(); + let queue = worker.pending_submission_handle(); + queue + .accept( + "saved-submit".into(), + vec![Segment::Text { + content: "saved input".into(), + }], + false, + ) + .unwrap(); + queue + .accept_notification("saved-notification".into(), "saved notice".into()) + .unwrap(); + let orphan = store + .write_uploaded_file( + session_id, + "orphan.txt", + "text/plain", + b"orphan", + session_store::UploadedFileLimits::default(), + ) + .unwrap(); + store + .pin_uploaded_file(session_id, &orphan, "acceptance-without-checkpoint") + .unwrap(); + let mut metadata = WorkerMetadata::new( + &manifest.worker.name, + Some(WorkerActiveSegmentRef { + session_id, + segment_id: Some(segment_id), + }), + ); + // Activation replaces this stale derived path; preparation must not. + metadata.workspace_root = Some(temp.path().join("old-root")); + metadata.spawned_children.push(WorkerSpawnedChild { + worker_name: "missing-child".into(), + socket_path: temp.path().join("missing-child.sock"), + scope_delegated: Vec::new(), + callback_address: temp.path().join("unused-parent.sock"), + }); + metadata.resolved_manifest_snapshot = + Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap()); + store.write(&metadata).unwrap(); + drop(worker); + Self { + temp, + store, + manifest, + session_id, + segment_id, + orphan, + model_calls, + } + } + + async fn prepare(&self) -> PreparedRestoredWorker { + Worker::prepare_restored_from_worker_metadata_with_context_and_model_client( + &self.manifest.worker.name, + self.manifest.clone(), + self.store.clone(), + PromptCatalogSource::builtins_only(), + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::local( + self.temp.path().to_path_buf(), + self.temp.path().to_path_buf(), + ), + Some(Box::new(CountedClient(self.model_calls.clone()))), + ) + .await + .unwrap() + } + + fn allocate( + &self, + worker: &Worker, CandidateStore>, + ) -> Result { + Ok(worker_allocation::install_top_level_at_for_test( + self.temp.path().join("allocation.json"), + worker.manifest.worker.name.clone(), + std::process::id(), + self.temp.path().join("worker.sock"), + worker.scope.snapshot().allow_rules(), + worker.segment_id(), + )?) + } + + fn snapshot(&self) -> std::collections::BTreeMap> { + fn visit( + root: &Path, + path: &Path, + files: &mut std::collections::BTreeMap>, + ) { + for entry in std::fs::read_dir(path).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + visit(root, &path, files); + } else { + files.insert( + path.strip_prefix(root).unwrap().to_path_buf(), + std::fs::read(path).unwrap(), + ); + } + } + } + let mut files = std::collections::BTreeMap::new(); + for name in ["sessions", "workers"] { + visit(self.temp.path(), &self.temp.path().join(name), &mut files); + } + files + } + } + + #[tokio::test] + async fn prepare_and_discard_preserve_session_queue_pins_and_metadata() { + let fixture = Fixture::new().await; + let before = fixture.snapshot(); + let mut candidate = fixture.prepare().await; + assert!( + candidate + .worker() + .restore_allocation_cleanup_handle() + .is_none() + ); + assert!(!fixture.temp.path().join("worker.sock").exists()); + assert_eq!( + candidate.worker().pending_submissions().submissions.len(), + 1 + ); + assert_eq!( + candidate + .worker() + .session + .pending_activations + .lock() + .unwrap() + .pending_notifications + .len(), + 1 + ); + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + let session = candidate + .worker() + .workdir_sessions() + .resolve(None) + .unwrap() + .session; + assert!( + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root() + }) + .await + .is_ok() + ); + candidate.discard().await.unwrap(); + candidate.discard().await.unwrap(); + assert!( + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root() + }) + .await + .is_err() + ); + assert_eq!(fixture.snapshot(), before); + } + + #[tokio::test] + async fn failed_discard_allows_only_cleanup_retry_not_binding_or_activation() { + for retry_discard in [true, false] { + let fixture = Fixture::new().await; + let before = fixture.snapshot(); + let mut candidate = fixture.prepare().await; + let provider = Arc::new(FailCloseOnce { + inner: candidate.workdir_sessions().resolve(None).unwrap().session, + attempts: AtomicUsize::new(0), + }); + candidate + .worker_mut() + .unwrap() + .bind_single_workdir_session(Some(provider.clone())); + let cleanup = candidate.workdir_sessions(); + assert!(matches!( + candidate.discard().await, + Err(WorkerError::FeatureLifecycle(_)) + )); + assert_eq!(provider.attempts.load(Ordering::SeqCst), 1); + assert!(!cleanup.is_empty()); + assert_eq!(fixture.snapshot(), before); + assert!(matches!( + candidate.worker_mut(), + Err(WorkerError::RestoreCandidateClosing) + )); + assert_eq!( + candidate.manifest().worker.name, + fixture.manifest.worker.name + ); + assert!(Arc::ptr_eq(&candidate.workdir_sessions(), &cleanup)); + if retry_discard { + candidate.discard().await.unwrap(); + assert!(cleanup.is_empty()); + assert_eq!(provider.attempts.load(Ordering::SeqCst), 2); + assert!(matches!( + candidate.worker_mut(), + Err(WorkerError::RestoreCandidateClosing) + )); + candidate.discard().await.unwrap(); + } else { + let allocations = AtomicUsize::new(0); + let error = match candidate + .activate_with_allocation(|worker| { + allocations.fetch_add(1, Ordering::SeqCst); + fixture.allocate(worker) + }) + .await + { + Err(error) => error, + Ok(_) => panic!("partially closed candidate was activated"), + }; + let WorkerError::ConstructionCleanup { + source, + allocation, + sessions, + } = error + else { + panic!("cleanup ownership lost"); + }; + assert!(matches!(*source, WorkerError::RestoreCandidateClosing)); + assert!(allocation.is_none()); + assert_eq!(allocations.load(Ordering::SeqCst), 0); + assert_eq!(provider.attempts.load(Ordering::SeqCst), 1); + assert!(Arc::ptr_eq(&sessions, &cleanup)); + sessions.close_all().await.unwrap(); + assert!(cleanup.is_empty()); + assert_eq!(provider.attempts.load(Ordering::SeqCst), 2); + } + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + } + } + + #[tokio::test] + async fn dropping_candidate_closes_workdir_without_consuming_saved_inputs() { + let fixture = Fixture::new().await; + let before = fixture.snapshot(); + let candidate = fixture.prepare().await; + let sessions = candidate.worker().workdir_sessions(); + let session = sessions.resolve(None).unwrap().session; + drop(candidate); + tokio::time::timeout(Duration::from_secs(5), async { + while !sessions.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("candidate Drop must finish direct close"); + assert!( + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::root() + }) + .await + .is_err() + ); + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn activation_alone_reconciles_orphan_pins_and_updates_active_metadata() { + let fixture = Fixture::new().await; + let before_metadata = fixture + .store + .read_by_name(&fixture.manifest.worker.name) + .unwrap() + .unwrap(); + let before_entries = restore_snapshot_value( + &fixture + .store + .read_all(fixture.session_id, fixture.segment_id) + .unwrap(), + ) + .unwrap(); + let candidate = fixture.prepare().await; + assert!(matches!( + fixture + .store + .delete_uploaded_file(fixture.session_id, &fixture.orphan.artifact_id), + Err(StoreError::ArtifactAlreadyCommitted) + )); + let worker = candidate + .activate_with_allocation(|worker| fixture.allocate(worker)) + .await + .unwrap(); + assert!(worker.restore_allocation_cleanup_handle().is_some()); + let metadata = fixture + .store + .read_by_name(&fixture.manifest.worker.name) + .unwrap() + .unwrap(); + assert_ne!(metadata.workspace_root, before_metadata.workspace_root); + assert_eq!(metadata.active, before_metadata.active); + assert_eq!(before_metadata.spawned_children.len(), 1); + assert!(metadata.spawned_children.is_empty()); + assert_eq!(metadata.reclaimed_children.len(), 1); + assert!( + fixture + .store + .delete_uploaded_file(fixture.session_id, &fixture.orphan.artifact_id) + .unwrap() + ); + assert_eq!(worker.pending_submissions().submissions.len(), 1); + assert_eq!( + worker + .session + .pending_activations + .lock() + .unwrap() + .pending_notifications + .len(), + 1 + ); + assert_eq!( + restore_snapshot_value( + &fixture + .store + .read_all(fixture.session_id, fixture.segment_id) + .unwrap() + ) + .unwrap(), + before_entries + ); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + worker.workdir_sessions().close_all().await.unwrap(); + let allocation = worker.restore_allocation_cleanup_handle().unwrap(); + assert!(matches!( + allocation.release(), + Err(ScopeLockError::AllocationGuardAttached { .. }) + )); + drop(worker); + allocation.release().unwrap(); + } + + #[tokio::test] + async fn metadata_or_same_entry_count_history_drift_rejects_before_allocation() { + for component in ["metadata", "segment", "manifest"] { + let fixture = Fixture::new().await; + let mut candidate = fixture.prepare().await; + match component { + "metadata" => { + fixture + .store + .update_by_name(&fixture.manifest.worker.name, |metadata| { + metadata.workspace_id = Some("changed-workspace".into()); + }) + .unwrap(); + } + "segment" => { + let mut entries = fixture + .store + .read_all(fixture.session_id, fixture.segment_id) + .unwrap(); + let LogEntry::AnnotatedSegmentStart { system_prompt, .. } = &mut entries[0] + else { + panic!("fixture seed"); + }; + *system_prompt = Some("different same-length history".into()); + fixture + .store + .create_segment(fixture.session_id, fixture.segment_id, &entries) + .unwrap(); + } + "manifest" => { + candidate.worker_mut().unwrap().manifest.model.model_id = + Some("changed-model".into()); + } + _ => unreachable!(), + } + let before = fixture.snapshot(); + let allocations = AtomicUsize::new(0); + let error = match candidate + .activate_with_allocation(|worker| { + allocations.fetch_add(1, Ordering::SeqCst); + fixture.allocate(worker) + }) + .await + { + Err(error) => error, + Ok(_) => panic!("stale candidate accepted"), + }; + let WorkerError::ConstructionCleanup { + source, + allocation, + sessions, + } = error + else { + panic!("cleanup ownership lost"); + }; + assert!( + matches!(*source, WorkerError::RestoreCandidateChanged { component: changed, .. } if changed == component) + ); + assert!(allocation.is_none()); + assert_eq!(allocations.load(Ordering::SeqCst), 0); + sessions.close_all().await.unwrap(); + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + } + } + + #[tokio::test] + async fn post_allocation_metadata_drift_retains_cleanup_without_mutating_pins() { + let fixture = Fixture::new().await; + let candidate = fixture.prepare().await; + let error = match candidate + .activate_with_allocation(|worker| { + let guard = fixture.allocate(worker)?; + fixture + .store + .update_by_name(&fixture.manifest.worker.name, |metadata| { + metadata.workspace_id = Some("changed-under-allocation".into()); + })?; + Ok(guard) + }) + .await + { + Err(error) => error, + Ok(_) => panic!("stale candidate accepted"), + }; + let before = fixture.snapshot(); + let WorkerError::ConstructionCleanup { + source, + allocation, + sessions, + } = error + else { + panic!("cleanup ownership lost"); + }; + assert!(matches!( + *source, + WorkerError::RestoreCandidateChanged { + component: "metadata", + .. + } + )); + let allocation = + allocation.expect("allocation retained after second validation failure"); + assert!( + worker_allocation::LockFileGuard::open( + &fixture.temp.path().join("allocation.json") + ) + .unwrap() + .data() + .find(&fixture.manifest.worker.name) + .is_some() + ); + assert!(matches!( + fixture + .store + .delete_uploaded_file(fixture.session_id, &fixture.orphan.artifact_id), + Err(StoreError::ArtifactAlreadyCommitted) + )); + sessions.close_all().await.unwrap(); + worker_allocation::fail_next_save_for_test( + &fixture.temp.path().join("allocation.json"), + ); + assert!(allocation.release().is_err()); + allocation.release().unwrap(); + assert_eq!(fixture.snapshot(), before); + } + + #[tokio::test] + async fn pending_session_is_rejected_without_repair_or_session_creation() { + let fixture = Fixture::new().await; + fixture + .store + .update_by_name(&fixture.manifest.worker.name, |metadata| { + metadata.active.as_mut().unwrap().segment_id = None; + }) + .unwrap(); + let before = fixture.snapshot(); + let result = + Worker::prepare_restored_from_worker_metadata_with_context_and_model_client( + &fixture.manifest.worker.name, + fixture.manifest.clone(), + fixture.store.clone(), + PromptCatalogSource::builtins_only(), + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + Some(Box::new(CountedClient(fixture.model_calls.clone()))), + ) + .await; + assert!(matches!( + result, + Err(WorkerError::WorkerMetadataPending { .. }) + )); + assert_eq!(fixture.snapshot(), before); + assert_eq!(fixture.model_calls.load(Ordering::SeqCst), 0); + } + } + // Constructor finalization uses real Worker, allocation, Workdir, and storage // implementations. Only the model and metadata-write failure boundary are substituted. mod construction_cleanup_tests { diff --git a/crates/worker/tests/controller_test.rs b/crates/worker/tests/controller_test.rs index 1009b79fd..2fe26e3d6 100644 --- a/crates/worker/tests/controller_test.rs +++ b/crates/worker/tests/controller_test.rs @@ -1,3 +1,4 @@ +use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -150,6 +151,19 @@ impl LlmClient for MockClient { // Helpers // --------------------------------------------------------------------------- +#[path = "../../../tests/support/restore_environment.rs"] +mod restore_test_environment; +use restore_test_environment::require_restore_test_environment; + +fn owned_relative_runtime_base() -> (tempfile::TempDir, PathBuf) { + require_restore_test_environment(); + let temp = tempfile::tempdir_in(".").unwrap(); + // The relative reference and the TempDir name the same owned tree, without + // putting the checkout prefix into sockaddr_un or changing process cwd. + let runtime_base = Path::new(temp.path().file_name().unwrap()).join("runtime"); + (temp, runtime_base) +} + fn simple_text_events() -> Vec { vec![ LlmEvent::text_block_start(0), @@ -295,6 +309,7 @@ async fn make_worker_with_manifest_and_workspace_context( manifest: WorkerManifest, workspace_context: WorkerWorkspaceContext, ) -> (Worker, std::path::PathBuf) { + require_restore_test_environment(); let store_tmp = tempfile::tempdir().unwrap(); let store = CombinedStore::new( FsStore::new(store_tmp.path()).unwrap(), @@ -320,18 +335,21 @@ async fn make_worker_with_manifest_and_workspace_context( } async fn spawn_controller(worker: Worker) -> WorkerHandle { - let tmp = tempfile::tempdir().unwrap(); - let runtime_base = tmp.path().to_owned(); - std::mem::forget(tmp); - let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base) + let (tmp, runtime_base) = owned_relative_runtime_base(); + let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base) .await .unwrap(); + // Retain ownership until controller shutdown (or test-runtime teardown). + tokio::spawn(async move { + let _ = shutdown_rx.await; + drop(tmp); + }); handle } #[tokio::test] async fn canonical_worker_handle_uploads_and_submits_attachment() { - let root = tempfile::tempdir().unwrap(); + let (root, runtime_base) = owned_relative_runtime_base(); let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap(); let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap(); let store = CombinedStore::new( @@ -353,7 +371,7 @@ async fn canonical_worker_handle_uploads_and_submits_attachment() { ) .await .unwrap(); - let (handle, shutdown_rx) = WorkerController::spawn(worker, &root.path().join("runtime")) + let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base) .await .unwrap(); let mut rx = handle.subscribe(); @@ -1761,11 +1779,12 @@ permission = "write" "#; let client = MockClient::new(simple_text_events()); let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0; - let tmp = tempfile::tempdir().unwrap(); - let result = WorkerController::spawn(worker, tmp.path()).await; + let (_tmp, runtime_base) = owned_relative_runtime_base(); + let result = WorkerController::spawn(worker, &runtime_base).await; assert!( result.is_ok(), - "feature exposure must not imply delegation authority" + "feature exposure must not imply delegation authority: {:?}", + result.as_ref().err() ); } diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index 02c40beb4..dd117ae7b 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -19,6 +19,9 @@ pub mod repositories; pub mod repository_access; pub mod repository_source; pub mod resource_broker; +#[cfg(test)] +#[path = "../../../tests/support/restore_environment.rs"] +mod restore_test_environment; pub mod retention; pub mod runtime_settings; pub mod runtime_subscription; diff --git a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs index 91093f5fc..d1a4d6c61 100644 --- a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs +++ b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs @@ -11,6 +11,20 @@ use worker_runtime::worker_backend::{ WorkerRuntimeExecutionBackend, }; +pub(super) fn connected_restore_tempdir() -> tempfile::TempDir { + crate::restore_test_environment::require_restore_test_environment(); + let checkout = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap(); + tempfile::Builder::new() + .prefix("c") + .rand_bytes(3) + .tempdir_in(checkout.join("target")) + .unwrap() +} + const WATCHDOG: Duration = Duration::from_secs(45); #[derive(Clone)] @@ -82,27 +96,91 @@ impl RuntimeWorkerFactory for CapturingFactory { ) -> std::result::Result<(), String> { self.inner.preflight_restore(request).await } - async fn restore_controller( + async fn recover_restore_source( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.inner.recover_restore_source(request).await + } + + async fn prepare_restore( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + let worker_ref = request.worker_ref.clone(); + let inner = self.inner.prepare_restore(request).await?; + Ok(Box::new(CapturingCandidate { + inner, + worker_ref, + captures: self.captures.clone(), + })) + } + + async fn cleanup_failed_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.cleanup_failed_restore(request).await + } + + async fn reconcile_stopped_worker( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.reconcile_stopped_worker(worker_ref).await + } + + fn activate_restored_controller( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + workspace_id: Option<&str>, + handle: &worker::WorkerHandle, + ) { + self.inner + .activate_restored_controller(worker_ref, workspace_id, handle); + } +} + +struct CapturingCandidate { + inner: Box, + worker_ref: worker_runtime::identity::WorkerRef, + captures: Arc, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for CapturingCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.inner.initial_state() + } + + async fn activate( + &mut self, ) -> std::result::Result< RuntimeWorkerController, worker_runtime::worker_backend::RuntimeRestoreFailure, > { - let worker_ref = request.worker_ref.clone(); - let controller = self.inner.restore_controller(request).await?; + let controller = self.inner.activate().await?; self.captures .handles .lock() .unwrap() - .push((worker_ref.clone(), controller.handle.clone())); + .push((self.worker_ref.clone(), controller.handle.clone())); self.captures .restored_clients .lock() .unwrap() - .push((worker_ref, controller.workspace_client.clone())); + .push((self.worker_ref.clone(), controller.workspace_client.clone())); Ok(controller) } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.discard().await + } } struct Resources { @@ -262,7 +340,7 @@ async fn call(tools: &BTreeMap>, name: &str, args: Value) #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn restored_runtime_parent_lists_and_inputs_existing_child() { - let root = tempfile::tempdir().unwrap(); + let root = connected_restore_tempdir(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let base = format!("http://{}", listener.local_addr().unwrap()); let mut config = test_server_config(root.path()); diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs index 4cc4b5c24..3b9215113 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs @@ -11,6 +11,8 @@ use agen::llm_client::{ use serde_json::{Value, json}; use std::sync::{Arc, Mutex}; +use super::super::restored_parent_control_tests::connected_restore_tempdir; + #[derive(Clone, Default)] struct ConnectedJournalModel { dialogs: Arc>>, @@ -320,7 +322,7 @@ async fn connected_wait_for_recording( #[tokio::test(flavor = "multi_thread", worker_threads = 6)] async fn backend_real_runtime_journal_release_and_next_body_persisted_startup() { - let dir = tempfile::tempdir().unwrap(); + let dir = connected_restore_tempdir(); init_clean_git_workspace(dir.path()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let mut config = test_server_config(dir.path()); @@ -896,3 +898,1579 @@ fn connected_host_reader( ) .unwrap() } + +// This wrapper changes timing only. Production owns reconstruction, recovery, +// activation and cleanup; the real SQLite trust binding supplies the denial. +#[derive(Default)] +struct RestoreAuthProbe { + prepared: tokio::sync::Notify, + release: tokio::sync::Notify, + prepares: std::sync::atomic::AtomicUsize, + activations: std::sync::atomic::AtomicUsize, + publications: std::sync::atomic::AtomicUsize, + handles: Mutex>, + controller_tasks: Mutex>, + operations: Mutex>, + repairs: Mutex>, + repair_operations: Mutex>, + gate_repair: std::sync::atomic::AtomicBool, + repairing: tokio::sync::Notify, + release_repair: tokio::sync::Notify, +} + +struct RestoreAuthFactory { + inner: worker_runtime::worker_backend::ProfileRuntimeWorkerFactory, + probe: Arc, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeWorkerFactory for RestoreAuthFactory { + fn observe_workspace_prompt_projection( + &self, + projection: worker::WorkspacePromptProjection, + ) -> std::result::Result<(), String> { + self.inner.observe_workspace_prompt_projection(projection) + } + + async fn fetch_workspace_config( + &self, + request: worker_runtime::execution::WorkspaceConfigFetchRequest, + ) -> std::result::Result { + self.inner.fetch_workspace_config(request).await + } + + fn retained_session_snapshot( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result< + session_store::RetainedSessionSnapshot, + session_store::RetainedSnapshotReadError, + > { + self.inner.retained_session_snapshot(worker_ref) + } + + fn retained_session_history_page( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + cursor: Option<&str>, + limit: Option, + ) -> std::result::Result + { + self.inner + .retained_session_history_page(worker_ref, cursor, limit) + } + + fn retained_session_attachment( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + session_id: &str, + attachment_id: &str, + ) -> std::result::Result< + session_store::RetainedSessionAttachment, + session_store::RetainedAttachmentReadError, + > { + self.inner + .retained_session_attachment(worker_ref, session_id, attachment_id) + } + + async fn spawn_controller( + &self, + mut request: worker_runtime::execution::WorkerExecutionSpawnRequest, + ) -> std::result::Result { + if request.request.backend_job.is_some() { + return self.inner.spawn_controller(request).await; + } + // Save a no-auth model too, so a restore regression cannot reach developer + // credentials. Only the external model stream is replaced, not Host I/O. + let archive = worker_runtime::profile_archive::ProfileSourceArchive::build( + worker_runtime::profile_archive::ProfileSourceArchiveInput { + id: "profile-source-archive:restore-host-auth".into(), + entrypoints: BTreeMap::from([("builtin:companion".into(), "profiles/body.dcdl".into())]), + imports: BTreeMap::new(), + sources: BTreeMap::from([("profiles/body.dcdl".into(), r#"{ + slug = "body"; description = "Connected restore Host authority"; scope = "workspace_read"; + model = { scheme = "anthropic"; model_id = "test-model"; auth = { kind = "none"; }; }; + engine = { max_tokens = 100; }; + feature = { subjektiv = { enabled = true; extraction = { enabled = false; }; }; }; + }"#.into())]), + }, + ).map_err(|error| error.to_string())?; + request.request.profile_source = + worker_runtime::catalog::ProfileSourceArchiveSource::Embedded { archive }; + let controller = self.inner.spawn_controller(request).await?; + self.probe + .controller_tasks + .lock() + .unwrap() + .push(controller.controller_task.abort_handle()); + self.probe + .handles + .lock() + .unwrap() + .push(controller.handle.clone()); + Ok(controller) + } + + async fn preflight_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), String> { + self.inner.preflight_restore(request).await + } + + async fn recover_restore_source( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.probe + .repair_operations + .lock() + .unwrap() + .push(serde_json::to_value(request.operation_id).unwrap()); + if self + .probe + .gate_repair + .swap(false, std::sync::atomic::Ordering::SeqCst) + { + self.probe.repairing.notify_one(); + self.probe.release_repair.notified().await; + } + let repaired = self.inner.recover_restore_source(request).await?; + self.probe.repairs.lock().unwrap().push(repaired); + Ok(repaired) + } + + async fn prepare_restore( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + let operation = serde_json::to_value(request.operation_id).unwrap(); + let inner = self.inner.prepare_restore(request).await?; + self.probe.operations.lock().unwrap().push(operation); + if self + .probe + .prepares + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + == 0 + { + self.probe.prepared.notify_one(); + self.probe.release.notified().await; + } + Ok(Box::new(RestoreAuthCandidate { + inner, + probe: self.probe.clone(), + })) + } + + async fn cleanup_failed_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.cleanup_failed_restore(request).await + } + + async fn reconcile_stopped_worker( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.reconcile_stopped_worker(worker_ref).await + } + + fn activate_restored_controller( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + workspace_id: Option<&str>, + handle: &worker::WorkerHandle, + ) { + self.inner + .activate_restored_controller(worker_ref, workspace_id, handle); + } +} + +struct RestoreAuthCandidate { + inner: Box, + probe: Arc, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for RestoreAuthCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.inner.initial_state() + } + + async fn activate( + &mut self, + ) -> std::result::Result< + worker_runtime::worker_backend::RuntimeWorkerController, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + self.probe + .activations + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let controller = self.inner.activate().await?; + self.probe + .publications + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.probe + .controller_tasks + .lock() + .unwrap() + .push(controller.controller_task.abort_handle()); + self.probe + .handles + .lock() + .unwrap() + .push(controller.handle.clone()); + Ok(controller) + } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.discard().await + } +} + +#[derive(Clone, Default)] +struct RestoreAuthModel { + calls: Arc, + jobs: ConnectedJournalModel, +} + +#[async_trait::async_trait] +impl LlmClient for RestoreAuthModel { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + + async fn stream( + &self, + request: Request, + ) -> std::result::Result { + use futures::StreamExt; + if request.tools.iter().any(|tool| { + matches!( + tool.name.as_str(), + "ReadJournalState" | "SubmitBackendJobResult" | "SubmitMemorySurface" + ) + }) { + return self.jobs.stream(request).await; + } + // Measure the saved body, not independent Journal jobs admitted by its + // seed turn before Stop; those have a different execution owner. + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(Box::pin( + futures::stream::iter([ + Ok(Event::text_block_start(0)), + Ok(Event::text_delta( + 0, + "fixture turn held for durable queue setup", + )), + ]) + .chain(futures::stream::pending()), + )) + } +} + +struct ConnectedRestoreServer { + server: tokio::task::JoinHandle<()>, + probe: Arc, +} + +impl Drop for ConnectedRestoreServer { + fn drop(&mut self) { + // Unblock a prepared candidate even if an assertion unwinds at the gate. + self.probe.release.notify_one(); + self.probe.release_repair.notify_one(); + self.server.abort(); + for task in self.probe.controller_tasks.lock().unwrap().iter() { + task.abort(); + } + } +} + +async fn restore_auth_event( + events: &mut tokio::sync::broadcast::Receiver, + predicate: impl Fn(&protocol::Event) -> bool, +) -> protocol::Event { + tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let event = events.recv().await.expect("real Controller event stream"); + assert!( + !matches!( + event, + protocol::Event::Error { .. } + | protocol::Event::SubmissionRejected { .. } + | protocol::Event::NotificationRejected { .. } + ), + "{event:?}" + ); + if predicate(&event) { + return event; + } + } + }) + .await + .expect("real Controller acknowledgement timed out") +} + +// Only initial creation is substituted for the pending-head case: it writes a +// genuine first-Session Worker, but publishes no Controller/head. Every restore, +// repair, Host authorization, activation and Stop delegates to production. +struct PendingHeadSeedBackend { + inner: Arc>, + seed: std::sync::atomic::AtomicBool, + root: std::path::PathBuf, + model: RestoreAuthModel, +} + +impl worker_runtime::execution::WorkerExecutionBackend for PendingHeadSeedBackend { + fn backend_id(&self) -> &str { + self.inner.backend_id() + } + fn spawn_worker( + &self, + request: worker_runtime::execution::WorkerExecutionSpawnRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + if !self.seed.swap(false, std::sync::atomic::Ordering::SeqCst) { + return self.inner.spawn_worker(request); + } + let root = self + .root + .join("workers") + .join(request.worker_ref.worker_id.to_string()); + let name = format!("worker-runtime-{}", request.worker_ref.worker_id); + let model = self.model.clone(); + let settings = request.request.memory_settings.clone().unwrap(); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + use session_store::WorkerMetadataStore as _; + let mut manifest = manifest::WorkerManifest::from_toml(&format!( + r#" +[worker] +name = "{name}" +pwd = "./" +[scope] +allow = [] +[model] +scheme = "anthropic" +model_id = "test-model" +auth = {{ kind = "none" }} +[engine] +max_tokens = 100 +[feature.subjektiv.profile] +enabled = true +[feature.subjektiv.profile.extraction] +enabled = false +"# + )) + .unwrap(); + manifest + .feature + .subjektiv + .bind_workspace_settings(settings) + .unwrap(); + let sessions = + session_store::WorkerSessionStore::new(root.join("session")).unwrap(); + let metadata = session_store::WorkerAggregateStore::new(&root, &name).unwrap(); + let engine = agen::Engine::< + _, + agen::state::Mutable, + worker::SessionHistoryMetadata, + >::new_annotated(model); + let mut head = worker::Worker::new( + manifest.clone(), + engine, + session_store::CombinedStore::new(sessions.clone(), metadata.clone()), + worker::WorkerWorkspaceContext::unavailable( + Some(worker::WorkspaceId::new(TEST_WORKSPACE_ID).unwrap()), + "initial creation interrupted before first head", + ), + worker::WorkerFilesystemAuthority::None, + manifest::Scope::empty(), + ) + .await + .unwrap(); + head.enable_worker_metadata_write_through().unwrap(); + let mut saved = metadata.read_by_name(&name).unwrap().unwrap(); + assert!(saved.active.as_ref().unwrap().segment_id.is_none()); + saved.resolved_manifest_snapshot = Some( + manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap(), + ); + metadata.write(&saved).unwrap(); + drop(head); + }); + }) + .join() + .unwrap(); + worker_runtime::execution::WorkerExecutionSpawnResult::Connected { + worker_state: protocol::WorkerStateSnapshot::initial(), + workdir_attachments: Vec::new(), + } + } + fn fetch_workspace_config( + &self, + request: worker_runtime::execution::WorkspaceConfigFetchRequest, + ) -> std::result::Result { + self.inner.fetch_workspace_config(request) + } + fn observe_workspace_prompt_projection( + &self, + projection: worker::WorkspacePromptProjection, + ) -> std::result::Result<(), String> { + self.inner.observe_workspace_prompt_projection(projection) + } + fn preflight_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), worker_runtime::execution::WorkerExecutionResult> { + self.inner.preflight_restore(request) + } + fn completed_restore_failure( + &self, + id: worker_runtime::execution::WorkerLifecycleOperationId, + worker: &worker_runtime::identity::WorkerRef, + ) -> Option { + self.inner.completed_restore_failure(id, worker) + } + fn restore_worker( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.inner.restore_worker(request) + } + fn reconcile_restore( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.inner.reconcile_restore(request) + } + fn activate_restored_worker( + &self, + id: worker_runtime::execution::WorkerLifecycleOperationId, + worker: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.activate_restored_worker(id, worker) + } + fn dispatch_input( + &self, + worker: &worker_runtime::identity::WorkerRef, + input: worker_runtime::interaction::WorkerInput, + ) -> worker_runtime::execution::WorkerExecutionResult { + self.inner.dispatch_input(worker, input) + } + fn dispatch_method( + &self, + worker: &worker_runtime::identity::WorkerRef, + method: protocol::Method, + ) -> worker_runtime::execution::WorkerExecutionResult { + self.inner.dispatch_method(worker, method) + } + fn stop_worker_operation( + &self, + request: worker_runtime::execution::WorkerExecutionStopRequest, + ) -> worker_runtime::execution::WorkerExecutionResult { + self.inner.stop_worker_operation(request) + } + fn worker_snapshot( + &self, + worker: &worker_runtime::identity::WorkerRef, + ) -> Option { + self.inner.worker_snapshot(worker) + } +} + +// One real connected seed and stopped queue setup shared by the publication +// denial and explicit-repair regressions. Only timing and response delivery are +// intercepted: every callback still passes the production signer/router/store. +struct ConnectedRestoreFixture { + api: WorkspaceApi, + execution: + Arc>, + probe: Arc, + model: RestoreAuthModel, + worker: RuntimeWorkerRef, + subject_id: String, + requests: Arc>>, + callbacks: Arc>>, + lose_response: Arc, + _server: ConnectedRestoreServer, + _dir: tempfile::TempDir, +} + +async fn connected_restore_fixture() -> ConnectedRestoreFixture { + connected_restore_fixture_with_pending_head(false).await +} + +async fn connected_restore_fixture_with_pending_head( + pending_head: bool, +) -> ConnectedRestoreFixture { + use std::sync::atomic::Ordering::SeqCst; + + // Guard the resolved global paths before factory/config/allocation access. + // No Git or Workdir attachment is needed for this connected regression. + let dir = connected_restore_tempdir(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut config = test_server_config(dir.path()); + config.repositories.clear(); + config.backend_base_url = Some(format!("http://{}", listener.local_addr().unwrap())); + let control = Arc::new(test_control_store(&config)); + seed_test_registered_workspace(control.as_ref(), &config) + .await + .unwrap(); + let broker = BackendResourceBroker::default(); + let dispatcher = Arc::new( + crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new( + config.clone(), + control.clone(), + ), + ); + let model = RestoreAuthModel::default(); + let model_factory = model.clone(); + let audience = embedded_runtime_request_audience(&config).unwrap(); + let probe = Arc::new(RestoreAuthProbe::default()); + let factory = worker_runtime::worker_backend::ProfileRuntimeWorkerFactory::new( + config.workspace_execution_root.clone(), + ) + .with_controller_transport(worker::WorkerControllerTransport::InProcess) + .with_embedded_worker_mutation_dispatcher(EMBEDDED_WORKER_RUNTIME_ID, dispatcher.clone()) + .with_workspace_request_client( + worker_runtime::workspace_request::RuntimeWorkspaceRequestClient::new( + TEST_WORKSPACE_ID, + audience.clone(), + EMBEDDED_WORKER_RUNTIME_ID, + ) + .with_runtime_request_source(&EMBEDDED_RUNTIME_REQUEST_IDENTITY, audience), + ) + .with_runtime_store_dir(config.embedded_runtime_store_root.clone()) + .with_resource_client(Arc::new(broker.clone())) + .with_spawn_model_client_factory(move || Box::new(model_factory.clone())); + let execution = Arc::new( + worker_runtime::worker_backend::WorkerRuntimeExecutionBackend::new(RestoreAuthFactory { + inner: factory, + probe: probe.clone(), + }) + .unwrap(), + ); + let seeded_execution = Arc::new(PendingHeadSeedBackend { + inner: execution.clone(), + seed: std::sync::atomic::AtomicBool::new(pending_head), + root: config.embedded_runtime_store_root.clone(), + model: model.clone(), + }); + let api = WorkspaceApi::new_with_execution_backend_and_broker( + config, + control, + seeded_execution, + broker, + HostExecutors::default(), + Some(dispatcher), + ) + .await + .unwrap(); + seed_test_delivery_binding(&api); + let requests = Arc::new(Mutex::new(Vec::new())); + let observed = requests.clone(); + let callbacks = Arc::new(Mutex::new(Vec::new())); + let recorded = callbacks.clone(); + let lose_response = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let lose_once = lose_response.clone(); + let app = build_router(api.clone()).layer(axum::middleware::from_fn( + move |request: axum::extract::Request, next: axum::middleware::Next| { + let observed = observed.clone(); + let recorded = recorded.clone(); + let lose_once = lose_once.clone(); + async move { + let path = request.uri().path().to_owned(); + let is_session = path.ends_with("/subjektiv/sessions"); + let request = if is_session { + let (parts, body) = request.into_parts(); + let bytes = to_bytes(body, usize::MAX).await.unwrap(); + recorded + .lock() + .unwrap() + .push(serde_json::from_slice(&bytes).unwrap()); + axum::http::Request::from_parts(parts, Body::from(bytes)) + } else { + request + }; + let response = next.run(request).await; + observed.lock().unwrap().push((path, response.status())); + if is_session + && response.status() == StatusCode::OK + && lose_once.swap(false, SeqCst) + { + // The real handler has already persisted its immutable SQLite + // attribution. Lose only its acknowledgement, never fake a Host. + axum::http::Response::builder() + .status(StatusCode::BAD_GATEWAY) + .body(Body::from("fixture lost committed Host response")) + .unwrap() + } else { + response + } + } + }, + )); + let server = ConnectedRestoreServer { + server: tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }), + probe: probe.clone(), + }; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject(crate::subjektiv::SubjectRole::new("restore-host-auth").unwrap()) + .unwrap(); + let owner = + seed_test_workspace_owner_token(api.store.as_ref(), "restore-host-auth-owner").await; + let update = server_api::authorization::SubjectPermissionsUpdate { + operation_id: "restore-host-auth-permissions".into(), + expected_change_id: None, + execution: [ + "task", + "subjektiv", + "web", + "image", + "sub_worker", + "worker", + "workspace_worker_discovery", + "objective", + "manage_workdir", + "workdir_catalog", + "drive", + "ticket", + ] + .into_iter() + .map(|name| server_api::authorization::ScopedPermission { + operation: name.into(), + resource: server_api::authorization::PermissionResource::Feature, + }) + .collect(), + delegation: vec![], + }; + let response = build_router(api.clone()) + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(format!( + "/api/w/{TEST_WORKSPACE_ID}/subjektiv/subjects/{}/permissions", + subject.id + )) + .header("authorization", format!("Bearer {owner}")) + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_vec(&update).unwrap())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let Json(created) = scoped_create_workspace_worker( + State(api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.into(), + }), + HeaderMap::new(), + Json(CreateWorkspaceWorkerRequest { + runtime_id: EMBEDDED_WORKER_RUNTIME_ID.into(), + display_name: "restore-auth-body".into(), + singleton_key: None, + profile: Some("builtin:companion".into()), + ticket_assignment: None, + initial_submit: vec![], + workdir_attachments: vec![], + feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest { + subjektiv: Some(server_api::SubjektivWorkerConnectionRequest { + subject_id: subject.id.clone(), + }), + }, + control_operation_id: None, + execution_grant: None, + }), + ) + .await + .unwrap(); + let worker = RuntimeWorkerRef::new(created.runtime_id, created.worker_id); + if pending_head { + assert_eq!( + api.runtime + .stop_worker(&worker, WorkerLifecycleRequest::default()) + .unwrap() + .state, + InternalWorkerOperationState::Accepted + ); + } else { + let handle = probe.handles.lock().unwrap()[0].clone(); + let mut events = handle.subscribe(); + handle + .send(protocol::Method::submit_text( + "setup-running", + "hold for queue setup", + )) + .await + .unwrap(); + restore_auth_event(&mut events, |event| { + matches!(event, protocol::Event::TextDelta { .. }) + }) + .await; + let command = protocol::WorkerCommandEnvelope::new( + handle.shared_state.snapshot().last_command_id + 1, + ); + handle + .send(protocol::Method::Pause { command }) + .await + .unwrap(); + restore_auth_event(&mut events, |event| { + matches!(event, protocol::Event::WorkerState { snapshot } + if snapshot.catalog_status() == protocol::WorkerStatus::Paused) + }) + .await; + handle + .send(protocol::Method::submit_text( + "saved-submit", + "must remain queued", + )) + .await + .unwrap(); + restore_auth_event(&mut events, |event| { + matches!(event, protocol::Event::SubmissionAccepted { + submission_request_id, disposition: protocol::SubmissionDisposition::Queued, .. + } if submission_request_id == "saved-submit") + }) + .await; + handle + .send(protocol::Method::Notify { + notification_request_id: "saved-notice".into(), + message: "must remain queued too".into(), + }) + .await + .unwrap(); + restore_auth_event(&mut events, |event| { + matches!(event, protocol::Event::NotificationAccepted { + notification_request_id + } if notification_request_id == "saved-notice") + }) + .await; + let stopped = api + .runtime + .stop_worker(&worker, WorkerLifecycleRequest::default()) + .unwrap(); + assert_eq!(stopped.state, InternalWorkerOperationState::Accepted); + assert!(handle.protocol_is_closed()); + // Pause/Shutdown can admit real Journal work for the seed turn. Finish that + // owned work before measuring the restore window; it is not restore I/O. + let attr = store + .subject_session_attributions(&subject.id) + .unwrap() + .into_iter() + .find(|attr| attr.worker_id == worker.worker_id) + .unwrap(); + let host = JournalHostContext::validated_session(&store, attr.clone()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(30), async { + let mut ticks = tokio::time::interval(std::time::Duration::from_millis(25)); + loop { + ticks.tick().await; + let jobs = api + .store + .list_backend_journal_jobs(TEST_WORKSPACE_ID, &attr.session_id, None, 100) + .unwrap(); + let closed = store + .journal_for_session(&host) + .unwrap() + .is_none_or(|journal| { + store + .active_journal_generation(&host, &journal) + .unwrap() + .is_none() + }); + if closed + && !store.journal_retention_required(&host).unwrap() + && jobs + .iter() + .all(|job| job.state == BackendJobState::Completed) + { + break; + } + } + }) + .await + .expect("seed Journal work must settle before measuring restore"); + } + + ConnectedRestoreFixture { + api, + execution, + probe, + model, + worker, + subject_id: subject.id, + requests, + callbacks, + lose_response, + _server: server, + _dir: dir, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn restore_after_real_prepare_rejects_revoked_host_trust_without_publication_or_queue_consumption() + { + use session_store::WorkerMetadataStore as _; + use std::sync::atomic::Ordering::SeqCst; + use worker_runtime::execution::WorkerExecutionBackend as _; + + let fixture = connected_restore_fixture().await; + let api = &fixture.api; + let execution = &fixture.execution; + let probe = &fixture.probe; + let model = &fixture.model; + let worker = fixture.worker.clone(); + let requests = &fixture.requests; + let store = open_subjektiv_store(api).unwrap(); + let subject_id = &fixture.subject_id; + let aggregate = api + .config + .embedded_runtime_store_root + .join("workers") + .join(&worker.worker_id); + let name = format!("worker-runtime-{}", worker.worker_id); + let metadata = session_store::WorkerAggregateStore::open_read_only(&aggregate, &name).unwrap(); + let saved = metadata.read_by_name(&name).unwrap().unwrap(); + assert!(matches!( + saved.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Confirmed { .. }) + )); + let active = saved.active.as_ref().unwrap(); + let session = + session_store::WorkerSessionStore::open_read_only(aggregate.join("session")).unwrap(); + let entries = || { + session + .read_all_read_only(active.session_id, active.segment_id.unwrap()) + .unwrap() + }; + let before = serde_json::to_value(entries()).unwrap(); + let checkpoint = entries() + .into_iter() + .rev() + .find_map(|entry| match entry { + session_store::LogEntry::Extension { + domain, payload, .. + } if domain == "worker.pending_activations.v1" => Some(payload), + _ => None, + }) + .expect("real Controller persisted pending activation checkpoint"); + assert_eq!( + checkpoint["pending"][0]["submission_request_id"], + "saved-submit" + ); + assert_eq!( + checkpoint["pending_notifications"][0]["notification_request_id"], + "saved-notice" + ); + let attributions = store.subject_session_attributions(subject_id).unwrap(); + let key = crate::subjektiv::subject_worker_singleton_key(subject_id).unwrap(); + let singleton = api + .store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &key) + .unwrap(); + let model_before = model.calls.load(SeqCst); + let request = runtime_api::WorkerRestoreRequest { + expected_observation_token: api + .runtime + .worker(&worker) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: "restore-revoked-after-real-prepare".into(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }; + assert!( + requests.lock().unwrap().iter().any( + |(path, status)| path.ends_with("/subjektiv/sessions") && *status == StatusCode::OK + ), + "the same production signer must be accepted before trust revocation" + ); + requests.lock().unwrap().clear(); + let runtime = api.runtime.clone(); + let target = worker.clone(); + let intent = request.clone(); + let restoring = tokio::task::spawn_blocking(move || runtime.restore_worker(&target, intent)); + tokio::time::timeout( + std::time::Duration::from_secs(10), + probe.prepared.notified(), + ) + .await + .expect("real prepare must complete before revocation"); + assert_eq!(probe.activations.load(SeqCst), 0); + assert_eq!(model.calls.load(SeqCst), model_before); + assert_eq!( + serde_json::to_value(entries()).unwrap(), + before, + "preparation must not consume saved input or append history" + ); + assert_eq!(metadata.read_by_name(&name).unwrap().unwrap(), saved); + assert!( + requests + .lock() + .unwrap() + .iter() + .all(|(path, _)| !path.ends_with("/subjektiv/sessions")), + "publication authorization belongs after candidate preparation" + ); + let prepared_record: Value = + serde_json::from_slice(&std::fs::read(aggregate.join("worker.json")).unwrap()).unwrap(); + let operation = prepared_record["restore_guard"]["owners"]["restore-revoked-after-real-prepare"]["operation_id"].clone(); + assert!( + !operation.is_null(), + "admission must persist the operation owner before prepare" + ); + assert!( + api.config_store + .revoke_workspace_runtime_binding( + TEST_WORKSPACE_ID, + EMBEDDED_WORKER_RUNTIME_ID, + &now_registry_timestamp(), + ) + .unwrap() + ); + probe.release.notify_one(); + let first = tokio::time::timeout(std::time::Duration::from_secs(20), restoring) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{first:?}" + ); + assert!( + requests + .lock() + .unwrap() + .iter() + .any(|(path, status)| path.ends_with("/subjektiv/sessions") + && *status == StatusCode::UNAUTHORIZED), + "actual signed Host publication must see revoked persisted trust" + ); + assert_eq!(probe.prepares.load(SeqCst), 1); + assert_eq!(probe.activations.load(SeqCst), 1); + let runtime = api.runtime.clone(); + let target = worker.clone(); + let replay = tokio::time::timeout( + std::time::Duration::from_secs(20), + tokio::task::spawn_blocking(move || runtime.restore_worker(&target, request)), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{replay:?}" + ); + // Reconciliation may reconstruct after confirmed cleanup, but must never + // invent a new operation or repair an already confirmed attribution. + assert!( + probe + .operations + .lock() + .unwrap() + .iter() + .all(|id| *id == operation) + ); + assert!( + probe + .repairs + .lock() + .unwrap() + .iter() + .all(|repaired| !repaired) + ); + let worker_ref = worker_runtime::identity::WorkerRef::try_from(&worker).unwrap(); + assert!( + execution.worker_snapshot(&worker_ref).is_none(), + "no live Controller may be published" + ); + assert!( + requests + .lock() + .unwrap() + .iter() + .all(|(path, _)| !path.ends_with("/authorization/check")), + "restoring saved Features must not re-admit them" + ); + assert_eq!(probe.publications.load(SeqCst), 0); + assert_eq!( + probe.handles.lock().unwrap().len(), + 1, + "only stopped seed Controller existed" + ); + assert_eq!(model.calls.load(SeqCst), model_before); + assert_eq!( + serde_json::to_value(entries()).unwrap(), + before, + "Host denial must not consume either saved queue or execute Tools" + ); + assert_eq!( + metadata + .read_by_name(&name) + .unwrap() + .unwrap() + .subjektiv_session_attribution, + saved.subjektiv_session_attribution + ); + assert_eq!( + store.subject_session_attributions(subject_id).unwrap(), + attributions + ); + assert_eq!( + api.store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &key) + .unwrap(), + singleton + ); + // Activation may already reconcile metadata/pins/allocation. Denial is not a + // fictitious rollback/Stopped: the same operation must retain its owner. + let record: Value = + serde_json::from_slice(&std::fs::read(aggregate.join("worker.json")).unwrap()).unwrap(); + assert_eq!( + record["restore_guard"]["active_request_id"], + "restore-revoked-after-real-prepare" + ); + let owner = &record["restore_guard"]["owners"]["restore-revoked-after-real-prepare"]; + assert_eq!( + owner["operation_id"], operation, + "the durable operation owner must survive denial and replay" + ); + assert_eq!( + record["execution_state"]["state"], + "reconciliation_required" + ); + assert_eq!( + record["execution_state"]["execution"]["operation"], + "restore" + ); + assert_eq!( + record["execution_state"]["execution"]["intent"]["operation_id"], + operation + ); + assert!( + owner["receipt"].is_null(), + "no saved Accepted receipt may escape on replay" + ); + // Only this fixture's retained operation is cleaned up, after its durable + // evidence has been checked. Cleanup is not the restore outcome above. + let cleanup = execution.stop_worker(&worker_ref); + assert!( + cleanup.is_accepted(), + "fixture restore cleanup: {cleanup:?}" + ); + shutdown_workspace_feature_storage(&api).await.unwrap(); +} + +#[derive(Clone, Copy)] +enum ConnectedRepair { + IncompleteAttribution, + PendingHead, + RevokedAttribution, +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn explicit_repair_existing_attribution_replays_lost_real_host_response_once() { + connected_explicit_repair(ConnectedRepair::IncompleteAttribution).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn explicit_repair_pending_head_creates_one_session_after_lost_real_host_response() { + connected_explicit_repair(ConnectedRepair::PendingHead).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 6)] +async fn explicit_repair_replay_rejects_revoked_real_host_trust_without_false_accepted() { + connected_explicit_repair(ConnectedRepair::RevokedAttribution).await; +} + +async fn connected_explicit_repair(scenario: ConnectedRepair) { + use session_store::{Store as _, WorkerMetadataStore as _}; + use std::sync::atomic::Ordering::SeqCst; + use worker_runtime::execution::WorkerExecutionBackend as _; + + let pending = matches!(scenario, ConnectedRepair::PendingHead); + let fixture = connected_restore_fixture_with_pending_head(pending).await; + let api = &fixture.api; + let probe = &fixture.probe; + let worker = &fixture.worker; + let revoked = matches!(scenario, ConnectedRepair::RevokedAttribution); + let aggregate = api + .config + .embedded_runtime_store_root + .join("workers") + .join(&worker.worker_id); + let name = format!("worker-runtime-{}", worker.worker_id); + let metadata = session_store::WorkerAggregateStore::new(&aggregate, &name).unwrap(); + let mut saved = metadata.read_by_name(&name).unwrap().unwrap(); + let original_active = saved.active.clone().unwrap(); + // The genuine pending fixture has no Session manifest yet. Open its empty + // store before measuring repair; after publication always reread authority + // rather than the original Store's cached empty lineage. + let pending_store = + pending.then(|| session_store::WorkerSessionStore::new(aggregate.join("session")).unwrap()); + let sessions = + || match session_store::WorkerSessionStore::open_read_only(aggregate.join("session")) { + Ok(store) => store, + Err(session_store::StoreError::Io(error)) + if pending && error.kind() == std::io::ErrorKind::NotFound => + { + pending_store.as_ref().unwrap().clone() + } + Err(error) => panic!("read repaired Session: {error}"), + }; + let original_entries = match original_active.segment_id { + Some(segment_id) => serde_json::to_value( + sessions() + .read_all_read_only(original_active.session_id, segment_id) + .unwrap(), + ) + .unwrap(), + None => { + assert!(pending); + assert!( + sessions() + .list_segments(original_active.session_id) + .unwrap() + .is_empty() + ); + serde_json::json!([]) + } + }; + let store = open_subjektiv_store(api).unwrap(); + let initial_attributions = store + .subject_session_attributions(&fixture.subject_id) + .unwrap(); + let singleton_key = + crate::subjektiv::subject_worker_singleton_key(&fixture.subject_id).unwrap(); + let singleton = api + .store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &singleton_key) + .unwrap(); + + let session_id = saved.active.as_ref().unwrap().session_id; + assert_eq!( + initial_attributions + .iter() + .filter(|attr| attr.session_id == session_id.to_string()) + .count(), + usize::from(!pending), + "existing attribution repair is not first creation; pending-head repair is" + ); + // Deliberate initial interruption fixture, before admitting any restore. + // Subsequent transitions, including OutcomeUnknown, are production writes. + saved.subjektiv_session_attribution = Some( + session_store::SubjektivSessionAttributionState::OutcomeUnknown { + session_id, + diagnostic: "fixture interrupted Session attribution acknowledgement".into(), + }, + ); + metadata.write(&saved).unwrap(); + let model_before = fixture.model.calls.load(SeqCst); + fixture.requests.lock().unwrap().clear(); + fixture.callbacks.lock().unwrap().clear(); + fixture.lose_response.store(true, SeqCst); + let request = runtime_api::WorkerRestoreRequest { + expected_observation_token: api + .runtime + .worker(worker) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: "connected-explicit-repair".into(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }; + let runtime = api.runtime.clone(); + let target = worker.clone(); + let intent = request.clone(); + let first = tokio::time::timeout( + std::time::Duration::from_secs(20), + tokio::task::spawn_blocking(move || runtime.restore_worker(&target, intent)), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{first:?}" + ); + assert!( + !fixture.lose_response.load(SeqCst), + "lose acknowledgement only after real Host OK" + ); + assert_eq!( + probe.prepares.load(SeqCst), + 0, + "repair precedes candidate construction" + ); + assert_eq!(probe.activations.load(SeqCst), 0); + assert_eq!(probe.publications.load(SeqCst), 0); + assert_eq!(fixture.model.calls.load(SeqCst), model_before); + let failed = metadata.read_by_name(&name).unwrap().unwrap(); + assert_eq!( + failed.active, saved.active, + "response loss cannot publish a first head" + ); + assert!(matches!(failed.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::OutcomeUnknown { session_id: id, .. }) if id == session_id)); + let committed_attributions = store + .subject_session_attributions(&fixture.subject_id) + .unwrap(); + assert_eq!( + committed_attributions.len(), + initial_attributions.len() + usize::from(pending) + ); + assert_eq!( + committed_attributions + .iter() + .filter(|attr| attr.session_id == session_id.to_string() + && attr.worker_id == worker.worker_id) + .count(), + 1, + "actual SQLite attribution persisted before response loss" + ); + if !pending { + assert_eq!( + committed_attributions, initial_attributions, + "reconcile the existing immutable row" + ); + } + assert_eq!(fixture.callbacks.lock().unwrap().len(), 1); + { + let callbacks = fixture.callbacks.lock().unwrap(); + assert_eq!(callbacks[0]["session_id"], session_id.to_string()); + assert_eq!(callbacks[0]["create_if_missing"], true); + } + let record = || -> Value { + serde_json::from_slice(&std::fs::read(aggregate.join("worker.json")).unwrap()).unwrap() + }; + let first_record = record(); + let operation = + first_record["restore_guard"]["owners"]["connected-explicit-repair"]["operation_id"] + .clone(); + assert!(!operation.is_null()); + assert!( + first_record["restore_guard"]["owners"]["connected-explicit-repair"]["receipt"].is_null() + ); + assert_eq!( + first_record["restore_guard"]["active_request_id"], + "connected-explicit-repair" + ); + assert_eq!( + first_record["execution_state"]["state"], + "reconciliation_required" + ); + assert!( + fixture + .requests + .lock() + .unwrap() + .iter() + .any( + |(path, status)| path.ends_with("/subjektiv/sessions") && *status == StatusCode::OK + ) + ); + assert!( + !fixture + .requests + .lock() + .unwrap() + .iter() + .any(|(path, _)| path.ends_with("/subjektiv/journal/startup")), + "no startup before attribution acknowledgement" + ); + if pending { + assert!(sessions().list_segments(session_id).unwrap().is_empty()); + } + + // Re-enter the same public Runtime intent and durable operation. The repair + // gate changes timing only; trust revocation is the real persisted binding. + if revoked { + probe.gate_repair.store(true, SeqCst); + } + let mut repaired_seed = None; + let runtime = api.runtime.clone(); + let target = worker.clone(); + let intent = request.clone(); + let retry = tokio::task::spawn_blocking(move || runtime.restore_worker(&target, intent)); + if revoked { + tokio::time::timeout( + std::time::Duration::from_secs(10), + probe.repairing.notified(), + ) + .await + .unwrap(); + assert!( + api.config_store + .revoke_workspace_runtime_binding( + TEST_WORKSPACE_ID, + EMBEDDED_WORKER_RUNTIME_ID, + &now_registry_timestamp(), + ) + .unwrap() + ); + probe.release_repair.notify_one(); + } else { + tokio::time::timeout( + std::time::Duration::from_secs(10), + probe.prepared.notified(), + ) + .await + .unwrap(); + assert_eq!(probe.activations.load(SeqCst), 0); + assert_eq!(fixture.model.calls.load(SeqCst), model_before); + assert_eq!(*probe.repairs.lock().unwrap(), vec![true]); + let repaired = metadata.read_by_name(&name).unwrap().unwrap(); + assert_eq!(repaired.active.as_ref().unwrap().session_id, session_id); + assert!(matches!(repaired.subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Confirmed { session_id: id, ref subject_id }) + if id == session_id && subject_id == &fixture.subject_id)); + assert_eq!( + sessions().list_segments(session_id).unwrap().len(), + 1, + "exactly one durable seed" + ); + let segment_id = repaired.active.unwrap().segment_id.unwrap(); + let entries = sessions() + .read_all_read_only(session_id, segment_id) + .unwrap(); + let seeds = entries + .iter() + .filter(|entry| matches!(entry, session_store::LogEntry::AnnotatedSegmentStart { .. })) + .collect::>(); + assert_eq!(seeds.len(), 1); + repaired_seed = Some((segment_id, serde_json::to_value(seeds[0]).unwrap())); + if !pending { + assert_eq!( + serde_json::to_value(entries).unwrap(), + original_entries, + "attribution-only repair and normal prepare cannot reseed or consume saved input" + ); + } + assert_eq!( + fixture + .requests + .lock() + .unwrap() + .iter() + .filter(|(path, _)| path.ends_with("/subjektiv/journal/startup")) + .count(), + usize::from(pending) + ); + probe.release.notify_one(); + } + let replay = tokio::time::timeout(std::time::Duration::from_secs(20), retry) + .await + .unwrap() + .unwrap() + .unwrap(); + if revoked { + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{replay:?}" + ); + assert!( + fixture + .requests + .lock() + .unwrap() + .iter() + .any(|(path, status)| path.ends_with("/subjektiv/sessions") + && *status == StatusCode::UNAUTHORIZED) + ); + assert_eq!(probe.prepares.load(SeqCst), 0); + assert_eq!(probe.activations.load(SeqCst), 0); + assert_eq!(probe.publications.load(SeqCst), 0); + assert_eq!(fixture.model.calls.load(SeqCst), model_before); + assert!( + matches!(metadata.read_by_name(&name).unwrap().unwrap().subjektiv_session_attribution, + Some(session_store::SubjektivSessionAttributionState::Failed { session_id: id, .. }) if id == session_id) + ); + let denied = record(); + assert_eq!( + denied["restore_guard"]["active_request_id"], + "connected-explicit-repair" + ); + assert_eq!( + denied["restore_guard"]["owners"]["connected-explicit-repair"]["operation_id"], + operation + ); + assert!( + denied["restore_guard"]["owners"]["connected-explicit-repair"]["receipt"].is_null() + ); + assert_eq!( + denied["execution_state"]["state"], + "reconciliation_required" + ); + assert_eq!( + serde_json::to_value( + sessions() + .read_all_read_only( + original_active.session_id, + original_active.segment_id.unwrap(), + ) + .unwrap() + ) + .unwrap(), + original_entries + ); + } else { + assert_eq!( + replay.state, + server_api::WorkerRestoreState::Accepted, + "{replay:?}" + ); + let callbacks_after = fixture.callbacks.lock().unwrap().len(); + let repairs_after = probe.repair_operations.lock().unwrap().len(); + let runtime = api.runtime.clone(); + let target = worker.clone(); + let accepted = tokio::time::timeout( + std::time::Duration::from_secs(20), + tokio::task::spawn_blocking(move || runtime.restore_worker(&target, request)), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + accepted.state, + server_api::WorkerRestoreState::Accepted, + "{accepted:?}" + ); + assert_eq!(probe.prepares.load(SeqCst), 1); + assert_eq!(probe.publications.load(SeqCst), 1); + assert_eq!(probe.repair_operations.lock().unwrap().len(), repairs_after); + assert_eq!( + fixture.callbacks.lock().unwrap().len(), + callbacks_after, + "receipt replay performs no Host callback" + ); + assert_eq!(sessions().list_segments(session_id).unwrap().len(), 1); + let repaired = metadata.read_by_name(&name).unwrap().unwrap(); + let active = repaired.active.unwrap(); + let (segment_id, seed) = repaired_seed.unwrap(); + assert_eq!(active.session_id, session_id); + assert_eq!(active.segment_id, Some(segment_id)); + let entries = sessions() + .read_all_read_only(session_id, segment_id) + .unwrap(); + let seeds = entries + .iter() + .filter(|entry| matches!(entry, session_store::LogEntry::AnnotatedSegmentStart { .. })) + .collect::>(); + assert_eq!(seeds.len(), 1); + assert_eq!( + serde_json::to_value(seeds[0]).unwrap(), + seed, + "activation and receipt replay preserve the exact repaired seed" + ); + } + if pending { + assert_eq!( + fixture.model.calls.load(SeqCst), + model_before, + "pending first-head repair has no body input to execute" + ); + assert_eq!( + session_id, original_active.session_id, + "first head belongs to the originally pending Session" + ); + assert!(original_active.segment_id.is_none()); + } + assert_eq!( + store + .subject_session_attributions(&fixture.subject_id) + .unwrap(), + committed_attributions + ); + assert_eq!( + api.store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &singleton_key) + .unwrap(), + singleton + ); + assert!( + probe + .repair_operations + .lock() + .unwrap() + .iter() + .all(|id| *id == operation) + ); + assert!( + probe + .operations + .lock() + .unwrap() + .iter() + .all(|id| *id == operation) + ); + assert_eq!( + probe.repair_operations.lock().unwrap().len(), + 2, + "one failed repair and one same-operation replay" + ); + assert!( + fixture + .requests + .lock() + .unwrap() + .iter() + .all(|(path, _)| !path.ends_with("/authorization/check")), + "saved Features must not be re-admitted" + ); + { + let callbacks = fixture.callbacks.lock().unwrap(); + assert!( + callbacks + .iter() + .all(|callback| callback["session_id"] == session_id.to_string()) + ); + assert_eq!( + callbacks + .iter() + .filter(|callback| callback["create_if_missing"] == true) + .count(), + 2, + "repair retries the Session-keyed receipt; publication only verifies it" + ); + } + let worker_ref = worker_runtime::identity::WorkerRef::try_from(worker).unwrap(); + if revoked { + assert!(fixture.execution.worker_snapshot(&worker_ref).is_none()); + assert!( + fixture.execution.stop_worker(&worker_ref).is_accepted(), + "fixture-owned repair resource cleanup" + ); + } else { + assert!(fixture.execution.worker_snapshot(&worker_ref).is_some()); + // Teardown is outside the measured restore window: Stop may admit an + // independent Journal job, not a second execution of the restored body. + assert_eq!( + api.runtime + .stop_worker(worker, WorkerLifecycleRequest::default()) + .unwrap() + .state, + InternalWorkerOperationState::Accepted + ); + } + shutdown_workspace_feature_storage(api).await.unwrap(); +} diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs index dece1c3f4..cb88a2636 100644 --- a/crates/workspace-server/src/server_assignment_attention_tests.rs +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -490,21 +490,61 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFac self.controller(request.worker_ref.worker_id.to_string(), false) .await } - async fn restore_controller( + async fn prepare_restore( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + Ok(Box::new(AttentionRestoreCandidate { + inputs: Some(( + AttentionWorkerFactory { + root: self.root.clone(), + }, + request.worker_ref.worker_id.to_string(), + )), + })) + } +} + +// Preparation owns only fixture inputs; reconstruction and Controller startup +// remain lazy so discard never needs to shut down a started Worker. +struct AttentionRestoreCandidate { + inputs: Option<(AttentionWorkerFactory, String)>, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for AttentionRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + protocol::WorkerStateSnapshot::initial() + } + + async fn activate( + &mut self, ) -> std::result::Result< worker_runtime::worker_backend::RuntimeWorkerController, worker_runtime::worker_backend::RuntimeRestoreFailure, > { - self.controller(request.worker_ref.worker_id.to_string(), true) - .await - .map_err(|_| { - worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( - worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, - "attention fixture reconstruction failed", - ) - }) + let (factory, worker_id) = self.inputs.take().ok_or_else(|| { + worker_runtime::worker_backend::RuntimeRestoreFailure::refused( + worker_runtime::worker_backend::RuntimeRestoreStage::Controller, + "attention fixture candidate already consumed", + ) + })?; + factory.controller(worker_id, true).await.map_err(|_| { + worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( + worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, + "attention fixture reconstruction failed", + ) + }) + } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inputs.take(); + Ok(()) } } @@ -611,7 +651,7 @@ async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() { async fn real_attention_worker_start_restore(lose_response: bool) { use session_store::{LogEntry, Store, WorkerSessionStore}; - let dir = tempfile::tempdir().unwrap(); + let dir = restored_parent_control_tests::connected_restore_tempdir(); init_clean_git_workspace(dir.path()); let config = test_server_config(dir.path()); let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs index 43bddd74f..25031c8f6 100644 --- a/crates/workspace-server/src/server_assignment_delivery_tests.rs +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -14,17 +14,37 @@ async fn notification_diagnostics_http_auth_scope_and_readonly_evidence() { seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID); let ticket = delivery_ticket(&api, "Notification diagnostics auth fixture"); delivery_assign(&api, ticket.id.as_str(), "diagnostics-assignment").await; - let path = format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",ticket.id.as_str()); - let page = delivery_http(&api,"GET",&path,None,StatusCode::OK).await; - assert_eq!(page["items"].as_array().unwrap().len(),1); - assert_eq!(page["items"][0]["model_processing_confirmed"],false); + let path = format!( + "/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1", + ticket.id.as_str() + ); + let page = delivery_http(&api, "GET", &path, None, StatusCode::OK).await; + assert_eq!(page["items"].as_array().unwrap().len(), 1); + assert_eq!(page["items"][0]["model_processing_confirmed"], false); assert!(page["items"][0].get("content").is_none()); - let again=delivery_http(&api,"GET",&path,None,StatusCode::OK).await; - assert_eq!(again,page); - let response=build_router(api.clone()).oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap()).await.unwrap(); - assert_eq!(response.status(),StatusCode::UNAUTHORIZED); - delivery_http(&api,"GET","/api/w/other-workspace/notification-diagnostics",None,StatusCode::NOT_FOUND).await; - delivery_http(&api,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),None,StatusCode::BAD_REQUEST).await; + let again = delivery_http(&api, "GET", &path, None, StatusCode::OK).await; + assert_eq!(again, page); + let response = build_router(api.clone()) + .oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + delivery_http( + &api, + "GET", + "/api/w/other-workspace/notification-diagnostics", + None, + StatusCode::NOT_FOUND, + ) + .await; + delivery_http( + &api, + "GET", + &format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"), + None, + StatusCode::BAD_REQUEST, + ) + .await; } fn delivery_assignment_path(ticket: &str) -> String { @@ -828,21 +848,62 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFact self.controller(request.worker_ref.worker_id.to_string(), false) .await } - async fn restore_controller( + async fn prepare_restore( &self, request: delivery_execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + Ok(Box::new(DeliveryRestoreCandidate { + inputs: Some(( + DeliveryWorkerFactory { + root: self.root.clone(), + client: self.client.clone(), + }, + request.worker_ref.worker_id.to_string(), + )), + })) + } +} + +// Retain the same scripted client/receipt store without opening either store or +// starting a Controller until activation. Discard drops only these owned inputs. +struct DeliveryRestoreCandidate { + inputs: Option<(DeliveryWorkerFactory, String)>, +} + +#[async_trait::async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for DeliveryRestoreCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + protocol::WorkerStateSnapshot::initial() + } + + async fn activate( + &mut self, ) -> std::result::Result< worker_runtime::worker_backend::RuntimeWorkerController, worker_runtime::worker_backend::RuntimeRestoreFailure, > { - self.controller(request.worker_ref.worker_id.to_string(), true) - .await - .map_err(|_| { - worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( - worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, - "delivery fixture reconstruction failed", - ) - }) + let (factory, worker_id) = self.inputs.take().ok_or_else(|| { + worker_runtime::worker_backend::RuntimeRestoreFailure::refused( + worker_runtime::worker_backend::RuntimeRestoreStage::Controller, + "delivery fixture candidate already consumed", + ) + })?; + factory.controller(worker_id, true).await.map_err(|_| { + worker_runtime::worker_backend::RuntimeRestoreFailure::unproven( + worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction, + "delivery fixture reconstruction failed", + ) + }) + } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inputs.take(); + Ok(()) } } @@ -941,6 +1002,7 @@ async fn delivery_real_fixture( Arc, DeliveryScriptedClient, ) { + let _environment = restored_parent_control_tests::connected_restore_tempdir(); let config = test_server_config(root); let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap()); seed_test_registered_workspace(store.as_ref(), &config) @@ -1146,9 +1208,22 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi "uncertain delivery changed request id or payload: {attempts:?}" ); assert_eq!(delivery_receipts(&api, &worker).len(), 1); - let unknown = api.store.notification_diagnostics(TEST_WORKSPACE_ID, &server_api::NotificationDiagnosticsQuery { - assignment_id: Some(assignment.into()), ..Default::default() - }).unwrap().items.into_iter().find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown).expect("saved unknown acceptance must remain visible while Runtime already has its receipt"); + let unknown = api + .store + .notification_diagnostics( + TEST_WORKSPACE_ID, + &server_api::NotificationDiagnosticsQuery { + assignment_id: Some(assignment.into()), + ..Default::default() + }, + ) + .unwrap() + .items + .into_iter() + .find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown) + .expect( + "saved unknown acceptance must remain visible while Runtime already has its receipt", + ); assert!(!unknown.model_processing_confirmed); // Change mutable rendering context while the acknowledgement is unknown. // A new self-report is not a new input and cannot change the frozen retry. @@ -1209,11 +1284,24 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi } delivery_wait_idle(&reopened, &worker, 2).await; assert_eq!(delivery_receipts(&reopened, &worker).len(), 2); - let evidence = delivery_http(&reopened,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",unknown.source_id),None,StatusCode::OK).await; - assert_eq!(evidence["items"][0]["stage"],"runtime_accepted"); - assert_eq!(evidence["items"][0]["delivery_id"],json!(unknown.delivery_id)); - assert_eq!(evidence["items"][0]["model_processing_confirmed"],false); - assert_eq!(evidence["items"][0]["retry_candidate"],false); + let evidence = delivery_http( + &reopened, + "GET", + &format!( + "/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}", + unknown.source_id + ), + None, + StatusCode::OK, + ) + .await; + assert_eq!(evidence["items"][0]["stage"], "runtime_accepted"); + assert_eq!( + evidence["items"][0]["delivery_id"], + json!(unknown.delivery_id) + ); + assert_eq!(evidence["items"][0]["model_processing_confirmed"], false); + assert_eq!(evidence["items"][0]["retry_candidate"], false); assert_eq!( delivery_commits(&reopened, &worker) .iter() diff --git a/crates/workspace-server/src/server_workspace_config_wip_tests.rs b/crates/workspace-server/src/server_workspace_config_wip_tests.rs index 6c3fd0fd6..9e970e17d 100644 --- a/crates/workspace-server/src/server_workspace_config_wip_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_wip_tests.rs @@ -187,7 +187,14 @@ async fn config_call( runtime .invoke( path.into(), - if matches!(operation,"edit"|"write") {wip_protocol::InterfaceReference {scope:"/".into(),name:format!("yoi.file/{operation}-text/v1")}} else {config_interface(path)}, + if matches!(operation, "edit" | "write") { + wip_protocol::InterfaceReference { + scope: "/".into(), + name: format!("yoi.file/{operation}-text/v1"), + } + } else { + config_interface(path) + }, operation.into(), args, agen::tool::ToolExecutionContext::direct(), @@ -581,14 +588,88 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto *self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle()); Ok(controller) } - async fn restore_controller( + async fn preflight_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), String> { + self.inner.preflight_restore(request).await + } + + async fn recover_restore_source( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.inner.recover_restore_source(request).await + } + + async fn prepare_restore( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result< + Box, + worker_runtime::worker_backend::RuntimeRestoreFailure, + > { + let inner = self.inner.prepare_restore(request).await?; + Ok(Box::new(ConfigCaptureCandidate { + inner, + client: self.client.clone(), + controller_abort: self.controller_abort.clone(), + })) + } + + async fn cleanup_failed_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.cleanup_failed_restore(request).await + } + + async fn reconcile_stopped_worker( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.reconcile_stopped_worker(worker_ref).await + } + + fn activate_restored_controller( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + workspace_id: Option<&str>, + handle: &worker::WorkerHandle, + ) { + self.inner + .activate_restored_controller(worker_ref, workspace_id, handle); + } +} + +struct ConfigCaptureCandidate { + inner: Box, + client: Arc>>>, + controller_abort: Arc>>, +} + +#[async_trait] +impl worker_runtime::worker_backend::RuntimeRestoreCandidate for ConfigCaptureCandidate { + fn initial_state(&self) -> protocol::WorkerStateSnapshot { + self.inner.initial_state() + } + + async fn activate( + &mut self, ) -> std::result::Result< worker_runtime::worker_backend::RuntimeWorkerController, worker_runtime::worker_backend::RuntimeRestoreFailure, > { - self.inner.restore_controller(request).await + let controller = self.inner.activate().await?; + *self.client.lock().unwrap() = Some(controller.workspace_client.clone()); + *self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle()); + Ok(controller) + } + + async fn discard( + &mut self, + ) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> { + self.inner.discard().await } } @@ -717,6 +798,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION, decode_runtime_request_source_claims, }; + let root = restored_parent_control_tests::connected_restore_tempdir(); // Remote Runtime-owned HTTP adapter, including its real request signer. let mut fixture = manual_worker_assignment_fixture().await; let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap(); @@ -796,7 +878,6 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w // Actual embedded Runtime factory -> real Worker controller -> captured // production WorkspaceClient -> signed HTTP -> production Backend middleware. - let root = tempfile::tempdir().unwrap(); init_clean_git_workspace(root.path()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); diff --git a/deno.json b/deno.json index d10675008..b006ccf53 100644 --- a/deno.json +++ b/deno.json @@ -1,8 +1,8 @@ { "tasks": { - "test": "deno run --allow-read --allow-write=target --allow-run --allow-env tools/test.ts", + "test": "deno run --allow-read --allow-write=target --allow-run --allow-env --allow-sys=uid tools/test.ts", "test:ticket:real-api": "cargo test -p yoi-workspace-server --lib ticket_web_roundtrip_tests -- --ignored", - "test:runner": "deno test --allow-read --allow-write --allow-run --allow-env tools/test.test.ts" + "test:runner": "deno test --allow-read --allow-write --allow-run --allow-env --allow-sys=uid tools/test.test.ts" }, "fmt": { "proseWrap": "never" diff --git a/tests/support/restore_environment.rs b/tests/support/restore_environment.rs new file mode 100644 index 000000000..c045a2ed5 --- /dev/null +++ b/tests/support/restore_environment.rs @@ -0,0 +1,129 @@ +//! Fail-closed preflight shared by restore fixtures across Worker, Runtime and Server. +//! +//! The normal runner supplies a separate namespace per process; a manually +//! configured invocation may use the checkout-owned base itself. This module +//! never sets process-wide environment or creates directories/tables. + +use std::fs; +use std::path::{Component, Path, PathBuf}; + +fn validate_restore_test_paths( + base: &Path, + resolved: [Option; 5], +) -> Result { + let root = resolved[0] + .as_deref() + .filter(|config| config.file_name().is_some_and(|name| name == "config")) + .and_then(Path::parent) + .ok_or("config")?; + let namespace = root.strip_prefix(base).map_err(|_| "config")?; + if namespace + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err("config"); + } + for ((label, suffix), actual) in [ + ("config", "config"), + ("data", "data"), + ("secrets", "data"), + ("runtime", "run"), + ("allocation", "run/workers.json"), + ] + .into_iter() + .zip(resolved.iter()) + { + if actual.as_deref() != Some(root.join(suffix).as_path()) { + return Err(label); + } + } + Ok(root.to_path_buf()) +} + +fn require_owned_directory(checkout: &Path, directory: &Path) { + let mut checked = checkout.to_path_buf(); + for part in directory.strip_prefix(checkout).unwrap().components() { + assert!(matches!(part, Component::Normal(_))); + checked.push(part); + let metadata = fs::symlink_metadata(&checked) + .expect("create isolated restore test directories before launching the test process"); + assert!( + metadata.is_dir() && !metadata.file_type().is_symlink(), + "restore test directory must not traverse symlinks" + ); + } +} + +pub(crate) fn require_restore_test_environment() { + let checkout = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap(); + let base = checkout.join("target/restore-test-environment"); + // Compare all resolver results before inspecting even an absent allocation + // table (its lookup can create a lock). Never inspect a live fallback path. + let root = validate_restore_test_paths( + &base, + [ + manifest::paths::config_dir(), + manifest::paths::data_dir(), + manifest::paths::secret_data_dir(), + manifest::paths::runtime_dir(), + manifest::paths::worker_allocation_path(), + ], + ) + .unwrap_or_else(|label| { + panic!( + "restore fixture requires isolated {label}: run deno task test, or configure YOI_CONFIG_DIR=/target/restore-test-environment/config, YOI_DATA_DIR=/target/restore-test-environment/data, YOI_RUNTIME_DIR=/target/restore-test-environment/run before launch" + ) + }); + assert_eq!(fs::canonicalize(checkout).unwrap(), checkout); + for directory in [ + root.clone(), + root.join("config"), + root.join("data"), + root.join("run"), + ] { + require_owned_directory(checkout, &directory); + } + for name in ["workers.json", "workers.json.lock"] { + match fs::symlink_metadata(root.join("run").join(name)) { + Ok(metadata) => assert!( + metadata.is_file() && !metadata.file_type().is_symlink(), + "owned allocation paths must be regular files, not symlinks" + ), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => panic!("inspect owned restore allocation path: {error}"), + } + } +} + +#[test] +fn restore_fixture_path_validation_rejects_fallback_before_resource_access() { + let base = Path::new("/owned/test-environment"); + for root in [base.to_path_buf(), base.join("run-1/worker-shard-2")] { + let safe = ["config", "data", "data", "run", "run/workers.json"] + .map(|suffix| Some(root.join(suffix))); + assert_eq!( + validate_restore_test_paths(base, safe.clone()), + Ok(root.clone()) + ); + for index in 0..safe.len() { + for fallback in [None, Some(PathBuf::from("/not-a-fixture"))] { + let mut unsafe_paths = safe.clone(); + unsafe_paths[index] = fallback; + assert!(validate_restore_test_paths(base, unsafe_paths).is_err()); + } + } + } + for root in [ + base.join("../escape"), + base.join("safe/../../escape"), + PathBuf::from("/outside"), + ] { + let paths = ["config", "data", "data", "run", "run/workers.json"] + .map(|suffix| Some(root.join(suffix))); + assert!(validate_restore_test_paths(base, paths).is_err()); + } +} diff --git a/tools/test.test.ts b/tools/test.test.ts index aa8c69d12..bad5ee0c1 100644 --- a/tools/test.test.ts +++ b/tools/test.test.ts @@ -6,6 +6,7 @@ import { mapConcurrent, parseJobs, parseRustListing, + prepareTestIsolation, resultSummary, rustTestCommands, shardRustCommand, @@ -202,6 +203,39 @@ Deno.test("round-robin shards retain every listed name exactly once at threshold assertThrows(() => shardRustCommand(task, ["same", "same"])); }); +Deno.test("global runtime-path packages serialize every normal target without losing names", () => { + for (const pkg of ["worker", "worker-runtime", "yoi-workspace-server"]) { + for (const target of ["lib", "controller_restore_test"]) { + const task = { + name: `${pkg}/${target}`, + command: "/test-binary", + args: [], + cwd: "/pkg", + }; + for (const count of [0, 1, 59, 72, 73, 1135]) { + const names = listedNames(count); + const shards = shardRustCommand(task, names); + assert( + shards.flatMap((shard) => shard.selectedNames).sort().join() === + [...names].sort().join(), + ); + for (const shard of shards) { + const args = shard.task.args; + assert(args[args.indexOf("--test-threads") + 1] === "1"); + if (count > 72) { + assert(args[0] === "--exact"); + assert(args.slice(3).join() === shard.selectedNames.join()); + } else { + assert(args.join() === "--test-threads,1"); + } + assert(!args.includes("--ignored") && !args.includes("--skip")); + } + } + assert(task.args.length === 0, "do not mutate discovered commands"); + } + } +}); + Deno.test("scheduler drains active workers before propagating a callback error", async () => { let drained = false; let failed = false; @@ -266,6 +300,276 @@ function emergencyKill(pid: number) { } } +async function assertRejects( + run: () => Promise, + message: string, +) { + try { + await run(); + } catch (error) { + assert(String(error).includes(message), String(error)); + return; + } + throw new Error(`expected rejection containing ${message}`); +} + +Deno.test({ + name: + "isolated real children use bounded disjoint tables and sanitized Git/compiler environments", + ignore: Deno.build.os !== "linux", + async fn() { + const checkout = await Deno.realPath(await Deno.makeTempDir()); + const environmentSnapshot = () => + JSON.stringify( + Object.entries(process.env).sort(([a], [b]) => a.localeCompare(b)), + ); + const before = environmentSnapshot(); + try { + // An ancestor repository must not make an uninitialized tempfile look like + // a repository. No developer repository or configuration is borrowed. + for (const dir of [".git/objects", ".git/refs"]) { + await Deno.mkdir(`${checkout}/${dir}`, { recursive: true }); + } + await Deno.writeTextFile( + `${checkout}/.git/HEAD`, + "ref: refs/heads/main\n", + ); + const [first, second] = await Promise.all([ + prepareTestIsolation(checkout, "same/output/id"), + prepareTestIsolation(checkout, "same/output/id"), + ]); + assert( + first.root !== second.root, + "concurrent runner ids must not collide", + ); + const task = { + ...fixtureTask("isolation-env", checkout), + env: { + CARGO_MANIFEST_DIR: `${checkout}/crates/example`, + CARGO_HOME: `${checkout}/compiler-cache`, + RUSTUP_HOME: `${checkout}/toolchain-cache`, + YOI_HOME: "/live/yoi", + YOI_CONFIG_DIR: "/live/config", + YOI_DATA_DIR: "/live/data", + YOI_RUNTIME_DIR: "/live/run", + HOME: "/live/home", + TMPDIR: "/deleted/compiler-tmp", + GIT_DIR: "/live/git", + GIT_WORK_TREE: "/live/tree", + GIT_CONFIG: "/live/gitconfig", + GIT_CONFIG_GLOBAL: "/live/gitconfig", + GIT_CONFIG_SYSTEM: "/live/system-gitconfig", + GIT_TEMPLATE_DIR: "/live/template", + GIT_CONFIG_PARAMETERS: "'core.hooksPath'='/live/hooks'", + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "core.hooksPath", + GIT_CONFIG_VALUE_0: "/live/hooks", + HTTP_PROXY: "http://live.invalid", + https_proxy: "http://live.invalid", + ALL_PROXY: "http://live.invalid", + NO_PROXY: "live.invalid", + RUSTC_WRAPPER: "/live/sccache", + RUSTC_WORKSPACE_WRAPPER: "/live/wrapper", + SCCACHE_DISABLE: "0", + }, + }; + const taskBefore = JSON.stringify(task); + const commands = await Promise.all([ + first.wrapCommand(task, "pkg/a"), + first.wrapCommand(task, "pkg?a"), // same sanitized label + first.wrapCommand(task, "pkg/a"), // same exact label + second.wrapCommand(task, "pkg/a"), + ]); + assert(JSON.stringify(task) === taskBefore); + assert( + environmentSnapshot() === before, + "host environment changed", + ); + const results = await Promise.all( + commands.map((command, index) => + execute(command, { + output: checkout, + label: `environment-${index}`, + timeoutMs: 10_000, + }) + ), + ); + const tables = new Set(); + for (const [index, result] of results.entries()) { + assert(result.code === 0 && !result.error, JSON.stringify(result)); + const observed = JSON.parse(result.stdout); + const env = observed.env as Record; + assert(observed.cwd === checkout && commands[index].cwd === task.cwd); + assert(commands[index].args === task.args); + assert(env.CARGO_MANIFEST_DIR === task.env.CARGO_MANIFEST_DIR); + assert(env.CARGO_HOME === task.env.CARGO_HOME); + assert(env.RUSTUP_HOME === task.env.RUSTUP_HOME); + const namespace = env.YOI_RUNTIME_DIR.replace(/\/run$/, ""); + const prefix = `${checkout}/target/restore-test-environment/`; + assert(namespace.startsWith(prefix)); + const suffix = namespace.slice(prefix.length); + assert( + suffix.split("/").every((part) => /^[a-zA-Z0-9_-]+$/.test(part)), + ); + assert(env.YOI_CONFIG_DIR === `${namespace}/config`); + assert(env.YOI_DATA_DIR === `${namespace}/data`); + assert( + env.YOI_HOME === env.YOI_DATA_DIR, + "secret data must equal data", + ); + assert(env.HOME === `${namespace}/home`); + for ( + const dir of [ + namespace, + env.YOI_CONFIG_DIR, + env.YOI_DATA_DIR, + env.YOI_RUNTIME_DIR, + env.HOME, + ] + ) { + assert((await Deno.lstat(dir)).isDirectory); + assert(await Deno.realPath(dir) === dir); + } + for (const key of ["TMPDIR", "TMP", "TEMP"]) { + assert(env[key] === `${checkout}/target/t`); + } + for ( + const key of [ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_CONFIG", + "GIT_CONFIG_PARAMETERS", + "GIT_CONFIG_COUNT", + "GIT_CONFIG_KEY_0", + "GIT_CONFIG_VALUE_0", + "HTTP_PROXY", + "https_proxy", + "ALL_PROXY", + "NO_PROXY", + ] + ) { + assert(!(key in env), `${key} leaked to child`); + } + assert(env.GIT_CONFIG_GLOBAL === "/dev/null"); + assert(env.GIT_CONFIG_SYSTEM === "/dev/null"); + assert(env.GIT_CONFIG_NOSYSTEM === "1"); + assert(env.GIT_CEILING_DIRECTORIES === checkout); + assert(env.GIT_TEMPLATE_DIR === `${namespace}/git-template`); + assert(observed.globalConfig === ""); + assert(observed.ancestorDiscovered === false); + assert(observed.hooks.length === 0 && observed.info.length === 0); + assert(observed.author.includes("normal-tests@example.invalid")); + assert(env.GIT_AUTHOR_NAME === env.GIT_COMMITTER_NAME); + assert(env.GIT_AUTHOR_EMAIL === env.GIT_COMMITTER_EMAIL); + assert(env.RUSTC_WRAPPER === "" && env.RUSTC_WORKSPACE_WRAPPER === ""); + assert(env.SCCACHE_DISABLE === "1"); + const table = `${env.YOI_RUNTIME_DIR}/workers.json`; + tables.add(table); + assert(await Deno.readTextFile(table) === String(observed.pid)); + } + assert( + tables.size === commands.length, + "independent commands shared a table", + ); + assert(environmentSnapshot() === before); + } finally { + await Deno.remove(checkout, { recursive: true }); + } + }, +}); + +Deno.test("isolation refuses lexical aliases and symlinked owned paths before external writes", async () => { + for ( + const unsafe of ["target", "target/restore-test-environment", "target/t"] + ) { + const checkout = await Deno.realPath(await Deno.makeTempDir()); + const outside = await Deno.realPath(await Deno.makeTempDir()); + try { + await Deno.writeTextFile(`${outside}/sentinel`, "untouched"); + if (unsafe !== "target") await Deno.mkdir(`${checkout}/target`); + await Deno.symlink(outside, `${checkout}/${unsafe}`); + await assertRejects( + () => prepareTestIsolation(checkout, "run"), + "symlink", + ); + assert( + Array.from(Deno.readDirSync(outside)).map((entry) => entry.name) + .join() === + "sentinel", + ); + assert(await Deno.readTextFile(`${outside}/sentinel`) === "untouched"); + } finally { + await Deno.remove(checkout, { recursive: true }); + await Deno.remove(outside, { recursive: true }); + } + } + const checkout = await Deno.realPath(await Deno.makeTempDir()); + try { + await Deno.symlink(checkout, `${checkout}/alias`); + await assertRejects( + () => prepareTestIsolation(`${checkout}/alias`, "run"), + "symlink", + ); + for (const suffix of ["/../escape", "/./child", "//child"]) { + await assertRejects( + () => prepareTestIsolation(`${checkout}${suffix}`, "run"), + "normal-only", + ); + } + assert( + Array.from(Deno.readDirSync(checkout)).map((entry) => entry.name) + .join() === + "alias", + ); + } finally { + await Deno.remove(checkout, { recursive: true }); + } +}); + +Deno.test("command wrapping rechecks owned ancestors and refuses symlinked namespace children", async () => { + for ( + const unsafe of [ + "ancestor", + "tmp", + "config", + "data", + "run", + "home", + "git-template", + "git-template/info", + "git-template/hooks", + ] + ) { + const checkout = await Deno.realPath(await Deno.makeTempDir()); + const outside = await Deno.realPath(await Deno.makeTempDir()); + try { + const isolation = await prepareTestIsolation(checkout, "run"); + let path: string; + if (unsafe === "ancestor") { + path = isolation.root; + await Deno.remove(path); + } else if (unsafe === "tmp") { + path = `${checkout}/target/t`; + await Deno.remove(path); + } else { + const namespace = `${isolation.root}/unsafe-0`; + await Deno.mkdir(namespace); + if (unsafe.includes("/")) await Deno.mkdir(`${namespace}/git-template`); + path = `${namespace}/${unsafe}`; + } + await Deno.symlink(outside, path); + await assertRejects( + () => isolation.wrapCommand(fixtureTask("success", checkout), "unsafe"), + "symlink", + ); + assert(Array.from(Deno.readDirSync(outside)).length === 0); + } finally { + await Deno.remove(checkout, { recursive: true }); + await Deno.remove(outside, { recursive: true }); + } + } +}); + for (const [mode, code] of [["success", 0], ["nonzero", 7]] as const) { Deno.test({ name: `execute streams real ${mode} child output and records its exit`, @@ -436,15 +740,28 @@ Deno.test({ "runtime listing persists complete name inventory for large and small binaries", ignore: Deno.build.os !== "linux", async fn() { - const output = await Deno.makeTempDir(); + const output = await Deno.realPath(await Deno.makeTempDir()); try { + const isolation = await prepareTestIsolation(output, "discovery"); + const wrappedTables = new Set(); const discovery = await discoverRustCommands( [ fixtureTask("list-large", output), fixtureTask("list-small", output), ], 2, - { output }, + { + output, + run: async (task, label) => { + const wrapped = await isolation.wrapCommand(task, label); + wrappedTables.add(`${wrapped.env!.YOI_RUNTIME_DIR}/workers.json`); + return await execute(wrapped, { output, label, timeoutMs: 10_000 }); + }, + }, + ); + assert( + wrappedTables.size === 2, + "all discovery launches must use the callback", ); assert(discovery.inventory.complete); assert( @@ -635,6 +952,54 @@ if (import.meta.main) { ); } Deno.exit(mode === "list-nonzero" ? 7 : 0); + } else if (mode === "isolation-env") { + const env = Deno.env.toObject(); + const scratch = await Deno.makeTempDir({ dir: env.TMPDIR }); + const text = (bytes: Uint8Array) => new TextDecoder().decode(bytes); + async function git(args: string[]) { + return await new Deno.Command("git", { + args, + cwd: scratch, + stdout: "piped", + stderr: "piped", + }).output(); + } + try { + const ownedRepo = await git([ + "-C", + output, + "rev-parse", + "--show-toplevel", + ]); + assert( + ownedRepo.success && text(ownedRepo.stdout).trim() === output, + "owned ancestor repository fixture must be discoverable at its root", + ); + const ancestor = await git(["rev-parse", "--show-toplevel"]); + const init = await git(["init", "--quiet"]); + assert(init.success, text(init.stderr)); + const global = await git(["config", "--global", "--list"]); + assert(global.success, text(global.stderr)); + const author = await git(["var", "GIT_AUTHOR_IDENT"]); + assert(author.success, text(author.stderr)); + await Deno.writeTextFile( + `${env.YOI_RUNTIME_DIR}/workers.json`, + String(Deno.pid), + ); + console.log(JSON.stringify({ + env, + cwd: Deno.cwd(), + pid: Deno.pid, + ancestorDiscovered: ancestor.success, + globalConfig: text(global.stdout), + author: text(author.stdout), + hooks: Array.from(Deno.readDirSync(`${scratch}/.git/hooks`)), + info: Array.from(Deno.readDirSync(`${scratch}/.git/info`)), + })); + } finally { + await Deno.remove(scratch, { recursive: true }); + } + Deno.exit(0); } else if (mode === "runner-parent") { const cancellation = installCancellationHandlers(); try { diff --git a/tools/test.ts b/tools/test.ts index 14cb347d4..35bf8c769 100644 --- a/tools/test.ts +++ b/tools/test.ts @@ -1,9 +1,10 @@ -// Bound top-level suites/shards, not total processes; large shards use two test threads. +// Bound top-level suites/shards, not total processes; large shards normally use two threads. // Cargo selects targets/features. Retain all listed names; never skip cached successes. import { spawn } from "node:child_process"; import process from "node:process"; import { availableParallelism } from "node:os"; import { fileURLToPath } from "node:url"; +import { posix } from "node:path"; // Safety limits, NOT the separate 60s performance goal. export const SUITE_TIMEOUT_MS = 180_000; @@ -65,7 +66,19 @@ export function shardRustCommand( if (new Set(names).size !== names.length) { throw new Error("cannot shard duplicate test names"); } - if (names.length <= 72) return [{ task, selectedNames: [...names] }]; + // RuntimeDirSandbox changes process-global paths. Do not overlap its writes + // with restore guards in the same process, including small integration targets. + const serial = ["worker", "worker-runtime", "yoi-workspace-server"].includes( + task.name.split("/")[0], + ); + if (names.length <= 72) { + return [{ + task: serial + ? { ...task, args: [...task.args, "--test-threads", "1"] } + : task, + selectedNames: [...names], + }]; + } const count = Math.min(32, Math.ceil(names.length / 36)); const shards: string[][] = Array.from({ length: count }, () => []); names.forEach((name, index) => shards[index % count].push(name)); @@ -75,7 +88,13 @@ export function shardRustCommand( name: `${task.name}/shard-${ String(index + 1).padStart(2, "0") }-of-${count}`, - args: [...task.args, "--exact", "--test-threads", "2", ...selectedNames], + args: [ + ...task.args, + "--exact", + "--test-threads", + serial ? "1" : "2", + ...selectedNames, + ], }, selectedNames, })); @@ -84,18 +103,27 @@ export function shardRustCommand( export async function discoverRustCommands( commands: TestCommand[], jobs: number, - options: { output: string; signal?: AbortSignal }, + options: { + output: string; + signal?: AbortSignal; + run?: (task: TestCommand, label: string) => Promise; + }, ) { const binaries = await mapConcurrent(commands, jobs, async (task, index) => { - const listing = await execute({ + const listingTask = { ...task, name: `${task.name}/list`, args: [...task.args, "--list", "--format", "terse"], - }, { - ...options, - label: `list-${String(index).padStart(3, "0")}`, - timeoutMs: SUITE_TIMEOUT_MS, - }); + }; + const label = `list-${String(index).padStart(3, "0")}`; + const listing = await (options.run + ? options.run(listingTask, label) + : execute(listingTask, { + output: options.output, + signal: options.signal, + label, + timeoutMs: SUITE_TIMEOUT_MS, + })); let shards: RustShard[] = []; let error: string | undefined; try { @@ -124,7 +152,8 @@ export async function discoverRustCommands( error, selectedCases: error ? null - : shards.reduce((sum, shard) => sum + shard.selectedNames.length, 0), + : shards.reduce((sum, shard) => + sum + shard.selectedNames.length, 0), shards, }; }); @@ -426,12 +455,149 @@ export interface TestCommand { command: string; args: string[]; cwd: string; - env?: Record; + env?: Record; // Exact link from each execution result into inventory.json, even when two // Cargo targets happen to share the same package/target display name. inventoryRef?: { binaryIndex: number; shardIndex: number }; } +// Walk with lstat before creating any child: recursive mkdir/realPath alone +// would follow an existing symlink first. Reject aliases rather than silently +// canonicalizing a caller's checkout. As with the Rust guard, this assumes the +// owned checkout is not concurrently replaced by an adversarial filesystem actor. +async function ownedTestDirectory(checkout: string, relative: string) { + if ( + !posix.isAbsolute(checkout) || checkout === "/" || + posix.normalize(checkout) !== checkout + ) throw new Error("test checkout must be an absolute normal-only path"); + + async function checkDirectory(path: string, owned: boolean) { + const stat = await Deno.lstat(path); + if (!stat.isDirectory || stat.isSymlink) { + throw new Error( + `test isolation refuses non-directory or symlink: ${path}`, + ); + } + if (owned && process.getuid && stat.uid !== process.getuid()) { + throw new Error( + `test isolation directory is not owned by this user: ${path}`, + ); + } + if (await Deno.realPath(path) !== path) { + throw new Error( + `test isolation refuses non-canonical directory: ${path}`, + ); + } + } + let checkoutPath = ""; + for (const part of checkout.slice(1).split("/")) { + checkoutPath += `/${part}`; + await checkDirectory(checkoutPath, false); + } + let path = checkout; + for (const part of relative.split("/")) { + if (!part || part === "." || part === "..") { + throw new Error("invalid owned test directory component"); + } + path += `/${part}`; + try { + await Deno.mkdir(path, { mode: 0o700 }); + } catch (error) { + if (!(error instanceof Deno.errors.AlreadyExists)) throw error; + } + await checkDirectory(path, true); + } + return path; +} + +/** Prepare child-only isolation for the maintained normal-test entrypoint. + * Each wrap gets a fresh allocation table, even for repeated/sanitized labels or + * concurrent runners using the same output id. Resources stay under target for + * diagnosis; callers with temporary checkouts own their recursive cleanup. + */ +export async function prepareTestIsolation(checkout: string, runId: string) { + const ownedDirectory = (relative: string) => + ownedTestDirectory(checkout, relative); + const prefix = "target/restore-test-environment"; + await ownedDirectory(prefix); + // Keep tempfile Unix-socket references short. tempfile owns unique resources + // here; neither runners nor suites remove/replace this shared directory. + const tmp = await ownedDirectory("target/t"); + const digest = new Uint8Array( + await crypto.subtle.digest("SHA-256", new TextEncoder().encode(runId)), + ); + const id = Array.from( + digest.slice(0, 6), + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""); + const relativeRoot = `${prefix}/runner-${id}-${crypto.randomUUID()}`; + const root = await ownedDirectory(relativeRoot); + let next = 0; + return { + root, + async wrapCommand(task: TestCommand, label: string): Promise { + const safeLabel = label.replaceAll(/[^a-zA-Z0-9_-]/g, "-").slice(0, 40) || + "command"; + const relative = `${relativeRoot}/${safeLabel}-${next++}`; + // Recheck shared ancestors as well as the command namespace on every wrap. + await ownedDirectory("target/t"); + const namespace = await ownedDirectory(relative); + for ( + const dir of [ + "config", + "data", + "run", + "home", + "git-template/info", + "git-template/hooks", + ] + ) { + await ownedDirectory(`${relative}/${dir}`); + } + const inherited = { ...process.env, ...task.env }; + const env: NonNullable = { ...task.env }; + // Undefined shadows inherited entries in execute's merge; node spawn omits + // them. Strip indexed Git config and less common Git path overrides too. + for (const key of Object.keys(inherited)) { + if (/^GIT_/i.test(key) || /(^|_)proxy$/i.test(key)) { + env[key] = undefined; + } + } + Object.assign(env, { + HOME: `${namespace}/home`, + YOI_HOME: `${namespace}/data`, + YOI_CONFIG_DIR: `${namespace}/config`, + YOI_DATA_DIR: `${namespace}/data`, + YOI_RUNTIME_DIR: `${namespace}/run`, + TMPDIR: tmp, + TMP: tmp, + TEMP: tmp, + GIT_TEMPLATE_DIR: `${namespace}/git-template`, + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_SYSTEM: "/dev/null", + GIT_CONFIG_NOSYSTEM: "1", + GIT_ATTR_NOSYSTEM: "1", + GIT_CEILING_DIRECTORIES: checkout, + GIT_AUTHOR_NAME: "Normal test runner", + GIT_AUTHOR_EMAIL: "normal-tests@example.invalid", + GIT_COMMITTER_NAME: "Normal test runner", + GIT_COMMITTER_EMAIL: "normal-tests@example.invalid", + RUSTC_WRAPPER: "", + RUSTC_WORKSPACE_WRAPPER: "", + SCCACHE_DISABLE: "1", + }); + // HOME is isolated, but toolchain/cache discovery must keep its ordinary + // host locations (these are compiler settings, not live product state). + for (const key of ["CARGO_HOME", "RUSTUP_HOME"] as const) { + const fallback = key === "CARGO_HOME" ? ".cargo" : ".rustup"; + env[key] = inherited[key] || + (process.env.HOME ? `${process.env.HOME}/${fallback}` : undefined); + } + return { ...task, env }; + }, + }; +} + export function rustTestCommands( artifacts: CargoArtifact[], packages: { id: string; name: string; manifest_path: string }[], @@ -497,16 +663,26 @@ async function main() { "", ); const jobs = parseJobs(Deno.args); - const output = `${root}/target/normal-tests/${ + const outputId = `${ new Date().toISOString().replaceAll(":", "-") - }`; - await Deno.mkdir(output, { recursive: true }); + }-${crypto.randomUUID()}`; + const isolation = await prepareTestIsolation(root, outputId); + const output = await ownedTestDirectory( + root, + `target/normal-tests/${outputId}`, + ); const cancellation = installCancellationHandlers(); - const run = ( + const run = async ( task: TestCommand, label: string, timeoutMs = SUITE_TIMEOUT_MS, - ) => execute(task, { output, label, timeoutMs, signal: cancellation.signal }); + ) => + execute(await isolation.wrapCommand(task, label), { + output, + label, + timeoutMs, + signal: cancellation.signal, + }); const summary: Record = { phase: "metadata", failed: false }; let executionStart: number | undefined; const writeSummary = () => @@ -516,7 +692,7 @@ async function main() { { jobs, concurrency: - "top-level suites/shards; large Rust shards use 2 test threads, other suites retain internal parallelism", + "top-level suites/shards; worker, worker-runtime and yoi-workspace-server use 1 test thread; other large Rust shards use 2, other suites retain internal parallelism", platform: "linux", suiteTimeoutMs: SUITE_TIMEOUT_MS, buildTimeoutMs: BUILD_TIMEOUT_MS, @@ -600,6 +776,7 @@ async function main() { const discovery = await discoverRustCommands(rust, jobs, { output, signal: cancellation.signal, + run, }); summary.discoverySeconds = (performance.now() - discoveryStart) / 1000; summary.inventory = "inventory.json"; @@ -655,6 +832,7 @@ async function main() { "--allow-write", "--allow-run", "--allow-env", + "--allow-sys=uid", "tools/test.test.ts", "tools/check-state-contract-names.test.ts", ], @@ -663,7 +841,7 @@ async function main() { ...discovery.tasks, ]; console.log( - `Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (large Rust shards: 2 test threads; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`, + `Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (worker/worker-runtime/yoi-workspace-server: 1 test thread; other large Rust shards: 2; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`, ); const results: ReturnType[] = []; summary.results = results; From 6776973e4c3960ee4e072a963259ff87ee6fc065 Mon Sep 17 00:00:00 2001 From: Hare Date: Thu, 22 Oct 2026 23:17:25 +0900 Subject: [PATCH 20/46] fix: preserve original session coordinates through capture selection --- crates/session-store/src/capture.rs | 50 ++++--- crates/session-store/src/capture/body.rs | 52 +++++++- .../session-store/tests/capture_contract.rs | 125 +++++++++++++++++- .../src/feature/builtin/flow_transition.rs | 2 +- .../src/feature/builtin/memory_lifecycle.rs | 99 +++++++++++++- .../feature/builtin/memory_staging_output.rs | 49 +++++++ .../src/feature/builtin/subjektiv_memory.rs | 6 +- crates/worker/src/session_capture.rs | 15 ++- crates/worker/src/worker.rs | 22 ++- 9 files changed, 368 insertions(+), 52 deletions(-) diff --git a/crates/session-store/src/capture.rs b/crates/session-store/src/capture.rs index 4c9b4d3bc..07da4c1f7 100644 --- a/crates/session-store/src/capture.rs +++ b/crates/session-store/src/capture.rs @@ -237,6 +237,7 @@ pub struct SessionEntryEvidence { pub segment_id: String, pub entry_ref: SessionEntryRef, pub origin: SessionEntryProvenance, + /// Inclusive original-source positions in `segment_id`, not view/UI order. pub entry_range: [u64; 2], pub kind: ReferenceKind, pub tool_part: Option, @@ -259,6 +260,7 @@ pub struct SessionCapture { session_id: String, segment_id: String, capture_id: String, + view_entry_count: u64, entries: Arc>, overview: Vec, index: Vec, @@ -271,6 +273,7 @@ impl SessionCapture { snapshot: SessionSnapshot, ) -> Self { let segment_id = segment_id.into(); + let view_entry_count = snapshot.entries.len() as u64; let entries = snapshot .entries .into_iter() @@ -341,7 +344,7 @@ impl SessionCapture { }) }) .collect(); - Self::from_projected(session_id.into(), segment_id, entries) + Self::from_projected(session_id.into(), segment_id, view_entry_count, entries) } pub fn new( @@ -350,6 +353,7 @@ impl SessionCapture { items: Vec, ) -> Self { let segment_id = segment_id.into(); + let view_entry_count = items.len() as u64; let entries = items .into_iter() .enumerate() @@ -371,15 +375,19 @@ impl SessionCapture { }) }) .collect(); - Self::from_projected(session_id.into(), segment_id, entries) + Self::from_projected(session_id.into(), segment_id, view_entry_count, entries) } - pub fn from_history_entries( + /// Project a complete Host-fixed history whose source positions start at zero. + /// Select a suffix with `select_order_range` on the resulting value instead + /// of passing a rebased history vector: annotations do not carry source positions. + pub fn from_complete_history( session_id: impl Into, segment_id: impl Into, entries: Vec>, ) -> Self { let segment_id = segment_id.into(); + let view_entry_count = entries.len() as u64; let entries = entries .into_iter() .enumerate() @@ -432,14 +440,20 @@ impl SessionCapture { }) }) .collect(); - Self::from_projected(session_id.into(), segment_id, entries) + Self::from_projected(session_id.into(), segment_id, view_entry_count, entries) } - fn from_projected(session_id: String, segment_id: String, dtos: Vec) -> Self { + fn from_projected( + session_id: String, + segment_id: String, + view_entry_count: u64, + dtos: Vec, + ) -> Self { let dto = CaptureDto { version: CAPTURE_VERSION, session_id: session_id.clone(), segment_id: segment_id.clone(), + view_entry_count, entries: dtos, }; let capture_id = dto.fingerprint(); @@ -589,6 +603,7 @@ impl SessionCapture { session_id, segment_id, capture_id, + view_entry_count, entries: Arc::new(entries), overview, index, @@ -604,7 +619,8 @@ impl SessionCapture { &self.overview } - pub fn source_index_for_ref(&self, reference: &SessionEntryRef) -> Option { + /// Capture-local UI position, not the original source position used by evidence. + pub fn view_order_for_ref(&self, reference: &SessionEntryRef) -> Option { self.index .iter() .find(|entry| entry.id == *reference) @@ -624,12 +640,12 @@ impl SessionCapture { let min_entry_index = options .from .as_ref() - .and_then(|reference| self.source_index_for_ref(reference)) + .and_then(|reference| self.view_order_for_ref(reference)) .unwrap_or_else(|| options.min_entry_index.unwrap_or(0)); let max_entry_index = options .through .as_ref() - .and_then(|reference| self.source_index_for_ref(reference)) + .and_then(|reference| self.view_order_for_ref(reference)) .unwrap_or(u64::MAX); let mut skipped = 0usize; let mut hits = Vec::new(); @@ -787,24 +803,16 @@ impl SessionCapture { if entry.use_policy.is_some_and(|p| !p.memory_evidence) { return None; } - let item = &self + let captured = self .entries .iter() - .find(|captured| captured.dto.order == entry.entry_range[0])? - .item; - let excerpt = render_item(item, entry, ReadDetail::Compact, 2 * 1024); + .find(|captured| captured.dto.order == entry.entry_range[0])?; + let excerpt = render_item(&captured.item, entry, ReadDetail::Compact, 2 * 1024); Some(SessionEntryEvidence { - segment_id: self - .entries - .iter() - .find(|captured| captured.dto.order == entry.entry_range[0])? - .dto - .source - .segment_id - .clone(), + segment_id: captured.dto.source.segment_id.clone(), entry_ref: entry.id.clone(), origin: entry.origin.clone(), - entry_range: entry.entry_range, + entry_range: [captured.dto.source.entry_index; 2], kind: entry.kind, tool_part: entry.tool_part, label: entry.label.clone(), diff --git a/crates/session-store/src/capture/body.rs b/crates/session-store/src/capture/body.rs index 747586889..eb7768622 100644 --- a/crates/session-store/src/capture/body.rs +++ b/crates/session-store/src/capture/body.rs @@ -307,6 +307,8 @@ pub struct CaptureDto { pub version: u32, pub session_id: String, pub segment_id: String, + /// Extent of view order, including positions omitted by public projection. + pub view_entry_count: u64, pub entries: Vec, } @@ -409,6 +411,9 @@ impl CaptureDto { { return Err(CaptureError::Invalid("entry identity")); } + if entry.order >= self.view_entry_count { + return Err(CaptureError::Invalid("entry outside view extent")); + } if previous.is_some_and(|order| order >= entry.order) { return Err(CaptureError::Invalid("entry order")); } @@ -598,6 +603,7 @@ impl SessionCapture { version: CAPTURE_VERSION, session_id: self.session_id.clone(), segment_id: self.segment_id.clone(), + view_entry_count: self.view_entry_count, entries: self.entries.iter().map(|e| e.dto.clone()).collect(), } } @@ -607,6 +613,7 @@ impl SessionCapture { Ok(Self::from_projected( dto.session_id, dto.segment_id, + dto.view_entry_count, dto.entries, )) } @@ -618,11 +625,48 @@ impl SessionCapture { Self::try_from_dto(CaptureDto::decode(bytes, limits)?, limits) } + /// Select fixed view positions. UI order is rebased, while every original + /// source identity/position, body, provenance and policy stays unchanged. + pub fn select_order_range(&self, range: std::ops::Range) -> Result { + if range.start > range.end || range.end > self.view_entry_count { + return Err(CaptureError::InvalidRange); + } + let entries = self + .entries() + .filter(|entry| range.contains(&entry.order)) + .map(|entry| { + let mut entry = entry.clone(); + entry.order -= range.start; + entry + }) + .collect(); + Ok(Self::from_projected( + self.session_id.clone(), + self.segment_id.clone(), + range.end - range.start, + entries, + )) + } + + /// Host-owned target selection may exclude entries without renumbering either + /// the fixed view or its original sources. No source I/O or business policy is applied. + pub fn retain_entries(&self, mut include: impl FnMut(&CaptureEntryDto) -> bool) -> Self { + Self::from_projected( + self.session_id.clone(), + self.segment_id.clone(), + self.view_entry_count, + self.entries() + .filter(|entry| include(entry)) + .cloned() + .collect(), + ) + } + pub fn scoped_ref( &self, reference: &SessionEntryRef, ) -> Result { - if self.source_index_for_ref(reference).is_none() { + if self.view_order_for_ref(reference).is_none() { return Err(CaptureError::UnknownReference); } Ok(ScopedSessionEntryRef { @@ -679,7 +723,7 @@ impl SessionCapture { .from .as_ref() .map(|r| { - self.source_index_for_ref(r) + self.view_order_for_ref(r) .ok_or(CaptureError::UnknownReference) }) .transpose()?; @@ -687,7 +731,7 @@ impl SessionCapture { .through .as_ref() .map(|r| { - self.source_index_for_ref(r) + self.view_order_for_ref(r) .ok_or(CaptureError::UnknownReference) }) .transpose()?; @@ -710,7 +754,7 @@ impl SessionCapture { match selector { ReadSelector::Id(id) => { let reference = SessionEntryRef::parse(id).ok_or(CaptureError::UnknownReference)?; - self.source_index_for_ref(&reference) + self.view_order_for_ref(&reference) .ok_or(CaptureError::UnknownReference)?; } ReadSelector::EntryRange([start, end]) => { diff --git a/crates/session-store/tests/capture_contract.rs b/crates/session-store/tests/capture_contract.rs index a605d6d6e..6925fb9df 100644 --- a/crates/session-store/tests/capture_contract.rs +++ b/crates/session-store/tests/capture_contract.rs @@ -77,7 +77,7 @@ fn public_consumer_roundtrip_retains_body_original_source_order_origin_and_indep Some(HistoryInputMetadata::new(usage(false, false, true), None, None).unwrap()), ), ]; - let capture = SessionCapture::from_history_entries("synthetic-session", "selected", entries); + let capture = SessionCapture::from_complete_history("synthetic-session", "selected", entries); let mut dto = capture.to_dto(); // The checked boundary also accepts entries retained from different original segments. dto.entries[0].source.segment_id = "original-a".into(); @@ -115,6 +115,7 @@ fn public_consumer_roundtrip_retains_body_original_source_order_origin_and_indep assert!(restored.evidence_for("Erecording").is_none()); let evidence = restored.evidence_for("Ememory").unwrap(); assert_eq!(evidence.segment_id, "original-b"); + assert_eq!(evidence.entry_range, [3, 3]); assert!(evidence.excerpt.contains("memory-only")); let recorded = restored .slice(CapturePurpose::Recording, None, 10, 16_000) @@ -159,7 +160,7 @@ fn common_data_drops_reasoning_raw_system_and_private_payload_even_when_unindexe HistoryInputMetadata::new(usage(true, true, true), Some(payload.clone()), Some(public)) .unwrap(); let private = HistoryInputMetadata::new(usage(true, true, true), Some(payload), None).unwrap(); - let capture = SessionCapture::from_history_entries( + let capture = SessionCapture::from_complete_history( "s", "segment", vec![ @@ -195,7 +196,7 @@ fn common_data_drops_reasoning_raw_system_and_private_payload_even_when_unindexe // Filtered entries do not renumber the public source positions. assert_eq!(capture.to_dto().entries[0].order, 3); assert_eq!( - capture.source_index_for_ref(&SessionEntryRef::parse("Eordinary").unwrap()), + capture.view_order_for_ref(&SessionEntryRef::parse("Eordinary").unwrap()), Some(4) ); } @@ -318,6 +319,7 @@ fn malformed_dtos_and_oversized_representations_are_rejected() { .is_err() ); let mut reversed = capture.to_dto(); + reversed.view_entry_count = 2; let mut second = reversed.entries[0].clone(); second.entry_id = "second".into(); second.source.entry_index = 4; @@ -325,6 +327,16 @@ fn malformed_dtos_and_oversized_representations_are_rejected() { reversed.entries[0].order = 1; reversed.entries.push(second); assert!(SessionCapture::try_from_dto(reversed, CaptureLimits::default()).is_err()); + let mut duplicate_source = capture.to_dto(); + duplicate_source.view_entry_count = 2; + let mut second = duplicate_source.entries[0].clone(); + second.entry_id = "distinct-id".into(); + second.order = 1; + duplicate_source.entries.push(second); + assert!(matches!( + SessionCapture::try_from_dto(duplicate_source, CaptureLimits::default()), + Err(CaptureError::Invalid("duplicate source position")) + )); } #[test] @@ -503,7 +515,7 @@ fn snapshot_immutability_and_scoped_operations_reject_foreign_or_unknown_refs_an 2 ); let unknown = SessionEntryRef::parse("E99999999").unwrap(); - assert_eq!(capture.source_index_for_ref(&unknown), None); + assert_eq!(capture.view_order_for_ref(&unknown), None); assert!(matches!( capture.scoped_ref(&unknown), Err(CaptureError::UnknownReference) @@ -592,3 +604,108 @@ fn slices_preserve_empty_and_long_result_fields_under_item_and_metadata_budgets( Err(CaptureError::BudgetTooSmall) )); } + +#[test] +fn evidence_uses_original_segment_positions_after_roundtrip_and_view_selection() { + let full = SessionCapture::new( + "s", + "selected", + vec![ + Item::user_message("first"), + Item::assistant_message("second"), + ], + ); + let mut dto = full.to_dto(); + dto.entries[0].source.segment_id = "original-a".into(); + dto.entries[0].source.entry_index = 80; + dto.entries[1].source.segment_id = "original-b".into(); + dto.entries[1].source.entry_index = 3; + let encoded = dto.encode(CaptureLimits::default()).unwrap(); + let restored = SessionCapture::decode(&encoded, CaptureLimits::default()).unwrap(); + for (id, segment, index, order) in [ + ("E00000000", "original-a", 80, 0), + ("E00000001", "original-b", 3, 1), + ] { + let evidence = restored.evidence_for(id).unwrap(); + assert_eq!(evidence.segment_id, segment); + assert_eq!(evidence.entry_range, [index; 2]); + let read = restored.read(ReadSelector::Id(id), ReadOptions::default()); + assert_eq!(read.entries[0].entry_range, [order; 2]); + } + let selected = restored.select_order_range(1..2).unwrap(); + assert_eq!(selected.overview()[0].entry_range, [0, 0]); + let evidence = selected.evidence_for("E00000001").unwrap(); + assert_eq!(evidence.segment_id, "original-b"); + assert_eq!(evidence.entry_range, [3, 3]); + let roundtrip = SessionCapture::decode( + &selected.encode(CaptureLimits::default()).unwrap(), + CaptureLimits::default(), + ) + .unwrap(); + assert_eq!(roundtrip.to_dto(), selected.to_dto()); + assert_eq!( + roundtrip.evidence_for("E00000001").unwrap().entry_range, + [3, 3] + ); +} + +#[test] +fn fixed_history_selection_preserves_original_coordinates_and_checks_private_position_bounds() { + let history = vec![ + history("old", Item::user_message("old"), None), + history("hidden", Item::reasoning("PRIVATE"), None), + history("selected", Item::user_message("selected"), None), + history("private-tail", Item::system_message("PRIVATE"), None), + ]; + let full = SessionCapture::from_complete_history("s", "seg", history); + let suffix = full.select_order_range(1..4).unwrap(); + let original = full + .entries() + .find(|entry| entry.entry_id == "selected") + .unwrap(); + let selected = suffix.entries().next().unwrap(); + assert_eq!(selected.entry_id, original.entry_id); + assert_eq!(selected.source, original.source); + assert_eq!(selected.body, original.body); + assert_eq!(selected.policy, original.policy); + assert!(suffix.evidence_for("Eold").is_none()); + assert_eq!(suffix.search(&SearchOptions::default()).len(), 1); + assert_eq!( + suffix.view_order_for_ref(&SessionEntryRef::parse("Eselected").unwrap()), + Some(1) + ); + assert_eq!(selected.source.entry_index, 2); + assert_eq!(selected.order, 1); + assert_eq!(suffix.to_dto().view_entry_count, 3); + assert_eq!( + suffix.evidence_for("Eselected").unwrap().entry_range, + [2, 2] + ); + let nested = suffix.select_order_range(1..2).unwrap(); + assert_eq!(nested.entries().next().unwrap().order, 0); + assert_eq!(nested.entries().next().unwrap().source, original.source); + let restored = SessionCapture::decode( + &suffix.encode(CaptureLimits::default()).unwrap(), + CaptureLimits::default(), + ) + .unwrap(); + assert_eq!(restored.to_dto(), suffix.to_dto()); + assert!(full.select_order_range(4..5).is_err()); + let reversed = std::ops::Range { start: 3, end: 2 }; + assert!(full.select_order_range(reversed).is_err()); + let hidden_only = full.select_order_range(3..4).unwrap(); + assert_eq!(hidden_only.entries().len(), 0); + assert_eq!(hidden_only.to_dto().view_entry_count, 1); + let mut invalid = full.to_dto(); + invalid.view_entry_count = 2; // the selected entry's view order is outside this extent + assert!(SessionCapture::try_from_dto(invalid, CaptureLimits::default()).is_err()); + assert_eq!( + full.retain_entries(|entry| entry.entry_id == "selected") + .entries() + .next() + .unwrap() + .source, + original.source + ); + assert_eq!(full.to_dto().view_entry_count, 4); +} diff --git a/crates/worker/src/feature/builtin/flow_transition.rs b/crates/worker/src/feature/builtin/flow_transition.rs index 4c9cd98e3..39115d664 100644 --- a/crates/worker/src/feature/builtin/flow_transition.rs +++ b/crates/worker/src/feature/builtin/flow_transition.rs @@ -474,7 +474,7 @@ where self.segment_id, &entries, )?; - Ok(SessionCapture::from_history_entries( + Ok(SessionCapture::from_complete_history( self.session_id.to_string(), self.segment_id.to_string(), history, diff --git a/crates/worker/src/feature/builtin/memory_lifecycle.rs b/crates/worker/src/feature/builtin/memory_lifecycle.rs index 7db770eb0..7af3b867e 100644 --- a/crates/worker/src/feature/builtin/memory_lifecycle.rs +++ b/crates/worker/src/feature/builtin/memory_lifecycle.rs @@ -456,11 +456,13 @@ impl MemoryLifecycleTask { .await; return Ok(()); } - let view = SessionCapture::from_history_entries( + let view = SessionCapture::from_complete_history( capture.session_id.clone(), capture.segment_id.clone(), - capture.history[history_start..history_end].to_vec(), - ); + capture.history.clone(), + ) + .select_order_range(history_start as u64..history_end as u64) + .map_err(hook_internal)?; let start_entry = pointer .as_ref() .map(|pointer| pointer.processed_through_entry + 1) @@ -1557,7 +1559,7 @@ permission = "write" #[tokio::test] async fn run_committed_background_task_stages_non_empty_extraction_and_commits_pointer() { let source = capture(2, 250); - let entry_ref = SessionCapture::from_history_entries( + let entry_ref = SessionCapture::from_complete_history( source.session_id.clone(), source.segment_id.clone(), source.history.clone(), @@ -1610,10 +1612,95 @@ permission = "write" ); } + // Real background extraction -> common fixed selection -> StageMemoryCandidate + // -> typed backend operation and pointer write. Only model/backend transport + // are replaced; the second extraction uses the first run's saved pointer. + #[tokio::test] + async fn second_extraction_preserves_original_evidence_positions_after_nonzero_pointer() { + let first = capture(2, 250); + let mut second = capture(4, 750); + second.history[..2].clone_from_slice(&first.history); + second + .usage_history + .insert(0, first.usage_history[0].clone()); + let selected_ref = + crate::SessionEntryRef::from_history_entry_id(&second.history[2].annotation.entry_id) + .to_string(); + let extension_writes = Arc::new(Mutex::new(Vec::new())); + let workspace_client = Arc::new(RecordingWorkspaceClient::default()); + let (event_tx, _) = broadcast::channel(64); + run_background_task(test_task( + first, + Box::new(ScriptClient::new(vec![ + finish_empty_events("first-empty"), + completed_events(), + ])), + extension_writes.clone(), + event_tx.clone(), + workspace_client.clone(), + )) + .await; + { + let writes = extension_writes.lock().unwrap(); + assert_eq!(writes.len(), 1); + let pointer: memory::ExtractPointerPayload = + serde_json::from_value(writes[0].2.clone()).unwrap(); + assert_eq!(pointer.processed_through_history_len, 2); + second + .extensions + .push((writes[0].1.clone(), writes[0].2.clone())); + } + run_background_task(test_task( + second, + Box::new(ScriptClient::new(vec![ + stage_candidate_events("second-stage", &selected_ref), + finish_events("second-finish", 1), + completed_events(), + ])), + extension_writes.clone(), + event_tx, + workspace_client.clone(), + )) + .await; + let operations = workspace_client + .requests + .lock() + .unwrap() + .iter() + .filter_map(|request| { + request.body.as_deref().and_then(|body| { + serde_json::from_str::(body).ok() + }) + }) + .filter_map(|operation| match operation { + memory::backend::MemoryBackendOperation::StageCandidate(operation) => { + Some(operation) + } + _ => None, + }) + .collect::>(); + assert_eq!(operations.len(), 1); + let operation = &operations[0]; + assert_eq!(operation.source.range, [2, 3]); + assert_eq!(operation.evidence[0].id, selected_ref); + assert_eq!(operation.evidence[0].entry_range, Some([2, 2])); + assert_eq!( + operation.source_refs[0].segment_id.as_deref(), + Some("segment-1") + ); + assert_eq!(operation.source_refs[0].entry_range, Some([2, 2])); + let writes = extension_writes.lock().unwrap(); + assert_eq!(writes.len(), 2); + let pointer: memory::ExtractPointerPayload = + serde_json::from_value(writes[1].2.clone()).unwrap(); + assert_eq!(pointer.processed_through_history_len, 4); + assert_eq!(pointer.processed_through_entry, 3); + } + #[tokio::test] async fn subjektiv_extraction_stages_once_and_commits_its_independent_pointer() { let source = capture(2, 250); - let entry_ref = SessionCapture::from_history_entries( + let entry_ref = SessionCapture::from_complete_history( source.session_id.clone(), source.segment_id.clone(), source.history.clone(), @@ -1722,7 +1809,7 @@ permission = "write" #[tokio::test] async fn subjektiv_candidate_remains_staged_when_later_extraction_fails_without_pointer() { let source = capture(2, 250); - let entry_ref = SessionCapture::from_history_entries( + let entry_ref = SessionCapture::from_complete_history( source.session_id.clone(), source.segment_id.clone(), source.history.clone(), diff --git a/crates/worker/src/feature/builtin/memory_staging_output.rs b/crates/worker/src/feature/builtin/memory_staging_output.rs index dd830349d..2ac95d8bf 100644 --- a/crates/worker/src/feature/builtin/memory_staging_output.rs +++ b/crates/worker/src/feature/builtin/memory_staging_output.rs @@ -639,4 +639,53 @@ mod tests { .unwrap_err(); assert!(format!("{error:?}").contains("unknown SessionEntryRef")); } + #[tokio::test] + async fn staging_roundtripped_mixed_sources_uses_original_evidence_ranges_not_view_order() { + let client = Arc::new(RecordingSubjektivClient::default()); + let mut dto = SessionCapture::new( + "session", + "view", + vec![ + Item::user_message("first"), + Item::assistant_message("second"), + ], + ) + .to_dto(); + dto.entries[0].source.segment_id = "original-a".into(); + dto.entries[0].source.entry_index = 80; + dto.entries[1].source.segment_id = "original-b".into(); + dto.entries[1].source.entry_index = 3; + let limits = crate::session_capture::CaptureLimits::default(); + let view = SessionCapture::decode(&dto.encode(limits).unwrap(), limits).unwrap(); + let state = MemoryStagingOutputState::new_subjektiv( + view, + crate::subjektiv::test_connection(client.clone()), + SourceRef { + segment_id: "view".into(), + range: [0, 1], + }, + "run".into(), + ); + StageMemoryCandidateTool { state }.execute( + r#"{"kind":"decision","claim":"claim","why_useful":"useful","entry_refs":["E00000000","E00000001"]}"#, + agen::tool::ToolExecutionContext::direct(), + ).await.unwrap(); + let requests = client.requests.lock().unwrap(); + let request: server_api::SubjektivStageCandidateRequest = + serde_json::from_str(requests[0].body.as_deref().unwrap()).unwrap(); + for (index, segment, position) in [(0, "original-a", 80), (1, "original-b", 3)] { + assert_eq!( + request.operation.evidence[index].entry_range, + Some([position; 2]) + ); + assert_eq!( + request.operation.source_refs[index].segment_id.as_deref(), + Some(segment) + ); + assert_eq!( + request.operation.source_refs[index].entry_range, + Some([position; 2]) + ); + } + } } diff --git a/crates/worker/src/feature/builtin/subjektiv_memory.rs b/crates/worker/src/feature/builtin/subjektiv_memory.rs index 82ce970f9..9b5f3c387 100644 --- a/crates/worker/src/feature/builtin/subjektiv_memory.rs +++ b/crates/worker/src/feature/builtin/subjektiv_memory.rs @@ -571,7 +571,7 @@ impl SubjektivMemoryState { .capture() .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?; let session_id = capture.session_id.clone(); - let view = SessionCapture::from_history_entries( + let view = SessionCapture::from_complete_history( capture.session_id.clone(), capture.segment_id, capture.history, @@ -680,7 +680,7 @@ fn replay_pending_committed_memory_with_policy( .capture .capture() .map_err(|error| HookError::new(HookErrorCategory::Dependency, error.to_string()))?; - let view = SessionCapture::from_history_entries( + let view = SessionCapture::from_complete_history( capture.session_id.clone(), capture.segment_id.clone(), capture.history.clone(), @@ -815,7 +815,7 @@ fn resolve_entry_refs( capture: &crate::feature::session::CommittedSessionCapture, entry_refs: &[String], ) -> Result, ToolError> { - let view = SessionCapture::from_history_entries( + let view = SessionCapture::from_complete_history( capture.session_id.clone(), capture.segment_id.clone(), capture.history.clone(), diff --git a/crates/worker/src/session_capture.rs b/crates/worker/src/session_capture.rs index 979c8d44a..e24d8c1e0 100644 --- a/crates/worker/src/session_capture.rs +++ b/crates/worker/src/session_capture.rs @@ -30,7 +30,7 @@ mod tests { }, ), ]; - let capture = SessionCapture::from_history_entries("session", "segment", entries); + let capture = SessionCapture::from_complete_history("session", "segment", entries); let overview = capture.overview(); assert_eq!(overview.len(), 1); assert!(matches!( @@ -54,8 +54,9 @@ mod tests { }, ); let expected_ref = SessionEntryRef::from_history_entry_id(&retained.annotation.entry_id); - let before = SessionCapture::from_history_entries("session", "old", vec![retained.clone()]); - let after = SessionCapture::from_history_entries("session", "new", vec![retained]); + let before = + SessionCapture::from_complete_history("session", "old", vec![retained.clone()]); + let after = SessionCapture::from_complete_history("session", "new", vec![retained]); assert_eq!(before.overview()[0].id, expected_ref); assert_eq!(after.overview()[0].id, expected_ref); assert_eq!( @@ -96,7 +97,7 @@ mod tests { .iter() .map(|entry| SessionEntryRef::from_history_entry_id(&entry.annotation.entry_id)) .collect::>(); - let capture = SessionCapture::from_history_entries("session", "segment", entries); + let capture = SessionCapture::from_complete_history("session", "segment", entries); assert!(capture.overview().is_empty()); assert!( capture @@ -235,7 +236,7 @@ mod tests { let snapshot: SessionSnapshot = serde_json::from_value(serde_json::to_value(snapshot).unwrap()).unwrap(); for capture in [ - SessionCapture::from_history_entries( + SessionCapture::from_complete_history( session.to_string(), segment.to_string(), restored, @@ -369,7 +370,7 @@ mod tests { assert_eq!(entry.item, items[index]); assert_eq!(entry.annotation, history[index].metadata); } - let capture = SessionCapture::from_history_entries( + let capture = SessionCapture::from_complete_history( session.to_string(), fork.to_string(), restored, @@ -757,7 +758,7 @@ mod tests { let restored = crate::session_history::restore_history_entries(session, first, &log).unwrap(); let fixed = - SessionCapture::from_history_entries(session.to_string(), first.to_string(), restored); + SessionCapture::from_complete_history(session.to_string(), first.to_string(), restored); let overview = fixed.overview()[0].clone(); let evidence = fixed.evidence_for(overview.id.as_str()).unwrap(); let extract = render_extract_input(&fixed); diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 859784866..d47dc40c9 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -7338,6 +7338,20 @@ impl Worker { .cloned() .collect::>(); project_compactor_capture_attachments(&mut entries_to_summarise); + // Project before Host selection so omitted summary/reference inputs never + // rebase the original source positions of the retained prefix entries. + let mut exploration_entries = history_entries[..retain_from].to_vec(); + project_compactor_capture_attachments(&mut exploration_entries); + let summarised_ids = entries_to_summarise + .iter() + .map(|entry| entry.annotation.entry_id.0.as_str()) + .collect::>(); + let exploration_view = crate::session_capture::SessionCapture::from_complete_history( + self.session_id().to_string(), + self.segment_id().to_string(), + exploration_entries, + ) + .retain_entries(|entry| summarised_ids.contains(entry.entry_id.as_str())); let items_to_summarise = entries_to_summarise .iter() .map(|entry| entry.item.clone()) @@ -7500,11 +7514,7 @@ impl Worker { .with_module( crate::feature::builtin::session_explore::SessionExploreFeature::new( crate::feature::builtin::session_explore::SessionExploreState::new( - crate::session_capture::SessionCapture::from_history_entries( - self.session_id().to_string(), - self.segment_id().to_string(), - entries_to_summarise.clone(), - ), + exploration_view, ), ), ) @@ -17375,7 +17385,7 @@ permission = "write" producer: "arbitrary-context-provider".into() } ); - let view = crate::session_capture::SessionCapture::from_history_entries( + let view = crate::session_capture::SessionCapture::from_complete_history( "session", "startup", replay, ); let extract = From 907392cce8d7915913c2ab785056867ebd76246d Mon Sep 17 00:00:00 2001 From: Hare Date: Fri, 23 Oct 2026 00:21:02 +0900 Subject: [PATCH 21/46] fix: preserve restore effects and fence prepared attachment replay --- crates/worker-runtime/src/runtime.rs | 112 +++- crates/worker-runtime/src/worker_backend.rs | 69 +- .../worker_backend_restore_candidate_tests.rs | 206 ++++++ .../worker_backend_restore_failure_tests.rs | 595 +++++++++++++++++- 4 files changed, 952 insertions(+), 30 deletions(-) diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index eaec092dc..fc53df360 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -2150,6 +2150,16 @@ impl Runtime { { return Err(conflict()); } + if let Err(error) = owner.validate_workdir_preparation_authority(&candidate) { + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_authority_changed", + sanitize_worker_create_failure_message(&error.to_string()), + )), + None, + )); + } } else { // Startup/older persisted pending operations have a lifecycle // owner but no public request receipt. Bind recovery to that @@ -2215,12 +2225,18 @@ impl Runtime { candidate.pending_restore = Some(pending); candidate.restore_guard.rotate(); candidate.restore_guard.active_request_id = Some(request.request_id.clone()); + let workdir_attachment_preparation_base = request + .preparation + .as_ref() + .and_then(|preparation| preparation.workdir_attachments.as_ref()) + .map(|_| candidate.logical_workdir_attachments.clone()); candidate.restore_guard.owners.insert( request.request_id.clone(), RestoreRequestOwner { fingerprint, operation_id, request, + workdir_attachment_preparation_base, preparation_applied: recovering_pending, execution_started: recovering_pending, receipt: None, @@ -2433,14 +2449,15 @@ impl Runtime { { return Ok(RuntimeWorkerRestoreResult::accepted(worker.detail())); } - let preparation = worker + let owner = worker .restore_guard .active_request_id .as_ref() - .and_then(|id| worker.restore_guard.owners.get(id)) - .filter(|owner| !owner.preparation_applied) - .and_then(|owner| owner.request.preparation.as_ref()); - let prepared_worker = match worker.prepared_restore_record(preparation) { + .and_then(|id| worker.restore_guard.owners.get(id)); + let prepared_worker = match owner.map_or_else( + || Ok(worker.clone()), + |owner| owner.prepared_worker_record(worker), + ) { Ok(prepared) => prepared, Err(error) => { return Ok(RuntimeWorkerRestoreResult::failed( @@ -4398,8 +4415,7 @@ impl Runtime { worker_id: worker_ref.worker_id, }); } - let mut candidate = - previous.prepared_restore_record(owner.request.preparation.as_ref())?; + let mut candidate = owner.prepared_worker_record(previous)?; candidate .restore_guard .owners @@ -6277,12 +6293,48 @@ pub(crate) struct RestoreRequestOwner { fingerprint: String, operation_id: WorkerLifecycleOperationId, request: runtime_api::WorkerRestoreRequest, + // This is the admission comparison input, not a second attachment authority. + // Without it an old preparation could hide a later canonical revocation. + // Older owners lacking this fact cannot safely reapply attachment overrides. + #[serde(default, skip_serializing_if = "Option::is_none")] + workdir_attachment_preparation_base: Option>, preparation_applied: bool, #[serde(default)] execution_started: bool, receipt: Option, } +impl RestoreRequestOwner { + fn validate_workdir_preparation_authority( + &self, + worker: &WorkerRecord, + ) -> Result<(), RuntimeError> { + if !self.preparation_applied + && self + .request + .preparation + .as_ref() + .is_some_and(|preparation| preparation.workdir_attachments.is_some()) + && self.workdir_attachment_preparation_base.as_ref() + != Some(&worker.logical_workdir_attachments) + { + return Err(RuntimeError::InvalidRequest( + "current Worker authority differs from admitted Restore attachment preparation" + .into(), + )); + } + Ok(()) + } + + fn prepared_worker_record(&self, worker: &WorkerRecord) -> Result { + if self.preparation_applied { + return Ok(worker.clone()); + } + self.validate_workdir_preparation_authority(worker)?; + worker.prepared_restore_record(self.request.preparation.as_ref()) + } +} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct RestoreGuard { @@ -11995,7 +12047,7 @@ mod tests { let mut request = runtime.test_restore_request(&worker.worker_ref); request.preparation = Some(serde_json::from_value(serde_json::json!({ "workspace_api": { "workspace_id": "workspace-a", "base_url": "https://prepared.example" }, - "workdir_attachments": [], "repository_access": [] + "workdir_attachments": [{ "alias": "checkout", "working_directory_id": "prepared-logical-workdir" }], "repository_access": [] })).unwrap()); let gate = Arc::new(RestoreGate::default()); *backend.restore_gate.lock().unwrap() = Some(gate.clone()); @@ -12014,6 +12066,26 @@ mod tests { let state = runtime.lock().unwrap(); let record = state.worker(&worker.worker_ref).unwrap(); assert!(record.request.as_ref().unwrap().workspace_api.is_none()); + assert!(record.logical_workdir_attachments.is_empty()); + let owner = record.restore_guard.owners.values().next().unwrap(); + let mut saved_owner = serde_json::to_value(owner).unwrap(); + let decoded: RestoreRequestOwner = serde_json::from_value(saved_owner.clone()).unwrap(); + assert!( + decoded + .validate_workdir_preparation_authority(record) + .is_ok() + ); + saved_owner + .as_object_mut() + .unwrap() + .remove("workdir_attachment_preparation_base"); + let legacy: RestoreRequestOwner = serde_json::from_value(saved_owner).unwrap(); + assert!( + legacy + .validate_workdir_preparation_authority(record) + .is_err(), + "a missing historical comparison input must not be invented from current authority" + ); assert!( !record .restore_guard @@ -12033,6 +12105,11 @@ mod tests { .base_url, "https://prepared.example" ); + assert_eq!( + backend.restore_requests.lock().unwrap()[0].logical_workdir_attachments[0] + .working_directory_id, + "prepared-logical-workdir" + ); assert!(backend.activation_operation_ids.lock().unwrap().is_empty()); gate.release(); assert_eq!( @@ -12053,6 +12130,25 @@ mod tests { .base_url, "https://prepared.example" ); + let record = state.worker(&worker.worker_ref).unwrap(); + assert_eq!( + record.logical_workdir_attachments[0].working_directory_id, + "prepared-logical-workdir" + ); + let owner = record.restore_guard.owners.values().next().unwrap(); + assert!(owner.preparation_applied); + let mut revoked = record.clone(); + revoked + .apply_logical_workdir_attachments(Vec::new()) + .unwrap(); + assert!( + owner + .prepared_worker_record(&revoked) + .unwrap() + .logical_workdir_attachments + .is_empty(), + "a committed preparation is not a perpetual grant" + ); } #[cfg(feature = "ws-server")] diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index b5ace780f..695a03379 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -360,20 +360,21 @@ async fn prepare_restore_operation( failure })?; if repaired { - Ok(Box::new(RepairedRestoreCandidate { candidate })) + Ok(Box::new(EffectRetainingRestoreCandidate { candidate })) } else { Ok(candidate) } } -// A completed repair is a canonical effect even when later activation fails -// before its own mutation boundary. Carry that evidence with the resource owner. -struct RepairedRestoreCandidate { +// Completed repair or an earlier activation may have changed canonical state. +// Keep that operation-owned evidence through reconstruction and even an +// activation failure before the replacement candidate's own mutation boundary. +struct EffectRetainingRestoreCandidate { candidate: Box, } #[async_trait] -impl RuntimeRestoreCandidate for RepairedRestoreCandidate { +impl RuntimeRestoreCandidate for EffectRetainingRestoreCandidate { fn initial_state(&self) -> protocol::WorkerStateSnapshot { self.candidate.initial_state() } @@ -2953,6 +2954,9 @@ struct PendingRuntimeRestore { task: tokio::sync::Mutex>>, result: Mutex, RuntimeRestoreFailure>>>, activated: Mutex>>, + // Monotonic for this operation, not a claim that its current candidate has + // started. A reconstructed candidate has a new result and no completion in + // `activated`; prior activation/effect evidence must survive that handoff. activation_started: AtomicBool, discard_started: AtomicBool, discard_result: Mutex>>, @@ -2960,6 +2964,16 @@ struct PendingRuntimeRestore { } impl PendingRuntimeRestore { + fn owns_prepared_candidate(&self) -> bool { + matches!( + self.result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref(), + Some(Ok(_)) + ) + } + async fn wait(&self, timeout: Duration) -> Result<(), String> { let mut task = self.task.lock().await; if let Some(message) = self @@ -3279,23 +3293,40 @@ where && (failure.durable_effects_possible || owner.activation_started.load(Ordering::Acquire)) { + let prior_effects = failure.durable_effects_possible + || owner.activation_started.load(Ordering::Acquire); let result = match prepare_restore_operation(factory.as_ref(), owner.request.clone()) .await { - Ok(candidate) => Ok(candidate), - Err(failure) => Err(finish_restore_failure( - factory.as_ref(), - &owner.request, - failure, - ) - .await), + Ok(candidate) => { + Ok(Box::new(EffectRetainingRestoreCandidate { candidate }) + as Box) + } + Err(mut failure) => { + failure.durable_effects_possible |= prior_effects; + Err(finish_restore_failure( + factory.as_ref(), + &owner.request, + failure, + ) + .await) + } }; - owner.activation_started.store(false, Ordering::Release); + // A completed reprepare replaces attempt resources, never + // the operation's effect history. Only a ready candidate + // may clear the old activation completion for a new attempt. + let activated = match &result { + Ok(_) => None, + Err(failure) if owner.activation_started.load(Ordering::Acquire) => { + Some(Err(failure.clone())) + } + Err(_) => None, + }; *owner .activated .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = None; + .unwrap_or_else(std::sync::PoisonError::into_inner) = activated; *owner .result .lock() @@ -4019,7 +4050,9 @@ where failure.to_string(), ); } - if pending.activation_started.load(Ordering::Acquire) { + if pending.activation_started.load(Ordering::Acquire) + && !pending.owns_prepared_candidate() + { let mut activated = match pending.activated.lock() { Ok(result) => result, Err(_) => { @@ -5205,7 +5238,8 @@ where }; } } - if pending.activation_started.load(Ordering::Acquire) { + if pending.activation_started.load(Ordering::Acquire) && !pending.owns_prepared_candidate() + { let activated = pending .activated .lock() @@ -5330,7 +5364,8 @@ where return Err("restored Worker operation identity does not match".into()); } self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; - if !pending.activation_started.swap(true, Ordering::AcqRel) { + if pending.owns_prepared_candidate() { + pending.activation_started.store(true, Ordering::Release); let owner = pending.clone(); let factory = self.factory.clone(); let task = self.spawn_on_adapter_runtime(async move { diff --git a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs index a5b0312f9..443fbac6f 100644 --- a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs @@ -895,4 +895,210 @@ mod restore_candidate { assert_eq!(model.call_count.load(Ordering::SeqCst), 0); } } + + #[test] + #[serial_test::serial(worker_allocation)] + fn attachment_preparation_replay_cannot_hide_logical_authority_revocation() { + use crate::fs_store::{ + FsRuntimeStoreOptions, PersistedWorkerExecution, PersistedWorkerExecutionState, + PersistedWorkerRecord, + }; + let mut fixture = Fixture::new(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_secs(2); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let options = FsRuntimeStoreOptions { + root: fixture._root.path().join("catalog"), + runtime_id: "candidate-runtime".into(), + display_name: None, + }; + let attachment = crate::catalog::LogicalWorkdirAttachment { + alias: WorkdirAttachmentAlias::new("checkout").unwrap(), + working_directory_id: "workspace-only-workdir".into(), + capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY, + }; + // Only the initial Runtime catalog is seeded. The saved Profile, queue, + // restore admission, revocation, replay and Stop use their real owners. + let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( + options.clone(), + fixture.backend.clone(), + ) + .unwrap(); + let create = fixture.request.request.clone(); + runtime + .test_runtime_store() + .write_worker_record(&PersistedWorkerRecord { + restore_guard: crate::runtime::RestoreGuard::default(), + worker_ref: worker_ref.clone(), + worker_id: worker_ref.worker_id, + profile: create.profile.clone(), + display_name: None, + profile_source: create.profile_source.reference(), + config_bundle: create.config_bundle.clone(), + created_at_ms: None, + status: crate::catalog::WorkerStatus::Stopped, + last_finished_submission_request_id: None, + execution_state: PersistedWorkerExecutionState::Available( + PersistedWorkerExecution { + request: create, + binding: None, + restore_intent: crate::catalog::WorkerRestoreIntent::Explicit, + }, + ), + workspace_id: Some(scope.workspace_id.clone()), + workdir_attachments: Vec::new(), + logical_workdir_attachments: vec![attachment.clone()], + }) + .unwrap(); + drop(runtime); + let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( + options.clone(), + fixture.backend.clone(), + ) + .unwrap(); + assert!( + runtime + .worker_detail(&worker_ref) + .unwrap() + .workdir_attachments + .is_empty() + ); + let before = fixture.canonical_files(); + let authority = || -> serde_json::Value { + serde_json::from_slice( + &fs::read( + options + .root + .join("workers") + .join(worker_ref.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap() + }; + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .factory_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .factory_entered + .lock() + .unwrap() = Some(entered_tx); + let mut request = runtime.test_restore_request(&worker_ref); + request.preparation.as_mut().unwrap().workdir_attachments = + Some(serde_json::from_value(serde_json::to_value(vec![attachment]).unwrap()).unwrap()); + let first = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request.clone()) + .unwrap(); + entered.recv_timeout(WAIT).unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{first:?}" + ); + let pending = fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .unwrap(); + assert!(!pending.request.logical_workdir_attachments.is_empty()); + assert!(pending.request.previous_workdir_attachments.is_empty()); + assert!(pending.request.deferred_workdir_sessions.is_empty()); + runtime + .replace_worker_workdir_attachments_scoped(&scope, &worker_ref, Vec::new()) + .unwrap(); + assert!(authority().get("logical_workdir_attachments").is_none()); + release.notify_one(); + let replay = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "revocation hidden by saved preparation: {replay:?}" + ); + assert!( + replay + .message + .as_deref() + .unwrap_or_default() + .contains("current Worker authority differs"), + "{replay:?}" + ); + let record = authority(); + assert!(record.get("logical_workdir_attachments").is_none()); + let owner = &record["restore_guard"]["owners"][&request.request_id]; + assert_eq!( + owner["operation_id"], + serde_json::to_value(pending.request.operation_id).unwrap() + ); + assert!(owner["receipt"].is_null()); + assert_eq!( + owner["workdir_attachment_preparation_base"] + .as_array() + .unwrap() + .len(), + 1 + ); + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + let durable_guard = + serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap(); + assert_eq!(durable_guard["owners"][&request.request_id], *owner); + let (coordinated, preparation) = runtime + .coordinate_worker_restore_operation( + &worker_ref, + runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: None, + }, + Some(&scope), + ) + .unwrap(); + assert_eq!( + coordinated.unwrap().state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + assert!( + preparation.is_none(), + "coordinator must not export the revoked preparation again" + ); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!(Arc::ptr_eq( + &pending, + &fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .unwrap() + )); + assert!(!pending.activation_started.load(Ordering::SeqCst)); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before); + runtime + .stop_worker_scoped(&scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + fixture.assert_quiet(); + assert_eq!(fixture.canonical_files(), before); + assert!(authority().get("logical_workdir_attachments").is_none()); + assert_ne!( + runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request) + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + } } diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs index 40de8168c..91530ab28 100644 --- a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -100,6 +100,148 @@ impl WorkingDirectoryMaterializer for RestoreOpenCountingMaterializer { } } +// Substitute only initial creation: a new aggregate/Session is left pending +// before its first head, without ever materializing or rewinding a saved head. +// All Restore preflight, repair, retry, activation and Stop use the real backend. +struct PendingProfileSeedBackend { + inner: Arc, +} + +impl WorkerExecutionBackend for PendingProfileSeedBackend { + fn backend_id(&self) -> &str { + self.inner.backend_id() + } + + fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { + let aggregate = self + .inner + .factory + .worker_aggregate_dir(&request.worker_ref) + .unwrap(); + let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&request.worker_ref); + self.inner + .run_on_adapter_runtime(async move { + assert!( + !aggregate.join("session").exists(), + "pending fixture requires a new Session aggregate" + ); + let manifest = manifest::WorkerManifest::from_toml(&format!( + r#" +[worker] +name = "{name}" +pwd = "./" +[scope] +allow = [] +[model] +scheme = "anthropic" +model_id = "test-model" +auth = {{ kind = "none" }} +[engine] +max_tokens = 100 +"# + )) + .unwrap(); + let sessions = WorkerSessionStore::new(aggregate.join("session")).unwrap(); + let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap(); + assert!(metadata.read_by_name(&name).unwrap().is_none()); + let engine = + Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated( + MockClient::new(Vec::new()), + ); + let mut head = Worker::new( + manifest.clone(), + engine, + CombinedStore::new(sessions.clone(), metadata.clone()), + WorkerWorkspaceContext::unavailable( + None, + "initial creation interrupted before first head", + ), + WorkerFilesystemAuthority::None, + Scope::empty(), + ) + .await + .unwrap(); + head.enable_worker_metadata_write_through().unwrap(); + let mut saved = metadata.read_by_name(&name).unwrap().unwrap(); + let active = saved.active.as_ref().unwrap(); + assert!(active.segment_id.is_none()); + assert!(sessions.list_segments(active.session_id).unwrap().is_empty()); + saved.resolved_manifest_snapshot = + Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap()); + metadata.write(&saved).unwrap(); + drop(head); + Ok(()) + }) + .unwrap(); + WorkerExecutionSpawnResult::Connected { + worker_state: protocol::WorkerStateSnapshot::initial(), + workdir_attachments: Vec::new(), + } + } + + fn fetch_workspace_config( + &self, + request: WorkspaceConfigFetchRequest, + ) -> Result { + self.inner.fetch_workspace_config(request) + } + + fn observe_workspace_prompt_projection( + &self, + projection: worker::WorkspacePromptProjection, + ) -> Result<(), String> { + self.inner.observe_workspace_prompt_projection(projection) + } + + fn preflight_restore( + &self, + request: &WorkerExecutionRestoreRequest, + ) -> Result<(), WorkerExecutionResult> { + self.inner.preflight_restore(request) + } + + fn completed_restore_failure( + &self, + operation_id: crate::execution::WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Option { + self.inner + .completed_restore_failure(operation_id, worker_ref) + } + + fn restore_worker(&self, request: WorkerExecutionRestoreRequest) -> WorkerExecutionSpawnResult { + self.inner.restore_worker(request) + } + + fn reconcile_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + self.inner.reconcile_restore(request) + } + + fn activate_restored_worker( + &self, + operation_id: crate::execution::WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Result<(), String> { + self.inner + .activate_restored_worker(operation_id, worker_ref) + } + + fn dispatch_input(&self, worker_ref: &WorkerRef, input: WorkerInput) -> WorkerExecutionResult { + self.inner.dispatch_input(worker_ref, input) + } + + fn dispatch_method(&self, worker_ref: &WorkerRef, method: Method) -> WorkerExecutionResult { + self.inner.dispatch_method(worker_ref, method) + } + + fn stop_worker_operation(&self, request: WorkerExecutionStopRequest) -> WorkerExecutionResult { + self.inner.stop_worker_operation(request) + } +} + struct ProfileFailureFixture { runtime: EmbeddedRuntime, backend: Arc, @@ -115,6 +257,14 @@ impl ProfileFailureFixture { } fn with_saved_inputs(timeout: Duration, model: Option) -> Self { + Self::with_seed(timeout, model, false) + } + + fn with_pending_head(timeout: Duration) -> Self { + Self::with_seed(timeout, None, true) + } + + fn with_seed(timeout: Duration, model: Option, pending_head: bool) -> Self { let root = restore_failure_tempdir(); let repo = root.path().join("g"); fs::create_dir(&repo).unwrap(); @@ -141,22 +291,31 @@ impl ProfileFailureFixture { opens: provider_opens.clone(), }), ); + let execution: Arc = if pending_head { + Arc::new(PendingProfileSeedBackend { + inner: backend.clone(), + }) + } else { + backend.clone() + }; let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( crate::fs_store::FsRuntimeStoreOptions { root: store, runtime_id: "r".into(), display_name: None, }, - backend.clone(), + execution, ) .unwrap(); runtime.store_config_bundle(test_bundle()).unwrap(); let mut request = create_request("restore failure recovery"); request.profile = ProfileSelector::Builtin("default".into()); - request.workdir_attachment_requests = vec![WorkingDirectoryAttachmentRequest { - alias: WorkdirAttachmentAlias::new("checkout").unwrap(), - working_directory: working_directory_request(&repo), - }]; + if !pending_head { + request.workdir_attachment_requests = vec![WorkingDirectoryAttachmentRequest { + alias: WorkdirAttachmentAlias::new("checkout").unwrap(), + working_directory: working_directory_request(&repo), + }]; + } let worker = runtime.create_worker(request).unwrap(); let seed_handle = queue_inputs.then(|| { backend.workers.lock().unwrap()[&worker.worker_ref] @@ -203,6 +362,127 @@ impl ProfileFailureFixture { *self.backend.factory.restore_faults.fail_at.lock().unwrap() = Some(stage); } + fn pin_unclaimed_restore_upload(&self) -> (PathBuf, BTreeMap>) { + let aggregate = self + .backend + .factory + .worker_aggregate_dir(&self.worker.worker_ref) + .unwrap(); + let name = + ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&self.worker.worker_ref); + let metadata = WorkerAggregateStore::open_read_only(&aggregate, name.clone()).unwrap(); + let session_id = metadata + .read_by_name(&name) + .unwrap() + .unwrap() + .active + .unwrap() + .session_id; + let session = WorkerSessionStore::new(aggregate.join("session")).unwrap(); + let orphan = session + .write_uploaded_file( + session_id, + "orphan.txt", + "text/plain", + b"restore effect evidence", + session_store::UploadedFileLimits::default(), + ) + .unwrap(); + session + .pin_uploaded_file(session_id, &orphan, "uncommitted-restore-input") + .unwrap(); + let artifacts = aggregate.join("session/artifacts/paste"); + let pinned = persisted_files(&artifacts); + assert!(!pinned.is_empty()); + (artifacts, pinned) + } + + fn assert_retained_restore_effects( + &self, + request: &runtime_api::WorkerRestoreRequest, + pending: &Arc, + stage: RuntimeRestoreStage, + ) { + let retained = self + .backend + .pending_restore(&self.worker.worker_ref) + .unwrap() + .expect("resource cleanup must not remove the operation's effect owner"); + assert!(Arc::ptr_eq(pending, &retained)); + { + let result = retained.result.lock().unwrap(); + let failure = result.as_ref().unwrap().as_ref().err().unwrap(); + assert_eq!(failure.stage, stage); + assert_eq!(failure.cleanup, RuntimeRestoreCleanup::Confirmed); + assert!( + failure.durable_effects_possible, + "resource cleanup cannot erase prior canonical effects: {failure:?}" + ); + } + let record: serde_json::Value = serde_json::from_slice( + &fs::read( + self.root + .path() + .join("s/workers") + .join(self.worker.worker_ref.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!( + record["restore_guard"]["active_request_id"], + request.request_id + ); + let operation = serde_json::to_value(pending.request.operation_id).unwrap(); + let owner = &record["restore_guard"]["owners"][&request.request_id]; + assert_eq!(owner["operation_id"], operation); + assert!( + owner["receipt"].is_null(), + "false terminal receipt: {owner:?}" + ); + assert_eq!( + record["execution_state"]["state"], + "reconciliation_required" + ); + assert_eq!( + record["execution_state"]["execution"]["operation"], + "restore" + ); + assert_eq!( + record["execution_state"]["execution"]["intent"]["operation_id"], + operation + ); + assert!( + self.backend + .completed_restore_failures + .lock() + .unwrap() + .is_empty() + ); + assert!(self.backend.workers.lock().unwrap().is_empty()); + assert!( + self.backend + .factory + .failed_restore_resources + .lock() + .unwrap() + .is_empty(), + "cleanup confirmed resources, not rollback of canonical effects" + ); + assert!( + self.backend + .factory + .observation_hub + .get(&self.worker.worker_ref) + .is_none() + ); + if !pending.request.deferred_workdir_sessions.is_empty() { + self.assert_closed_deferred_sessions(pending); + } + assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0); + } + fn assert_released(&self) { assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0); assert!(self.backend.pending_restores.lock().unwrap().is_empty()); @@ -422,6 +702,311 @@ impl ProfileFailureFixture { } } +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_prior_effects_survive_read_only_reprepare_failure() { + for (activation_stage, read_stage) in [ + (RuntimeRestoreStage::Feature, RuntimeRestoreStage::Metadata), + ( + RuntimeRestoreStage::Controller, + RuntimeRestoreStage::Reconstruction, + ), + ] { + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload(); + fixture.fail(activation_stage); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let first = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{activation_stage:?}: {first:?}" + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + assert!(pending.activation_started.load(Ordering::SeqCst)); + fixture.assert_retained_restore_effects(&request, &pending, activation_stage); + let effected = persisted_files(&artifacts); + assert_ne!( + effected, pinned, + "activation must actually reconcile the orphan pin" + ); + + // The next attempt needs no repair, and fails before any new mutation. + // Resource cleanup is confirmed, but the first attempt's pin change + // was not rolled back. Repeated sends must retain the same owner. + for _ in 0..2 { + fixture.fail(read_stage); + let replay = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{activation_stage:?} then {read_stage:?}: {replay:?}" + ); + fixture.assert_retained_restore_effects(&request, &pending, read_stage); + assert!(pending.activation_started.load(Ordering::SeqCst)); + assert_eq!(persisted_files(&artifacts), effected); + } + fixture.resume_same_operation(request, pending.request.operation_id); + } +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_prior_effects_from_pending_head_repair_survive_read_only_reprepare_failure() { + let fixture = ProfileFailureFixture::with_pending_head(Duration::from_secs(5)); + let aggregate = fixture + .backend + .factory + .worker_aggregate_dir(&fixture.worker.worker_ref) + .unwrap(); + let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&fixture.worker.worker_ref); + let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap(); + let initial = metadata.read_by_name(&name).unwrap().unwrap(); + let pending_session = initial.active.as_ref().unwrap().session_id; + assert!(initial.active.as_ref().unwrap().segment_id.is_none()); + { + let before = WorkerSessionStore::new(aggregate.join("session")).unwrap(); + assert!(before.list_sessions().unwrap().is_empty()); + assert!(before.list_segments(pending_session).unwrap().is_empty()); + } + // The independent new Worker has never published a head. Its first segment + // and active pointer must be created by production repair, not fixture + // rewinding/deletion of a materialized Session. + fixture.fail(RuntimeRestoreStage::Metadata); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let first = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{first:?}" + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + assert!(!pending.activation_started.load(Ordering::SeqCst)); + assert!(pending.request.deferred_workdir_sessions.is_empty()); + fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Metadata); + let failure = pending + .result + .lock() + .unwrap() + .as_ref() + .unwrap() + .as_ref() + .err() + .unwrap() + .clone(); + let repaired = WorkerAggregateStore::open_read_only(&aggregate, &name) + .unwrap() + .read_by_name(&name) + .unwrap() + .unwrap(); + eprintln!( + "pending-head repair first={first:?} failure={failure:?} active={:?}", + repaired.active + ); + let active = repaired + .active + .as_ref() + .expect("real repair must retain the pending Session's active pointer"); + assert_eq!(active.session_id, pending_session); + // Store handles opened before the first Manifest retain session_id=None; + // they cannot observe that another owner has since published this Session. + // Reopen read-only to require the canonical Manifest as well as its logs. + let session = WorkerSessionStore::open_read_only(aggregate.join("session")) + .unwrap_or_else(|error| { + panic!( + "repair Manifest reread failed: {error}; first={first:?}; failure={failure:?}; active={active:?}" + ) + }); + assert_eq!(session.list_sessions().unwrap(), vec![pending_session]); + let segments = session.list_segments(pending_session).unwrap(); + assert_eq!( + segments.len(), + 1, + "real repair must commit exactly one seed; failure={failure:?}; active={active:?}" + ); + assert_eq!(active.segment_id, Some(segments[0])); + let entries = session + .read_all_read_only(pending_session, segments[0]) + .unwrap(); + assert!(matches!( + entries.first(), + Some(LogEntry::AnnotatedSegmentStart { .. }) + )); + assert!( + session_store::collect_state(&entries) + .system_prompt + .is_some() + ); + assert_ne!( + repaired, initial, + "repair must publish the first active head" + ); + let repaired_session = persisted_files(&aggregate.join("session")); + + for _ in 0..2 { + fixture.fail(RuntimeRestoreStage::Reconstruction); + let replay = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "repair effect cannot become a rollback on the next read failure: {replay:?}" + ); + fixture.assert_retained_restore_effects( + &request, + &pending, + RuntimeRestoreStage::Reconstruction, + ); + let current_metadata = WorkerAggregateStore::open_read_only(&aggregate, &name).unwrap(); + assert_eq!( + current_metadata.read_by_name(&name).unwrap().unwrap(), + repaired + ); + let current_session = + WorkerSessionStore::open_read_only(aggregate.join("session")).unwrap(); + assert_eq!( + current_session.list_sessions().unwrap(), + vec![pending_session] + ); + assert_eq!( + current_session.list_segments(pending_session).unwrap(), + segments + ); + assert_eq!( + persisted_files(&aggregate.join("session")), + repaired_session + ); + } + fixture.resume_same_operation(request, pending.request.operation_id); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn profile_restore_prior_effects_survive_rebuilt_candidate_pre_effect_activation_failure() { + let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload(); + fixture.fail(RuntimeRestoreStage::Feature); + let request = fixture + .runtime + .test_restore_request(&fixture.worker.worker_ref); + let first = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{first:?}" + ); + let pending = fixture + .backend + .pending_restore(&fixture.worker.worker_ref) + .unwrap() + .unwrap(); + fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Feature); + let effected = persisted_files(&artifacts); + assert_ne!(effected, pinned); + + let release = Arc::new(tokio::sync::Notify::new()); + let (entered_tx, entered) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .factory_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .factory_entered + .lock() + .unwrap() = Some(entered_tx); + let runtime = fixture.runtime.clone(); + let worker_ref = fixture.worker.worker_ref.clone(); + let resend = request.clone(); + let retry = std::thread::spawn(move || runtime.restore_worker_operation(&worker_ref, resend)); + let reached = entered.recv_timeout(Duration::from_secs(5)); + // This gate is after real read-only reconstruction. Change only saved + // metadata, so the rebuilt candidate's first activation revalidation fails + // before allocation/pin/metadata/delegation mutation. No factory or repair + // implementation is replaced by a fake. + let aggregate = fixture + .backend + .factory + .worker_aggregate_dir(&fixture.worker.worker_ref) + .unwrap(); + let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&fixture.worker.worker_ref); + let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap(); + let mut changed = metadata.read_by_name(&name).unwrap().unwrap(); + changed.workspace_root = Some(fixture.root.path().join("changed-after-reconstruction")); + metadata.write(&changed).unwrap(); + release.notify_one(); + let rebuilt = retry.join().unwrap().unwrap(); + reached.expect("same-operation retry must reach the real reconstructed candidate"); + assert_eq!( + rebuilt.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{rebuilt:?}" + ); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 2 + ); + assert!(pending.activation_started.load(Ordering::SeqCst)); + fixture.assert_retained_restore_effects( + &request, + &pending, + RuntimeRestoreStage::Reconstruction, + ); + assert_eq!(metadata.read_by_name(&name).unwrap().unwrap(), changed); + assert_eq!(persisted_files(&artifacts), effected); + + fixture.fail(RuntimeRestoreStage::Metadata); + let replay = fixture + .runtime + .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "successful reconstruction must not erase the older effect: {replay:?}" + ); + fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Metadata); + assert_eq!(metadata.read_by_name(&name).unwrap().unwrap(), changed); + assert_eq!(persisted_files(&artifacts), effected); + fixture.resume_same_operation(request, pending.request.operation_id); +} + #[test] #[serial_test::serial(worker_allocation)] fn profile_restore_timeout_attachment_revocation_cannot_reactivate_old_candidate() { From 85f70d4bc1916b2c35a02a3f5856796910be9464 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 06:20:22 +0900 Subject: [PATCH 22/46] test: adapt fixed Worker restore fixtures to integration lane --- Cargo.lock | 6 + crates/worker-runtime/Cargo.toml | 2 + crates/worker-runtime/README.md | 31 + crates/worker-runtime/src/fs_store.rs | 4 + crates/worker-runtime/src/lib.rs | 2 + crates/worker-runtime/src/test_support.rs | 859 +++ .../worker-runtime/src/test_support_tests.rs | 885 +++ crates/worker-runtime/src/worker_backend.rs | 9 + crates/workspace-server/Cargo.toml | 4 + crates/workspace-server/src/lib.rs | 2 + crates/workspace-server/src/store.rs | 42 +- crates/workspace-server/src/test_support.rs | 688 ++ tests/e2e/Cargo.toml | 12 + tests/e2e/README.md | 38 + .../fixtures/worker-codex-v1/expected.json | 52 + .../fixtures/worker-codex-v1/manifest.json | 34 + .../worker-codex-v1/worker-template.json | 5605 +++++++++++++++++ .../worker-codex-v1/workspace-config.dcdl | 22 + tests/e2e/src/fixture_source.rs | 336 + tests/e2e/src/lib.rs | 40 +- tests/e2e/src/worker_environment.rs | 1178 +++- tests/e2e/src/worker_session.rs | 284 + tests/e2e/tests/panel.rs | 10 +- tests/e2e/tests/worker_restore.rs | 274 + tests/e2e/tests/worker_restore/fixture.rs | 257 + tests/e2e/tests/worker_restore/seed.rs | 482 ++ tools/state-contract-name-exceptions.json | 33 + 27 files changed, 11036 insertions(+), 155 deletions(-) create mode 100644 crates/worker-runtime/src/test_support.rs create mode 100644 crates/worker-runtime/src/test_support_tests.rs create mode 100644 crates/workspace-server/src/test_support.rs create mode 100644 tests/e2e/fixtures/worker-codex-v1/expected.json create mode 100644 tests/e2e/fixtures/worker-codex-v1/manifest.json create mode 100644 tests/e2e/fixtures/worker-codex-v1/worker-template.json create mode 100644 tests/e2e/fixtures/worker-codex-v1/workspace-config.dcdl create mode 100644 tests/e2e/src/fixture_source.rs create mode 100644 tests/e2e/src/worker_session.rs create mode 100644 tests/e2e/tests/worker_restore.rs create mode 100644 tests/e2e/tests/worker_restore/fixture.rs create mode 100644 tests/e2e/tests/worker_restore/seed.rs diff --git a/Cargo.lock b/Cargo.lock index f2901cb1f..c88fc1d23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6707,15 +6707,21 @@ name = "yoi-e2e" version = "0.0.0" dependencies = [ "chrono", + "config-source", "libc", + "manifest", + "protocol", "reqwest", "serde", "serde_json", "server-api", + "session-store", "sha2 0.11.0", "tempfile", + "tungstenite 0.29.0", "webauthn-authenticator-rs", "webauthn-rs-proto", + "worker-runtime", "yoi-workspace-server", ] diff --git a/crates/worker-runtime/Cargo.toml b/crates/worker-runtime/Cargo.toml index 0eba6a589..32f6ea537 100644 --- a/crates/worker-runtime/Cargo.toml +++ b/crates/worker-runtime/Cargo.toml @@ -19,6 +19,8 @@ required-features = ["ws-server", "fs-store"] [features] default = ["ws-server", "fs-store"] fs-store = ["dep:fs4"] +# Offline fixture tooling only; never enabled by the product defaults. +test-support = ["fs-store", "ws-server"] http-server = ["dep:axum", "dep:tower", "dep:reqwest"] ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"] diff --git a/crates/worker-runtime/README.md b/crates/worker-runtime/README.md index 4984c2e18..e4f041745 100644 --- a/crates/worker-runtime/README.md +++ b/crates/worker-runtime/README.md @@ -8,3 +8,34 @@ Manages Worker creation, stopping, and Restore, execution connections within the - [`execution`] — execution backends and operation-specific results. - [`working_directory`] — repository materialization, session resources, and safe deletion. - [`workspace_issuer`] — Workspace signature verification and replay prevention. + +## Offline stopped-Worker fixtures + +The nondefault `test-support` feature exposes `test_support::StoppedWorkerTemplate`. +It is intentionally limited to current-schema, stopped, attachment-free artificial +conversations with one Segment, text history, and no pending execution/activation. +It does not add a product endpoint or change create/restore authorization. + +1. With the source Runtime offline, call `StoppedWorkerTemplate::extract(worker_dir)`. + The source must have its existing Runtime owner lock and identity. Extraction is + read-only; the returned serde template excludes source auth, saved Manifest, + request/config references, run directories, and Runtime operation history. +2. Initialize **new** Runtime identity and Workspace trust in a fresh destination + using normal setup. Preserve the template's Runtime and Workspace IDs. +3. Call `template.install(&options, request, &resolved_manifest)` before starting + the Runtime. Supply a fresh typed `CreateWorkerRequest` (including config/profile + references and trusted memory settings) and `manifest::WorkerManifest` resolved + for that environment. Its Worker name must equal `template.worker_name()`. +4. Start normally and use the ordinary explicit restore operation. Installation + itself creates no execution and leaves the Worker stopped with a fresh guard. + +The template preserves the original Session manifest/JSONL bytes, including stable +entry IDs, timestamps, and settled submission receipts. It rejects resumable queues +and Feature invocation history. A saved system prompt is required for restore; +checkpoint receipts must match the Worker's typed receipt schema. Preserved records +must use canonical writer field shapes: unknown/duplicate fields (including nested +fields), discarded optional fields, and opaque Feature history payloads are rejected, +not stripped or rewritten. Portability/secret-marker checks do not replace +review of the supplied artificial conversation or its independently verified hashes. +The installer refuses any destination content other than initialized `auth.toml`; +interrupted publication is fail-closed, not an overwrite/recovery facility. diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index 21ecfeca8..dcfb1b42c 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -39,6 +39,10 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; +#[cfg(feature = "test-support")] +#[path = "test_support.rs"] +pub mod test_support; + const SCHEMA_VERSION: u32 = 12; // Frozen source schemas: do not move their meaning when adding a migration. const SOURCE_SCHEMA_VERSION: u32 = 11; diff --git a/crates/worker-runtime/src/lib.rs b/crates/worker-runtime/src/lib.rs index f4a5b02ff..bb72c71e3 100644 --- a/crates/worker-runtime/src/lib.rs +++ b/crates/worker-runtime/src/lib.rs @@ -35,6 +35,8 @@ pub mod workspace_issuer; #[cfg(feature = "http-server")] pub mod workspace_request; +#[cfg(feature = "test-support")] +pub use fs_store::test_support; #[cfg(feature = "fs-store")] pub use fs_store::{FsRuntimeStore, FsRuntimeStoreOptions}; pub use management::RuntimeOptions; diff --git a/crates/worker-runtime/src/test_support.rs b/crates/worker-runtime/src/test_support.rs new file mode 100644 index 000000000..fb0f32693 --- /dev/null +++ b/crates/worker-runtime/src/test_support.rs @@ -0,0 +1,859 @@ +//! Offline support for the approved, attachment-free stopped Worker fixture. +//! +//! Enabled only by `test-support`. This is not an import API: extraction accepts a +//! current-schema, settled, single-Segment aggregate; installation accepts only a +//! fresh Runtime directory containing its newly initialized `auth.toml`. No +//! Runtime or execution backend is constructed. Original Session bytes are never +//! rewritten. The caller must separately review/attest the artificial conversation; +//! structural checks and secret-marker checks cannot prove arbitrary prose safe. + +use super::*; +use crate::catalog::ProfileSourceArchiveSource; +use crate::workspace_issuer::{ + WorkspaceIssuerTrustRecord, WorkspaceIssuerTrustState, validate_workspace_issuer_trust_records, +}; +use session_store::{ + SegmentId, SessionId, WorkerActiveSegmentRef, WorkerMetadata, WorkerSessionStore, +}; + +#[cfg(test)] +#[path = "test_support_tests.rs"] +mod tests; + +const TEMPLATE_VERSION: u32 = 1; +const MAX_SESSION_BYTES: usize = 8 * 1024 * 1024; + +/// Portable fixture, intentionally without Debug (Session material is private). +/// Fields are private and every deserialized template is validated on install. +/// Serialize this value to retain the fixture; do not serialize its source aggregate. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StoppedWorkerTemplate { + template_version: u32, + runtime_schema_version: u32, + runtime_id: String, + workspace_id: String, + worker_id: WorkerId, + created_at_ms: Option, + session_id: SessionId, + segment_id: SegmentId, + session_manifest: Vec, + segment_jsonl: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct SessionManifest { + schema_version: u32, + session_id: SessionId, +} + +// Read-only projection: never retain source key material in a template. +#[derive(Deserialize)] +struct AuthIdentity { + identity_id: String, +} + +#[derive(Deserialize)] +struct SourceAuth { + identity: AuthIdentity, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct InitializedAuth { + identity: crate::auth::RuntimeIdentityMaterial, + workspace_issuers: Vec, + #[serde(default)] + archived_legacy_issuers: Vec, +} + +#[derive(Deserialize)] +struct GuardState { + active_request_id: Option, + owners: BTreeMap, +} + +#[derive(Deserialize)] +struct GuardOwner { + receipt: Option, +} + +fn reject(message: &'static str) -> RuntimeError { + RuntimeError::InvalidRequest(format!("stopped Worker fixture: {message}")) +} + +fn io(result: std::io::Result) -> Result { + result.map_err(|_| reject("filesystem operation failed")) +} + +fn decode(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(|_| reject("invalid typed document")) +} + +fn no_symlink_components(path: &Path) -> Result<(), RuntimeError> { + let mut current = PathBuf::new(); + for part in path.components() { + if matches!(part, std::path::Component::ParentDir) { + return Err(reject("parent traversal is unsupported")); + } + current.push(part); + let metadata = io(fs::symlink_metadata(¤t))?; + if metadata.file_type().is_symlink() { + return Err(reject("symlinks are unsupported")); + } + } + Ok(()) +} + +fn check_tree(path: &Path, budget: &mut usize) -> Result<(), RuntimeError> { + if *budget == 0 { + return Err(reject("file count exceeds fixture limit")); + } + *budget -= 1; + let metadata = io(fs::symlink_metadata(path))?; + if metadata.is_dir() { + for entry in io(fs::read_dir(path))? { + check_tree(&io(entry)?.path(), budget)?; + } + } else if !metadata.is_file() { + return Err(reject("symlinks and special files are unsupported")); + } + Ok(()) +} + +fn regular_file(path: &Path) -> Result<(), RuntimeError> { + no_symlink_components(path)?; + if !io(fs::symlink_metadata(path))?.is_file() { + return Err(reject("expected a regular file")); + } + Ok(()) +} + +fn bytes(path: &Path) -> Result, RuntimeError> { + regular_file(path)?; + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOATIME | libc::O_NOFOLLOW); + } + let file = io(options.open(path))?; + let mut result = Vec::new(); + io(file + .take(MAX_SESSION_BYTES as u64 + 1) + .read_to_end(&mut result))?; + if result.len() > MAX_SESSION_BYTES { + return Err(reject("file exceeds fixture limit")); + } + Ok(result) +} + +fn text(path: &Path) -> Result { + String::from_utf8(bytes(path)?).map_err(|_| reject("invalid UTF-8 document")) +} + +fn portable_id(id: &str) -> bool { + !id.is_empty() + && id.len() <= 256 + && id + .bytes() + .all(|c| c.is_ascii_alphanumeric() || b"-_".contains(&c)) +} + +impl StoppedWorkerTemplate { + pub fn runtime_id(&self) -> &str { + &self.runtime_id + } + pub fn workspace_id(&self) -> &str { + &self.workspace_id + } + pub fn worker_id(&self) -> WorkerId { + self.worker_id + } + pub fn worker_name(&self) -> String { + runtime_worker_name(self.worker_id) + } + pub fn session_id(&self) -> SessionId { + self.session_id + } + pub fn segment_id(&self) -> SegmentId { + self.segment_id + } + + /// Extract without migration, rewriting metadata, or copying credentials, + /// resolved manifests, execution-operation receipts, or run directories. + /// The source Runtime must be offline; its existing owner lock is held for + /// the entire read. The path must be `/workers/`. + pub fn extract(worker_dir: &Path) -> Result { + no_symlink_components(worker_dir)?; + let workers = worker_dir + .parent() + .ok_or_else(|| reject("invalid aggregate path"))?; + if workers.file_name().and_then(|s| s.to_str()) != Some(WORKERS_DIR) { + return Err(reject("invalid aggregate path")); + } + let root = workers + .parent() + .ok_or_else(|| reject("invalid Runtime path"))?; + // Require the existing lock, rather than creating a file in the source. + let lock_path = root.with_file_name(format!( + ".{}.runtime-owner.lock", + root.file_name() + .and_then(|s| s.to_str()) + .ok_or_else(|| reject("invalid Runtime path"))? + )); + regular_file(&lock_path)?; + let lock = io(OpenOptions::new().read(true).open(lock_path))?; + if !io(FileExt::try_lock_shared(&lock))? { + return Err(reject("source Runtime is not offline")); + } + let _lock = RuntimeStoreOwnerLock { file: lock }; + check_tree(worker_dir, &mut 256)?; + let snapshot: RuntimeSnapshot = decode(&bytes(&root.join(RUNTIME_FILE))?)?; + snapshot + .validate(&root.join(RUNTIME_FILE)) + .map_err(|_| reject("unsupported Runtime schema"))?; + if !snapshot.subject_create_recovery_fences.is_empty() + || !snapshot.subject_releases.is_empty() + { + return Err(reject("source has lifecycle operations")); + } + let record: WorkerAggregateRecord = decode(&bytes(&worker_dir.join(WORKER_FILE))?)?; + let record = record + .validate(&worker_dir.join(WORKER_FILE)) + .map_err(|_| reject("invalid Worker schema or identity"))?; + let guard: GuardState = decode( + &serde_json::to_vec(&record.restore_guard) + .map_err(|_| reject("invalid restore guard"))?, + )?; + if record.status != WorkerStatus::Stopped + || guard.active_request_id.is_some() + || guard.owners.values().any(|owner| owner.receipt.is_none()) + { + return Err(reject("Worker is not settled and stopped")); + } + let PersistedWorkerExecutionState::Available(execution) = &record.execution_state else { + return Err(reject("pending or unavailable execution metadata")); + }; + // A stopped aggregate may retain the empty historical binding marker. + // It is deliberately not copied to the installed Worker. + if execution.restore_intent != WorkerRestoreIntent::Explicit { + return Err(reject("Worker requires explicit restore")); + } + if !record.workdir_attachments.is_empty() || !record.logical_workdir_attachments.is_empty() + { + return Err(reject("attachments are unsupported")); + } + check_request(&execution.request, false)?; + let workspace_id = record + .workspace_id + .ok_or_else(|| reject("missing Workspace identity"))?; + let metadata: WorkerMetadata = decode(&bytes(&worker_dir.join(WORKER_METADATA_FILE))?)?; + crate::worker_backend::validate_fixture_restore_metadata(&metadata, &execution.request) + .map_err(|_| reject("invalid source restore Manifest or settings"))?; + if metadata.worker_name != runtime_worker_name(record.worker_id) + || metadata.workspace_id.as_deref() != Some(&workspace_id) + || !metadata.spawned_children.is_empty() + || !metadata.reclaimed_children.is_empty() + || !metadata.peers.is_empty() + || metadata.subjektiv_session_attribution.is_some() + { + return Err(reject("unsupported or inconsistent Worker metadata")); + } + let active = metadata + .active + .ok_or_else(|| reject("missing active Session"))?; + let segment_id = active + .segment_id + .ok_or_else(|| reject("unmaterialized Session"))?; + let source_auth: SourceAuth = toml::from_str(&text(&root.join("auth.toml"))?) + .map_err(|_| reject("missing Runtime identity"))?; + let session = worker_dir.join("session"); + let fixture = Self { + template_version: TEMPLATE_VERSION, + runtime_schema_version: SCHEMA_VERSION, + runtime_id: source_auth.identity.identity_id, + workspace_id, + worker_id: record.worker_id, + created_at_ms: record.created_at_ms, + session_id: active.session_id, + segment_id, + session_manifest: bytes(&session.join("session.json"))?, + segment_jsonl: bytes(&session.join("segments").join(format!("{segment_id}.jsonl")))?, + }; + fixture.validate_shape()?; + fixture.validate_session(&session)?; + Ok(fixture) + } + + /// Install after normal identity/trust initialization, before Runtime start. + /// `request` and `manifest` supply NEW backend/config/profile references and + /// freshly resolved policy, not a launch. The saved source Manifest is never + /// reused. `manifest.worker.name` must equal `worker_name()`. + /// Only `auth.toml` may already exist. The owner lock fences live Runtimes; + /// staged files are synced and published without overwriting any Worker. + /// An interrupted publication fails closed as an incomplete Runtime layout. + pub fn install( + &self, + options: &FsRuntimeStoreOptions, + request: CreateWorkerRequest, + manifest: &manifest::WorkerManifest, + ) -> Result<(), RuntimeError> { + self.validate_shape()?; + check_request(&request, true)?; + if manifest.worker.name != self.worker_name() { + return Err(reject("resolved Manifest Worker name mismatch")); + } + let mut metadata = WorkerMetadata::new( + self.worker_name(), + Some(WorkerActiveSegmentRef { + session_id: self.session_id, + segment_id: Some(self.segment_id), + }), + ) + .with_workspace_id(&self.workspace_id); + metadata.resolved_manifest_snapshot = Some( + manifest::write_persisted_worker_manifest_snapshot(manifest) + .map_err(|_| reject("invalid resolved Manifest"))?, + ); + crate::worker_backend::validate_fixture_restore_metadata(&metadata, &request) + .map_err(|_| reject("resolved Manifest does not match fresh request settings"))?; + if options.runtime_id != self.runtime_id + || request.worker_id != self.worker_id + || request + .workspace_api + .as_ref() + .map(|api| api.workspace_id.as_str()) + != Some(&self.workspace_id) + { + return Err(reject("installation identity mismatch")); + } + no_symlink_components(&options.root)?; + check_tree(&options.root, &mut 256)?; + // Validate the lock location before the ordinary owner-lock opener. + let lock_path = options.root.with_file_name(format!( + ".{}.runtime-owner.lock", + options + .root + .file_name() + .and_then(|s| s.to_str()) + .ok_or_else(|| reject("invalid Runtime path"))? + )); + if fs::symlink_metadata(&lock_path).is_ok() { + regular_file(&lock_path)?; + } + let lock = acquire_runtime_store_owner_lock(&options.root)?; + validate_fresh_runtime_store_layout(&options.root) + .map_err(|_| reject("destination is not fresh; refusing overwrite"))?; + let auth: InitializedAuth = toml::from_str(&text(&options.root.join("auth.toml"))?) + .map_err(|_| reject("initialize Runtime identity and trust before install"))?; + if auth.identity.identity_id != self.runtime_id + || !auth.archived_legacy_issuers.is_empty() + || auth.identity.signing_key().is_err() + { + return Err(reject("invalid destination Runtime identity")); + } + validate_workspace_issuer_trust_records(&auth.workspace_issuers) + .map_err(|_| reject("invalid destination Workspace trust"))?; + let api = request + .workspace_api + .as_ref() + .expect("validated Workspace binding"); + if !auth.workspace_issuers.iter().any(|trust| { + trust.workspace_id == self.workspace_id + && trust.backend_url == api.base_url + && trust.state == WorkspaceIssuerTrustState::Active + }) { + return Err(reject("request does not match initialized Workspace trust")); + } + let stage = options + .root + .join(format!(".fixture-{}", uuid::Uuid::now_v7())); + io(fs::create_dir(&stage))?; + let store = FsRuntimeStore { + root: stage.clone(), + _owner_lock: Some(lock), + worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), + }; + let result = (|| { + let worker = PersistedWorkerRecord { + restore_guard: crate::runtime::RestoreGuard::default(), + worker_ref: WorkerRef::new(self.worker_id), + worker_id: self.worker_id, + profile: request.profile.clone(), + display_name: request.display_name.clone(), + profile_source: request.profile_source.reference(), + config_bundle: request.config_bundle.clone(), + created_at_ms: self.created_at_ms, + status: WorkerStatus::Stopped, + last_finished_submission_request_id: None, + workspace_id: Some(self.workspace_id.clone()), + workdir_attachments: Vec::new(), + logical_workdir_attachments: Vec::new(), + execution_state: PersistedWorkerExecutionState::Available( + PersistedWorkerExecution { + request, + binding: None, + restore_intent: WorkerRestoreIntent::Explicit, + }, + ), + }; + WorkerAggregateRecord::from_persisted(&worker) + .validate(&store.worker_dir(&self.worker_id).join(WORKER_FILE)) + .map_err(|_| reject("invalid installation request"))?; + store.write_worker_record(&worker)?; + let worker_dir = store.worker_dir(&self.worker_id); + atomic_write_json( + &worker_dir.join(WORKER_METADATA_FILE), + &metadata, + "write fixture metadata", + )?; + let session = worker_dir.join("session"); + io(fs::create_dir_all(session.join("segments")))?; + write_new(&session.join("session.json"), &self.session_manifest)?; + write_new( + &session + .join("segments") + .join(format!("{}.jsonl", self.segment_id)), + &self.segment_jsonl, + )?; + self.validate_session(&session)?; + store.write_runtime_snapshot(&PersistedRuntimeState { + subject_create_recovery_fences: BTreeMap::new(), + subject_releases: Vec::new(), + // Match normal store initialization: this is catalog availability, + // not a process or Worker execution start. + display_name: options.display_name.clone(), + status: RuntimeStatus::Running, + next_diagnostic_id: 1, + workers: BTreeMap::new(), + workspace_owners: BTreeMap::new(), + diagnostics: Vec::new(), + })?; + let loaded = store.load_runtime_state()?; + if loaded.workers.len() != 1 || !loaded.diagnostics.is_empty() { + return Err(reject("staged aggregate failed ordinary store roundtrip")); + } + sync_runtime_tree(&stage)?; + // No destination existed when checked under the owner lock. Hard-link + // the snapshot last with create-new semantics; never replace a file. + // Reserve the destination catalog with create-new semantics. Even + // an empty preexisting workers directory must never be replaced. + let destination_workers = options.root.join(WORKERS_DIR); + io(fs::create_dir(&destination_workers))?; + io(fs::rename( + store.worker_dir(&self.worker_id), + destination_workers.join(self.worker_id.to_string()), + ))?; + sync_directory(&destination_workers, "publish stopped Worker aggregate")?; + io(fs::hard_link( + stage.join(RUNTIME_FILE), + options.root.join(RUNTIME_FILE), + ))?; + sync_directory(&options.root, "publish stopped fixture")?; + Ok(()) + })(); + // Only remove our private staging directory, never existing destination data. + let cleanup = fs::remove_dir_all(&stage); + result?; + io(cleanup)?; + sync_directory(&options.root, "finish stopped fixture") + } + + fn validate_shape(&self) -> Result<(), RuntimeError> { + if self.template_version != TEMPLATE_VERSION + || self.runtime_schema_version != SCHEMA_VERSION + { + return Err(reject("unsupported template schema")); + } + if !portable_id(&self.runtime_id) + || !portable_id(&self.workspace_id) + || self.session_id.get_version_num() != 7 + || self.segment_id.get_version_num() != 7 + || self.session_manifest.len() > MAX_SESSION_BYTES + || self.segment_jsonl.len() > MAX_SESSION_BYTES + { + return Err(reject("invalid portable identity or size")); + } + let manifest: SessionManifest = decode(&self.session_manifest)?; + if manifest.session_id != self.session_id + || manifest.schema_version == 0 + || !self.segment_jsonl.ends_with(b"\n") + { + return Err(reject("Session identity or committed boundary mismatch")); + } + // Read-only JSON traversal, never mutation/reconstruction of Session bytes. + // Reject known secret/host markers even in otherwise valid typed records. + for line in self + .segment_jsonl + .split(|b| *b == b'\n') + .filter(|line| !line.is_empty()) + { + let _: PortableJson = decode(line)?; + validate_preserved_record(line)?; + } + Ok(()) + } + + fn validate_history_entry( + &self, + entry: &session_store::LoggedHistoryEntry, + ids: &mut std::collections::BTreeSet, + ) -> Result<(), RuntimeError> { + if !portable_id(&entry.metadata.entry_id.0) + || !ids.insert(entry.metadata.entry_id.0.clone()) + { + return Err(reject("duplicate or invalid entry identity")); + } + if entry + .metadata + .input_metadata + .as_ref() + .is_some_and(|metadata| { + metadata.feature_payload().is_some() || metadata.public_item().is_some() + }) + { + // These are opaque owner-specific payloads, not part of the narrow + // artificial conversation contract. Never preserve unchecked JSON. + return Err(reject("Feature history payloads are unsupported")); + } + if !matches!( + entry.item, + session_store::LoggedItem::Message { .. } + | session_store::LoggedItem::AssistantResponseBoundary { .. } + ) { + return Err(reject("only text conversation records are supported")); + } + if let session_store::LoggedSessionHistoryOrigin::ModelOutput { worker } = + &entry.metadata.origin + { + // Session producer IDs identify the original execution, not the + // Workspace catalog Worker. Preserve them; do not rebind/rewrite them. + if !portable_id(&worker.worker_id) + || worker + .workspace_id + .as_ref() + .is_some_and(|id| id != &self.workspace_id) + || worker + .runtime_id + .as_ref() + .is_some_and(|id| id != &self.runtime_id) + { + return Err(reject("Session origin identity mismatch")); + } + } + Ok(()) + } + + fn validate_session(&self, root: &Path) -> Result<(), RuntimeError> { + for entry in io(fs::read_dir(root))? { + let entry = io(entry)?; + match entry.file_name().to_str() { + Some("session.json" | "segments") => {} + Some("artifacts") => check_empty_artifacts(&entry.path())?, + _ => return Err(reject("unsupported Session files")), + } + } + let entries = io(fs::read_dir(root.join("segments")))?.collect::, _>>(); + let entries = io(entries)?; + if entries.len() != 1 + || entries[0].file_name() != format!("{}.jsonl", self.segment_id).as_str() + { + return Err(reject("only the fixed single-Segment fixture is supported")); + } + let store = WorkerSessionStore::open_read_only(root) + .map_err(|_| reject("unsupported Session schema"))?; + let logs = store + .read_all_read_only_bounded(self.session_id, self.segment_id, MAX_SESSION_BYTES as u64) + .map_err(|_| reject("invalid canonical Session records"))? + .0; + if !matches!(logs.first(), Some(session_store::LogEntry::AnnotatedSegmentStart { + session_id, system_prompt: Some(_), forked_from: None, compacted_from: None, .. + }) if *session_id == self.session_id) + { + return Err(reject( + "missing system prompt, invalid Segment seed, or unsupported lineage", + )); + } + let mut ids = std::collections::BTreeSet::new(); + for (index, log) in logs.iter().enumerate() { + use session_store::LogEntry; + match log { + LogEntry::AnnotatedSegmentStart { history, .. } + if index == 0 && history.is_empty() => {} + LogEntry::AnnotatedUserInput { + history, + extensions, + .. + } => { + for extension in extensions { + validate_settled_activation(extension)?; + } + for entry in history { + self.validate_history_entry(entry, &mut ids)?; + } + } + LogEntry::AnnotatedAssistantItem { entry, .. } => { + self.validate_history_entry(entry, &mut ids)? + } + LogEntry::RunCompleted { + entry_id: Some(id), .. + } => { + if !portable_id(&id.0) || !ids.insert(id.0.clone()) { + return Err(reject("duplicate or invalid entry identity")); + } + } + LogEntry::Invoke { .. } | LogEntry::LlmUsage { .. } | LogEntry::TurnEnd { .. } => {} + _ => { + return Err(reject( + "unsupported conversation record or pending operation", + )); + } + } + } + let state = session_store::collect_state(&logs); + if state.last_run_interrupted || state.active_run_turn_count.is_some() { + return Err(reject("Session has pending execution")); + } + for segments in &state.user_segments { + if segments + .iter() + .any(|segment| !matches!(segment, protocol::Segment::Text { .. })) + { + return Err(reject("Session attachments are unsupported")); + } + } + Ok(()) + } +} + +// Compare *fields*, not re-encoded bytes or scalar values (e.g. f32 JSON +// normalization). Serde's permissive/untagged/flattened compatibility readers +// can drop unknown data without surfacing it to an ignored-field callback. A +// retained field must survive the owner's typed projection at every depth. +// This intentionally accepts only canonical writer shapes: even an explicit +// empty optional field omitted by that writer is rejected, not repaired. +// The projection is validation-only; installation always writes original bytes. +fn validate_preserved_record(bytes: &[u8]) -> Result<(), RuntimeError> { + let original: serde_json::Value = decode(bytes)?; + let record: session_store::LogEntry = decode(bytes)?; + let projection = + serde_json::to_value(&record).map_err(|_| reject("invalid typed Session record"))?; + require_preserved_fields(&original, &projection) +} + +fn require_preserved_fields( + original: &serde_json::Value, + projection: &serde_json::Value, +) -> Result<(), RuntimeError> { + use serde_json::Value; + match (original, projection) { + (Value::Object(original), Value::Object(projection)) => { + for (key, value) in original { + let projected = projection + .get(key) + .ok_or_else(|| reject("unknown or noncanonical Session field"))?; + require_preserved_fields(value, projected)?; + } + } + (Value::Array(original), Value::Array(projection)) + if original.len() == projection.len() => + { + for (value, projected) in original.iter().zip(projection) { + require_preserved_fields(value, projected)?; + } + } + (Value::Null, Value::Null) + | (Value::Bool(_), Value::Bool(_)) + | (Value::Number(_), Value::Number(_)) + | (Value::String(_), Value::String(_)) => {} + _ => return Err(reject("noncanonical Session field shape")), + } + Ok(()) +} + +// Exact receipt wire fields owned by worker::PendingActivationState. Keep these +// fixture-only mirrors strict; do not change the production compatibility reader. +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct SubmissionReceipt { + submission_request_id: String, + source_namespace: String, + submission_id: String, + payload_digest: String, + disposition: protocol::SubmissionDisposition, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct NotificationReceipt { + notification_request_id: String, + source_namespace: String, + payload_digest: String, +} + +// The approved conversation has committed submission checkpoints in its original +// Session records. Keep their receipt bytes, but reject any resumable queue or +// Feature invocation history. This projection confers no execution authority. +fn validate_settled_activation( + extension: &session_store::SessionExtension, +) -> Result<(), RuntimeError> { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct Checkpoint { + #[serde(rename = "next_activation_sequence")] + _next_activation_sequence: u64, + activating: Option, + activating_notification: Vec, + pending: Vec, + pending_notifications: Vec, + #[serde(rename = "receipts")] + _receipts: Vec, + #[serde(rename = "notification_receipts")] + _notification_receipts: Vec, + invocation_receipts: Vec, + invocation_recovery_blocked: bool, + } + if extension.domain != "worker.pending_activations.v1" { + return Err(reject("unsupported Session extension")); + } + let checkpoint: Checkpoint = serde_json::from_value(extension.payload.clone()) + .map_err(|_| reject("invalid Session activation checkpoint"))?; + if checkpoint.activating.is_some() + || !checkpoint.activating_notification.is_empty() + || !checkpoint.pending.is_empty() + || !checkpoint.pending_notifications.is_empty() + || !checkpoint.invocation_receipts.is_empty() + || checkpoint.invocation_recovery_blocked + { + return Err(reject( + "Session has pending activation or invocation history", + )); + } + Ok(()) +} + +fn check_request(request: &CreateWorkerRequest, install: bool) -> Result<(), RuntimeError> { + if !request.workdir_attachment_requests.is_empty() + || !request.workdir_attachments.is_empty() + || request.backend_job.is_some() + || request.subjektiv_attached + || request.worker_observation_enabled + || !request.worker_observation_grants.is_empty() + || (install && request.initial_input.is_some()) + { + return Err(reject( + "unsupported capabilities, attachments, or initial input", + )); + } + if install + && (request.create_fingerprint.trim().is_empty() + || request.config_bundle.is_none() + || !matches!( + request.profile_source, + ProfileSourceArchiveSource::WorkspaceConfig { .. } + )) + { + return Err(reject( + "fresh config/profile references and create fingerprint are required", + )); + } + Ok(()) +} + +fn check_empty_artifacts(root: &Path) -> Result<(), RuntimeError> { + for entry in io(fs::read_dir(root))? { + let entry = io(entry)?; + let metadata = io(entry.metadata())?; + if metadata.is_dir() { + check_empty_artifacts(&entry.path())?; + } else if entry.file_name() != ".aggregate.lock" || metadata.len() != 0 { + return Err(reject("Session artifacts are unsupported")); + } + } + Ok(()) +} + +fn write_new(path: &Path, bytes: &[u8]) -> Result<(), RuntimeError> { + let mut file = io(OpenOptions::new().write(true).create_new(true).open(path))?; + io(file.write_all(bytes))?; + io(file.sync_all()) +} + +// A streaming validation visitor, not a generic document available to callers. +struct PortableJson; +impl<'de> Deserialize<'de> for PortableJson { + fn deserialize>(deserializer: D) -> Result { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = PortableJson; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("portable fixture JSON") + } + fn visit_bool(self, _: bool) -> Result { + Ok(PortableJson) + } + fn visit_i64(self, _: i64) -> Result { + Ok(PortableJson) + } + fn visit_u64(self, _: u64) -> Result { + Ok(PortableJson) + } + fn visit_f64(self, _: f64) -> Result { + Ok(PortableJson) + } + fn visit_unit(self) -> Result { + Ok(PortableJson) + } + fn visit_str(self, value: &str) -> Result { + let lower = value.to_ascii_lowercase(); + if [ + "://", + "private key", + "bearer ", + "sk-", + "api_key", + "access_token", + "refresh_token", + "private_key", + "authorization", + ] + .iter() + .any(|marker| lower.contains(marker)) + || value + .split_whitespace() + .any(|word| word.starts_with('/') || word.contains(":\\")) + { + return Err(E::custom("nonportable or sensitive fixture material")); + } + Ok(PortableJson) + } + fn visit_seq>( + self, + mut seq: A, + ) -> Result { + while seq.next_element::()?.is_some() {} + Ok(PortableJson) + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut keys = std::collections::BTreeSet::new(); + while let Some(key) = map.next_key::()? { + // A duplicate key can hide discarded data even inside an + // otherwise opaque extension payload. Never retain it. + if !keys.insert(key.clone()) { + return Err(serde::de::Error::custom("duplicate fixture JSON field")); + } + serde::de::Visitor::visit_str::(Visitor, &key)?; + map.next_value::()?; + } + Ok(PortableJson) + } + } + deserializer.deserialize_any(Visitor) + } +} diff --git a/crates/worker-runtime/src/test_support_tests.rs b/crates/worker-runtime/src/test_support_tests.rs new file mode 100644 index 000000000..203434258 --- /dev/null +++ b/crates/worker-runtime/src/test_support_tests.rs @@ -0,0 +1,885 @@ +use super::*; +use crate::auth::RuntimeIdentityMaterial; +use crate::catalog::{ProfileSelector, WorkspaceApiRef}; +use crate::profile_archive::ProfileSourceGraphSummary; +use session_store::{LogEntry, Store}; +use sha2::{Digest, Sha256}; + +const RUNTIME: &str = "fixture-runtime"; +const WORKSPACE: &str = "fixture-workspace"; +const NEW_URL: &str = "http://127.0.0.1:34567"; + +#[derive(Serialize)] +#[serde(untagged)] +enum ReceiptFixture { + Valid(T), + Malformed(u8), +} + +#[derive(Default, Serialize)] +struct CheckpointFixture { + next_activation_sequence: u64, + activating: Option, + activating_notification: Vec, + pending: Vec, + pending_notifications: Vec, + receipts: Vec>, + notification_receipts: Vec>, + invocation_receipts: Vec, + invocation_recovery_blocked: bool, +} + +fn settled_checkpoint() -> CheckpointFixture { + CheckpointFixture { + next_activation_sequence: 2, + receipts: vec![ReceiptFixture::Valid(SubmissionReceipt { + submission_request_id: "completed-submission-request".into(), + source_namespace: "fixture-source".into(), + submission_id: "completed-submission".into(), + payload_digest: "fixture-payload-digest".into(), + disposition: protocol::SubmissionDisposition::Started, + })], + notification_receipts: vec![ReceiptFixture::Valid(NotificationReceipt { + notification_request_id: "completed-notification-request".into(), + source_namespace: "fixture-source".into(), + payload_digest: "fixture-notification-digest".into(), + })], + ..Default::default() + } +} + +fn checkpoint_extension(checkpoint: &CheckpointFixture) -> session_store::SessionExtension { + session_store::SessionExtension::new( + "worker.pending_activations.v1", + serde_json::to_value(checkpoint).unwrap(), + ) +} + +fn worker_id() -> WorkerId { + "01a1657d-b762-7791-87ae-a49f28cadbce".parse().unwrap() +} + +fn resolved() -> manifest::WorkerManifest { + let mut config = manifest::WorkerManifestConfig::resolution_defaults(); + config.worker.name = Some(runtime_worker_name(worker_id())); + manifest::WorkerManifest::try_from(config).unwrap() +} + +fn request() -> CreateWorkerRequest { + CreateWorkerRequest { + worker_id: worker_id(), + create_fingerprint: "new-create-fingerprint".into(), + profile: ProfileSelector::Builtin("default".into()), + display_name: Some("fixture".into()), + profile_source: ProfileSourceArchiveSource::WorkspaceConfig { + archive: ProfileSourceArchiveRef { + id: "new-profile".into(), + digest: "new-profile-digest".into(), + size_bytes: 1, + source_graph: ProfileSourceGraphSummary { + source_count: 1, + total_source_bytes: 1, + entrypoints: BTreeMap::from([("builtin:default".into(), "default.dcl".into())]), + import_count: 0, + }, + }, + }, + config_bundle: Some(ConfigBundleRef { + id: "new-config".into(), + digest: "new-config-digest".into(), + }), + workspace_api: Some(WorkspaceApiRef { + workspace_id: WORKSPACE.into(), + base_url: NEW_URL.into(), + }), + initial_input: None, + workdir_attachment_requests: Vec::new(), + workdir_attachments: Vec::new(), + worker_observation_enabled: false, + worker_observation_grants: Vec::new(), + memory_settings: Some(manifest::WorkspaceMemorySettingsSnapshot { + workspace_id: WORKSPACE.into(), + language: "en".into(), + }), + subjektiv_attached: false, + backend_job: None, + } +} + +fn initialize(root: &Path, runtime_id: &str, workspace_id: &str, url: &str) { + #[derive(Serialize)] + struct Auth { + identity: RuntimeIdentityMaterial, + workspace_issuers: Vec, + } + let issuer = RuntimeIdentityMaterial::generate("fixture-issuer").unwrap(); + let digest = Sha256::digest(crate::auth::decode_public_key(&issuer.public_key).unwrap()); + let fingerprint = format!( + "sha256:{}", + digest + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + ); + let trust = WorkspaceIssuerTrustRecord::from_bundle( + server_api::WorkspacePublicIdentityBundle { + workspace_id: workspace_id.into(), + backend_url: url.into(), + key_id: "fixture-issuer".into(), + algorithm: "ed25519".into(), + public_key: issuer.public_key, + public_key_fingerprint: fingerprint, + }, + "fixture-trust".into(), + 100, + ) + .unwrap(); + fs::create_dir_all(root).unwrap(); + fs::write( + root.join("auth.toml"), + toml::to_string(&Auth { + identity: RuntimeIdentityMaterial::generate(runtime_id).unwrap(), + workspace_issuers: vec![trust], + }) + .unwrap(), + ) + .unwrap(); +} + +fn record(request: CreateWorkerRequest) -> PersistedWorkerRecord { + PersistedWorkerRecord { + restore_guard: crate::runtime::RestoreGuard::default(), + worker_ref: WorkerRef::new(request.worker_id), + worker_id: request.worker_id, + profile: request.profile.clone(), + display_name: request.display_name.clone(), + profile_source: request.profile_source.reference(), + config_bundle: request.config_bundle.clone(), + created_at_ms: Some(123456), + status: WorkerStatus::Stopped, + last_finished_submission_request_id: Some("old-submission-operation".into()), + workspace_id: Some(WORKSPACE.into()), + workdir_attachments: vec![], + logical_workdir_attachments: vec![], + execution_state: PersistedWorkerExecutionState::Available(PersistedWorkerExecution { + request, + binding: Some(PersistedWorkerExecutionBinding {}), + restore_intent: WorkerRestoreIntent::Explicit, + }), + } +} + +fn source() -> (tempfile::TempDir, PathBuf, StoppedWorkerTemplate) { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("runtime"); + initialize(&root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456"); + drop( + crate::Runtime::with_fs_store(FsRuntimeStoreOptions::new(&root).with_runtime_id(RUNTIME)) + .unwrap(), + ); + let opened = FsRuntimeStore::open_or_create(root.clone(), RUNTIME).unwrap(); + let mut old = request(); + old.workspace_api.as_mut().unwrap().base_url = "http://127.0.0.1:23456".into(); + old.config_bundle.as_mut().unwrap().id = "old-config".into(); + opened.store.write_worker_record(&record(old)).unwrap(); + let worker = opened.store.worker_dir(&worker_id()); + let session_id = "01a1657d-b800-7777-8888-111111111111".parse().unwrap(); + let segment_id = "01a1657d-b818-77c1-bd73-176172a4bd82".parse().unwrap(); + let mut metadata = WorkerMetadata::new( + runtime_worker_name(worker_id()), + Some(WorkerActiveSegmentRef { + session_id, + segment_id: Some(segment_id), + }), + ) + .with_workspace_id(WORKSPACE) + .with_workspace_root(PathBuf::from("/old/host/path")); + metadata.resolved_manifest_snapshot = + Some(manifest::write_persisted_worker_manifest_snapshot(&resolved()).unwrap()); + atomic_write_json( + &worker.join(WORKER_METADATA_FILE), + &metadata, + "test metadata", + ) + .unwrap(); + let sessions = WorkerSessionStore::new(worker.join("session")).unwrap(); + let history = session_store::LoggedHistoryEntry { + item: session_store::LoggedItem::Message { + role: session_store::LoggedRole::Assistant, + content: vec![session_store::LoggedContentPart::Text { + text: "artificial fixture reply".into(), + }], + }, + metadata: session_store::LoggedSessionHistoryMetadata { + entry_id: session_store::LoggedSessionHistoryEntryId("fixed-entry-id".into()), + origin: session_store::LoggedSessionHistoryOrigin::ModelOutput { + worker: session_store::LoggedWorkerSubject { + workspace_id: Some(WORKSPACE.into()), + runtime_id: Some(RUNTIME.into()), + worker_id: worker_id().to_string(), + }, + }, + derivation: None, + input_metadata: None, + }, + }; + sessions + .create_segment( + session_id, + segment_id, + &[ + LogEntry::AnnotatedSegmentStart { + ts: 123, + session_id, + system_prompt: Some("Harmless artificial fixture instructions.".into()), + config: Default::default(), + history: vec![], + forked_from: None, + compacted_from: None, + }, + LogEntry::Invoke { + ts: 124, + trigger: protocol::InvokeKind::UserSend, + }, + LogEntry::AnnotatedUserInput { + ts: 124, + segments: vec![protocol::Segment::text( + "Harmless artificial fixture input.", + )], + extensions: vec![checkpoint_extension(&settled_checkpoint())], + history: vec![session_store::LoggedHistoryEntry { + item: session_store::LoggedItem::Message { + role: session_store::LoggedRole::User, + content: vec![session_store::LoggedContentPart::Text { + text: "Harmless artificial fixture input.".into(), + }], + }, + metadata: session_store::LoggedSessionHistoryMetadata { + entry_id: session_store::LoggedSessionHistoryEntryId( + "fixed-user-id".into(), + ), + origin: session_store::LoggedSessionHistoryOrigin::BackendInstruction { + operation_id: None, + }, + derivation: None, + input_metadata: None, + }, + }], + }, + LogEntry::AnnotatedAssistantItem { + ts: 125, + entry: history, + }, + LogEntry::LlmUsage { + ts: 125, + history_len: 2, + input_total_tokens: 1, + cache_read_tokens: 0, + cache_write_tokens: 0, + output_tokens: 1, + }, + LogEntry::TurnEnd { + ts: 126, + turn_count: 1, + }, + LogEntry::RunCompleted { + entry_id: Some(session_store::LoggedSessionHistoryEntryId( + "fixed-terminal-id".into(), + )), + ts: 126, + interrupted: false, + result: agen::EngineResult::Finished, + active_run_turn_count: None, + }, + ], + ) + .unwrap(); + fs::create_dir_all(worker.join("runs/old-run")).unwrap(); + fs::write(worker.join("runs/old-run/private.log"), "not exported").unwrap(); + drop(opened); + let template = StoppedWorkerTemplate::extract(&worker).unwrap(); + (temp, worker, template) +} + +fn destination(template: &StoppedWorkerTemplate) -> (tempfile::TempDir, FsRuntimeStoreOptions) { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("runtime"); + initialize( + &root, + template.runtime_id(), + template.workspace_id(), + NEW_URL, + ); + ( + temp, + FsRuntimeStoreOptions::new(root).with_runtime_id(template.runtime_id()), + ) +} + +fn assert_fresh(options: &FsRuntimeStoreOptions) { + assert_eq!(fs::read_dir(&options.root).unwrap().count(), 1); + assert!(options.root.join("auth.toml").is_file()); +} + +#[test] +fn extraction_install_roundtrip_preserves_bytes_and_is_inactive() { + let (_source, worker, template) = source(); + let original_worker = fs::read(worker.join(WORKER_FILE)).unwrap(); + let original_metadata = fs::read(worker.join(WORKER_METADATA_FILE)).unwrap(); + let exported = serde_json::to_vec(&template).unwrap(); + let exported_text = String::from_utf8(exported.clone()).unwrap(); + for forbidden in [ + "old-config", + "old-submission-operation", + "restore_guard", + "private_key", + "http://", + "/old/host", + "private.log", + "resolved_manifest_snapshot", + ] { + assert!(!exported_text.contains(forbidden)); + } + let template: StoppedWorkerTemplate = serde_json::from_slice(&exported).unwrap(); + let (_target, options) = destination(&template); + let auth_before = fs::read(options.root.join("auth.toml")).unwrap(); + template.install(&options, request(), &resolved()).unwrap(); + assert_eq!( + fs::read(options.root.join("auth.toml")).unwrap(), + auth_before + ); + assert_eq!(fs::read(worker.join(WORKER_FILE)).unwrap(), original_worker); + assert_eq!( + fs::read(worker.join(WORKER_METADATA_FILE)).unwrap(), + original_metadata + ); + let installed = options + .root + .join(WORKERS_DIR) + .join(template.worker_id().to_string()); + assert_eq!( + fs::read(installed.join("session/session.json")).unwrap(), + template.session_manifest + ); + assert_eq!( + fs::read(installed.join(format!("session/segments/{}.jsonl", template.segment_id()))) + .unwrap(), + template.segment_jsonl + ); + assert!(!installed.join("runs").exists()); + let metadata: WorkerMetadata = + decode(&fs::read(installed.join(WORKER_METADATA_FILE)).unwrap()).unwrap(); + assert_eq!(metadata.active.unwrap().session_id, template.session_id()); + assert!(metadata.workspace_root.is_none() && metadata.resolved_manifest_snapshot.is_some()); + + let opened = FsRuntimeStore::open_or_create(options.root.clone(), RUNTIME).unwrap(); + let state = opened.state.unwrap(); + assert!(state.diagnostics.is_empty()); + let saved = state.workers.get(&worker_id()).unwrap(); + assert_eq!(saved.status, WorkerStatus::Stopped); + assert!(saved.last_finished_submission_request_id.is_none()); + let PersistedWorkerExecutionState::Available(execution) = &saved.execution_state else { + panic!("not available") + }; + assert_eq!(execution.restore_intent, WorkerRestoreIntent::Explicit); + assert!(execution.binding.is_none()); + assert_eq!(execution.request, request()); + // Fresh restore guard, no operation owner/receipt history. + assert_ne!( + serde_json::to_vec(&saved.restore_guard).unwrap(), + serde_json::to_vec( + &decode::(&original_worker) + .unwrap() + .restore_guard + ) + .unwrap() + ); + let guard: GuardState = decode(&serde_json::to_vec(&saved.restore_guard).unwrap()).unwrap(); + assert!(guard.active_request_id.is_none() && guard.owners.is_empty()); + drop(opened.store); + + struct NoExecution; + impl crate::execution::WorkerExecutionBackend for NoExecution { + fn backend_id(&self) -> &str { + "fixture-no-execution" + } + fn spawn_worker( + &self, + _: crate::execution::WorkerExecutionSpawnRequest, + ) -> crate::execution::WorkerExecutionSpawnResult { + panic!("fixture executed") + } + fn dispatch_input( + &self, + _: &WorkerRef, + _: crate::interaction::WorkerInput, + ) -> crate::execution::WorkerExecutionResult { + panic!("fixture submitted") + } + fn restore_worker( + &self, + _: crate::execution::WorkerExecutionRestoreRequest, + ) -> crate::execution::WorkerExecutionSpawnResult { + panic!("fixture auto-restored") + } + } + let runtime = + crate::Runtime::with_fs_store_and_execution_backend(options.clone(), Arc::new(NoExecution)) + .unwrap(); + assert_eq!(runtime.list_workers().unwrap().len(), 1); + assert_eq!(runtime.summary().unwrap().active_worker_count, 0); + assert_eq!( + runtime + .worker_detail(&WorkerRef::new(worker_id())) + .unwrap() + .status, + WorkerStatus::Stopped + ); + drop(runtime); + let runtime = crate::Runtime::with_fs_store(options).unwrap(); + assert_eq!( + runtime.list_workers().unwrap()[0].status, + WorkerStatus::Stopped + ); +} + +#[test] +fn install_rejects_schema_identity_input_and_overwrite() { + let (_source, _, template) = source(); + for change in [ + "template", + "runtime-schema", + "session-schema", + "session-id", + "segment-id", + "runtime-id", + "workspace-id", + "worker-id", + "origin-id", + "duplicate-id", + "truncated", + ] { + let (_target, options) = destination(&template); + let mut invalid = template.clone(); + match change { + "template" => invalid.template_version += 1, + "runtime-schema" => invalid.runtime_schema_version += 1, + "session-schema" => { + invalid.session_manifest = format!( + "{{\"schema_version\":999,\"session_id\":\"{}\"}}", + invalid.session_id + ) + .into_bytes() + } + "session-id" => invalid.session_id = uuid::Uuid::now_v7(), + "segment-id" => invalid.segment_id = uuid::Uuid::nil(), // no external attestation: valid new Segment ID is otherwise portable + "runtime-id" => invalid.runtime_id = "other-runtime".into(), + "workspace-id" => invalid.workspace_id = "other-workspace".into(), + "worker-id" => invalid.worker_id = WorkerId::now_v7(), + "origin-id" => { + invalid.segment_jsonl = String::from_utf8(invalid.segment_jsonl) + .unwrap() + .replace(RUNTIME, "other-runtime") + .into_bytes() + } + "duplicate-id" => { + let last = invalid + .segment_jsonl + .split_inclusive(|b| *b == b'\n') + .last() + .unwrap() + .to_vec(); + invalid.segment_jsonl.extend(last); + } + "truncated" => { + invalid.segment_jsonl.pop(); + } + _ => unreachable!(), + } + assert!( + invalid.install(&options, request(), &resolved()).is_err(), + "accepted {change}" + ); + assert_fresh(&options); + } + let (_target, options) = destination(&template); + let mut launch = request(); + launch.initial_input = Some(crate::interaction::WorkerInput::user("do not execute")); + assert!(template.install(&options, launch, &resolved()).is_err()); + assert_fresh(&options); + template.install(&options, request(), &resolved()).unwrap(); + let worker_path = options + .root + .join(WORKERS_DIR) + .join(worker_id().to_string()) + .join(WORKER_FILE); + let before = fs::read(&worker_path).unwrap(); + assert!(template.install(&options, request(), &resolved()).is_err()); + assert_eq!(fs::read(worker_path).unwrap(), before); +} + +#[test] +fn rejects_uninitialized_untrusted_live_and_partial_destination() { + let (_source, _, template) = source(); + let (_target, options) = destination(&template); + let held = acquire_runtime_store_owner_lock(&options.root).unwrap(); + assert!(template.install(&options, request(), &resolved()).is_err()); + assert_fresh(&options); + drop(held); + initialize(&options.root, "wrong-runtime", WORKSPACE, NEW_URL); + assert!(template.install(&options, request(), &resolved()).is_err()); + initialize(&options.root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456"); + assert!(template.install(&options, request(), &resolved()).is_err()); + fs::remove_file(options.root.join("auth.toml")).unwrap(); + assert!(template.install(&options, request(), &resolved()).is_err()); + initialize(&options.root, RUNTIME, WORKSPACE, NEW_URL); + fs::create_dir(options.root.join(WORKERS_DIR)).unwrap(); + assert!(template.install(&options, request(), &resolved()).is_err()); +} + +#[test] +fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() { + for change in [ + "running", + "pending", + "guard-pending", + "attachment", + "schema", + "worker-id", + "metadata-workspace", + "metadata-session", + "artifact", + "secret", + "url", + "path", + ] { + let (_source, worker, _) = source(); + let mut aggregate: WorkerAggregateRecord = + decode(&fs::read(worker.join(WORKER_FILE)).unwrap()).unwrap(); + match change { + "running" => aggregate.status = WorkerStatus::Running, + "pending" => { + aggregate.execution_state = WorkerExecutionStateRecord::ReconciliationRequired( + PersistedWorkerLifecycleOperation::Restore(PersistedWorkerRestoreOperation { + operation_id: WorkerLifecycleOperationId::new(), + mode: PersistedWorkerRestoreMode::Explicit, + request: request(), + binding: None, + last_settled_status: WorkerStatus::Stopped, + }), + ) + } + "guard-pending" => aggregate.restore_guard = serde_json::from_str(r#"{"observation_token":"old-token","active_request_id":"pending-restore","owners":{}}"#).unwrap(), + "attachment" => aggregate.logical_workdir_attachments.push(LogicalWorkdirAttachment { + alias: workdir::WorkdirAttachmentAlias::new("source").unwrap(), + working_directory_id: "old-workdir".into(), + capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY, + }), + "schema" => aggregate.schema_version += 1, + "worker-id" => aggregate.worker_id = WorkerId::now_v7(), + "metadata-workspace" | "metadata-session" => { + let mut metadata: WorkerMetadata = + decode(&fs::read(worker.join(WORKER_METADATA_FILE)).unwrap()).unwrap(); + if change == "metadata-workspace" { + metadata.workspace_id = Some("wrong-workspace".into()); + } else { + metadata.active.as_mut().unwrap().session_id = uuid::Uuid::now_v7(); + } + atomic_write_json( + &worker.join(WORKER_METADATA_FILE), + &metadata, + "test mismatch", + ) + .unwrap(); + } + "artifact" => { + fs::create_dir_all(worker.join("session/artifacts/paste")).unwrap(); + fs::write( + worker.join("session/artifacts/paste/payload"), + "unsupported", + ) + .unwrap(); + } + "secret" | "url" | "path" => { + let segment = fs::read_dir(worker.join("session/segments")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let replacement = match change { + "secret" => "Bearer secret", + "url" => "https://old-host.invalid", + _ => "/old/host/path", + }; + let text = fs::read_to_string(&segment) + .unwrap() + .replace("artificial fixture reply", replacement); + fs::write(segment, text).unwrap(); + } + _ => unreachable!(), + } + atomic_write_json(&worker.join(WORKER_FILE), &aggregate, "test aggregate").unwrap(); + assert!( + StoppedWorkerTemplate::extract(&worker).is_err(), + "accepted {change}" + ); + } + let (_source, worker, _) = source(); + let held = + acquire_runtime_store_owner_lock(worker.parent().unwrap().parent().unwrap()).unwrap(); + assert!(StoppedWorkerTemplate::extract(&worker).is_err()); + drop(held); +} + +#[test] +fn fresh_manifest_and_memory_settings_are_required() { + let (_source, _, template) = source(); + let (_target, options) = destination(&template); + let mut manifest = resolved(); + manifest.worker.name = "wrong-worker".into(); + assert!(template.install(&options, request(), &manifest).is_err()); + let mut binding = request(); + binding.memory_settings = None; + assert!(template.install(&options, binding, &resolved()).is_err()); + let mut binding = request(); + binding.memory_settings.as_mut().unwrap().workspace_id = "other-workspace".into(); + assert!(template.install(&options, binding, &resolved()).is_err()); + assert_fresh(&options); +} + +#[test] +fn settled_checkpoints_preserve_receipts_but_refuse_pending_work() { + let mut checkpoint = settled_checkpoint(); + assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_ok()); + checkpoint.pending.push(1); + assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err()); + checkpoint.pending.clear(); + checkpoint.activating = Some(1); + assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err()); + checkpoint.activating = None; + checkpoint.invocation_receipts.push(1); + assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err()); + checkpoint.invocation_receipts.clear(); + checkpoint.invocation_recovery_blocked = true; + assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err()); +} + +// Deliberately corrupt only test-owned synthetic source files, never the lab. +fn assert_rejected_session_bytes(worker: &Path, template: &StoppedWorkerTemplate, bytes: Vec) { + let path = worker.join(format!("session/segments/{}.jsonl", template.segment_id())); + fs::write(path, &bytes).unwrap(); + let extraction_error = StoppedWorkerTemplate::extract(worker) + .err() + .expect("invalid source accepted"); + assert!( + !extraction_error + .to_string() + .contains("synthetic-discarded-material") + ); + let (_target, options) = destination(template); + let mut invalid = template.clone(); + invalid.segment_jsonl = bytes; + let installation_error = invalid + .install(&options, request(), &resolved()) + .unwrap_err(); + assert!( + !installation_error + .to_string() + .contains("synthetic-discarded-material") + ); + assert_fresh(&options); +} + +fn synthetic_logs(template: &StoppedWorkerTemplate) -> Vec { + template + .segment_jsonl + .split(|byte| *byte == b'\n') + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_slice(line).unwrap()) + .collect() +} + +fn encode_synthetic_logs(logs: &[LogEntry]) -> Vec { + let mut bytes = Vec::new(); + for log in logs { + serde_json::to_writer(&mut bytes, log).unwrap(); + bytes.push(b'\n'); + } + bytes +} + +#[tokio::test] +async fn missing_system_prompt_matches_worker_restore_rejection() { + let (_source, worker_dir, template) = source(); + let mut logs = synthetic_logs(&template); + assert!(session_store::collect_state(&logs).system_prompt.is_some()); + let LogEntry::AnnotatedSegmentStart { system_prompt, .. } = &mut logs[0] else { + panic!("missing seed") + }; + *system_prompt = None; + assert_rejected_session_bytes(&worker_dir, &template, encode_synthetic_logs(&logs)); + // The owning Worker constructor rejects this before model-client creation, + // allocation, or execution. Do not start a controller or submit any input. + let store = session_store::CombinedStore::new( + WorkerSessionStore::open_read_only(worker_dir.join("session")).unwrap(), + session_store::WorkerAggregateStore::open_read_only(&worker_dir, template.worker_name()) + .unwrap(), + ); + let restored = worker::Worker::restore_from_manifest( + template.session_id(), + template.segment_id(), + resolved(), + store, + worker::PromptCatalogSource::builtins_only(), + ) + .await; + assert!(matches!( + restored, + Err(worker::WorkerError::SegmentSystemPromptMissing { .. }) + )); +} + +#[test] +fn malformed_submission_and_notification_receipts_fail_extraction_and_install() { + let (_source, worker, template) = source(); + for notification in [false, true] { + let mut checkpoint = settled_checkpoint(); + if notification { + checkpoint.notification_receipts = + vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)]; + } else { + checkpoint.receipts = vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)]; + } + let mut logs = synthetic_logs(&template); + for log in &mut logs { + if let LogEntry::AnnotatedUserInput { extensions, .. } = log { + *extensions = vec![checkpoint_extension(&checkpoint)]; + } + } + assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs)); + } +} + +#[test] +fn unknown_fields_at_every_retained_record_depth_are_rejected_without_secret_diagnostics() { + let (_source, worker, template) = source(); + let original = String::from_utf8(template.segment_jsonl.clone()).unwrap(); + // These values are harmless synthetic markers, not lab/private material. + // The compatibility decoder normally discards many of these fields. + let locations = [ + r#""kind":"annotated_segment_start""#, + r#""kind":"invoke""#, + r#""kind":"annotated_user_input""#, + r#""kind":"annotated_assistant_item""#, + r#""kind":"llm_usage""#, + r#""kind":"turn_end""#, + r#""kind":"run_completed""#, + r#""max_tokens":null"#, + r#""kind":"message","role":"assistant""#, + r#""kind":"text","text":"artificial fixture reply""#, + r#""kind":"text","content":"Harmless artificial fixture input.""#, + r#""entry_id":"fixed-entry-id""#, + r#""kind":"model_output""#, + r#""worker_id":"01a1657d-b762-7791-87ae-a49f28cadbce""#, + r#""domain":"worker.pending_activations.v1""#, + r#""next_activation_sequence":2"#, + r#""submission_request_id":"completed-submission-request""#, + r#""notification_request_id":"completed-notification-request""#, + ]; + for location in locations { + assert!(original.contains(location)); + for extra in [ + r#", "reasoning":{"text":"synthetic-discarded-material"}"#, + r#", "encrypted_content":"synthetic-discarded-material""#, + ] { + let altered = original.replacen(location, &format!("{location}{extra}"), 1); + assert_rejected_session_bytes(&worker, &template, altered.into_bytes()); + } + } + let duplicate = original.replacen( + r#""role":"assistant""#, + r#""role":"assistant","role":"assistant""#, + 1, + ); + assert_rejected_session_bytes(&worker, &template, duplicate.into_bytes()); +} + +#[test] +fn opaque_feature_payloads_cannot_bypass_strict_session_validation() { + let (_source, worker, template) = source(); + let mut logs = synthetic_logs(&template); + for log in &mut logs { + if let LogEntry::AnnotatedUserInput { history, .. } = log { + if let session_store::LoggedItem::Message { role, .. } = &mut history[0].item { + *role = session_store::LoggedRole::System; + } + history[0].metadata.input_metadata = Some( + session_store::HistoryInputMetadata::new( + session_store::HistoryInputUsePolicy { + recording_material: session_store::InputEvidenceUse::Exclude, + memory_evidence: session_store::InputEvidenceUse::Exclude, + observation: session_store::InputObservationUse::Hidden, + compaction: session_store::InputCompactionUse::Retain, + }, + Some( + session_store::FeatureInputPayload::encode( + "fixture.test", + 1, + &BTreeMap::from([("reasoning", "harmless synthetic payload")]), + ) + .unwrap(), + ), + None, + ) + .unwrap(), + ); + } + } + assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs)); +} + +#[cfg(unix)] +#[test] +fn refuses_source_and_destination_symlinks() { + let (_source, worker, template) = source(); + let (_target, options) = destination(&template); + std::os::unix::fs::symlink(worker.join(WORKER_FILE), options.root.join("linked")).unwrap(); + assert!(template.install(&options, request(), &resolved()).is_err()); + fs::remove_file(options.root.join("linked")).unwrap(); + std::os::unix::fs::symlink(worker.join(WORKER_FILE), worker.join("runs/link")).unwrap(); + assert!(StoppedWorkerTemplate::extract(&worker).is_err()); +} + +/// Opt-in only: source path supplied explicitly; assertions never display content. +#[test] +#[ignore = "requires the approved artificial lab aggregate via YOI_STOPPED_FIXTURE_SOURCE"] +fn approved_lab_extract_install() { + let path = PathBuf::from( + std::env::var_os("YOI_STOPPED_FIXTURE_SOURCE").expect("explicit source required"), + ); + let template = StoppedWorkerTemplate::extract(&path).unwrap(); + let (_target, options) = destination(&template); + let mut bind = request(); + bind.worker_id = template.worker_id(); + bind.workspace_api.as_mut().unwrap().workspace_id = template.workspace_id().into(); + bind.memory_settings.as_mut().unwrap().workspace_id = template.workspace_id().into(); + let mut manifest = resolved(); + manifest.worker.name = template.worker_name(); + template.install(&options, bind, &manifest).unwrap(); + let installed = options + .root + .join(WORKERS_DIR) + .join(template.worker_id().to_string()); + for relative in [ + "session/session.json".to_string(), + format!("session/segments/{}.jsonl", template.segment_id()), + ] { + assert!( + fs::read(path.join(&relative)).unwrap() == fs::read(installed.join(relative)).unwrap() + ); + } + let runtime = crate::Runtime::with_fs_store(options).unwrap(); + assert_eq!(runtime.list_workers().unwrap().len(), 1); + assert_eq!(runtime.summary().unwrap().active_worker_count, 0); +} diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 695a03379..87c0f676c 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -1682,6 +1682,15 @@ fn bind_workspace_memory_settings( Ok(()) } +/// Read-only validation shared with the offline, feature-gated fixture installer. +#[cfg(feature = "test-support")] +pub(crate) fn validate_fixture_restore_metadata( + metadata: &session_store::WorkerMetadata, + request: &CreateWorkerRequest, +) -> Result<(), String> { + ProfileRuntimeWorkerFactory::manifest_for_restore(metadata, request).map(|_| ()) +} + fn validate_worker_memory_settings( manifest: &manifest::WorkerManifest, request: &CreateWorkerRequest, diff --git a/crates/workspace-server/Cargo.toml b/crates/workspace-server/Cargo.toml index b561ca4eb..0879f2b07 100644 --- a/crates/workspace-server/Cargo.toml +++ b/crates/workspace-server/Cargo.toml @@ -10,6 +10,10 @@ autobins = false name = "yoi-server" path = "src/main.rs" +[features] +# Offline fixture preparation only; build product binaries without this feature. +test-support = [] + [dependencies] api-macros = { workspace = true, features = ["axum"] } async-trait.workspace = true diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index dd117ae7b..c4a95e8c0 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -28,6 +28,8 @@ pub mod runtime_subscription; pub mod server; pub mod skills; pub mod store; +#[cfg(feature = "test-support")] +pub use store::fixture_support as test_support; pub mod subjektiv; #[cfg(test)] mod ticket_browse_performance_tests; diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index d742f0ec6..f12c54334 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -2589,6 +2589,10 @@ pub struct SqliteWorkspaceStore { db_executor: crate::db_executor::DbExecutor, } +#[cfg(feature = "test-support")] +#[path = "test_support.rs"] +pub mod fixture_support; + impl SqliteWorkspaceStore { pub fn open(path: impl AsRef) -> Result { let conn = Connection::open(path)?; @@ -2900,6 +2904,33 @@ impl SqliteWorkspaceStore { request_fingerprint: &str, singleton_key: Option<&str>, current_memory_settings: &WorkspaceMemorySettingsRecord, + ) -> Result { + self.reserve_worker_create_with( + workspace_id, + runtime_id, + allocation_key, + request_fingerprint, + singleton_key, + current_memory_settings, + |_| Ok(None), + |_, _| Ok(()), + ) + } + + // Keep reservation validation, Memory binding and writes in one transaction. The + // nondefault test-support module supplies fixed identity / offline catalog hooks; + // ordinary creation always supplies a newly generated ID and no completion hook. + #[allow(clippy::too_many_arguments)] + fn reserve_worker_create_with( + &self, + workspace_id: &str, + runtime_id: &str, + allocation_key: &str, + request_fingerprint: &str, + singleton_key: Option<&str>, + current_memory_settings: &WorkspaceMemorySettingsRecord, + prepare: impl FnOnce(&rusqlite::Transaction<'_>) -> Result>, + complete: impl FnOnce(&rusqlite::Transaction<'_>, &WorkerCreateReservation) -> Result<()>, ) -> Result { if allocation_key.trim().is_empty() || request_fingerprint.trim().is_empty() { return Err(Error::InvalidInput( @@ -2912,6 +2943,7 @@ impl SqliteWorkspaceStore { validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?; self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let fixed_worker_id = prepare(&tx)?; let existing = tx .query_row( "SELECT worker_id, runtime_id, request_fingerprint, create_fingerprint, \ @@ -3094,7 +3126,7 @@ impl SqliteWorkspaceStore { validate_workspace_memory_settings_snapshot(&snapshot, workspace_id)?; let create_fingerprint = bound_worker_create_fingerprint(request_fingerprint, &snapshot); - let worker_id = WorkerId::now_v7(); + let worker_id = fixed_worker_id.unwrap_or_else(WorkerId::now_v7); let worker_id_text = worker_id.to_string(); let now = chrono::Utc::now().to_rfc3339(); let singleton_generation = match singleton_owner.as_ref() { @@ -3183,13 +3215,15 @@ impl SqliteWorkspaceStore { )); } } - tx.commit()?; - Ok(WorkerCreateReservation { + let reservation = WorkerCreateReservation { worker_id, create_fingerprint, memory_settings: snapshot, singleton, - }) + }; + complete(&tx, &reservation)?; + tx.commit()?; + Ok(reservation) }) } diff --git a/crates/workspace-server/src/test_support.rs b/crates/workspace-server/src/test_support.rs new file mode 100644 index 000000000..fcdf63087 --- /dev/null +++ b/crates/workspace-server/src/test_support.rs @@ -0,0 +1,688 @@ +//! Offline Backend fixture preparation, available only with `test-support`. +//! +//! Use a new isolated database and the existing account, Workspace signing-identity / +//! `ControlPlaneStore::create_workspace_bootstrap`, and Runtime binding APIs first. +//! Generate fresh account credentials and signing keys for each environment; no fixture +//! credentials are accepted here. Install Profile configuration through the existing +//! config-source mechanisms before starting the product processes. +//! +//! This module seeds Backend ownership only. It neither writes Runtime storage nor +//! launches a Worker, fabricates Runtime observations, changes authorization, or sends +//! initial input. The caller must separately install the matching stopped Runtime +//! fixture. Build product binaries in a separate Cargo invocation **without** this +//! feature (Cargo features unify within an invocation). + +use super::*; +use crate::hosts::{WorkerSpawnIntent, WorkerSpawnRequest, worker_spawn_create_fingerprint}; +use worker_runtime::catalog::ProfileSelector; + +/// One ordinary, attachment-free Worker from an externally validated fixture. +/// +/// IDs must match the Runtime fixture. `request` is the unresolved canonical create +/// intent, not a Runtime launch request containing credentials or execution grants. +/// The Profile selector must already be installed by the harness. `initial_submit` +/// must be empty: saved Session contents belong to the Runtime fixture, not new input. +/// Singleton/service, Ticket-assigned and attachment-bearing Workers are deliberately +/// unsupported. +pub struct FixedWorkerSeed<'a> { + pub workspace_id: &'a str, + pub runtime_id: &'a str, + pub worker_id: WorkerId, + pub allocation_key: &'a str, + pub request: &'a WorkerSpawnRequest, +} + +/// Insert a fixed Worker catalog entry and completed creation reservation atomically. +/// +/// Uses the normal reservation validators, authoritative Memory snapshot, fingerprint +/// binding, resource-key allocator and immediate transaction. Even an exact repeat is +/// rejected: this is not an upsert/import/recovery API. Existing allocations, Worker +/// identities and residual lifecycle/attachment records are never adopted or repaired. +/// Returns the creation binding needed to check the corresponding Runtime fixture. +pub fn seed_fixed_worker( + store: &SqliteWorkspaceStore, + seed: FixedWorkerSeed<'_>, +) -> Result { + validate_identifier("workspace_id", seed.workspace_id)?; + validate_identifier("runtime_id", seed.runtime_id)?; + let request = seed.request; + if !request.initial_submit.is_empty() { + return Err(Error::InvalidInput( + "fixture Worker cannot contain initial_submit; offline seeding accepts no new input" + .into(), + )); + } + if !matches!(request.intent, WorkerSpawnIntent::WorkspaceCoding) + || request.singleton_key.is_some() + || request.ticket_assignment.is_some() + || request.resolved_subjektiv_attached + || !request.workdir_attachment_requests.is_empty() + || !request.resolved_workdir_attachment_requests.is_empty() + || !request.resolved_workdir_attachments.is_empty() + || request.resolved_config_bundle.is_some() + || request.resolved_workspace_api.is_some() + || request.resolved_memory_settings.is_some() + || request.resolved_worker_observation_enabled + || !request.resolved_worker_observation_grants.is_empty() + || request.resolved_control_operation.is_some() + || request.resolved_execution_grant.is_some() + { + return Err(Error::InvalidInput( + "fixture Worker must be ordinary unresolved create intent without singleton, assignments, attachments or grants".into(), + )); + } + // Builtin selectors already carry their canonical `builtin:` prefix. + let profile = match &request.profile { + ProfileSelector::Builtin(name) | ProfileSelector::Named(name) => name, + }; + validate_non_empty("fixture Profile selector", profile)?; + let worker_id = seed.worker_id.to_string(); + let display_name = request + .requested_worker_name + .as_deref() + .unwrap_or(&worker_id); + validate_non_empty("fixture Worker display name", display_name)?; + let fingerprint = worker_spawn_create_fingerprint(request).map_err(Error::InvalidInput)?; + let memory = store.get_workspace_memory_settings(seed.workspace_id)?; + store.reserve_worker_create_with( + seed.workspace_id, + seed.runtime_id, + seed.allocation_key, + &fingerprint, + None, + &memory, + |tx| { + require_seed_target(tx, &seed)?; + Ok(Some(seed.worker_id)) + }, + |tx, reservation| { + let now = chrono::Utc::now().to_rfc3339(); + tx.execute( + "INSERT INTO worker_registry ( + workspace_id, runtime_id, worker_id, display_name, profile, + retention_state, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, 'normal', ?6, ?6)", + params![seed.workspace_id, seed.runtime_id, worker_id, display_name, profile, now], + )?; + let changed = tx.execute( + "UPDATE worker_create_reservations SET state = 'created', updated_at = ?4 + WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 AND state = 'reserved'", + params![seed.workspace_id, seed.runtime_id, reservation.worker_id.to_string(), now], + )?; + if changed != 1 { + return Err(Error::Store("fixture creation reservation disappeared".into())); + } + Ok(()) + }, + ) +} + +fn require_seed_target(tx: &rusqlite::Transaction<'_>, seed: &FixedWorkerSeed<'_>) -> Result<()> { + let active: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM workspaces w + JOIN workspace_runtime_bindings r ON r.workspace_id = w.workspace_id + JOIN workspace_signing_identities s ON s.workspace_id = w.workspace_id + WHERE w.workspace_id = ?1 AND w.state = 'active' + AND r.runtime_id = ?2 AND r.revoked_at IS NULL + AND r.authentication_mode = 'workspace_identity' AND s.state = 'active')", + params![seed.workspace_id, seed.runtime_id], + |row| row.get(0), + )?; + if !active { + return Err(Error::InvalidInput( + "fixture requires an active bootstrapped Workspace and authenticated Runtime binding" + .into(), + )); + } + let worker_id = seed.worker_id.to_string(); + let allocation_exists: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM worker_create_reservations + WHERE workspace_id = ?1 AND allocation_key = ?2)", + params![seed.workspace_id, seed.allocation_key], + |row| row.get(0), + )?; + if allocation_exists { + return Err(Error::InvalidInput( + "fixture Worker allocation already exists".into(), + )); + } + // Check by WorkerId, not just the proposed Runtime tuple: orphaned and terminal + // ownership is still a collision. These tables are owned by this store schema. + for table in [ + "worker_create_reservations", + "worker_registry", + "worker_registry_observations", + "worker_registry_projection_removals", + "worker_removal_operations", + "worker_singleton_owners", + "worker_workdir_links", + "worker_restore_intents", + "worker_tombstones", + "worker_orphan_diagnostics", + "worker_registry_projection_diagnostics", + "worker_session_archives", + "worker_diagnostics_archives", + "ticket_assignment_worker_tombstones", + ] { + let exists: bool = tx.query_row( + &format!("SELECT EXISTS(SELECT 1 FROM {table} WHERE worker_id = ?1)"), + [&worker_id], + |row| row.get(0), + )?; + if exists { + return Err(Error::InvalidInput(format!( + "fixture Worker identity already exists in {table}" + ))); + } + } + // A resource key without a catalog/reservation must not be silently reused. + let mut statement = tx.prepare( + "SELECT resource_id FROM workspace_resource_keys WHERE resource_kind = 'worker'", + )?; + let resources = statement.query_map([], |row| row.get::<_, String>(0))?; + for resource in resources { + let resource = resource?; + let existing = resource + .is_ascii() + .then(|| parse_worker_resource_id(&resource)) + .flatten() + .ok_or_else(|| { + Error::InvalidInput("unknown fixture Worker resource identity collision".into()) + })?; + if existing.worker_id == worker_id { + return Err(Error::InvalidInput( + "fixture Worker resource identity already exists".into(), + )); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::hosts::{ + WorkerSpawnAcceptanceRequirement, WorkerSpawnWorkingDirectoryAttachmentRequest, + WorkerSpawnWorkingDirectoryRequest, + }; + use crate::workspace_signing_identity::{ + InMemoryWorkspaceSigningMaterialStore, WorkspaceSigningIdentityService, + }; + + const WORKSPACE: &str = "0197a949-4b6b-7f2a-9d9a-1f87e3a4c501"; + const RUNTIME: &str = "fixture-runtime"; + const WORKER: &str = "0197a949-4b6b-7f2a-9d9a-1f87e3a4c502"; + + // Exercises the same public bootstrap APIs the offline harness uses. All account + // and key identities are newly generated, even across stores with fixed fixture IDs. + fn bootstrap() -> SqliteWorkspaceStore { + let store = SqliteWorkspaceStore::in_memory_for_test().unwrap(); + let account = AccountRecord { + account_id: format!("fixture-account-{}", Uuid::now_v7()), + kind: "user".into(), + handle: format!("fixture-{}", Uuid::now_v7()), + display_name: "Fixture owner".into(), + created_at: "2026-10-24T00:00:00Z".into(), + updated_at: "2026-10-24T00:00:00Z".into(), + }; + store.upsert_account(&account).unwrap(); + let identities = WorkspaceSigningIdentityService::new( + Arc::new(store.clone()), + Arc::new(InMemoryWorkspaceSigningMaterialStore::default()), + ); + let (identity, operation) = identities + .prepare_workspace_creation( + "fixture-bootstrap", + "fixture-bootstrap", + WORKSPACE, + &account.account_id, + ) + .unwrap(); + store + .create_workspace_bootstrap( + &WorkspaceBootstrapRecord { + operation_key: "fixture-bootstrap".into(), + request_fingerprint: "fixture-bootstrap".into(), + workspace: WorkspaceRecord { + workspace_id: WORKSPACE.into(), + owner_account_id: account.account_id, + display_name: "Fixture".into(), + state: "active".into(), + created_at: account.created_at, + updated_at: account.updated_at, + }, + repository: None, + }, + &identity, + &operation, + ) + .unwrap(); + let runtime_key = worker_runtime::auth::RuntimeIdentityMaterial::generate(RUNTIME).unwrap(); + store + .upsert_workspace_runtime_binding( + WorkspaceRuntimeBinding { + workspace_id: WORKSPACE.into(), + runtime_id: RUNTIME.into(), + display_name: "Fixture Runtime".into(), + base_url: "http://127.0.0.1:12345".into(), + public_key: runtime_key.public_key, + public_key_fingerprint: String::new(), + binding_id: String::new(), + authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity, + created_at: "2026-10-24T00:00:00Z".into(), + updated_at: "2026-10-24T00:00:00Z".into(), + revoked_at: None, + }, + false, + ) + .unwrap(); + store + } + + fn request() -> WorkerSpawnRequest { + WorkerSpawnRequest { + intent: WorkerSpawnIntent::WorkspaceCoding, + requested_worker_name: Some("Fixture Worker".into()), + singleton_key: None, + acceptance: WorkerSpawnAcceptanceRequirement::SocketReady, + profile: ProfileSelector::Builtin("builtin:coder".into()), + ticket_assignment: None, + initial_submit: Vec::new(), + workdir_attachment_requests: Vec::new(), + resolved_workdir_attachment_requests: Vec::new(), + resolved_workdir_attachments: Vec::new(), + resolved_config_bundle: None, + resolved_workspace_api: None, + resolved_memory_settings: None, + resolved_subjektiv_attached: false, + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: Vec::new(), + resolved_control_operation: None, + resolved_execution_grant: None, + } + } + + fn seed(request: &WorkerSpawnRequest) -> FixedWorkerSeed<'_> { + FixedWorkerSeed { + workspace_id: WORKSPACE, + runtime_id: RUNTIME, + worker_id: WORKER.parse().unwrap(), + allocation_key: "fixture-allocation", + request, + } + } + + fn worker() -> RuntimeWorkerRef { + RuntimeWorkerRef::new(RUNTIME, WORKER) + } + + #[test] + fn fixed_ids_are_isolated_and_seed_has_no_runtime_observation() { + let a = bootstrap(); + let b = bootstrap(); + let request = request(); + let first = seed_fixed_worker(&a, seed(&request)).unwrap(); + let second = seed_fixed_worker(&b, seed(&request)).unwrap(); + assert_eq!(first, second); + assert_eq!(first.worker_id.to_string(), WORKER); + assert_eq!(first.memory_settings.workspace_id, WORKSPACE); + assert_eq!(first.memory_settings.language, "English"); + assert!(first.singleton.is_none()); + assert_ne!( + a.get_workspace_signing_identity(WORKSPACE) + .unwrap() + .unwrap() + .public_key, + b.get_workspace_signing_identity(WORKSPACE) + .unwrap() + .unwrap() + .public_key + ); + assert_ne!( + a.get_workspace_runtime_binding(WORKSPACE, RUNTIME) + .unwrap() + .unwrap() + .public_key, + b.get_workspace_runtime_binding(WORKSPACE, RUNTIME) + .unwrap() + .unwrap() + .public_key + ); + for store in [&a, &b] { + assert_eq!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .as_deref(), + Some("W-1") + ); + assert_eq!( + store + .resolve_worker_resource_reference(WORKSPACE, "W-1") + .unwrap(), + Some(worker()) + ); + assert!( + !store + .has_active_worker_create_reservation(WORKSPACE, &worker()) + .unwrap() + ); + let registry = store + .get_worker_registry(WORKSPACE, &worker()) + .unwrap() + .unwrap(); + assert_eq!(registry.profile.as_deref(), Some("builtin:coder")); + assert_eq!(registry.display_name, "Fixture Worker"); + assert_eq!(registry.retention_state, "normal"); + let projection = store + .with_conn(|conn| read_worker_registry_projection(conn, WORKSPACE, &worker())) + .unwrap() + .unwrap(); + assert!(projection.observation.is_none()); + assert!(projection.job.is_none()); + } + } + + #[test] + fn initial_submit_is_refused_without_registration_or_allocation() { + let store = bootstrap(); + let mut request = request(); + // Even an empty text segment is new input; the vector itself must be empty. + for text in ["new input must not be seeded", ""] { + request.initial_submit = vec![protocol::Segment::text(text)]; + let error = seed_fixed_worker(&store, seed(&request)).unwrap_err(); + assert!( + matches!(error, Error::InvalidInput(ref message) if message.contains("initial_submit")) + ); + assert!( + store + .get_worker_registry(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + assert!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + assert!( + !store + .has_active_worker_create_reservation(WORKSPACE, &worker()) + .unwrap() + ); + } + // The rejected attempts must not consume the allocation or resource counter. + request.initial_submit.clear(); + seed_fixed_worker(&store, seed(&request)).unwrap(); + assert_eq!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .as_deref(), + Some("W-1") + ); + } + + #[test] + fn exact_replay_changed_input_and_cross_runtime_identity_are_refused() { + let store = bootstrap(); + let mut request = request(); + seed_fixed_worker(&store, seed(&request)).unwrap(); + let before = store.get_worker_registry(WORKSPACE, &worker()).unwrap(); + assert!(seed_fixed_worker(&store, seed(&request)).is_err()); + request.requested_worker_name = Some("Must not overwrite".into()); + assert!(seed_fixed_worker(&store, seed(&request)).is_err()); + let mut new_allocation = seed(&request); + new_allocation.allocation_key = "another-allocation"; + assert!(seed_fixed_worker(&store, new_allocation).is_err()); + let mut same_allocation = seed(&request); + same_allocation.worker_id = WorkerId::now_v7(); + assert!(seed_fixed_worker(&store, same_allocation).is_err()); + assert_eq!( + store.get_worker_registry(WORKSPACE, &worker()).unwrap(), + before + ); + + let other = bootstrap(); + let mut record = before.unwrap(); + record.worker.runtime_id = "different-runtime".into(); + other.upsert_worker_registry(&record).unwrap(); + assert!(seed_fixed_worker(&other, seed(&request)).is_err()); + assert!( + other + .get_worker_registry(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + } + + #[test] + fn reserved_and_orphaned_resource_identities_are_not_adopted() { + let store = bootstrap(); + let request = request(); + let memory = store.get_workspace_memory_settings(WORKSPACE).unwrap(); + let ordinary = store + .reserve_worker_create(WORKSPACE, RUNTIME, "ordinary", "fingerprint", None, &memory) + .unwrap(); + let mut collision = seed(&request); + collision.worker_id = ordinary.worker_id; + assert!(seed_fixed_worker(&store, collision).is_err()); + // Store-level fault setup only: the external harness never writes SQL. + store + .with_conn(|conn| { + allocate_resource_key( + conn, + WORKSPACE, + WorkspaceResourceKind::Worker, + &worker_resource_id(&worker()), + "now", + )?; + Ok(()) + }) + .unwrap(); + assert!(seed_fixed_worker(&store, seed(&request)).is_err()); + assert!( + store + .get_worker_registry(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + } + + #[test] + fn unknown_resource_identity_and_retained_removal_fence_are_refused() { + let request = request(); + let malformed = bootstrap(); + malformed + .with_conn(|conn| { + allocate_resource_key( + conn, + WORKSPACE, + WorkspaceResourceKind::Worker, + "unknown-identity", + "now", + )?; + Ok(()) + }) + .unwrap(); + assert!(matches!( + seed_fixed_worker(&malformed, seed(&request)), + Err(Error::InvalidInput(_)) + )); + assert!( + malformed + .get_worker_registry(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + + let removed = bootstrap(); + removed.with_conn(|conn| { + conn.execute( + "INSERT INTO worker_registry_projection_removals(workspace_id, runtime_id, worker_id) VALUES (?1, ?2, ?3)", + params![WORKSPACE, RUNTIME, WORKER], + )?; + Ok(()) + }).unwrap(); + assert!(matches!( + seed_fixed_worker(&removed, seed(&request)), + Err(Error::InvalidInput(_)) + )); + assert!( + removed + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + } + + #[test] + fn unknown_workspace_runtime_and_revoked_binding_are_refused() { + let store = bootstrap(); + let request = request(); + let mut unknown = seed(&request); + unknown.workspace_id = "unknown"; + assert!(seed_fixed_worker(&store, unknown).is_err()); + let mut unknown = seed(&request); + unknown.runtime_id = "unknown"; + assert!(seed_fixed_worker(&store, unknown).is_err()); + store + .revoke_workspace_runtime_binding(WORKSPACE, RUNTIME, "2026-10-24T01:00:00Z") + .unwrap(); + assert!(seed_fixed_worker(&store, seed(&request)).is_err()); + assert!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + } + + #[test] + fn unsupported_intent_is_refused_without_partial_registration() { + let store = bootstrap(); + let base = request(); + let mut requests = Vec::new(); + let mut request = base.clone(); + request.singleton_key = Some("fixture-singleton".into()); + requests.push(request); + let mut request = base.clone(); + request.intent = WorkerSpawnIntent::WorkspaceOrchestrator; + requests.push(request); + let mut request = base.clone(); + request.resolved_subjektiv_attached = true; + requests.push(request); + let mut request = base.clone(); + request + .workdir_attachment_requests + .push(WorkerSpawnWorkingDirectoryAttachmentRequest { + alias: workdir::WorkdirAttachmentAlias::new("repo").unwrap(), + working_directory: WorkerSpawnWorkingDirectoryRequest { + repository_key: "repo".into(), + selector: None, + }, + }); + requests.push(request); + for request in requests { + assert!(seed_fixed_worker(&store, seed(&request)).is_err()); + } + assert!( + store + .get_worker_registry(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + assert!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + assert_eq!( + seed_fixed_worker(&store, seed(&base)) + .unwrap() + .worker_id + .to_string(), + WORKER + ); + } + + #[test] + fn late_catalog_failure_rolls_back_reservation_and_resource_allocation() { + let store = bootstrap(); + let request = request(); + store + .with_conn(|conn| { + conn.execute_batch( + "CREATE TEMP TRIGGER fail_fixture_catalog BEFORE INSERT ON worker_registry + BEGIN SELECT RAISE(ABORT, 'injected catalog failure'); END", + )?; + Ok(()) + }) + .unwrap(); + assert!(seed_fixed_worker(&store, seed(&request)).is_err()); + assert!( + !store + .has_active_worker_create_reservation(WORKSPACE, &worker()) + .unwrap() + ); + assert!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .is_none() + ); + store + .with_conn(|conn| { + conn.execute_batch("DROP TRIGGER fail_fixture_catalog")?; + Ok(()) + }) + .unwrap(); + seed_fixed_worker(&store, seed(&request)).unwrap(); + assert_eq!( + store + .worker_resource_key(WORKSPACE, &worker()) + .unwrap() + .as_deref(), + Some("W-1") + ); + } + + #[test] + fn ordinary_creation_still_allocates_and_replays_without_catalog_insertion() { + let store = bootstrap(); + let memory = store.get_workspace_memory_settings(WORKSPACE).unwrap(); + let reserve = |key| { + store + .reserve_worker_create(WORKSPACE, RUNTIME, key, "fingerprint", None, &memory) + .unwrap() + }; + let first = reserve("ordinary-one"); + assert_eq!(first, reserve("ordinary-one")); + let second = reserve("ordinary-two"); + assert_ne!(first.worker_id, second.worker_id); + assert_ne!(first.worker_id.to_string(), WORKER); + assert_eq!( + first.worker_id.as_uuid().get_version(), + Some(uuid::Version::SortRand) + ); + let worker = RuntimeWorkerRef::new(RUNTIME, first.worker_id.to_string()); + assert!( + store + .has_active_worker_create_reservation(WORKSPACE, &worker) + .unwrap() + ); + assert!( + store + .get_worker_registry(WORKSPACE, &worker) + .unwrap() + .is_none() + ); + assert!( + store + .reserve_worker_create(WORKSPACE, RUNTIME, "ordinary-one", "changed", None, &memory) + .is_err() + ); + } +} diff --git a/tests/e2e/Cargo.toml b/tests/e2e/Cargo.toml index b308eeb49..53b60d5e3 100644 --- a/tests/e2e/Cargo.toml +++ b/tests/e2e/Cargo.toml @@ -9,6 +9,7 @@ autotests = false [features] default = [] e2e = [] +fixture-support = ["yoi-workspace-server/test-support", "worker-runtime/test-support"] [dependencies] libc.workspace = true @@ -17,11 +18,17 @@ serde = { workspace = true, features = ["derive"] } serde_json.workspace = true sha2.workspace = true tempfile.workspace = true +protocol.workspace = true +tungstenite = { version = "0.29", default-features = false, features = ["handshake"] } [dev-dependencies] chrono = { version = "0.4", default-features = false, features = ["clock"] } yoi-workspace-server.workspace = true server-api.workspace = true +worker-runtime.workspace = true +session-store.workspace = true +config-source.workspace = true +manifest.workspace = true webauthn-rs-proto = "0.5.5" webauthn-authenticator-rs = { version = "0.5.5", default-features = false, features = ["softpasskey"] } @@ -44,3 +51,8 @@ required-features = ["e2e"] name = "worker_environment" path = "tests/worker_environment.rs" required-features = ["e2e"] + +[[test]] +name = "worker_restore" +path = "tests/worker_restore.rs" +required-features = ["e2e", "fixture-support"] diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 678300006..5766c9471 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -7,3 +7,41 @@ - [`PanelHarness`]: PTY経由の入力、観測イベントによる同期、失敗時のartifact保存。 - [`FixtureWorkspace`]: TUI用WorkspaceとHOME／XDG領域の隔離。 - [`WorkerE2eEnvironment`]: Backend／Runtime用プロセスとfixtureの管理。 +- [`FixtureSource`]: 提供資産の全ファイルSHA-256記録、独立copy、原本不変の検証。 +- [`WorkerSessionSubscription`]: 公開Worker WebSocketの初期full Snapshot照合と停止確認。モデル呼出しは行わない。 + +## 固定Workerの段階起動 + +1. 提供fixtureを所有crateの型で検証し、`FixtureSource`で原本hashを記録する。各環境の一時領域へcopyする。fixtureが未提供・不適合なら失敗とし、代替Workerを生成しない。 +2. `initialize_runtime_identity`で固定Runtime IDに新しい鍵を発行する。Backendの型付きstore APIで新しいAccount認証・Workspace署名identity・Runtime binding・固定Worker登録を準備する。 +3. `start_seeded_backend`へ固定Workspace/Runtime IDとその環境の新しいbearerを渡す。Server起動後の公開APIからWorkspaceの公開identityを取得する。 +4. `start_runtime_with_seed`はtrust設定後、Runtime processの起動前にseed closureを呼ぶ。ここで実Server URLを使いRuntime所有crateの型付きinstallを呼ぶ。Runtime store rootは`runtime_data_dir.join("runtime")`。 +5. Worker一覧とdetailから停止状態・観測tokenを確認し、一つの固定request IDで`restore_worker`を一度だけ呼ぶ。結果不明・拒否は成功扱いせず、tokenを自動更新しない。 +6. `WorkerSessionSubscription::connect`で期待`protocol::SessionSnapshot`全体（entry ID、timestamp、順序、本文を含む）を照合する。二環境を同時に保持し、一方のstop/cleanup後も他方の購読・一覧・内容を再確認する。 +7. `stop_worker`、購読の`expect_stopped`、環境cleanup、原本`assert_unchanged`を確認する。 + +所有crateの`test-support`依存はテスト側の非default `fixture-support` featureだけで有効にする。製品binary providerは別Cargo invocationと専用`target/e2e-products`へ`yoi`・`yoi-server`・`yoi-runtime`を追加featureなしでビルドする。support test自身をこのtargetへビルドしてはならない。`YOI_E2E_BIN`等の明示binary overrideを使用する場合も、呼出し側が同じ分離を保証する。既存TUI用`e2e-test` featureは別の`target/e2e-tui-products`へビルドし、restore製品binaryへ混入しない。 + +固定資産の正常restore・並行隔離E2E: + +```sh +cargo test -p yoi-e2e --features e2e,fixture-support --test worker_restore +# 親から提供された候補の検証（読取専用、原本の自動採用や生成はしない） +YOI_E2E_RESTORE_FIXTURE=target/fixture-candidate-v1 \ + cargo test -p yoi-e2e --features e2e,fixture-support --test worker_restore +``` + +標準資産は`tests/e2e/fixtures/worker-codex-v1`の`manifest.json`・`worker-template.json`・`expected.json`・`workspace-config.dcdl`。manifestの各asset hash/size、schemaと固定ID、生成元commit、modelを検証する。Workspace configのimport参照を黙って最新へ置換せず、Gitで生成元commitに対する`resources`と`crates/builtin-source`の一致を確認する（Git履歴がない場合も失敗）。移植したSessionの正規公開projectionを供給された会話行と照合し、その全snapshotを実executionのWebSocketと比較する。欠落fixture・hash不一致・未実行は成功扱いしない。 + +通常のharness回帰試験（製品processは起動しない）: + +```sh +cargo test -p yoi-e2e --lib --no-default-features +``` + +既存の製品process E2Eは別結果として実行・報告する: + +```sh +cargo test -p yoi-e2e --features e2e --test worker_environment +cargo test -p yoi-e2e --features e2e --test panel --test rewind +``` diff --git a/tests/e2e/fixtures/worker-codex-v1/expected.json b/tests/e2e/fixtures/worker-codex-v1/expected.json new file mode 100644 index 000000000..485398932 --- /dev/null +++ b/tests/e2e/fixtures/worker-codex-v1/expected.json @@ -0,0 +1,52 @@ +{ + "source_revision": "1639be98411921308238b3a6a6e75c784c045156", + "model": "codex-oauth/gpt-6.1-sol", + "worker_id": "01a1657d-b762-7791-87ae-a49f28cadbce", + "session_id": "01a1657d-b818-77c1-bd73-175a27562f63", + "rows": [ + { + "entry_id": "01a1657f-18bf-7a03-b8b8-953d27041fe0", + "timestamp": 1792704190655, + "role": "user", + "parts": [ + { + "kind": "text", + "content": "これは保存と復元を確認するための人工的な会話です。合言葉は「青い灯台」です。覚えたことを一文で返してください。Toolは使わないでください。" + } + ] + }, + { + "entry_id": "01a1657f-1f55-71d3-b418-8199b863f59a", + "timestamp": 1792704192341, + "role": "assistant", + "parts": [ + { + "kind": "text", + "text": "合言葉は「青い灯台」と覚えました。" + } + ] + }, + { + "entry_id": "01a1657f-3053-71f3-bf4f-91290d82488b", + "timestamp": 1792704196691, + "role": "user", + "parts": [ + { + "kind": "text", + "content": "先ほどの合言葉は何でしたか？合言葉だけを答えてください。" + } + ] + }, + { + "entry_id": "01a1657f-36ef-78b1-b0c8-b141f13b0c7b", + "timestamp": 1792704198383, + "role": "assistant", + "parts": [ + { + "kind": "text", + "text": "青い灯台" + } + ] + } + ] +} diff --git a/tests/e2e/fixtures/worker-codex-v1/manifest.json b/tests/e2e/fixtures/worker-codex-v1/manifest.json new file mode 100644 index 000000000..6c26e18ef --- /dev/null +++ b/tests/e2e/fixtures/worker-codex-v1/manifest.json @@ -0,0 +1,34 @@ +{ + "format_version": 1, + "source_revision": "1639be98411921308238b3a6a6e75c784c045156", + "runtime_schema_version": 12, + "session_schema_version": 6, + "workspace_id": "01a16553-b901-7191-8ceb-8e2306fe3da9", + "runtime_id": "fixture-951707ee-93ca-4205-9c12-45f08a104a74", + "worker_id": "01a1657d-b762-7791-87ae-a49f28cadbce", + "session_id": "01a1657d-b818-77c1-bd73-175a27562f63", + "segment_id": "01a1657d-b818-77c1-bd73-176172a4bd82", + "model": "codex-oauth/gpt-6.1-sol", + "state": "stopped", + "acquisition": "Real isolated Codex OAuth two-turn artificial conversation; typed offline StoppedWorkerTemplate extraction. Session bytes unchanged; environment credentials and execution receipts excluded.", + "source_file_sha256": { + "worker.json": "912e620013a17f05c1aaf900da48c9ee0ecb1f54121efc3721e03f8755eb5eca", + "metadata.json": "657928a89ec621bd092b86924201ed7b562e9683261789ce18bdb1d248bdc450", + "session/session.json": "22b01daab68a28b9777a1f7abb9ef9f7f6cbae113089446602f5d15421e7c7f3", + "session/segments/01a1657d-b818-77c1-bd73-176172a4bd82.jsonl": "7c5663bb06d9406d5bd0a5086159b31c0d7799458fcb1c6f81e18b63c49f4df5" + }, + "files": { + "worker-template.json": { + "bytes": 48343, + "sha256": "8bf6ce3c71c4f05c82314de93c1e8e51b7502b00915b6adb8b33046944e463b2" + }, + "expected.json": { + "bytes": 1474, + "sha256": "b9846db47ecb8f3f53b2abd9d5f0c766122f86015a7a7df7addf8beab3e9c55a" + }, + "workspace-config.dcdl": { + "bytes": 1141, + "sha256": "9b07507b71733081f077e2e9968a7f1f676c916a243bb355d1813f067a55c415" + } + } +} diff --git a/tests/e2e/fixtures/worker-codex-v1/worker-template.json b/tests/e2e/fixtures/worker-codex-v1/worker-template.json new file mode 100644 index 000000000..dfd032e64 --- /dev/null +++ b/tests/e2e/fixtures/worker-codex-v1/worker-template.json @@ -0,0 +1,5605 @@ +{ + "template_version": 1, + "runtime_schema_version": 12, + "runtime_id": "fixture-951707ee-93ca-4205-9c12-45f08a104a74", + "workspace_id": "01a16553-b901-7191-8ceb-8e2306fe3da9", + "worker_id": "01a1657d-b762-7791-87ae-a49f28cadbce", + "created_at_ms": 1792704100293, + "session_id": "01a1657d-b818-77c1-bd73-175a27562f63", + "segment_id": "01a1657d-b818-77c1-bd73-176172a4bd82", + "session_manifest": [ + 123, + 10, + 32, + 32, + 34, + 115, + 99, + 104, + 101, + 109, + 97, + 95, + 118, + 101, + 114, + 115, + 105, + 111, + 110, + 34, + 58, + 32, + 54, + 44, + 10, + 32, + 32, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 32, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 100, + 45, + 98, + 56, + 49, + 56, + 45, + 55, + 55, + 99, + 49, + 45, + 98, + 100, + 55, + 51, + 45, + 49, + 55, + 53, + 97, + 50, + 55, + 53, + 54, + 50, + 102, + 54, + 51, + 34, + 10, + 125, + 10 + ], + "segment_jsonl": [ + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 115, + 101, + 103, + 109, + 101, + 110, + 116, + 95, + 115, + 116, + 97, + 114, + 116, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 48, + 48, + 52, + 48, + 55, + 44, + 34, + 115, + 101, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 100, + 45, + 98, + 56, + 49, + 56, + 45, + 55, + 55, + 99, + 49, + 45, + 98, + 100, + 55, + 51, + 45, + 49, + 55, + 53, + 97, + 50, + 55, + 53, + 54, + 50, + 102, + 54, + 51, + 34, + 44, + 34, + 115, + 121, + 115, + 116, + 101, + 109, + 95, + 112, + 114, + 111, + 109, + 112, + 116, + 34, + 58, + 34, + 195, + 163, + 194, + 129, + 194, + 147, + 195, + 163, + 194, + 130, + 194, + 140, + 195, + 163, + 194, + 129, + 194, + 175, + 195, + 169, + 194, + 154, + 194, + 148, + 195, + 169, + 194, + 155, + 194, + 162, + 195, + 167, + 194, + 146, + 194, + 176, + 195, + 165, + 194, + 162, + 194, + 131, + 195, + 163, + 194, + 129, + 194, + 174, + 195, + 164, + 194, + 186, + 194, + 186, + 195, + 165, + 194, + 183, + 194, + 165, + 195, + 167, + 194, + 154, + 194, + 132, + 195, + 163, + 194, + 129, + 194, + 170, + 195, + 164, + 194, + 188, + 194, + 154, + 195, + 168, + 194, + 169, + 194, + 177, + 195, + 163, + 194, + 131, + 194, + 134, + 195, + 163, + 194, + 130, + 194, + 185, + 195, + 163, + 194, + 131, + 194, + 136, + 195, + 163, + 194, + 129, + 194, + 167, + 195, + 163, + 194, + 129, + 194, + 153, + 195, + 163, + 194, + 128, + 194, + 130, + 195, + 166, + 194, + 151, + 194, + 165, + 195, + 166, + 194, + 156, + 194, + 172, + 195, + 168, + 194, + 170, + 194, + 158, + 195, + 163, + 194, + 129, + 194, + 167, + 195, + 167, + 194, + 176, + 194, + 161, + 195, + 166, + 194, + 189, + 194, + 148, + 195, + 163, + 194, + 129, + 194, + 171, + 195, + 165, + 194, + 191, + 194, + 156, + 195, + 167, + 194, + 173, + 194, + 148, + 195, + 163, + 194, + 129, + 194, + 151, + 195, + 163, + 194, + 129, + 194, + 166, + 195, + 163, + 194, + 129, + 194, + 143, + 195, + 163, + 194, + 129, + 194, + 160, + 195, + 163, + 194, + 129, + 194, + 149, + 195, + 163, + 194, + 129, + 194, + 132, + 195, + 163, + 194, + 128, + 194, + 130, + 84, + 111, + 111, + 108, + 195, + 163, + 194, + 130, + 194, + 132, + 195, + 165, + 194, + 164, + 194, + 150, + 195, + 169, + 194, + 131, + 194, + 168, + 195, + 166, + 194, + 147, + 194, + 141, + 195, + 164, + 194, + 189, + 194, + 156, + 195, + 163, + 194, + 129, + 194, + 175, + 195, + 168, + 194, + 161, + 194, + 140, + 195, + 163, + 194, + 130, + 194, + 143, + 195, + 163, + 194, + 129, + 194, + 154, + 195, + 163, + 194, + 128, + 194, + 129, + 195, + 163, + 194, + 131, + 194, + 166, + 195, + 163, + 194, + 131, + 194, + 188, + 195, + 163, + 194, + 130, + 194, + 182, + 195, + 163, + 194, + 131, + 194, + 188, + 195, + 163, + 194, + 129, + 194, + 140, + 195, + 164, + 194, + 188, + 194, + 154, + 195, + 168, + 194, + 169, + 194, + 177, + 195, + 164, + 194, + 184, + 194, + 173, + 195, + 163, + 194, + 129, + 194, + 171, + 195, + 166, + 194, + 143, + 194, + 144, + 195, + 167, + 194, + 164, + 194, + 186, + 195, + 163, + 194, + 129, + 194, + 151, + 195, + 163, + 194, + 129, + 194, + 159, + 195, + 166, + 194, + 131, + 194, + 133, + 195, + 165, + 194, + 160, + 194, + 177, + 195, + 163, + 194, + 129, + 194, + 160, + 195, + 163, + 194, + 129, + 194, + 145, + 195, + 163, + 194, + 130, + 194, + 146, + 195, + 164, + 194, + 189, + 194, + 191, + 195, + 163, + 194, + 129, + 194, + 163, + 195, + 163, + 194, + 129, + 194, + 166, + 195, + 163, + 194, + 129, + 194, + 143, + 195, + 163, + 194, + 129, + 194, + 160, + 195, + 163, + 194, + 129, + 194, + 149, + 195, + 163, + 194, + 129, + 194, + 132, + 195, + 163, + 194, + 128, + 194, + 130, + 92, + 110, + 92, + 110, + 92, + 110, + 45, + 45, + 45, + 92, + 110, + 35, + 35, + 32, + 87, + 111, + 114, + 107, + 105, + 110, + 103, + 32, + 98, + 111, + 117, + 110, + 100, + 97, + 114, + 105, + 101, + 115, + 34, + 44, + 34, + 99, + 111, + 110, + 102, + 105, + 103, + 34, + 58, + 123, + 34, + 109, + 97, + 120, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 50, + 48, + 52, + 56, + 44, + 34, + 116, + 101, + 109, + 112, + 101, + 114, + 97, + 116, + 117, + 114, + 101, + 34, + 58, + 110, + 117, + 108, + 108, + 44, + 34, + 116, + 111, + 112, + 95, + 112, + 34, + 58, + 110, + 117, + 108, + 108, + 44, + 34, + 116, + 111, + 112, + 95, + 107, + 34, + 58, + 110, + 117, + 108, + 108, + 44, + 34, + 115, + 116, + 111, + 112, + 95, + 115, + 101, + 113, + 117, + 101, + 110, + 99, + 101, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 114, + 101, + 97, + 115, + 111, + 110, + 105, + 110, + 103, + 34, + 58, + 34, + 108, + 111, + 119, + 34, + 125, + 44, + 34, + 104, + 105, + 115, + 116, + 111, + 114, + 121, + 34, + 58, + 91, + 93, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 105, + 110, + 118, + 111, + 107, + 101, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 48, + 54, + 53, + 53, + 44, + 34, + 116, + 114, + 105, + 103, + 103, + 101, + 114, + 34, + 58, + 34, + 117, + 115, + 101, + 114, + 95, + 115, + 101, + 110, + 100, + 34, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 117, + 115, + 101, + 114, + 95, + 105, + 110, + 112, + 117, + 116, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 48, + 54, + 53, + 53, + 44, + 34, + 115, + 101, + 103, + 109, + 101, + 110, + 116, + 115, + 34, + 58, + 91, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 101, + 120, + 116, + 34, + 44, + 34, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 34, + 58, + 34, + 227, + 129, + 147, + 227, + 130, + 140, + 227, + 129, + 175, + 228, + 191, + 157, + 229, + 173, + 152, + 227, + 129, + 168, + 229, + 190, + 169, + 229, + 133, + 131, + 227, + 130, + 146, + 231, + 162, + 186, + 232, + 170, + 141, + 227, + 129, + 153, + 227, + 130, + 139, + 227, + 129, + 159, + 227, + 130, + 129, + 227, + 129, + 174, + 228, + 186, + 186, + 229, + 183, + 165, + 231, + 154, + 132, + 227, + 129, + 170, + 228, + 188, + 154, + 232, + 169, + 177, + 227, + 129, + 167, + 227, + 129, + 153, + 227, + 128, + 130, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 175, + 227, + 128, + 140, + 233, + 157, + 146, + 227, + 129, + 132, + 231, + 129, + 175, + 229, + 143, + 176, + 227, + 128, + 141, + 227, + 129, + 167, + 227, + 129, + 153, + 227, + 128, + 130, + 232, + 166, + 154, + 227, + 129, + 136, + 227, + 129, + 159, + 227, + 129, + 147, + 227, + 129, + 168, + 227, + 130, + 146, + 228, + 184, + 128, + 230, + 150, + 135, + 227, + 129, + 167, + 232, + 191, + 148, + 227, + 129, + 151, + 227, + 129, + 166, + 227, + 129, + 143, + 227, + 129, + 160, + 227, + 129, + 149, + 227, + 129, + 132, + 227, + 128, + 130, + 84, + 111, + 111, + 108, + 227, + 129, + 175, + 228, + 189, + 191, + 227, + 130, + 143, + 227, + 129, + 170, + 227, + 129, + 132, + 227, + 129, + 167, + 227, + 129, + 143, + 227, + 129, + 160, + 227, + 129, + 149, + 227, + 129, + 132, + 227, + 128, + 130, + 34, + 125, + 93, + 44, + 34, + 101, + 120, + 116, + 101, + 110, + 115, + 105, + 111, + 110, + 115, + 34, + 58, + 91, + 123, + 34, + 100, + 111, + 109, + 97, + 105, + 110, + 34, + 58, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 46, + 112, + 101, + 110, + 100, + 105, + 110, + 103, + 95, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 111, + 110, + 115, + 46, + 118, + 49, + 34, + 44, + 34, + 112, + 97, + 121, + 108, + 111, + 97, + 100, + 34, + 58, + 123, + 34, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 110, + 103, + 34, + 58, + 110, + 117, + 108, + 108, + 44, + 34, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 110, + 103, + 95, + 110, + 111, + 116, + 105, + 102, + 105, + 99, + 97, + 116, + 105, + 111, + 110, + 34, + 58, + 91, + 93, + 44, + 34, + 105, + 110, + 118, + 111, + 99, + 97, + 116, + 105, + 111, + 110, + 95, + 114, + 101, + 99, + 101, + 105, + 112, + 116, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 105, + 110, + 118, + 111, + 99, + 97, + 116, + 105, + 111, + 110, + 95, + 114, + 101, + 99, + 111, + 118, + 101, + 114, + 121, + 95, + 98, + 108, + 111, + 99, + 107, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 110, + 101, + 120, + 116, + 95, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 111, + 110, + 95, + 115, + 101, + 113, + 117, + 101, + 110, + 99, + 101, + 34, + 58, + 49, + 44, + 34, + 110, + 111, + 116, + 105, + 102, + 105, + 99, + 97, + 116, + 105, + 111, + 110, + 95, + 114, + 101, + 99, + 101, + 105, + 112, + 116, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 112, + 101, + 110, + 100, + 105, + 110, + 103, + 34, + 58, + 91, + 93, + 44, + 34, + 112, + 101, + 110, + 100, + 105, + 110, + 103, + 95, + 110, + 111, + 116, + 105, + 102, + 105, + 99, + 97, + 116, + 105, + 111, + 110, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 114, + 101, + 99, + 101, + 105, + 112, + 116, + 115, + 34, + 58, + 91, + 123, + 34, + 100, + 105, + 115, + 112, + 111, + 115, + 105, + 116, + 105, + 111, + 110, + 34, + 58, + 34, + 115, + 116, + 97, + 114, + 116, + 101, + 100, + 34, + 44, + 34, + 112, + 97, + 121, + 108, + 111, + 97, + 100, + 95, + 100, + 105, + 103, + 101, + 115, + 116, + 34, + 58, + 34, + 100, + 53, + 51, + 102, + 53, + 53, + 52, + 52, + 52, + 56, + 98, + 99, + 50, + 55, + 55, + 53, + 50, + 97, + 99, + 49, + 49, + 98, + 56, + 54, + 57, + 50, + 48, + 51, + 53, + 48, + 57, + 50, + 49, + 98, + 51, + 54, + 54, + 54, + 97, + 53, + 101, + 54, + 100, + 54, + 53, + 49, + 51, + 97, + 49, + 48, + 98, + 56, + 53, + 102, + 49, + 102, + 56, + 102, + 50, + 52, + 48, + 100, + 97, + 50, + 34, + 44, + 34, + 115, + 111, + 117, + 114, + 99, + 101, + 95, + 110, + 97, + 109, + 101, + 115, + 112, + 97, + 99, + 101, + 34, + 58, + 34, + 98, + 97, + 99, + 107, + 101, + 110, + 100, + 58, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 101, + 45, + 55, + 98, + 98, + 48, + 45, + 56, + 50, + 53, + 101, + 45, + 56, + 50, + 54, + 102, + 97, + 102, + 54, + 51, + 48, + 98, + 51, + 53, + 34, + 44, + 34, + 115, + 117, + 98, + 109, + 105, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 102, + 45, + 55, + 97, + 48, + 51, + 45, + 98, + 56, + 98, + 56, + 45, + 57, + 53, + 50, + 98, + 98, + 98, + 98, + 54, + 100, + 50, + 50, + 57, + 34, + 44, + 34, + 115, + 117, + 98, + 109, + 105, + 115, + 115, + 105, + 111, + 110, + 95, + 114, + 101, + 113, + 117, + 101, + 115, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 101, + 45, + 55, + 98, + 98, + 48, + 45, + 56, + 50, + 53, + 101, + 45, + 56, + 50, + 54, + 102, + 97, + 102, + 54, + 51, + 48, + 98, + 51, + 53, + 34, + 125, + 93, + 125, + 125, + 93, + 44, + 34, + 104, + 105, + 115, + 116, + 111, + 114, + 121, + 34, + 58, + 91, + 123, + 34, + 105, + 116, + 101, + 109, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 101, + 115, + 115, + 97, + 103, + 101, + 34, + 44, + 34, + 114, + 111, + 108, + 101, + 34, + 58, + 34, + 117, + 115, + 101, + 114, + 34, + 44, + 34, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 34, + 58, + 91, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 101, + 120, + 116, + 34, + 44, + 34, + 116, + 101, + 120, + 116, + 34, + 58, + 34, + 227, + 129, + 147, + 227, + 130, + 140, + 227, + 129, + 175, + 228, + 191, + 157, + 229, + 173, + 152, + 227, + 129, + 168, + 229, + 190, + 169, + 229, + 133, + 131, + 227, + 130, + 146, + 231, + 162, + 186, + 232, + 170, + 141, + 227, + 129, + 153, + 227, + 130, + 139, + 227, + 129, + 159, + 227, + 130, + 129, + 227, + 129, + 174, + 228, + 186, + 186, + 229, + 183, + 165, + 231, + 154, + 132, + 227, + 129, + 170, + 228, + 188, + 154, + 232, + 169, + 177, + 227, + 129, + 167, + 227, + 129, + 153, + 227, + 128, + 130, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 175, + 227, + 128, + 140, + 233, + 157, + 146, + 227, + 129, + 132, + 231, + 129, + 175, + 229, + 143, + 176, + 227, + 128, + 141, + 227, + 129, + 167, + 227, + 129, + 153, + 227, + 128, + 130, + 232, + 166, + 154, + 227, + 129, + 136, + 227, + 129, + 159, + 227, + 129, + 147, + 227, + 129, + 168, + 227, + 130, + 146, + 228, + 184, + 128, + 230, + 150, + 135, + 227, + 129, + 167, + 232, + 191, + 148, + 227, + 129, + 151, + 227, + 129, + 166, + 227, + 129, + 143, + 227, + 129, + 160, + 227, + 129, + 149, + 227, + 129, + 132, + 227, + 128, + 130, + 84, + 111, + 111, + 108, + 227, + 129, + 175, + 228, + 189, + 191, + 227, + 130, + 143, + 227, + 129, + 170, + 227, + 129, + 132, + 227, + 129, + 167, + 227, + 129, + 143, + 227, + 129, + 160, + 227, + 129, + 149, + 227, + 129, + 132, + 227, + 128, + 130, + 34, + 125, + 93, + 125, + 44, + 34, + 109, + 101, + 116, + 97, + 100, + 97, + 116, + 97, + 34, + 58, + 123, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 102, + 45, + 55, + 97, + 48, + 51, + 45, + 98, + 56, + 98, + 56, + 45, + 57, + 53, + 51, + 100, + 50, + 55, + 48, + 52, + 49, + 102, + 101, + 48, + 34, + 44, + 34, + 111, + 114, + 105, + 103, + 105, + 110, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 98, + 97, + 99, + 107, + 101, + 110, + 100, + 95, + 105, + 110, + 115, + 116, + 114, + 117, + 99, + 116, + 105, + 111, + 110, + 34, + 44, + 34, + 111, + 112, + 101, + 114, + 97, + 116, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 101, + 45, + 55, + 98, + 98, + 48, + 45, + 56, + 50, + 53, + 101, + 45, + 56, + 50, + 54, + 102, + 97, + 102, + 54, + 51, + 48, + 98, + 51, + 53, + 34, + 125, + 125, + 125, + 93, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 95, + 105, + 116, + 101, + 109, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 48, + 54, + 53, + 54, + 44, + 34, + 101, + 110, + 116, + 114, + 121, + 34, + 58, + 123, + 34, + 105, + 116, + 101, + 109, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 95, + 114, + 101, + 115, + 112, + 111, + 110, + 115, + 101, + 95, + 98, + 111, + 117, + 110, + 100, + 97, + 114, + 121, + 34, + 44, + 34, + 114, + 101, + 115, + 112, + 111, + 110, + 115, + 101, + 95, + 105, + 100, + 34, + 58, + 34, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 45, + 114, + 101, + 115, + 112, + 111, + 110, + 115, + 101, + 45, + 48, + 45, + 48, + 34, + 125, + 44, + 34, + 109, + 101, + 116, + 97, + 100, + 97, + 116, + 97, + 34, + 58, + 123, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 99, + 48, + 45, + 55, + 56, + 56, + 49, + 45, + 97, + 98, + 101, + 101, + 45, + 56, + 97, + 54, + 51, + 102, + 100, + 49, + 56, + 50, + 57, + 55, + 49, + 34, + 44, + 34, + 111, + 114, + 105, + 103, + 105, + 110, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 111, + 100, + 101, + 108, + 95, + 111, + 117, + 116, + 112, + 117, + 116, + 34, + 44, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 34, + 58, + 123, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 100, + 45, + 98, + 56, + 49, + 56, + 45, + 55, + 55, + 99, + 49, + 45, + 98, + 100, + 55, + 51, + 45, + 49, + 55, + 53, + 97, + 50, + 55, + 53, + 54, + 50, + 102, + 54, + 51, + 34, + 125, + 125, + 125, + 125, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 95, + 105, + 116, + 101, + 109, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 50, + 51, + 52, + 49, + 44, + 34, + 101, + 110, + 116, + 114, + 121, + 34, + 58, + 123, + 34, + 105, + 116, + 101, + 109, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 101, + 115, + 115, + 97, + 103, + 101, + 34, + 44, + 34, + 114, + 111, + 108, + 101, + 34, + 58, + 34, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 34, + 44, + 34, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 34, + 58, + 91, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 101, + 120, + 116, + 34, + 44, + 34, + 116, + 101, + 120, + 116, + 34, + 58, + 34, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 175, + 227, + 128, + 140, + 233, + 157, + 146, + 227, + 129, + 132, + 231, + 129, + 175, + 229, + 143, + 176, + 227, + 128, + 141, + 227, + 129, + 168, + 232, + 166, + 154, + 227, + 129, + 136, + 227, + 129, + 190, + 227, + 129, + 151, + 227, + 129, + 159, + 227, + 128, + 130, + 34, + 125, + 93, + 125, + 44, + 34, + 109, + 101, + 116, + 97, + 100, + 97, + 116, + 97, + 34, + 58, + 123, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 102, + 53, + 53, + 45, + 55, + 49, + 100, + 51, + 45, + 98, + 52, + 49, + 56, + 45, + 56, + 49, + 57, + 57, + 98, + 56, + 54, + 51, + 102, + 53, + 57, + 97, + 34, + 44, + 34, + 111, + 114, + 105, + 103, + 105, + 110, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 111, + 100, + 101, + 108, + 95, + 111, + 117, + 116, + 112, + 117, + 116, + 34, + 44, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 34, + 58, + 123, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 100, + 45, + 98, + 56, + 49, + 56, + 45, + 55, + 55, + 99, + 49, + 45, + 98, + 100, + 55, + 51, + 45, + 49, + 55, + 53, + 97, + 50, + 55, + 53, + 54, + 50, + 102, + 54, + 51, + 34, + 125, + 125, + 125, + 125, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 108, + 108, + 109, + 95, + 117, + 115, + 97, + 103, + 101, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 50, + 51, + 52, + 49, + 44, + 34, + 104, + 105, + 115, + 116, + 111, + 114, + 121, + 95, + 108, + 101, + 110, + 34, + 58, + 49, + 44, + 34, + 105, + 110, + 112, + 117, + 116, + 95, + 116, + 111, + 116, + 97, + 108, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 49, + 54, + 53, + 50, + 44, + 34, + 99, + 97, + 99, + 104, + 101, + 95, + 114, + 101, + 97, + 100, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 48, + 44, + 34, + 99, + 97, + 99, + 104, + 101, + 95, + 119, + 114, + 105, + 116, + 101, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 48, + 44, + 34, + 111, + 117, + 116, + 112, + 117, + 116, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 49, + 56, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 117, + 114, + 110, + 95, + 101, + 110, + 100, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 50, + 51, + 52, + 50, + 44, + 34, + 116, + 117, + 114, + 110, + 95, + 99, + 111, + 117, + 110, + 116, + 34, + 58, + 49, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 114, + 117, + 110, + 95, + 99, + 111, + 109, + 112, + 108, + 101, + 116, + 101, + 100, + 34, + 44, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 102, + 53, + 54, + 45, + 55, + 54, + 101, + 51, + 45, + 57, + 53, + 101, + 97, + 45, + 48, + 100, + 97, + 100, + 101, + 49, + 98, + 48, + 52, + 101, + 53, + 102, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 50, + 51, + 52, + 50, + 44, + 34, + 105, + 110, + 116, + 101, + 114, + 114, + 117, + 112, + 116, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 114, + 101, + 115, + 117, + 108, + 116, + 34, + 58, + 34, + 102, + 105, + 110, + 105, + 115, + 104, + 101, + 100, + 34, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 105, + 110, + 118, + 111, + 107, + 101, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 54, + 54, + 57, + 49, + 44, + 34, + 116, + 114, + 105, + 103, + 103, + 101, + 114, + 34, + 58, + 34, + 117, + 115, + 101, + 114, + 95, + 115, + 101, + 110, + 100, + 34, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 117, + 115, + 101, + 114, + 95, + 105, + 110, + 112, + 117, + 116, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 54, + 54, + 57, + 49, + 44, + 34, + 115, + 101, + 103, + 109, + 101, + 110, + 116, + 115, + 34, + 58, + 91, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 101, + 120, + 116, + 34, + 44, + 34, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 34, + 58, + 34, + 229, + 133, + 136, + 227, + 129, + 187, + 227, + 129, + 169, + 227, + 129, + 174, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 175, + 228, + 189, + 149, + 227, + 129, + 167, + 227, + 129, + 151, + 227, + 129, + 159, + 227, + 129, + 139, + 239, + 188, + 159, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 160, + 227, + 129, + 145, + 227, + 130, + 146, + 231, + 173, + 148, + 227, + 129, + 136, + 227, + 129, + 166, + 227, + 129, + 143, + 227, + 129, + 160, + 227, + 129, + 149, + 227, + 129, + 132, + 227, + 128, + 130, + 34, + 125, + 93, + 44, + 34, + 101, + 120, + 116, + 101, + 110, + 115, + 105, + 111, + 110, + 115, + 34, + 58, + 91, + 123, + 34, + 100, + 111, + 109, + 97, + 105, + 110, + 34, + 58, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 46, + 112, + 101, + 110, + 100, + 105, + 110, + 103, + 95, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 111, + 110, + 115, + 46, + 118, + 49, + 34, + 44, + 34, + 112, + 97, + 121, + 108, + 111, + 97, + 100, + 34, + 58, + 123, + 34, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 110, + 103, + 34, + 58, + 110, + 117, + 108, + 108, + 44, + 34, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 110, + 103, + 95, + 110, + 111, + 116, + 105, + 102, + 105, + 99, + 97, + 116, + 105, + 111, + 110, + 34, + 58, + 91, + 93, + 44, + 34, + 105, + 110, + 118, + 111, + 99, + 97, + 116, + 105, + 111, + 110, + 95, + 114, + 101, + 99, + 101, + 105, + 112, + 116, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 105, + 110, + 118, + 111, + 99, + 97, + 116, + 105, + 111, + 110, + 95, + 114, + 101, + 99, + 111, + 118, + 101, + 114, + 121, + 95, + 98, + 108, + 111, + 99, + 107, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 110, + 101, + 120, + 116, + 95, + 97, + 99, + 116, + 105, + 118, + 97, + 116, + 105, + 111, + 110, + 95, + 115, + 101, + 113, + 117, + 101, + 110, + 99, + 101, + 34, + 58, + 50, + 44, + 34, + 110, + 111, + 116, + 105, + 102, + 105, + 99, + 97, + 116, + 105, + 111, + 110, + 95, + 114, + 101, + 99, + 101, + 105, + 112, + 116, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 112, + 101, + 110, + 100, + 105, + 110, + 103, + 34, + 58, + 91, + 93, + 44, + 34, + 112, + 101, + 110, + 100, + 105, + 110, + 103, + 95, + 110, + 111, + 116, + 105, + 102, + 105, + 99, + 97, + 116, + 105, + 111, + 110, + 115, + 34, + 58, + 91, + 93, + 44, + 34, + 114, + 101, + 99, + 101, + 105, + 112, + 116, + 115, + 34, + 58, + 91, + 123, + 34, + 100, + 105, + 115, + 112, + 111, + 115, + 105, + 116, + 105, + 111, + 110, + 34, + 58, + 34, + 115, + 116, + 97, + 114, + 116, + 101, + 100, + 34, + 44, + 34, + 112, + 97, + 121, + 108, + 111, + 97, + 100, + 95, + 100, + 105, + 103, + 101, + 115, + 116, + 34, + 58, + 34, + 100, + 53, + 51, + 102, + 53, + 53, + 52, + 52, + 52, + 56, + 98, + 99, + 50, + 55, + 55, + 53, + 50, + 97, + 99, + 49, + 49, + 98, + 56, + 54, + 57, + 50, + 48, + 51, + 53, + 48, + 57, + 50, + 49, + 98, + 51, + 54, + 54, + 54, + 97, + 53, + 101, + 54, + 100, + 54, + 53, + 49, + 51, + 97, + 49, + 48, + 98, + 56, + 53, + 102, + 49, + 102, + 56, + 102, + 50, + 52, + 48, + 100, + 97, + 50, + 34, + 44, + 34, + 115, + 111, + 117, + 114, + 99, + 101, + 95, + 110, + 97, + 109, + 101, + 115, + 112, + 97, + 99, + 101, + 34, + 58, + 34, + 98, + 97, + 99, + 107, + 101, + 110, + 100, + 58, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 101, + 45, + 55, + 98, + 98, + 48, + 45, + 56, + 50, + 53, + 101, + 45, + 56, + 50, + 54, + 102, + 97, + 102, + 54, + 51, + 48, + 98, + 51, + 53, + 34, + 44, + 34, + 115, + 117, + 98, + 109, + 105, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 102, + 45, + 55, + 97, + 48, + 51, + 45, + 98, + 56, + 98, + 56, + 45, + 57, + 53, + 50, + 98, + 98, + 98, + 98, + 54, + 100, + 50, + 50, + 57, + 34, + 44, + 34, + 115, + 117, + 98, + 109, + 105, + 115, + 115, + 105, + 111, + 110, + 95, + 114, + 101, + 113, + 117, + 101, + 115, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 49, + 56, + 98, + 101, + 45, + 55, + 98, + 98, + 48, + 45, + 56, + 50, + 53, + 101, + 45, + 56, + 50, + 54, + 102, + 97, + 102, + 54, + 51, + 48, + 98, + 51, + 53, + 34, + 125, + 44, + 123, + 34, + 100, + 105, + 115, + 112, + 111, + 115, + 105, + 116, + 105, + 111, + 110, + 34, + 58, + 34, + 115, + 116, + 97, + 114, + 116, + 101, + 100, + 34, + 44, + 34, + 112, + 97, + 121, + 108, + 111, + 97, + 100, + 95, + 100, + 105, + 103, + 101, + 115, + 116, + 34, + 58, + 34, + 98, + 100, + 101, + 101, + 49, + 53, + 56, + 98, + 99, + 57, + 50, + 53, + 102, + 101, + 98, + 56, + 49, + 100, + 54, + 56, + 52, + 48, + 102, + 100, + 49, + 54, + 100, + 50, + 100, + 101, + 56, + 98, + 50, + 49, + 97, + 51, + 48, + 50, + 52, + 98, + 49, + 53, + 102, + 57, + 50, + 99, + 102, + 53, + 48, + 54, + 50, + 53, + 97, + 97, + 56, + 50, + 101, + 97, + 48, + 52, + 49, + 97, + 101, + 98, + 34, + 44, + 34, + 115, + 111, + 117, + 114, + 99, + 101, + 95, + 110, + 97, + 109, + 101, + 115, + 112, + 97, + 99, + 101, + 34, + 58, + 34, + 98, + 97, + 99, + 107, + 101, + 110, + 100, + 58, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 48, + 53, + 50, + 45, + 55, + 54, + 100, + 49, + 45, + 97, + 52, + 99, + 49, + 45, + 49, + 51, + 56, + 100, + 101, + 55, + 56, + 101, + 52, + 51, + 56, + 100, + 34, + 44, + 34, + 115, + 117, + 98, + 109, + 105, + 115, + 115, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 48, + 53, + 51, + 45, + 55, + 49, + 102, + 51, + 45, + 98, + 102, + 52, + 102, + 45, + 57, + 49, + 49, + 55, + 48, + 55, + 57, + 102, + 49, + 100, + 102, + 97, + 34, + 44, + 34, + 115, + 117, + 98, + 109, + 105, + 115, + 115, + 105, + 111, + 110, + 95, + 114, + 101, + 113, + 117, + 101, + 115, + 116, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 48, + 53, + 50, + 45, + 55, + 54, + 100, + 49, + 45, + 97, + 52, + 99, + 49, + 45, + 49, + 51, + 56, + 100, + 101, + 55, + 56, + 101, + 52, + 51, + 56, + 100, + 34, + 125, + 93, + 125, + 125, + 93, + 44, + 34, + 104, + 105, + 115, + 116, + 111, + 114, + 121, + 34, + 58, + 91, + 123, + 34, + 105, + 116, + 101, + 109, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 101, + 115, + 115, + 97, + 103, + 101, + 34, + 44, + 34, + 114, + 111, + 108, + 101, + 34, + 58, + 34, + 117, + 115, + 101, + 114, + 34, + 44, + 34, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 34, + 58, + 91, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 101, + 120, + 116, + 34, + 44, + 34, + 116, + 101, + 120, + 116, + 34, + 58, + 34, + 229, + 133, + 136, + 227, + 129, + 187, + 227, + 129, + 169, + 227, + 129, + 174, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 175, + 228, + 189, + 149, + 227, + 129, + 167, + 227, + 129, + 151, + 227, + 129, + 159, + 227, + 129, + 139, + 239, + 188, + 159, + 229, + 144, + 136, + 232, + 168, + 128, + 232, + 145, + 137, + 227, + 129, + 160, + 227, + 129, + 145, + 227, + 130, + 146, + 231, + 173, + 148, + 227, + 129, + 136, + 227, + 129, + 166, + 227, + 129, + 143, + 227, + 129, + 160, + 227, + 129, + 149, + 227, + 129, + 132, + 227, + 128, + 130, + 34, + 125, + 93, + 125, + 44, + 34, + 109, + 101, + 116, + 97, + 100, + 97, + 116, + 97, + 34, + 58, + 123, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 48, + 53, + 51, + 45, + 55, + 49, + 102, + 51, + 45, + 98, + 102, + 52, + 102, + 45, + 57, + 49, + 50, + 57, + 48, + 100, + 56, + 50, + 52, + 56, + 56, + 98, + 34, + 44, + 34, + 111, + 114, + 105, + 103, + 105, + 110, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 98, + 97, + 99, + 107, + 101, + 110, + 100, + 95, + 105, + 110, + 115, + 116, + 114, + 117, + 99, + 116, + 105, + 111, + 110, + 34, + 44, + 34, + 111, + 112, + 101, + 114, + 97, + 116, + 105, + 111, + 110, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 48, + 53, + 50, + 45, + 55, + 54, + 100, + 49, + 45, + 97, + 52, + 99, + 49, + 45, + 49, + 51, + 56, + 100, + 101, + 55, + 56, + 101, + 52, + 51, + 56, + 100, + 34, + 125, + 125, + 125, + 93, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 95, + 105, + 116, + 101, + 109, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 54, + 54, + 57, + 50, + 44, + 34, + 101, + 110, + 116, + 114, + 121, + 34, + 58, + 123, + 34, + 105, + 116, + 101, + 109, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 95, + 114, + 101, + 115, + 112, + 111, + 110, + 115, + 101, + 95, + 98, + 111, + 117, + 110, + 100, + 97, + 114, + 121, + 34, + 44, + 34, + 114, + 101, + 115, + 112, + 111, + 110, + 115, + 101, + 95, + 105, + 100, + 34, + 58, + 34, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 45, + 114, + 101, + 115, + 112, + 111, + 110, + 115, + 101, + 45, + 49, + 45, + 49, + 34, + 125, + 44, + 34, + 109, + 101, + 116, + 97, + 100, + 97, + 116, + 97, + 34, + 58, + 123, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 48, + 53, + 52, + 45, + 55, + 48, + 54, + 49, + 45, + 97, + 49, + 102, + 54, + 45, + 52, + 100, + 55, + 101, + 101, + 100, + 51, + 98, + 52, + 102, + 53, + 53, + 34, + 44, + 34, + 111, + 114, + 105, + 103, + 105, + 110, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 111, + 100, + 101, + 108, + 95, + 111, + 117, + 116, + 112, + 117, + 116, + 34, + 44, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 34, + 58, + 123, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 100, + 45, + 98, + 56, + 49, + 56, + 45, + 55, + 55, + 99, + 49, + 45, + 98, + 100, + 55, + 51, + 45, + 49, + 55, + 53, + 97, + 50, + 55, + 53, + 54, + 50, + 102, + 54, + 51, + 34, + 125, + 125, + 125, + 125, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 97, + 110, + 110, + 111, + 116, + 97, + 116, + 101, + 100, + 95, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 95, + 105, + 116, + 101, + 109, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 56, + 51, + 56, + 51, + 44, + 34, + 101, + 110, + 116, + 114, + 121, + 34, + 58, + 123, + 34, + 105, + 116, + 101, + 109, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 101, + 115, + 115, + 97, + 103, + 101, + 34, + 44, + 34, + 114, + 111, + 108, + 101, + 34, + 58, + 34, + 97, + 115, + 115, + 105, + 115, + 116, + 97, + 110, + 116, + 34, + 44, + 34, + 99, + 111, + 110, + 116, + 101, + 110, + 116, + 34, + 58, + 91, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 101, + 120, + 116, + 34, + 44, + 34, + 116, + 101, + 120, + 116, + 34, + 58, + 34, + 233, + 157, + 146, + 227, + 129, + 132, + 231, + 129, + 175, + 229, + 143, + 176, + 34, + 125, + 93, + 125, + 44, + 34, + 109, + 101, + 116, + 97, + 100, + 97, + 116, + 97, + 34, + 58, + 123, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 54, + 101, + 102, + 45, + 55, + 56, + 98, + 49, + 45, + 98, + 48, + 99, + 56, + 45, + 98, + 49, + 52, + 49, + 102, + 49, + 51, + 98, + 48, + 99, + 55, + 98, + 34, + 44, + 34, + 111, + 114, + 105, + 103, + 105, + 110, + 34, + 58, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 109, + 111, + 100, + 101, + 108, + 95, + 111, + 117, + 116, + 112, + 117, + 116, + 34, + 44, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 34, + 58, + 123, + 34, + 119, + 111, + 114, + 107, + 101, + 114, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 100, + 45, + 98, + 56, + 49, + 56, + 45, + 55, + 55, + 99, + 49, + 45, + 98, + 100, + 55, + 51, + 45, + 49, + 55, + 53, + 97, + 50, + 55, + 53, + 54, + 50, + 102, + 54, + 51, + 34, + 125, + 125, + 125, + 125, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 108, + 108, + 109, + 95, + 117, + 115, + 97, + 103, + 101, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 56, + 51, + 56, + 52, + 44, + 34, + 104, + 105, + 115, + 116, + 111, + 114, + 121, + 95, + 108, + 101, + 110, + 34, + 58, + 52, + 44, + 34, + 105, + 110, + 112, + 117, + 116, + 95, + 116, + 111, + 116, + 97, + 108, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 49, + 54, + 57, + 54, + 44, + 34, + 99, + 97, + 99, + 104, + 101, + 95, + 114, + 101, + 97, + 100, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 49, + 53, + 51, + 54, + 44, + 34, + 99, + 97, + 99, + 104, + 101, + 95, + 119, + 114, + 105, + 116, + 101, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 48, + 44, + 34, + 111, + 117, + 116, + 112, + 117, + 116, + 95, + 116, + 111, + 107, + 101, + 110, + 115, + 34, + 58, + 56, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 116, + 117, + 114, + 110, + 95, + 101, + 110, + 100, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 56, + 51, + 56, + 52, + 44, + 34, + 116, + 117, + 114, + 110, + 95, + 99, + 111, + 117, + 110, + 116, + 34, + 58, + 50, + 125, + 10, + 123, + 34, + 107, + 105, + 110, + 100, + 34, + 58, + 34, + 114, + 117, + 110, + 95, + 99, + 111, + 109, + 112, + 108, + 101, + 116, + 101, + 100, + 34, + 44, + 34, + 101, + 110, + 116, + 114, + 121, + 95, + 105, + 100, + 34, + 58, + 34, + 48, + 49, + 97, + 49, + 54, + 53, + 55, + 102, + 45, + 51, + 54, + 102, + 48, + 45, + 55, + 102, + 53, + 48, + 45, + 97, + 52, + 56, + 102, + 45, + 56, + 55, + 55, + 99, + 98, + 54, + 97, + 100, + 100, + 53, + 56, + 54, + 34, + 44, + 34, + 116, + 115, + 34, + 58, + 49, + 55, + 57, + 50, + 55, + 48, + 52, + 49, + 57, + 56, + 51, + 56, + 52, + 44, + 34, + 105, + 110, + 116, + 101, + 114, + 114, + 117, + 112, + 116, + 101, + 100, + 34, + 58, + 102, + 97, + 108, + 115, + 101, + 44, + 34, + 114, + 101, + 115, + 117, + 108, + 116, + 34, + 58, + 34, + 102, + 105, + 110, + 105, + 115, + 104, + 101, + 100, + 34, + 125, + 10 + ] +} \ No newline at end of file diff --git a/tests/e2e/fixtures/worker-codex-v1/workspace-config.dcdl b/tests/e2e/fixtures/worker-codex-v1/workspace-config.dcdl new file mode 100644 index 000000000..60afe9221 --- /dev/null +++ b/tests/e2e/fixtures/worker-codex-v1/workspace-config.dcdl @@ -0,0 +1,22 @@ +{ + profile = { + default_profile = "project:fixture-chat"; + entries = [{ + selector = "project:fixture-chat"; + label = "Isolated Codex fixture conversation"; + description = "Artificial conversation only; no Workdir, Tools, Subject or background work."; + profile = (import "$builtin/profiles/job.dcdl") // { + slug = "fixture-chat"; + worker = { mode = "tools"; }; + model = { ref = "codex-oauth/gpt-6.1-sol"; }; + scope = "workspace_read"; + delegation_scope = "workspace_read"; + permissions = { default_action = "deny"; rule = []; }; + engine = { instruction = "default"; reasoning = "low"; max_tokens = 2048; max_turns = 4; language = "Japanese"; }; + session = { record_event_trace = false; }; + feature = { workdir_catalog = { enabled = false; }; }; + }; + }]; + }; + prompts = { default_prompt = "これは隔離環境の人工的な会話テストです。日本語で簡潔に応答してください。Toolや外部操作は行わず、ユーザーが会話中に提示した情報だけを使ってください。"; }; +} as WorkspaceConfigSchema diff --git a/tests/e2e/src/fixture_source.rs b/tests/e2e/src/fixture_source.rs new file mode 100644 index 000000000..de5100275 --- /dev/null +++ b/tests/e2e/src/fixture_source.rs @@ -0,0 +1,336 @@ +//! Bounded byte-preserving transport, independent of the owning store's schema validation. +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use sha2::{Digest, Sha256}; + +use crate::{HarnessError, Result}; + +#[derive(Debug, Clone, Copy)] +struct Limits { + file_bytes: u64, + total_bytes: u64, + entries: usize, + depth: usize, +} + +const LIMITS: Limits = Limits { + file_bytes: 8 * 1024 * 1024, + total_bytes: 32 * 1024 * 1024, + entries: 256, + depth: 8, +}; + +#[derive(Debug, Clone)] +pub struct FixtureSource { + root: PathBuf, + hashes: BTreeMap, + flat_files: Option>, +} + +fn reject(message: &'static str) -> HarnessError { + HarnessError::Protocol(message.into()) +} + +impl FixtureSource { + /// Capture a bounded tree without generating or repairing store data. + pub fn capture(root: &Path) -> Result { + Self::capture_with(root, None) + } + + /// Validate the complete flat file set before reading any content. The same + /// allowlist is retained for copy and later source-integrity checks. + pub fn capture_flat(root: &Path, names: &[&str]) -> Result { + let files: BTreeSet<_> = names.iter().map(PathBuf::from).collect(); + if files.len() != names.len() + || files.iter().any(|name| { + name.components().count() != 1 + || !matches!( + name.components().next(), + Some(std::path::Component::Normal(_)) + ) + }) + { + return Err(reject("invalid fixture file allowlist")); + } + Self::capture_with(root, Some(files)) + } + + fn capture_with(root: &Path, flat_files: Option>) -> Result { + let hashes = inventory(root, flat_files.as_ref(), LIMITS)?; + if hashes.is_empty() { + return Err(reject("provided fixture is empty")); + } + Ok(Self { + root: root.to_owned(), + hashes, + flat_files, + }) + } + + /// Read a known asset through a descriptor with a real byte limit, including + /// when its size grows after metadata inspection. Errors never include paths. + pub fn read_bounded(path: &Path) -> Result> { + read_regular(path, LIMITS.file_bytes) + } + + pub fn hashes(&self) -> &BTreeMap { + &self.hashes + } + + pub fn assert_unchanged(&self) -> Result<()> { + if inventory(&self.root, self.flat_files.as_ref(), LIMITS)? != self.hashes { + return Err(reject("provided fixture changed during execution")); + } + Ok(()) + } + + /// The destination must not exist. A copy is never a link to the source. + pub fn copy_to(&self, destination: &Path) -> Result<()> { + self.assert_unchanged()?; + let parent = destination + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + if parent + .canonicalize()? + .starts_with(self.root.canonicalize()?) + { + return Err(reject("fixture copy destination is inside the original")); + } + fs::create_dir(destination)?; + let mut remaining = LIMITS.total_bytes; + for (relative, expected) in &self.hashes { + let bytes = read_regular(&self.root.join(relative), LIMITS.file_bytes.min(remaining))?; + remaining -= bytes.len() as u64; + if hash(&bytes) != *expected { + return Err(reject("fixture changed while copying")); + } + let target = destination.join(relative); + fs::create_dir_all(target.parent().unwrap())?; + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(target)?; + file.write_all(&bytes)?; + } + if inventory(destination, self.flat_files.as_ref(), LIMITS)? != self.hashes { + return Err(reject("fixture copy failed hash verification")); + } + self.assert_unchanged() + } +} + +fn hash(bytes: &[u8]) -> String { + format!( + "sha256:{}", + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + ) +} + +fn read_limited(reader: impl Read, limit: u64) -> Result> { + let mut bytes = Vec::new(); + reader + .take(limit + 1) + .read_to_end(&mut bytes) + .map_err(|_| reject("fixture content read failed"))?; + if bytes.len() as u64 > limit { + return Err(reject("fixture byte budget exceeded")); + } + Ok(bytes) +} + +fn read_regular(path: &Path, limit: u64) -> Result> { + let mut options = fs::OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + // Do not follow a replaced final symlink or block on a replaced FIFO. + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + let file = options + .open(path) + .map_err(|_| reject("fixture asset open failed"))?; + let metadata = file + .metadata() + .map_err(|_| reject("fixture asset metadata failed"))?; + if !metadata.is_file() || metadata.len() > limit { + return Err(reject("fixture asset is not a bounded regular file")); + } + read_limited(file, limit) +} + +fn inventory( + root: &Path, + flat: Option<&BTreeSet>, + limits: Limits, +) -> Result> { + struct Budget { + entries: usize, + bytes: u64, + } + fn visit( + root: &Path, + path: &Path, + depth: usize, + limits: Limits, + flat: Option<&BTreeSet>, + budget: &mut Budget, + files: &mut BTreeSet, + ) -> Result<()> { + if depth > limits.depth || budget.entries == 0 { + return Err(reject("fixture tree budget exceeded")); + } + budget.entries -= 1; + let metadata = fs::symlink_metadata(path).map_err(|_| reject("fixture metadata failed"))?; + let relative = path.strip_prefix(root).unwrap(); + if depth != 0 && flat.is_some_and(|names| !names.contains(relative)) { + return Err(reject("unexpected fixture file set")); + } + if metadata.is_dir() && (depth == 0 || flat.is_none()) { + for entry in fs::read_dir(path).map_err(|_| reject("fixture directory read failed"))? { + let entry = entry.map_err(|_| reject("fixture directory entry failed"))?; + visit(root, &entry.path(), depth + 1, limits, flat, budget, files)?; + } + } else if metadata.is_file() && depth != 0 { + if metadata.len() > limits.file_bytes || metadata.len() > budget.bytes { + return Err(reject("fixture byte budget exceeded")); + } + budget.bytes -= metadata.len(); + files.insert(relative.to_owned()); + } else { + return Err(reject("fixture contains a symlink or special file")); + } + Ok(()) + } + // Preflight the entire shape and metadata budget before reading any body. + let mut files = BTreeSet::new(); + visit( + root, + root, + 0, + limits, + flat, + &mut Budget { + entries: limits.entries, + bytes: limits.total_bytes, + }, + &mut files, + )?; + if flat.is_some_and(|expected| expected != &files) { + return Err(reject("unexpected fixture file set")); + } + let mut remaining = limits.total_bytes; + let mut hashes = BTreeMap::new(); + for relative in files { + let bytes = read_regular(&root.join(&relative), limits.file_bytes.min(remaining))?; + remaining -= bytes.len() as u64; + hashes.insert(relative, hash(&bytes)); + } + Ok(hashes) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Transport-only synthetic bytes, never a substitute saved Worker fixture. + #[test] + fn copies_are_independent_and_source_mutation_is_detected() -> Result<()> { + let temp = tempfile::tempdir()?; + let source = temp.path().join("source"); + fs::create_dir(&source)?; + fs::write(source.join("bytes"), b"original")?; + let fixture = FixtureSource::capture_flat(&source, &["bytes"])?; + assert!(fixture.copy_to(&source.join("nested-copy")).is_err()); + assert!(!source.join("nested-copy").exists()); + fixture.assert_unchanged()?; + let first = temp.path().join("first"); + let second = temp.path().join("second"); + fixture.copy_to(&first)?; + fixture.copy_to(&second)?; + fs::write(first.join("bytes"), b"changed")?; + fs::remove_dir_all(first)?; + assert_eq!(fs::read(second.join("bytes"))?, b"original"); + fixture.assert_unchanged()?; + fs::write(source.join("bytes"), b"changed")?; + assert!(fixture.assert_unchanged().is_err()); + Ok(()) + } + + #[test] + fn rejects_missing_empty_existing_destination_and_extra_file() -> Result<()> { + let temp = tempfile::tempdir()?; + assert!(FixtureSource::capture(&temp.path().join("missing")).is_err()); + assert!(FixtureSource::capture(temp.path()).is_err()); + fs::write(temp.path().join("bytes"), b"original")?; + let source = FixtureSource::capture(temp.path())?; + assert!(source.copy_to(temp.path()).is_err()); + fs::write(temp.path().join("extra"), b"new")?; + assert!(source.assert_unchanged().is_err()); + Ok(()) + } + + #[test] + fn flat_allowlist_rejects_extra_entries_before_reading_content() -> Result<()> { + let temp = tempfile::tempdir()?; + fs::write(temp.path().join("asset"), b"ok")?; + // Sparse oversized input is never read. Even an allowed file that cannot + // be decoded does not take precedence over an unexpected file set. + fs::File::create(temp.path().join("extra"))?.set_len(LIMITS.file_bytes + 1)?; + let error = FixtureSource::capture_flat(temp.path(), &["asset"]).unwrap_err(); + assert!(error.to_string().contains("unexpected fixture file set")); + fs::remove_file(temp.path().join("extra"))?; + assert!(FixtureSource::capture_flat(temp.path(), &["missing"]).is_err()); + fs::create_dir(temp.path().join("dir"))?; + assert!(FixtureSource::capture_flat(temp.path(), &["asset", "dir"]).is_err()); + Ok(()) + } + + #[test] + fn tree_count_depth_total_and_actual_read_budgets_are_enforced() -> Result<()> { + let temp = tempfile::tempdir()?; + fs::write(temp.path().join("a"), b"1234")?; + fs::write(temp.path().join("b"), b"1234")?; + for limits in [ + Limits { + entries: 2, + ..LIMITS + }, + Limits { + total_bytes: 7, + ..LIMITS + }, + Limits { + file_bytes: 3, + ..LIMITS + }, + ] { + assert!(inventory(temp.path(), None, limits).is_err()); + } + fs::create_dir_all(temp.path().join("nested/deep"))?; + assert!(inventory(temp.path(), None, Limits { depth: 1, ..LIMITS }).is_err()); + // A reader can return more bytes than earlier metadata reported. Bound + // actual IO rather than trusting that observation or an EOF at stat size. + assert!(read_limited(std::io::Cursor::new(b"12345"), 4).is_err()); + assert_eq!(read_limited(std::io::Cursor::new(b"1234"), 4)?, b"1234"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn rejects_symbolic_links() -> Result<()> { + let temp = tempfile::tempdir()?; + fs::write(temp.path().join("bytes"), b"original")?; + std::os::unix::fs::symlink("bytes", temp.path().join("link"))?; + assert!(FixtureSource::capture(temp.path()).is_err()); + assert!(FixtureSource::read_bounded(&temp.path().join("link")).is_err()); + Ok(()) + } +} diff --git a/tests/e2e/src/lib.rs b/tests/e2e/src/lib.rs index 176c7c2ff..9fcdb4ff0 100644 --- a/tests/e2e/src/lib.rs +++ b/tests/e2e/src/lib.rs @@ -137,7 +137,9 @@ fn tested_yoi_env_policy_overview() -> TestedYoiEnvPolicy { #[derive(Debug)] pub enum HarnessError { Io(io::Error), - Json(serde_json::Error), + // Never retain decode errors: serde messages can contain rejected field names + // or values from private fixture/protocol payloads, including through Debug. + Json, CommandFailed { program: PathBuf, args: Vec, @@ -163,7 +165,7 @@ impl std::fmt::Display for HarnessError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Io(err) => write!(f, "io error: {err}"), - Self::Json(err) => write!(f, "json error: {err}"), + Self::Json => write!(f, "invalid JSON document"), Self::CommandFailed { program, args, @@ -209,8 +211,8 @@ impl From for HarnessError { } impl From for HarnessError { - fn from(value: serde_json::Error) -> Self { - Self::Json(value) + fn from(_: serde_json::Error) -> Self { + Self::Json } } @@ -1268,7 +1270,11 @@ impl Drop for PanelHarness { } } +mod fixture_source; +pub use fixture_source::FixtureSource; mod worker_environment; +mod worker_session; +pub use worker_session::{WorkerSessionSubscription, validate_full_snapshot}; pub use worker_environment::{ WorkerBinaryProviderInfo, WorkerE2eEnvironment, WorkerEnvironmentCleanupReport, @@ -1627,6 +1633,8 @@ fn resolve_yoi_binary() -> Result { "e2e-test".to_owned(), "--bin".to_owned(), "yoi".to_owned(), + "--target-dir".to_owned(), + tui_product_target_dir()?.to_string_lossy().into_owned(), ]; if test_profile() == "release" { args.push("--release".to_owned()); @@ -1648,7 +1656,9 @@ fn resolve_yoi_binary() -> Result { }); } - let binary = current_target_profile_dir()?.join(binary_name()); + let binary = tui_product_target_dir()? + .join(test_profile()) + .join(binary_name()); let info = BinaryProviderInfo { provider: "cargo-build".to_owned(), binary, @@ -1675,6 +1685,26 @@ fn workspace_root() -> Result { .ok_or_else(|| HarnessError::Protocol("could not resolve workspace root".to_owned())) } +/// Product processes must never reuse the target used by test-support builds. +fn product_target_dir() -> Result { + dedicated_product_target("e2e-products") +} + +// Existing PTY tests need their observer feature, never on restore binaries. +fn tui_product_target_dir() -> Result { + dedicated_product_target("e2e-tui-products") +} + +fn dedicated_product_target(name: &str) -> Result { + let target = workspace_root()?.join("target").join(name); + if current_target_profile_dir()?.parent() == Some(target.as_path()) { + return Err(HarnessError::Protocol( + "run support tests outside reserved product binary targets".to_owned(), + )); + } + Ok(target) +} + fn current_target_profile_dir() -> Result { let mut path = std::env::current_exe()?; while let Some(name) = path.file_name().and_then(|name| name.to_str()) { diff --git a/tests/e2e/src/worker_environment.rs b/tests/e2e/src/worker_environment.rs index 2183560cd..1fbdc4b0b 100644 --- a/tests/e2e/src/worker_environment.rs +++ b/tests/e2e/src/worker_environment.rs @@ -13,12 +13,13 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use super::{ - FixtureAllocation, HarnessError, Result, allocate_fixture, command_display, - current_target_profile_dir, now_ms, test_profile, workspace_root, yoi_binary_info, + FixtureAllocation, HarnessError, Result, allocate_fixture, command_display, now_ms, + product_target_dir, test_profile, workspace_root, }; const STARTUP_WAIT: Duration = Duration::from_secs(15); const EXIT_WAIT: Duration = Duration::from_secs(5); +const SETUP_WAIT: Duration = Duration::from_secs(15); const HTTP_WAIT_STEP: Duration = Duration::from_millis(50); const MAX_PROCESS_LOG_BYTES: usize = 256 * 1024; const MAX_COMMAND_OUTPUT_BYTES: usize = 64 * 1024; @@ -61,22 +62,20 @@ pub fn worker_binary_provider_info() -> Result { fn resolve_worker_binaries() -> Result { let workspace_root = workspace_root()?; let profile = test_profile(); - let yoi = yoi_binary_info()?; - let mut records = vec![binary_record( - "yoi", - yoi.binary, - yoi.provider, - yoi.build_command, - )?]; - + let mut records = Vec::new(); + let yoi_override = resolve_override("YOI_E2E_BIN", &workspace_root)?; let server_override = resolve_override("YOI_E2E_SERVER_BIN", &workspace_root)?; let runtime_override = resolve_override("YOI_E2E_RUNTIME_BIN", &workspace_root)?; let mut build_command = None; - if server_override.is_none() || runtime_override.is_none() { + if yoi_override.is_none() || server_override.is_none() || runtime_override.is_none() { let cargo = PathBuf::from(std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into())); let mut args = vec![ "build".to_string(), "-p".to_string(), + "yoi".to_string(), + "--bin".to_string(), + "yoi".to_string(), + "-p".to_string(), "yoi-workspace-server".to_string(), "--bin".to_string(), "yoi-server".to_string(), @@ -84,6 +83,8 @@ fn resolve_worker_binaries() -> Result { "worker-runtime".to_string(), "--bin".to_string(), "yoi-runtime".to_string(), + "--target-dir".to_string(), + product_target_dir()?.to_string_lossy().into_owned(), ]; if profile == "release" { args.push("--release".to_string()); @@ -106,7 +107,16 @@ fn resolve_worker_binaries() -> Result { build_command = Some(rendered); } - let profile_dir = current_target_profile_dir()?; + let profile_dir = product_target_dir()?.join(&profile); + let (yoi_path, yoi_provider, yoi_build) = match yoi_override { + Some(path) => (path, "YOI_E2E_BIN".to_string(), None), + None => ( + profile_dir.join(executable_name("yoi")), + "cargo-build".to_string(), + build_command.clone(), + ), + }; + records.push(binary_record("yoi", yoi_path, yoi_provider, yoi_build)?); let (server_path, server_provider, server_build) = match server_override { Some(path) => (path, "YOI_E2E_SERVER_BIN".to_string(), None), None => ( @@ -196,6 +206,17 @@ fn executable_name(name: &str) -> String { format!("{name}{}", std::env::consts::EXE_SUFFIX) } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum WorkerProcessShutdownOutcome { + #[default] + NotStarted, + Succeeded, + Failed { + error: String, + }, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct WorkerEnvironmentCleanupReport { pub fixture_root: PathBuf, @@ -204,6 +225,10 @@ pub struct WorkerEnvironmentCleanupReport { pub cleanup_success: bool, pub fixture_root_exists_after: bool, pub cleanup_error: Option, + pub runtime_shutdown: WorkerProcessShutdownOutcome, + pub server_shutdown: WorkerProcessShutdownOutcome, + pub root_cleanup_success: bool, + pub root_cleanup_error: Option, pub report_path: PathBuf, } @@ -231,6 +256,10 @@ pub struct WorkerE2eEnvironment { pub runtime_url: Option, server: Option, runtime: Option, + runtime_shutdown: WorkerProcessShutdownOutcome, + server_shutdown: WorkerProcessShutdownOutcome, + shutdown_error: Option, + runtime_public_identity: Option, access_token: Option, client: Client, binaries: WorkerBinaryProviderInfo, @@ -292,6 +321,10 @@ impl WorkerE2eEnvironment { runtime_url: None, server: None, runtime: None, + runtime_shutdown: WorkerProcessShutdownOutcome::NotStarted, + server_shutdown: WorkerProcessShutdownOutcome::NotStarted, + shutdown_error: None, + runtime_public_identity: None, access_token: None, client, binaries, @@ -319,8 +352,9 @@ impl WorkerE2eEnvironment { "Backend-only restart requires a running Server and no Runtime".to_string(), )); } - if let Some(mut server) = self.server.take() { - server.shutdown(EXIT_WAIT)?; + shutdown_process(&mut self.server, &mut self.server_shutdown); + if let WorkerProcessShutdownOutcome::Failed { error } = &self.server_shutdown { + return Err(HarnessError::Protocol(error.clone())); } self.server_url = None; self.start_server_process() @@ -374,6 +408,47 @@ impl WorkerE2eEnvironment { Ok(()) } + /// Start a store populated by the Backend's typed test-support API. The seed + /// supplies fixed catalog IDs, but the bearer must be freshly issued for this + /// environment. No database mutation or fixture credential import occurs here. + pub fn start_seeded_backend( + &mut self, + workspace_id: String, + runtime_id: String, + access_token: String, + ) -> Result<()> { + if self.server.is_some() || self.runtime.is_some() || self.workspace_id.is_some() { + return Err(HarnessError::Protocol( + "fixture seed requires an unstarted environment".into(), + )); + } + if workspace_id.is_empty() || runtime_id.is_empty() || access_token.is_empty() { + return Err(HarnessError::Protocol( + "fixture seed omitted an identity or fresh bearer".into(), + )); + } + if !self.server_data_dir.join("server.db").is_file() { + return Err(HarnessError::Protocol( + "typed Backend seed has not created server.db".into(), + )); + } + if self.runtime_public_identity.is_some() { + if self.runtime_id != runtime_id { + return Err(HarnessError::Protocol( + "seeded Runtime ID differs from initialized identity".into(), + )); + } + } else { + assert_absent_or_empty(&self.runtime_data_dir, "Runtime data directory")?; + } + self.workspace_id = Some(workspace_id); + self.runtime_id = runtime_id; + self.access_token = Some(access_token); + self.start_server_process()?; + self.export_workspace_public_identity()?; + self.write_metadata("seeded-backend-running") + } + pub fn start_backend(&mut self) -> Result<()> { self.start_empty_backend()?; let server_binary = self.binaries.binary("yoi-server")?; @@ -417,7 +492,15 @@ impl WorkerE2eEnvironment { "create Workspace", )?; let workspace_id = json_string(&created, "/workspace/workspace_id")?; - self.workspace_id = Some(workspace_id.clone()); + self.workspace_id = Some(workspace_id); + self.export_workspace_public_identity()?; + self.write_metadata("backend-running")?; + Ok(()) + } + + fn export_workspace_public_identity(&self) -> Result<()> { + let server_url = self.required_server_url()?; + let workspace_id = self.required_workspace_id()?; let identity = self.request_json( self.client.get(format!( "{server_url}/api/w/{workspace_id}/settings/signing-identity" @@ -429,17 +512,27 @@ impl WorkerE2eEnvironment { })?; let bundle_path = self.root.join("workspace-public-identity.json"); fs::write(&bundle_path, serde_json::to_vec_pretty(&public_bundle)?)?; - self.write_metadata("backend-running")?; Ok(()) } fn start_runtime(&mut self) -> Result<()> { - let bundle_path = self.root.join("workspace-public-identity.json"); - let server_url = self.required_server_url()?.to_string(); - let workspace_id = self.required_workspace_id()?.to_string(); - let runtime_binary = self.binaries.binary("yoi-runtime")?; + self.start_runtime_with_seed(|_| Ok(()))?; + self.assert_connected_and_empty() + } + + /// Generate fresh keys before offline Backend registration when fixed IDs are + /// required. Returns only the public bundle; no fixture keys are accepted. + pub fn initialize_runtime_identity(&mut self, runtime_id: String) -> Result { + if self.runtime_public_identity.is_some() || self.runtime.is_some() { + return Err(HarnessError::Protocol( + "Runtime identity is already initialized".into(), + )); + } + assert_absent_or_empty(&self.runtime_data_dir, "Runtime data directory")?; + self.runtime_id = runtime_id; + let binary = self.binaries.binary("yoi-runtime")?; self.run_command( - &runtime_binary, + &binary, &[ "identity", "init", @@ -450,8 +543,8 @@ impl WorkerE2eEnvironment { ], false, )?; - let runtime_identity = self.run_command( - &runtime_binary, + let identity = self.run_command( + &binary, &[ "identity", "show", @@ -461,7 +554,31 @@ impl WorkerE2eEnvironment { ], false, )?; - let runtime_identity: Value = serde_json::from_slice(&runtime_identity.stdout)?; + let identity: Value = serde_json::from_slice(&identity.stdout)?; + self.runtime_public_identity = Some(identity.clone()); + Ok(identity) + } + + /// Initialize fresh Runtime keys/trust, then invoke the typed Runtime seed + /// before starting its process. The actual Backend URL is now available. + pub fn start_runtime_with_seed( + &mut self, + seed: impl FnOnce(&Self) -> Result<()>, + ) -> Result<()> { + if self.runtime.is_some() { + return Err(HarnessError::Protocol("Runtime is already running".into())); + } + if self.runtime_public_identity.is_none() { + self.initialize_runtime_identity(self.runtime_id.clone())?; + } + let runtime_identity = self + .runtime_public_identity + .clone() + .expect("initialized identity"); + let bundle_path = self.root.join("workspace-public-identity.json"); + let server_url = self.required_server_url()?.to_string(); + let workspace_id = self.required_workspace_id()?.to_string(); + let runtime_binary = self.binaries.binary("yoi-runtime")?; self.run_command( &runtime_binary, &[ @@ -497,6 +614,8 @@ impl WorkerE2eEnvironment { )); } + seed(self)?; + self.write_metadata("runtime-seeded")?; let mut command = self.isolated_command(&runtime_binary); command.args([ "--bind", @@ -519,22 +638,38 @@ impl WorkerE2eEnvironment { self.runtime = Some(runtime); self.runtime_url = Some(runtime_url.clone()); + let registered = self.request_json( + self.client + .get(format!("{server_url}/api/w/{workspace_id}/runtimes")), + "observe Runtime registration before endpoint update", + )?; + let expected_binding_id = registered + .get("items") + .and_then(Value::as_array) + .and_then(|items| { + items.iter().find(|item| { + item.get("runtime_id").and_then(Value::as_str) == Some(self.runtime_id.as_str()) + }) + }) + .and_then(|item| item.pointer("/management/binding/binding_id")) + .cloned(); self.request_json( self.client .post(format!("{server_url}/api/w/{workspace_id}/runtimes")) .json(&serde_json::json!({ "public_bundle": runtime_identity, "display_name": format!("E2E {}", self.runtime_id), - "endpoint": runtime_url + "endpoint": runtime_url, + "expected_binding_id": expected_binding_id, })), "register Runtime", )?; - self.assert_connected_and_empty()?; + self.assert_connected()?; self.write_metadata("running")?; Ok(()) } - pub fn assert_connected_and_empty(&self) -> Result<()> { + pub fn assert_connected(&self) -> Result<()> { let server_url = self.required_server_url()?; let workspace_id = self.required_workspace_id()?; // connection testは実署名要求。保存済みの接続flagを認証成功の代わりにしない。 @@ -545,47 +680,125 @@ impl WorkerE2eEnvironment { )), "verify Runtime connection", )?; - if connection.get("status").and_then(Value::as_str) != Some("compatible") { - return Err(HarnessError::Protocol(format!( - "Runtime connection was not compatible: {connection}" - ))); - } + require_compatible_connection(&connection)?; let binding_id = json_string(&connection, "/binding_id")?; let runtimes = self.request_json( self.client .get(format!("{server_url}/api/w/{workspace_id}/runtimes")), "list Runtimes", )?; - let connected = runtimes - .get("items") - .and_then(Value::as_array) - .is_some_and(|items| { - items.iter().any(|item| { - item.get("runtime_id").and_then(Value::as_str) == Some(self.runtime_id.as_str()) - && item - .pointer("/management/binding/binding_id") - .and_then(Value::as_str) - == Some(binding_id.as_str()) - }) - }); - if !connected { - return Err(HarnessError::Protocol(format!( - "Runtime {} is not verified in {runtimes}", - self.runtime_id - ))); - } - let workers = self.request_json( + require_registered_runtime(&runtimes, &self.runtime_id, &binding_id) + } + + pub fn list_workers(&self) -> Result { + let server_url = self.required_server_url()?; + let workspace_id = self.required_workspace_id()?; + self.request_json( self.client .get(format!("{server_url}/api/w/{workspace_id}/workers")), "list Workers", - )?; - if workers - .get("items") - .and_then(Value::as_array) - .is_none_or(|items| !items.is_empty()) - { + ) + } + + pub fn assert_connected_and_empty(&self) -> Result<()> { + self.assert_connected()?; + let workers = self.list_workers()?; + require_empty_worker_list(&workers) + } + + fn worker_url(&self, worker_id: &str, suffix: &str) -> Result { + let mut url = reqwest::Url::parse(self.required_server_url()?) + .map_err(|error| HarnessError::Protocol(format!("invalid Server URL: {error}")))?; + url.path_segments_mut() + .map_err(|_| HarnessError::Protocol("Server URL has no path segments".into()))? + .extend([ + "api", + "w", + self.required_workspace_id()?, + "runtimes", + &self.runtime_id, + "workers", + worker_id, + ]); + if !suffix.is_empty() { + url.path_segments_mut().unwrap().extend(suffix.split('/')); + } + Ok(url.into()) + } + + pub fn worker_detail(&self, worker_id: &str) -> Result { + self.request_json( + self.client.get(self.worker_url(worker_id, "")?), + "read Worker detail", + ) + } + + /// Exactly one restore submission. No retries, replacement token or new intent. + pub fn restore_worker( + &self, + worker_id: &str, + request_id: &str, + expected_observation_token: &str, + ) -> Result { + self.request_json( + self.client + .post(self.worker_url(worker_id, "restore")?) + .timeout(STARTUP_WAIT) + .json(&serde_json::json!({ + "request_id": request_id, + "expected_observation_token": expected_observation_token, + })), + "restore Worker", + ) + } + + pub fn stop_worker(&self, worker_id: &str) -> Result { + self.request_json( + self.client + .post(self.worker_url(worker_id, "stop")?) + .json(&serde_json::json!({})), + "stop Worker", + ) + } + + /// Authorization stays in memory; callers must not persist or debug this request. + pub fn worker_protocol_request(&self, worker_id: &str) -> Result { + let token = self.access_token.as_deref().ok_or_else(|| { + HarnessError::Protocol("Backend access token is not initialized".into()) + })?; + self.client + .get(self.worker_url(worker_id, "protocol/ws")?) + .bearer_auth(token) + .build() + .map_err(|_| HarnessError::Protocol("build Worker protocol request".into())) + } + + /// The other environment must reject this environment's fresh bearer even + /// when both contain identical Workspace/Runtime/Worker IDs. Never logs tokens. + pub fn assert_authentication_isolated_from(&self, other: &Self) -> Result<()> { + let token = self + .access_token + .as_deref() + .ok_or_else(|| HarnessError::Protocol("missing source bearer".into()))?; + if other.access_token.as_deref() == Some(token) { + return Err(HarnessError::Protocol( + "environments share authentication material".into(), + )); + } + let (server, workspace) = other.backend_target()?; + let response = self + .client + .get(format!("{server}/api/w/{workspace}/workers")) + .bearer_auth(token) + .send() + .map_err(|_| { + HarnessError::Protocol("cross-environment authorization request failed".into()) + })?; + let status = response.status().as_u16(); + other.append_operation("reject other environment bearer", status)?; + if status != 401 { return Err(HarnessError::Protocol(format!( - "fresh Worker list is not empty: {workers}" + "cross-environment bearer was not rejected with 401: {status}" ))); } Ok(()) @@ -636,7 +849,7 @@ impl WorkerE2eEnvironment { "assignee": null, "readiness": null, "risk_flags": [], - "workflow_state": "planning", + "workflow_state": "open", "queued_by": null, "queued_at": null, "repository_key": null, @@ -697,30 +910,53 @@ impl WorkerE2eEnvironment { } pub fn shutdown(&mut self) -> Result<()> { - if let Some(mut runtime) = self.runtime.take() { - runtime.shutdown(EXIT_WAIT)?; + // 一方の異常でも他方を回収し、再呼出しやroot削除で失敗を消さない。 + shutdown_process(&mut self.runtime, &mut self.runtime_shutdown); + shutdown_process(&mut self.server, &mut self.server_shutdown); + let mut errors = Vec::new(); + for (name, outcome) in [ + ("Runtime", &self.runtime_shutdown), + ("Server", &self.server_shutdown), + ] { + if let WorkerProcessShutdownOutcome::Failed { error } = outcome { + errors.push(format!("{name}: {error}")); + } } - if let Some(mut server) = self.server.take() { - server.shutdown(EXIT_WAIT)?; + if self.write_metadata("stopped").is_err() { + errors.push("write stopped metadata failed".to_string()); + } + if self.shutdown_error.is_none() && !errors.is_empty() { + self.shutdown_error = Some(errors.join("; ")); + } + match &self.shutdown_error { + Some(error) => Err(HarnessError::Protocol(error.clone())), + None => Ok(()), } - self.write_metadata("stopped")?; - Ok(()) } pub fn cleanup(mut self) -> Result { - self.shutdown()?; self.cleanup_inner(true) } fn cleanup_inner(&mut self, strict: bool) -> Result { - let mut cleanup_error = None; + let _ = self.shutdown(); + let mut root_cleanup_error = None; if let Some(temp_root) = self.temp_root.take() - && let Err(error) = temp_root.close() + && temp_root.close().is_err() { - cleanup_error = Some(error.to_string()); + root_cleanup_error = Some("remove fixture root failed".to_string()); } let fixture_root_exists_after = self.root.exists(); - let cleanup_success = cleanup_error.is_none() && !fixture_root_exists_after; + if fixture_root_exists_after && root_cleanup_error.is_none() { + root_cleanup_error = Some("fixture root still exists".to_string()); + } + let root_cleanup_success = root_cleanup_error.is_none() && !fixture_root_exists_after; + let cleanup_error = match (&self.shutdown_error, &root_cleanup_error) { + (Some(shutdown), Some(root)) => Some(format!("{shutdown}; {root}")), + (Some(error), _) | (_, Some(error)) => Some(error.clone()), + (None, None) => None, + }; + let cleanup_success = self.shutdown_error.is_none() && root_cleanup_success; let report = WorkerEnvironmentCleanupReport { fixture_root: self.root.clone(), artifacts_dir: self.artifacts_dir.clone(), @@ -728,6 +964,10 @@ impl WorkerE2eEnvironment { cleanup_success, fixture_root_exists_after, cleanup_error, + runtime_shutdown: self.runtime_shutdown.clone(), + server_shutdown: self.server_shutdown.clone(), + root_cleanup_success, + root_cleanup_error, report_path: self.artifacts_dir.join("cleanup.json"), }; fs::write(&report.report_path, serde_json::to_vec_pretty(&report)?)?; @@ -773,18 +1013,8 @@ impl WorkerE2eEnvironment { let response = request .send() .map_err(|error| HarnessError::Protocol(format!("{operation}: {error}")))?; - let status = response.status(); - let body = response - .bytes() - .map_err(|error| HarnessError::Protocol(format!("{operation}: {error}")))?; - self.append_operation(operation, status.as_u16())?; - if !status.is_success() { - return Err(HarnessError::Protocol(format!( - "{operation} returned {status}: {}", - bounded_text(&body, MAX_COMMAND_OUTPUT_BYTES) - ))); - } - serde_json::from_slice(&body).map_err(HarnessError::from) + self.append_operation(operation, response.status().as_u16())?; + decode_api_response(response, operation) } fn append_operation(&self, operation: &str, status: u16) -> Result<()> { @@ -809,13 +1039,33 @@ impl WorkerE2eEnvironment { binary: &Path, args: &[&str], redact_stdout: bool, + ) -> Result { + self.run_command_with_timeout(binary, args, redact_stdout, SETUP_WAIT) + } + + fn run_command_with_timeout( + &self, + binary: &Path, + args: &[&str], + redact_stdout: bool, + timeout: Duration, ) -> Result { let args_owned = args .iter() .map(|arg| (*arg).to_string()) .collect::>(); let mut command = self.isolated_command(binary); - let output = command.args(args).output()?; + command.args(args).stdin(Stdio::null()); + // credentialを含むstdoutは呼出元へだけ返す。raw fileは作らない。 + let mut process = ManagedProcess::spawn_captured( + "setup command", + command, + None, + None, + MAX_COMMAND_OUTPUT_BYTES + 1, + )?; + let result = process.collect_output(timeout); + let output = result.as_ref().ok(); let mut log = OpenOptions::new() .create(true) .append(true) @@ -826,17 +1076,21 @@ impl WorkerE2eEnvironment { "ts_ms": now_ms(), "binary": binary, "args": args, - "status": output.status.code(), + "status": output.and_then(|output| output.status.code()), "stdout": if redact_stdout { "".to_string() } else { - bounded_text(&output.stdout, MAX_COMMAND_OUTPUT_BYTES) + output.map(|output| bounded_text(&output.stdout, MAX_COMMAND_OUTPUT_BYTES)) + .unwrap_or_default() }, - "stderr": bounded_text(&output.stderr, MAX_COMMAND_OUTPUT_BYTES), + "stderr": output.map(|output| bounded_text(&output.stderr, MAX_COMMAND_OUTPUT_BYTES)) + .unwrap_or_default(), + "error": result.as_ref().err().map(ToString::to_string), "env_clear": true, }), )?; writeln!(log)?; + let output = result?; if !output.status.success() { return Err(HarnessError::CommandFailed { program: binary.to_path_buf(), @@ -908,16 +1162,52 @@ impl WorkerE2eEnvironment { } } +// The operation is a caller-owned static label, never a response field. Refused +// response bodies may contain secrets; do not even collect them for diagnostics. +fn decode_api_response(response: reqwest::blocking::Response, operation: &str) -> Result { + let status = response.status(); + if !status.is_success() { + return Err(HarnessError::Protocol(format!( + "{operation} returned HTTP {}", + status.as_u16() + ))); + } + let mut body = std::io::Read::take(response, MAX_COMMAND_OUTPUT_BYTES as u64 + 1); + let mut bytes = Vec::new(); + std::io::Read::read_to_end(&mut body, &mut bytes) + .map_err(|_| HarnessError::Protocol(format!("{operation}: response read failed")))?; + if bytes.len() > MAX_COMMAND_OUTPUT_BYTES { + return Err(HarnessError::Protocol(format!( + "{operation}: response byte budget exceeded" + ))); + } + serde_json::from_slice(&bytes).map_err(HarnessError::from) +} + impl Drop for WorkerE2eEnvironment { fn drop(&mut self) { - if let Some(mut runtime) = self.runtime.take() { - let _ = runtime.shutdown(EXIT_WAIT); - } - if let Some(mut server) = self.server.take() { - let _ = server.shutdown(EXIT_WAIT); - } if self.temp_root.is_some() { let _ = self.cleanup_inner(false); + } else if self.runtime.is_some() || self.server.is_some() { + let _ = self.shutdown(); + } + } +} + +fn shutdown_process( + process: &mut Option, + outcome: &mut WorkerProcessShutdownOutcome, +) { + if let Some(mut process) = process.take() { + let next = match process.shutdown(EXIT_WAIT) { + Ok(_) => WorkerProcessShutdownOutcome::Succeeded, + // shutdownはstatus／OSエラーだけを返し、captureしたstdout/stderrは含めない。 + Err(error) => WorkerProcessShutdownOutcome::Failed { + error: error.to_string(), + }, + }; + if !matches!(outcome, WorkerProcessShutdownOutcome::Failed { .. }) { + *outcome = next; } } } @@ -928,15 +1218,26 @@ struct ManagedProcess { child: Child, stdout: Arc>>, stderr: Arc>>, - readers: Vec>, + readers: Vec>>, } impl ManagedProcess { - fn spawn( + fn spawn(name: &str, command: Command, stdout_path: &Path, stderr_path: &Path) -> Result { + Self::spawn_captured( + name, + command, + Some(stdout_path.to_path_buf()), + Some(stderr_path.to_path_buf()), + MAX_PROCESS_LOG_BYTES, + ) + } + + fn spawn_captured( name: &str, mut command: Command, - stdout_path: &Path, - stderr_path: &Path, + stdout_path: Option, + stderr_path: Option, + capture_limit: usize, ) -> Result { command.stdout(Stdio::piped()).stderr(Stdio::piped()); let mut child = command.spawn()?; @@ -951,8 +1252,8 @@ impl ManagedProcess { let stdout = Arc::new(Mutex::new(Vec::new())); let stderr = Arc::new(Mutex::new(Vec::new())); let readers = vec![ - capture_stream(stdout_pipe, stdout_path.to_path_buf(), Arc::clone(&stdout)), - capture_stream(stderr_pipe, stderr_path.to_path_buf(), Arc::clone(&stderr)), + capture_stream(stdout_pipe, stdout_path, Arc::clone(&stdout), capture_limit), + capture_stream(stderr_pipe, stderr_path, Arc::clone(&stderr), capture_limit), ]; Ok(Self { name: name.to_string(), @@ -1023,64 +1324,161 @@ impl ManagedProcess { ) } - fn shutdown(&mut self, timeout: Duration) -> Result { - if let Some(status) = self.child.try_wait()? { - self.join_readers(); - return Ok(status); + fn collect_output(&mut self, timeout: Duration) -> Result { + let deadline = Instant::now() + timeout; + let result = self.wait_until(deadline); + let status = match result { + Ok(Some(status)) => status, + other => { + // timeoutでも所有childをkillしてreapする。出力は診断へコピーしない。 + let _ = self.kill_and_reap(EXIT_WAIT); + let _ = self.join_readers_until(Instant::now() + EXIT_WAIT); + return match other { + Err(error) => Err(error), + _ => Err(HarnessError::Protocol( + "setup command deadline exceeded".into(), + )), + }; + } + }; + // child終了後もpipeが保持される場合を、同じsetup deadlineで失敗にする。 + self.join_readers_until(deadline)?; + let stdout = self + .stdout + .lock() + .map_err(|_| HarnessError::Protocol("setup stdout capture poisoned".into()))? + .clone(); + let stderr = self + .stderr + .lock() + .map_err(|_| HarnessError::Protocol("setup stderr capture poisoned".into()))? + .clone(); + if stdout.len() > MAX_COMMAND_OUTPUT_BYTES || stderr.len() > MAX_COMMAND_OUTPUT_BYTES { + return Err(HarnessError::Protocol( + "setup command output byte budget exceeded".into(), + )); } - let signal_result = unsafe { libc::kill(self.child.id() as i32, libc::SIGTERM) }; - if signal_result != 0 { - return Err(io_error("send SIGTERM", std::io::Error::last_os_error())); - } - let start = Instant::now(); + Ok(std::process::Output { + status, + stdout, + stderr, + }) + } + + fn wait_until(&mut self, deadline: Instant) -> Result> { loop { if let Some(status) = self.child.try_wait()? { - self.join_readers(); - if status.success() { - return Ok(status); - } - return Err(HarnessError::Protocol(format!( - "{} did not complete graceful shutdown: {status}", - self.name - ))); + return Ok(Some(status)); } - if start.elapsed() >= timeout { - let _ = self.child.kill(); - let _ = self.child.wait(); - self.join_readers(); - return Err(HarnessError::Protocol(format!( - "{} did not exit within {timeout:?}", - self.name - ))); + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Ok(None); } - thread::sleep(Duration::from_millis(20)); + thread::sleep(remaining.min(Duration::from_millis(20))); } } - fn join_readers(&mut self) { - for reader in self.readers.drain(..) { - let _ = reader.join(); + fn kill_and_reap(&mut self, timeout: Duration) -> Result { + if let Ok(Some(status)) = self.child.try_wait() { + return Ok(status); } + // killと自然終了の競合ではtry_waitで回収を確認する。 + let _ = self.child.kill(); + self.wait_until(Instant::now() + timeout)?.ok_or_else(|| { + HarnessError::Protocol(format!( + "{} could not be reaped within {timeout:?}", + self.name + )) + }) + } + + fn shutdown(&mut self, timeout: Duration) -> Result { + let result = (|| { + if let Some(status) = self.child.try_wait()? { + return Ok(status); + } + let signal_result = unsafe { libc::kill(self.child.id() as i32, libc::SIGTERM) }; + if signal_result != 0 { + return Err(io_error("send SIGTERM", std::io::Error::last_os_error())); + } + self.wait_until(Instant::now() + timeout)?.ok_or_else(|| { + HarnessError::Protocol(format!("{} did not exit within {timeout:?}", self.name)) + }) + })(); + if result.is_err() { + let _ = self.kill_and_reap(EXIT_WAIT); + } + let readers = self.join_readers_until(Instant::now() + EXIT_WAIT); + let status = result?; + readers?; + self.require_successful_exit(status) + } + + fn require_successful_exit(&self, status: ExitStatus) -> Result { + if status.success() { + Ok(status) + } else { + Err(HarnessError::Protocol(format!( + "{} did not complete graceful shutdown: {status}", + self.name + ))) + } + } + + fn join_readers_until(&mut self, deadline: Instant) -> Result<()> { + let mut failed = false; + while !self.readers.is_empty() { + let mut index = 0; + while index < self.readers.len() { + if self.readers[index].is_finished() { + let reader = self.readers.swap_remove(index); + if !matches!(reader.join(), Ok(Ok(()))) { + failed = true; + } + } else { + index += 1; + } + } + if self.readers.is_empty() { + break; + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + // stdのblocking Readは取消せない。異常なpipe保持時はthreadをdetachし、 + // 呼出元の待機だけをboundedにする。子孫processの回収までは保証しない。 + self.readers.clear(); + return Err(HarnessError::Protocol(format!( + "{} output readers did not finish before deadline", + self.name + ))); + } + thread::sleep(remaining.min(Duration::from_millis(20))); + } + if failed { + return Err(HarnessError::Protocol(format!( + "{} output reader failed", + self.name + ))); + } + Ok(()) } } impl Drop for ManagedProcess { fn drop(&mut self) { - if self.child.try_wait().ok().flatten().is_none() { - let _ = self.child.kill(); - let _ = self.child.wait(); - } - self.join_readers(); + let _ = self.kill_and_reap(EXIT_WAIT); + let _ = self.join_readers_until(Instant::now() + EXIT_WAIT); } } fn capture_stream( mut stream: impl Read + Send + 'static, - path: PathBuf, + path: Option, shared: Arc>>, -) -> JoinHandle<()> { + capture_limit: usize, +) -> JoinHandle> { thread::spawn(move || { - let mut file = File::create(path).ok(); + let mut file = path.and_then(|path| File::create(path).ok()); let mut written = 0usize; let mut truncated = false; let mut buffer = [0u8; 4096]; @@ -1089,12 +1487,12 @@ fn capture_stream( Ok(0) => break, Ok(count) => count, Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, - Err(_) => break, + Err(error) => return Err(error), }; if let Ok(mut captured) = shared.lock() - && captured.len() < MAX_PROCESS_LOG_BYTES + && captured.len() < capture_limit { - let remaining = MAX_PROCESS_LOG_BYTES - captured.len(); + let remaining = capture_limit - captured.len(); captured.extend_from_slice(&buffer[..count.min(remaining)]); } if let Some(file) = file.as_mut() { @@ -1110,6 +1508,7 @@ fn capture_stream( } } } + Ok(()) }) } @@ -1145,12 +1544,55 @@ fn assert_absent_or_empty(path: &Path, label: &str) -> Result<()> { ))) } -fn json_string(value: &Value, pointer: &str) -> Result { +fn require_compatible_connection(connection: &Value) -> Result<()> { + if connection.get("status").and_then(Value::as_str) != Some("compatible") { + return Err(HarnessError::Protocol( + "Runtime connection was not compatible".to_owned(), + )); + } + Ok(()) +} + +fn require_registered_runtime(runtimes: &Value, runtime_id: &str, binding_id: &str) -> Result<()> { + let connected = runtimes + .get("items") + .and_then(Value::as_array) + .is_some_and(|items| { + items.iter().any(|item| { + item.get("runtime_id").and_then(Value::as_str) == Some(runtime_id) + && item + .pointer("/management/binding/binding_id") + .and_then(Value::as_str) + == Some(binding_id) + }) + }); + if !connected { + return Err(HarnessError::Protocol( + "Runtime registration did not match the verified binding".to_owned(), + )); + } + Ok(()) +} + +fn require_empty_worker_list(workers: &Value) -> Result<()> { + if workers + .get("items") + .and_then(Value::as_array) + .is_none_or(|items| !items.is_empty()) + { + return Err(HarnessError::Protocol( + "fresh Worker list is not empty or has invalid shape".to_owned(), + )); + } + Ok(()) +} + +fn json_string(value: &Value, pointer: &'static str) -> Result { value .pointer(pointer) .and_then(Value::as_str) .map(str::to_owned) - .ok_or_else(|| HarnessError::Protocol(format!("JSON response omitted {pointer}: {value}"))) + .ok_or_else(|| HarnessError::Protocol(format!("JSON response omitted string at {pointer}"))) } fn url_port(url: &str) -> Result { @@ -1175,7 +1617,357 @@ fn io_error(operation: &str, error: std::io::Error) -> HarnessError { #[cfg(test)] mod startup_address_tests { - use super::parse_address_after; + use super::*; + + // 製品binary providerを呼ばず、同じ所有・収集・cleanup経路へ合成childだけを接続する。 + fn synthetic_environment(artifacts_dir: &Path) -> Result { + let temp_root = tempfile::tempdir()?; + let root = temp_root.path().to_path_buf(); + fs::create_dir_all(artifacts_dir)?; + Ok(WorkerE2eEnvironment { + temp_root: Some(temp_root), + home: root.clone(), + xdg_data_home: root.clone(), + xdg_state_home: root.clone(), + xdg_config_home: root.clone(), + xdg_runtime_dir: root.clone(), + server_data_dir: root.join("server"), + runtime_data_dir: root.join("runtime"), + server_config: root.join("server.toml"), + artifacts_dir: artifacts_dir.to_path_buf(), + workspace_id: None, + runtime_id: "synthetic-runtime".into(), + account_handle: "synthetic-account".into(), + server_url: None, + runtime_url: None, + server: None, + runtime: None, + runtime_shutdown: WorkerProcessShutdownOutcome::NotStarted, + server_shutdown: WorkerProcessShutdownOutcome::NotStarted, + shutdown_error: None, + runtime_public_identity: None, + access_token: None, + client: Client::builder().no_proxy().build().unwrap(), + binaries: WorkerBinaryProviderInfo { + workspace_root: root.clone(), + profile: "synthetic".into(), + binaries: Vec::new(), + }, + root, + }) + } + + fn synthetic_process(root: &Path, name: &str, script: &str) -> Result { + let mut command = Command::new("/bin/sh"); + command + .env_clear() + .current_dir(root) + .stdin(Stdio::null()) + .args(["-c", script]); + ManagedProcess::spawn_captured(name, command, None, None, MAX_COMMAND_OUTPUT_BYTES + 1) + } + + fn assert_secret_free(diagnostic: impl std::fmt::Debug) { + assert!(!format!("{diagnostic:?}").contains("private-test-marker")); + } + + #[test] + fn setup_stdout_is_returned_but_redacted_in_logs_and_nonzero_errors() -> Result<()> { + let artifacts = tempfile::tempdir()?; + let environment = synthetic_environment(artifacts.path())?; + // script自体はsecretの完全値を含めず、stdoutの漏洩だけを検出する。 + let script = "printf 'private-%s' test-marker"; + let output = environment.run_command(Path::new("/bin/sh"), &["-c", script], true)?; + assert_eq!(output.stdout, b"private-test-marker"); + let error = environment + .run_command( + Path::new("/bin/sh"), + &["-c", "printf 'private-%s' test-marker; exit 7"], + true, + ) + .unwrap_err(); + assert!( + matches!(&error, HarnessError::CommandFailed { status, stdout, .. } + if status.code() == Some(7) && stdout == "") + ); + assert_secret_free(&error); + assert_secret_free(error.to_string()); + let log = fs::read_to_string(artifacts.path().join("setup-commands.jsonl"))?; + assert_secret_free(&log); + for line in log.lines() { + let record: Value = serde_json::from_str(line)?; + assert_eq!(record["stdout"], ""); + } + // setup経路はraw stdout/stderr fileを作らない。 + assert_eq!(fs::read_dir(artifacts.path())?.count(), 1); + let report = environment.cleanup()?; + assert!(report.cleanup_success && report.root_cleanup_success); + Ok(()) + } + + #[test] + fn setup_deadline_kills_and_reaps_owned_child_without_logging_stdout() -> Result<()> { + let artifacts = tempfile::tempdir()?; + let environment = synthetic_environment(artifacts.path())?; + let script = "trap '' TERM; printf 'private-%s' test-marker; while :; do :; done"; + let timeout = Duration::from_millis(100); + let mut process = synthetic_process(&environment.root, "setup command", script)?; + let started = Instant::now(); + let error = process.collect_output(timeout).unwrap_err(); + assert!(error.to_string().contains("deadline exceeded")); + assert_secret_free(&error); + assert!(started.elapsed() < timeout + EXIT_WAIT + EXIT_WAIT); + assert!(process.child.try_wait()?.is_some()); + assert!(process.readers.is_empty()); + let error = environment + .run_command_with_timeout(Path::new("/bin/sh"), &["-c", script], true, timeout) + .unwrap_err(); + assert_secret_free(&error); + let log = fs::read_to_string(artifacts.path().join("setup-commands.jsonl"))?; + assert_secret_free(&log); + let record: Value = serde_json::from_str(log.trim())?; + assert_eq!(record["stdout"], ""); + assert!( + record["error"] + .as_str() + .unwrap() + .contains("deadline exceeded") + ); + assert_eq!(fs::read_dir(artifacts.path())?.count(), 1); + Ok(()) + } + + #[test] + fn setup_output_overflow_is_bounded_drained_and_rejected() -> Result<()> { + let root = tempfile::tempdir()?; + let mut process = synthetic_process( + root.path(), + "setup command", + "i=0; while [ \"$i\" -lt 20000 ]; do printf 0123456789abcdef; printf fedcba9876543210 >&2; i=$((i + 1)); done", + )?; + let error = process.collect_output(SETUP_WAIT).unwrap_err(); + assert!(error.to_string().contains("output byte budget exceeded")); + // 両pipeを上限後もdrainしなければこの成功exitまで到達しない。 + assert!(process.child.try_wait()?.unwrap().success()); + assert!(process.readers.is_empty()); + assert_eq!( + process.stdout.lock().unwrap().len(), + MAX_COMMAND_OUTPUT_BYTES + 1 + ); + assert_eq!( + process.stderr.lock().unwrap().len(), + MAX_COMMAND_OUTPUT_BYTES + 1 + ); + assert_eq!(fs::read_dir(root.path())?.count(), 0); + Ok(()) + } + + #[test] + fn setup_output_at_byte_limit_is_returned_without_truncation() -> Result<()> { + let root = tempfile::tempdir()?; + let mut process = synthetic_process( + root.path(), + "setup command", + "i=0; while [ \"$i\" -lt 4096 ]; do printf 0123456789abcdef; i=$((i + 1)); done", + )?; + let output = process.collect_output(SETUP_WAIT)?; + assert!(output.status.success()); + assert_eq!(output.stdout.len(), MAX_COMMAND_OUTPUT_BYTES); + assert!( + output + .stdout + .chunks_exact(16) + .all(|chunk| chunk == b"0123456789abcdef") + ); + assert!(output.stderr.is_empty()); + Ok(()) + } + + #[test] + fn failed_output_reader_rejects_successful_child_without_raw_error() -> Result<()> { + struct FailedReader; + impl Read for FailedReader { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + Err(std::io::Error::other("private-test-marker")) + } + } + let root = tempfile::tempdir()?; + let mut process = synthetic_process(root.path(), "setup command", "exit 0")?; + process.readers.push(capture_stream( + FailedReader, + None, + Arc::new(Mutex::new(Vec::new())), + MAX_COMMAND_OUTPUT_BYTES + 1, + )); + let error = process.collect_output(SETUP_WAIT).unwrap_err(); + assert!(error.to_string().contains("output reader failed")); + assert_secret_free(&error); + assert_secret_free(error.to_string()); + assert!(process.child.try_wait()?.unwrap().success()); + Ok(()) + } + + #[test] + fn unfinished_output_reader_has_bounded_join_and_is_not_success() -> Result<()> { + let root = tempfile::tempdir()?; + let mut process = synthetic_process(root.path(), "setup command", "exit 0")?; + assert!( + process + .wait_until(Instant::now() + EXIT_WAIT)? + .unwrap() + .success() + ); + // 子孫を残さず、pipe保持と同じ未完readerをchannelで決定的に再現する。 + let (release, pending) = std::sync::mpsc::channel::<()>(); + let (finished, completion) = std::sync::mpsc::channel(); + process.readers.push(thread::spawn(move || { + let _ = pending.recv(); + let _ = finished.send(()); + Ok(()) + })); + let timeout = Duration::from_millis(50); + let started = Instant::now(); + let error = process.collect_output(timeout).unwrap_err(); + drop(release); + completion.recv_timeout(EXIT_WAIT).unwrap(); + assert!( + error + .to_string() + .contains("readers did not finish before deadline") + ); + assert!(started.elapsed() < timeout + EXIT_WAIT); + assert!(process.readers.is_empty()); + Ok(()) + } + + #[test] + fn runtime_failure_still_reaps_live_server_and_survives_repeated_shutdown() -> Result<()> { + let artifacts = tempfile::tempdir()?; + let mut environment = synthetic_environment(artifacts.path())?; + let mut runtime = synthetic_process(&environment.root, "runtime", "exit 7")?; + assert_eq!( + runtime + .wait_until(Instant::now() + EXIT_WAIT)? + .unwrap() + .code(), + Some(7) + ); + environment.runtime = Some(runtime); + let server = synthetic_process( + &environment.root, + "server", + "trap 'exit 0' TERM; printf 'ready 127.0.0.1:12345\\n'; while :; do :; done", + )?; + server.wait_for_address("ready ", EXIT_WAIT)?; + environment.server = Some(server); + for _ in 0..2 { + assert!(environment.shutdown().is_err()); + assert!(environment.runtime.is_none() && environment.server.is_none()); + assert!(matches!( + environment.runtime_shutdown, + WorkerProcessShutdownOutcome::Failed { .. } + )); + assert_eq!( + environment.server_shutdown, + WorkerProcessShutdownOutcome::Succeeded + ); + } + let root = environment.root.clone(); + assert!(environment.cleanup().is_err()); + assert!(!root.exists()); + let report: WorkerEnvironmentCleanupReport = + serde_json::from_slice(&fs::read(artifacts.path().join("cleanup.json"))?)?; + assert!(!report.cleanup_success); + assert!(report.root_cleanup_success); + assert!(report.cleanup_error.is_some()); + Ok(()) + } + + #[test] + fn cleanup_and_drop_keep_process_failures_separate_from_root_cleanup() -> Result<()> { + for explicit_cleanup in [false, true] { + for (runtime_code, server_code) in [(0, 0), (7, 0), (0, 9), (7, 9)] { + let artifacts = tempfile::tempdir()?; + let mut environment = synthetic_environment(artifacts.path())?; + for (name, code) in [("runtime", runtime_code), ("server", server_code)] { + let mut process = synthetic_process( + &environment.root, + name, + &format!("printf 'private-%s' test-marker; exit {code}"), + )?; + assert_eq!( + process + .wait_until(Instant::now() + EXIT_WAIT)? + .unwrap() + .code(), + Some(code) + ); + if name == "runtime" { + environment.runtime = Some(process); + } else { + environment.server = Some(process); + } + } + let root = environment.root.clone(); + let expected_success = runtime_code == 0 && server_code == 0; + if explicit_cleanup { + assert_eq!(environment.cleanup().is_ok(), expected_success); + } else { + drop(environment); + } + let raw_report = fs::read_to_string(artifacts.path().join("cleanup.json"))?; + assert_secret_free(&raw_report); + let report: WorkerEnvironmentCleanupReport = serde_json::from_str(&raw_report)?; + assert_eq!(report.cleanup_success, expected_success); + assert_eq!(report.cleanup_error.is_none(), expected_success); + assert!(report.root_cleanup_success); + assert!(report.root_cleanup_error.is_none()); + assert!(!report.fixture_root_exists_after && !root.exists()); + for (outcome, code) in [ + (report.runtime_shutdown, runtime_code), + (report.server_shutdown, server_code), + ] { + if code == 0 { + assert_eq!(outcome, WorkerProcessShutdownOutcome::Succeeded); + } else { + assert!(matches!( + outcome, + WorkerProcessShutdownOutcome::Failed { .. } + )); + } + } + } + } + Ok(()) + } + + #[test] + fn shutdown_rejects_nonzero_exit_even_when_process_already_exited() -> super::Result<()> { + let root = tempfile::tempdir()?; + for code in [0, 7] { + let mut command = std::process::Command::new("/bin/sh"); + command + .env_clear() + .current_dir(root.path()) + .args(["-c", &format!("exit {code}")]); + let mut process = super::ManagedProcess::spawn( + "already-exited", + command, + &root.path().join(format!("{code}.stdout")), + &root.path().join(format!("{code}.stderr")), + )?; + // Reap first to deterministically exercise the initial try_wait branch. + assert_eq!(process.child.wait()?.code(), Some(code)); + let result = process.shutdown(std::time::Duration::from_secs(1)); + if code == 0 { + assert!(result?.success()); + } else { + assert!(matches!(result, Err(super::HarnessError::Protocol(message)) + if message.contains("did not complete graceful shutdown"))); + } + } + Ok(()) + } #[test] fn startup_address_waits_for_complete_valid_socket_record() { @@ -1205,3 +1997,107 @@ mod startup_address_tests { } } } + +#[cfg(test)] +mod response_diagnostic_tests { + use super::*; + use serde_json::json; + use std::io::{Read, Write}; + use std::net::TcpListener; + + fn assert_secret_free(error: &HarnessError) { + for diagnostic in [error.to_string(), format!("{error:?}")] { + assert!(!diagnostic.contains("private-test-marker"), "{diagnostic}"); + } + assert!(std::error::Error::source(error).is_none()); + } + + #[test] + fn json_decode_errors_do_not_retain_rejected_fields_values_or_enum_names() { + #[derive(Debug, Deserialize)] + #[serde(deny_unknown_fields)] + struct Document { + #[allow(dead_code)] + count: u64, + } + for raw in [ + r#"{"count":"private-test-marker"}"#, + r#"{"count":1,"private-test-marker":true}"#, + ] { + let error: HarnessError = serde_json::from_str::(raw).unwrap_err().into(); + assert_secret_free(&error); + } + let raw = serde_json::json!({"kind":"private-test-marker"}); + let error: HarnessError = serde_json::from_value::(raw) + .unwrap_err() + .into(); + assert_secret_free(&error); + } + + #[test] + fn valid_json_contract_failures_do_not_retain_response_fields_or_values() -> Result<()> { + let response: Value = serde_json::from_str( + r#"{"status":"private-test-marker","items":[{"private-test-marker":"private-test-marker"}],"binding_id":{ "private-test-marker":true }}"#, + )?; + for error in [ + require_compatible_connection(&response).unwrap_err(), + require_registered_runtime(&response, "runtime-test", "binding-test").unwrap_err(), + require_empty_worker_list(&response).unwrap_err(), + json_string(&response, "/binding_id").unwrap_err(), + json_string(&response, "/workspace/workspace_id").unwrap_err(), + ] { + assert_secret_free(&error); + } + require_compatible_connection(&json!({"status":"compatible"}))?; + require_registered_runtime( + &json!({"items":[{"runtime_id":"runtime-test","management":{"binding":{"binding_id":"binding-test"}}}]}), + "runtime-test", + "binding-test", + )?; + require_empty_worker_list(&json!({"items":[]}))?; + assert_eq!( + json_string(&json!({"id":"worker-test"}), "/id")?, + "worker-test" + ); + Ok(()) + } + + #[test] + fn http_failure_bodies_and_malformed_success_bodies_are_not_diagnostics() -> Result<()> { + for (status, body) in [ + ("400 Bad Request", "private-test-marker"), + ("200 OK", "private-test-marker"), + ] { + let listener = TcpListener::bind("127.0.0.1:0")?; + let address = listener.local_addr()?; + let server = thread::spawn(move || -> std::io::Result<()> { + let (mut connection, _) = listener.accept()?; + connection.set_read_timeout(Some(Duration::from_secs(3)))?; + let mut request = [0; 4096]; + connection.read(&mut request)?; + write!( + connection, + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + )?; + Ok(()) + }); + let response = Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(3)) + .build() + .unwrap() + .get(format!("http://{address}")) + .send() + .unwrap(); + let error = decode_api_response(response, "test response").unwrap_err(); + server.join().unwrap()?; + assert_secret_free(&error); + if status.starts_with("400") { + assert!(error.to_string().contains("HTTP 400")); + } + } + Ok(()) + } +} diff --git a/tests/e2e/src/worker_session.rs b/tests/e2e/src/worker_session.rs new file mode 100644 index 000000000..fb2a454e4 --- /dev/null +++ b/tests/e2e/src/worker_session.rs @@ -0,0 +1,284 @@ +//! Live public Worker protocol assertions. No Submit/Resume/model request is sent. +use std::net::TcpStream; +use std::time::{Duration, Instant}; + +use protocol::{Event, SessionSnapshot, WorkerState}; +use tungstenite::{Message, WebSocket, client::IntoClientRequest}; + +use crate::{HarnessError, Result, WorkerE2eEnvironment}; + +const WS_WAIT: Duration = Duration::from_secs(10); + +pub struct WorkerSessionSubscription { + socket: WebSocket, +} + +impl WorkerSessionSubscription { + pub fn connect( + environment: &WorkerE2eEnvironment, + worker_id: &str, + expected: &SessionSnapshot, + ) -> Result { + let authorized = environment.worker_protocol_request(worker_id)?; + let mut url = authorized.url().clone(); + url.set_scheme("ws") + .map_err(|_| error("invalid protocol URL"))?; + let mut request = url + .as_str() + .into_client_request() + .map_err(|_| error("build protocol handshake"))?; + request.headers_mut().extend(authorized.headers().clone()); + // Only this environment's loopback endpoint is permitted, never a proxy. + let address = url + .socket_addrs(|| None) + .map_err(|_| error("resolve protocol address"))? + .into_iter() + .find(|address| address.ip().is_loopback()) + .ok_or_else(|| error("protocol address is not loopback"))?; + let stream = TcpStream::connect_timeout(&address, WS_WAIT)?; + stream.set_read_timeout(Some(WS_WAIT))?; + stream.set_write_timeout(Some(WS_WAIT))?; + let (socket, _) = + tungstenite::client(request, stream).map_err(|_| error("protocol handshake failed"))?; + let mut subscription = Self { socket }; + let deadline = Instant::now() + WS_WAIT; + loop { + match subscription.read(deadline)? { + Message::Text(text) => { + let raw: serde_json::Value = serde_json::from_str(&text)?; + validate_full_snapshot(&raw, expected)?; + // Only validated public content is retained, never request headers. + std::fs::write( + environment.artifacts_dir.join("restored-session.json"), + serde_json::to_vec_pretty(&raw)?, + )?; + return Ok(subscription); + } + Message::Ping(_) => subscription + .socket + .flush() + .map_err(|_| error("flush protocol Pong"))?, + Message::Pong(_) => {} + _ => return Err(error("protocol ended before its initial full Snapshot")), + } + } + } + + /// Prove an already-open subscription remains usable after the other + /// environment is stopped and cleaned up. This does not invoke the model. + pub fn assert_alive(&mut self) -> Result<()> { + let payload = b"restore-e2e-isolation".to_vec(); + self.socket + .send(Message::Ping(payload.clone().into())) + .map_err(|_| error("send protocol Ping"))?; + let deadline = Instant::now() + WS_WAIT; + loop { + match self.read(deadline)? { + Message::Pong(bytes) if bytes.as_ref() == payload.as_slice() => return Ok(()), + Message::Ping(_) => self + .socket + .flush() + .map_err(|_| error("flush protocol Pong"))?, + Message::Pong(_) => {} + _ => { + return Err(error( + "idle subscription changed or ended during isolation check", + )); + } + } + } + } + + /// A public stop must terminate the connected execution, not merely update + /// a retained catalog row. Require the shutdown event or websocket close. + pub fn expect_stopped(mut self) -> Result<()> { + let deadline = Instant::now() + WS_WAIT; + loop { + match self.read(deadline)? { + Message::Close(_) => return Ok(()), + Message::Text(text) => match serde_json::from_str::(&text)? { + Event::Shutdown => return Ok(()), + Event::WorkerState { .. } => {} + _ => return Err(error("unexpected protocol event after Worker stop")), + }, + Message::Ping(_) => self + .socket + .flush() + .map_err(|_| error("flush protocol Pong"))?, + Message::Pong(_) => {} + _ => return Err(error("unexpected protocol frame after Worker stop")), + } + } + } + + fn read(&mut self, deadline: Instant) -> Result { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|duration| !duration.is_zero()) + .ok_or_else(|| error("protocol deadline exceeded"))?; + self.socket.get_mut().set_read_timeout(Some(remaining))?; + self.socket + .read() + .map_err(|_| error("protocol read failed or timed out")) + } +} + +fn error(message: &str) -> HarnessError { + HarnessError::Protocol(message.into()) +} + +pub fn validate_full_snapshot(raw: &serde_json::Value, expected: &SessionSnapshot) -> Result<()> { + let event: Event = serde_json::from_value(raw.clone())?; + // Product DTOs intentionally tolerate unknown fields for compatibility. This + // fixture boundary does not: anything discarded by typed decoding must never + // be persisted to an artifact, including fields outside the Session payload. + if serde_json::to_value(&event)? != *raw { + return Err(error( + "noncanonical or unknown fields in full protocol Snapshot envelope", + )); + } + let Event::Snapshot { + session, + state, + in_flight, + internal_workers, + .. + } = event + else { + return Err(error("first protocol event is not a full Snapshot")); + }; + if state.state != WorkerState::Idle || !in_flight.is_empty() || !internal_workers.is_empty() { + return Err(error( + "restored Worker is not an idle, completed standalone Session", + )); + } + // Typed equality covers entry order/ID/timestamp/provenance/content; raw + // equality also refuses unknown fields that could leak private data. + if session != *expected + || raw.pointer("/data/session") != Some(&serde_json::to_value(expected)?) + { + return Err(error( + "full Session snapshot differs from supplied expected rows", + )); + } + if !session.pending_submissions.submissions.is_empty() { + return Err(error("fixture has pending submissions")); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use protocol::{ + Greeting, InFlightSnapshot, Segment, SessionEntryProvenance, SessionSnapshotEntry, + SessionSnapshotEntryData, WorkerStateSnapshot, WorkerStatus, + }; + + // Protocol-validation samples only, not a substitute saved Worker fixture. + fn sample() -> (serde_json::Value, SessionSnapshot) { + let session = SessionSnapshot { + pending_submissions: Default::default(), + entries: vec![SessionSnapshotEntry { + entry_id: "E-unit-1".into(), + timestamp: 123, + provenance: SessionEntryProvenance::HumanInput, + derived_from: vec![], + use_policy: None, + data: SessionSnapshotEntryData::UserInput { + segments: vec![Segment::text("unit input")], + }, + }], + }; + let event = Event::Snapshot { + session: session.clone(), + state: WorkerStateSnapshot::from(WorkerStatus::Idle), + in_flight: InFlightSnapshot::default(), + internal_workers: vec![], + greeting: Greeting { + worker_name: "unit".into(), + cwd: ".".into(), + provider: "unit".into(), + model: "unit".into(), + reasoning: None, + scope_summary: "none".into(), + tools: vec![], + context_window: 0, + context_tokens: 0, + context_usage: None, + }, + }; + (serde_json::to_value(event).unwrap(), session) + } + + #[test] + fn full_snapshot_requires_exact_ids_timestamps_contents_and_no_unknown_fields() { + let (raw, expected) = sample(); + validate_full_snapshot(&raw, &expected).unwrap(); + for (key, value) in [ + ("entry_id", serde_json::json!("different")), + ("timestamp", serde_json::json!(124)), + ( + "hidden_reasoning", + serde_json::json!("must never be exposed"), + ), + ("segments", serde_json::json!([])), + ] { + let mut changed = raw.clone(); + changed["data"]["session"]["entries"][0][key] = value; + assert!( + validate_full_snapshot(&changed, &expected).is_err(), + "{key}" + ); + } + let mut missing = raw.clone(); + missing["data"]["session"]["entries"] = serde_json::json!([]); + assert!(validate_full_snapshot(&missing, &expected).is_err()); + let mut extra = raw; + let entry = extra["data"]["session"]["entries"][0].clone(); + extra["data"]["session"]["entries"] + .as_array_mut() + .unwrap() + .push(entry); + assert!(validate_full_snapshot(&extra, &expected).is_err()); + } + + #[test] + fn full_snapshot_rejects_unknown_fields_outside_session_before_artifact_write() { + let (raw, expected) = sample(); + for pointer in [ + "", + "/data", + "/data/greeting", + "/data/state", + "/data/state/state", + ] { + let mut changed = raw.clone(); + changed + .pointer_mut(pointer) + .unwrap() + .as_object_mut() + .unwrap() + .insert( + "hidden_reasoning".into(), + serde_json::json!("must not reach artifact"), + ); + assert!( + validate_full_snapshot(&changed, &expected).is_err(), + "{pointer}" + ); + } + let mut changed = raw; + changed["data"]["greeting"]["private_key"] = serde_json::json!("must not reach artifact"); + assert!(validate_full_snapshot(&changed, &expected).is_err()); + } + + #[test] + fn non_snapshot_event_cannot_stand_in_for_live_snapshot() { + let (_, expected) = sample(); + assert!( + validate_full_snapshot(&serde_json::to_value(Event::Shutdown).unwrap(), &expected) + .is_err() + ); + } +} diff --git a/tests/e2e/tests/panel.rs b/tests/e2e/tests/panel.rs index 058e27827..87636bb62 100644 --- a/tests/e2e/tests/panel.rs +++ b/tests/e2e/tests/panel.rs @@ -41,13 +41,13 @@ fn backend_panel_loads_backend_owned_tickets_and_supports_reload_navigation_and_ &event.data, &ready_ticket_id, "Planning E2E Ticket A", - "planning", + "open", ) && has_backend_ticket( &event.data, &planning_ticket_id, "Planning E2E Ticket B", - "planning", + "open", ) }, )?; @@ -82,13 +82,13 @@ fn backend_panel_loads_backend_owned_tickets_and_supports_reload_navigation_and_ &event.data, &ready_ticket_id, "Planning E2E Ticket A", - "planning", + "open", ) && has_backend_ticket( &event.data, &planning_ticket_id, "Planning E2E Ticket B", - "planning", + "open", ) }, )?; @@ -137,7 +137,7 @@ fn backend_panel_shell_enter_path_uses_the_same_backend_authority() -> yoi_e2e:: &event.data, &ready_ticket_id, "Planning E2E Ticket A", - "planning", + "open", ) }, )?; diff --git a/tests/e2e/tests/worker_restore.rs b/tests/e2e/tests/worker_restore.rs new file mode 100644 index 000000000..3d1619847 --- /dev/null +++ b/tests/e2e/tests/worker_restore.rs @@ -0,0 +1,274 @@ +//! Fixed, externally supplied Worker restore. Never generates a fixture or sends model input. +#[path = "worker_restore/fixture.rs"] +mod fixture; +#[path = "worker_restore/seed.rs"] +mod seed; + +use std::fs; +use std::time::{Duration, Instant}; + +use fixture::{Fixture, failure}; +use seed::Installed; +use server_api::{ + RuntimeWorkerLifecycleResult, RuntimeWorkerShowResponse, WorkerListResponse, + WorkerOperationState, WorkerRestoreResponse, WorkerRestoreState, WorkerSummary, +}; +use yoi_e2e::{FixtureSource, Result, WorkerE2eEnvironment, WorkerSessionSubscription}; + +#[test] +fn supplied_stopped_worker_restores_exact_live_session_and_cleans_up() -> Result<()> { + let fixture = Fixture::load()?; + let result = (|| { + let installed = seed::start(&fixture)?; + let subscription = restore(&installed, &fixture, "fixed-fixture-restore-single")?; + stop_and_cleanup(installed, vec![subscription], &fixture) + })(); + fixture.source.assert_unchanged()?; + result +} + +#[test] +fn concurrent_fixed_id_restores_keep_sessions_storage_and_cleanup_isolated() -> Result<()> { + let fixture = Fixture::load()?; + let result = (|| { + let (first, second) = std::thread::scope(|scope| { + let first = scope.spawn(|| seed::start(&fixture)); + let second = seed::start(&fixture); + ( + first + .join() + .map_err(|_| failure("first environment startup panicked")) + .and_then(|result| result), + second, + ) + }); + let first = first?; + let second = second?; + assert_eq!( + first.environment.workspace_id, + second.environment.workspace_id + ); + assert_eq!(first.environment.runtime_id, second.environment.runtime_id); + assert_ne!(first.environment.root, second.environment.root); + assert_ne!( + first.environment.server_data_dir, + second.environment.server_data_dir + ); + assert_ne!( + first.environment.runtime_data_dir, + second.environment.runtime_data_dir + ); + assert_ne!( + first.environment.artifacts_dir, + second.environment.artifacts_dir + ); + assert_ne!( + first.environment.account_handle, + second.environment.account_handle + ); + assert_ne!(first.runtime_public_key, second.runtime_public_key); + assert_ne!(first.workspace_public_key, second.workspace_public_key); + assert_ne!( + first.environment.server_port()?, + second.environment.server_port()? + ); + assert_ne!( + first.environment.runtime_port()?, + second.environment.runtime_port()? + ); + assert_ne!( + first.environment.server_pid(), + second.environment.server_pid() + ); + assert_ne!( + first.environment.runtime_pid(), + second.environment.runtime_pid() + ); + first + .environment + .assert_authentication_isolated_from(&second.environment)?; + second + .environment + .assert_authentication_isolated_from(&first.environment)?; + let (first_subscription, second_subscription) = std::thread::scope(|scope| { + // Deliberately reuse request identity in isolated stores, just like fixed IDs. + let a = scope.spawn(|| restore(&first, &fixture, "fixed-fixture-restore-concurrent")); + let b = restore(&second, &fixture, "fixed-fixture-restore-concurrent"); + ( + a.join() + .map_err(|_| failure("first restore panicked")) + .and_then(|result| result), + b, + ) + }); + let first_subscription = first_subscription?; + let mut second_subscription = second_subscription?; + let second_session_root = second + .environment + .runtime_data_dir + .join("runtime/workers") + .join(&fixture.manifest.worker_id) + .join("session"); + let second_session = FixtureSource::capture(&second_session_root)?; + let first_root = first.environment.root.clone(); + let first_artifacts = first.environment.artifacts_dir.clone(); + stop_and_cleanup(first, vec![first_subscription], &fixture)?; + assert!(!first_root.exists()); + assert!(first_artifacts.join("cleanup.json").is_file()); + assert!(second.environment.root.is_dir()); + assert!(second.environment.server_data_dir.is_dir()); + assert!(second.environment.runtime_data_dir.is_dir()); + assert!( + second + .environment + .artifacts_dir + .join("restored-session.json") + .is_file() + ); + second.environment.assert_connected()?; + assert_single_state(&second.environment, &fixture, "idle")?; + second_subscription.assert_alive()?; + // A second live connection independently verifies content after peer cleanup. + let mut reconnected = WorkerSessionSubscription::connect( + &second.environment, + &fixture.manifest.worker_id, + &second.snapshot, + )?; + reconnected.assert_alive()?; + second_session.assert_unchanged()?; + stop_and_cleanup(second, vec![second_subscription, reconnected], &fixture)?; + Ok(()) + })(); + fixture.source.assert_unchanged()?; + result +} + +fn restore( + installed: &Installed, + fixture: &Fixture, + request_id: &str, +) -> Result { + phase(&installed.environment, "observe-stopped")?; + let stopped = assert_single_state(&installed.environment, fixture, "stopped")?; + let token = stopped + .restore_observation_token + .as_deref() + .filter(|token| !token.is_empty()) + .ok_or_else(|| failure("stopped Worker did not expose a restore observation token"))?; + phase(&installed.environment, "restore")?; + let result: WorkerRestoreResponse = serde_json::from_value( + installed + .environment + .restore_worker(&fixture.manifest.worker_id, request_id, token)?, + )?; + if result.workspace_id != fixture.manifest.workspace_id + || result.runtime_id != fixture.manifest.runtime_id + || result.worker_id != fixture.manifest.worker_id + || result.result.state != WorkerRestoreState::Accepted + { + return Err(failure(format!( + "restore did not commit accepted: {:?}", + result.result.state + ))); + } + assert_single_state(&installed.environment, fixture, "idle")?; + let detail: RuntimeWorkerShowResponse = serde_json::from_value( + installed + .environment + .worker_detail(&fixture.manifest.worker_id)?, + )?; + if detail.worker.worker_id != fixture.manifest.worker_id + || detail.worker.runtime_id != fixture.manifest.runtime_id + || detail.worker.state != "idle" + || detail + .worker + .worker_state + .as_ref() + .is_none_or(|state| state.state != protocol::WorkerState::Idle) + { + return Err(failure( + "restored Worker detail is not the single live idle execution", + )); + } + phase(&installed.environment, "subscribe-live-session")?; + let mut subscription = WorkerSessionSubscription::connect( + &installed.environment, + &fixture.manifest.worker_id, + &installed.snapshot, + )?; + subscription.assert_alive()?; + phase(&installed.environment, "live-session-verified")?; + Ok(subscription) +} + +fn assert_single_state( + environment: &WorkerE2eEnvironment, + fixture: &Fixture, + state: &str, +) -> Result { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let list: WorkerListResponse = serde_json::from_value(environment.list_workers()?)?; + if list.workspace_id != fixture.manifest.workspace_id || list.items.len() > 1 { + return Err(failure( + "Workspace Worker list has wrong identity or duplicate Workers", + )); + } + if let Some(worker) = list.items.into_iter().next() { + if worker.worker_id != fixture.manifest.worker_id + || worker.runtime_id != fixture.manifest.runtime_id + { + return Err(failure("Workspace listed an unexpected Worker identity")); + } + if worker.state == state { + return Ok(worker); + } + } + if Instant::now() >= deadline { + return Err(failure(format!( + "Worker list did not become exactly one {state} Worker" + ))); + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn stop_and_cleanup( + installed: Installed, + subscriptions: Vec, + fixture: &Fixture, +) -> Result<()> { + phase(&installed.environment, "stop")?; + let result: RuntimeWorkerLifecycleResult = serde_json::from_value( + installed + .environment + .stop_worker(&fixture.manifest.worker_id)?, + )?; + if result.state != WorkerOperationState::Accepted { + return Err(failure("Worker stop was not accepted")); + } + installed.environment.assert_connected()?; + for subscription in subscriptions { + subscription.expect_stopped()?; + // A transport close caused by Server/Runtime shutdown is not stop evidence. + installed.environment.assert_connected()?; + } + assert_single_state(&installed.environment, fixture, "stopped")?; + phase(&installed.environment, "shutdown-cleanup")?; + let report = installed.environment.cleanup()?; + if !report.cleanup_success || report.fixture_root_exists_after { + return Err(failure("environment cleanup failed")); + } + if !report.report_path.is_file() { + return Err(failure("cleanup evidence is missing")); + } + Ok(()) +} + +fn phase(environment: &WorkerE2eEnvironment, phase: &str) -> Result<()> { + fs::write( + environment.artifacts_dir.join("restore-phase.json"), + serde_json::to_vec_pretty(&serde_json::json!({"phase":phase}))?, + )?; + Ok(()) +} diff --git a/tests/e2e/tests/worker_restore/fixture.rs b/tests/e2e/tests/worker_restore/fixture.rs new file mode 100644 index 000000000..707a159fe --- /dev/null +++ b/tests/e2e/tests/worker_restore/fixture.rs @@ -0,0 +1,257 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use protocol::{SessionMessageRole, SessionSnapshot, SessionSnapshotEntryData}; +use serde::Deserialize; +use serde_json::{Value, json}; +use worker_runtime::test_support::StoppedWorkerTemplate; +use yoi_e2e::{FixtureSource, HarnessError, Result}; + +pub const PROFILE: &str = "project:fixture-chat"; +const ASSETS: [&str; 3] = [ + "expected.json", + "worker-template.json", + "workspace-config.dcdl", +]; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Manifest { + pub format_version: u32, + #[serde(rename = "source_revision")] + pub source_commit: String, + pub runtime_schema_version: u32, + pub session_schema_version: u32, + pub workspace_id: String, + pub runtime_id: String, + pub worker_id: String, + pub session_id: String, + pub segment_id: String, + pub model: String, + pub state: String, + pub acquisition: String, + pub source_file_sha256: BTreeMap, + pub files: BTreeMap, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Asset { + pub bytes: u64, + pub sha256: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Expected { + #[serde(rename = "source_revision")] + source_commit: String, + model: String, + worker_id: String, + session_id: String, + rows: Vec, +} + +pub struct Fixture { + pub root: PathBuf, + pub source: FixtureSource, + pub manifest: Manifest, + expected: Expected, +} + +pub fn failure(message: impl Into) -> HarnessError { + HarnessError::Protocol(message.into()) +} +pub fn support(result: std::result::Result) -> Result { + result.map_err(|error| failure(error.to_string())) +} + +impl Fixture { + pub fn load() -> Result { + // Candidate override is explicit and read-only. Missing assets are failures, + // never skips, and no test generates or repairs a fixture. + let root = std::env::var_os("YOI_E2E_RESTORE_FIXTURE") + .map(PathBuf::from) + .map(|path| { + if path.is_absolute() { + path + } else { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join(path) + } + }) + .unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/worker-codex-v1") + }); + let file_names: Vec<_> = std::iter::once("manifest.json").chain(ASSETS).collect(); + let source = FixtureSource::capture_flat(&root, &file_names)?; + let manifest: Manifest = + serde_json::from_slice(&bounded_read(&root.join("manifest.json"))?)?; + if manifest.format_version != 1 + || manifest.state != "stopped" + || manifest.runtime_schema_version == 0 + || manifest.session_schema_version == 0 + || manifest.acquisition.is_empty() + || manifest.source_file_sha256.is_empty() + || manifest.source_commit.len() != 40 + || !manifest + .source_commit + .bytes() + .all(|c| c.is_ascii_hexdigit()) + { + return Err(failure("unsupported/incomplete fixture manifest")); + } + let names: BTreeSet<_> = manifest.files.keys().map(String::as_str).collect(); + if names != BTreeSet::from(ASSETS) { + return Err(failure("unexpected fixture asset set")); + } + for (name, asset) in &manifest.files { + if asset.bytes != fs::metadata(root.join(name))?.len() + || source.hashes().get(Path::new(name)) != Some(&format!("sha256:{}", asset.sha256)) + { + return Err(failure(format!("fixture asset hash/size mismatch: {name}"))); + } + } + let expected: Expected = + serde_json::from_slice(&bounded_read(&root.join("expected.json"))?)?; + if expected.source_commit != manifest.source_commit + || expected.model != manifest.model + || expected.worker_id != manifest.worker_id + || expected.session_id != manifest.session_id + || expected.rows.is_empty() + { + return Err(failure( + "expected conversation identity differs from manifest", + )); + } + verify_builtin_commit(&manifest.source_commit)?; + let fixture = Self { + root, + source, + manifest, + expected, + }; + fixture.template(&fixture.root)?; + fixture.source.assert_unchanged()?; + Ok(fixture) + } + + pub fn template(&self, copy: &Path) -> Result { + let bytes = bounded_read(©.join("worker-template.json"))?; + let template: StoppedWorkerTemplate = serde_json::from_slice(&bytes)?; + let shape: Value = serde_json::from_slice(&bytes)?; + let session_manifest_bytes: Vec = + serde_json::from_value(shape["session_manifest"].clone())?; + let segment_bytes: Vec = serde_json::from_value(shape["segment_jsonl"].clone())?; + for (name, bytes) in [ + ( + "session/session.json".to_owned(), + session_manifest_bytes.as_slice(), + ), + ( + format!("session/segments/{}.jsonl", self.manifest.segment_id), + segment_bytes.as_slice(), + ), + ] { + use sha2::{Digest, Sha256}; + let digest = Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + if self.manifest.source_file_sha256.get(&name) != Some(&digest) { + return Err(failure( + "exported Session bytes differ from acquisition hash evidence", + )); + } + } + let session_manifest: Value = serde_json::from_slice(&session_manifest_bytes)?; + let m = &self.manifest; + if template.workspace_id() != m.workspace_id + || template.runtime_id() != m.runtime_id + || template.worker_id().to_string() != m.worker_id + || template.session_id().to_string() != m.session_id + || template.segment_id().to_string() != m.segment_id + || shape["runtime_schema_version"] != m.runtime_schema_version + || session_manifest["schema_version"] != m.session_schema_version + { + return Err(failure("template schema/identity differs from manifest")); + } + Ok(template) + } + + pub fn verify_conversation(&self, snapshot: &SessionSnapshot) -> Result<()> { + let mut rows = Vec::new(); + for entry in &snapshot.entries { + let (role, parts) = match &entry.data { + SessionSnapshotEntryData::UserInput { segments } => { + ("user", serde_json::to_value(segments)?) + } + SessionSnapshotEntryData::Message { + role: SessionMessageRole::Assistant, + content, + } => ("assistant", serde_json::to_value(content)?), + // Display-only accounting is not a conversation row. Its exact + // IDs, timestamp and values are still checked in the full WS snapshot. + SessionSnapshotEntryData::RunStats { .. } => continue, + _ => { + return Err(failure( + "unexpected non-conversation entry in full Session snapshot", + )); + } + }; + rows.push(json!({"entry_id":entry.entry_id,"timestamp":entry.timestamp,"role":role,"parts":parts})); + } + if rows != self.expected.rows { + return Err(failure( + "full public conversation rows differ from expected IDs/time/roles/text/order", + )); + } + Ok(()) + } +} + +fn bounded_read(path: &Path) -> Result> { + FixtureSource::read_bounded(path) +} + +fn verify_builtin_commit(source_commit: &str) -> Result<()> { + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap(); + // Resolve the immutable commit before diff; unavailable commits must not be treated + // as an empty diff. All resources are checked, including transitive imports. + let commit = Command::new("git") + .args(["cat-file", "-t", source_commit]) + .current_dir(workspace) + .output()?; + if !commit.status.success() || commit.stdout != b"commit\n" { + return Err(failure( + "fixture source commit is unavailable for builtin verification", + )); + } + let status = Command::new("git") + .args([ + "diff", + "--quiet", + source_commit, + "--", + "resources", + "crates/builtin-source", + ]) + .current_dir(workspace) + .status()?; + if !status.success() { + return Err(failure( + "builtin/config resources differ from fixture source commit; provide a new version or frozen references", + )); + } + Ok(()) +} diff --git a/tests/e2e/tests/worker_restore/seed.rs b/tests/e2e/tests/worker_restore/seed.rs new file mode 100644 index 000000000..afc7804e4 --- /dev/null +++ b/tests/e2e/tests/worker_restore/seed.rs @@ -0,0 +1,482 @@ +use std::fs; +use std::net::TcpListener; +use std::path::Path; +use std::sync::Arc; + +use config_source::{ConfigTreeChange, VirtualPath}; +use protocol::SessionSnapshot; +use serde_json::Value; +use worker_runtime::FsRuntimeStoreOptions; +use worker_runtime::catalog::{ + ConfigBundleRef, CreateWorkerRequest, ProfileSelector, ProfileSourceArchiveSource, + WorkspaceApiRef, +}; +use yoi_e2e::{Result, WorkerE2eEnvironment}; +use yoi_workspace_server::auth::{mint_secret, new_id, now_rfc3339, token_hash}; +use yoi_workspace_server::config_source::{ConfigCommitRequest, WorkspaceConfigSchemaRegistry}; +use yoi_workspace_server::hosts::{ + WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest, +}; +use yoi_workspace_server::profile_settings::{ + ProfileConfigSchemaProvider, build_virtual_profile_config_bundle, + project_profiles_from_workspace_config, +}; +use yoi_workspace_server::store::{ + AccountRecord, ApiTokenRecord, UserRecord, WorkspaceBootstrapRecord, + WorkspaceRuntimeAuthenticationMode, WorkspaceRuntimeBinding, +}; +use yoi_workspace_server::test_support::{FixedWorkerSeed, seed_fixed_worker}; +use yoi_workspace_server::workspace_signing_identity::{ + FsWorkspaceSigningMaterialStore, WorkspaceSigningIdentityService, + workspace_signing_material_root, +}; +use yoi_workspace_server::{ControlPlaneStore, SqliteWorkspaceStore, WorkspaceRecord}; + +use super::fixture::{Fixture, PROFILE, failure, support}; + +pub struct Installed { + pub environment: WorkerE2eEnvironment, + pub snapshot: SessionSnapshot, + pub runtime_public_key: String, + pub workspace_public_key: String, +} + +pub fn start(fixture: &Fixture) -> Result { + let mut environment = WorkerE2eEnvironment::new()?; + super::phase(&environment, "fixture-copy-verify")?; + let copy = environment.root.join("provided-fixture"); + fixture.source.copy_to(©)?; + fs::write( + environment.artifacts_dir.join("fixture-manifest.json"), + fs::read(copy.join("manifest.json"))?, + )?; + let template = fixture.template(©)?; + let runtime_identity = + environment.initialize_runtime_identity(fixture.manifest.runtime_id.clone())?; + let runtime_public_key = runtime_identity["public_key"] + .as_str() + .ok_or_else(|| failure("Runtime public bundle omitted key"))? + .to_owned(); + + // Reserve an environment-local placeholder endpoint only while the Runtime is + // offline. Its real :0 endpoint replaces this binding through public API CAS. + let reserved_endpoint = TcpListener::bind("127.0.0.1:0")?; + super::phase( + &environment, + "offline-backend-bootstrap-profile-worker-seed", + )?; + let bootstrap = bootstrap( + &environment, + fixture, + ©, + &runtime_identity, + &format!("http://{}", reserved_endpoint.local_addr()?), + &template.worker_name(), + )?; + // Close the placeholder before Backend probes it: an accepting TCP listener + // with no HTTP service would stall discovery until its transport timeout. + drop(reserved_endpoint); + super::phase(&environment, "start-seeded-backend")?; + environment.start_seeded_backend( + fixture.manifest.workspace_id.clone(), + fixture.manifest.runtime_id.clone(), + bootstrap.token, + )?; + let mut snapshot = None; + super::phase(&environment, "initialize-runtime-trust")?; + environment.start_runtime_with_seed(|environment| { + super::phase(environment, "install-runtime-fixture-and-validate-session")?; + let options = FsRuntimeStoreOptions::new(environment.runtime_data_dir.join("runtime")) + .with_runtime_id(&fixture.manifest.runtime_id); + let request = CreateWorkerRequest { + worker_id: template.worker_id(), + create_fingerprint: bootstrap.create_fingerprint.clone(), + profile: ProfileSelector::Named(PROFILE.into()), + display_name: Some("Restore E2E fixture".into()), + profile_source: ProfileSourceArchiveSource::WorkspaceConfig { + archive: bootstrap.archive.clone(), + }, + config_bundle: Some(bootstrap.config_bundle.clone()), + workspace_api: Some(WorkspaceApiRef { + workspace_id: fixture.manifest.workspace_id.clone(), + base_url: environment.backend_target()?.0.into(), + }), + initial_input: None, + workdir_attachment_requests: vec![], + workdir_attachments: vec![], + worker_observation_enabled: false, + worker_observation_grants: vec![], + memory_settings: Some(bootstrap.memory_settings.clone()), + subjektiv_attached: false, + backend_job: None, + }; + support(template.install(&options, request, &bootstrap.manifest))?; + let sessions = support(session_store::WorkerSessionStore::open_read_only( + options + .root + .join("workers") + .join(template.worker_id().to_string()) + .join("session"), + ))?; + let (logs, _) = support(sessions.read_all_read_only_bounded( + template.session_id(), + template.segment_id(), + 8 * 1024 * 1024, + ))?; + let projected = session_store::project_session_snapshot(template.session_id(), &logs); + fixture.verify_conversation(&projected)?; + snapshot = Some(full_restored_snapshot(&sessions, &logs)); + Ok(()) + })?; + fixture.source.assert_unchanged()?; + Ok(Installed { + environment, + snapshot: snapshot.ok_or_else(|| failure("Runtime seed was not invoked"))?, + runtime_public_key, + workspace_public_key: bootstrap.workspace_public_key, + }) +} + +fn full_restored_snapshot( + store: &dyn session_store::Store, + logs: &[session_store::LogEntry], +) -> SessionSnapshot { + session_store::public_snapshot::project_current_session_snapshot_with_accounting(store, logs) +} + +#[cfg(test)] +mod tests { + use super::*; + use protocol::{ + Event, Greeting, InFlightSnapshot, InvokeKind, SessionSnapshotEntryData, + WorkerStateSnapshot, WorkerStatus, + }; + use session_store::{ + LogEntry, LoggedContentPart, LoggedHistoryEntry, LoggedItem, LoggedRole, + LoggedSessionHistoryEntryId, LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, + }; + + #[test] + fn completed_run_full_snapshot_retains_invoke_and_usage_accounting() -> Result<()> { + // Projection regression only: no saved Worker or substitute E2E fixture. + let temp = tempfile::tempdir()?; + let store = support(session_store::WorkerSessionStore::new(temp.path()))?; + let session_id = session_store::new_session_id(); + let logs = vec![ + LogEntry::AnnotatedSegmentStart { + ts: 1, + session_id, + system_prompt: None, + config: Default::default(), + history: vec![], + forked_from: None, + compacted_from: None, + }, + LogEntry::Invoke { + ts: 10, + trigger: InvokeKind::UserSend, + }, + LogEntry::AnnotatedAssistantItem { + ts: 20, + entry: LoggedHistoryEntry { + item: LoggedItem::Message { + role: LoggedRole::Assistant, + content: vec![LoggedContentPart::Text { + text: "unit response".into(), + }], + }, + metadata: LoggedSessionHistoryMetadata { + entry_id: LoggedSessionHistoryEntryId("unit-response".into()), + origin: LoggedSessionHistoryOrigin::LegacyUnknown, + derivation: None, + input_metadata: None, + }, + }, + }, + LogEntry::LlmUsage { + ts: 30, + history_len: 1, + input_total_tokens: 100, + cache_read_tokens: 20, + cache_write_tokens: 0, + output_tokens: 5, + }, + serde_json::from_value(serde_json::json!({"kind":"run_completed", "ts":40, + "interrupted":false,"result":"finished"}))?, + ]; + let conversation = session_store::project_session_snapshot(session_id, &logs); + let expected = full_restored_snapshot(&store, &logs); + assert_eq!(expected.entries.len(), conversation.entries.len() + 2); + assert!(matches!( + expected.entries[expected.entries.len() - 2].data, + SessionSnapshotEntryData::Invoke { + trigger: InvokeKind::UserSend + } + )); + assert_eq!(expected.entries[expected.entries.len() - 2].timestamp, 10); + assert!(matches!( + expected.entries.last().unwrap().data, + SessionSnapshotEntryData::Usage { + input_tokens: 100, + cache_read_input_tokens: 20, + output_tokens: 5 + } + )); + assert_eq!(expected.entries.last().unwrap().timestamp, 30); + let live = session_store::public_snapshot::project_current_session_snapshot_with_accounting( + &store, &logs, + ); + let event = Event::Snapshot { + session: live, + state: WorkerStateSnapshot::from(WorkerStatus::Idle), + in_flight: InFlightSnapshot::default(), + internal_workers: vec![], + greeting: Greeting { + worker_name: "unit".into(), + cwd: ".".into(), + provider: "unit".into(), + model: "unit".into(), + reasoning: None, + scope_summary: "none".into(), + tools: vec![], + context_window: 0, + context_tokens: 0, + context_usage: None, + }, + }; + let raw = serde_json::to_value(event)?; + yoi_e2e::validate_full_snapshot(&raw, &expected)?; + assert!(yoi_e2e::validate_full_snapshot(&raw, &conversation).is_err()); + Ok(()) + } +} + +struct Bootstrap { + token: String, + workspace_public_key: String, + create_fingerprint: String, + archive: worker_runtime::profile_archive::ProfileSourceArchiveRef, + config_bundle: ConfigBundleRef, + manifest: manifest::WorkerManifest, + memory_settings: manifest::WorkspaceMemorySettingsSnapshot, +} + +fn bootstrap( + environment: &WorkerE2eEnvironment, + fixture: &Fixture, + copy: &Path, + runtime_identity: &Value, + endpoint: &str, + worker_name: &str, +) -> Result { + fs::create_dir_all(&environment.server_data_dir)?; + let database = environment.server_data_dir.join("server.db"); + if database.exists() { + return Err(failure("Backend seed refuses an existing database")); + } + let store = support(SqliteWorkspaceStore::open(&database))?; + let now = now_rfc3339(); + let account_id = new_id("acct-user"); + let user_id = new_id("user"); + support(store.upsert_account(&AccountRecord { + account_id: account_id.clone(), + kind: "user".into(), + handle: environment.account_handle.clone(), + display_name: "Restore E2E owner".into(), + created_at: now.clone(), + updated_at: now.clone(), + }))?; + support(store.upsert_user(&UserRecord { + user_id: user_id.clone(), + account_id: account_id.clone(), + handle: environment.account_handle.clone(), + display_name: "Restore E2E owner".into(), + created_at: now.clone(), + updated_at: now.clone(), + }))?; + let token = mint_secret("yoi_api"); + support(store.create_api_token(&ApiTokenRecord { + token_id: new_id("api-token"), + token_hash: token_hash(&token), + user_id, + label: "restore-e2e".into(), + created_at: now.clone(), + expires_at: None, + revoked_at: None, + last_used_at: None, + }))?; + let identities = WorkspaceSigningIdentityService::new( + Arc::new(store.clone()), + Arc::new(FsWorkspaceSigningMaterialStore::new( + workspace_signing_material_root(&database), + )), + ); + let (identity, operation) = support(identities.prepare_workspace_creation( + "restore-e2e-bootstrap", + "restore-e2e-bootstrap", + &fixture.manifest.workspace_id, + &account_id, + ))?; + let workspace_public_key = identity.public_key.clone(); + support(store.create_workspace_bootstrap( + &WorkspaceBootstrapRecord { + operation_key: "restore-e2e-bootstrap".into(), + request_fingerprint: "restore-e2e-bootstrap".into(), + workspace: WorkspaceRecord { + workspace_id: fixture.manifest.workspace_id.clone(), + owner_account_id: account_id, + display_name: "Restore E2E".into(), + state: "active".into(), + created_at: now.clone(), + updated_at: now.clone(), + }, + repository: None, + }, + &identity, + &operation, + ))?; + support( + store.upsert_workspace_runtime_binding( + WorkspaceRuntimeBinding { + workspace_id: fixture.manifest.workspace_id.clone(), + runtime_id: fixture.manifest.runtime_id.clone(), + display_name: format!("E2E {}", fixture.manifest.runtime_id), + base_url: endpoint.into(), + public_key: runtime_identity["public_key"] + .as_str() + .ok_or_else(|| failure("missing Runtime public key"))? + .into(), + public_key_fingerprint: String::new(), + binding_id: String::new(), + authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity, + created_at: now.clone(), + updated_at: now.clone(), + revoked_at: None, + }, + false, + ), + )?; + + let schema = support( + WorkspaceConfigSchemaRegistry::default() + .with_provider(Arc::new(ProfileConfigSchemaProvider)) + .with_provider(Arc::new( + yoi_workspace_server::prompt_settings::PromptConfigSchemaProvider, + )) + .with_provider(Arc::new( + yoi_workspace_server::runtime_settings::RuntimeConfigSchemaProvider, + )) + .with_provider(Arc::new( + yoi_workspace_server::repository_access::RepositoryAccessConfigSchemaProvider, + )) + .with_provider(Arc::new( + yoi_workspace_server::skills::SkillConfigSchemaProvider, + )) + .compose(), + )?; + let state = support(store.ensure_workspace_config_materialized_with_schema( + &fixture.manifest.workspace_id, + &now, + schema.clone(), + ))?; + let main = support(VirtualPath::parse("main.dcdl"))?; + let current = state + .snapshot + .get(&main) + .ok_or_else(|| failure("Workspace config omitted main.dcdl"))?; + let candidate = support(store.evaluate_workspace_config_candidate_with_schema( + &fixture.manifest.workspace_id, + &ConfigCommitRequest { + base_digest: state.snapshot.digest.clone(), + changes: vec![ConfigTreeChange::Update { + path: main.clone(), + expected_digest: current.content_digest.clone(), + content: fs::read_to_string(copy.join("workspace-config.dcdl"))?, + }], + entrypoints: vec![main], + }, + schema, + ))?; + let state = support( + store.commit_evaluated_workspace_config(&fixture.manifest.workspace_id, &candidate), + )?; + let projection = support(project_profiles_from_workspace_config( + &fixture.manifest.workspace_id, + &state, + ))?; + let bundle = support(build_virtual_profile_config_bundle( + &projection, + &state, + &fixture.manifest.workspace_id, + &now, + PROFILE, + ))? + .ok_or_else(|| failure("fixture Profile is not registered"))?; + let archive = bundle + .profile_source_archive + .as_ref() + .ok_or_else(|| failure("fixture Profile archive is absent"))?; + let mut resolved = support( + support(archive.verify())?.resolve_profile( + PROFILE, + &environment + .runtime_data_dir + .join("runtime/workers") + .join(&fixture.manifest.worker_id), + worker_name, + ), + )?; + if resolved.model.ref_.as_deref() != Some(fixture.manifest.model.as_str()) { + return Err(failure("resolved Profile changed the supplied model")); + } + let request = WorkerSpawnRequest { + intent: WorkerSpawnIntent::WorkspaceCoding, + requested_worker_name: Some("Restore E2E fixture".into()), + singleton_key: None, + acceptance: WorkerSpawnAcceptanceRequirement::SocketReady, + profile: ProfileSelector::Named(PROFILE.into()), + ticket_assignment: None, + initial_submit: vec![], + workdir_attachment_requests: vec![], + resolved_workdir_attachment_requests: vec![], + resolved_workdir_attachments: vec![], + resolved_config_bundle: None, + resolved_workspace_api: None, + resolved_memory_settings: None, + resolved_subjektiv_attached: false, + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: vec![], + resolved_control_operation: None, + resolved_execution_grant: None, + }; + let reservation = support(seed_fixed_worker( + &store, + FixedWorkerSeed { + workspace_id: &fixture.manifest.workspace_id, + runtime_id: &fixture.manifest.runtime_id, + worker_id: support(fixture.manifest.worker_id.parse())?, + allocation_key: "restore-e2e-worker", + request: &request, + }, + ))?; + if resolved.feature.memory.profile.enabled { + support( + resolved + .feature + .memory + .bind_workspace_settings(reservation.memory_settings.clone()), + )?; + } + Ok(Bootstrap { + manifest: resolved, + memory_settings: reservation.memory_settings, + token, + workspace_public_key, + create_fingerprint: reservation.create_fingerprint, + archive: archive.reference.clone(), + config_bundle: ConfigBundleRef { + id: bundle.metadata.id, + digest: bundle.metadata.digest, + }, + }) +} diff --git a/tools/state-contract-name-exceptions.json b/tools/state-contract-name-exceptions.json index 933d7a696..bebae656f 100644 --- a/tools/state-contract-name-exceptions.json +++ b/tools/state-contract-name-exceptions.json @@ -1,4 +1,37 @@ [ + { + "path": "tests/e2e/fixtures/worker-codex-v1/manifest.json", + "kind": "saved_data", + "reason": "ユーザー提供済v1固定資産の生成元Git commitの保存key。原本の取得証拠とSHA-256を保持するため変更しない。Worker実行状態や競合制御の番号ではなく、生成元resources照合のみに使う。", + "lines": [ + { + "sha256": "fbf8f92576eb6afbb03502d2f95144651373ce8cd7b4401bc06f801fdb4db28d", + "count": 1 + } + ] + }, + { + "path": "tests/e2e/fixtures/worker-codex-v1/expected.json", + "kind": "saved_data", + "reason": "ユーザー提供済v1固定資産の生成元Git commitの保存key。原本の取得証拠とSHA-256を保持するため変更しない。Worker実行状態や競合制御の番号ではなく、生成元resources照合のみに使う。", + "lines": [ + { + "sha256": "fbf8f92576eb6afbb03502d2f95144651373ce8cd7b4401bc06f801fdb4db28d", + "count": 1 + } + ] + }, + { + "path": "tests/e2e/tests/worker_restore/fixture.rs", + "kind": "saved_data", + "reason": "提供済v1原本の生成元Git commitを読むための固定保存key対応。原本bytesとhashを変更しない。実行状態の観測tokenや更新番号に利用せず、現行Rust内ではsource_commitとして扱う。", + "lines": [ + { + "sha256": "4dd4810d1a31b812596b9d6cc8a8b27f398476174dfa6f0c3c1881aeadb5ade4", + "count": 2 + } + ] + }, { "path": "crates/subjektiv/src/journal/memory_links/tests.rs", "kind": "saved_data", From a77453338cf3d23c3ff886cf537c3cdf092edd15 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 09:16:36 +0900 Subject: [PATCH 23/46] feat: mcp retain stdio process ownership before initialization Allow startup cleanup to own and reap the exact spawned client across handshake failure or cancellation. --- crates/mcp/src/stdio.rs | 267 +++++++++++++++++++++++++++------------- 1 file changed, 184 insertions(+), 83 deletions(-) diff --git a/crates/mcp/src/stdio.rs b/crates/mcp/src/stdio.rs index a3eb426b3..174afd401 100644 --- a/crates/mcp/src/stdio.rs +++ b/crates/mcp/src/stdio.rs @@ -617,7 +617,8 @@ impl fmt::Display for McpRequestStage { } } -/// A running initialized stdio MCP client. +/// An owned stdio MCP process, initialized by `connect` or explicitly after +/// `spawn_uninitialized` when startup cleanup must retain it before any await. pub struct McpStdioClient { server_name: String, limits: McpStdioLimits, @@ -636,6 +637,9 @@ pub struct McpStdioClient { impl McpStdioClient { /// Spawn, initialize, negotiate capabilities, and send notifications/initialized. + /// Initialization errors attempt shutdown, but this convenience API cannot + /// return an unreaped client on failure/cancellation. Its error is not proof + /// of cleanup; use `spawn_uninitialized` with retained ownership when needed. pub async fn connect( spec: McpStdioServerSpec, limits: McpStdioLimits, @@ -667,6 +671,21 @@ impl McpStdioClient { async fn spawn( spec: McpStdioServerSpec, limits: McpStdioLimits, + ) -> Result { + Self::spawn_uninitialized(spec, limits) + } + + /// Spawn without awaiting the handshake, so a cleanup owner can retain the + /// client before initialization can fail or be cancelled. Call + /// `initialize_with_timeout` before discovery; on error/cancellation, shut + /// down this same client rather than spawning a replacement. + /// + /// Once the process spawns, this returns its owner even if an expected pipe + /// is missing. Initialization then fails, but cleanup can still reap the + /// actual child rather than relying on kill-on-drop for a post-spawn error. + pub fn spawn_uninitialized( + spec: McpStdioServerSpec, + limits: McpStdioLimits, ) -> Result { let redactor = spec.redactor(); let mut command = Command::new(&spec.command); @@ -688,29 +707,23 @@ impl McpStdioClient { McpErrorKind::Io(redactor.redact(&err.to_string())), ) })?; - let stdin = child.stdin.take().ok_or_else(|| { - McpClientError::new( - &spec.name, - McpPhase::Spawn, - McpErrorKind::Protocol("child stdin was not piped".into()), - ) - })?; - let stdout = child.stdout.take().ok_or_else(|| { - McpClientError::new( - &spec.name, - McpPhase::Spawn, - McpErrorKind::Protocol("child stdout was not piped".into()), - ) - })?; - let stderr = child.stderr.take().ok_or_else(|| { - McpClientError::new( - &spec.name, - McpPhase::Spawn, - McpErrorKind::Protocol("child stderr was not piped".into()), - ) - })?; + let stdin = child.stdin.take(); + let stdout = child.stdout.take(); + let stderr = child.stderr.take(); + let missing_pipe = if stdin.is_none() { + Some("stdin") + } else if stdout.is_none() { + Some("stdout") + } else if stderr.is_none() { + Some("stderr") + } else { + None + }; - let stdin: SharedStdin = Arc::new(Mutex::new(Some(Box::new(stdin)))); + let stdin: SharedStdin = + Arc::new(Mutex::new(stdin.map(|stdin| { + Box::new(stdin) as Box + }))); let diagnostics = Arc::new(Mutex::new(BoundedDiagnostics::new( spec.name.clone(), limits.max_diagnostic_lines, @@ -718,16 +731,32 @@ impl McpStdioClient { ))); let list_changes = Arc::new(Mutex::new(BoundedListChanged::new(spec.name.clone()))); let (tx, rx) = mpsc::channel(16); - let reader_task = spawn_stdout_reader( - spec.name.clone(), - stdout, - stdin.clone(), - tx, - limits.clone(), - redactor.clone(), - list_changes.clone(), - ); - let stderr_task = spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone()); + // Do not return an error that discards an already spawned child. Queue + // setup failure before reader events so initialization cannot succeed. + if let Some(pipe) = missing_pipe { + tx.try_send(ReaderEvent::Error(McpClientError::new( + &spec.name, + McpPhase::Spawn, + McpErrorKind::Protocol(format!("child {pipe} was not piped")), + ))) + .expect("new response channel has capacity"); + } + let reader_task = match stdout { + Some(stdout) => spawn_stdout_reader( + spec.name.clone(), + stdout, + stdin.clone(), + tx, + limits.clone(), + redactor.clone(), + list_changes.clone(), + ), + None => tokio::spawn(async {}), + }; + let stderr_task = match stderr { + Some(stderr) => spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone()), + None => tokio::spawn(async {}), + }; Ok(Self { server_name: spec.name, @@ -746,6 +775,25 @@ impl McpStdioClient { }) } + /// Perform the bounded handshake on an already retained client. Failure or + /// cancellation does not drop or shut down the client: its owner must do so. + /// Do not retry initialization on a partially initialized connection. + pub async fn initialize_with_timeout(&mut self) -> Result<(), McpClientError> { + let started = Instant::now(); + match timeout(self.limits.startup_timeout, self.initialize()).await { + Ok(result) => result, + Err(_) => Err(McpClientError::new( + &self.server_name, + McpPhase::Initialize, + McpErrorKind::Timeout { + operation: "startup".to_string(), + elapsed: started.elapsed(), + }, + ) + .with_diagnostics(self.snapshot_diagnostics().await)), + } + } + async fn initialize(&mut self) -> Result<(), McpClientError> { let result: InitializeResult = self .request( @@ -1080,6 +1128,8 @@ impl McpStdioClient { } /// Close stdin and wait for process exit, falling back to terminate and kill. + /// Errors and cancellation retain the child for another attempt on this + /// client. Only confirmed reaping releases the process handle. pub async fn shutdown(&mut self) -> Result { self.shutdown_started = true; let deadline = Instant::now() + self.limits.shutdown_timeout; @@ -1091,64 +1141,55 @@ impl McpStdioClient { .await .is_ok(); - let mut child = match self.child.take() { - Some(child) => child, - None => return Ok(ShutdownReport::already_finished()), - }; + // Keep the actual process handle in the client across every await and + // error. Taking it before wait loses retry/reaping authority on cancel. + if self.child.is_none() { + return Ok(ShutdownReport::already_finished()); + } if !closed { self.reader_task.abort(); - return self.terminate_then_kill(child).await; + return self.terminate_then_kill().await; } + let child = self.child.as_mut().expect("checked child"); if let Ok(Some(status)) = child.try_wait() { - self.reader_task.abort(); - self.stderr_task.abort(); - return Ok(ShutdownReport { - exit_status: Some(status), - terminated: false, - killed: false, - }); + return Ok(self.finish_shutdown(status, false, false)); } match timeout_at(deadline, child.wait()).await { - Ok(Ok(status)) => { - self.reader_task.abort(); - self.stderr_task.abort(); - Ok(ShutdownReport { - exit_status: Some(status), - terminated: false, - killed: false, - }) - } + Ok(Ok(status)) => Ok(self.finish_shutdown(status, false, false)), Ok(Err(err)) => Err(McpClientError::new( &self.server_name, McpPhase::Shutdown, McpErrorKind::Io(self.redactor.redact(&err.to_string())), ) .with_diagnostics(self.snapshot_diagnostics().await)), - Err(_) => self.terminate_then_kill(child).await, + Err(_) => self.terminate_then_kill().await, } } - async fn terminate_then_kill( + fn finish_shutdown( &mut self, - mut child: Child, - ) -> Result { - let mut terminated = false; - let mut killed = false; - if send_terminate(&mut child).is_ok() { - terminated = true; + status: ExitStatus, + terminated: bool, + killed: bool, + ) -> ShutdownReport { + // No await separates confirmed reaping from releasing the handle. + drop(self.child.take()); + self.reader_task.abort(); + self.stderr_task.abort(); + ShutdownReport { + exit_status: Some(status), + terminated, + killed, } + } + + async fn terminate_then_kill(&mut self) -> Result { + let child = self.child.as_mut().expect("shutdown retains child"); + let terminated = send_terminate(child).is_ok(); match timeout(self.limits.kill_timeout, child.wait()).await { - Ok(Ok(status)) => { - self.reader_task.abort(); - self.stderr_task.abort(); - Ok(ShutdownReport { - exit_status: Some(status), - terminated, - killed, - }) - } + Ok(Ok(status)) => Ok(self.finish_shutdown(status, terminated, false)), Ok(Err(err)) => Err(McpClientError::new( &self.server_name, McpPhase::Shutdown, @@ -1163,7 +1204,6 @@ impl McpStdioClient { McpErrorKind::Io(self.redactor.redact(&err.to_string())), ) })?; - killed = true; let status = timeout(self.limits.kill_timeout, child.wait()) .await .map_err(|_| { @@ -1183,13 +1223,7 @@ impl McpStdioClient { McpErrorKind::Io(self.redactor.redact(&err.to_string())), ) })?; - self.reader_task.abort(); - self.stderr_task.abort(); - Ok(ShutdownReport { - exit_status: Some(status), - terminated, - killed, - }) + Ok(self.finish_shutdown(status, terminated, true)) } } } @@ -1197,10 +1231,10 @@ impl McpStdioClient { impl Drop for McpStdioClient { fn drop(&mut self) { - if !self.shutdown_started { - if let Some(child) = &mut self.child { - let _ = child.start_kill(); - } + // An attempted shutdown is not proof of exit. Errors and cancellation + // leave the child here so Drop must still make a best-effort kill. + if let Some(child) = &mut self.child { + let _ = child.start_kill(); } self.reader_task.abort(); self.stderr_task.abort(); @@ -1784,3 +1818,70 @@ fn send_terminate(child: &mut Child) -> Result<(), ()> { #[cfg(test)] mod deadline_tests; + +#[cfg(all(test, unix))] +mod shutdown_retention_tests { + //! The real client and OS process are used; only the external server is a + //! local shell that acknowledges initialization and ignores TERM/EOF. + use super::*; + use std::future::{Future, poll_fn}; + use std::task::Poll; + + #[tokio::test] + async fn cancelled_graceful_or_terminate_wait_retries_reaping_the_same_child() { + for cancel_in_fallback in [false, true] { + let script = r#"trap '' TERM +read init || exit 1 +printf '%s\n' '{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-11-25","capabilities":{},"serverInfo":{"name":"retained","version":"1"}}}' +while :; do :; done +"#; + let mut client = McpStdioClient::connect( + McpStdioServerSpec::new("retained", "/bin/sh").args(["-c", script]), + McpStdioLimits { + shutdown_timeout: if cancel_in_fallback { + Duration::ZERO + } else { + Duration::from_secs(30) + }, + kill_timeout: Duration::from_secs(30), + ..Default::default() + }, + ) + .await + .expect("initialize after installing TERM handler"); + let pid = client.child.as_ref().unwrap().id(); + let mut shutdown = Box::pin(client.shutdown()); + poll_fn(|cx| { + assert!(shutdown.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + drop(shutdown); + assert_eq!(client.child.as_ref().and_then(Child::id), pid); + assert!(client.shutdown_started); + + client.limits.shutdown_timeout = Duration::from_millis(20); + client.limits.kill_timeout = Duration::from_millis(50); + let report = timeout(Duration::from_secs(2), client.shutdown()) + .await + .expect("retry bounded") + .expect("retry reaps original child"); + assert!( + report.exit_status.is_some(), + "not an already-finished receipt" + ); + assert!(report.killed, "TERM was ignored"); + assert!(client.child.is_none()); + let mut status = 0; + assert_eq!( + unsafe { libc::waitpid(pid.unwrap() as libc::pid_t, &mut status, libc::WNOHANG) }, + -1 + ); + assert_eq!( + std::io::Error::last_os_error().raw_os_error(), + Some(libc::ECHILD), + "shutdown, not a later observer, reaped the original process" + ); + } + } +} From 848a1e9a5b5c8d366617e0dda93d7f828e59941f Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 09:16:37 +0900 Subject: [PATCH 24/46] feat: worker separate restored controller preparation from publication Prepare the real receiver, full snapshot and paired streams privately; retain activation effects and startup cleanup until the host decision. --- crates/worker/src/bootstrap.rs | 220 +++++++++++++ crates/worker/src/controller.rs | 473 ++++++++++++++++++++++++---- crates/worker/src/feature/mcp.rs | 308 ++++++++++++++++-- crates/worker/src/spawn/registry.rs | 426 ++++++++++++++++++++++--- crates/worker/src/worker.rs | 163 ++++++++-- 5 files changed, 1412 insertions(+), 178 deletions(-) diff --git a/crates/worker/src/bootstrap.rs b/crates/worker/src/bootstrap.rs index 00e6c5ddf..2a591cb3f 100644 --- a/crates/worker/src/bootstrap.rs +++ b/crates/worker/src/bootstrap.rs @@ -323,3 +323,223 @@ where } } } + +/// Host bindings for one unpublished restored Controller. These do not expose +/// Worker execution or permit arbitrary mutation during preparation. +pub struct RestoredControllerBindings { + pub backend_job: Option, + pub workdir_sessions: std::sync::Arc, + pub observation_provider: + Option>, + pub subjektiv_attached: bool, +} + +/// The same real receiver, snapshot sources and paired subscription retained +/// privately across the Runtime's durable restore decision. +pub struct UnpublishedRestoredController { + initialized: Option, St>>, + activation: Option, + streams: Option, + subjektiv_attached: bool, + effects_applied: bool, + activation_failed: bool, + closing: bool, +} + +pub struct ActivatedRestoredController { + pub controller: BootstrappedWorker, + pub streams: crate::ipc::protocol_session::WorkerProtocolSessionStreams, +} + +impl PreparedRestoredWorker +where + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, +{ + pub fn workspace_client_handle( + &mut self, + ) -> Result, WorkerError> { + Ok(self.worker_mut()?.workspace_client_handle()) + } + + pub fn retain_startup_feature_cleanup( + &mut self, + ) -> Result { + Ok(self.worker_mut()?.retain_startup_feature_cleanup()) + } + + /// Initialize the real Controller without allocation, canonical restore + /// writes, attribution changes, registry reclamation or input consumption. + /// MCP discovery and Feature tasks belong to the retained startup cleanup. + pub async fn prepare_runtime_controller( + mut self, + run_dir: &std::path::Path, + transport: WorkerControllerTransport, + bindings: RestoredControllerBindings, + ) -> Result, WorkerBootstrapError> { + let worker = self.worker_mut().map_err(WorkerBootstrapError::Worker)?; + if let Some(binding) = bindings.backend_job { + worker.bind_backend_job(binding).map_err(|error| { + WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.into())) + })?; + } + worker + .workdir_sessions() + .close_all() + .await + .map_err(|error| { + WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.to_string())) + })?; + worker.bind_workdir_sessions(bindings.workdir_sessions); + worker.bind_worker_observation_provider(bindings.observation_provider); + let flow_enabled = worker.manifest().feature.flow.enabled; + if flow_enabled { + let report = worker + .install_runtime_flow_transition_feature() + .map_err(|error| { + WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error)) + })?; + if report.reports.iter().any(|report| !report.installed) { + return Err(WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle( + "Flow transition installation refused".into(), + ))); + } + } + let (worker, activation) = self + .into_controller_parts() + .map_err(WorkerBootstrapError::Worker)?; + let initialized = WorkerController::initialize_restored(worker, run_dir, transport) + .await + .map_err(|source| WorkerBootstrapError::Controller { + source, + cleanup_failed: true, + })?; + if flow_enabled { + initialized.handle.shared_state.enable_flow_transition(); + } + let streams = { + let _guard = initialized.handle.protocol_snapshot_publish_guard(); + crate::ipc::protocol_session::subscribe_worker_protocol_session(&initialized.handle) + }; + Ok(UnpublishedRestoredController { + initialized: Some(initialized), + activation: Some(activation), + streams: Some(streams), + subjektiv_attached: bindings.subjektiv_attached, + effects_applied: false, + activation_failed: false, + closing: false, + }) + } +} + +impl UnpublishedRestoredController +where + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, +{ + pub fn snapshot(&self) -> &protocol::Event { + &self + .streams + .as_ref() + .expect("unpublished restore streams") + .snapshot_event + } + + /// Only publication failures after confirmed effects can retry on this + /// exact owner. Partial activation must be reconciled, never blindly rerun. + pub fn can_retry_publication(&self) -> bool { + self.initialized.is_some() + && self.effects_applied + && !self.activation_failed + && !self.closing + } + + /// Apply canonical restore effects after the host's durable decision, while + /// retaining the same private Controller for subsequent publication. + pub async fn apply_effects(&mut self) -> Result<(), WorkerBootstrapError> { + let initialized = self + .initialized + .as_mut() + .ok_or_else(|| WorkerBootstrapError::Worker(WorkerError::RestoreCandidateClosing))?; + if self.activation_failed || self.closing { + return Err(WorkerBootstrapError::Worker( + WorkerError::RestoreCandidateClosing, + )); + } + if !self.effects_applied { + self.activation_failed = true; + self.activation + .as_ref() + .expect("restore activation") + .activate(&mut initialized.worker) + .await + .map_err(WorkerBootstrapError::Worker)?; + initialized + .worker + .finalize_subjektiv_session_attribution( + self.subjektiv_attached, + crate::SubjektivSessionAttributionLifecycle::RestoredSession, + ) + .map_err(WorkerBootstrapError::Worker)?; + self.effects_applied = true; + self.activation_failed = false; + self.activation = None; + } + Ok(()) + } + + pub fn allocation_cleanup( + &self, + ) -> Option { + self.initialized + .as_ref() + .and_then(|initialized| initialized.worker.restore_allocation_cleanup_handle()) + } + + pub async fn activate(&mut self) -> Result { + self.apply_effects().await?; + let initialized = self + .initialized + .as_mut() + .expect("same initialized Controller"); + let publication = initialized.prepare_publication().await.map_err(|source| { + WorkerBootstrapError::Controller { + source, + cleanup_failed: true, + } + })?; + let initialized = self + .initialized + .take() + .expect("same initialized Controller"); + let (handle, shutdown, controller_task) = initialized.start(publication); + Ok(ActivatedRestoredController { + controller: BootstrappedWorker { + handle, + shutdown, + controller_task, + }, + streams: self.streams.take().expect("same paired subscription"), + }) + } + + pub async fn discard(&mut self) -> Result<(), WorkerError> { + self.closing = true; + if let Some(initialized) = self.initialized.as_mut() { + initialized + .worker + .retain_startup_feature_cleanup() + .shutdown() + .await + .map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?; + initialized + .worker + .workdir_sessions() + .close_all() + .await + .map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?; + } + self.streams = None; + self.initialized = None; + Ok(()) + } +} diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index 6721753d6..c20d5d833 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -23,7 +23,7 @@ use crate::shutdown_after_idle::{ ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role, take_shutdown_request_after_status, }; -use crate::spawn::registry::SpawnedWorkerRegistry; +use crate::spawn::registry::{PreparedSpawnedWorkerRegistryLoad, SpawnedWorkerRegistry}; use crate::spawn::tool::sub_worker_spawn_tool; use crate::worker::{ PendingSubmissionHandle, SystemItemCommitter, Worker, WorkerError, WorkerRunResult, @@ -688,6 +688,28 @@ pub enum WorkerControllerTransport { pub struct WorkerController; +// Own the real command receiver and all snapshot sources before starting any +// protocol listener or controller loop. Fresh startup may materialize the head; +// Runtime restore requires an already materialized source and defers canonical +// restore effects until its durable decision. Tool discovery can own external +// resources, so private initialization is not synonymous with side-effect-free. +pub(crate) struct InitializedWorkerController { + pub(crate) worker: Worker, + pub(crate) handle: WorkerHandle, + method_rx: mpsc::Receiver, + shutdown_tx: oneshot::Sender<()>, + shutdown_rx: ShutdownReceiver, + registry_reclaim: PreparedSpawnedWorkerRegistryLoad, + shutdown_after_idle: ShutdownAfterIdleRequest, + transport: WorkerControllerTransport, + command_events: Option, +} + +pub(crate) struct PreparedControllerPublication { + socket_server: Option, + command_observer: Option>, +} + impl WorkerController { /// Explicit recovery of an unmaterialized Session head. This installs the /// saved Feature composition needed to produce the seed, but creates no @@ -877,9 +899,8 @@ impl WorkerController { } async fn spawn_initialized( - mut worker: Worker, + worker: Worker, runtime_base: &Path, - runtime_managed: bool, runtime_run: Option<&Path>, transport: WorkerControllerTransport, @@ -888,6 +909,71 @@ impl WorkerController { C: LlmClient + Clone + 'static, St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, { + Self::initialize( + worker, + runtime_base, + runtime_managed, + runtime_run, + transport, + ) + .await? + .publish() + .await + } + + async fn initialize( + worker: Worker, + runtime_base: &Path, + runtime_managed: bool, + runtime_run: Option<&Path>, + transport: WorkerControllerTransport, + ) -> Result, std::io::Error> + where + C: LlmClient + Clone + 'static, + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + Self::initialize_inner( + worker, + runtime_base, + runtime_managed, + runtime_run, + transport, + true, + ) + .await + } + + pub(crate) async fn initialize_restored( + worker: Worker, St>, + run_dir: &Path, + transport: WorkerControllerTransport, + ) -> std::io::Result, St>> + where + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + let parent = run_dir + .parent() + .ok_or_else(|| std::io::Error::other("run path has no parent"))?; + Self::initialize_inner(worker, parent, true, Some(run_dir), transport, false).await + } + + async fn initialize_inner( + mut worker: Worker, + runtime_base: &Path, + runtime_managed: bool, + runtime_run: Option<&Path>, + transport: WorkerControllerTransport, + canonical_startup: bool, + ) -> Result, std::io::Error> + where + C: LlmClient + Clone + 'static, + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + if !canonical_startup && worker.needs_initial_session_head_materialization() { + return Err(std::io::Error::other( + "restore source requires operation-owned head recovery", + )); + } worker .validate_subjektiv_body_publication() .map_err(std::io::Error::other)?; @@ -913,22 +999,13 @@ impl WorkerController { }); let spawner_name = worker.manifest().worker.name.clone(); - let self_parent_socket = worker.callback_socket().cloned(); - let loaded_registry = SpawnedWorkerRegistry::load_from_worker_state_with_reclaim( + let registry_reclaim = SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim( runtime_dir.clone(), worker.store().clone(), spawner_name.clone(), Some(worker.scope().clone()), - ) - .await?; - let reclaimed_unreachable = loaded_registry.reclaimed_unreachable; - let spawned_registry = loaded_registry.registry; - if reclaimed_unreachable { - worker.push_notify( - "Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume." - .to_string(), - ); - } + )?; + let spawned_registry = registry_reclaim.registry.clone(); // Hand the alerter to the Worker so internal operations (compaction, // AGENTS.md ingestion during the first turn) can emit user-facing @@ -1020,9 +1097,6 @@ impl WorkerController { if let Some(fs_for_view) = fs_for_view.as_ref() { shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view.clone())); } - runtime_dir.write_manifest(&manifest_toml).await?; - runtime_dir.write_status(&shared_state).await?; - let artifact_store: Arc = Arc::new(worker.store().clone()); let session_id = worker.session_id(); let pending_activations = worker.pending_activation_state(); @@ -1047,25 +1121,88 @@ impl WorkerController { pending_activations, }; - // Do not start an observer before fallible head/recovery/runtime-file - // initialization. The only remaining fallible step is socket startup, - // whose error path must abort AND join this task before returning. - let command_observer = fs_for_view + let command_events = fs_for_view .as_ref() - .and_then(|session| wire_workdir_command_events(session, &in_flight)); - let socket_server = match transport { - WorkerControllerTransport::UnixSocket => match SocketServer::start(&handle).await { - Ok(server) => Some(server), - Err(error) => { - if let Some(observer) = command_observer { - observer.abort(); - let _ = observer.await; - } - return Err(error); - } - }, + .map(|session| PreparedWorkdirCommandEvents::new(session, &handle.in_flight)); + Ok(InitializedWorkerController { + worker, + handle, + method_rx, + shutdown_tx, + shutdown_rx, + registry_reclaim, + shutdown_after_idle, + transport, + command_events, + }) + } +} + +impl InitializedWorkerController +where + C: LlmClient + Clone + 'static, + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, +{ + async fn publish( + mut self, + ) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error> { + let publication = self.prepare_publication().await?; + Ok(self.start(publication)) + } + + pub(crate) async fn prepare_publication( + &mut self, + ) -> std::io::Result { + let runtime_dir = &self.handle.runtime_dir; + runtime_dir + .write_manifest(&self.handle.shared_state.manifest_toml) + .await?; + runtime_dir.write_status(&self.handle.shared_state).await?; + let loaded_registry = self.registry_reclaim.apply().await?; + debug_assert!(Arc::ptr_eq( + &loaded_registry.registry, + &self.handle.spawned_registry + )); + if loaded_registry.reclaimed_unreachable { + self.worker.push_notify("Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume.".to_string()); + } + let socket_server = match self.transport { + WorkerControllerTransport::UnixSocket => Some(SocketServer::start(&self.handle).await?), WorkerControllerTransport::InProcess => None, }; + let command_observer = self + .command_events + .take() + .and_then(PreparedWorkdirCommandEvents::start); + Ok(PreparedControllerPublication { + socket_server, + command_observer, + }) + } + + pub(crate) fn start( + self, + publication: PreparedControllerPublication, + ) -> (WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>) { + let Self { + mut worker, + handle, + method_rx, + shutdown_tx, + shutdown_rx, + shutdown_after_idle, + .. + } = self; + let PreparedControllerPublication { + socket_server, + command_observer, + } = publication; + let runtime_dir = handle.runtime_dir.clone(); + let shared_state = handle.shared_state.clone(); + let working_event_tx = handle.working_event_tx.clone(); + let spawned_registry = handle.spawned_registry.clone(); + let self_parent_socket = worker.callback_socket().cloned(); + let spawner_name = worker.manifest().worker.name.clone(); // === 5. controller_loop === // Clone cancel sender and notification buffer before moving worker @@ -1094,7 +1231,7 @@ impl WorkerController { command_observer, )); - Ok((handle, shutdown_rx, controller_task)) + (handle, shutdown_rx, controller_task) } } @@ -1102,30 +1239,51 @@ pub(crate) fn wire_workdir_command_events( session: &Arc, in_flight: &InFlightEvents, ) -> Option> { - in_flight.replace_command_snapshot(protocol_command_snapshots(session.as_ref())); - let Some(mut events) = session.subscribe_command_events() else { - return None; - }; - // Keep only a weak reference in the observer task. Holding the session - // strongly here would keep its broadcast sender alive forever and prevent - // the receiver from observing closure during Worker teardown. - let session = Arc::downgrade(session); - let in_flight = in_flight.clone(); - Some(tokio::spawn(async move { - loop { - match events.recv().await { - Ok(event) => in_flight.publish_command_event(protocol_command_event(event)), - Err(broadcast::error::RecvError::Lagged(_)) => { - let Some(session) = session.upgrade() else { - break; - }; - in_flight - .replace_command_snapshot(protocol_command_snapshots(session.as_ref())); - } - Err(broadcast::error::RecvError::Closed) => break, - } + PreparedWorkdirCommandEvents::new(session, in_flight).start() +} + +struct PreparedWorkdirCommandEvents { + session: std::sync::Weak, + in_flight: InFlightEvents, + events: Option>, +} + +impl PreparedWorkdirCommandEvents { + fn new(session: &Arc, in_flight: &InFlightEvents) -> Self { + // Subscribe before reading command state; hold the receiver privately + // across the durable decision instead of creating a publication gap. + let events = session.subscribe_command_events(); + in_flight.replace_command_snapshot(protocol_command_snapshots(session.as_ref())); + Self { + session: Arc::downgrade(session), + in_flight: in_flight.clone(), + events, } - })) + } + + fn start(self) -> Option> { + let Self { + session, + in_flight, + events, + } = self; + let mut events = events?; + Some(tokio::spawn(async move { + loop { + match events.recv().await { + Ok(event) => in_flight.publish_command_event(protocol_command_event(event)), + Err(broadcast::error::RecvError::Lagged(_)) => { + let Some(session) = session.upgrade() else { + break; + }; + in_flight + .replace_command_snapshot(protocol_command_snapshots(session.as_ref())); + } + Err(broadcast::error::RecvError::Closed) => break, + } + } + })) + } } fn protocol_command_snapshots(session: &dyn WorkdirSession) -> Vec { @@ -1851,9 +2009,17 @@ where .add_module(crate::feature::builtin::orchestration::orchestration_feature()); } if let Some(workspace_root) = local_workspace_root.as_ref() { - if let Some(module) = + let module = if let Some(cleanup) = worker.startup_feature_cleanup_handle() { + crate::feature::mcp::discover_stdio_tool_feature_with_cleanup( + &mcp_config, + workspace_root, + &cleanup.mcp_discovery(), + ) + .await? + } else { crate::feature::mcp::discover_stdio_tool_feature(&mcp_config, workspace_root).await - { + }; + if let Some(module) = module { feature_registry = feature_registry.with_module(module); } } @@ -4123,6 +4289,171 @@ mod tests { } } + // Component connection: real Worker/Store, initialized endpoint, paired + // protocol subscription and Controller loop. The model boundary is an + // empty non-network script; this test must not request any generation. + #[tokio::test] + async fn initialized_controller_publishes_the_same_endpoint_and_subscription() { + let root = TempDir::new().unwrap(); + let manifest = manifest::WorkerManifest::from_toml( + "[worker]\nname = 'p'\n[model]\nscheme = 'anthropic'\nmodel_id = 'test'\n[engine]\n[scope]\n", + ) + .unwrap(); + let client = ScriptedClient::new(vec![]); + let requests = client.requests.clone(); + let engine = + Engine::<_, agen::state::Mutable, crate::SessionHistoryMetadata>::new_annotated( + client, + ); + let store = session_store::CombinedStore::new( + session_store::FsStore::new(root.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(root.path().join("workers")).unwrap(), + ); + let mut worker = Worker::new( + manifest, + engine, + store, + crate::worker::WorkerWorkspaceContext::local_filesystem(None), + crate::worker::WorkerFilesystemAuthority::None, + manifest::Scope::empty(), + ) + .await + .unwrap(); + worker.set_system_prompt_template( + crate::SystemPromptTemplate::parse( + "default", + crate::PromptCatalogSource::builtins_only(), + ) + .unwrap(), + ); + let prepared = WorkerController::initialize( + worker, + root.path(), + false, + None, + WorkerControllerTransport::UnixSocket, + ) + .await + .unwrap(); + let endpoint = prepared.handle.clone(); + let mut streams = + crate::ipc::protocol_session::subscribe_worker_protocol_session(&endpoint); + let entries = endpoint.committed_entries(); + assert!(matches!( + &streams.snapshot_event, + Event::Snapshot { greeting, state, session, .. } + if !greeting.tools.is_empty() + && state.state == WorkerState::Idle + && session == &session_store::public_snapshot::project_current_session_snapshot(&entries) + )); + assert!(!endpoint.protocol_is_closed()); + assert!(!endpoint.runtime_dir.socket_path().exists()); + assert!(!endpoint.runtime_dir.path().join("status.json").exists()); + assert!(!endpoint.runtime_dir.path().join("manifest.toml").exists()); + assert!(matches!( + streams.events.try_recv(), + Err(broadcast::error::TryRecvError::Empty) + )); + assert!(requests.lock().unwrap().is_empty()); + + let (handle, shutdown, task) = prepared.publish().await.unwrap(); + assert!(handle.same_controller(&endpoint)); + assert!(handle.runtime_dir.socket_path().exists()); + assert!(handle.runtime_dir.path().join("status.json").exists()); + handle.send(Method::ListPendingSubmissions).await.unwrap(); + let pending = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Event::PendingSubmissionsChanged { pending } = + streams.events.recv().await.unwrap() + { + break pending; + } + } + }) + .await + .unwrap(); + assert_eq!(pending, protocol::PendingSubmissionsSnapshot::default()); + assert_eq!( + serde_json::to_value(handle.committed_entries()).unwrap(), + serde_json::to_value(entries).unwrap(), + ); + handle + .send(Method::Shutdown { + command: WorkerCommandEnvelope { command_id: 1 }, + }) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), shutdown) + .await + .unwrap() + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap(); + assert!(endpoint.protocol_is_closed()); + assert!(!endpoint.runtime_dir.socket_path().exists()); + assert!(requests.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn tool_initialization_failure_does_not_reclaim_saved_children() { + let root = TempDir::new().unwrap(); + let mut manifest = manifest::WorkerManifest::from_toml( + "[worker]\nname = 'p'\n[model]\nscheme = 'anthropic'\nmodel_id = 'test'\n[engine]\n[scope]\n", + ).unwrap(); + manifest.feature.ticket.enabled = true; + let client = ScriptedClient::new(vec![]); + let requests = client.requests.clone(); + let store = session_store::CombinedStore::new( + session_store::FsStore::new(root.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(root.path().join("workers")).unwrap(), + ); + // New feature activation is refused by the real unavailable host; + // legacy child metadata must not be consumed before that refusal. + let worker = Worker::new( + manifest, + Engine::<_, agen::state::Mutable, crate::SessionHistoryMetadata>::new_annotated( + client, + ), + store.clone(), + crate::worker::WorkerWorkspaceContext::no_workspace(), + crate::worker::WorkerFilesystemAuthority::None, + manifest::Scope::empty(), + ) + .await + .unwrap(); + let mut metadata = session_store::WorkerMetadata::new( + "p", + Some(session_store::WorkerActiveSegmentRef { + session_id: worker.session_id(), + segment_id: Some(worker.segment_id()), + }), + ); + metadata + .spawned_children + .push(session_store::WorkerSpawnedChild { + worker_name: "legacy".into(), + socket_path: root.path().join("legacy.sock"), + callback_address: root.path().join("p/sock"), + scope_delegated: Vec::new(), + }); + store.write(&metadata).unwrap(); + let before = store.read_by_name("p").unwrap(); + let result = WorkerController::initialize( + worker, + root.path(), + false, + None, + WorkerControllerTransport::InProcess, + ) + .await; + assert!(result.is_err()); + assert_eq!(store.read_by_name("p").unwrap(), before); + assert!(!root.path().join("p/spawned_workers.json").exists()); + assert!(requests.lock().unwrap().is_empty()); + } + #[derive(Debug)] struct DriveWorkspaceHost { available: bool, @@ -4450,20 +4781,24 @@ mod tests { assert!(feature_install < materialize_gate); assert!(materialize_gate < materialize); assert!(materialize < exposure); - let observer = startup.find("let command_observer = fs_for_view").unwrap(); + let observer_preparation = startup + .find("PreparedWorkdirCommandEvents::new(session") + .unwrap(); let recover = startup.find(".recover_unfinished_compaction()").unwrap(); let files = startup - .find("runtime_dir.write_status(&shared_state).await?") + .find(".write_status(&self.handle.shared_state)") + .unwrap(); + let listener = startup + .find("Some(SocketServer::start(&self.handle).await?)") + .unwrap(); + let observer_start = startup + .find(".and_then(PreparedWorkdirCommandEvents::start)") .unwrap(); let loop_start = startup.find("tokio::spawn(controller_loop(").unwrap(); - assert!(recover < observer && files < observer && observer < loop_start); - let socket_startup = &startup[observer..loop_start]; - let abort = socket_startup.find("observer.abort()").unwrap(); - let join = socket_startup.find("observer.await").unwrap(); - let failure = socket_startup.find("return Err(error)").unwrap(); + assert!(recover < observer_preparation && observer_preparation < files); assert!( - abort < join && join < failure, - "startup cannot detach its observer on error" + files < listener && listener < observer_start && observer_start < loop_start, + "observer must not start before the last fallible listener preparation" ); assert!( startup[materialize_gate..materialize] diff --git a/crates/worker/src/feature/mcp.rs b/crates/worker/src/feature/mcp.rs index 259af6db3..a617acb72 100644 --- a/crates/worker/src/feature/mcp.rs +++ b/crates/worker/src/feature/mcp.rs @@ -8,6 +8,7 @@ //! the regular registry decides Tool-name collisions. use std::collections::BTreeMap; +use std::io; use std::path::Path; use std::sync::Arc; @@ -25,6 +26,7 @@ use mcp::stdio::{ }; use serde::Serialize; use serde_json::{Map, Value}; +use tokio::sync::Mutex; use super::{ FeatureDescriptor, FeatureDiagnostic, FeatureInstallContext, FeatureInstallError, @@ -52,8 +54,88 @@ const MAX_LIST_ITEMS: usize = 128; const MAX_RESOURCE_CONTENTS: usize = 16; const MAX_PROMPT_MESSAGES: usize = 32; +/// Owns discovery clients until process reaping is confirmed. Retain a clone in +/// startup cleanup *before* awaiting discovery. Stop/cancel discovery before +/// calling `shutdown`; the shared lock serializes discovery and cleanup, not +/// cancellation of an in-flight discovery task. +/// +/// Error or cancellation leaves the actual clients here for another shutdown +/// attempt. Dropping the final owner only provides best-effort process killing. +#[derive(Clone, Default)] +pub struct McpDiscoveryCleanup { + clients: Arc>>, +} + +impl McpDiscoveryCleanup { + pub fn new() -> Self { + Self::default() + } + + /// Try every retained client once, removing only confirmed shutdowns. An + /// error is not a cleanup receipt; retry this same owner, not discovery. + pub async fn shutdown(&self) -> io::Result<()> { + let mut clients = self.clients.lock().await; + let mut first_error = None; + let mut index = 0; + while index < clients.len() { + match clients[index].shutdown().await { + Ok(_) => { + clients.remove(index); + } + Err(err) => { + first_error.get_or_insert_with(|| discovery_shutdown_error(&err)); + index += 1; + } + } + } + match first_error { + Some(err) => Err(err), + None => Ok(()), + } + } +} + +fn discovery_shutdown_error(err: &McpClientError) -> io::Error { + io::Error::other(bounded_diagnostic(format!( + "MCP discovery shutdown unconfirmed; client retained for cleanup retry: {err}" + ))) +} + +/// Restore preparation must fail if discovery cannot confirm process shutdown. +/// The caller retains `cleanup` before this future is polled, including during +/// initialization. Ordinary discovery failures remain isolated contributions +/// only after their client has been reaped. +pub async fn discover_stdio_tool_feature_with_cleanup( + config: &McpConfig, + workspace_root: &Path, + cleanup: &McpDiscoveryCleanup, +) -> Result, io::Error> { + if config.stdio_servers.is_empty() { + return Ok(None); + } + let mut feature = McpStdioToolFeature::new(); + for server in &config.stdio_servers { + match resolve_stdio_server(server, workspace_root, None) { + Ok(spec) => { + feature.add_contribution( + discover_server_tools_with_cleanup(spec, cleanup, McpStdioLimits::default()) + .await?, + ); + } + Err(err) => { + feature.add_diagnostic(FeatureDiagnostic::error(bounded_diagnostic(format!( + "failed to resolve MCP stdio server `{}`: {err}", + server.name + )))) + } + } + } + Ok(Some(feature)) +} + /// Discover enabled MCP stdio server tools and return a single feature module /// containing startup contributions for normal ToolRegistry installation. +/// Callers needing retryable startup cleanup must use the cleanup-aware API. pub async fn discover_stdio_tool_feature( config: &McpConfig, workspace_root: &Path, @@ -81,21 +163,66 @@ pub async fn discover_stdio_tool_feature( } async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderContribution { + let declaration = provider_declaration(&spec.name, None); + let cleanup = McpDiscoveryCleanup::new(); + match discover_server_tools_with_cleanup(spec, &cleanup, McpStdioLimits::default()).await { + Ok(contribution) => contribution, + Err(err) => { + ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string())) + } + } +} + +async fn discover_server_tools_with_cleanup( + spec: McpStdioServerSpec, + cleanup: &McpDiscoveryCleanup, + limits: McpStdioLimits, +) -> io::Result { + let declaration = provider_declaration(&spec.name, None); + let mut clients = cleanup.clients.lock().await; + if !clients.is_empty() { + return Err(io::Error::other( + "MCP discovery cleanup is pending; shut down the retained owner before retrying discovery", + )); + } + let client = match McpStdioClient::spawn_uninitialized(spec.clone(), limits) { + Ok(client) => client, + Err(err) => { + // The synchronous spawn API only errors before a process exists; + // pipe/handshake failures retain a client and use the shutdown path. + return Ok(ProtocolProviderContribution::failed( + declaration, + bounded_diagnostic(err.to_string()), + )); + } + }; + // Spawn is synchronous: the owner holds the client before the first await + // that can leave a running process on initialization failure/cancellation. + clients.push(client); + let client = clients.last_mut().expect("retained discovery client"); + let contribution = match client.initialize_with_timeout().await { + Ok(()) => discover_connected_server_tools(spec, client).await, + Err(err) => { + ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string())) + } + }; + client + .shutdown() + .await + .map_err(|err| discovery_shutdown_error(&err))?; + drop(clients.pop()); + Ok(contribution) +} + +async fn discover_connected_server_tools( + spec: McpStdioServerSpec, + client: &mut McpStdioClient, +) -> ProtocolProviderContribution { let declaration = provider_declaration(&spec.name, None); let mut contribution = ProtocolProviderContribution::ready(declaration.clone()); let server_namespace = sanitize_segment(&spec.name); let execution_spec = spec.clone(); - let mut client = match McpStdioClient::connect(spec, McpStdioLimits::default()).await { - Ok(client) => client, - Err(err) => { - return ProtocolProviderContribution::failed( - declaration, - bounded_diagnostic(err.to_string()), - ); - } - }; - let (server_version, has_tools, has_resources, has_prompts) = client .initialize_result() .map(|result| { @@ -133,18 +260,10 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont { Ok(list) => list, Err(err) => { - let mut failed = ProtocolProviderContribution::failed( + return ProtocolProviderContribution::failed( declaration, bounded_diagnostic(err.to_string()), ); - if let Err(shutdown_err) = client.shutdown().await { - failed = failed.with_diagnostic(FeatureDiagnostic::warning( - bounded_diagnostic(format!( - "MCP server shutdown after discovery failure failed: {shutdown_err}" - )), - )); - } - return failed; } }; let list_changes = client.snapshot_list_changes().await; @@ -166,20 +285,12 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont { Ok(list) => list, Err(err) => { - let mut failed = ProtocolProviderContribution::failed( + return ProtocolProviderContribution::failed( declaration, bounded_diagnostic(format!( "MCP server sent notifications/tools/list_changed during tool discovery and refresh failed: {err}" )), ); - if let Err(shutdown_err) = client.shutdown().await { - failed = failed.with_diagnostic(FeatureDiagnostic::warning( - bounded_diagnostic(format!( - "MCP server shutdown after discovery refresh failure failed: {shutdown_err}" - )), - )); - } - return failed; } }; let refresh_changes = client.snapshot_list_changes().await; @@ -197,13 +308,6 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont } else { None }; - let shutdown_result = client.shutdown().await; - if let Err(err) = shutdown_result { - contribution = - contribution.with_diagnostic(FeatureDiagnostic::warning(bounded_diagnostic(format!( - "MCP server shutdown after capability discovery failed: {err}" - )))); - } if let Some(list) = list { contribution = normalize_listed_tools( @@ -1819,6 +1923,140 @@ read exit_notification || true McpStdioServerSpec::new("shell-mock", "/bin/sh").args(["-c".to_string(), script]) } + #[cfg(unix)] + fn discovery_config(spec: McpStdioServerSpec) -> McpConfig { + McpConfig { + stdio_servers: vec![manifest::McpStdioServerConfig { + name: spec.name, + command: spec.command, + args: spec.args, + cwd: None, + env: Default::default(), + }], + } + } + + #[cfg(unix)] + #[tokio::test] + async fn cancelled_discovery_retains_initializing_client_in_cleanup_clone() { + use std::future::{Future, poll_fn}; + use std::task::Poll; + use std::time::Duration; + + let config = discovery_config( + McpStdioServerSpec::new("initializing", "/bin/sh") + .args(["-c", "while read line; do :; done"]), + ); + let cleanup = McpDiscoveryCleanup::new(); + let retained = cleanup.clone(); + let mut discovery = Box::pin(discover_stdio_tool_feature_with_cleanup( + &config, + Path::new("."), + &cleanup, + )); + poll_fn(|cx| { + assert!(discovery.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + drop(discovery); + drop(cleanup); + assert_eq!(retained.clients.lock().await.len(), 1); + let retry_error = + discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &retained) + .await + .err() + .expect("discovery must not replace an unreaped client"); + assert!(retry_error.to_string().contains("cleanup is pending")); + assert_eq!(retained.clients.lock().await.len(), 1); + tokio::time::timeout(Duration::from_secs(3), retained.shutdown()) + .await + .expect("cleanup bounded") + .expect("initializing process reaped"); + assert!(retained.clients.lock().await.is_empty()); + retained + .shutdown() + .await + .expect("confirmed cleanup repeatable"); + } + + #[cfg(unix)] + #[tokio::test] + async fn cancelled_owner_shutdown_keeps_client_for_retry() { + use std::future::{Future, poll_fn}; + use std::task::Poll; + use std::time::Duration; + + let cleanup = McpDiscoveryCleanup::new(); + cleanup.clients.lock().await.push( + McpStdioClient::spawn_uninitialized( + McpStdioServerSpec::new("shutdown-pending", "/bin/sh") + .args(["-c", "while :; do :; done"]), + McpStdioLimits { + shutdown_timeout: Duration::from_millis(50), + kill_timeout: Duration::from_millis(100), + ..Default::default() + }, + ) + .expect("local process spawned"), + ); + let mut shutdown = Box::pin(cleanup.shutdown()); + poll_fn(|cx| { + assert!(shutdown.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + drop(shutdown); + assert_eq!(cleanup.clients.lock().await.len(), 1); + tokio::time::timeout(Duration::from_secs(2), cleanup.shutdown()) + .await + .expect("retry bounded") + .expect("retained client shutdown"); + assert!(cleanup.clients.lock().await.is_empty()); + } + + #[cfg(unix)] + #[tokio::test] + async fn cleanup_aware_discovery_reaps_success_and_initialize_failure() { + let cleanup = McpDiscoveryCleanup::new(); + let config = discovery_config(shell_capability_server(r#"{"resources":{}}"#)); + let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup) + .await + .expect("discovery cleanup confirmed") + .expect("configured feature"); + assert!( + feature.contributions[0] + .tools + .iter() + .any(|tool| { tool.name() == "Mcp_shell_mock_resources_read" }) + ); + assert!(cleanup.clients.lock().await.is_empty()); + + let spec = McpStdioServerSpec::new("init-rejected", "/bin/sh").args([ + "-c", + r#"read init || exit 1 +printf '%s\n' '{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"initialization rejected"}}' +while read line; do :; done +"#, + ]); + let config = discovery_config(spec); + let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup) + .await + .expect("failure isolated only after cleanup") + .expect("failed contribution preserved"); + assert!(feature.contributions[0].tools.is_empty()); + assert!( + feature.contributions[0] + .diagnostics + .iter() + .any(|diagnostic| { + diagnostic.severity == FeatureDiagnosticSeverity::Error + && diagnostic.message.contains("initialization rejected") + }) + ); + assert!(cleanup.clients.lock().await.is_empty()); + } + #[test] fn mcp_resource_prompt_operation_tools_are_namespaced_and_untrusted() { let declaration = provider_declaration("demo", Some("1.0.0")); diff --git a/crates/worker/src/spawn/registry.rs b/crates/worker/src/spawn/registry.rs index dd45b6f47..f2528b4e0 100644 --- a/crates/worker/src/spawn/registry.rs +++ b/crates/worker/src/spawn/registry.rs @@ -340,6 +340,84 @@ pub struct SpawnedWorkerRegistryLoad { pub reclaimed_unreachable: bool, } +/// Read-only restore preparation using the registry that will be published by the Host. +/// The Host must retain its lifecycle exclusion from preparation through application; +/// the metadata reread detects source drift, but is not an atomic store transaction. +/// Dropping an unapplied plan does not reclaim legacy child state. +pub(crate) struct PreparedSpawnedWorkerRegistryLoad { + pub(crate) registry: Arc, + /// Projected outcome, not evidence that reclamation has already happened. + pub(crate) reclaimed_unreachable: bool, + runtime_dir: Arc, + store: St, + worker_name: String, + parent_scope: Option, + persisted_children: Vec, + valid_records: Vec, + legacy_projection_exists: bool, +} + +impl PreparedSpawnedWorkerRegistryLoad +where + St: WorkerMetadataStore + Clone + Send + Sync + 'static, +{ + /// Apply only after all other fallible tool initialization, before socket publication. + /// Effects retain their original order: projection, canonical reclaim history, then + /// allocation/scope. An error may follow committed effects; no rollback is attempted. + pub(crate) async fn apply(&mut self) -> io::Result { + let metadata = self + .store + .read_by_name(&self.worker_name) + .map_err(store_error_to_io)?; + let reclaimed: Vec<_> = self + .persisted_children + .iter() + .map(reclaimed_child_from_metadata) + .collect(); + // A prior apply may have committed the canonical reclaim and then failed + // in allocation/scope cleanup. Under the host's same lifecycle lock, + // confirm its exact retained facts rather than appending them again. + let already_reclaimed = !reclaimed.is_empty() + && metadata.as_ref().is_some_and(|metadata| { + metadata.spawned_children.is_empty() + && metadata.reclaimed_children.ends_with(&reclaimed) + }); + if !already_reclaimed + && metadata + .as_ref() + .map(|metadata| metadata.spawned_children.as_slice()) + .unwrap_or_default() + != self.persisted_children.as_slice() + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "spawned-child metadata changed after registry preparation", + )); + } + + // Runtime projection is migration input only; the normal Internal registry is never + // materialized into spawned_workers.json. Only existence was captured, not its contents. + if !self.persisted_children.is_empty() || self.legacy_projection_exists { + self.runtime_dir.write_spawned_workers(&[]).await?; + } + if !reclaimed.is_empty() && !already_reclaimed { + self.store + .reclaim_spawned_children(&self.worker_name, reclaimed) + .map_err(store_error_to_io)?; + } + for record in &self.valid_records { + reclaim_record(&self.worker_name, self.parent_scope.as_ref(), record)?; + } + + let reclaimed_unreachable = self.reclaimed_unreachable; + self.reclaimed_unreachable = false; + Ok(SpawnedWorkerRegistryLoad { + registry: self.registry.clone(), + reclaimed_unreachable, + }) + } +} + impl SpawnedWorkerRegistry { pub(crate) fn bash_output(&self) -> tools::bash_output::BashOutputView { self.bash_output.lock().unwrap().clone() @@ -426,6 +504,23 @@ impl SpawnedWorkerRegistry { worker_name: String, parent_scope: Option, ) -> io::Result + where + St: WorkerMetadataStore + Clone + Send + Sync + 'static, + { + Self::prepare_from_worker_state_with_reclaim(runtime_dir, store, worker_name, parent_scope)? + .apply() + .await + } + + /// Capture legacy child facts without starting children or changing metadata, + /// projection, allocation, or scope. Tool initialization must clone this registry, + /// not construct a replacement, and defer reclamation to the owned plan's `apply`. + pub(crate) fn prepare_from_worker_state_with_reclaim( + runtime_dir: Arc, + store: St, + worker_name: String, + parent_scope: Option, + ) -> io::Result> where St: WorkerMetadataStore + Clone + Send + Sync + 'static, { @@ -442,54 +537,33 @@ impl SpawnedWorkerRegistry { Ok(record) => { warn!( worker = %record.worker_name, - "reclaiming legacy persisted process SubWorker during Internal session restore" + "captured legacy persisted process SubWorker for reclamation during Internal session restore" ); valid_records.push(record); } Err(error) => warn!( error = %error, worker = %child.worker_name, - "clearing corrupt legacy persisted process SubWorker record" + "captured corrupt legacy persisted process SubWorker record for clearing" ), } } - // Runtime projection is migration input only; the normal Internal registry is never - // materialized into spawned_workers.json. let legacy_projection_exists = runtime_dir.path().join("spawned_workers.json").exists(); - if !persisted_children.is_empty() || legacy_projection_exists { - runtime_dir.write_spawned_workers(&[]).await?; - } - if !persisted_children.is_empty() { - let reclaimed = persisted_children - .iter() - .map(reclaimed_child_from_metadata) - .collect(); - store - .reclaim_spawned_children(&worker_name, reclaimed) - .map_err(store_error_to_io)?; - } - for record in &valid_records { - reclaim_record(&worker_name, parent_scope.as_ref(), record)?; - } - - Ok(SpawnedWorkerRegistryLoad { - registry: Arc::new(Self { - internal_records: std::sync::Mutex::new(Vec::new()), - service_records: std::sync::Mutex::new(Vec::new()), - #[cfg(test)] - fail_service_stops: std::sync::Mutex::new(HashMap::new()), - internal_names: std::sync::Mutex::new(HashSet::new()), - internal_shutting_down: AtomicBool::new(false), - pending_internal_spawns: AtomicUsize::new(0), - pending_internal_notify: Notify::new(), - internal_spawn_cleanup_failed: AtomicBool::new(false), - parent_scope, - bash_output: Mutex::new(tools::bash_output::BashOutputView::new()), - parent_protocol: Mutex::new(None), - protocol_publish_lock: Arc::new(Mutex::new(())), - }), + let registry = match &parent_scope { + Some(scope) => Self::new_internal(worker_name.clone(), scope.clone()), + None => Self::new(runtime_dir.clone()), + }; + Ok(PreparedSpawnedWorkerRegistryLoad { + registry, reclaimed_unreachable: !persisted_children.is_empty(), + runtime_dir, + store, + worker_name, + parent_scope, + persisted_children, + valid_records, + legacy_projection_exists, }) } @@ -1140,6 +1214,10 @@ impl Drop for SpawnedWorkerRegistry { .filter(|record| !record.scope_reclaimed.load(Ordering::Acquire)) .flat_map(delegated_write_rules) .collect::>(); + // Empty prepared registries own no delegations, including when abandoned or rejected. + if write_rules.is_empty() { + return; + } let _ = parent_scope.update(|current| current.with_removed_deny_rules(write_rules)); } } @@ -1165,19 +1243,18 @@ fn reclaim_record( parent_scope: Option<&SharedScope>, record: &SpawnedWorkerRecord, ) -> io::Result<()> { - if let Ok(path) = worker_allocation::default_allocation_path() { - if let Ok(mut guard) = worker_allocation::LockFileGuard::open(&path) { - match worker_allocation::reclaim_delegated_scope( - &mut guard, - parent_name, - &record.worker_name, - &record.scope_delegated, - ) { - Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {} - Err(error) => return Err(io::Error::other(error)), - } - } + let path = worker_allocation::default_allocation_path().map_err(io::Error::other)?; + let mut guard = worker_allocation::LockFileGuard::open(&path).map_err(io::Error::other)?; + match worker_allocation::reclaim_delegated_scope( + &mut guard, + parent_name, + &record.worker_name, + &record.scope_delegated, + ) { + Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {} + Err(error) => return Err(io::Error::other(error)), } + drop(guard); if let Some(parent_scope) = parent_scope { let write_rules = record .scope_delegated @@ -1240,10 +1317,18 @@ fn store_error_to_io(error: WorkerStoreError) -> io::Error { #[cfg(test)] mod tests { + //! Legacy restore cases connect the registry to canonical aggregate storage, + //! runtime projections, and temporary SharedScope rules. Successful apply uses + //! a corrupt legacy record to exercise durable clearing without consulting the + //! process-global legacy allocation file; valid-record scope reclaim is also + //! covered by the existing external restore test. + + use std::path::PathBuf; use std::time::Duration; use manifest::{Scope, ScopeConfig}; - use session_store::LogEntry; + use session_store::{LogEntry, WorkerAggregateStore, WorkerMetadata, WorkerSpawnedScopeRule}; + use tempfile::TempDir; use super::*; use crate::internal_worker::{ @@ -1251,6 +1336,249 @@ mod tests { test_internal_worker_session_with_greeting, }; + struct LegacyRestoreFixture { + runtime_dir: Arc, + store: WorkerAggregateStore, + parent_scope: SharedScope, + scope_root: PathBuf, + metadata_path: PathBuf, + // Keep all storage and scope paths alive until the other fixture fields are dropped. + _root: TempDir, + } + + impl LegacyRestoreFixture { + async fn new(permission: &str) -> Self { + let root = TempDir::new().unwrap(); + let scope_root = root.path().join("scope"); + std::fs::create_dir(&scope_root).unwrap(); + let aggregate = root.path().join("aggregate"); + let store = WorkerAggregateStore::new(&aggregate, "parent").unwrap(); + let runtime_dir = Arc::new( + RuntimeDir::create(&root.path().join("runtime"), "parent") + .await + .unwrap(), + ); + let mut metadata = + WorkerMetadata::new("parent", None).with_workspace_id("restore-test-workspace"); + metadata.spawned_children.push(WorkerSpawnedChild { + worker_name: "legacy-child".into(), + socket_path: root.path().join("legacy.sock"), + callback_address: runtime_dir.socket_path(), + scope_delegated: vec![WorkerSpawnedScopeRule { + target: scope_root.clone(), + permission: permission.into(), + recursive: true, + symlink_policy: Default::default(), + }], + }); + store.write(&metadata).unwrap(); + let write_rule = ScopeRule { + target: scope_root.clone(), + permission: Permission::Write, + recursive: true, + symlink_policy: Default::default(), + }; + let parent_scope = SharedScope::new( + Scope::from_config(&ScopeConfig { + allow: vec![write_rule.clone()], + deny: vec![write_rule], + }) + .unwrap(), + ); + Self { + runtime_dir, + store, + parent_scope, + scope_root, + metadata_path: aggregate.join("metadata.json"), + _root: root, + } + } + + fn prepare(&self) -> PreparedSpawnedWorkerRegistryLoad { + SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim( + self.runtime_dir.clone(), + self.store.clone(), + "parent".into(), + Some(self.parent_scope.clone()), + ) + .unwrap() + } + + fn projection_path(&self) -> PathBuf { + self.runtime_dir.path().join("spawned_workers.json") + } + } + + #[tokio::test] + async fn registry_preparation_and_abandonment_leave_legacy_state_unchanged() { + for projection_exists in [false, true] { + let fixture = LegacyRestoreFixture::new("write").await; + if projection_exists { + // Projection contents are deliberately not decoded during preparation. + std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap(); + } + let metadata_before = std::fs::read(&fixture.metadata_path).unwrap(); + let projection_before = std::fs::read(fixture.projection_path()).ok(); + let scope_before = fixture.parent_scope.snapshot(); + assert!(!scope_before.is_writable(&fixture.scope_root)); + + let prepared = fixture.prepare(); + + assert!(prepared.reclaimed_unreachable); + assert!(prepared.registry.list_internal().is_empty()); + assert!(prepared.registry.internal_worker_snapshots().is_empty()); + assert_eq!( + std::fs::read(&fixture.metadata_path).unwrap(), + metadata_before + ); + assert_eq!( + std::fs::read(fixture.projection_path()).ok(), + projection_before + ); + assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot())); + assert!(!fixture.runtime_dir.socket_path().exists()); + drop(prepared); + assert_eq!( + std::fs::read(&fixture.metadata_path).unwrap(), + metadata_before + ); + assert_eq!( + std::fs::read(fixture.projection_path()).ok(), + projection_before + ); + assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot())); + } + } + + #[tokio::test] + async fn prepared_apply_clears_canonical_children_and_projection_on_same_registry() { + let fixture = LegacyRestoreFixture::new("unsupported-legacy-permission").await; + std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap(); + let scope_before = fixture.parent_scope.snapshot(); + let mut prepared = fixture.prepare(); + let shared_registry = prepared.registry.clone(); + // Unrelated metadata may advance while tools are initialized. + fixture.store.add_peer("parent", "new-peer").unwrap(); + let mut expected = fixture.store.read_by_name("parent").unwrap().unwrap(); + expected.reclaimed_children.extend( + expected + .spawned_children + .iter() + .map(reclaimed_child_from_metadata), + ); + expected.spawned_children.clear(); + + let loaded = prepared.apply().await.unwrap(); + + assert!(loaded.reclaimed_unreachable); + assert!(Arc::ptr_eq(&shared_registry, &loaded.registry)); + assert!(shared_registry.list_internal().is_empty()); + assert_eq!( + fixture.store.read_by_name("parent").unwrap(), + Some(expected) + ); + let projection: serde_json::Value = + serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap(); + assert_eq!(projection, serde_json::json!([])); + // Corrupt records are audited and cleared, not interpreted as scope grants. + assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot())); + assert!( + !fixture + .parent_scope + .snapshot() + .is_writable(&fixture.scope_root) + ); + assert!(!fixture.runtime_dir.socket_path().exists()); + } + + #[tokio::test] + async fn prepared_apply_rejects_child_source_drift_before_any_effects() { + for drift in ["scope", "socket", "removal", "metadata-deletion"] { + let fixture = LegacyRestoreFixture::new("write").await; + std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap(); + let scope_before = fixture.parent_scope.snapshot(); + let mut prepared = fixture.prepare(); + if drift == "metadata-deletion" { + fixture.store.delete_by_name("parent").unwrap(); + } else { + fixture + .store + .update_by_name("parent", |metadata| match drift { + "scope" => { + metadata.spawned_children[0].scope_delegated[0].recursive = false + } + "socket" => { + metadata.spawned_children[0].socket_path = + fixture.runtime_dir.path().join("replacement.sock"); + } + "removal" => metadata.spawned_children.clear(), + _ => unreachable!(), + }) + .unwrap(); + } + let metadata_before = std::fs::read(&fixture.metadata_path).ok(); + let projection_before = std::fs::read(fixture.projection_path()).unwrap(); + + let error = prepared + .apply() + .await + .err() + .expect("source drift must fail"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(std::fs::read(&fixture.metadata_path).ok(), metadata_before); + assert_eq!( + std::fs::read(fixture.projection_path()).unwrap(), + projection_before + ); + assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot())); + assert!(!fixture.runtime_dir.socket_path().exists()); + } + } + + #[tokio::test] + async fn prepared_apply_clears_projection_only_without_reporting_child_reclaim() { + let fixture = LegacyRestoreFixture::new("write").await; + fixture + .store + .set_spawned_children("parent", Vec::new()) + .unwrap(); + std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap(); + let metadata_before = std::fs::read(&fixture.metadata_path).unwrap(); + let mut prepared = fixture.prepare(); + assert!(!prepared.reclaimed_unreachable); + + let loaded = prepared.apply().await.unwrap(); + + assert!(!loaded.reclaimed_unreachable); + assert_eq!( + std::fs::read(&fixture.metadata_path).unwrap(), + metadata_before + ); + let projection: serde_json::Value = + serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap(); + assert_eq!(projection, serde_json::json!([])); + } + + #[tokio::test] + async fn prepared_apply_projection_failure_keeps_canonical_children_and_scope() { + let fixture = LegacyRestoreFixture::new("write").await; + std::fs::create_dir(fixture.projection_path()).unwrap(); + let metadata_before = std::fs::read(&fixture.metadata_path).unwrap(); + let scope_before = fixture.parent_scope.snapshot(); + let mut prepared = fixture.prepare(); + + assert!(prepared.apply().await.is_err()); + + assert_eq!( + std::fs::read(&fixture.metadata_path).unwrap(), + metadata_before + ); + assert!(fixture.projection_path().is_dir()); + assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot())); + } + fn registry() -> Arc { let scope = Scope::from_config(&ScopeConfig { allow: vec![ScopeRule { diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 59eb439a7..14be7b385 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -3335,6 +3335,7 @@ pub struct WorkerStartupFeatureCleanup { } struct StartupFeatureCleanupInner { + mcp_discovery: crate::feature::mcp::McpDiscoveryCleanup, registries: Mutex>, settlement: tokio::sync::Mutex< Option>>, @@ -3345,12 +3346,17 @@ impl WorkerStartupFeatureCleanup { fn new(registry: FeatureBackgroundTaskRegistry) -> Self { Self { inner: Arc::new(StartupFeatureCleanupInner { + mcp_discovery: crate::feature::mcp::McpDiscoveryCleanup::new(), registries: Mutex::new(vec![registry]), settlement: tokio::sync::Mutex::new(None), }), } } + pub(crate) fn mcp_discovery(&self) -> crate::feature::mcp::McpDiscoveryCleanup { + self.inner.mcp_discovery.clone() + } + fn retain(&self, registry: FeatureBackgroundTaskRegistry) { self.inner .registries @@ -3370,6 +3376,7 @@ impl WorkerStartupFeatureCleanup { .lock() .expect("startup Feature cleanup poisoned") .clone(); + let mcp_discovery = self.inner.mcp_discovery.clone(); Box::pin(async move { let mut failure = None; for registry in registries { @@ -3377,6 +3384,14 @@ impl WorkerStartupFeatureCleanup { failure.get_or_insert(error); } } + if let Err(error) = mcp_discovery.shutdown().await { + failure.get_or_insert_with(|| { + crate::hook::HookError::new( + crate::hook::HookErrorCategory::Dependency, + error.to_string(), + ) + }); + } failure.map_or(Ok(()), Err) }) }); @@ -3414,11 +3429,84 @@ impl RestoreCandidateWorkdirCleanup { pub struct PreparedRestoredWorker { worker: Option, St>>, closing: bool, + activation: Option, +} + +/// Exact existing restore source retained while a Controller is initialized. +/// +/// This carries no new Worker identity or execution authority. The host must keep +/// the per-Worker operation lock and invoke activation only after durable commit. +/// Controller initialization must not change the captured source before then. +pub(crate) struct RestoredWorkerActivation { metadata: Option, entries: serde_json::Value, manifest: serde_json::Value, } +impl RestoredWorkerActivation { + fn revalidate(&self, worker: &Worker, St>) -> Result<(), WorkerError> + where + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + let changed = |component| WorkerError::RestoreCandidateChanged { + worker_name: worker.manifest.worker.name.clone(), + component, + }; + if worker.store.read_by_name(&worker.manifest.worker.name)? != self.metadata { + return Err(changed("metadata")); + } + if restore_snapshot_value(&worker.manifest)? != self.manifest { + return Err(changed("manifest")); + } + let entries = worker + .store + .read_all(worker.session_id(), worker.segment_id())?; + if restore_snapshot_value(&entries)? != self.entries { + return Err(changed("segment")); + } + Ok(()) + } + + /// Acquire the existing top-level writer allocation after commit, checking + /// exact source both before and after acquisition, then apply restore effects. + /// On failure the Controller still owns the Worker; cleanup handles do not + /// establish rollback and allocation release must wait until it is dropped. + pub(crate) async fn activate( + &self, + worker: &mut Worker, St>, + ) -> Result<(), WorkerError> + where + St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, + { + self.revalidate(worker) + .map_err(|error| worker.construction_failure(error))?; + let allocate = || -> Result { + let socket_path = dir::default_base() + .map_err(ScopeLockError::from)? + .join(&worker.manifest.worker.name) + .join("sock"); + Ok(worker_allocation::install_top_level_with_deny( + worker.manifest.worker.name.clone(), + std::process::id(), + socket_path, + worker.scope.snapshot().allow_rules(), + worker.scope.snapshot().deny_rules(), + worker.segment_id(), + )?) + }; + worker.scope_allocation = + Some(allocate().map_err(|error| worker.construction_failure(error))?); + self.revalidate(worker) + .map_err(|error| worker.construction_failure(error))?; + worker + .finish_restored_construction_in_place(false) + .await + .map_err(|source| WorkerError::RestoreActivationEffects { + source: Box::new(source), + }) + } +} + impl PreparedRestoredWorker where St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static, @@ -3460,23 +3548,32 @@ where } fn revalidate(&self, worker: &Worker, St>) -> Result<(), WorkerError> { - let changed = |component| WorkerError::RestoreCandidateChanged { - worker_name: worker.manifest.worker.name.clone(), - component, - }; - if worker.store.read_by_name(&worker.manifest.worker.name)? != self.metadata { - return Err(changed("metadata")); + self.activation + .as_ref() + .expect("restore candidate activation already transferred") + .revalidate(worker) + } + + /// Transfer the read-only reconstruction for Controller initialization, not + /// execution. Validate once without allocation or canonical restore effects; + /// the Controller must retain activation until its durable commit succeeds. + pub(crate) fn into_controller_parts( + mut self, + ) -> Result<(Worker, St>, RestoredWorkerActivation), WorkerError> { + let worker = self + .worker + .take() + .ok_or(WorkerError::RestoreCandidateClosing)?; + if self.closing { + return Err(worker.construction_failure(WorkerError::RestoreCandidateClosing)); } - if restore_snapshot_value(&worker.manifest)? != self.manifest { - return Err(changed("manifest")); - } - let entries = worker - .store - .read_all(worker.session_id(), worker.segment_id())?; - if restore_snapshot_value(&entries)? != self.entries { - return Err(changed("segment")); - } - Ok(()) + self.revalidate(&worker) + .map_err(|error| worker.construction_failure(error))?; + let activation = self + .activation + .take() + .expect("restore candidate activation already transferred"); + Ok((worker, activation)) } /// Acquire writer allocation only after exact source validation, validate @@ -9730,9 +9827,11 @@ where let mut candidate = PreparedRestoredWorker { worker: Some(worker), closing: false, - metadata, - entries: entries_snapshot, - manifest: manifest_snapshot, + activation: Some(RestoredWorkerActivation { + metadata, + entries: entries_snapshot, + manifest: manifest_snapshot, + }), }; if let Err(error) = candidate.revalidate(candidate.worker()) { let worker = candidate.worker.take().expect("candidate just constructed"); @@ -9742,6 +9841,16 @@ where } async fn finish_restored_construction(mut self) -> Result { + self.finish_restored_construction_in_place(true).await?; + Ok(self) + } + + /// Apply the same restore effects while an initialized Controller retains + /// ownership. Errors retain cleanup handles, but do not drop this Worker. + async fn finish_restored_construction_in_place( + &mut self, + reclaim_delegations: bool, + ) -> Result<(), WorkerError> { self.pending_submission_handle() .reconcile_uploaded_file_pins() .map_err(|error| self.construction_failure(error.into()))?; @@ -9752,10 +9861,12 @@ where segment_id: self.segment_id(), }) .map_err(|error| self.construction_failure(error))?; - self.reconcile_restored_delegations() - .await - .map_err(|error| self.construction_failure(error))?; - Ok(self) + if reclaim_delegations { + self.reconcile_restored_delegations() + .await + .map_err(|error| self.construction_failure(error))?; + } + Ok(()) } async fn reconcile_restored_delegations(&mut self) -> Result<(), WorkerError> { @@ -10596,8 +10707,10 @@ fn compact_failure_category(error: &WorkerError) -> CompactFailureCategory { #[derive(Debug, thiserror::Error)] pub enum WorkerError { - /// A constructor failed after acquiring resources. Retain these handles until - /// explicit cleanup succeeds; the Worker has dropped before this is returned. + /// A construction or activation step failed after acquiring resources. Retain + /// these handles until explicit cleanup succeeds. Consuming constructors drop + /// the Worker before returning; in-place activation leaves its owner responsible + /// for dropping it before allocation release. #[error("{source}")] ConstructionCleanup { #[source] From 1b13563e196f7a199cd1422dbe3472b0e90d61bb Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 09:16:37 +0900 Subject: [PATCH 25/46] feat: runtime commit restore decisions before execution publication Persist explicit restore commit evidence, reconcile canonical write uncertainty, and publish the same prepared controller before Accepted. Keep owned cleanup and publication failures distinct. Preserve optional transport/store feature boundaries and adapt external controller constructors. --- crates/worker-runtime/Cargo.toml | 4 +- crates/worker-runtime/src/execution.rs | 27 + crates/worker-runtime/src/fs_store.rs | 587 ++++++++++- crates/worker-runtime/src/http_server.rs | 84 ++ .../src/http_server/runtime_management_api.rs | 55 +- crates/worker-runtime/src/resource.rs | 2 + .../src/retention/runtime_v5_tests.rs | 38 +- crates/worker-runtime/src/runtime.rs | 674 +++++++++--- crates/worker-runtime/src/test_support.rs | 11 +- .../worker-runtime/src/test_support_tests.rs | 33 + crates/worker-runtime/src/worker_backend.rs | 969 +++++++++++++----- .../src/worker_backend/restore_recovery.rs | 10 +- .../worker_backend_restore_candidate_tests.rs | 781 ++++++++++++-- .../worker_backend_restore_failure_tests.rs | 79 +- ...orker_backend_restore_publication_tests.rs | 535 ++++++++++ crates/worker-runtime/src/worker_source.rs | 55 +- .../src/server_assignment_attention_tests.rs | 1 + .../src/server_assignment_delivery_tests.rs | 1 + 18 files changed, 3348 insertions(+), 598 deletions(-) create mode 100644 crates/worker-runtime/src/worker_backend_restore_publication_tests.rs diff --git a/crates/worker-runtime/Cargo.toml b/crates/worker-runtime/Cargo.toml index 32f6ea537..e61b4fe4f 100644 --- a/crates/worker-runtime/Cargo.toml +++ b/crates/worker-runtime/Cargo.toml @@ -22,14 +22,14 @@ fs-store = ["dep:fs4"] # Offline fixture tooling only; never enabled by the product defaults. test-support = ["fs-store", "ws-server"] http-server = ["dep:axum", "dep:tower", "dep:reqwest"] -ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"] +ws-server = ["http-server", "axum/ws", "tokio/sync"] [dependencies] agen.workspace = true async-trait.workspace = true base64.workspace = true axum = { workspace = true, optional = true } -futures = { workspace = true, optional = true } +futures.workspace = true decodal.workspace = true flow = { path = "../flow" } fs4 = { workspace = true, optional = true } diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index 719811ad0..b95347102 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -650,6 +650,24 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { self.restore_worker(request) } + /// Discard only the exact unpublished operation after Runtime has proved + /// its durable decision uncommitted. This is not Stop and must not affect a + /// connected execution. RolledBack requires confirmed cleanup and no repair. + fn discard_uncommitted_restore( + &self, + _operation_id: WorkerLifecycleOperationId, + _worker_ref: &WorkerRef, + ) -> WorkerExecutionSpawnResult { + WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::unsupported( + WorkerExecutionOperation::Restore, + "backend cannot confirm unpublished restore cleanup", + ), + worker_state: None, + workdir_attachments: Vec::new(), + } + } + fn activate_restored_worker( &self, _operation_id: WorkerLifecycleOperationId, @@ -927,6 +945,15 @@ impl WorkerExecutionBackendRef { self.backend.reconcile_restore(request) } + pub(crate) fn discard_uncommitted_restore( + &self, + operation_id: WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> WorkerExecutionSpawnResult { + self.backend + .discard_uncommitted_restore(operation_id, worker_ref) + } + pub(crate) fn activate_restored_worker( &self, operation_id: WorkerLifecycleOperationId, diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index dcfb1b42c..5418550d1 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -43,7 +43,8 @@ use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; #[path = "test_support.rs"] pub mod test_support; -const SCHEMA_VERSION: u32 = 12; +pub(crate) const SCHEMA_VERSION: u32 = 13; +const RESTORE_ADMISSION_SCHEMA_VERSION: u32 = 12; // Frozen source schemas: do not move their meaning when adding a migration. const SOURCE_SCHEMA_VERSION: u32 = 11; const EXECUTION_SCHEMA_VERSION: u32 = 10; @@ -124,6 +125,8 @@ pub struct FsRuntimeStore { root: PathBuf, _owner_lock: Option>, worker_write_fault: Arc, + #[cfg(test)] + worker_reconcile_sync_fault: Arc, } impl PartialEq for FsRuntimeStore { @@ -164,6 +167,12 @@ impl FsRuntimeStore { .store(fault as u8, Ordering::Release); } + #[cfg(test)] + pub(crate) fn fail_next_worker_reconcile_sync(&self) { + self.worker_reconcile_sync_fault + .store(true, Ordering::Release); + } + pub(crate) fn open_or_create( root: PathBuf, runtime_id: &str, @@ -223,6 +232,8 @@ impl FsRuntimeStore { root, _owner_lock: Some(owner_lock), worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), + #[cfg(test)] + worker_reconcile_sync_fault: Default::default(), }; let state = if snapshot_exists { Some(store.load_runtime_state()?) @@ -261,6 +272,124 @@ impl FsRuntimeStore { Ok(()) } + /// Read one canonical aggregate without migration, cleanup, or tolerant + /// startup enumeration. Reconciliation must not interpret an unreadable + /// record as absent or replace it with an older in-memory projection. + #[cfg(test)] + pub(crate) fn read_worker_record( + &self, + worker_ref: &WorkerRef, + ) -> Result { + self.read_worker_record_file(worker_ref) + .map(|(record, _)| record) + } + + /// Reobserve and sync canonical authority after an uncertain aggregate write. + /// + /// The caller must hold the Worker operation lock; this store retains the + /// Runtime owner lock. Sync the same file that was validated, then its directory + /// and the directories owning newly created Worker paths. Read equality alone + /// cannot discharge a failed post-rename sync. Any error leaves the caller's + /// write outcome unresolved; this never writes an older memory projection. + /// A returned record proves neither a matching operation nor live activation: + /// the lifecycle owner must still compare request/operation, Stop and receipt. + pub(crate) fn read_durable_worker_record( + &self, + worker_ref: &WorkerRef, + ) -> Result { + self.read_durable_worker_record_with_sync(worker_ref, |file| { + #[cfg(test)] + if self + .worker_reconcile_sync_fault + .swap(false, Ordering::AcqRel) + { + return Err(std::io::Error::other( + "injected reconciliation sync failure", + )); + } + file.sync_all() + }) + } + + /// Compare the entire typed aggregate, including request owner, receipt, + /// attachments and Stop, after durability is confirmed under owner locks. + pub(crate) fn reconcile_worker_record( + &self, + expected: &PersistedWorkerRecord, + ) -> Result<(PersistedWorkerRecord, bool), RuntimeError> { + let observed = self.read_durable_worker_record(&expected.worker_ref)?; + let document = |record: &PersistedWorkerRecord| { + serde_json::to_value(WorkerAggregateRecord::from_persisted(record)) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string())) + }; + let matches = document(&observed)? == document(expected)?; + Ok((observed, matches)) + } + + fn read_durable_worker_record_with_sync( + &self, + worker_ref: &WorkerRef, + mut sync: impl FnMut(&File) -> std::io::Result<()>, + ) -> Result { + if self._owner_lock.is_none() { + return Err(RuntimeError::InvalidRequest( + "Worker aggregate reconciliation requires the Runtime store owner lock".to_string(), + )); + } + let (record, file) = self.read_worker_record_file(worker_ref)?; + let worker_dir = self.worker_dir(&worker_ref.worker_id); + let path = worker_dir.join(WORKER_FILE); + let unknown = |source| RuntimeError::StoreCommitOutcomeUnknown { + operation: "sync reconciled Worker aggregate", + path: path.clone(), + source, + }; + sync(&file).map_err(&unknown)?; + for directory in [worker_dir, self.root.join(WORKERS_DIR), self.root.clone()] { + let file = File::open(directory).map_err(&unknown)?; + sync(&file).map_err(&unknown)?; + } + Ok(record) + } + + fn read_worker_record_file( + &self, + worker_ref: &WorkerRef, + ) -> Result<(PersistedWorkerRecord, File), RuntimeError> { + let worker_dir = self.worker_dir(&worker_ref.worker_id); + for directory in [&self.root, &self.root.join(WORKERS_DIR), &worker_dir] { + let metadata = fs::symlink_metadata(directory).map_err(|source| { + if source.kind() == std::io::ErrorKind::NotFound { + RuntimeError::StoreMissing { + operation: "reconcile Worker aggregate", + path: directory.to_path_buf(), + } + } else { + runtime_io_error("reconcile Worker aggregate", directory, source) + } + })?; + if !metadata.file_type().is_dir() { + return Err(RuntimeError::StoreCorrupt { + operation: "reconcile Worker aggregate", + path: directory.to_path_buf(), + message: "Worker aggregate directory is not a regular directory".to_string(), + }); + } + } + let path = worker_dir.join(WORKER_FILE); + let (aggregate, file): (WorkerAggregateRecord, File) = + read_bounded_json_file(&path, "reconcile Worker aggregate")?; + let record = aggregate.validate(&path)?; + if record.worker_ref != *worker_ref || record.worker_id != worker_ref.worker_id { + return Err(RuntimeError::StoreCorrupt { + operation: "reconcile Worker aggregate", + path, + message: "Worker aggregate does not match the requested identity".to_string(), + }); + } + Ok((record, file)) + } + pub(crate) fn delete_worker_record(&self, worker_id: &WorkerId) -> Result<(), RuntimeError> { self.delete_worker_record_with_intent( worker_id, @@ -523,6 +652,7 @@ pub(crate) enum PersistedWorkerRestoreMode { pub(crate) struct PersistedWorkerRestoreOperation { pub(crate) operation_id: WorkerLifecycleOperationId, pub(crate) mode: PersistedWorkerRestoreMode, + pub(crate) commit: crate::runtime::RestoreCommitFact, pub(crate) request: CreateWorkerRequest, pub(crate) binding: Option, pub(crate) last_settled_status: WorkerStatus, @@ -648,8 +778,8 @@ fn plan_runtime_store_migration( format!("Runtime store schema version {schema_version} is out of range"), ) })?; - let staging = migration_sibling(root, "schema-v12-staging")?; - let backup = migration_sibling(root, "pre-schema-v12-backup")?; + let staging = migration_sibling(root, "schema-v13-staging")?; + let backup = migration_sibling(root, "pre-schema-v13-backup")?; if staging.exists() || backup.exists() { return Err(runtime_store_corrupt( root, @@ -698,11 +828,12 @@ fn plan_runtime_store_migration( | AGGREGATE_SCHEMA_VERSION | EXECUTION_SCHEMA_VERSION | SOURCE_SCHEMA_VERSION + | RESTORE_ADMISSION_SCHEMA_VERSION ) { return Err(runtime_store_corrupt( &runtime_path, format!( - "unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, or {SCHEMA_VERSION}" + "unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, or {SCHEMA_VERSION}" ), )); } @@ -1034,10 +1165,10 @@ fn migrate_worker_record( ) -> Result { if matches!( source_schema_version, - EXECUTION_SCHEMA_VERSION | SOURCE_SCHEMA_VERSION + EXECUTION_SCHEMA_VERSION | SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION ) { - // Schemas 10 and 11 already use the current Worker aggregate layout. - // Only advance its version; never regenerate Restore admission identity. + // Admission identity survives migration. Older pending records did not + // distinguish a committed decision from preparation or failed cleanup. if identity_document .get("schema_version") .and_then(serde_json::Value::as_u64) @@ -1048,6 +1179,7 @@ fn migrate_worker_record( "Worker schema does not match Runtime migration source".into(), )); } + migrate_restore_commit_fact(&mut identity_document, identity_path)?; identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION); let aggregate: WorkerAggregateRecord = serde_json::from_value(identity_document) .map_err(|error| runtime_store_corrupt(identity_path, error.to_string()))?; @@ -1059,6 +1191,7 @@ fn migrate_worker_record( remove_legacy_job_input_counter(&mut identity_document); migrate_legacy_request_evidence(&mut identity_document) .map_err(|message| runtime_store_corrupt(identity_path, message.into()))?; + migrate_restore_commit_fact(&mut identity_document, identity_path)?; identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION); let aggregate: WorkerAggregateRecord = serde_json::from_value(identity_document).map_err(|error| { @@ -1125,6 +1258,42 @@ fn migrate_worker_record( Ok(WorkerAggregateRecord::from_v8(identity, execution)) } +// Missing commit evidence is not proof of rollback. This conversion applies +// only to versioned legacy aggregates, including Restore nested under Stop. +fn migrate_restore_commit_fact( + document: &mut serde_json::Value, + path: &Path, +) -> Result<(), RuntimeError> { + let execution = &mut document["execution_state"]; + if execution["state"] != "reconciliation_required" { + return Ok(()); + } + let operation = &mut execution["execution"]; + let restore = match operation["operation"].as_str() { + Some("restore") => Some(&mut operation["intent"]), + Some("stop") => operation["intent"] + .as_object_mut() + .and_then(|stop| stop.get_mut("pending_restore")) + .filter(|restore| !restore.is_null()), + _ => None, + }; + if let Some(restore) = restore { + let restore = restore.as_object_mut().ok_or_else(|| { + runtime_store_corrupt(path, "legacy pending Restore must be an object".into()) + })?; + if restore + .insert("commit".into(), serde_json::json!("legacy_unknown")) + .is_some() + { + return Err(runtime_store_corrupt( + path, + "legacy Restore contains unexpected commit evidence".into(), + )); + } + } + Ok(()) +} + // Frozen schema-v9 conversion. Preserve the exact token and all Restore owners // and receipts; generating a fresh token here would lose retry/admission authority. fn migrate_v9_restore_guard( @@ -1437,7 +1606,10 @@ fn migrate_runtime_document( object .entry("subject_create_recovery_fences") .or_insert_with(|| serde_json::json!({})); - if source_schema_version != SOURCE_SCHEMA_VERSION { + if !matches!( + source_schema_version, + SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION + ) { object.remove("workers"); // This legacy cache is not Runtime lifecycle or restore authority. Bundles // are reacquired from Workspace content authority, never renumbered/rehash-pinned here. @@ -1505,6 +1677,8 @@ fn validate_current_runtime_store(root: &Path) -> Result<(), RuntimeError> { root: root.to_path_buf(), _owner_lock: None, worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), + #[cfg(test)] + worker_reconcile_sync_fault: Default::default(), }; store.load_runtime_state().map(|_| ()) } @@ -1514,6 +1688,8 @@ fn validate_completed_migration_staging(root: &Path) -> Result<(), RuntimeError> root: root.to_path_buf(), _owner_lock: None, worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), + #[cfg(test)] + worker_reconcile_sync_fault: Default::default(), }; let state = store.load_runtime_state()?; if let Some(worker) = state.workers.values().find(|worker| { @@ -1559,14 +1735,16 @@ fn recover_runtime_store_migration(root: &Path) -> Result<(), RuntimeError> { "pre-schema-v10-backup", "schema-v11-staging", "pre-schema-v11-backup", + "schema-v12-staging", + "pre-schema-v12-backup", ] { let earlier = migration_sibling(root, suffix)?; if earlier.exists() { - return Err(runtime_store_corrupt(root, "unfinished earlier Runtime migration requires explicit recovery before schema-v12 startup".into())); + return Err(runtime_store_corrupt(root, "unfinished earlier Runtime migration requires explicit recovery before schema-v13 startup".into())); } } - let staging = migration_sibling(root, "schema-v12-staging")?; - let backup = migration_sibling(root, "pre-schema-v12-backup")?; + let staging = migration_sibling(root, "schema-v13-staging")?; + let backup = migration_sibling(root, "pre-schema-v13-backup")?; let root_exists = migration_directory_exists(root)?; let staging_exists = migration_directory_exists(&staging)?; let backup_exists = migration_directory_exists(&backup)?; @@ -1768,8 +1946,8 @@ fn migrate_runtime_store( if !plan.migration_required { return Ok(plan); } - let staging = migration_sibling(root, "schema-v12-staging")?; - let backup = migration_sibling(root, "pre-schema-v12-backup")?; + let staging = migration_sibling(root, "schema-v13-staging")?; + let backup = migration_sibling(root, "pre-schema-v13-backup")?; if staging.exists() || backup.exists() { return Err(runtime_store_corrupt( root, @@ -1966,9 +2144,12 @@ fn migrate_runtime_store_in_place( )); } } - // Schema 11 metadata was validated above but needs no conversion. + // Schema 11/12 metadata was validated above but needs no conversion. if let Some(metadata) = metadata - && plan.current_schema_version != SOURCE_SCHEMA_VERSION + && !matches!( + plan.current_schema_version, + SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION + ) { atomic_write_json( &migrated_dir.join(WORKER_METADATA_FILE), @@ -2597,6 +2778,19 @@ impl WorkerExecutionRecord { } fn read_bounded_json(path: &Path, operation: &'static str) -> Result +where + T: for<'de> Deserialize<'de>, +{ + read_bounded_json_file(path, operation).map(|(value, _)| value) +} + +// Keep the validated file open so reconciliation syncs the observed authority, +// not a separately reopened projection. The Runtime owner and Worker locks +// exclude cooperating writers throughout the read and sync. +fn read_bounded_json_file( + path: &Path, + operation: &'static str, +) -> Result<(T, File), RuntimeError> where T: for<'de> Deserialize<'de>, { @@ -2628,13 +2822,14 @@ where ), }); } - let file = File::open(path).map_err(|source| RuntimeError::StoreIo { + let mut file = File::open(path).map_err(|source| RuntimeError::StoreIo { operation, path: path.to_path_buf(), source, })?; let mut bytes = Vec::with_capacity(metadata.len() as usize); - file.take(MAX_WORKER_RECORD_BYTES + 1) + (&mut file) + .take(MAX_WORKER_RECORD_BYTES + 1) .read_to_end(&mut bytes) .map_err(|source| RuntimeError::StoreIo { operation, @@ -2648,11 +2843,12 @@ where message: format!("record exceeds limit {MAX_WORKER_RECORD_BYTES} while reading"), }); } - serde_json::from_slice(&bytes).map_err(|source| RuntimeError::StoreCorrupt { + let value = serde_json::from_slice(&bytes).map_err(|source| RuntimeError::StoreCorrupt { operation, path: path.to_path_buf(), message: source.to_string(), - }) + })?; + Ok((value, file)) } fn read_json(path: &Path, operation: &'static str) -> Result @@ -2913,7 +3109,7 @@ mod tests { let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-test"); let plan = FsRuntimeStore::migration_plan(&options).unwrap(); assert_eq!(plan.current_schema_version, 11); - assert_eq!(plan.target_schema_version, 12); + assert_eq!(plan.target_schema_version, SCHEMA_VERSION); assert!(plan.migration_required); assert_eq!(plan.worker_count, 1); assert_eq!(plan.migrated_worker_aggregate_count, 0); @@ -2928,13 +3124,13 @@ mod tests { let opened = FsRuntimeStore::open_or_create(root.clone(), "runtime-test").unwrap(); let state = opened.state.as_ref().unwrap(); let persisted_worker = state.workers.get(&worker_id).unwrap(); - expected_worker["schema_version"] = serde_json::json!(12); + expected_worker["schema_version"] = serde_json::json!(SCHEMA_VERSION); assert_eq!( serde_json::to_value(WorkerAggregateRecord::from_persisted(persisted_worker)) .unwrap(), expected_worker ); - runtime["schema_version"] = serde_json::json!(12); + runtime["schema_version"] = serde_json::json!(SCHEMA_VERSION); if fence_state == "absent" { runtime["subject_create_recovery_fences"] = serde_json::json!({}); } @@ -2970,8 +3166,8 @@ mod tests { drop(reopened); let plan = FsRuntimeStore::migrate(&options).unwrap(); assert!(!plan.migration_required); - assert_eq!(plan.current_schema_version, 12); - assert_eq!(plan.target_schema_version, 12); + assert_eq!(plan.current_schema_version, SCHEMA_VERSION); + assert_eq!(plan.target_schema_version, SCHEMA_VERSION); assert!( !FsRuntimeStore::migration_plan(&options) .unwrap() @@ -2982,12 +3178,12 @@ mod tests { assert_eq!(fs::read(&metadata_path).unwrap(), metadata); assert_eq!(fs::read(&history_path).unwrap(), history); assert!( - !migration_sibling(&root, "schema-v12-staging") + !migration_sibling(&root, "schema-v13-staging") .unwrap() .exists() ); assert!( - !migration_sibling(&root, "pre-schema-v12-backup") + !migration_sibling(&root, "pre-schema-v13-backup") .unwrap() .exists() ); @@ -3456,7 +3652,7 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime-store"); write_empty_schema_v8_store(&root); - let staging = migration_sibling(&root, "schema-v12-staging").unwrap(); + let staging = migration_sibling(&root, "schema-v13-staging").unwrap(); fs::create_dir_all(&staging).unwrap(); fs::write(staging.join("partial"), b"partial").unwrap(); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); @@ -3471,7 +3667,7 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime-store"); write_empty_schema_v8_store(&root); - let backup = migration_sibling(&root, "pre-schema-v12-backup").unwrap(); + let backup = migration_sibling(&root, "pre-schema-v13-backup").unwrap(); fs::rename(&root, &backup).unwrap(); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); @@ -3485,10 +3681,10 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime-store"); write_empty_schema_v8_store(&root); - let staging = migration_sibling(&root, "schema-v12-staging").unwrap(); + let staging = migration_sibling(&root, "schema-v13-staging").unwrap(); copy_runtime_tree(&root, &root, &staging).unwrap(); migrate_runtime_store_in_place(&staging, "test-runtime").unwrap(); - let backup = migration_sibling(&root, "pre-schema-v12-backup").unwrap(); + let backup = migration_sibling(&root, "pre-schema-v13-backup").unwrap(); fs::rename(&root, &backup).unwrap(); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); @@ -3503,10 +3699,10 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime-store"); write_empty_schema_v8_store(&root); - let staging = migration_sibling(&root, "schema-v12-staging").unwrap(); + let staging = migration_sibling(&root, "schema-v13-staging").unwrap(); fs::create_dir_all(&staging).unwrap(); fs::write(staging.join(RUNTIME_FILE), b"{\"schema_version\":8}").unwrap(); - let backup = migration_sibling(&root, "pre-schema-v12-backup").unwrap(); + let backup = migration_sibling(&root, "pre-schema-v13-backup").unwrap(); fs::rename(&root, &backup).unwrap(); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); @@ -3523,7 +3719,7 @@ mod tests { let root = temp.path().join("runtime-store"); write_empty_schema_v8_store(&root); migrate_runtime_store_in_place(&root, "test-runtime").unwrap(); - let backup = migration_sibling(&root, "pre-schema-v12-backup").unwrap(); + let backup = migration_sibling(&root, "pre-schema-v13-backup").unwrap(); write_empty_schema_v8_store(&backup); let store = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); @@ -3538,6 +3734,8 @@ mod tests { "pre-schema-v10-backup", "schema-v11-staging", "pre-schema-v11-backup", + "schema-v12-staging", + "pre-schema-v12-backup", ] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime-store"); @@ -3692,6 +3890,7 @@ mod tests { let error = opened.store.write_worker_record(&worker).unwrap_err(); assert!(matches!(error, RuntimeError::StoreIo { .. })); assert!(!aggregate_path.exists()); + assert!(opened.store.read_worker_record(&worker.worker_ref).is_err()); worker.created_at_ms = Some(1); opened.store.write_worker_record(&worker).unwrap(); @@ -3707,6 +3906,17 @@ mod tests { let committed: WorkerAggregateRecord = read_bounded_json(&aggregate_path, "read test Worker aggregate").unwrap(); assert_eq!(committed.created_at_ms, Some(2)); + let before = fs::read(&aggregate_path).unwrap(); + assert_eq!( + opened + .store + .read_worker_record(&worker.worker_ref) + .unwrap() + .created_at_ms, + Some(2), + "unknown write must be reconciled from the canonical record" + ); + assert_eq!(fs::read(&aggregate_path).unwrap(), before); assert!( !aggregate_path .with_file_name(WORKER_EXECUTION_FILE) @@ -3714,6 +3924,317 @@ mod tests { ); } + #[test] + fn canonical_worker_read_rejects_unreadable_or_wrong_identity_without_repair() { + let temp = tempfile::tempdir().unwrap(); + let opened = + FsRuntimeStore::open_or_create(temp.path().join("runtime-store"), "test-runtime") + .unwrap(); + let requested = WorkerRef::new(WorkerId::now_v7()); + let path = opened + .store + .worker_dir(&requested.worker_id) + .join(WORKER_FILE); + assert!(opened.store.read_worker_record(&requested).is_err()); + assert!(!path.exists()); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let foreign = WorkerId::now_v7(); + let foreign_record = migrate_worker_record( + schema_v7_worker_document(foreign), + LEGACY_SCHEMA_VERSION, + None, + &path + .parent() + .unwrap() + .parent() + .unwrap() + .join(foreign.to_string()) + .join(WORKER_FILE), + ) + .unwrap(); + let documents = [ + b"not-json".to_vec(), + serde_json::to_vec(&foreign_record).unwrap(), + ]; + for document in documents { + fs::write(&path, &document).unwrap(); + assert!(opened.store.read_worker_record(&requested).is_err()); + assert_eq!(fs::read(&path).unwrap(), document); + } + File::create(&path) + .unwrap() + .set_len(MAX_WORKER_RECORD_BYTES + 1) + .unwrap(); + assert!(opened.store.read_worker_record(&requested).is_err()); + assert_eq!( + fs::metadata(&path).unwrap().len(), + MAX_WORKER_RECORD_BYTES + 1 + ); + } + + fn canonical_worker_store() -> ( + tempfile::TempDir, + FsRuntimeStore, + PersistedWorkerRecord, + PathBuf, + ) { + let temp = tempfile::tempdir().unwrap(); + let opened = + FsRuntimeStore::open_or_create(temp.path().join("runtime-store"), "test-runtime") + .unwrap(); + let worker_id = WorkerId::now_v7(); + let path = opened.store.worker_dir(&worker_id).join(WORKER_FILE); + let mut worker = migrate_worker_record( + schema_v7_worker_document(worker_id), + LEGACY_SCHEMA_VERSION, + None, + &path, + ) + .unwrap() + .validate(&path) + .unwrap(); + worker.created_at_ms = Some(1); + opened.store.write_worker_record(&worker).unwrap(); + (temp, opened.store, worker, path) + } + + #[test] + fn legacy_pending_restore_migrates_as_unknown_and_preserves_stop_ownership() { + use crate::runtime::RestoreCommitFact; + for under_stop in [false, true] { + let (_temp, store, mut worker, path) = canonical_worker_store(); + let root = store.root.clone(); + let PersistedWorkerExecutionState::Available(execution) = + worker.execution_state.clone() + else { + panic!("fixture execution unavailable") + }; + let restore = PersistedWorkerRestoreOperation { + operation_id: WorkerLifecycleOperationId::new(), + mode: PersistedWorkerRestoreMode::Explicit, + commit: RestoreCommitFact::Committed, + request: execution.request.clone(), + binding: execution.binding.clone(), + last_settled_status: worker.status, + }; + let operation = if under_stop { + PersistedWorkerLifecycleOperation::Stop(PersistedWorkerStopOperation { + operation_id: WorkerLifecycleOperationId::new(), + pending_restore: Some(restore), + request: Some(execution.request), + binding: execution.binding, + restore_intent: execution.restore_intent, + last_settled_status: worker.status, + }) + } else { + PersistedWorkerLifecycleOperation::Restore(restore) + }; + worker.execution_state = + PersistedWorkerExecutionState::ReconciliationRequired(operation); + let mut legacy = + serde_json::to_value(WorkerAggregateRecord::from_persisted(&worker)).unwrap(); + legacy["schema_version"] = serde_json::json!(RESTORE_ADMISSION_SCHEMA_VERSION); + let intent = &mut legacy["execution_state"]["execution"]["intent"]; + let restore = if under_stop { + &mut intent["pending_restore"] + } else { + intent + }; + restore.as_object_mut().unwrap().remove("commit"); + let expected_request = restore["request"].clone(); + let before_guard = legacy["restore_guard"].clone(); + fs::write(&path, serde_json::to_vec(&legacy).unwrap()).unwrap(); + write_empty_schema_v8_store(&root); + let runtime_path = root.join(RUNTIME_FILE); + let mut runtime: serde_json::Value = read_json(&runtime_path, "read fixture").unwrap(); + runtime["schema_version"] = serde_json::json!(RESTORE_ADMISSION_SCHEMA_VERSION); + fs::write(&runtime_path, serde_json::to_vec(&runtime).unwrap()).unwrap(); + drop(store); + + let opened = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); + let saved = opened.store.read_worker_record(&worker.worker_ref).unwrap(); + let PersistedWorkerExecutionState::ReconciliationRequired(operation) = + saved.execution_state + else { + panic!("migration discarded pending ownership") + }; + let restored = match operation { + PersistedWorkerLifecycleOperation::Restore(restore) => restore, + PersistedWorkerLifecycleOperation::Stop(stop) => stop.pending_restore.unwrap(), + }; + assert_eq!(restored.commit, RestoreCommitFact::LegacyUnknown); + assert_eq!( + serde_json::to_value(restored.request).unwrap(), + expected_request + ); + let disk: serde_json::Value = read_json(&path, "read migrated Worker").unwrap(); + assert_eq!(disk["restore_guard"], before_guard); + assert_eq!(disk["worker_ref"], legacy["worker_ref"]); + let settled = disk["execution_state"]["execution"]["intent"].clone(); + let legacy_intent = &legacy["execution_state"]["execution"]["intent"]; + if under_stop { + assert_eq!(settled["operation_id"], legacy_intent["operation_id"]); + assert_eq!( + settled["pending_restore"]["operation_id"], + legacy_intent["pending_restore"]["operation_id"] + ); + } else { + assert_eq!(settled["operation_id"], legacy_intent["operation_id"]); + } + drop(opened); + let reopened = FsRuntimeStore::open_or_create(root, "test-runtime").unwrap(); + assert!( + reopened + .store + .read_worker_record(&worker.worker_ref) + .is_ok() + ); + } + } + + #[test] + fn uncertain_worker_write_sync_failure_stays_unknown_until_sync_recovers() { + // Real filesystem aggregate/rename/read/validation; substitute only a + // failing file/directory sync syscall, never the reconciliation owner. + let (_temp, store, mut worker, path) = canonical_worker_store(); + worker.created_at_ms = Some(2); + store.fail_next_worker_write(AtomicWriteFault::AfterRename); + assert!(matches!( + store.write_worker_record(&worker), + Err(RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); + let bytes = fs::read(&path).unwrap(); + assert_eq!( + store + .read_worker_record(&worker.worker_ref) + .unwrap() + .created_at_ms, + Some(2) + ); + // The readable replacement is not a durability confirmation. Inject + // failure at the file and at each directory owning its canonical path. + for fail_at in 0..4 { + let mut sync_count = 0; + let result = store.read_durable_worker_record_with_sync(&worker.worker_ref, |file| { + let index = sync_count; + sync_count += 1; + if index == fail_at { + Err(std::io::Error::other( + "injected reconciliation sync failure", + )) + } else { + file.sync_all() + } + }); + assert!(matches!( + result, + Err(RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); + assert_eq!(fs::read(&path).unwrap(), bytes); + } + let reconciled = store + .read_durable_worker_record(&worker.worker_ref) + .unwrap(); + assert_eq!(reconciled.created_at_ms, Some(2)); + assert_eq!(reconciled.execution_state, worker.execution_state); + assert_eq!(fs::read(&path).unwrap(), bytes); + } + + #[test] + fn pre_rename_failure_durable_reread_preserves_old_canonical_record() { + let (_temp, store, mut worker, path) = canonical_worker_store(); + let bytes = fs::read(&path).unwrap(); + worker.created_at_ms = Some(2); + store.fail_next_worker_write(AtomicWriteFault::BeforeRename); + assert!(matches!( + store.write_worker_record(&worker), + Err(RuntimeError::StoreIo { .. }) + )); + let reconciled = store + .read_durable_worker_record(&worker.worker_ref) + .unwrap(); + assert_eq!(reconciled.created_at_ms, Some(1)); + assert_eq!(fs::read(&path).unwrap(), bytes); + } + + #[test] + fn durable_worker_reread_rejects_invalid_authority_before_sync_without_repair() { + let (_temp, store, worker, path) = canonical_worker_store(); + let bytes = fs::read(&path).unwrap(); + let foreign = WorkerId::now_v7(); + let foreign_record = migrate_worker_record( + schema_v7_worker_document(foreign), + LEGACY_SCHEMA_VERSION, + None, + &path + .parent() + .unwrap() + .parent() + .unwrap() + .join(foreign.to_string()) + .join(WORKER_FILE), + ) + .unwrap(); + for document in [ + b"not-json".to_vec(), + serde_json::to_vec(&foreign_record).unwrap(), + ] { + fs::write(&path, &document).unwrap(); + assert!(matches!( + store.read_durable_worker_record_with_sync(&worker.worker_ref, |_| { + panic!("invalid authority must not reach durability confirmation") + }), + Err(RuntimeError::StoreCorrupt { .. }) + )); + assert_eq!(fs::read(&path).unwrap(), document); + } + File::create(&path) + .unwrap() + .set_len(MAX_WORKER_RECORD_BYTES + 1) + .unwrap(); + assert!(matches!( + store.read_durable_worker_record_with_sync(&worker.worker_ref, |_| { + panic!("oversized authority must not reach durability confirmation") + }), + Err(RuntimeError::StoreCorrupt { .. }) + )); + assert_eq!( + fs::metadata(&path).unwrap().len(), + MAX_WORKER_RECORD_BYTES + 1 + ); + fs::remove_file(&path).unwrap(); + assert!(matches!( + store.read_durable_worker_record_with_sync(&worker.worker_ref, |_| { + panic!("missing authority must not reach durability confirmation") + }), + Err(RuntimeError::StoreMissing { .. }) + )); + assert!(!path.exists()); + fs::write(&path, &bytes).unwrap(); + assert!(store.read_durable_worker_record(&worker.worker_ref).is_ok()); + assert_eq!(fs::read(&path).unwrap(), bytes); + } + + #[cfg(unix)] + #[test] + fn canonical_worker_reread_rejects_symlinked_worker_directory() { + let (_temp, store, worker, path) = canonical_worker_store(); + let worker_dir = path.parent().unwrap(); + let retained_dir = worker_dir.with_extension("retained"); + fs::rename(worker_dir, &retained_dir).unwrap(); + std::os::unix::fs::symlink(&retained_dir, worker_dir).unwrap(); + let bytes = fs::read(retained_dir.join(WORKER_FILE)).unwrap(); + assert!(matches!( + store.read_worker_record(&worker.worker_ref), + Err(RuntimeError::StoreCorrupt { .. }) + )); + assert!(matches!( + store.read_durable_worker_record(&worker.worker_ref), + Err(RuntimeError::StoreCorrupt { .. }) + )); + assert_eq!(fs::read(retained_dir.join(WORKER_FILE)).unwrap(), bytes); + } + #[test] fn schema_v8_pair_migrates_to_one_worker_aggregate() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 57b7d0eb6..0fd7d7e91 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -25,6 +25,7 @@ use crate::interaction::{WorkerInput, WorkerInteractionAck}; use crate::management::{RuntimeSummary, WorkerDeleteResult}; #[cfg(feature = "ws-server")] use crate::observation::WorkerObservationCursor; +#[cfg(feature = "fs-store")] use crate::retention::{ WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory, }; @@ -1567,6 +1568,7 @@ fn protocol_error_event(message: impl Into) -> protocol::Event { } } +#[cfg(feature = "fs-store")] #[allow(dead_code)] async fn worker_retention_inventory( State(state): State, @@ -1588,6 +1590,7 @@ async fn worker_retention_inventory( .map_err(RuntimeHttpRestError::runtime) } +#[cfg(feature = "fs-store")] #[allow(dead_code)] async fn execute_worker_retention( State(state): State, @@ -2620,6 +2623,87 @@ mod tests { .unwrap() } + // Generated management routes and real Workspace capability middleware stay + // connected even when this process has no filesystem retention provider. + #[cfg(not(feature = "fs-store"))] + #[tokio::test] + async fn memory_retention_routes_preserve_auth_and_return_typed_unavailable() { + let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap(); + let trust = workspace_trust(&identity); + let runtime = Runtime::new_memory(); + let app = runtime_http_router_with_workspace_auth( + runtime, + None, + workspace_test_auth(vec![trust.clone()]), + ); + let worker_id = WorkerId::now_v7(); + let body = runtime_api::WorkerRetentionExecutionRequest { + operation_id: "memory-retention".into(), + input_fingerprint: "input".into(), + archive_id: None, + workspace_id: "workspace-a".into(), + source_runtime_id: "runtime-test".into(), + worker_id, + source_created_at: "2026-01-01T00:00:00Z".into(), + removed_at: "2026-01-02T00:00:00Z".into(), + effective_profile: None, + retention_class: None, + policy_id: "policy".into(), + session_disposition: runtime_api::SessionDisposition::Purge, + diagnostics_disposition: runtime_api::DiagnosticsDisposition::Purge, + }; + for (method, path, body) in [ + ( + "GET", + format!("/v1/workers/{worker_id}/retention/inventory"), + vec![], + ), + ( + "POST", + format!("/v1/workers/{worker_id}/retention/execute"), + serde_json::to_vec(&body).unwrap(), + ), + ] { + let unsigned = app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(&path) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(body.clone())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED); + let claims = signed_claims(&trust, method, &path, &body); + let response = + signed_request(app.clone(), &identity, &claims, body, "workspace-a").await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + let error: RuntimeHttpErrorResponse = read_json(response).await; + assert_eq!(error.error.code, "worker_retention_unavailable"); + } + for (mut value, expected_status, expected_code) in [ + (body.clone(), StatusCode::NOT_FOUND, "worker_not_found"), + (body.clone(), StatusCode::BAD_REQUEST, "worker_id_mismatch"), + ] { + if expected_status == StatusCode::NOT_FOUND { + value.workspace_id = "workspace-b".into(); + } else { + value.worker_id = WorkerId::now_v7(); + } + let path = format!("/v1/workers/{worker_id}/retention/execute"); + let bytes = serde_json::to_vec(&value).unwrap(); + let claims = signed_claims(&trust, "POST", &path, &bytes); + let response = + signed_request(app.clone(), &identity, &claims, bytes, "workspace-a").await; + assert_eq!(response.status(), expected_status); + let error: RuntimeHttpErrorResponse = read_json(response).await; + assert_eq!(error.error.code, expected_code); + } + } + #[tokio::test] async fn ordinary_signed_ping_succeeds_without_handshake_and_rejects_replay() { let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap(); diff --git a/crates/worker-runtime/src/http_server/runtime_management_api.rs b/crates/worker-runtime/src/http_server/runtime_management_api.rs index e7daf4906..8bcdb123c 100644 --- a/crates/worker-runtime/src/http_server/runtime_management_api.rs +++ b/crates/worker-runtime/src/http_server/runtime_management_api.rs @@ -856,13 +856,25 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { ) })?; let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?; - let inventory = self - .state - .runtime - .worker_retention_inventory(&scope.workspace_id, &worker_ref) - .map_err(RuntimeHttpRestError::runtime) - .map_err(api_error)?; - response(inventory) + #[cfg(feature = "fs-store")] + { + let inventory = self + .state + .runtime + .worker_retention_inventory(&scope.workspace_id, &worker_ref) + .map_err(RuntimeHttpRestError::runtime) + .map_err(api_error)?; + response(inventory) + } + #[cfg(not(feature = "fs-store"))] + { + let _ = (scope, worker_ref); + Err(runtime_api::RuntimeApiError::new( + 503, + "worker_retention_unavailable", + "Worker retention requires filesystem storage", + )) + } } async fn execute_retention( @@ -880,27 +892,36 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { ) })?; let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?; - let request: WorkerRetentionExecutionRequest = request(value)?; - if request.workspace_id != scope.workspace_id { + if value.workspace_id != scope.workspace_id { return Err(runtime_api::RuntimeApiError::new( StatusCode::NOT_FOUND.as_u16(), "worker_not_found", "worker does not belong to the authenticated Workspace", )); } - if request.worker_id != worker_ref.worker_id { + if value.worker_id != worker_ref.worker_id { return Err(runtime_api::RuntimeApiError::new( StatusCode::BAD_REQUEST.as_u16(), "worker_id_mismatch", "Worker retention request worker_id does not match route worker_id", )); } - let result = self - .state - .runtime - .execute_worker_retention(&request) - .map_err(RuntimeHttpRestError::runtime) - .map_err(api_error)?; - response(result) + #[cfg(feature = "fs-store")] + { + let request: WorkerRetentionExecutionRequest = request(value)?; + let result = self + .state + .runtime + .execute_worker_retention(&request) + .map_err(RuntimeHttpRestError::runtime) + .map_err(api_error)?; + response(result) + } + #[cfg(not(feature = "fs-store"))] + Err(runtime_api::RuntimeApiError::new( + 503, + "worker_retention_unavailable", + "Worker retention requires filesystem storage", + )) } } diff --git a/crates/worker-runtime/src/resource.rs b/crates/worker-runtime/src/resource.rs index c9a4dc0db..04eb2ca91 100644 --- a/crates/worker-runtime/src/resource.rs +++ b/crates/worker-runtime/src/resource.rs @@ -1,6 +1,8 @@ +#[cfg(feature = "http-server")] use crate::auth::BACKEND_RESOURCE_FETCH_PERMISSION; use crate::identity::WorkerId; use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef, sha256_hex}; +#[cfg(feature = "http-server")] use crate::workspace_request::RuntimeWorkspaceRequestClient; use async_trait::async_trait; use serde::{Deserialize, Serialize}; diff --git a/crates/worker-runtime/src/retention/runtime_v5_tests.rs b/crates/worker-runtime/src/retention/runtime_v5_tests.rs index 47d75c940..b14a77e80 100644 --- a/crates/worker-runtime/src/retention/runtime_v5_tests.rs +++ b/crates/worker-runtime/src/retention/runtime_v5_tests.rs @@ -3,7 +3,7 @@ //! only the empty segment envelope is built with current APIs. use super::tests::{request, write_json}; use super::*; -use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions}; +use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions, SCHEMA_VERSION}; use serde_json::{Value, json}; use session_store::{LogEntry, SegmentId, SessionId, Store, WorkerSessionStore}; use std::collections::BTreeMap; @@ -226,7 +226,7 @@ fn completed_v4_archive( #[test] fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() { - for runtime_schema in [11, 12] { + for runtime_schema in [11, 12, SCHEMA_VERSION] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); let (worker, session, segment) = runtime_fixture(&root, runtime_schema); @@ -329,7 +329,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() { (storage.checksum_sha256, storage.bytes, storage.file_count), (checksum, bytes, count) ); - if runtime_schema == 12 { + if runtime_schema == SCHEMA_VERSION { for name in [ PathBuf::from("runtime.json"), PathBuf::from(format!("workers/{worker}/worker.json")), @@ -346,8 +346,8 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() { ); drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap()); assert_eq!(tree_bytes(&root), once, "retry rewrote migrated evidence"); - assert!(!temp.path().join(".runtime.schema-v12-staging").exists()); - assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists()); + assert!(!temp.path().join(".runtime.schema-v13-staging").exists()); + assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists()); // session-storage, not the immutable receipt's old checksum, attests // current bytes. Tampering with those bytes must remain detectable. let path = archive_session @@ -360,7 +360,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() { #[test] fn corrupt_archive_does_not_block_startup_or_healthy_session_migration_and_repair_converges() { - for runtime_schema in [11, 12] { + for runtime_schema in [11, 12, SCHEMA_VERSION] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); let (worker, session, segment) = runtime_fixture(&root, runtime_schema); @@ -440,8 +440,8 @@ fn unmigratable_archive_preserves_evidence_across_startup_and_reopen() { ] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); - let (live_worker, live_session, live_segment) = runtime_fixture(&root, 12); - let (worker, _, segment) = stopped_worker(&root, 12); + let (live_worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION); + let (worker, _, segment) = stopped_worker(&root, SCHEMA_VERSION); let provider = FsWorkerRetentionProvider::new(&root); let path = provider .worker_dir(worker) @@ -542,8 +542,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() { ] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); - let (worker, _, _) = runtime_fixture(&root, 12); - completed_v4_archive(&root, 12); + let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION); + completed_v4_archive(&root, SCHEMA_VERSION); let provider = FsWorkerRetentionProvider::new(&root); fs::remove_dir_all(provider.worker_dir(worker)).unwrap(); let archive = provider.archive_dir("archive-a").unwrap(); @@ -575,8 +575,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() { fn archive_isolation_does_not_bypass_corrupt_erasure_authority() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); - runtime_fixture(&root, 12); - let (session, _, _, _) = completed_v4_archive(&root, 12); + runtime_fixture(&root, SCHEMA_VERSION); + let (session, _, _, _) = completed_v4_archive(&root, SCHEMA_VERSION); let ledger = root.join("retention/session-sources"); fs::create_dir_all(&ledger).unwrap(); fs::write(ledger.join(format!("{session}.json")), b"{broken\n").unwrap(); @@ -591,7 +591,7 @@ fn archive_isolation_does_not_bypass_corrupt_erasure_authority() { fn runtime_v5_probe_and_staging_skip_erased_originals() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); - let (worker, live_session, live_segment) = runtime_fixture(&root, 12); + let (worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION); let erased = root .join("orphaned-workers") .join(WorkerId::now_v7().to_string()); @@ -643,7 +643,7 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() { for corruption in ["broken-json", "future-schema"] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); - let (worker, session, segment) = runtime_fixture(&root, 12); + let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION); let session_root = root .join("workers") .join(worker.to_string()) @@ -674,8 +674,8 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() { assert_eq!(tree_bytes(&root), before); assert!(FsRuntimeStore::migrate(&options).is_err()); assert_eq!(tree_bytes(&root), before); - assert!(!temp.path().join(".runtime.schema-v12-staging").exists()); - assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists()); + assert!(!temp.path().join(".runtime.schema-v13-staging").exists()); + assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists()); } } @@ -686,7 +686,7 @@ fn current_runtime_session_migration_still_rejects_live_worker_sockets() { // or alias is needed, and the real live-socket migration refusal stays tested. let temp = tempfile::tempdir_in(".").unwrap(); let root = Path::new(temp.path().file_name().unwrap()).join("runtime"); - let (worker, _, _) = runtime_fixture(&root, 12); + let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION); let before = tree_bytes(&root); let runs = root.join("workers").join(worker.to_string()).join("runs"); fs::create_dir_all(runs.join("1")).unwrap(); @@ -697,7 +697,7 @@ fn current_runtime_session_migration_still_rejects_live_worker_sockets() { drop(listener); fs::remove_dir_all(runs).unwrap(); assert_eq!(tree_bytes(&root), before); - assert!(!temp.path().join(".runtime.schema-v12-staging").exists()); + assert!(!temp.path().join(".runtime.schema-v13-staging").exists()); } #[test] @@ -705,7 +705,7 @@ fn current_runtime_upgrades_v5_role_metadata_through_staging_without_reserializi { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("runtime"); - let (worker, session, segment) = runtime_fixture(&root, 12); + let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION); let aggregate = root.join("workers").join(worker.to_string()); let path = aggregate .join("session/segments") diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index fc53df360..251e423c2 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -2105,6 +2105,16 @@ impl Runtime { // Authorization is read-only until token admission succeeds. state.ensure_workspace_owner(scope, false)?; } + if let Err(error) = self.lock()?.refresh_uncertain_worker_write(worker_ref) { + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_canonical_write_unknown", + sanitize_worker_create_failure_message(&error.to_string()), + )), + None, + )); + } if request.request_id.trim().is_empty() || request.request_id.len() > 512 || request.expected_observation_token.is_empty() @@ -2219,6 +2229,7 @@ impl Runtime { .unwrap_or_else(|| PendingWorkerRestore { operation_id: WorkerLifecycleOperationId::new(), mode: WorkerRestoreMode::Explicit, + commit: RestoreCommitFact::NotCommitted, last_settled_status: candidate.status, }); let operation_id = pending.operation_id; @@ -2242,10 +2253,9 @@ impl Runtime { receipt: None, }, ); - match state.persist_worker_record(&candidate) { + match state.persist_restore_worker_record(&candidate) { Ok(()) => {} Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { - state.workers.insert(worker_ref.worker_id, candidate); return Ok(( Some(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, @@ -2389,10 +2399,13 @@ impl Runtime { if candidate.restore_guard.active_request_id.is_some() || candidate.pending_restore.is_some() { + if result.state != WorkerRestoreState::Accepted { + candidate.ensure_restore_rollback_allowed()?; + } candidate.restore_guard.finish(result.clone()); candidate.pending_restore = None; candidate.restore_candidate_context = None; - state.persist_worker_record(&candidate)?; + state.persist_restore_worker_record(&candidate)?; state.workers.insert(worker_ref.worker_id, candidate); if result.state == WorkerRestoreState::Accepted { state.publish_worker_upsert(worker_ref.worker_id)?; @@ -2431,6 +2444,15 @@ impl Runtime { )); } let pending_restore = worker.pending_restore; + if pending_restore + .is_some_and(|pending| pending.commit == RestoreCommitFact::LegacyUnknown) + { + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_legacy_decision_unknown", + "The saved Restore lacks commit evidence; do not replay execution or infer rollback", + )); + } let reconcile = pending_restore.is_some() && worker .restore_guard @@ -2603,7 +2625,7 @@ impl Runtime { workdir_attachments.clone(), ); match commit { - Ok(worker) => { + Ok(_) => { if let Err(error) = backend.activate_restored_worker(operation_id, worker_ref) { @@ -2631,6 +2653,7 @@ impl Runtime { sanitize_worker_create_failure_message(&error.to_string()), )); } + let worker = self.lock()?.worker(worker_ref)?.detail(); let result = RuntimeWorkerRestoreResult::accepted(worker); if let Err(error) = self.finish_restore_receipt(worker_ref, &result) { return Ok(RuntimeWorkerRestoreResult::failed( @@ -2642,10 +2665,38 @@ impl Runtime { Ok(result) } Err(error) => { + let known_uncommitted = + !matches!(error, RuntimeError::StoreCommitOutcomeUnknown { .. }) + && self + .lock()? + .worker(worker_ref)? + .pending_restore + .is_some_and(|pending| { + pending.operation_id == operation_id + && pending.commit == RestoreCommitFact::NotCommitted + }); + if known_uncommitted { + if let WorkerExecutionSpawnResult::RolledBack(_) = + backend.discard_uncommitted_restore(operation_id, worker_ref) + { + self.discard_restore_candidate_context( + worker_ref, + &candidate_context, + )?; + let receipt = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "worker_restore_decision_not_committed", + sanitize_worker_create_failure_message(&error.to_string()), + ); + if self.settle_restore_rollback(worker_ref, &receipt).is_ok() { + return Ok(receipt); + } + } + } tracing::error!( worker_id = %worker_ref.worker_id, error = %error, - "Worker restore activated but commit did not confirm success; preserving operation and execution" + "Worker restore decision did not confirm success; preserving operation ownership" ); if let Err(retain_error) = self.retain_restore_execution_evidence( worker_ref, @@ -3601,6 +3652,7 @@ impl Runtime { self.ensure_worker_in_workspace(scope, worker_ref)?; } + self.lock()?.refresh_uncertain_worker_write(worker_ref)?; self.lock()?.ensure_subject_not_reserved(worker_ref, true)?; let (backend, pending_stop, pending_restore, status) = { let state = self.lock()?; @@ -4355,18 +4407,18 @@ impl Runtime { candidate.pending_restore = Some(PendingWorkerRestore { operation_id, mode, + commit: candidate + .pending_restore + .map_or(RestoreCommitFact::NotCommitted, |pending| pending.commit), last_settled_status: candidate.status, }); candidate.restore_candidate_context = Some(candidate_context); - match state.persist_worker_record(&candidate) { + match state.persist_restore_worker_record(&candidate) { Ok(()) => { state.workers.insert(worker_ref.worker_id, candidate); Ok(()) } - Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { - state.workers.insert(worker_ref.worker_id, candidate); - Err(error) - } + Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => Err(error), Err(error) => Err(error), } } @@ -4378,11 +4430,12 @@ impl Runtime { ) -> Result<(), RuntimeError> { let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); + candidate.ensure_restore_rollback_allowed()?; candidate.restore_guard.rotate(); candidate.restore_guard.finish(receipt.clone()); candidate.pending_restore = None; candidate.restore_candidate_context = None; - state.persist_worker_record(&candidate)?; + state.persist_restore_worker_record(&candidate)?; state.workers.insert(worker_ref.worker_id, candidate); Ok(()) } @@ -4398,7 +4451,11 @@ impl Runtime { let mut state = self.lock()?; state.ensure_worker_ref(worker_ref)?; let previous = state.worker(worker_ref)?; - if previous.pending_stop.is_some() { + if previous.pending_stop.is_some() + || previous + .pending_restore + .is_none_or(|pending| pending.commit == RestoreCommitFact::LegacyUnknown) + { return Err(RuntimeError::RestoreObservationConflict { worker_id: worker_ref.worker_id, }); @@ -4446,6 +4503,7 @@ impl Runtime { } let _ = candidate.apply_worker_state(&worker_state); if let Some(pending) = candidate.pending_restore.as_mut() { + pending.commit = RestoreCommitFact::Committed; pending.last_settled_status = candidate.status; } candidate.restore_intent = restore_intent_for_status(candidate.status); @@ -4459,7 +4517,7 @@ impl Runtime { // Persist the execution projection before publishing observations. Terminal // receipt settlement follows successful observation activation separately. - state.persist_worker_record(&candidate)?; + state.persist_restore_worker_record(&candidate)?; let detail = candidate.detail(); state.workers.insert(worker_ref.worker_id, candidate); @@ -4499,28 +4557,15 @@ impl Runtime { fn retain_restore_execution_evidence( &self, worker_ref: &WorkerRef, - _worker_state: protocol::WorkerStateSnapshot, - workdir_attachments: Vec, + _workdir_attachments: Vec, candidate_context: crate::execution::WorkerExecutionContext, ) -> Result<(), RuntimeError> { let mut state = self.lock()?; - let mut candidate = state.worker(worker_ref)?.clone(); - if candidate.pending_restore.is_none() { - return Err(RuntimeError::InvalidRequest(format!( - "worker {} has no pending restore operation", - worker_ref.worker_id - ))); - } - - candidate.restore_guard.rotate(); - candidate.execution_metadata_available = true; - candidate.execution_bound = true; - candidate.restore_candidate_context = Some(candidate_context); - candidate.workdir_attachments = workdir_attachments; - state.persist_worker_record(&candidate)?; - state.workers.insert(worker_ref.worker_id, candidate); - Ok(()) + // Keep the actual resource owner even if canonical authority is unreadable. + // Execution evidence cannot authorize overwriting a post-rename decision. + state.worker_mut(worker_ref)?.restore_candidate_context = Some(candidate_context); + state.refresh_uncertain_worker_write(worker_ref) } fn retain_create_reconciliation_pending( @@ -5080,6 +5125,14 @@ impl Runtime { ) })?; let provider = FsWorkerRetentionProvider::new(store.runtime_dir()); + if state + .unreconciled_worker_writes + .contains(&request.worker_id) + { + return Err(RuntimeError::InvalidRequest( + "Worker canonical write outcome must be reconciled before retention effects".into(), + )); + } if let Some(completed) = provider.completed_for(request)? { state.workers.remove(&request.worker_id); state.persist_runtime_snapshot()?; @@ -5178,6 +5231,9 @@ struct RuntimeState { #[cfg(feature = "fs-store")] next_diagnostic_id: u64, workers: BTreeMap, + // Ephemeral write barrier, not authority: only canonical reread + sync can + // release it. Prevents another mutation from persisting an older projection. + unreconciled_worker_writes: std::collections::BTreeSet, subject_create_recovery_fences: BTreeMap, subject_release_operations: BTreeMap< (WorkerId, String), @@ -5216,6 +5272,7 @@ impl RuntimeState { #[cfg(feature = "fs-store")] next_diagnostic_id: 1, workers: BTreeMap::new(), + unreconciled_worker_writes: Default::default(), subject_create_recovery_fences: BTreeMap::new(), subject_release_operations: BTreeMap::new(), workdir_operations: BTreeMap::new(), @@ -5249,6 +5306,7 @@ impl RuntimeState { #[cfg(feature = "fs-store")] next_diagnostic_id: 1, workers: BTreeMap::new(), + unreconciled_worker_writes: Default::default(), subject_create_recovery_fences: BTreeMap::new(), subject_release_operations: BTreeMap::new(), workdir_operations: BTreeMap::new(), @@ -5277,112 +5335,13 @@ impl RuntimeState { let diagnostics = persisted.diagnostics; let next_diagnostic_id = persisted.next_diagnostic_id; for (worker_id, worker) in persisted.workers { - let ( - request, - execution_metadata_available, - execution_bound, - restore_intent, - pending_restore, - pending_stop, - ) = match worker.execution_state { - PersistedWorkerExecutionState::Available(execution) => ( - Some(execution.request), - true, - execution.binding.is_some(), - execution.restore_intent, - None, - None, - ), - PersistedWorkerExecutionState::ReconciliationRequired(operation) => match operation - { - PersistedWorkerLifecycleOperation::Restore(operation) => { - let restore_intent = - restore_intent_for_status(operation.last_settled_status); - ( - Some(operation.request), - true, - operation.binding.is_some(), - restore_intent, - Some(PendingWorkerRestore { - operation_id: operation.operation_id, - mode: operation.mode.into(), - last_settled_status: operation.last_settled_status, - }), - None, - ) - } - PersistedWorkerLifecycleOperation::Stop(operation) => ( - operation.request.clone(), - operation.request.is_some(), - operation.binding.is_some(), - operation.restore_intent, - operation - .pending_restore - .map(|restore| PendingWorkerRestore { - operation_id: restore.operation_id, - mode: restore.mode.into(), - last_settled_status: restore.last_settled_status, - }), - Some(PendingWorkerStop { - operation_id: operation.operation_id, - last_settled_status: operation.last_settled_status, - }), - ), - }, - PersistedWorkerExecutionState::Unavailable => ( - None, - false, - false, - restore_intent_for_status(worker.status), - None, - None, - ), - }; - let logical_workdir_attachments = if worker.logical_workdir_attachments.is_empty() { - let claims = request - .as_ref() - .map(|request| request.workdir_attachments.as_slice()) - .unwrap_or_default(); - logical_workdir_attachments_from_statuses(&worker.workdir_attachments, claims) - } else { - worker.logical_workdir_attachments - }; - workers.insert( - worker_id, - WorkerRecord { - restore_guard: { - let mut guard = worker.restore_guard; - // A prior active controller is not live in this process. - // Stopped observations remain stable across restart. - if worker.status.is_active() && execution_bound { - guard.rotate(); - } - guard - }, - worker_ref: worker.worker_ref, - worker_id: worker.worker_id, - status: worker.status, - worker_state: None, - last_finished_submission_request_id: worker.last_finished_submission_request_id, - execution_reconstructed: true, - workspace_id: worker.workspace_id, - profile: worker.profile, - display_name: worker.display_name, - profile_source: worker.profile_source, - config_bundle: worker.config_bundle, - request, - created_at_ms: worker.created_at_ms, - execution_metadata_available, - execution_bound, - restore_intent, - pending_restore, - pending_stop, - restore_candidate_context: None, - workdir_attachments: worker.workdir_attachments, - logical_workdir_attachments, - internal_workers: InternalWorkerActivityProjection::default(), - }, - ); + let mut record = WorkerRecord::from_persisted(worker); + // A prior active Controller is not live in this process. Keep + // stopped observations stable; canonical rereads never rotate them. + if record.status.is_active() && record.execution_bound { + record.restore_guard.rotate(); + } + workers.insert(worker_id, record); } Ok(Self { @@ -5396,6 +5355,7 @@ impl RuntimeState { workspace_backend_resource_clients: BTreeMap::new(), next_diagnostic_id, workers, + unreconciled_worker_writes: Default::default(), subject_create_recovery_fences: persisted.subject_create_recovery_fences, subject_release_operations: persisted .subject_releases @@ -5458,8 +5418,80 @@ impl RuntimeState { Ok(()) } + /// An uncertain Restore write is resolved only from the locked canonical + /// aggregate and a successful durability barrier, never a second blind write. + fn persist_restore_worker_record(&mut self, worker: &WorkerRecord) -> Result<(), RuntimeError> { + match self.persist_worker_record(worker) { + Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + self.unreconciled_worker_writes.insert(worker.worker_id); + #[cfg(feature = "fs-store")] + if let RuntimePersistence::Fs(store) = &self.persistence { + match store.reconcile_worker_record(&worker.persisted_record()) { + Ok((observed, matches)) => { + self.apply_canonical_worker_record(observed)?; + if matches { + return Ok(()); + } + } + Err(reconcile_error) => { + tracing::warn!(worker_id = %worker.worker_id, error = %reconcile_error, + "Restore aggregate remains uncertain after canonical reconciliation"); + } + } + } + Err(error) + } + outcome => outcome, + } + } + + #[cfg(feature = "fs-store")] + fn apply_canonical_worker_record( + &mut self, + observed: PersistedWorkerRecord, + ) -> Result<(), RuntimeError> { + let previous = self.worker(&observed.worker_ref)?; + let mut current = WorkerRecord::from_persisted(observed); + // Canonical authority replaces only the saved projection. A read is + // neither process reconstruction nor evidence that a Controller closed. + current.execution_reconstructed = previous.execution_reconstructed; + current.restore_candidate_context = previous.restore_candidate_context.clone(); + current.internal_workers = previous.internal_workers.clone(); + if current.status == previous.status { + current.worker_state = previous.worker_state.clone(); + } + self.unreconciled_worker_writes.remove(¤t.worker_id); + self.workers.insert(current.worker_id, current); + Ok(()) + } + + fn refresh_uncertain_worker_write( + &mut self, + worker_ref: &WorkerRef, + ) -> Result<(), RuntimeError> { + if !self + .unreconciled_worker_writes + .contains(&worker_ref.worker_id) + { + return Ok(()); + } + #[cfg(feature = "fs-store")] + if let RuntimePersistence::Fs(store) = &self.persistence { + let observed = store.read_durable_worker_record(worker_ref)?; + return self.apply_canonical_worker_record(observed); + } + Err(RuntimeError::InvalidRequest( + "Worker canonical write outcome remains unresolved".into(), + )) + } + #[cfg(feature = "fs-store")] fn persist_worker_record(&self, worker: &WorkerRecord) -> Result<(), RuntimeError> { + if self.unreconciled_worker_writes.contains(&worker.worker_id) { + return Err(RuntimeError::InvalidRequest( + "Worker canonical write outcome must be reconciled before another mutation".into(), + )); + } if let Some(store) = self.fs_store() { store.write_worker_record(&worker.persisted_record())?; } @@ -5468,20 +5500,25 @@ impl RuntimeState { #[cfg(feature = "fs-store")] fn persist_worker(&self, worker_id: &WorkerId) -> Result<(), RuntimeError> { - if let Some(store) = self.fs_store() { + if self.fs_store().is_some() { let worker = self.workers .get(worker_id) .ok_or_else(|| RuntimeError::WorkerNotFound { worker_id: *worker_id, })?; - store.write_worker_record(&worker.persisted_record())?; + self.persist_worker_record(worker)?; } Ok(()) } #[cfg(feature = "fs-store")] fn delete_worker_record(&self, worker_id: &WorkerId) -> Result<(), RuntimeError> { + if self.unreconciled_worker_writes.contains(worker_id) { + return Err(RuntimeError::InvalidRequest( + "Worker canonical write outcome must be reconciled before removal".into(), + )); + } if let Some(store) = self.fs_store() { store.delete_worker_record(worker_id)?; } @@ -5950,6 +5987,7 @@ impl RuntimeState { .and_then(|worker| worker.workspace_id.clone()) .unwrap_or_else(|| "".to_string()); let mut candidate = self.worker(worker_ref)?.clone(); + candidate.ensure_restore_rollback_allowed()?; candidate.execution_bound = false; candidate.restore_guard.finish(receipt.clone()); @@ -6274,10 +6312,21 @@ impl InternalWorkerActivityProjection { } } +/// A durable decision is separate from both admission and a live Controller. +/// Legacy pending operations lack sufficient evidence to choose either branch. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum RestoreCommitFact { + NotCommitted, + Committed, + LegacyUnknown, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct PendingWorkerRestore { operation_id: WorkerLifecycleOperationId, mode: WorkerRestoreMode, + commit: RestoreCommitFact, last_settled_status: WorkerStatus, } @@ -6410,6 +6459,17 @@ struct WorkerRecord { } impl WorkerRecord { + fn ensure_restore_rollback_allowed(&self) -> Result<(), RuntimeError> { + if self + .pending_restore + .is_some_and(|pending| pending.commit != RestoreCommitFact::NotCommitted) + { + return Err(RuntimeError::InvalidRequest( + "A committed or legacy-unknown Restore cannot be settled as a rollback".into(), + )); + } + Ok(()) + } fn apply_logical_workdir_attachments( &mut self, mut attachments: Vec, @@ -6592,6 +6652,105 @@ impl WorkerRecord { } } + #[cfg(feature = "fs-store")] + fn from_persisted(worker: PersistedWorkerRecord) -> Self { + let ( + request, + execution_metadata_available, + execution_bound, + restore_intent, + pending_restore, + pending_stop, + ) = match worker.execution_state { + PersistedWorkerExecutionState::Available(execution) => ( + Some(execution.request), + true, + execution.binding.is_some(), + execution.restore_intent, + None, + None, + ), + PersistedWorkerExecutionState::ReconciliationRequired(operation) => match operation { + PersistedWorkerLifecycleOperation::Restore(operation) => { + let restore_intent = restore_intent_for_status(operation.last_settled_status); + ( + Some(operation.request), + true, + operation.binding.is_some(), + restore_intent, + Some(PendingWorkerRestore { + operation_id: operation.operation_id, + mode: operation.mode.into(), + commit: operation.commit, + last_settled_status: operation.last_settled_status, + }), + None, + ) + } + PersistedWorkerLifecycleOperation::Stop(operation) => ( + operation.request.clone(), + operation.request.is_some(), + operation.binding.is_some(), + operation.restore_intent, + operation + .pending_restore + .map(|restore| PendingWorkerRestore { + operation_id: restore.operation_id, + mode: restore.mode.into(), + commit: restore.commit, + last_settled_status: restore.last_settled_status, + }), + Some(PendingWorkerStop { + operation_id: operation.operation_id, + last_settled_status: operation.last_settled_status, + }), + ), + }, + PersistedWorkerExecutionState::Unavailable => ( + None, + false, + false, + restore_intent_for_status(worker.status), + None, + None, + ), + }; + let logical_workdir_attachments = if worker.logical_workdir_attachments.is_empty() { + let claims = request + .as_ref() + .map(|request| request.workdir_attachments.as_slice()) + .unwrap_or_default(); + logical_workdir_attachments_from_statuses(&worker.workdir_attachments, claims) + } else { + worker.logical_workdir_attachments + }; + Self { + restore_guard: worker.restore_guard, + worker_ref: worker.worker_ref, + worker_id: worker.worker_id, + status: worker.status, + worker_state: None, + last_finished_submission_request_id: worker.last_finished_submission_request_id, + execution_reconstructed: true, + workspace_id: worker.workspace_id, + profile: worker.profile, + display_name: worker.display_name, + profile_source: worker.profile_source, + config_bundle: worker.config_bundle, + request, + created_at_ms: worker.created_at_ms, + execution_metadata_available, + execution_bound, + restore_intent, + pending_restore, + pending_stop, + restore_candidate_context: None, + workdir_attachments: worker.workdir_attachments, + logical_workdir_attachments, + internal_workers: InternalWorkerActivityProjection::default(), + } + } + #[cfg(feature = "fs-store")] fn persisted_record(&self) -> PersistedWorkerRecord { let execution_state = if let Some(pending) = self.pending_stop { @@ -6602,6 +6761,7 @@ impl WorkerRecord { PersistedWorkerRestoreOperation { operation_id: restore.operation_id, mode: restore.mode.into(), + commit: restore.commit, request: self .request .clone() @@ -6625,6 +6785,7 @@ impl WorkerRecord { PersistedWorkerLifecycleOperation::Restore(PersistedWorkerRestoreOperation { operation_id: pending.operation_id, mode: pending.mode.into(), + commit: pending.commit, request, binding: self .execution_bound @@ -8994,6 +9155,7 @@ mod tests { .pending_restore = Some(PendingWorkerRestore { operation_id: WorkerLifecycleOperationId::new(), mode: WorkerRestoreMode::Explicit, + commit: RestoreCommitFact::NotCommitted, last_settled_status: WorkerStatus::Stopped, }); let id = "workdir-irrelevant-to-recovery"; @@ -10933,6 +11095,7 @@ mod tests { })).unwrap()); runtime_store(&runtime) .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + runtime_store(&runtime).fail_next_worker_reconcile_sync(); assert_eq!( runtime .restore_worker_operation(&worker.worker_ref, request.clone()) @@ -11031,6 +11194,7 @@ mod tests { .operation_id; runtime_store(&runtime) .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + runtime_store(&runtime).fail_next_worker_reconcile_sync(); request.preparation = None; assert_eq!( runtime @@ -11221,6 +11385,7 @@ mod tests { ); runtime_store(&runtime) .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + runtime_store(&runtime).fail_next_worker_reconcile_sync(); assert_eq!( runtime .restore_worker_operation(&worker.worker_ref, request.clone()) @@ -11228,9 +11393,21 @@ mod tests { .state, WorkerRestoreState::ReconciliationRequired ); + // Resolve admission authority before injecting a different write fault. + // Otherwise the next failed sync would only exercise admission reread. + { + let lease = runtime.worker_operation_lock(worker.worker_id).unwrap(); + let _guard = lease.lock().unwrap(); + runtime + .lock() + .unwrap() + .refresh_uncertain_worker_write(&worker.worker_ref) + .unwrap(); + } // This write interrupts idempotent attachment completion after admission. runtime_store(&runtime) .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + runtime_store(&runtime).fail_next_worker_reconcile_sync(); assert_eq!( runtime .restore_worker_operation(&worker.worker_ref, request.clone()) @@ -11873,6 +12050,9 @@ mod tests { .unwrap() .operation_id; runtime_store(&runtime).fail_next_worker_write(fault); + if matches!(fault, crate::fs_store::AtomicWriteFault::AfterRename) { + runtime_store(&runtime).fail_next_worker_reconcile_sync(); + } gate.release(); assert_eq!( restoring.join().unwrap().unwrap().state, @@ -13137,6 +13317,7 @@ mod tests { assert_eq!(summary.worker_count, 0); } + #[cfg(feature = "fs-store")] #[test] fn delete_worker_persistence_failure_retains_identity_and_bounded_diagnostic() { let root = tempfile::tempdir().unwrap(); @@ -13236,6 +13417,7 @@ mod tests { )); } + #[cfg(feature = "fs-store")] #[test] fn delete_worker_waits_for_execution_shutdown_before_removing_snapshot() { let root = tempfile::tempdir().unwrap(); @@ -13406,7 +13588,7 @@ mod tests { }) .unwrap_err(); assert!(error.to_string().contains( - "unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, or 12" + "unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, or 13" )); let _ = std::fs::remove_dir_all(root); @@ -13448,7 +13630,10 @@ mod tests { let worker_aggregate: serde_json::Value = serde_json::from_slice(&std::fs::read(worker_store_dir.join("worker.json")).unwrap()) .unwrap(); - assert_eq!(worker_aggregate["schema_version"], serde_json::json!(12)); + assert_eq!( + worker_aggregate["schema_version"], + serde_json::json!(crate::fs_store::SCHEMA_VERSION) + ); assert_eq!(worker_aggregate["status"], serde_json::json!("stopped")); assert_eq!( worker_aggregate["execution_state"]["state"], @@ -14802,7 +14987,10 @@ mod tests { ); let migrated_aggregate: serde_json::Value = serde_json::from_slice(&std::fs::read(&worker_path).unwrap()).unwrap(); - assert_eq!(migrated_aggregate["schema_version"], serde_json::json!(12)); + assert_eq!( + migrated_aggregate["schema_version"], + serde_json::json!(crate::fs_store::SCHEMA_VERSION) + ); assert!(migrated_aggregate.get("request").is_none()); assert!(migrated_aggregate.get("execution").is_none()); assert_eq!( @@ -14826,6 +15014,184 @@ mod tests { let _ = std::fs::remove_dir_all(root); } + #[cfg(feature = "fs-store")] + #[test] + fn restore_decision_reread_preserves_commit_across_uncertainty_stop_and_restart() { + // Component scope: real Runtime aggregate owner/store, substituted + // execution backend. This does not attest Controller readiness or input. + for sync_failure in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("decision-reconciliation"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("decision fact")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let lease = runtime.worker_operation_lock(worker.worker_id).unwrap(); + let guard = lease.lock().unwrap(); + let operation_id = WorkerLifecycleOperationId::new(); + runtime + .begin_restore_operation( + &worker.worker_ref, + operation_id, + WorkerRestoreMode::Explicit, + runtime.restore_candidate_execution_context(worker.worker_ref.clone()), + ) + .unwrap(); + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + if sync_failure { + runtime_store(&runtime).fail_next_worker_reconcile_sync(); + } + let result = runtime.commit_restored_worker_execution( + &worker.worker_ref, + protocol::WorkerStatus::Idle.into(), + WorkerStatus::Idle, + Vec::new(), + ); + assert_eq!(result.is_ok(), !sync_failure); + let path = options + .root + .join("workers") + .join(worker.worker_id.to_string()) + .join("worker.json"); + let bytes = std::fs::read(&path).unwrap(); + let observed = runtime_store(&runtime) + .read_worker_record(&worker.worker_ref) + .unwrap(); + let PersistedWorkerExecutionState::ReconciliationRequired( + PersistedWorkerLifecycleOperation::Restore(pending), + ) = observed.execution_state + else { + panic!("committed decision lost its operation") + }; + assert_eq!(pending.operation_id, operation_id); + assert_eq!(pending.commit, RestoreCommitFact::Committed); + assert_eq!( + pending.request, + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .request + .clone() + .unwrap() + ); + let rollback = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "must-not-revert", + "committed decision cannot become rollback", + ); + assert!( + runtime + .settle_restore_rollback(&worker.worker_ref, &rollback) + .is_err() + ); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + // Cleanup confirmation from a backend is not evidence that the + // Runtime's durable decision was uncommitted. + assert!( + runtime + .lock() + .unwrap() + .record_restore_failure( + &worker.worker_ref, + &WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "backend confirmed cleanup after reconstruction failure", + ), + &rollback, + ) + .is_err() + ); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + if sync_failure { + assert!( + runtime + .lock() + .unwrap() + .unreconciled_worker_writes + .contains(&worker.worker_id) + ); + // Substitute unreadable storage only. No production recovery + // path repairs it from an old memory record. + std::fs::write(&path, b"not-json").unwrap(); + assert!( + runtime + .lock() + .unwrap() + .refresh_uncertain_worker_write(&worker.worker_ref) + .is_err() + ); + assert!( + runtime + .lock() + .unwrap() + .persist_worker(&worker.worker_id) + .is_err() + ); + assert!( + runtime + .begin_stop_operation(&worker.worker_ref, WorkerLifecycleOperationId::new()) + .is_err() + ); + assert_eq!(std::fs::read(&path).unwrap(), b"not-json"); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + // End the storage fault with the exact post-rename bytes, not + // an inferred projection; recovery still performs its real sync. + std::fs::write(&path, &bytes).unwrap(); + runtime + .lock() + .unwrap() + .refresh_uncertain_worker_write(&worker.worker_ref) + .unwrap(); + } + let current = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap(); + assert_eq!(current.commit, RestoreCommitFact::Committed); + assert_eq!(current.operation_id, operation_id); + runtime + .begin_stop_operation(&worker.worker_ref, WorkerLifecycleOperationId::new()) + .unwrap(); + let saved = runtime_store(&runtime) + .read_worker_record(&worker.worker_ref) + .unwrap(); + let PersistedWorkerExecutionState::ReconciliationRequired( + PersistedWorkerLifecycleOperation::Stop(stop), + ) = saved.execution_state + else { + panic!("Stop ownership was not saved") + }; + assert_eq!( + stop.pending_restore.unwrap().commit, + RestoreCommitFact::Committed + ); + drop(guard); + drop(lease); + drop(runtime); + let reopened = Runtime::with_fs_store(options).unwrap(); + let state = reopened.lock().unwrap(); + let restored = state.worker(&worker.worker_ref).unwrap(); + assert!(restored.pending_stop.is_some()); + assert_eq!( + restored.pending_restore.unwrap().commit, + RestoreCommitFact::Committed + ); + assert_eq!(restored.pending_restore.unwrap().operation_id, operation_id); + } + } + #[cfg(feature = "fs-store")] #[test] fn fs_store_preserves_an_unknown_restore_operation_commit() { @@ -14849,9 +15215,11 @@ mod tests { runtime_store(&runtime) .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + runtime_store(&runtime).fail_next_worker_reconcile_sync(); + let operation_id = WorkerLifecycleOperationId::new(); let result = runtime.begin_restore_operation( &worker.worker_ref, - WorkerLifecycleOperationId::new(), + operation_id, WorkerRestoreMode::Explicit, runtime.restore_candidate_execution_context(worker.worker_ref.clone()), ); @@ -14866,14 +15234,18 @@ mod tests { runtime.worker_detail(&worker.worker_ref).unwrap().status, WorkerStatus::Stopped ); - let operation_id = runtime - .lock() - .unwrap() - .worker(&worker.worker_ref) - .unwrap() - .pending_restore - .unwrap() - .operation_id; + { + let state = runtime.lock().unwrap(); + assert!(state.unreconciled_worker_writes.contains(&worker.worker_id)); + assert!( + state + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .is_none(), + "failed sync must not promote an unconfirmed in-memory candidate" + ); + } let aggregate: serde_json::Value = serde_json::from_slice( &std::fs::read( root.join("workers") diff --git a/crates/worker-runtime/src/test_support.rs b/crates/worker-runtime/src/test_support.rs index fb0f32693..ca9192f8f 100644 --- a/crates/worker-runtime/src/test_support.rs +++ b/crates/worker-runtime/src/test_support.rs @@ -376,6 +376,8 @@ impl StoppedWorkerTemplate { root: stage.clone(), _owner_lock: Some(lock), worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), + #[cfg(test)] + worker_reconcile_sync_fault: Default::default(), }; let result = (|| { let worker = PersistedWorkerRecord { @@ -463,8 +465,15 @@ impl StoppedWorkerTemplate { } fn validate_shape(&self) -> Result<(), RuntimeError> { + // Version 1 templates carry only settled Session material, never a + // lifecycle operation. Schema 12 -> 13 changes pending decision evidence, + // not that payload. Accept the fixed schema-12 artifact read-only and + // install through the current aggregate owner; never rewrite its bytes. if self.template_version != TEMPLATE_VERSION - || self.runtime_schema_version != SCHEMA_VERSION + || !matches!( + self.runtime_schema_version, + RESTORE_ADMISSION_SCHEMA_VERSION | SCHEMA_VERSION + ) { return Err(reject("unsupported template schema")); } diff --git a/crates/worker-runtime/src/test_support_tests.rs b/crates/worker-runtime/src/test_support_tests.rs index 203434258..a4931def8 100644 --- a/crates/worker-runtime/src/test_support_tests.rs +++ b/crates/worker-runtime/src/test_support_tests.rs @@ -563,6 +563,7 @@ fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() { PersistedWorkerLifecycleOperation::Restore(PersistedWorkerRestoreOperation { operation_id: WorkerLifecycleOperationId::new(), mode: PersistedWorkerRestoreMode::Explicit, + commit: crate::runtime::RestoreCommitFact::NotCommitted, request: request(), binding: None, last_settled_status: WorkerStatus::Stopped, @@ -632,6 +633,38 @@ fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() { drop(held); } +#[test] +fn schema_v12_settled_template_installs_current_aggregate_without_rewriting_session() { + let (_source, _, mut template) = source(); + template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION; + let bytes = serde_json::to_vec(&template).unwrap(); + let (_target, options) = destination(&template); + template.install(&options, request(), &resolved()).unwrap(); + assert_eq!(serde_json::to_vec(&template).unwrap(), bytes); + let worker = options.root.join(WORKERS_DIR).join(worker_id().to_string()); + assert_eq!( + fs::read(worker.join("session/session.json")).unwrap(), + template.session_manifest + ); + assert_eq!( + fs::read(worker.join(format!("session/segments/{}.jsonl", template.segment_id))).unwrap(), + template.segment_jsonl + ); + let aggregate: WorkerAggregateRecord = + read_bounded_json(&worker.join(WORKER_FILE), "read installed fixture").unwrap(); + let saved = aggregate.validate(&worker.join(WORKER_FILE)).unwrap(); + assert_eq!(saved.status, WorkerStatus::Stopped); + assert!(matches!( + saved.execution_state, + PersistedWorkerExecutionState::Available(_) + )); + // Unsupported template schemas do not enter the versioned reader. + template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION - 1; + assert!(template.validate_shape().is_err()); + template.runtime_schema_version = SCHEMA_VERSION + 1; + assert!(template.validate_shape().is_err()); +} + #[test] fn fresh_manifest_and_memory_settings_are_required() { let (_source, _, template) = source(); diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 87c0f676c..cf6bfe5d5 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -25,13 +25,17 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, RwLock, mpsc}; use std::time::Duration; -use crate::auth::{BACKEND_RESOURCE_FETCH_PERMISSION, RuntimeIdentityMaterial}; +#[cfg(feature = "http-server")] +use crate::auth::BACKEND_RESOURCE_FETCH_PERMISSION; +use crate::auth::RuntimeIdentityMaterial; use crate::catalog::{ CreateWorkerRequest, ProfileSourceArchiveSource, RepositoryRefObservation, RepositoryRefObservationRequest, WorkingDirectoryAttachmentStatus, WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest, WorkingDirectoryStatus, }; -use crate::config_bundle::{ConfigBundle, workspace_config_etag}; +use crate::config_bundle::ConfigBundle; +#[cfg(feature = "http-server")] +use crate::config_bundle::workspace_config_etag; #[cfg(feature = "ws-server")] use crate::execution::WorkerProtocolTransport; use crate::execution::{ @@ -43,12 +47,13 @@ use crate::execution::{ use crate::identity::WorkerRef; use crate::interaction::{WorkerInput, WorkerInputKind}; use crate::resource::BackendResourceClient; -use crate::worker_source::{ - EmbeddedWorkerMutationDispatcher, RuntimeOwnedWorkspaceClient, RuntimeWorkerMutationForwarder, -}; +use crate::worker_source::EmbeddedWorkerMutationDispatcher; +#[cfg(feature = "http-server")] +use crate::worker_source::{RuntimeOwnedWorkspaceClient, RuntimeWorkerMutationForwarder}; use crate::working_directory::{ WorkingDirectoryBinding, WorkingDirectoryDiagnostic, WorkingDirectoryMaterializer, }; +#[cfg(feature = "http-server")] use crate::workspace_request::{RuntimeWorkspaceRequest, RuntimeWorkspaceRequestClient}; use async_trait::async_trait; #[cfg(test)] @@ -137,6 +142,8 @@ use workdir::{ #[cfg(test)] use worker::WorkerController; +#[cfg(any(feature = "http-server", test))] +use worker::WorkspaceId; use worker::feature::builtin::manage_workdir::WorkspaceAttachedWorkdirSession; use worker::feature::builtin::{ CompositeWorkerObservationProvider, WorkerObservationError, WorkerObservationProvider, @@ -146,10 +153,10 @@ use worker::feature::builtin::{ #[cfg(feature = "ws-server")] use worker::ipc::protocol_session::{WorkerProtocolSessionStreams, live_log_entry_event}; use worker::{ - PreparedWorker, PromptCatalogSource, SegmentLogSink, SubjektivSessionAttributionLifecycle, - Worker, WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout, - WorkerControllerTransport, WorkerError, WorkerFilesystemAuthority, WorkerHandle, - WorkerSharedState, WorkerWorkspaceContext, WorkspaceClient, WorkspaceId, + PromptCatalogSource, SegmentLogSink, SubjektivSessionAttributionLifecycle, Worker, + WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout, WorkerControllerTransport, + WorkerError, WorkerFilesystemAuthority, WorkerHandle, WorkerSharedState, + WorkerWorkspaceContext, WorkspaceClient, }; #[cfg(feature = "ws-server")] @@ -165,8 +172,13 @@ const DEFAULT_BACKEND_ID: &str = "worker-crate"; const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10); const SPAWN_RESTORE_TASK_TIMEOUT: Duration = Duration::from_secs(60); const USER_INPUT_TASK_TIMEOUT: Duration = Duration::from_secs(10); +#[cfg(feature = "http-server")] const WORKSPACE_CONFIG_HTTP_TIMEOUT: Duration = Duration::from_secs(8); +#[cfg(feature = "http-server")] const MAX_WORKSPACE_CONFIG_RESPONSE_BYTES: usize = 72 * 1024 * 1024; +#[cfg(not(feature = "http-server"))] +const WORKSPACE_HTTP_TRANSPORT_UNAVAILABLE: &str = + "Workspace API HTTP transport is unavailable: worker-runtime requires the http-server feature"; // Leave adapter cancellation margin after the durable submission deadline. const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(9); @@ -175,6 +187,7 @@ pub struct RuntimeWorkerController { pub shutdown: Arc>>, pub controller_task: tokio::task::JoinHandle<()>, pub workspace_client: Arc, + pub restore_streams: Option, } /// An unpublished restore resource owner. No protocol endpoint is available @@ -184,6 +197,14 @@ pub struct RuntimeWorkerController { pub trait RuntimeRestoreCandidate: Send { fn initial_state(&self) -> protocol::WorkerStateSnapshot; + fn has_durable_effects(&self) -> bool { + false + } + + fn can_retry_publication(&self) -> bool { + false + } + async fn activate(&mut self) -> Result; async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure>; @@ -198,6 +219,7 @@ pub enum RuntimeRestoreStage { Recovery, Binding, Feature, + ActivationEffects, Controller, } @@ -379,6 +401,14 @@ impl RuntimeRestoreCandidate for EffectRetainingRestoreCandidate { self.candidate.initial_state() } + fn has_durable_effects(&self) -> bool { + true + } + + fn can_retry_publication(&self) -> bool { + self.candidate.can_retry_publication() + } + async fn activate(&mut self) -> Result { self.candidate.activate().await.map_err(|mut failure| { failure.durable_effects_possible = true; @@ -493,7 +523,8 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { _worker_ref: &WorkerRef, _workspace_id: Option<&str>, _handle: &WorkerHandle, - ) { + ) -> Result<(), String> { + Ok(()) } } @@ -512,17 +543,25 @@ struct RuntimeObservedWorker { } impl RuntimeWorkerObservationHub { - fn register(&self, worker_ref: WorkerRef, workspace_id: Option, handle: &WorkerHandle) { - if let Ok(mut workers) = self.workers.lock() { - workers.insert( - worker_ref, - RuntimeObservedWorker { - workspace_id, - shared_state: Arc::downgrade(&handle.shared_state), - sink: handle.sink.clone(), - }, - ); - } + fn register( + &self, + worker_ref: WorkerRef, + workspace_id: Option, + handle: &WorkerHandle, + ) -> Result<(), String> { + let mut workers = self + .workers + .lock() + .map_err(|_| "Worker observation registry lock is poisoned".to_string())?; + workers.insert( + worker_ref, + RuntimeObservedWorker { + workspace_id, + shared_state: Arc::downgrade(&handle.shared_state), + sink: handle.sink.clone(), + }, + ); + Ok(()) } fn get( @@ -739,6 +778,7 @@ pub struct ProfileRuntimeWorkerFactory { prompt_projection_cache: Arc, runtime_id: Option, worker_mutation_identity: Option, + #[cfg(feature = "http-server")] workspace_request_clients: Arc>, embedded_worker_mutation_dispatcher: Option>, controller_transport: WorkerControllerTransport, @@ -763,8 +803,11 @@ struct RestoreFaults { executions: std::sync::atomic::AtomicUsize, cleanup_fail: AtomicBool, socket_failure: AtomicBool, + publication_failure: AtomicBool, factory_gate: Mutex>>, factory_entered: Mutex>>, + prepared_gate: Mutex>>, + prepared_entered: Mutex>>, cleanup_gate: Mutex>>, cleanup_entered: Mutex>>, } @@ -825,6 +868,7 @@ impl ProfileRuntimeWorkerFactory { prompt_projection_cache: Arc::new(WorkspacePromptProjectionCache::default()), runtime_id: None, worker_mutation_identity: None, + #[cfg(feature = "http-server")] workspace_request_clients: Arc::new(HashMap::new()), embedded_worker_mutation_dispatcher: None, controller_transport: WorkerControllerTransport::InProcess, @@ -860,6 +904,7 @@ impl ProfileRuntimeWorkerFactory { self } + #[cfg(feature = "http-server")] pub fn with_workspace_request_client(mut self, client: RuntimeWorkspaceRequestClient) -> Self { self.runtime_id = Some(client.runtime_id().to_string()); Arc::make_mut(&mut self.workspace_request_clients) @@ -1168,6 +1213,7 @@ impl ProfileRuntimeWorkerFactory { #[derive(Debug, Clone)] enum RuntimeWorkspaceBackendRef { None, + #[cfg(feature = "http-server")] Http { workspace_id: String, base_url: String, @@ -1176,28 +1222,45 @@ enum RuntimeWorkspaceBackendRef { } impl RuntimeWorkspaceBackendRef { - fn from_worker_request(request: &CreateWorkerRequest, runtime_id: Option<&str>) -> Self { - if let (Some(api), Some(runtime_id)) = (request.workspace_api.as_ref(), runtime_id) { - return Self::Http { + fn from_worker_request( + request: &CreateWorkerRequest, + runtime_id: Option<&str>, + ) -> Result { + #[cfg(not(feature = "http-server"))] + if request.workspace_api.is_some() { + return Err(WORKSPACE_HTTP_TRANSPORT_UNAVAILABLE); + } + match (request.workspace_api.as_ref(), runtime_id) { + #[cfg(feature = "http-server")] + (Some(api), Some(runtime_id)) => Ok(Self::Http { workspace_id: api.workspace_id.clone(), base_url: api.base_url.clone(), runtime_id: runtime_id.to_string(), - }; + }), + _ => Ok(Self::None), } - Self::None } fn worker_context( &self, - worker_ref: &WorkerRef, - workspace_scope: Option<&crate::runtime::RuntimeWorkspaceScope>, - mutation_identity: Option<&RuntimeIdentityMaterial>, - workspace_request_client: Option<&RuntimeWorkspaceRequestClient>, - embedded_dispatcher: Option<&Arc>, - prompt_projection_cache: Option>, + #[cfg(feature = "http-server")] worker_ref: &WorkerRef, + #[cfg(feature = "http-server")] workspace_scope: Option< + &crate::runtime::RuntimeWorkspaceScope, + >, + #[cfg(feature = "http-server")] mutation_identity: Option<&RuntimeIdentityMaterial>, + #[cfg(feature = "http-server")] workspace_request_client: Option< + &RuntimeWorkspaceRequestClient, + >, + #[cfg(feature = "http-server")] embedded_dispatcher: Option< + &Arc, + >, + #[cfg(feature = "http-server")] prompt_projection_cache: Option< + Arc, + >, ) -> WorkerWorkspaceContext { match self { Self::None => WorkerWorkspaceContext::no_workspace(), + #[cfg(feature = "http-server")] Self::Http { workspace_id, base_url, @@ -1327,14 +1390,6 @@ async fn fetch_workspace_config_http( Ok(WorkspaceConfigFetchResult::Modified(bundle)) } -#[cfg(not(feature = "http-server"))] -async fn fetch_workspace_config_http( - _request: &WorkspaceConfigFetchRequest, - _client: &T, -) -> Result { - Err("Workspace Config fetch requires the worker-runtime http-server feature".to_string()) -} - fn initial_workdir_session_capabilities( alias: &WorkdirAttachmentAlias, claims: &[crate::catalog::WorkingDirectoryAttachmentClaim], @@ -1767,16 +1822,24 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { &self, request: WorkspaceConfigFetchRequest, ) -> Result { - let client = self - .workspace_request_clients - .get(&request.workspace_api.workspace_id) - .ok_or_else(|| { - format!( - "Workspace request client is unavailable for workspace {}", - request.workspace_api.workspace_id - ) - })?; - fetch_workspace_config_http(&request, client).await + #[cfg(feature = "http-server")] + { + let client = self + .workspace_request_clients + .get(&request.workspace_api.workspace_id) + .ok_or_else(|| { + format!( + "Workspace request client is unavailable for workspace {}", + request.workspace_api.workspace_id + ) + })?; + fetch_workspace_config_http(&request, client).await + } + #[cfg(not(feature = "http-server"))] + { + let _ = request; + Err("Workspace Config fetch is unsupported: worker-runtime requires the http-server feature".to_string()) + } } fn retained_session_snapshot( @@ -1866,7 +1929,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { let workspace_backend_ref = RuntimeWorkspaceBackendRef::from_worker_request( &request.request, self.runtime_id.as_deref(), - ); + )?; let observation_runtime_id = self.runtime_id.clone(); let observation_workspace_id = request .request @@ -1875,17 +1938,24 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .map(|api| api.workspace_id.clone()); let observation_grants = request.request.worker_observation_grants.clone(); let observation_enabled = request.request.worker_observation_enabled; + #[cfg(feature = "http-server")] let workspace_request_client = request .request .workspace_api .as_ref() .and_then(|api| self.workspace_request_clients.get(&api.workspace_id)); let workspace_context = workspace_backend_ref.worker_context( + #[cfg(feature = "http-server")] &request.worker_ref, + #[cfg(feature = "http-server")] request.workspace_scope.as_ref(), + #[cfg(feature = "http-server")] self.worker_mutation_identity.as_ref(), + #[cfg(feature = "http-server")] workspace_request_client, + #[cfg(feature = "http-server")] self.embedded_worker_mutation_dispatcher.as_ref(), + #[cfg(feature = "http-server")] Some(self.prompt_projection_cache.clone()), ); let selector = profile.as_ref(); @@ -2044,16 +2114,19 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { if flow_transition_enabled { handle.shared_state.enable_flow_transition(); } - self.observation_hub.register( + if let Err(error) = self.observation_hub.register( request.worker_ref.clone(), observation_workspace_id, &handle, - ); + ) { + tracing::warn!(%error, "Worker observation registration unavailable"); + } Ok(RuntimeWorkerController { handle, shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), controller_task, workspace_client, + restore_streams: None, }) } @@ -2061,6 +2134,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { &self, request: &WorkerExecutionRestoreRequest, ) -> Result<(), String> { + #[cfg(not(feature = "http-server"))] + RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + self.runtime_id.as_deref(), + )?; let worker_name = Self::runtime_worker_name_for_ref(&request.worker_ref); let worker_aggregate_dir = self.worker_aggregate_dir(&request.worker_ref)?; if !worker_aggregate_dir.is_dir() { @@ -2134,6 +2212,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { &self, request: &WorkerExecutionRestoreRequest, ) -> Result { + #[cfg(not(feature = "http-server"))] + RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + self.runtime_id.as_deref(), + ) + .map_err(|cause| RuntimeRestoreFailure::refused(RuntimeRestoreStage::Preflight, cause))?; self.recover_saved_restore_source(request) .await .map_err(|mut failure| { @@ -2146,6 +2230,12 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { &self, request: WorkerExecutionRestoreRequest, ) -> Result, RuntimeRestoreFailure> { + #[cfg(not(feature = "http-server"))] + RuntimeWorkspaceBackendRef::from_worker_request( + &request.request, + self.runtime_id.as_deref(), + ) + .map_err(|cause| RuntimeRestoreFailure::refused(RuntimeRestoreStage::Preflight, cause))?; self.preflight_restore(&request).await.map_err(|_| { RuntimeRestoreFailure::refused( RuntimeRestoreStage::Preflight, @@ -2157,6 +2247,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .executions .fetch_add(1, Ordering::SeqCst); let mut stage = RuntimeRestoreStage::Metadata; + let mut preparation_failure = None; // Reconstruction owns read-only resources; cleanup confirmation must // come from that owner even when it cannot return a candidate. let result: Result, &'static str> = async { @@ -2178,18 +2269,25 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { let workspace_backend_ref = RuntimeWorkspaceBackendRef::from_worker_request( &request.request, self.runtime_id.as_deref(), - ); + )?; + #[cfg(feature = "http-server")] let workspace_request_client = request .request .workspace_api .as_ref() .and_then(|api| self.workspace_request_clients.get(&api.workspace_id)); let workspace_context = workspace_backend_ref.worker_context( + #[cfg(feature = "http-server")] &request.worker_ref, + #[cfg(feature = "http-server")] request.workspace_scope.as_ref(), + #[cfg(feature = "http-server")] self.worker_mutation_identity.as_ref(), + #[cfg(feature = "http-server")] workspace_request_client, + #[cfg(feature = "http-server")] self.embedded_worker_mutation_dispatcher.as_ref(), + #[cfg(feature = "http-server")] Some(self.prompt_projection_cache.clone()), ); @@ -2272,14 +2370,19 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .map_err(|_| "injected restore boundary failure")?; validate_worker_memory_settings(candidate.manifest(), &request.request) .map_err(|_| "saved memory settings mismatch")?; - Ok(Box::new(ProfileRestoreCandidate { - prepared: Some(candidate), - factory: self.clone(), - request: request.clone(), - }) as Box) + self.prepare_unpublished_restore(candidate, &request) + .await + .map(|candidate| Box::new(candidate) as Box) + .map_err(|failure| { + let cause = failure.cause; + preparation_failure = Some(failure); + cause + }) } .await; - result.map_err(|cause| RuntimeRestoreFailure::started(stage, cause)) + result.map_err(|cause| { + preparation_failure.unwrap_or_else(|| RuntimeRestoreFailure::started(stage, cause)) + }) } async fn cleanup_failed_restore( @@ -2429,51 +2532,128 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { worker_ref: &WorkerRef, workspace_id: Option<&str>, handle: &WorkerHandle, - ) { + ) -> Result<(), String> { self.observation_hub - .register(worker_ref.clone(), workspace_id.map(str::to_string), handle); + .register(worker_ref.clone(), workspace_id.map(str::to_string), handle) } } type ProfileRestoreStore = CombinedStore; struct ProfileRestoreCandidate { - prepared: Option>, + prepared: Option>, + workspace_client: Arc, factory: ProfileRuntimeWorkerFactory, request: WorkerExecutionRestoreRequest, + #[cfg(test)] + effects_boundary_failed: bool, } #[async_trait] impl RuntimeRestoreCandidate for ProfileRestoreCandidate { fn initial_state(&self) -> protocol::WorkerStateSnapshot { - protocol::WorkerStateSnapshot::initial() + match self + .prepared + .as_ref() + .expect("same unpublished Controller") + .snapshot() + { + protocol::Event::Snapshot { state, .. } => state.clone(), + _ => unreachable!("prepared Controller full snapshot"), + } + } + + fn can_retry_publication(&self) -> bool { + #[cfg(test)] + if self.effects_boundary_failed { + return false; + } + self.prepared + .as_ref() + .is_some_and(|prepared| prepared.can_retry_publication()) } async fn activate(&mut self) -> Result { - let prepared = self.prepared.take().ok_or_else(|| { + let prepared = self.prepared.as_mut().ok_or_else(|| { RuntimeRestoreFailure::unproven( RuntimeRestoreStage::Controller, "restore candidate already consumed", ) })?; - let worker = prepared.activate().await.map_err(|error| { - self.factory - .retain_construction_failure(&self.request, &error); - let mut failure = RuntimeRestoreFailure::started( - RuntimeRestoreStage::Reconstruction, - restore_worker_cause(&error), - ); - failure.durable_effects_possible = - matches!(error, WorkerError::RestoreActivationEffects { .. }); + let map_error = |error: WorkerBootstrapError| { + let (stage, cause, io_kind) = match &error { + WorkerBootstrapError::Worker(error) => { + self.factory + .retain_construction_failure(&self.request, error); + ( + RuntimeRestoreStage::Reconstruction, + restore_worker_cause(error), + None, + ) + } + WorkerBootstrapError::Controller { source, .. } => ( + RuntimeRestoreStage::Controller, + "restored Worker publication failed", + Some(source.kind()), + ), + }; + let mut failure = RuntimeRestoreFailure::started(stage, cause); + failure.durable_effects_possible = true; + failure.io_kind = io_kind; failure - })?; + }; + prepared.apply_effects().await.map_err(&map_error)?; self.factory - .activate_prepared_restore(worker, &self.request) - .await - .map_err(|mut failure| { - failure.durable_effects_possible = true; - failure - }) + .failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get_mut(&self.request.worker_ref) + .expect("retained restore owner") + .allocation = prepared.allocation_cleanup(); + #[cfg(test)] + if self + .factory + .restore_checkpoint(RuntimeRestoreStage::ActivationEffects) + .is_err() + { + self.effects_boundary_failed = true; + let mut failure = RuntimeRestoreFailure::started( + RuntimeRestoreStage::ActivationEffects, + "injected restore boundary failure", + ); + failure.durable_effects_possible = true; + return Err(failure); + } + let started = async { + #[cfg(test)] + if self + .factory + .restore_faults + .publication_failure + .swap(false, Ordering::SeqCst) + { + return Err(WorkerBootstrapError::Controller { + source: std::io::Error::other("injected publication boundary failure"), + cleanup_failed: true, + }); + } + prepared.activate().await + } + .await + .map_err(map_error)?; + self.prepared = None; + self.factory + .failed_restore_resources + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&self.request.worker_ref); + Ok(RuntimeWorkerController { + handle: started.controller.handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.controller.shutdown))), + controller_task: started.controller.controller_task, + workspace_client: self.workspace_client.clone(), + restore_streams: Some(started.streams), + }) } async fn discard(&mut self) -> Result<(), RuntimeRestoreCleanupFailure> { @@ -2489,52 +2669,32 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate { } impl ProfileRuntimeWorkerFactory { - async fn activate_prepared_restore( + async fn prepare_unpublished_restore( &self, - mut worker: Worker< - Box, - CombinedStore, - >, + mut candidate: worker::bootstrap::PreparedRestoredWorker, request: &WorkerExecutionRestoreRequest, - ) -> Result { + ) -> Result { let mut stage = RuntimeRestoreStage::Binding; let mut io_kind = None; - let result: Result = async { - let feature_tasks = worker.retain_startup_feature_cleanup(); + let result: Result = async { + let feature_tasks = candidate + .retain_startup_feature_cleanup() + .map_err(|_| "restore cleanup ownership unavailable")?; self.failed_restore_resources .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .insert( - request.worker_ref.clone(), - FailedRestoreResources { - operation_id: request.operation_id, - sessions: vec![worker.workdir_sessions()], - candidate_workdir_cleanup: None, - allocation: worker.restore_allocation_cleanup_handle(), - feature_tasks: Some(feature_tasks), - }, - ); - let observation_runtime_id = self.runtime_id.clone(); - let observation_workspace_id = request - .request - .workspace_api - .as_ref() - .map(|api| api.workspace_id.clone()); - let observation_grants = request.request.worker_observation_grants.clone(); - let observation_enabled = request.request.worker_observation_enabled; - if let Some(binding) = request.request.backend_job.as_ref() { - worker - .bind_backend_job(binding.clone()) - .map_err(|_| "restored Backend Job binding failed")?; - } - validate_worker_memory_settings(worker.manifest(), &request.request) - .map_err(|_| "saved memory settings mismatch")?; + .get_mut(&request.worker_ref) + .expect("retained restore owner") + .feature_tasks = Some(feature_tasks); + let workspace_client = candidate + .workspace_client_handle() + .map_err(|_| "restore Workspace binding unavailable")?; let workdir_sessions = restored_workdir_router( &request.workdir_attachments, &request.logical_workdir_attachments, &request.deferred_workdir_sessions, &request.request.workdir_attachments, - worker.workspace_client_handle(), + workspace_client.clone(), ) .map_err(|_| "restored Workdir binding failed")?; self.failed_restore_resources @@ -2544,56 +2704,43 @@ impl ProfileRuntimeWorkerFactory { .expect("retained restore owner") .sessions .push(workdir_sessions.clone()); - worker - .workdir_sessions() - .close_all() - .await - .map_err(|_| "initial Workdir close failed")?; - worker.bind_workdir_sessions(workdir_sessions); - if let (Some(runtime_id), Some(workspace_id)) = - (observation_runtime_id, observation_workspace_id.clone()) - && observation_enabled + let mut providers: Vec> = vec![Arc::new( + WorkspaceClientWorkerObservationProvider::new(workspace_client.clone()), + )]; + let observation_provider = if let (Some(runtime_id), Some(api)) = ( + self.runtime_id.clone(), + request.request.workspace_api.as_ref(), + ) && request.request.worker_observation_enabled { - let mut providers: Vec> = vec![Arc::new( - WorkspaceClientWorkerObservationProvider::new(worker.workspace_client_handle()), - )]; - if !observation_grants.is_empty() { + if !request.request.worker_observation_grants.is_empty() { providers.push(Arc::new(RuntimeGrantedWorkerObservationProvider { runtime_id, - workspace_id, - grants: observation_grants.into_iter().take(100).collect(), + workspace_id: api.workspace_id.clone(), + grants: request + .request + .worker_observation_grants + .iter() + .take(100) + .cloned() + .collect(), hub: self.observation_hub.clone(), })); } - worker.bind_worker_observation_provider(Some(Arc::new( - CompositeWorkerObservationProvider::new(providers), - ))); - } - worker - .finalize_subjektiv_session_attribution( - request.request.subjektiv_attached, - SubjektivSessionAttributionLifecycle::RestoredSession, - ) - .map_err(|error| restore_worker_cause(&error))?; - let flow_transition_enabled = worker.manifest().feature.flow.enabled; - stage = RuntimeRestoreStage::Feature; + Some(Arc::new(CompositeWorkerObservationProvider::new(providers)) + as Arc) + } else { + None + }; #[cfg(test)] - self.restore_checkpoint(stage) - .map_err(|_| "injected restore boundary failure")?; - if flow_transition_enabled { - let report = worker - .install_runtime_flow_transition_feature() - .map_err(|_| "Flow transition installation failed")?; - if report.reports.iter().any(|report| !report.installed) { - return Err("Flow transition installation refused"); - } + { + stage = RuntimeRestoreStage::Feature; + self.restore_checkpoint(stage) + .map_err(|_| "injected restore boundary failure")?; } - stage = RuntimeRestoreStage::Controller; #[cfg(test)] self.restore_checkpoint(stage) .map_err(|_| "injected restore boundary failure")?; - let workspace_client = worker.workspace_client_handle(); let run_dir = self .worker_restore_run_dir(&request.worker_ref, request.operation_id) .map_err(|_| "restore run directory unavailable")?; @@ -2604,36 +2751,48 @@ impl ProfileRuntimeWorkerFactory { } else { transport }; - let started = PreparedWorker::new( - worker, - WorkerBootstrapLayout::RuntimeManagedRun { - run_dir: run_dir.clone(), - }, - transport, - ) - .start() - .await - .map_err(|error| match error { - WorkerBootstrapError::Worker(error) => restore_worker_cause(&error), - WorkerBootstrapError::Controller { source, .. } => { - io_kind = Some(source.kind()); - "restored Worker controller startup failed" + let prepared = candidate + .prepare_runtime_controller( + &run_dir, + transport, + worker::bootstrap::RestoredControllerBindings { + backend_job: request.request.backend_job.clone(), + workdir_sessions, + observation_provider, + subjektiv_attached: request.request.subjektiv_attached, + }, + ) + .await + .map_err(|error| match error { + WorkerBootstrapError::Worker(error) => { + stage = RuntimeRestoreStage::Reconstruction; + self.retain_construction_failure(request, &error); + restore_worker_cause(&error) + } + WorkerBootstrapError::Controller { source, .. } => { + io_kind = Some(source.kind()); + "restored Worker preparation failed" + } + })?; + #[cfg(test)] + { + let gate = self.restore_faults.prepared_gate.lock().unwrap().take(); + if let Some(gate) = gate { + if let Some(entered) = + self.restore_faults.prepared_entered.lock().unwrap().take() + { + let _ = entered.send(prepared.snapshot().clone()); + } + gate.notified().await; } - })?; - self.failed_restore_resources - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .remove(&request.worker_ref); - let (handle, shutdown_rx, controller_task) = - (started.handle, started.shutdown, started.controller_task); - if flow_transition_enabled { - handle.shared_state.enable_flow_transition(); } - Ok(RuntimeWorkerController { - handle, - shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), - controller_task, + Ok(ProfileRestoreCandidate { + prepared: Some(prepared), workspace_client, + factory: self.clone(), + request: request.clone(), + #[cfg(test)] + effects_boundary_failed: false, }) } .await; @@ -3402,7 +3561,6 @@ where }) } - #[cfg(feature = "ws-server")] fn validate_protocol_execution_under_lock( &self, worker_ref: &WorkerRef, @@ -3750,6 +3908,7 @@ where restore_operation_id: Option, workdir_attachment_statuses: Vec, workspace_client: Option>, + restore_streams: Option, ) -> WorkerExecutionSpawnResult { self.connect_controller_scope( operation, @@ -3762,6 +3921,7 @@ where restore_operation_id, workdir_attachment_statuses, workspace_client, + restore_streams, ) } @@ -3780,9 +3940,10 @@ where restore_operation_id: Option, workdir_attachment_statuses: Vec, workspace_client: Option>, + restore_streams: Option, ) -> WorkerExecutionSpawnResult { #[cfg(feature = "ws-server")] - let streams = subscribe_worker_protocol_session(&handle); + let streams = restore_streams.unwrap_or_else(|| subscribe_worker_protocol_session(&handle)); #[cfg(feature = "ws-server")] let worker_state_snapshot = match &streams.snapshot_event { Event::Snapshot { state, .. } => state.clone(), @@ -3841,7 +4002,15 @@ where let mut entry_events = streams.log_entries; let bridge_worker_state = worker_state.clone(); let bridge_handle = handle.clone(); - let bridge_task = match self.spawn_on_adapter_runtime(async move { + let initial_publication = if restore_operation_id.is_some() { + bridge_context + .publish_protocol_event(streams.snapshot_event.clone()) + .map(|_| ()) + .map_err(|error| error.to_string()) + } else { + Ok(()) + }; + let bridge_task = match initial_publication.and_then(|()| self.spawn_on_adapter_runtime(async move { loop { tokio::select! { event = events.recv() => { @@ -3898,7 +4067,7 @@ where } } } - }) { + })) { Ok(task) => task, Err(message) => { drop(workers); @@ -4442,14 +4611,27 @@ where Ok(guard) => guard, Err(_) => return unavailable("worker operation lock is poisoned".to_string()), }; - let workers = match self.workers.lock() { - Ok(workers) => workers, + let execution = match self.workers.lock() { + Ok(workers) => workers.get(&request.worker_ref).cloned(), Err(_) => return unavailable("worker adapter registry lock is poisoned".to_string()), }; - if workers.contains_key(&request.worker_ref) { - return runtime_api::WorkerSessionAvailability::LiveProtocol; + if let Some(execution) = execution { + // Match protocol attach against the same actual Controller, without + // holding the registry lock across validation or retained reads. + match self.validate_protocol_execution_under_lock(&request.worker_ref, &execution) { + Ok(()) => return runtime_api::WorkerSessionAvailability::LiveProtocol, + Err(error) + if error.outcome == crate::execution::WorkerExecutionOutcome::Errored => + { + return unavailable(error.message.unwrap_or_else(|| { + "Worker protocol availability could not be checked".to_string() + })); + } + // Closed or pending executions can still expose canonical + // read-only history, but must not advertise a live endpoint. + Err(_) => {} + } } - drop(workers); match self.factory.retained_session_snapshot(&request.worker_ref) { Ok(retained) => runtime_api::WorkerSessionAvailability::RetainedSnapshot { @@ -4766,6 +4948,7 @@ where None, Vec::new(), Some(controller.workspace_client), + controller.restore_streams, ); } return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( @@ -4894,6 +5077,7 @@ where None, workdir_attachment_statuses(&workdir_attachments), Some(controller.workspace_client), + controller.restore_streams, ) } @@ -5247,6 +5431,21 @@ where }; } } + if let Some(Err(failure)) = pending + .discard_result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + failure.to_string(), + ), + worker_state: None, + workdir_attachments: pending.request.previous_workdir_attachments.clone(), + }; + } if pending.activation_started.load(Ordering::Acquire) && !pending.owns_prepared_candidate() { let activated = pending @@ -5352,6 +5551,139 @@ where } } + fn discard_uncommitted_restore( + &self, + operation_id: crate::execution::WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> WorkerExecutionSpawnResult { + let discarded = (|| -> Result { + let operation_lock = self.worker_lock(worker_ref)?; + let _guard = operation_lock + .lock() + .map_err(|_| "worker operation lock is poisoned")?; + let pending = self + .pending_restore(worker_ref)? + .ok_or("unpublished restore owner is unavailable")?; + if pending.request.operation_id != operation_id + || pending.activation_started.load(Ordering::Acquire) + { + return Err( + "uncommitted discard cannot target another or activated operation".into(), + ); + } + if self + .workers + .lock() + .map_err(|_| "worker registry lock is poisoned")? + .contains_key(worker_ref) + { + return Err("uncommitted discard cannot target a connected execution".into()); + } + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + if !pending.discard_started.swap(true, Ordering::AcqRel) { + let owner = pending.clone(); + let task = + self.spawn_on_adapter_runtime(async move { + let retained = owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let cleanup = + match retained { + Some(Ok(mut candidate)) => { + let effects = candidate.has_durable_effects(); + match candidate.discard().await { + Ok(()) => { + let mut failure = RuntimeRestoreFailure::started( + RuntimeRestoreStage::Controller, + "restore decision was not committed", + ); + failure.cleanup = RuntimeRestoreCleanup::Confirmed; + failure.durable_effects_possible = effects; + *owner.result.lock().unwrap_or_else( + std::sync::PoisonError::into_inner, + ) = Some(Err(failure)); + Ok(()) + } + Err(error) => { + *owner.result.lock().unwrap_or_else( + std::sync::PoisonError::into_inner, + ) = Some(Ok(candidate)); + Err(error) + } + } + } + Some(Err(failure)) => { + let result = if failure.needs_cleanup() { + Err(RuntimeRestoreCleanupFailure::Ownership) + } else { + Ok(()) + }; + *owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Err(failure)); + result + } + None => Err(RuntimeRestoreCleanupFailure::Ownership), + }; + *owner + .discard_result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(cleanup); + })?; + *pending + .task + .try_lock() + .map_err(|_| "restore discard task is already owned")? = Some(task); + } + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + let cleanup = *pending + .discard_result + .lock() + .map_err(|_| "restore discard result lock is poisoned")?; + match cleanup { + Some(Ok(())) => {} + Some(Err(error)) => { + pending.discard_started.store(false, Ordering::Release); + return Err(error.to_string()); + } + None => return Err("restore discard is still operation-owned".into()), + } + let failure = pending + .result + .lock() + .map_err(|_| "restore result lock is poisoned")? + .as_ref() + .and_then(|result| result.as_ref().err()) + .cloned() + .ok_or("restore discard has no confirmed result")?; + if failure.durable_effects_possible { + return Err("uncommitted candidate cleanup confirmed, but prior canonical repair effects require reconciliation".into()); + } + self.completed_restore_failures + .lock() + .map_err(|_| "restore failure receipts lock is poisoned")? + .insert(worker_ref.clone(), (operation_id, failure.clone())); + self.remove_pending_restore(worker_ref)?; + Ok(WorkerExecutionSpawnResult::RolledBack( + WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + failure.to_string(), + ), + )) + })(); + discarded.unwrap_or_else( + |message| WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message), + worker_state: None, + workdir_attachments: Vec::new(), + }, + ) + } + fn activate_restored_worker( &self, operation_id: crate::execution::WorkerLifecycleOperationId, @@ -5377,58 +5709,67 @@ where pending.activation_started.store(true, Ordering::Release); let owner = pending.clone(); let factory = self.factory.clone(); - let task = self.spawn_on_adapter_runtime(async move { - let candidate = owner - .result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .take(); - let result = match candidate { - Some(Ok(mut candidate)) => { - let result = candidate.activate().await; - match result { - Ok(controller) => Ok(controller), - Err(mut failure) => { - *owner - .result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(Err(failure.clone())); - match candidate.discard().await { - Ok(()) => { - failure.cleanup = RuntimeRestoreCleanup::Confirmed + let task = + self.spawn_on_adapter_runtime(async move { + let candidate = owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + let result = + match candidate { + Some(Ok(mut candidate)) => { + let result = candidate.activate().await; + match result { + Ok(controller) => Ok(controller), + Err(failure) if candidate.can_retry_publication() => { + // The durable decision owns this exact receiver and + // paired streams. Do not discard it or rerun restore + // effects merely because publication was unavailable. + *owner.result.lock().unwrap_or_else( + std::sync::PoisonError::into_inner, + ) = Some(Ok(candidate)); + Err(failure) } - Err(error) => { - failure.cleanup_failure.get_or_insert(error); - failure.cleanup = RuntimeRestoreCleanup::Failed(error); + Err(mut failure) => { + *owner.result.lock().unwrap_or_else( + std::sync::PoisonError::into_inner, + ) = Some(Err(failure.clone())); + match candidate.discard().await { + Ok(()) => { + failure.cleanup = + RuntimeRestoreCleanup::Confirmed + } + Err(error) => { + failure.cleanup_failure.get_or_insert(error); + failure.cleanup = + RuntimeRestoreCleanup::Failed(error); + } + } + let failure = finish_restore_failure( + factory.as_ref(), + &owner.request, + failure, + ) + .await; + *owner.result.lock().unwrap_or_else( + std::sync::PoisonError::into_inner, + ) = Some(Err(failure.clone())); + Err(failure) } } - let failure = finish_restore_failure( - factory.as_ref(), - &owner.request, - failure, - ) - .await; - *owner - .result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(Err(failure.clone())); - Err(failure) } - } - } - Some(Err(failure)) => Err(failure), - None => Err(RuntimeRestoreFailure::unproven( - RuntimeRestoreStage::Controller, - "restore candidate ownership unavailable", - )), - }; - *owner - .activated - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(result); - })?; + Some(Err(failure)) => Err(failure), + None => Err(RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "restore candidate ownership unavailable", + )), + }; + *owner + .activated + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(result); + })?; *pending .task .try_lock() @@ -5456,6 +5797,7 @@ where Some(operation_id), pending.request.previous_workdir_attachments.clone(), Some(controller.workspace_client), + controller.restore_streams, ) { WorkerExecutionSpawnResult::Connected { .. } => { self.remove_pending_restore(worker_ref)? @@ -5473,11 +5815,16 @@ where if execution.restore_operation_id != Some(operation_id) { return Err("restored Worker operation identity does not match".into()); } + if execution.shutdown_requested.load(Ordering::Acquire) + || execution.handle.protocol_is_closed() + { + return Err("restored Worker protocol endpoint is not attachable".into()); + } self.factory.activate_restored_controller( worker_ref, execution.workspace_id.as_deref(), &execution.handle, - ); + )?; Ok(()) } @@ -5956,6 +6303,7 @@ mod tests { use crate::management::RuntimeOptions; #[cfg(feature = "ws-server")] use crate::observation::WorkerObservationCursor; + use crate::restore_test_environment::require_restore_test_environment; use crate::working_directory::RuntimeGitMaterializer; use agen::Engine; use agen::llm_client::event::{Event as LlmEvent, ResponseStatus, StatusEvent}; @@ -5968,6 +6316,25 @@ mod tests { LoggedSessionHistoryEntryId, LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, Store, WorkerActiveSegmentRef, WorkerMetadata, WorkerMetadataStore, }; + use worker::PreparedWorker; + + // Both candidate and failure suites need the same guarded, checkout-owned + // temporary root, independently of Runtime's optional filesystem catalog. + fn restore_failure_tempdir() -> tempfile::TempDir { + require_restore_test_environment(); + let target = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("target"); + fs::create_dir_all(&target).unwrap(); + tempfile::Builder::new() + .prefix("r") + .rand_bytes(3) + .tempdir_in(target) + .unwrap() + } #[test] fn production_source_has_no_repository_derived_runtime_store() { @@ -6165,6 +6532,10 @@ mod tests { (saved, request) } + // This Subject-backed fixture verifies attribution through an explicit + // loopback Workspace transport. Its Profile lifecycle consumers require + // HTTP; no-HTTP tests instead check the explicit transport refusal. + #[cfg(feature = "http-server")] async fn saved_subjektiv_restore_fixture( root: &Path, materialize_head: bool, @@ -6351,6 +6722,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn profile_backend_existing_restore_attestation_is_read_only_under_concurrency() { let root = tempfile::tempdir().unwrap(); let runtime_store = root.path().join("runtime"); @@ -6692,6 +7064,7 @@ mod tests { assert!(backend.stop_worker(&request.worker_ref).is_accepted()); } + #[cfg(feature = "http-server")] fn profile_stop_fixture( root: &Path, ) -> (WorkerRuntimeExecutionBackend, WorkerExecutionRestoreRequest) { @@ -6767,6 +7140,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn stop_rejected_shutdown_resets_request_and_retries_from_actual_snapshot() { for stale in [false, true] { let root = tempfile::tempdir().unwrap(); @@ -6810,6 +7184,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn stop_late_shutdown_rejection_after_timeout_retains_waiter_without_double_enqueue() { let root = tempfile::tempdir().unwrap(); let (backend, request) = profile_stop_fixture(root.path()); @@ -6943,6 +7318,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn unconnected_cleanup_rejected_shutdown_retains_scope_for_retry() { let root = tempfile::tempdir().unwrap(); let (backend, request) = profile_stop_fixture(root.path()); @@ -6979,6 +7355,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn duplicate_connect_rejected_shutdown_retains_actual_candidate_until_stop_retry() { for operation in [ WorkerExecutionOperation::Spawn, @@ -7024,6 +7401,7 @@ mod tests { Some(request.operation_id), Vec::new(), candidate.workspace_client.clone(), + None, ) }; match result { @@ -7111,6 +7489,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn duplicate_connect_timeout_retains_candidate_waiter_and_no_false_stop() { let root = tempfile::tempdir().unwrap(); let candidate_root = tempfile::tempdir().unwrap(); @@ -7155,6 +7534,7 @@ mod tests { Some(request.operation_id), Vec::new(), candidate.workspace_client.clone(), + None, ) }; assert!(matches!( @@ -7317,6 +7697,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn stop_joins_healthy_self_terminated_controller_with_closed_endpoint() { let root = tempfile::tempdir().unwrap(); let (backend, request) = profile_stop_fixture(root.path()); @@ -7388,6 +7769,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn stop_and_unconnected_cleanup_retain_abnormal_controller_completion_across_retries() { let root = tempfile::tempdir().unwrap(); let (backend, request) = profile_stop_fixture(root.path()); @@ -7465,6 +7847,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] async fn profile_factory_saved_subjektiv_manifest_survives_restart_and_explicit_restore() { let root = tempfile::tempdir().unwrap(); let runtime_store = root.path().join("runtime"); @@ -7535,6 +7918,7 @@ mod tests { #[test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] fn profile_backend_restores_saved_workdir_reference_without_a_materializer() { let root = tempfile::tempdir().unwrap(); let store = root.path().join("runtime"); @@ -7599,6 +7983,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] async fn profile_factory_pending_subjektiv_restore_requires_operation_owned_launch_material() { let root = tempfile::tempdir().unwrap(); let runtime_store = root.path().join("runtime"); @@ -7796,6 +8181,7 @@ mod tests { // The local input matrix belongs to saved_manifest_policy_* above. #[tokio::test] #[serial_test::serial(worker_allocation)] + #[cfg(feature = "http-server")] async fn profile_factory_saved_manifest_mismatch_rejected_before_live_work() { let root = tempfile::tempdir().unwrap(); let runtime_store = root.path().join("runtime"); @@ -8154,6 +8540,98 @@ mod tests { .unwrap(); } + #[cfg(not(feature = "http-server"))] + #[tokio::test] + async fn workspace_api_without_http_rejects_spawn_and_restore_before_resource_creation() { + let root = tempfile::tempdir().unwrap(); + let profile_dir = root.path().join("profiles"); + let store_dir = root.path().join("runtime"); + for runtime_id in [None, Some("runtime-a")] { + let mut factory = + ProfileRuntimeWorkerFactory::new(&profile_dir).with_runtime_store_dir(&store_dir); + if let Some(runtime_id) = runtime_id { + factory = factory.with_runtime_id(runtime_id); + } + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let mut restore = direct_restore_request(worker_ref.clone()); + restore.request.worker_id = worker_ref.worker_id; + restore.request.workspace_api = Some(WorkspaceApiRef { + workspace_id: "workspace-a".to_string(), + base_url: "https://workspace.invalid".to_string(), + }); + let spawn = WorkerExecutionSpawnRequest { + worker_ref, + request: restore.request.clone(), + workspace_scope: None, + context: restore.context.clone(), + workdir_attachments: BTreeMap::new(), + config_bundle: None, + }; + + let error = factory + .spawn_controller(spawn) + .await + .err() + .expect("HTTP is unavailable"); + assert!(error.contains("HTTP transport is unavailable"), "{error}"); + let error = factory.preflight_restore(&restore).await.unwrap_err(); + assert!(error.contains("HTTP transport is unavailable"), "{error}"); + let failure = factory.recover_restore_source(&restore).await.unwrap_err(); + assert_eq!(failure.stage, RuntimeRestoreStage::Preflight); + assert_eq!(failure.resources, RuntimeRestoreResources::None); + assert!(!failure.needs_cleanup()); + assert!(!failure.durable_effects_possible); + assert!( + failure + .to_string() + .contains("HTTP transport is unavailable") + ); + let failure = factory + .prepare_restore(restore) + .await + .err() + .expect("HTTP is unavailable"); + assert_eq!(failure.stage, RuntimeRestoreStage::Preflight); + assert_eq!(failure.resources, RuntimeRestoreResources::None); + assert!(!failure.needs_cleanup()); + assert!(!failure.durable_effects_possible); + assert!( + failure + .to_string() + .contains("HTTP transport is unavailable") + ); + } + assert!(!profile_dir.exists()); + assert!(!store_dir.exists()); + } + + #[cfg(not(feature = "http-server"))] + #[tokio::test] + async fn workspace_config_without_http_returns_unsupported_instead_of_cached_success() { + let root = tempfile::tempdir().unwrap(); + let factory = ProfileRuntimeWorkerFactory::new(root.path()); + let bundle = test_bundle(); + let bundle_ref = ConfigBundleRef { + id: bundle.metadata.id, + digest: bundle.metadata.digest, + }; + let result = factory + .fetch_workspace_config(WorkspaceConfigFetchRequest { + workspace_api: WorkspaceApiRef { + workspace_id: "workspace-a".to_string(), + base_url: "https://workspace.invalid".to_string(), + }, + profile: ProfileSelector::Builtin("builtin:companion".to_string()), + expected: bundle_ref.clone(), + cached: Some(bundle_ref), + }) + .await; + let error = result.unwrap_err(); + assert!(error.contains("unsupported"), "{error}"); + assert!(error.contains("http-server"), "{error}"); + } + + #[cfg(feature = "http-server")] #[test] fn profile_factory_routes_workspace_requests_by_workspace_id() { let profiles = tempfile::tempdir().unwrap(); @@ -8192,6 +8670,7 @@ mod tests { ); } + #[cfg(feature = "http-server")] #[tokio::test] async fn workspace_config_refresh_uses_workspace_scoped_request_client() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -8514,6 +8993,7 @@ mod tests { ); } + #[cfg(feature = "http-server")] #[test] fn restart_restore_reconstructs_runtime_owned_worker_mutation_client() { let identity = RuntimeIdentityMaterial::generate("runtime-source").unwrap(); @@ -8695,6 +9175,7 @@ mod tests { } } + #[cfg(feature = "fs-store")] #[test] fn create_timeout_cancels_factory_and_removes_persisted_worker() { let root = tempfile::tempdir().unwrap(); @@ -8901,13 +9382,19 @@ mod tests { let workspace_backend_ref = RuntimeWorkspaceBackendRef::from_worker_request( &request.request, Some("runtime-test"), - ); + )?; let workspace_context = workspace_backend_ref.worker_context( + #[cfg(feature = "http-server")] &request.worker_ref, + #[cfg(feature = "http-server")] request.workspace_scope.as_ref(), + #[cfg(feature = "http-server")] None, + #[cfg(feature = "http-server")] None, + #[cfg(feature = "http-server")] None, + #[cfg(feature = "http-server")] None, ); let workspace_context = if workspace_context.client().is_available() @@ -8954,6 +9441,7 @@ mod tests { shutdown: Arc::new(tokio::sync::Mutex::new(Some(shutdown_rx))), controller_task: tokio::spawn(async {}), workspace_client, + restore_streams: None, }) } async fn restore_controller( @@ -10385,6 +10873,7 @@ mod tests { assert!(!socket_path.exists()); } + #[cfg(feature = "fs-store")] #[test] #[serial_test::serial(worker_allocation)] fn shared_bootstrap_preserves_in_process_transport_for_fresh_and_restored_runtime_workers() { @@ -10649,6 +11138,7 @@ mod tests { } #[test] + #[cfg(feature = "ws-server")] fn running_worker_waits_for_durable_notify_receipt() { let client = MockClient::sequential(vec![MockResponse::Hang(vec![])]); let runtime_base = tempfile::tempdir().unwrap(); @@ -11336,6 +11826,7 @@ mod tests { } #[test] + #[cfg(feature = "http-server")] fn subject_release_reserves_real_controller_and_excludes_runtime_lifecycle() { use runtime_api::{SubjectReleaseAction as Action, SubjectReleaseState as State}; let runtime_base = tempfile::tempdir().unwrap(); diff --git a/crates/worker-runtime/src/worker_backend/restore_recovery.rs b/crates/worker-runtime/src/worker_backend/restore_recovery.rs index 8dd47ee15..54feb63bd 100644 --- a/crates/worker-runtime/src/worker_backend/restore_recovery.rs +++ b/crates/worker-runtime/src/worker_backend/restore_recovery.rs @@ -47,18 +47,26 @@ impl ProfileRuntimeWorkerFactory { let workspace = RuntimeWorkspaceBackendRef::from_worker_request( &request.request, self.runtime_id.as_deref(), - ); + ) + .map_err(cause)?; + #[cfg(feature = "http-server")] let request_client = request .request .workspace_api .as_ref() .and_then(|api| self.workspace_request_clients.get(&api.workspace_id)); let context = workspace.worker_context( + #[cfg(feature = "http-server")] &request.worker_ref, + #[cfg(feature = "http-server")] request.workspace_scope.as_ref(), + #[cfg(feature = "http-server")] self.worker_mutation_identity.as_ref(), + #[cfg(feature = "http-server")] request_client, + #[cfg(feature = "http-server")] self.embedded_worker_mutation_dispatcher.as_ref(), + #[cfg(feature = "http-server")] Some(self.prompt_projection_cache.clone()), ); if pending && context.workspace_id().is_some() { diff --git a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs index 443fbac6f..690a4025f 100644 --- a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs @@ -165,23 +165,35 @@ mod restore_candidate { } fn with_transport(transport: WorkerControllerTransport) -> Self { + Self::with_seed(transport, false) + } + + fn no_model() -> Self { + Self::with_seed(WorkerControllerTransport::InProcess, true) + } + + fn with_seed(transport: WorkerControllerTransport, no_model: bool) -> Self { let root = restore_failure_tempdir(); - let model = MockClient::sequential(vec![ - MockResponse::Complete(vec![ - LlmEvent::tool_use_start(0, "candidate-task-probe", "TaskCreate"), - LlmEvent::tool_input_delta( - 0, - r#"{"subject":"Activated candidate","description":"Execution probe"}"#, - ), - LlmEvent::tool_use_stop(0), - LlmEvent::Status(StatusEvent { - status: ResponseStatus::Completed, - }), - ]), - MockResponse::Complete(simple_text_events()), - MockResponse::Complete(simple_text_events()), - MockResponse::Complete(simple_text_events()), - ]); + let model = if no_model { + MockClient::new(Vec::new()) + } else { + MockClient::sequential(vec![ + MockResponse::Complete(vec![ + LlmEvent::tool_use_start(0, "candidate-task-probe", "TaskCreate"), + LlmEvent::tool_input_delta( + 0, + r#"{"subject":"Activated candidate","description":"Execution probe"}"#, + ), + LlmEvent::tool_use_stop(0), + LlmEvent::Status(StatusEvent { + status: ResponseStatus::Completed, + }), + ]), + MockResponse::Complete(simple_text_events()), + MockResponse::Complete(simple_text_events()), + MockResponse::Complete(simple_text_events()), + ]) + }; let injected = model.clone(); let mut factory = ProfileRuntimeWorkerFactory::new(root.path()) .with_runtime_store_dir(root.path()) @@ -264,15 +276,19 @@ mod restore_candidate { let aggregate = fixture_root.join("workers").join(worker_ref.worker_id.to_string()); let session = WorkerSessionStore::new(aggregate.join("session")).unwrap(); let metadata = WorkerAggregateStore::new(&aggregate, &manifest.worker.name).unwrap(); - let mut engine = Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated( - MockClient::sequential(vec![MockResponse::Hang(simple_text_events().into_iter().take(2).collect())]), - ); + let seed_model = if no_model { + MockClient::new(Vec::new()) + } else { + MockClient::sequential(vec![MockResponse::Hang(simple_text_events().into_iter().take(2).collect())]) + }; + let mut engine = Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(seed_model); engine.set_system_prompt("Committed restore candidate fixture prompt"); let mut worker = Worker::new( manifest.clone(), engine, CombinedStore::new(session.clone(), metadata.clone()), RuntimeWorkspaceBackendRef::from_worker_request(&create, Some("candidate-runtime")) + .unwrap() .worker_context(&worker_ref, Some(&crate::runtime::RuntimeWorkspaceScope::new( "workspace-saved-subjektiv", "server-main", )), None, None, None, None), @@ -282,51 +298,60 @@ mod restore_candidate { worker.enable_worker_metadata_write_through().unwrap(); worker.materialize_durable_session_head().await.unwrap(); let (session_id, segment_id) = (worker.session_id(), worker.segment_id()); - let workspace_client = worker.workspace_client_handle(); - let started = PreparedWorker::new( - worker, - WorkerBootstrapLayout::RuntimeManagedRun { run_dir: fixture_root.join("seed") }, - WorkerControllerTransport::InProcess, - ).start().await.unwrap(); - let mut seed = SeedController(Some(RuntimeWorkerController { - handle: started.handle, - shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), - controller_task: started.controller_task, - workspace_client, - })); - let handle = seed.0.as_ref().unwrap().handle.clone(); - let mut started_events = handle.subscribe(); - submit(&handle, "candidate-setup-turn").await; - // Running and streaming observations can arrive in either - // order. Do not discard TextDelta while awaiting WorkerState. - let mut running = handle.shared_state.catalog_status() == WorkerStatus::Running; - let mut streaming = false; - receive_matching(&mut started_events, "seed running and streaming", |event| { - if let Event::WorkerState { snapshot } = event { - running |= snapshot.catalog_status() == protocol::WorkerStatus::Running; - } - streaming |= matches!(event, Event::TextDelta { .. }); - running && streaming - }).await; - let state = Arc::new(RwLock::new(handle.shared_state.snapshot())); - let mut events = handle.subscribe(); - handle.send(Method::Pause { command: next_internal_command(&state).unwrap() }) - .await.unwrap(); - receive_matching(&mut events, "seed pause acknowledgement", |event| { - matches!(event, Event::WorkerState { snapshot } - if snapshot.catalog_status() == protocol::WorkerStatus::Paused) - }).await; - let mut receipts = Vec::new(); - for id in [FIRST, SECOND] { - let receipt = submit(&handle, id).await; - assert!(matches!(receipt, Event::SubmissionAccepted { - disposition: protocol::SubmissionDisposition::Queued, .. + let receipts = if no_model { + // Persist a real empty Session without starting a seed + // Controller or submitting any model input. + drop(worker); + Vec::new() + } else { + let workspace_client = worker.workspace_client_handle(); + let started = PreparedWorker::new( + worker, + WorkerBootstrapLayout::RuntimeManagedRun { run_dir: fixture_root.join("seed") }, + WorkerControllerTransport::InProcess, + ).start().await.unwrap(); + let mut seed = SeedController(Some(RuntimeWorkerController { + handle: started.handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), + controller_task: started.controller_task, + workspace_client, + restore_streams: None, })); - receipts.push(serde_json::to_value(receipt).unwrap()); - if id == FIRST { notify(&handle).await; } - } - assert_eq!(pending(&handle).await.submissions.len(), 2); - shutdown_profile_controller(seed.0.take().unwrap()).await; + let handle = seed.0.as_ref().unwrap().handle.clone(); + let mut started_events = handle.subscribe(); + submit(&handle, "candidate-setup-turn").await; + // Running and streaming observations can arrive in either + // order. Do not discard TextDelta while awaiting WorkerState. + let mut running = handle.shared_state.catalog_status() == WorkerStatus::Running; + let mut streaming = false; + receive_matching(&mut started_events, "seed running and streaming", |event| { + if let Event::WorkerState { snapshot } = event { + running |= snapshot.catalog_status() == protocol::WorkerStatus::Running; + } + streaming |= matches!(event, Event::TextDelta { .. }); + running && streaming + }).await; + let state = Arc::new(RwLock::new(handle.shared_state.snapshot())); + let mut events = handle.subscribe(); + handle.send(Method::Pause { command: next_internal_command(&state).unwrap() }) + .await.unwrap(); + receive_matching(&mut events, "seed pause acknowledgement", |event| { + matches!(event, Event::WorkerState { snapshot } + if snapshot.catalog_status() == protocol::WorkerStatus::Paused) + }).await; + let mut receipts = Vec::new(); + for id in [FIRST, SECOND] { + let receipt = submit(&handle, id).await; + assert!(matches!(receipt, Event::SubmissionAccepted { + disposition: protocol::SubmissionDisposition::Queued, .. + })); + receipts.push(serde_json::to_value(receipt).unwrap()); + if id == FIRST { notify(&handle).await; } + } + assert_eq!(pending(&handle).await.submissions.len(), 2); + shutdown_profile_controller(seed.0.take().unwrap()).await; + receipts + }; // This orphan pin would be removed by the normal activation // reconciliation. Preparation and discard must leave it alone. let orphan = session.write_uploaded_file( @@ -358,15 +383,17 @@ mod restore_candidate { host, _root: root, }; - let checkpoint = queue_checkpoint(&fixture.entries()); - assert_eq!(checkpoint["pending"].as_array().unwrap().len(), 2); - assert_eq!( - checkpoint["pending_notifications"] - .as_array() - .unwrap() - .len(), - 1 - ); + if !no_model { + let checkpoint = queue_checkpoint(&fixture.entries()); + assert_eq!(checkpoint["pending"].as_array().unwrap().len(), 2); + assert_eq!( + checkpoint["pending_notifications"] + .as_array() + .unwrap() + .len(), + 1 + ); + } fixture.assert_quiet(); fixture } @@ -443,14 +470,23 @@ mod restore_candidate { .get(&self.request.worker_ref) .is_none() ); - assert!( - !self - .backend - .factory - .worker_restore_run_dir(&self.request.worker_ref, self.request.operation_id,) - .unwrap() - .exists() - ); + let run_dir = self.backend.factory.worker_restore_run_dir( + &self.request.worker_ref, self.request.operation_id, + ).unwrap(); + // Private initialization owns only disposable operation-scoped run + // artifacts. It has not published a listener, liveness files or a + // delegated-child projection, and canonical files are checked by + // each caller independently. + let mut artifacts = std::fs::read_dir(&run_dir).unwrap() + .map(|entry| entry.unwrap().file_name().into_string().unwrap()).collect::>(); + artifacts.sort(); + assert_eq!(artifacts, ["artifacts", "spawned", "worker.err.log", "worker.out.log"]); + for log in ["worker.err.log", "worker.out.log"] { + assert_eq!(std::fs::metadata(run_dir.join(log)).unwrap().len(), 0); + } + for directory in ["artifacts", "spawned"] { + assert_eq!(std::fs::read_dir(run_dir.join(directory)).unwrap().count(), 0); + } let table = worker::runtime::worker_allocation::LockFileGuard::open( &worker::runtime::worker_allocation::default_allocation_path().unwrap(), ) @@ -521,6 +557,588 @@ mod restore_candidate { } } + #[test] + #[serial_test::serial(worker_allocation)] + fn session_availability_requires_attachable_current_controller() { + let fixture = Fixture::no_model(); + let worker_ref = &fixture.request.worker_ref; + let observe = || { + fixture + .backend + .worker_session(WorkerSessionObservationRequest { + worker_ref: worker_ref.clone(), + }) + }; + let assert_retained = |availability| { + assert!(matches!( + availability, + runtime_api::WorkerSessionAvailability::RetainedSnapshot { identity, .. } + if identity.session_id == fixture.session_id.to_string() + && identity.segment_id == fixture.segment_id.to_string() + )); + }; + fixture.prepare(); + assert_retained(observe()); + let pending = fixture + .backend + .pending_restore(worker_ref) + .unwrap() + .unwrap(); + fixture + .backend + .activate_restored_worker(fixture.request.operation_id, worker_ref) + .unwrap(); + let execution = fixture.backend.workers.lock().unwrap()[worker_ref].clone(); + assert!(!execution.handle.protocol_is_closed()); + assert!(!execution.shutdown_requested.load(Ordering::Acquire)); + assert!(matches!( + observe(), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + #[cfg(feature = "ws-server")] + let transport = fixture + .backend + .clone() + .attach_worker_protocol(worker_ref) + .unwrap(); + #[cfg(feature = "ws-server")] + transport.validate().unwrap(); + + // Retain the same real operation owner alongside its registered actual + // Controller to exercise the pending-resource visibility guard alone. + let operation_lock = fixture.backend.worker_lock(worker_ref).unwrap(); + { + let _guard = operation_lock.lock().unwrap(); + fixture + .backend + .pending_restores + .lock() + .unwrap() + .insert(worker_ref.clone(), pending); + } + let pending_session = observe(); + #[cfg(feature = "ws-server")] + let pending_protocol = ( + transport.validate(), + fixture.backend.clone().attach_worker_protocol(worker_ref), + ); + // Reset injected visibility before assertions so Fixture can also stop + // and release its allocation when an assertion unwinds. + { + let _guard = operation_lock.lock().unwrap(); + fixture.backend.remove_pending_restore(worker_ref).unwrap(); + } + assert_retained(pending_session); + #[cfg(feature = "ws-server")] + { + assert!(pending_protocol.0.is_err()); + assert!(matches!( + pending_protocol.1, + Err(error) if error.outcome == crate::execution::WorkerExecutionOutcome::Busy + )); + } + assert!(matches!( + observe(), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + { + let _guard = operation_lock.lock().unwrap(); + execution.shutdown_requested.store(true, Ordering::Release); + } + let stopping_session = observe(); + #[cfg(feature = "ws-server")] + let stopping_protocol = ( + transport.validate(), + fixture.backend.clone().attach_worker_protocol(worker_ref), + ); + { + let _guard = operation_lock.lock().unwrap(); + execution.shutdown_requested.store(false, Ordering::Release); + } + assert!(!execution.handle.protocol_is_closed()); + assert_retained(stopping_session); + #[cfg(feature = "ws-server")] + { + assert!(stopping_protocol.0.is_err()); + assert!(stopping_protocol.1.is_err()); + transport.validate().unwrap(); + } + assert!(matches!( + observe(), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + let handle = execution.handle.clone(); + let command = next_internal_command(&execution.worker_state).unwrap(); + fixture + .backend + .run_on_adapter_runtime(async move { + handle + .send(Method::Shutdown { command }) + .await + .map_err(|error| error.to_string())?; + tokio::time::timeout(WAIT, handle.protocol_closed()) + .await + .map_err(|_| "Controller did not close".to_string())?; + Ok(()) + }) + .unwrap(); + assert!(execution.handle.protocol_is_closed()); + assert!(!execution.shutdown_requested.load(Ordering::Acquire)); + assert!( + fixture + .backend + .workers + .lock() + .unwrap() + .contains_key(worker_ref) + ); + assert_retained(observe()); + #[cfg(feature = "ws-server")] + { + assert!(transport.validate().is_err()); + assert!( + fixture + .backend + .clone() + .attach_worker_protocol(worker_ref) + .is_err() + ); + } + assert!(fixture.backend.stop_worker(worker_ref).is_accepted()); + #[cfg(feature = "ws-server")] + drop(transport); + drop(execution); + fixture.assert_released(); + fixture.assert_quiet(); + } + + // Connects Profile restore activation to ordinary protocol input and the + // durable Paused boundary; Runtime admission/Accepted ordering is outside it. + #[test] + #[cfg(feature = "ws-server")] + #[serial_test::serial(worker_allocation)] + fn activated_empty_candidate_accepts_body_input_and_queues_while_paused() { + use crate::execution::WorkerProtocolTransport; + + const S1: &str = "candidate-body-s1"; + const S2: &str = "candidate-body-s2"; + const IDLE_ONLY: &str = "candidate-body-idle-only"; + const OUTPUT: &str = "committed candidate output before pause"; + + // Keep one real connection (paired full snapshot + subscription) for + // every method and observation. Never reconnect to hide missing events. + fn observe( + backend: &WorkerRuntimeExecutionBackend, + mut transport: WorkerProtocolTransport, + phase: &'static str, + no_new_run: bool, + mut matches: impl FnMut(&Event) -> bool + Send + 'static, + ) -> (WorkerProtocolTransport, Event) { + backend + .run_on_adapter_runtime(async move { + let mut seen = Vec::new(); + let matched = tokio::time::timeout(WAIT, async { + loop { + let event = transport + .events + .recv() + .await + .expect("attached protocol stream"); + seen.push(format!("{event:?}")); + if no_new_run { + assert!( + !matches!( + event, + Event::InvokeStart { .. } + | Event::TurnStart { .. } + | Event::UserMessage { .. } + | Event::TextDelta { .. } + | Event::TextDone { .. } + ), + "paused input started another run during {phase}: {event:?}" + ); + } + if matches(&event) { + break event; + } + assert!( + !matches!( + event, + Event::Error { .. } + | Event::SubmissionRejected { .. } + | Event::NotificationRejected { .. } + | Event::Shutdown + ), + "unexpected protocol failure during {phase}: {event:?}" + ); + } + }) + .await + .unwrap_or_else(|_| { + panic!("protocol deadline during {phase}; observed {seen:?}") + }); + Ok((transport, matched)) + }) + .unwrap() + } + + fn user_inputs(entries: &[LogEntry]) -> Vec> { + entries + .iter() + .filter_map(|entry| match entry { + LogEntry::AnnotatedUserInput { segments, .. } => Some(segments.clone()), + _ => None, + }) + .collect() + } + + // Declaration order keeps the bootstrap runtime (which owns the + // existing loopback Host task) alive through Fixture::drop, even on panic. + let bootstrap_backend; + let mut fixture = Fixture::no_model(); + let script = MockClient::sequential(vec![MockResponse::Hang(vec![ + LlmEvent::text_block_start(0), + LlmEvent::text_delta(0, OUTPUT), + LlmEvent::text_block_stop(0, None), + ])]); + // Hang explicitly chains stream::pending() after a complete text block. + // Hang([]) would exercise pre-token rollback instead of durable Paused. + let injected = script.clone(); + let factory = fixture + .backend + .factory + .as_ref() + .clone() + .with_spawn_model_client_factory(move || Box::new(injected.clone())); + bootstrap_backend = std::mem::replace( + &mut fixture.backend, + Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap()), + ); + let worker_ref = &fixture.request.worker_ref; + assert!(user_inputs(&fixture.entries()).is_empty()); + assert!( + !fixture + .entries() + .iter() + .any(|entry| matches!(entry, LogEntry::Invoke { .. })) + ); + let before = fixture.canonical_files(); + fixture.prepare(); // actual Profile prepare_restore; not MockFactory::spawn + fixture.assert_quiet(); // independent empty bootstrap client remains unused + assert_eq!(script.call_count.load(Ordering::SeqCst), 0); + assert!(script.captured.lock().unwrap().is_empty()); + assert_eq!(fixture.canonical_files(), before); + assert!( + fixture + .backend + .clone() + .attach_worker_protocol(worker_ref) + .is_err() + ); + fixture + .backend + .activate_restored_worker(fixture.request.operation_id, worker_ref) + .unwrap(); + let execution = fixture.backend.workers.lock().unwrap()[worker_ref].clone(); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(worker_ref) + .unwrap(); + transport.validate().unwrap(); + assert!( + matches!( + &transport.snapshot, + Event::Snapshot { session, state, in_flight, .. } + if state.state == protocol::WorkerState::Idle + && session.pending_submissions == protocol::PendingSubmissionsSnapshot::default() + && session == &session_store::public_snapshot::project_current_session_snapshot(&fixture.entries()) + && in_flight.is_empty() + ), + "expected full input-ready snapshot: {:?}", + transport.snapshot + ); + assert!(user_inputs(&fixture.entries()).is_empty()); + assert_eq!(script.call_count.load(Ordering::SeqCst), 0); + + assert!( + transport + .dispatch(Method::submit_text(S1, S1)) + .unwrap() + .is_empty() + ); + let (transport, _) = observe( + &fixture.backend, + transport, + "S1 committed and streaming", + false, + { + let (mut accepted, mut user_message, mut running, mut text_done) = + (false, false, false, false); + move |event| { + match event { + Event::SubmissionAccepted { + submission_request_id, + disposition, + .. + } if submission_request_id == S1 => { + assert_eq!(*disposition, protocol::SubmissionDisposition::Started); + accepted = true; + } + Event::UserMessage { segments, .. } => { + assert_eq!(segments, &vec![protocol::Segment::text(S1)]); + user_message = true; + } + Event::WorkerState { snapshot } => { + running |= snapshot.state + == protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )); + } + Event::TextDone { text } => { + assert_eq!(text, OUTPUT); + text_done = true; + } + _ => {} + } + accepted && user_message && running && text_done + } + }, + ); + // Started is only an admission receipt. Require the actual canonical + // AnnotatedUserInput plus its model-visible history before sending Pause. + assert_eq!( + user_inputs(&fixture.entries()), + vec![vec![protocol::Segment::text(S1)]] + ); + let history = session_store::collect_state(&fixture.entries()).annotated_history; + assert!(history.iter().any(|entry| matches!( + &entry.item, LoggedItem::Message { content, .. } + if content.iter().any(|part| matches!(part, LoggedContentPart::Text { text } if text == S1)) + ))); + assert_eq!(script.call_count.load(Ordering::SeqCst), 1); + let pause = next_internal_command(&execution.worker_state).unwrap(); + transport + .dispatch(Method::Pause { command: pause }) + .unwrap(); + let (transport, _) = observe(&fixture.backend, transport, "Pause completion", false, { + let (mut acknowledged, mut paused, mut ended) = (false, false, false); + move |event| { + match event { + Event::CommandAcknowledged { acknowledgement } + if acknowledgement.command_id == pause.command_id => + { + assert_eq!(acknowledgement.command, protocol::WorkerCommandKind::Pause); + assert_eq!( + acknowledgement.disposition, + protocol::WorkerCommandDisposition::Accepted + ); + acknowledged = true; + } + Event::WorkerState { snapshot } => { + paused |= snapshot.state + == protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Paused, + )); + } + Event::RunEnd { result } => { + assert_eq!(*result, protocol::RunResult::Paused); + ended = true; + } + _ => {} + } + acknowledged && paused && ended + } + }); + assert_eq!( + execution.handle.shared_state.snapshot().state, + protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Paused + )) + ); + let paused_entries = fixture.entries(); + assert_eq!( + user_inputs(&paused_entries), + vec![vec![protocol::Segment::text(S1)]] + ); + let invokes = paused_entries + .iter() + .filter(|entry| matches!(entry, LogEntry::Invoke { .. })) + .count(); + assert_eq!(invokes, 1); + // The hanging model response has not appended a final assistant item. + // Prove the actual durable Paused boundary, not a streaming TextDone's + // assumed equivalence to a completed assistant history message. + assert!(paused_entries.iter().any(|entry| matches!( + entry, + LogEntry::RunCompleted { + interrupted: true, + result: agen::EngineResult::Paused, + .. + } + ))); + + transport.dispatch(Method::submit_text(S2, S2)).unwrap(); + let (transport, queued) = observe( + &fixture.backend, + transport, + "S2 Queued receipt", + true, + |event| matches!(event, Event::SubmissionAccepted { submission_request_id, .. } if submission_request_id == S2), + ); + let submission_id = match &queued { + Event::SubmissionAccepted { + submission_id, + disposition, + .. + } => { + assert_eq!(*disposition, protocol::SubmissionDisposition::Queued); + submission_id.clone() + } + _ => unreachable!(), + }; + let checkpoint = queue_checkpoint(&fixture.entries()); + assert_eq!(checkpoint["pending"].as_array().unwrap().len(), 1); + assert_eq!(checkpoint["pending"][0]["submission_request_id"], S2); + assert_eq!(checkpoint["pending"][0]["submission_id"], submission_id); + assert_eq!( + checkpoint["pending"][0]["input"], + serde_json::to_value(vec![protocol::Segment::text(S2)]).unwrap() + ); + let receipts = checkpoint["receipts"].as_array().unwrap(); + let s2_receipts: Vec<_> = receipts + .iter() + .filter(|receipt| receipt["submission_request_id"] == S2) + .collect(); + assert_eq!(s2_receipts.len(), 1); + assert_eq!(s2_receipts[0]["submission_id"], submission_id); + assert_eq!(s2_receipts[0]["disposition"], "queued"); + transport.dispatch(Method::ListPendingSubmissions).unwrap(); + let (transport, listed) = observe( + &fixture.backend, + transport, + "S2 pending list", + true, + |event| matches!(event, Event::PendingSubmissionsChanged { .. }), + ); + let pending = match listed { + Event::PendingSubmissionsChanged { pending } => pending, + _ => unreachable!(), + }; + assert_eq!(pending.submissions.len(), 1); + assert_eq!(pending.head_id.as_deref(), Some(submission_id.as_str())); + assert_eq!(pending.submissions[0].submission_id, submission_id); + + let before_retry = fixture.canonical_files(); + transport.dispatch(Method::submit_text(S2, S2)).unwrap(); + let (transport, replay) = observe( + &fixture.backend, + transport, + "S2 exact retry", + true, + |event| matches!(event, Event::SubmissionAccepted { submission_request_id, .. } if submission_request_id == S2), + ); + assert_eq!( + serde_json::to_value(replay).unwrap(), + serde_json::to_value(queued).unwrap() + ); + assert_eq!(fixture.canonical_files(), before_retry); + transport.dispatch(Method::ListPendingSubmissions).unwrap(); + let (transport, listed) = observe( + &fixture.backend, + transport, + "retry retains one pending", + true, + |event| matches!(event, Event::PendingSubmissionsChanged { .. }), + ); + assert!( + matches!(listed, Event::PendingSubmissionsChanged { pending: actual } if actual == pending) + ); + + // Idle-only admission is a different contract from ordinary Submit: + // observe the real rejection on this connection, not a Queued receipt. + let mut idle_only = WorkerInput::user(IDLE_ONLY); + idle_only.kind = WorkerInputKind::UserIfIdle; + idle_only.submission_request_id = Some(IDLE_ONLY.into()); + let rejected = fixture.backend.dispatch_input(worker_ref, idle_only); + assert!(!rejected.is_accepted(), "{rejected:?}"); + assert!( + rejected.message_or_default().contains("WorkerNotify"), + "{rejected:?}" + ); + let (transport, _) = observe( + &fixture.backend, + transport, + "Paused idle-only rejection", + true, + |event| { + matches!(event, Event::SubmissionRejected { submission_request_id, message } + if submission_request_id == IDLE_ONLY && message.contains("WorkerNotify")) + }, + ); + assert_eq!(fixture.canonical_files(), before_retry); + assert_eq!(queue_checkpoint(&fixture.entries()), checkpoint); + + transport + .dispatch(Method::CancelPendingSubmission { submission_id }) + .unwrap(); + let (transport, _) = observe(&fixture.backend, transport, "cancel S2", true, |event| { + matches!(event, Event::PendingSubmissionsChanged { pending } + if pending.submissions.is_empty() && pending.head_id.is_none()) + }); + transport.dispatch(Method::ListPendingSubmissions).unwrap(); + let (transport, _) = observe( + &fixture.backend, + transport, + "empty pending list after cancel", + true, + |event| { + matches!(event, Event::PendingSubmissionsChanged { pending } + if pending == &protocol::PendingSubmissionsSnapshot::default()) + }, + ); + let entries = fixture.entries(); + assert!( + queue_checkpoint(&entries)["pending"] + .as_array() + .unwrap() + .is_empty() + ); + assert_eq!( + user_inputs(&entries), + vec![vec![protocol::Segment::text(S1)]] + ); + assert_eq!( + entries + .iter() + .filter(|entry| matches!(entry, LogEntry::Invoke { .. })) + .count(), + invokes + ); + assert_eq!( + execution.handle.shared_state.snapshot().state, + protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Paused + )) + ); + assert_eq!(script.call_count.load(Ordering::SeqCst), 1); + assert_eq!(script.captured.lock().unwrap().len(), 1); + fixture.assert_quiet(); + transport.validate().unwrap(); + // Normal Stop owns shutdown and the Controller/transport abort+join + // barrier; do not simulate closure or inject a terminal state. + assert!(fixture.backend.stop_worker(worker_ref).is_accepted()); + assert!(execution.handle.protocol_is_closed()); + assert!(execution.tasks.tasks.lock().unwrap().is_empty()); + assert!(transport.validate().is_err()); + drop(transport); + drop(execution); + fixture.assert_released(); + assert_eq!(script.call_count.load(Ordering::SeqCst), 1); + drop(fixture); + drop(bootstrap_backend); + } + #[test] #[serial_test::serial(worker_allocation)] fn prepare_saved_inputs_rejects_new_input_and_discard_preserves_authority() { @@ -896,6 +1514,7 @@ mod restore_candidate { } } + #[cfg(feature = "fs-store")] #[test] #[serial_test::serial(worker_allocation)] fn attachment_preparation_replay_cannot_hide_logical_authority_revocation() { @@ -1101,4 +1720,6 @@ mod restore_candidate { server_api::WorkerRestoreState::Accepted ); } + + include!("worker_backend_restore_publication_tests.rs"); } diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs index 91530ab28..620fd140d 100644 --- a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -2,26 +2,6 @@ // Worker/bootstrap, local Git Workdir provider, and allocation table. External // model I/O is replaced for saved-input coverage; this is not product E2E. -use crate::restore_test_environment::require_restore_test_environment; - -fn restore_failure_tempdir() -> tempfile::TempDir { - require_restore_test_environment(); - // Explicitly stay inside this checkout, even when TMPDIR points elsewhere. - // Keep fixture components short; no symlink/alias to evade socket limits. - let target = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap() - .join("target"); - fs::create_dir_all(&target).unwrap(); - tempfile::Builder::new() - .prefix("r") - .rand_bytes(3) - .tempdir_in(target) - .unwrap() -} - struct RestoreOpenCountingMaterializer { inner: RuntimeGitMaterializer, opens: Arc, @@ -705,13 +685,8 @@ impl ProfileFailureFixture { #[test] #[serial_test::serial(worker_allocation)] fn profile_restore_prior_effects_survive_read_only_reprepare_failure() { - for (activation_stage, read_stage) in [ - (RuntimeRestoreStage::Feature, RuntimeRestoreStage::Metadata), - ( - RuntimeRestoreStage::Controller, - RuntimeRestoreStage::Reconstruction, - ), - ] { + for read_stage in [RuntimeRestoreStage::Metadata, RuntimeRestoreStage::Reconstruction] { + let activation_stage = RuntimeRestoreStage::ActivationEffects; let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload(); fixture.fail(activation_stage); @@ -908,7 +883,7 @@ fn profile_restore_prior_effects_from_pending_head_repair_survive_read_only_repr fn profile_restore_prior_effects_survive_rebuilt_candidate_pre_effect_activation_failure() { let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload(); - fixture.fail(RuntimeRestoreStage::Feature); + fixture.fail(RuntimeRestoreStage::ActivationEffects); let request = fixture .runtime .test_restore_request(&fixture.worker.worker_ref); @@ -926,7 +901,7 @@ fn profile_restore_prior_effects_survive_rebuilt_candidate_pre_effect_activation .pending_restore(&fixture.worker.worker_ref) .unwrap() .unwrap(); - fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Feature); + fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::ActivationEffects); let effected = persisted_files(&artifacts); assert_ne!(effected, pinned); @@ -1215,7 +1190,7 @@ fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() { #[serial_test::serial(worker_allocation)] fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_deferred_provider() { let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); - fixture.fail(RuntimeRestoreStage::Feature); + fixture.fail(RuntimeRestoreStage::ActivationEffects); fixture .backend .factory @@ -1256,7 +1231,7 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer { let retained = pending.result.lock().unwrap(); let failure = retained.as_ref().unwrap().as_ref().err().unwrap(); - assert_eq!(failure.stage, RuntimeRestoreStage::Feature); + assert_eq!(failure.stage, RuntimeRestoreStage::ActivationEffects); assert_eq!(failure.resources, RuntimeRestoreResources::WorkerStartup); assert_eq!( failure.cleanup, @@ -1281,12 +1256,17 @@ fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_defer assert!(!pending.request.deferred_workdir_sessions.is_empty()); assert_eq!(owned.operation_id, pending.request.operation_id); assert!(owned.allocation.is_some()); + // The private Controller has already closed its router before the later + // factory cleanup fault. Keep the remaining cleanup owner, not live aliases. + assert!(!owned.sessions.is_empty()); assert!( owned .sessions .iter() - .any(|sessions| !sessions.aliases().is_empty()) + .all(|sessions| sessions.aliases().is_empty()) ); + // Raw deferred handles are closed by factory cleanup after this injected + // fault is removed; do not open them while cleanup is still pending. let repeat = fixture .runtime .restore_worker_operation(&fixture.worker.worker_ref, request.clone()) @@ -1484,7 +1464,7 @@ async fn failed_restore_raw_cleanup_is_operation_bound_and_does_not_follow_reuse fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconciles_without_execution() { let fixture = ProfileFailureFixture::new(Duration::from_secs(2)); - fixture.fail(RuntimeRestoreStage::Feature); + fixture.fail(RuntimeRestoreStage::ActivationEffects); let release = Arc::new(tokio::sync::Notify::new()); let (entered_tx, entered) = mpsc::channel(); *fixture @@ -1579,7 +1559,7 @@ fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconcile #[cfg(unix)] #[test] #[serial_test::serial(worker_allocation)] -fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation() { +fn profile_restore_real_socket_failure_retains_committed_candidate_until_cleanup() { let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); fixture .backend @@ -1601,7 +1581,7 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( ); let diagnostic = serde_json::to_string(&result).unwrap(); assert!( - diagnostic.contains("controller startup failed"), + diagnostic.contains("publication failed"), "{diagnostic}" ); assert!(diagnostic.contains("InvalidInput"), "{diagnostic}"); @@ -1624,15 +1604,14 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( assert!(!diagnostic.contains(&fixture.root.path().to_string_lossy().to_string())); assert!(!socket.exists()); assert!(fixture.backend.workers.lock().unwrap().is_empty()); - assert!( - fixture - .backend - .factory - .failed_restore_resources - .lock() - .unwrap() - .is_empty() - ); + assert!(pending.owns_prepared_candidate()); + assert!(pending.result.lock().unwrap().as_ref().unwrap().as_ref().unwrap().can_retry_publication()); + assert!(fixture.backend.factory.failed_restore_resources.lock().unwrap()[&fixture.worker.worker_ref].allocation.is_some()); + let durable: serde_json::Value = serde_json::from_slice(&fs::read( + fixture.root.path().join("s/workers").join(fixture.worker.worker_ref.worker_id.to_string()).join("worker.json") + ).unwrap()).unwrap(); + assert_eq!(durable["execution_state"]["execution"]["intent"]["commit"], "committed"); + assert!(durable["restore_guard"]["owners"][&request.request_id]["receipt"].is_null()); assert!( fixture .backend @@ -1647,10 +1626,14 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation( .restore_faults .socket_failure .store(false, Ordering::SeqCst); - eprintln!( - "restore-fault stage=socket-start real-bootstrap=true partial-run=removed allocation=released" - ); - fixture.resume_same_operation(request, operation_id); + // A real permanent path-limit error cannot be repaired by silently + // constructing another endpoint/transport. Explicit cleanup settles the + // retained owner. This case does not assert autonomous terminal handling. + drop(pending); + fixture.runtime.stop_worker(&fixture.worker.worker_ref, None).unwrap(); + fixture.assert_released(); + assert!(!socket.parent().unwrap().exists()); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 1); } #[test] diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs new file mode 100644 index 000000000..394fc64d9 --- /dev/null +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -0,0 +1,535 @@ +// Included in restore_candidate: real Runtime catalog/store, Profile candidate, +// Controller and protocol; only the non-network model client and loopback +// Workspace transport are substituted. These are ordinary integration tests, +// not product-process E2E or live model generation. +#[cfg(all(feature = "fs-store", feature = "ws-server"))] +mod publication { + use super::*; + use crate::fs_store::{ + AtomicWriteFault, FsRuntimeStoreOptions, PersistedWorkerExecution, + PersistedWorkerExecutionState, PersistedWorkerRecord, + }; + + fn stopped_runtime(fixture: &Fixture) -> EmbeddedRuntime { + let options = FsRuntimeStoreOptions { + root: fixture._root.path().join("catalog"), + runtime_id: "candidate-runtime".into(), + display_name: None, + }; + let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend( + options.clone(), + fixture.backend.clone(), + ) + .unwrap(); + let create = fixture.request.request.clone(); + let worker_ref = fixture.request.worker_ref.clone(); + runtime + .test_runtime_store() + .write_worker_record(&PersistedWorkerRecord { + restore_guard: crate::runtime::RestoreGuard::default(), + worker_ref: worker_ref.clone(), + worker_id: worker_ref.worker_id, + profile: create.profile.clone(), + display_name: None, + profile_source: create.profile_source.reference(), + config_bundle: create.config_bundle.clone(), + created_at_ms: None, + status: crate::catalog::WorkerStatus::Stopped, + last_finished_submission_request_id: None, + execution_state: PersistedWorkerExecutionState::Available( + PersistedWorkerExecution { + request: create, + binding: None, + restore_intent: crate::catalog::WorkerRestoreIntent::Explicit, + }, + ), + workspace_id: Some( + fixture + .request + .workspace_scope + .as_ref() + .unwrap() + .workspace_id + .clone(), + ), + workdir_attachments: Vec::new(), + logical_workdir_attachments: Vec::new(), + }) + .unwrap(); + drop(runtime); + EmbeddedRuntime::with_fs_store_and_execution_backend(options, fixture.backend.clone()) + .unwrap() + } + + fn durable_operation( + runtime: &EmbeddedRuntime, + worker_ref: &WorkerRef, + ) -> Option { + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + match &loaded.workers[&worker_ref.worker_id].execution_state { + PersistedWorkerExecutionState::ReconciliationRequired( + crate::fs_store::PersistedWorkerLifecycleOperation::Restore(operation), + ) => Some(operation.clone()), + _ => None, + } + } + + fn prepared_gate(fixture: &Fixture) -> (Arc, mpsc::Receiver) { + let release = Arc::new(tokio::sync::Notify::new()); + let (entered, received) = mpsc::channel(); + *fixture + .backend + .factory + .restore_faults + .prepared_gate + .lock() + .unwrap() = Some(release.clone()); + *fixture + .backend + .factory + .restore_faults + .prepared_entered + .lock() + .unwrap() = Some(entered); + (release, received) + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn runtime_accepted_follows_real_prepared_snapshot_commit_attach_and_input() { + let bootstrap_backend; + let mut fixture = Fixture::no_model(); + let script = MockClient::sequential(vec![MockResponse::Complete(simple_text_events())]); + let injected = script.clone(); + let factory = fixture + .backend + .factory + .as_ref() + .clone() + .with_spawn_model_client_factory(move || Box::new(injected.clone())); + bootstrap_backend = std::mem::replace( + &mut fixture.backend, + Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap()), + ); + let runtime = Arc::new(stopped_runtime(&fixture)); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let before = fixture.canonical_files(); + let (release, prepared) = prepared_gate(&fixture); + let restoring = { + let runtime = runtime.clone(); + let worker_ref = worker_ref.clone(); + let scope = scope.clone(); + let request = request.clone(); + std::thread::spawn(move || { + runtime.restore_worker_operation_scoped(&scope, &worker_ref, request) + }) + }; + let snapshot = prepared.recv_timeout(WAIT).unwrap(); + assert!( + matches!(&snapshot, Event::Snapshot { session, greeting, state, in_flight, .. } + if session == &session_store::public_snapshot::project_current_session_snapshot(&fixture.entries()) + && !greeting.tools.is_empty() && state.state == protocol::WorkerState::Idle && in_flight.is_empty()) + ); + let pending = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!( + pending.commit, + crate::runtime::RestoreCommitFact::NotCommitted + ); + assert_eq!(fixture.canonical_files(), before); + fixture.assert_quiet(); + fixture.assert_no_allocation(); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!( + fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .is_none() + ); + // External attach shares the in-flight backend operation lock; do not + // synchronously acquire that lock while this test holds its prepare gate. + // The empty execution registry and absent hub/allocation prove private + // ownership here; attach is exercised immediately after Accepted below. + assert_eq!(script.call_count.load(Ordering::SeqCst), 0); + release.notify_one(); + let accepted = restoring.join().unwrap().unwrap(); + assert_eq!( + accepted.state, + server_api::WorkerRestoreState::Accepted, + "{accepted:?}" + ); + assert!(durable_operation(&runtime, &worker_ref).is_none()); + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + let guard = + serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap(); + assert!(!guard["owners"][&request.request_id]["receipt"].is_null()); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + assert!( + fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .is_some() + ); + let mut transport = fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + transport.validate().unwrap(); + assert_eq!( + serde_json::to_value(&transport.snapshot).unwrap(), + serde_json::to_value(snapshot).unwrap() + ); + assert!(matches!( + fixture + .backend + .worker_session(crate::execution::WorkerSessionObservationRequest { + worker_ref: worker_ref.clone(), + }), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + transport + .dispatch(Method::submit_text( + "vertical-body-input", + "vertical-body-input", + )) + .unwrap(); + fixture + .backend + .run_on_adapter_runtime(async move { + let (mut admitted, mut committed, mut text, mut ended) = + (false, false, false, false); + tokio::time::timeout(WAIT, async { + while !(admitted && committed && text && ended) { + match transport.events.recv().await.unwrap() { + Event::SubmissionAccepted { + submission_request_id, + .. + } if submission_request_id == "vertical-body-input" => admitted = true, + Event::UserMessage { segments, .. } => { + assert_eq!( + segments, + vec![protocol::Segment::text("vertical-body-input")] + ); + committed = true; + } + Event::TextDone { text: output } => { + assert_eq!(output, "hello from worker"); + text = true; + } + Event::RunEnd { .. } => ended = true, + Event::SubmissionRejected { .. } | Event::Error { .. } => { + panic!("normal input rejected after Accepted") + } + _ => {} + } + } + }) + .await + .map_err(|_| "normal input did not settle".to_string())?; + // Capture and release the whole transport, not only its events field; + // its dispatch/validation closures retain the actual execution. + drop(transport); + Ok(()) + }) + .unwrap(); + assert!(fixture.entries().iter().any( + |entry| matches!(entry, LogEntry::AnnotatedUserInput { segments, .. } + if segments == &vec![protocol::Segment::text("vertical-body-input")]) + )); + assert_eq!(script.call_count.load(Ordering::SeqCst), 1); + let replay = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!(replay.state, server_api::WorkerRestoreState::Accepted); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + let mut stale = request; + stale.request_id = "vertical-new-stale-intent".into(); + assert!( + matches!(runtime.restore_worker_operation_scoped(&scope, &worker_ref, stale), Err(crate::error::RuntimeError::RestoreObservationConflict { worker_id }) if worker_id == worker_ref.worker_id) + ); + runtime + .stop_worker_scoped(&scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + assert_eq!(script.call_count.load(Ordering::SeqCst), 1); + drop(runtime); + drop(bootstrap_backend); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn decision_write_faults_preserve_prepared_owner_or_confirm_uncommitted_cleanup() { + for (fault, sync_unknown) in [ + (AtomicWriteFault::BeforeRename, false), + (AtomicWriteFault::AfterRename, false), + (AtomicWriteFault::AfterRename, true), + ] { + let fixture = Fixture::no_model(); + let runtime = Arc::new(stopped_runtime(&fixture)); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let before = fixture.canonical_files(); + let (release, prepared) = prepared_gate(&fixture); + let restoring = { + let runtime = runtime.clone(); + let worker_ref = worker_ref.clone(); + let scope = scope.clone(); + let request = request.clone(); + std::thread::spawn(move || { + runtime.restore_worker_operation_scoped(&scope, &worker_ref, request) + }) + }; + prepared.recv_timeout(WAIT).unwrap(); + let operation = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!( + operation.commit, + crate::runtime::RestoreCommitFact::NotCommitted + ); + runtime.test_runtime_store().fail_next_worker_write(fault); + if sync_unknown { + runtime + .test_runtime_store() + .fail_next_worker_reconcile_sync(); + } + release.notify_one(); + let result = restoring.join().unwrap().unwrap(); + match fault { + AtomicWriteFault::BeforeRename => { + assert_eq!( + result.state, + server_api::WorkerRestoreState::RolledBack, + "{result:?}" + ); + assert_eq!(fixture.canonical_files(), before); + fixture.assert_released(); + } + AtomicWriteFault::AfterRename if !sync_unknown => { + assert_eq!( + result.state, + server_api::WorkerRestoreState::Accepted, + "{result:?}" + ); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + transport.validate().unwrap(); + drop(transport); + runtime + .stop_worker_scoped(&scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + } + AtomicWriteFault::AfterRename => { + assert_eq!( + result.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{result:?}" + ); + assert_eq!( + durable_operation(&runtime, &worker_ref) + .unwrap() + .operation_id, + operation.operation_id + ); + assert_eq!( + durable_operation(&runtime, &worker_ref).unwrap().commit, + crate::runtime::RestoreCommitFact::Committed + ); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + fixture.assert_no_allocation(); + assert_eq!(fixture.canonical_files(), before); + let replay = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request) + .unwrap(); + assert_eq!( + replay.state, + server_api::WorkerRestoreState::Accepted, + "{replay:?}" + ); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + runtime + .stop_worker_scoped(&scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + } + _ => unreachable!(), + } + fixture.assert_quiet(); + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn uncommitted_cleanup_failure_fences_discarding_candidate_from_activation() { + let fixture = Fixture::no_model(); + let runtime = Arc::new(stopped_runtime(&fixture)); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let before = fixture.canonical_files(); + let (release, prepared) = prepared_gate(&fixture); + let restoring = { + let runtime = runtime.clone(); + let worker_ref = worker_ref.clone(); + let scope = scope.clone(); + let request = request.clone(); + std::thread::spawn(move || { + runtime.restore_worker_operation_scoped(&scope, &worker_ref, request) + }) + }; + prepared.recv_timeout(WAIT).unwrap(); + runtime.test_runtime_store().fail_next_worker_write(AtomicWriteFault::BeforeRename); + fixture.backend.factory.restore_faults.cleanup_fail.store(true, Ordering::SeqCst); + release.notify_one(); + let failed = restoring.join().unwrap().unwrap(); + assert_eq!(failed.state, server_api::WorkerRestoreState::ReconciliationRequired, "{failed:?}"); + let pending = fixture.backend.pending_restore(&worker_ref).unwrap().unwrap(); + assert!(matches!(pending.discard_result.lock().unwrap().as_ref(), Some(Err(_)))); + assert_eq!(durable_operation(&runtime, &worker_ref).unwrap().commit, crate::runtime::RestoreCommitFact::NotCommitted); + for cleanup_fails in [true, false] { + fixture.backend.factory.restore_faults.cleanup_fail.store(cleanup_fails, Ordering::SeqCst); + let replay = runtime.restore_worker_operation_scoped(&scope, &worker_ref, request.clone()).unwrap(); + assert_eq!(replay.state, server_api::WorkerRestoreState::ReconciliationRequired, "discarding candidate must not activate: {replay:?}"); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!(fixture.backend.factory.observation_hub.get(&worker_ref).is_none()); + assert!(fixture.backend.clone().attach_worker_protocol(&worker_ref).is_err()); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 1); + assert_eq!(fixture.canonical_files(), before); + fixture.assert_no_allocation(); + fixture.assert_quiet(); + } + // Explicit teardown verifies this retained owner's cleanup. Autonomous + // cleanup convergence is a separate operation-driver contract. + drop(pending); + runtime.stop_worker_scoped(&scope, &worker_ref, None).unwrap(); + fixture.assert_released(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn publication_failure_retains_same_committed_candidate_without_reconstruction() { + let fixture = Fixture::no_model(); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + fixture + .backend + .factory + .restore_faults + .publication_failure + .store(true, Ordering::SeqCst); + let failed = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + failed.state, + server_api::WorkerRestoreState::ReconciliationRequired, + "{failed:?}" + ); + let pending = fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .unwrap(); + assert!(pending.owns_prepared_candidate()); + assert!(pending.activation_started.load(Ordering::SeqCst)); + assert!( + pending + .result + .lock() + .unwrap() + .as_ref() + .unwrap() + .as_ref() + .unwrap() + .can_retry_publication() + ); + let durable = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!(durable.operation_id, pending.request.operation_id); + assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!( + fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .is_none() + ); + assert!( + fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .is_err() + ); + fixture.assert_quiet(); + let result = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request) + .unwrap(); + assert_eq!( + result.state, + server_api::WorkerRestoreState::Accepted, + "{result:?}" + ); + let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert_eq!(execution.restore_operation_id, Some(durable.operation_id)); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + transport.validate().unwrap(); + drop(transport); + drop(execution); + drop(pending); + runtime + .stop_worker_scoped(&scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + fixture.assert_quiet(); + } +} diff --git a/crates/worker-runtime/src/worker_source.rs b/crates/worker-runtime/src/worker_source.rs index a50fafc98..57f2e1f00 100644 --- a/crates/worker-runtime/src/worker_source.rs +++ b/crates/worker-runtime/src/worker_source.rs @@ -1,23 +1,32 @@ use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +#[cfg(feature = "http-server")] +use std::time::Duration; +use std::time::{SystemTime, UNIX_EPOCH}; +#[cfg(feature = "http-server")] use crate::workspace_request::{ RuntimeWorkspaceRequest, RuntimeWorkspaceRequestClient, RuntimeWorkspaceRequestError, }; +use worker::WorkspaceResponse; +#[cfg(feature = "http-server")] use worker::{ WorkspaceBinaryRequest, WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspacePromptCatalogResolution, WorkspacePromptProjection, WorkspaceRequest, - WorkspaceRequestMethod, WorkspaceResponse, WorkspaceServerOperation, + WorkspaceRequestMethod, WorkspaceServerOperation, }; use crate::auth::{ RuntimeAuthError, RuntimeIdentityMaterial, RuntimeWorkerMutationSourceSigner, - WORKER_REMOVE_PERMISSION, WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION, - WorkerMutationActorKind, WorkerMutationOperation, WorkerMutationSourceClaims, new_token_id, + WORKER_REMOVE_PERMISSION, WorkerMutationActorKind, WorkerMutationOperation, + WorkerMutationSourceClaims, new_token_id, }; +#[cfg(feature = "http-server")] +use crate::auth::{WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION}; use crate::runtime::RuntimeWorkspaceScope; +#[cfg(feature = "http-server")] use crate::worker_backend::WorkspacePromptProjectionCache; +#[cfg(feature = "http-server")] fn ticket_assignment_role_segment(role: server_api::TicketAssignmentRole) -> &'static str { match role { server_api::TicketAssignmentRole::Orchestrator => "orchestrator", @@ -144,6 +153,7 @@ pub trait EmbeddedWorkerMutationDispatcher: Send + Sync { #[derive(Clone)] enum RuntimeWorkerMutationTransport { + #[cfg(feature = "http-server")] Remote { request_client: RuntimeWorkspaceRequestClient, }, @@ -161,6 +171,7 @@ pub struct RuntimeWorkerMutationForwarder { } impl RuntimeWorkerMutationForwarder { + #[cfg(feature = "http-server")] pub fn remote( identity: &RuntimeIdentityMaterial, scope: RuntimeWorkspaceScope, @@ -206,6 +217,7 @@ impl RuntimeWorkerMutationForwarder { target_worker_id, )?; match (&self.transport, proof) { + #[cfg(feature = "http-server")] ( RuntimeWorkerMutationTransport::Remote { request_client }, RuntimeOwnedWorkerMutationProof::Remote(token), @@ -231,6 +243,7 @@ impl RuntimeWorkerMutationForwarder { } } +#[cfg(feature = "http-server")] struct RemoteWorkerRemoveHttpRequest { request_client: RuntimeWorkspaceRequestClient, source_worker_id: String, @@ -240,6 +253,7 @@ struct RemoteWorkerRemoveHttpRequest { reason: String, } +#[cfg(feature = "http-server")] fn execute_remote_worker_remove_http( request: RemoteWorkerRemoveHttpRequest, ) -> Result { @@ -263,6 +277,7 @@ fn execute_remote_worker_remove_http( execute_remote_worker_remove_http_blocking(request) } +#[cfg(feature = "http-server")] fn execute_remote_worker_remove_http_blocking( request: RemoteWorkerRemoveHttpRequest, ) -> Result { @@ -315,6 +330,7 @@ fn execute_remote_worker_remove_http_blocking( } } +#[cfg(feature = "http-server")] #[derive(Clone)] pub struct RuntimeOwnedWorkspaceClient { workspace_id: String, @@ -325,6 +341,7 @@ pub struct RuntimeOwnedWorkspaceClient { prompt_projection_cache: Option>, } +#[cfg(feature = "http-server")] impl RuntimeOwnedWorkspaceClient { pub fn new( workspace_id: impl Into, @@ -495,6 +512,7 @@ impl RuntimeOwnedWorkspaceClient { } } +#[cfg(feature = "http-server")] impl std::fmt::Debug for RuntimeOwnedWorkspaceClient { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter @@ -510,6 +528,7 @@ impl std::fmt::Debug for RuntimeOwnedWorkspaceClient { } } +#[cfg(feature = "http-server")] fn generated_workspace_response( response: Result>, ) -> Result { @@ -530,6 +549,7 @@ fn generated_workspace_response( } } +#[cfg(feature = "http-server")] impl WorkspaceClient for RuntimeOwnedWorkspaceClient { fn workspace_id(&self) -> Option<&str> { Some(self.request_client.workspace_id()) @@ -929,12 +949,17 @@ fn unix_now_seconds() -> u64 { #[cfg(test)] mod tests { use super::*; + #[cfg(feature = "http-server")] use crate::auth::{ - RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WorkerMutationSourceExpectation, - decode_runtime_request_source_claims, decode_worker_mutation_source_claims, - request_body_digest, verify_worker_mutation_source_proof, + RUNTIME_REQUEST_SOURCE_PROOF_HEADER, decode_runtime_request_source_claims, + request_body_digest, + }; + use crate::auth::{ + WorkerMutationSourceExpectation, decode_worker_mutation_source_claims, + verify_worker_mutation_source_proof, }; + #[cfg(feature = "http-server")] fn binary_loopback(response: Vec) -> (String, std::thread::JoinHandle<(String, Vec)>) { use std::io::{Read, Write}; let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); @@ -972,6 +997,7 @@ mod tests { (base_url, server) } + #[cfg(feature = "http-server")] #[test] fn generated_ticket_assignment_transport_signs_exact_authoring_and_bounded_read_subjects() { use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source}; @@ -1104,6 +1130,7 @@ mod tests { } } + #[cfg(feature = "http-server")] #[test] fn binary_roundtrip_signs_exact_query_method_worker_and_raw_body() { use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source}; @@ -1194,6 +1221,7 @@ mod tests { } } + #[cfg(feature = "http-server")] #[test] fn binary_responses_enforce_declared_and_streamed_limits_with_fixed_errors() { for response in [ @@ -1213,6 +1241,7 @@ mod tests { } } + #[cfg(feature = "http-server")] #[test] fn binary_redirect_returns_status_without_forwarding_authentication() { let (base_url, server) = binary_loopback(b"HTTP/1.1 307 Temporary Redirect\r\nLocation: http://127.0.0.1:1/private\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec()); @@ -1231,6 +1260,7 @@ mod tests { assert!(result.body.is_empty()); } + #[cfg(feature = "http-server")] #[test] fn binary_invalid_or_normalized_targets_fail_with_fixed_errors_before_http() { let client = RuntimeOwnedWorkspaceClient::new( @@ -1277,6 +1307,7 @@ mod tests { ); } + #[cfg(feature = "http-server")] #[test] fn current_prompt_projection_uses_the_shared_runtime_cache_without_http() { let cache = Arc::new(WorkspacePromptProjectionCache::default()); @@ -1312,6 +1343,7 @@ mod tests { assert!(Arc::ptr_eq(&projection.catalog, &second.catalog)); } + #[cfg(feature = "http-server")] #[test] fn prompt_projection_cache_miss_fetches_content_identity() { use std::io::{Read, Write}; @@ -1356,6 +1388,7 @@ mod tests { assert_eq!(fetched.projection.source_digest, "source-3"); } + #[cfg(feature = "http-server")] #[test] fn concurrent_prompt_projection_miss_fetches_once_and_shares_catalog() { use std::io::{Read, Write}; @@ -1427,6 +1460,7 @@ mod tests { ); } + #[cfg(feature = "http-server")] #[test] fn current_prompt_projection_rejects_cross_workspace_response() { use std::io::{Read, Write}; @@ -1473,6 +1507,7 @@ mod tests { assert!(error.to_string().contains("scope mismatch")); } + #[cfg(feature = "http-server")] #[test] fn ordinary_workspace_forwarding_stamps_runtime_identity_and_signs_path_and_query() { use std::io::{Read, Write}; @@ -1531,6 +1566,7 @@ mod tests { ); } + #[cfg(feature = "http-server")] #[test] fn workspace_worker_discovery_signs_dedicated_permission_and_encoded_query() { use std::io::{Read, Write}; @@ -1652,6 +1688,7 @@ mod tests { ); } + #[cfg(feature = "http-server")] #[tokio::test(flavor = "multi_thread")] async fn remote_forwarder_is_safe_in_async_runtime_and_stamps_signed_proof() { use std::io::{Read, Write}; @@ -1815,6 +1852,7 @@ mod tests { assert_eq!(reason, "retire obsolete Worker"); } + #[cfg(feature = "http-server")] #[test] fn runtime_owned_workspace_client_returns_successful_response_without_request_timeout() { // Effective deadline clearing is exercised with a short inherited deadline in @@ -1838,6 +1876,7 @@ mod tests { server.join().unwrap(); } + #[cfg(feature = "http-server")] #[test] fn runtime_owned_workspace_client_honors_per_request_timeouts() { let (base_url, server) = delayed_workspace_response(Duration::from_millis(75)); @@ -1857,6 +1896,7 @@ mod tests { server.join().unwrap(); } + #[cfg(feature = "http-server")] #[test] fn runtime_owned_workspace_client_reports_request_timeouts() { let (base_url, server) = delayed_workspace_response(Duration::from_millis(75)); @@ -1878,6 +1918,7 @@ mod tests { server.join().unwrap(); } + #[cfg(feature = "http-server")] fn delayed_workspace_response(delay: Duration) -> (String, std::thread::JoinHandle<()>) { use std::io::{Read, Write}; use std::net::TcpListener; diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs index cb88a2636..71d77f16f 100644 --- a/crates/workspace-server/src/server_assignment_attention_tests.rs +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -477,6 +477,7 @@ impl AttentionWorkerFactory { shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), controller_task: started.controller_task, workspace_client, + restore_streams: None, }) } } diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs index 25031c8f6..c713af5a7 100644 --- a/crates/workspace-server/src/server_assignment_delivery_tests.rs +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -835,6 +835,7 @@ impl DeliveryWorkerFactory { shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), controller_task: started.controller_task, workspace_client, + restore_streams: None, }) } } From ceb01f6eaa4a467a74274b1df2564ac092b38816 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 10:03:56 +0900 Subject: [PATCH 26/46] feat: runtime own restore continuation beyond caller timeout --- crates/worker-runtime/src/execution.rs | 84 ++- crates/worker-runtime/src/fs_store.rs | 93 +++- crates/worker-runtime/src/runtime.rs | 138 ++++- .../src/runtime/restore_continuation.rs | 498 ++++++++++++++++++ .../worker-runtime/src/test_support_tests.rs | 1 + crates/worker-runtime/src/worker_backend.rs | 94 +++- .../src/worker_backend/restore_recovery.rs | 4 + .../worker_backend_restore_failure_tests.rs | 2 + ...orker_backend_restore_publication_tests.rs | 138 +++++ 9 files changed, 1030 insertions(+), 22 deletions(-) create mode 100644 crates/worker-runtime/src/runtime/restore_continuation.rs diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index b95347102..39faadab5 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -242,6 +242,9 @@ enum WorkerExecutionPublicationState { #[derive(Clone)] pub struct WorkerExecutionContext { worker_ref: WorkerRef, + restore_progress: Option>, + restore_repair_authorizer: + Option Result<(), RuntimeError> + Send + Sync>>, #[cfg(feature = "ws-server")] observation_publisher: WorkerObservationPublisher, #[cfg(feature = "ws-server")] @@ -256,6 +259,8 @@ impl WorkerExecutionContext { ) -> Self { Self { worker_ref, + restore_progress: None, + restore_repair_authorizer: None, observation_publisher, publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Active)), } @@ -268,6 +273,8 @@ impl WorkerExecutionContext { ) -> Self { Self { worker_ref, + restore_progress: None, + restore_repair_authorizer: None, observation_publisher, publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Pending( VecDeque::new(), @@ -277,12 +284,56 @@ impl WorkerExecutionContext { #[cfg(not(feature = "ws-server"))] pub(crate) fn new(worker_ref: WorkerRef) -> Self { - Self { worker_ref } + Self { + worker_ref, + restore_progress: None, + restore_repair_authorizer: None, + } } #[cfg(not(feature = "ws-server"))] pub(crate) fn candidate(worker_ref: WorkerRef) -> Self { - Self { worker_ref } + Self { + worker_ref, + restore_progress: None, + restore_repair_authorizer: None, + } + } + + pub(crate) fn with_restore_repair_authorizer( + mut self, + authorizer: Arc< + dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync, + >, + ) -> Self { + self.restore_repair_authorizer = Some(authorizer); + self + } + + pub(crate) fn authorize_restore_source_repair( + &self, + operation: WorkerLifecycleOperationId, + ) -> Result<(), RuntimeError> { + match &self.restore_repair_authorizer { + Some(authorizer) => authorizer(operation), + // Direct factory/component users own their source repair separately; + // only Runtime contexts carry a Runtime aggregate authority callback. + None => Ok(()), + } + } + + pub(crate) fn with_restore_progress( + mut self, + progress: Arc, + ) -> Self { + self.restore_progress = Some(progress); + self + } + + pub(crate) fn restore_progressed(&self, operation_id: WorkerLifecycleOperationId) { + if let Some(progress) = &self.restore_progress { + progress(operation_id); + } } pub fn worker_ref(&self) -> &WorkerRef { @@ -424,6 +475,7 @@ pub struct WorkerExecutionSpawnRequest { /// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker. #[derive(Clone, Debug)] pub struct WorkerExecutionRestoreRequest { + pub source_repair_possible: bool, pub operation_id: WorkerLifecycleOperationId, pub worker_ref: WorkerRef, pub request: crate::catalog::CreateWorkerRequest, @@ -565,6 +617,15 @@ impl fmt::Debug for WorkerProtocolTransport { } } +/// Derived resource disposition for an existing restore owner. Not durable +/// authority: Runtime must reread its decision before choosing any action. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RestoreRecoveryDisposition { + InFlight, + Continue, + Discard, +} + /// The connection selects this backend; WorkerRef identifies an execution, /// never a capability or authorization token. pub trait WorkerExecutionBackend: Send + Sync + 'static { @@ -643,6 +704,16 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { None } + /// Opt in to Runtime-owned continuation. Waiting is signaled through the + /// original execution context; no callback may wait for Runtime locks. + fn restore_recovery_disposition( + &self, + _operation_id: WorkerLifecycleOperationId, + _worker_ref: &WorkerRef, + ) -> Option { + None + } + fn reconcile_restore( &self, request: WorkerExecutionRestoreRequest, @@ -938,6 +1009,15 @@ impl WorkerExecutionBackendRef { .completed_restore_failure(operation_id, worker_ref) } + pub(crate) fn restore_recovery_disposition( + &self, + operation_id: WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Option { + self.backend + .restore_recovery_disposition(operation_id, worker_ref) + } + pub(crate) fn reconcile_restore( &self, request: WorkerExecutionRestoreRequest, diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index 5418550d1..d2b9c86de 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -43,7 +43,8 @@ use std::sync::atomic::{AtomicU8, AtomicU64, Ordering}; #[path = "test_support.rs"] pub mod test_support; -pub(crate) const SCHEMA_VERSION: u32 = 13; +pub(crate) const SCHEMA_VERSION: u32 = 14; +const RESTORE_DECISION_SCHEMA_VERSION: u32 = 13; const RESTORE_ADMISSION_SCHEMA_VERSION: u32 = 12; // Frozen source schemas: do not move their meaning when adding a migration. const SOURCE_SCHEMA_VERSION: u32 = 11; @@ -653,6 +654,7 @@ pub(crate) struct PersistedWorkerRestoreOperation { pub(crate) operation_id: WorkerLifecycleOperationId, pub(crate) mode: PersistedWorkerRestoreMode, pub(crate) commit: crate::runtime::RestoreCommitFact, + pub(crate) source_repair_possible: bool, pub(crate) request: CreateWorkerRequest, pub(crate) binding: Option, pub(crate) last_settled_status: WorkerStatus, @@ -829,11 +831,12 @@ fn plan_runtime_store_migration( | EXECUTION_SCHEMA_VERSION | SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION + | RESTORE_DECISION_SCHEMA_VERSION ) { return Err(runtime_store_corrupt( &runtime_path, format!( - "unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, or {SCHEMA_VERSION}" + "unsupported Runtime store schema version {schema_version}; expected {OLDEST_SCHEMA_VERSION}, {LEGACY_SCHEMA_VERSION}, {PREVIOUS_SCHEMA_VERSION}, {AGGREGATE_SCHEMA_VERSION}, {EXECUTION_SCHEMA_VERSION}, {SOURCE_SCHEMA_VERSION}, {RESTORE_ADMISSION_SCHEMA_VERSION}, {RESTORE_DECISION_SCHEMA_VERSION}, or {SCHEMA_VERSION}" ), )); } @@ -1165,7 +1168,10 @@ fn migrate_worker_record( ) -> Result { if matches!( source_schema_version, - EXECUTION_SCHEMA_VERSION | SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION + EXECUTION_SCHEMA_VERSION + | SOURCE_SCHEMA_VERSION + | RESTORE_ADMISSION_SCHEMA_VERSION + | RESTORE_DECISION_SCHEMA_VERSION ) { // Admission identity survives migration. Older pending records did not // distinguish a committed decision from preparation or failed cleanup. @@ -1179,7 +1185,10 @@ fn migrate_worker_record( "Worker schema does not match Runtime migration source".into(), )); } - migrate_restore_commit_fact(&mut identity_document, identity_path)?; + if source_schema_version != RESTORE_DECISION_SCHEMA_VERSION { + migrate_restore_commit_fact(&mut identity_document, identity_path)?; + } + migrate_restore_repair_fact(&mut identity_document, identity_path)?; identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION); let aggregate: WorkerAggregateRecord = serde_json::from_value(identity_document) .map_err(|error| runtime_store_corrupt(identity_path, error.to_string()))?; @@ -1294,6 +1303,42 @@ fn migrate_restore_commit_fact( Ok(()) } +// Earlier writers could repair another canonical store before commit without +// retaining that authority in this aggregate. Missing evidence is never false. +fn migrate_restore_repair_fact( + document: &mut serde_json::Value, + path: &Path, +) -> Result<(), RuntimeError> { + let execution = &mut document["execution_state"]; + if execution["state"] != "reconciliation_required" { + return Ok(()); + } + let operation = &mut execution["execution"]; + let restore = match operation["operation"].as_str() { + Some("restore") => Some(&mut operation["intent"]), + Some("stop") => operation["intent"] + .as_object_mut() + .and_then(|stop| stop.get_mut("pending_restore")) + .filter(|restore| !restore.is_null()), + _ => None, + }; + if let Some(restore) = restore { + let restore = restore.as_object_mut().ok_or_else(|| { + runtime_store_corrupt(path, "legacy pending Restore must be an object".into()) + })?; + if restore + .insert("source_repair_possible".into(), serde_json::json!(true)) + .is_some() + { + return Err(runtime_store_corrupt( + path, + "legacy Restore contains unexpected source repair evidence".into(), + )); + } + } + Ok(()) +} + // Frozen schema-v9 conversion. Preserve the exact token and all Restore owners // and receipts; generating a fresh token here would lose retry/admission authority. fn migrate_v9_restore_guard( @@ -1608,7 +1653,7 @@ fn migrate_runtime_document( .or_insert_with(|| serde_json::json!({})); if !matches!( source_schema_version, - SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION + SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION | RESTORE_DECISION_SCHEMA_VERSION ) { object.remove("workers"); // This legacy cache is not Runtime lifecycle or restore authority. Bundles @@ -2148,7 +2193,9 @@ fn migrate_runtime_store_in_place( if let Some(metadata) = metadata && !matches!( plan.current_schema_version, - SOURCE_SCHEMA_VERSION | RESTORE_ADMISSION_SCHEMA_VERSION + SOURCE_SCHEMA_VERSION + | RESTORE_ADMISSION_SCHEMA_VERSION + | RESTORE_DECISION_SCHEMA_VERSION ) { atomic_write_json( @@ -4001,7 +4048,13 @@ mod tests { #[test] fn legacy_pending_restore_migrates_as_unknown_and_preserves_stop_ownership() { use crate::runtime::RestoreCommitFact; - for under_stop in [false, true] { + for (source_schema, under_stop) in [ + RESTORE_ADMISSION_SCHEMA_VERSION, + RESTORE_DECISION_SCHEMA_VERSION, + ] + .into_iter() + .flat_map(|schema| [false, true].map(move |stop| (schema, stop))) + { let (_temp, store, mut worker, path) = canonical_worker_store(); let root = store.root.clone(); let PersistedWorkerExecutionState::Available(execution) = @@ -4013,6 +4066,7 @@ mod tests { operation_id: WorkerLifecycleOperationId::new(), mode: PersistedWorkerRestoreMode::Explicit, commit: RestoreCommitFact::Committed, + source_repair_possible: false, request: execution.request.clone(), binding: execution.binding.clone(), last_settled_status: worker.status, @@ -4033,21 +4087,27 @@ mod tests { PersistedWorkerExecutionState::ReconciliationRequired(operation); let mut legacy = serde_json::to_value(WorkerAggregateRecord::from_persisted(&worker)).unwrap(); - legacy["schema_version"] = serde_json::json!(RESTORE_ADMISSION_SCHEMA_VERSION); + legacy["schema_version"] = serde_json::json!(source_schema); let intent = &mut legacy["execution_state"]["execution"]["intent"]; let restore = if under_stop { &mut intent["pending_restore"] } else { intent }; - restore.as_object_mut().unwrap().remove("commit"); + if source_schema == RESTORE_ADMISSION_SCHEMA_VERSION { + restore.as_object_mut().unwrap().remove("commit"); + } + restore + .as_object_mut() + .unwrap() + .remove("source_repair_possible"); let expected_request = restore["request"].clone(); let before_guard = legacy["restore_guard"].clone(); fs::write(&path, serde_json::to_vec(&legacy).unwrap()).unwrap(); write_empty_schema_v8_store(&root); let runtime_path = root.join(RUNTIME_FILE); let mut runtime: serde_json::Value = read_json(&runtime_path, "read fixture").unwrap(); - runtime["schema_version"] = serde_json::json!(RESTORE_ADMISSION_SCHEMA_VERSION); + runtime["schema_version"] = serde_json::json!(source_schema); fs::write(&runtime_path, serde_json::to_vec(&runtime).unwrap()).unwrap(); drop(store); @@ -4062,7 +4122,18 @@ mod tests { PersistedWorkerLifecycleOperation::Restore(restore) => restore, PersistedWorkerLifecycleOperation::Stop(stop) => stop.pending_restore.unwrap(), }; - assert_eq!(restored.commit, RestoreCommitFact::LegacyUnknown); + assert_eq!( + restored.commit, + if source_schema == RESTORE_DECISION_SCHEMA_VERSION { + RestoreCommitFact::Committed + } else { + RestoreCommitFact::LegacyUnknown + } + ); + assert!( + restored.source_repair_possible, + "older source-repair effects must remain possible" + ); assert_eq!( serde_json::to_value(restored.request).unwrap(), expected_request diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 251e423c2..6c396e6b4 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -1,3 +1,4 @@ +mod restore_continuation; use crate::catalog::{ ConfigBundleRef, CreateWorkerRequest, LogicalWorkdirAttachment, ProfileSelector, ProfileSourceArchiveSource, RepositoryRefObservation, RepositoryRefObservationRequest, @@ -323,6 +324,7 @@ impl Drop for RuntimeEventSelectorSubscription { pub struct Runtime { inner: Arc>, worker_operations: Arc>>>>, + restore_continuations: Arc, } struct WorkerOperationLease { @@ -586,6 +588,7 @@ impl Runtime { Self { inner: Arc::new(Mutex::new(state)), worker_operations: Arc::new(Mutex::new(BTreeMap::new())), + restore_continuations: Arc::default(), } } @@ -662,6 +665,7 @@ impl Runtime { let runtime = Self { inner: Arc::new(Mutex::new(state)), worker_operations: Arc::new(Mutex::new(BTreeMap::new())), + restore_continuations: Arc::default(), }; runtime.restore_persisted_worker_executions()?; Ok(runtime) @@ -2230,6 +2234,7 @@ impl Runtime { operation_id: WorkerLifecycleOperationId::new(), mode: WorkerRestoreMode::Explicit, commit: RestoreCommitFact::NotCommitted, + source_repair_possible: false, last_settled_status: candidate.status, }); let operation_id = pending.operation_id; @@ -2256,6 +2261,10 @@ impl Runtime { match state.persist_restore_worker_record(&candidate) { Ok(()) => {} Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + drop(state); + if !coordinate { + self.start_restore_continuation(worker_ref.clone(), operation_id)?; + } return Ok(( Some(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, @@ -2287,8 +2296,10 @@ impl Runtime { return Ok((None, owner.request.preparation.clone())); } } - self.restore_admitted_worker_under_lock(worker_ref, WorkerRestoreMode::Explicit) - .map(|result| (Some(result), None)) + let result = + self.restore_admitted_worker_under_lock(worker_ref, WorkerRestoreMode::Explicit); + self.ensure_restore_continuation(worker_ref)?; + result.map(|result| (Some(result), None)) } #[cfg(all(test, feature = "fs-store"))] @@ -2529,6 +2540,8 @@ impl Runtime { .cloned() }); let request = WorkerExecutionRestoreRequest { + source_repair_possible: pending_restore + .is_some_and(|pending| pending.source_repair_possible), operation_id, worker_ref: worker_ref.clone(), request: worker_request, @@ -4075,12 +4088,26 @@ impl Runtime { crate::execution::WorkerExecutionContext::new(worker_ref) } + fn restore_progress_notifier( + &self, + worker_ref: WorkerRef, + ) -> Arc { + let continuations = Arc::downgrade(&self.restore_continuations); + Arc::new(move |operation| { + if let Some(continuations) = continuations.upgrade() { + continuations.signal(&worker_ref, operation); + } + }) + } + #[cfg(feature = "ws-server")] fn restore_candidate_execution_context( &self, worker_ref: WorkerRef, ) -> crate::execution::WorkerExecutionContext { let runtime = Arc::downgrade(&self.inner); + let progress = self.restore_progress_notifier(worker_ref.clone()); + let repair = self.restore_repair_authorizer(worker_ref.clone()); crate::execution::WorkerExecutionContext::candidate( worker_ref, Arc::new(move |worker_ref, payload| { @@ -4101,6 +4128,8 @@ impl Runtime { Ok(state.push_worker_observation_event(worker_ref, payload)) }), ) + .with_restore_progress(progress) + .with_restore_repair_authorizer(repair) } #[cfg(not(feature = "ws-server"))] @@ -4108,7 +4137,11 @@ impl Runtime { &self, worker_ref: WorkerRef, ) -> crate::execution::WorkerExecutionContext { + let progress = self.restore_progress_notifier(worker_ref.clone()); + let repair = self.restore_repair_authorizer(worker_ref.clone()); crate::execution::WorkerExecutionContext::candidate(worker_ref) + .with_restore_progress(progress) + .with_restore_repair_authorizer(repair) } #[cfg(feature = "fs-store")] @@ -4410,6 +4443,9 @@ impl Runtime { commit: candidate .pending_restore .map_or(RestoreCommitFact::NotCommitted, |pending| pending.commit), + source_repair_possible: candidate + .pending_restore + .is_some_and(|pending| pending.source_repair_possible), last_settled_status: candidate.status, }); candidate.restore_candidate_context = Some(candidate_context); @@ -6327,6 +6363,9 @@ struct PendingWorkerRestore { operation_id: WorkerLifecycleOperationId, mode: WorkerRestoreMode, commit: RestoreCommitFact, + // False proves Runtime never granted this operation authority to repair + // canonical source before its decision. True includes legacy uncertainty. + source_repair_possible: bool, last_settled_status: WorkerStatus, } @@ -6460,10 +6499,9 @@ struct WorkerRecord { impl WorkerRecord { fn ensure_restore_rollback_allowed(&self) -> Result<(), RuntimeError> { - if self - .pending_restore - .is_some_and(|pending| pending.commit != RestoreCommitFact::NotCommitted) - { + if self.pending_restore.is_some_and(|pending| { + pending.commit != RestoreCommitFact::NotCommitted || pending.source_repair_possible + }) { return Err(RuntimeError::InvalidRequest( "A committed or legacy-unknown Restore cannot be settled as a rollback".into(), )); @@ -6682,6 +6720,7 @@ impl WorkerRecord { operation_id: operation.operation_id, mode: operation.mode.into(), commit: operation.commit, + source_repair_possible: operation.source_repair_possible, last_settled_status: operation.last_settled_status, }), None, @@ -6698,6 +6737,7 @@ impl WorkerRecord { operation_id: restore.operation_id, mode: restore.mode.into(), commit: restore.commit, + source_repair_possible: restore.source_repair_possible, last_settled_status: restore.last_settled_status, }), Some(PendingWorkerStop { @@ -6762,6 +6802,7 @@ impl WorkerRecord { operation_id: restore.operation_id, mode: restore.mode.into(), commit: restore.commit, + source_repair_possible: restore.source_repair_possible, request: self .request .clone() @@ -6786,6 +6827,7 @@ impl WorkerRecord { operation_id: pending.operation_id, mode: pending.mode.into(), commit: pending.commit, + source_repair_possible: pending.source_repair_possible, request, binding: self .execution_bound @@ -9156,6 +9198,7 @@ mod tests { operation_id: WorkerLifecycleOperationId::new(), mode: WorkerRestoreMode::Explicit, commit: RestoreCommitFact::NotCommitted, + source_repair_possible: false, last_settled_status: WorkerStatus::Stopped, }); let id = "workdir-irrelevant-to-recovery"; @@ -13588,7 +13631,7 @@ mod tests { }) .unwrap_err(); assert!(error.to_string().contains( - "unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, or 13" + "unsupported Runtime store schema version 2; expected 6, 7, 8, 9, 10, 11, 12, 13, or 14" )); let _ = std::fs::remove_dir_all(root); @@ -15014,6 +15057,87 @@ mod tests { let _ = std::fs::remove_dir_all(root); } + #[cfg(feature = "fs-store")] + #[test] + fn restore_source_repair_authority_is_durable_before_effects_without_reentering_lifecycle_lock() + { + for fault in [ + None, + Some(crate::fs_store::AtomicWriteFault::BeforeRename), + Some(crate::fs_store::AtomicWriteFault::AfterRename), + ] { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("repair-authority"); + let runtime = Runtime::with_fs_store_and_execution_backend( + options, + Arc::new(TestExecutionBackend::default()), + ) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("repair authority")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let lease = runtime.worker_operation_lock(worker.worker_id).unwrap(); + let _guard = lease.lock().unwrap(); + let operation = WorkerLifecycleOperationId::new(); + runtime + .begin_restore_operation( + &worker.worker_ref, + operation, + WorkerRestoreMode::Explicit, + runtime.restore_candidate_execution_context(worker.worker_ref.clone()), + ) + .unwrap(); + if let Some(fault) = fault { + runtime_store(&runtime).fail_next_worker_write(fault); + if matches!(fault, crate::fs_store::AtomicWriteFault::AfterRename) { + runtime_store(&runtime).fail_next_worker_reconcile_sync(); + } + } + let authorizer = runtime.restore_repair_authorizer(worker.worker_ref.clone()); + let callback = authorizer.clone(); + let (sent, received) = std::sync::mpsc::channel(); + let task = std::thread::spawn(move || sent.send(callback(operation)).unwrap()); + // The lifecycle lock above remains held while this exact factory + // callback persists its grant. Waiting on that lock would deadlock. + let granted = received + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("repair callback reentered lifecycle lock"); + task.join().unwrap(); + assert_eq!(granted.is_ok(), fault.is_none()); + let saved = runtime_store(&runtime) + .read_worker_record(&worker.worker_ref) + .unwrap(); + let PersistedWorkerExecutionState::ReconciliationRequired( + PersistedWorkerLifecycleOperation::Restore(saved), + ) = saved.execution_state + else { + panic!("repair grant lost operation") + }; + assert_eq!(saved.operation_id, operation); + assert_eq!(saved.commit, RestoreCommitFact::NotCommitted); + assert_eq!( + saved.source_repair_possible, + !matches!(fault, Some(crate::fs_store::AtomicWriteFault::BeforeRename)) + ); + // A denied/unknown grant must not authorize external repair. Once + // storage is healthy the same operation reconciles, never a new ID. + authorizer(operation).unwrap(); + let rollback = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "unsafe", + "repair effects may exist", + ); + assert!( + runtime + .settle_restore_rollback(&worker.worker_ref, &rollback) + .is_err() + ); + } + } + #[cfg(feature = "fs-store")] #[test] fn restore_decision_reread_preserves_commit_across_uncertainty_stop_and_restart() { diff --git a/crates/worker-runtime/src/runtime/restore_continuation.rs b/crates/worker-runtime/src/runtime/restore_continuation.rs new file mode 100644 index 000000000..78cda5fdc --- /dev/null +++ b/crates/worker-runtime/src/runtime/restore_continuation.rs @@ -0,0 +1,498 @@ +//! Runtime-owned continuation of an admitted restore, independent of its caller. +//! +//! Completion callbacks only signal a condition variable. They never acquire a +//! Runtime/lifecycle lock and are never joined by this driver while holding one. +//! Each step revalidates the canonical owner under the existing Worker lock; +//! waiting/backoff and driver teardown happen outside it. A repeated obstruction +//! parks this owner with its reason, rather than spinning or inventing rollback. + +use super::*; +use crate::execution::RestoreRecoveryDisposition; +use std::collections::HashMap; +use std::sync::{Condvar, Weak}; +use std::thread::JoinHandle; +use std::time::Duration; + +type Key = (WorkerRef, WorkerLifecycleOperationId); + +#[derive(Debug, Default)] +pub(super) struct RestoreContinuations { + owners: Mutex>>, + #[cfg(test)] + paused: std::sync::atomic::AtomicBool, +} + +#[derive(Debug, Default)] +struct Schedule { + signaled: bool, + cancelled: bool, + finished: bool, + blocked_reason: Option, + parked: bool, +} + +#[derive(Debug, Default)] +struct Continuation { + schedule: Mutex, + wake: Condvar, + task: Mutex>>, +} + +impl RestoreContinuations { + pub(super) fn signal(&self, worker_ref: &WorkerRef, operation: WorkerLifecycleOperationId) { + if let Some(owner) = self + .owners + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&(worker_ref.clone(), operation)) + { + let mut schedule = owner + .schedule + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + schedule.signaled = true; + owner.wake.notify_all(); + } + } +} + +impl Drop for RestoreContinuations { + fn drop(&mut self) { + let owners = self + .owners + .get_mut() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for owner in owners.values() { + let mut schedule = owner + .schedule + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + schedule.cancelled = true; + owner.wake.notify_all(); + } + for owner in owners.values() { + if let Some(task) = owner + .task + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + if task.thread().id() != std::thread::current().id() { + let _ = task.join(); + } + } + } + } +} + +#[cfg(test)] +pub(crate) struct ContinuationPause(Arc); + +#[cfg(test)] +impl Drop for ContinuationPause { + fn drop(&mut self) { + self.0 + .paused + .store(false, std::sync::atomic::Ordering::Release); + for owner in self.0.owners.lock().unwrap().values() { + owner.schedule.lock().unwrap().signaled = true; + owner.wake.notify_all(); + } + } +} + +enum Step { + Finished, + Waiting, + Contended, + Retry(String), +} + +impl Runtime { + pub(super) fn restore_repair_authorizer( + &self, + worker_ref: WorkerRef, + ) -> Arc Result<(), RuntimeError> + Send + Sync> { + let inner = Arc::downgrade(&self.inner); + Arc::new(move |operation| { + let inner = inner.upgrade().ok_or(RuntimeError::RuntimeStopped)?; + // The factory caller owns the lifecycle lock. Never reacquire it + // here: it may be awaiting this exact factory task. + let mut state = inner.lock().map_err(|_| RuntimeError::StatePoisoned)?; + state.ensure_running()?; + state.refresh_uncertain_worker_write(&worker_ref)?; + let worker = state.worker(&worker_ref)?; + state.ensure_subject_not_reserved(&worker_ref, false)?; + if worker.pending_stop.is_some() + || worker.pending_restore.is_none_or(|pending| { + pending.operation_id != operation + || pending.commit == RestoreCommitFact::LegacyUnknown + }) + { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + if let Some(owner) = worker + .restore_guard + .active_request_id + .as_ref() + .and_then(|id| worker.restore_guard.owners.get(id)) + { + owner.validate_workdir_preparation_authority(worker)?; + } + if worker + .pending_restore + .is_some_and(|pending| pending.source_repair_possible) + { + return Ok(()); + } + let mut candidate = worker.clone(); + candidate + .pending_restore + .as_mut() + .expect("validated restore owner") + .source_repair_possible = true; + // Grant before touching another canonical store. Unknown durability + // denies repair until authoritative reconciliation; no old-memory write. + state.persist_restore_worker_record(&candidate)?; + state.workers.insert(worker_ref.worker_id, candidate); + Ok(()) + }) + } + + #[cfg(test)] + pub(crate) fn test_wait_restore_continuation_parked( + &self, + worker_ref: &WorkerRef, + operation: WorkerLifecycleOperationId, + timeout: Duration, + ) -> String { + let owner = self + .restore_continuations + .owners + .lock() + .unwrap() + .get(&(worker_ref.clone(), operation)) + .cloned() + .unwrap(); + let (schedule, waited) = owner + .wake + .wait_timeout_while(owner.schedule.lock().unwrap(), timeout, |s| { + !s.parked && !s.finished + }) + .unwrap(); + assert!( + !waited.timed_out() && schedule.parked, + "continuation did not park: {:?}", + schedule.blocked_reason + ); + schedule.blocked_reason.clone().expect("parked reason") + } + + /// Gate autonomous steps when a component test observes a single fault + /// boundary. Dedicated continuation tests run without this gate. + #[cfg(test)] + pub(crate) fn test_pause_restore_continuation(&self) -> ContinuationPause { + assert!( + !self + .restore_continuations + .paused + .swap(true, std::sync::atomic::Ordering::AcqRel) + ); + ContinuationPause(self.restore_continuations.clone()) + } + + #[cfg(test)] + pub(crate) fn test_wait_restore_continuation( + &self, + worker_ref: &WorkerRef, + operation: WorkerLifecycleOperationId, + timeout: Duration, + ) { + let owner = self + .restore_continuations + .owners + .lock() + .unwrap() + .get(&(worker_ref.clone(), operation)) + .cloned() + .expect("continuation owner"); + let schedule = owner.schedule.lock().unwrap(); + let (schedule, waited) = owner + .wake + .wait_timeout_while(schedule, timeout, |s| !s.finished && !s.cancelled) + .unwrap(); + assert!( + !waited.timed_out() && schedule.finished, + "restore continuation did not settle: {:?}", + schedule.blocked_reason + ); + } + + pub(super) fn ensure_restore_continuation( + &self, + worker_ref: &WorkerRef, + ) -> Result<(), RuntimeError> { + let operation = { + let state = self.lock()?; + let worker = state.worker(worker_ref)?; + worker + .pending_restore + .map(|pending| pending.operation_id) + .or_else(|| { + worker + .restore_guard + .active_request_id + .as_ref() + .and_then(|id| worker.restore_guard.owners.get(id)) + .filter(|owner| owner.receipt.is_none()) + .map(|owner| owner.operation_id) + }) + }; + if let Some(operation) = operation { + self.start_restore_continuation(worker_ref.clone(), operation)?; + } + Ok(()) + } + + pub(super) fn start_restore_continuation( + &self, + worker_ref: WorkerRef, + operation: WorkerLifecycleOperationId, + ) -> Result<(), RuntimeError> { + { + let state = self.lock()?; + let Some(backend) = &state.execution_backend else { + return Ok(()); + }; + if backend + .restore_recovery_disposition(operation, &worker_ref) + .is_none() + { + return Ok(()); + } + } + let key = (worker_ref.clone(), operation); + let mut owners = self + .restore_continuations + .owners + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + if owners.contains_key(&key) { + return Ok(()); + } + let owner = Arc::new(Continuation::default()); + let retained = owner.clone(); + let state = Arc::downgrade(&self.inner); + let operations = Arc::downgrade(&self.worker_operations); + let continuations = Arc::downgrade(&self.restore_continuations); + let task = std::thread::Builder::new() + .name("restore-continuation".into()) + .spawn(move || { + run( + state, + operations, + continuations, + retained, + worker_ref, + operation, + ); + }) + .map_err(|error| { + RuntimeError::InvalidRequest(format!( + "Restore continuation could not start: {error}" + )) + })?; + *owner.task.lock().map_err(|_| RuntimeError::StatePoisoned)? = Some(task); + owners.insert(key, owner); + Ok(()) + } + + fn continue_restore_operation( + &self, + worker_ref: &WorkerRef, + operation: WorkerLifecycleOperationId, + ) -> Result { + #[cfg(test)] + if self + .restore_continuations + .paused + .load(std::sync::atomic::Ordering::Acquire) + { + return Ok(Step::Contended); + } + let lease = self.worker_operation_lock(worker_ref.worker_id)?; + // Another caller may be waiting for a backend task. Do not queue a join + // of that caller/driver while retaining the lock it needs to complete. + let _guard = match lease.try_lock() { + Ok(guard) => guard, + Err(std::sync::TryLockError::WouldBlock) => return Ok(Step::Contended), + Err(_) => return Err(RuntimeError::StatePoisoned), + }; + let (backend, pending) = { + let mut state = self.lock()?; + state.ensure_running()?; + state.refresh_uncertain_worker_write(worker_ref)?; + let worker = state.worker(worker_ref)?; + if worker.pending_stop.is_some() { + return Ok(Step::Finished); + } + let owner = worker + .restore_guard + .active_request_id + .as_ref() + .and_then(|id| worker.restore_guard.owners.get(id)); + if owner.is_some_and(|owner| owner.operation_id != operation || owner.receipt.is_some()) + { + return Ok(Step::Finished); + } + let Some(pending) = worker.pending_restore else { + return Ok(Step::Finished); + }; + if pending.operation_id != operation { + return Ok(Step::Finished); + } + if pending.commit == RestoreCommitFact::LegacyUnknown { + return Ok(Step::Retry( + "Saved Restore has no authoritative commit evidence".into(), + )); + } + state.ensure_subject_not_reserved(worker_ref, false)?; + if let Some(owner) = owner { + owner.validate_workdir_preparation_authority(worker)?; + } + let Some(backend) = state.execution_backend.clone() else { + return Ok(Step::Finished); + }; + (backend, pending) + }; + match backend.restore_recovery_disposition(operation, worker_ref) { + None => return Ok(Step::Finished), + Some(RestoreRecoveryDisposition::InFlight) => return Ok(Step::Waiting), + Some(RestoreRecoveryDisposition::Discard) + if pending.commit == RestoreCommitFact::NotCommitted => + { + match backend.discard_uncommitted_restore(operation, worker_ref) { + WorkerExecutionSpawnResult::RolledBack(result) => { + let receipt = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "worker_restore_rolled_back", + result.message_or_default(), + ); + self.settle_restore_rollback(worker_ref, &receipt)?; + return Ok(Step::Finished); + } + result => { + return Ok(Step::Retry(format!( + "Restore cleanup remains unresolved: {result:?}" + ))); + } + } + } + Some(_) => {} + } + let result = self.restore_admitted_worker_under_lock(worker_ref, pending.mode)?; + if backend.restore_recovery_disposition(operation, worker_ref) + == Some(RestoreRecoveryDisposition::InFlight) + { + return Ok(Step::Waiting); + } + if result.state == WorkerRestoreState::ReconciliationRequired { + Ok(Step::Retry(result.message.unwrap_or_else(|| { + "Restore reconciliation remains pending".into() + }))) + } else { + Ok(Step::Finished) + } + } +} + +fn run( + inner: Weak>, + operations: Weak>>>>, + continuations: Weak, + owner: Arc, + worker_ref: WorkerRef, + operation: WorkerLifecycleOperationId, +) { + let mut repeated = 0u32; + let mut last_reason = None; + loop { + let (Some(inner), Some(worker_operations), Some(restore_continuations)) = ( + inner.upgrade(), + operations.upgrade(), + continuations.upgrade(), + ) else { + break; + }; + owner + .schedule + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .parked = false; + let runtime = Runtime { + inner, + worker_operations, + restore_continuations, + }; + let step = runtime.continue_restore_operation(&worker_ref, operation); + // No lifecycle lock or strong Runtime/registry reference survives waiting. + drop(runtime); + let mut schedule = owner + .schedule + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if schedule.cancelled { + break; + } + let contended = matches!(&step, Ok(Step::Contended)); + let reason = match step { + Ok(Step::Finished) => { + schedule.finished = true; + owner.wake.notify_all(); + break; + } + Ok(Step::Waiting) | Ok(Step::Contended) => None, + Ok(Step::Retry(reason)) => Some(reason), + Err(error) => Some(sanitize_worker_create_failure_message(&error.to_string())), + }; + if let Some(reason) = &reason { + repeated = repeated.saturating_add(1); + last_reason = Some(reason.clone()); + schedule.blocked_reason = Some(reason.clone()); + } + // Completed failed attempts can have signaled while this step joined + // them. Do not let our own unchanged failure reset the retry budget. + if reason.is_some() { + schedule.signaled = false; + } + if (!contended && reason.is_none()) || repeated >= 4 { + if repeated >= 4 { + schedule.parked = true; + owner.wake.notify_all(); + tracing::warn!(worker_id = %worker_ref.worker_id, operation_id = %operation, reason = ?last_reason, + "Restore continuation parked after bounded attempts; operation remains unresolved"); + } + schedule = owner + .wake + .wait_while(schedule, |s| !s.signaled && !s.cancelled) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } else { + let delay = Duration::from_millis(100 * (1u64 << repeated.min(4))); + schedule = owner + .wake + .wait_timeout_while(schedule, delay, |s| !s.signaled && !s.cancelled) + .unwrap_or_else(std::sync::PoisonError::into_inner) + .0; + } + if schedule.cancelled { + break; + } + if schedule.signaled { + schedule.signaled = false; + // A completion wakes this same owner, not a new request. Its next + // observed result, not the signal itself, determines progress. + } + } +} diff --git a/crates/worker-runtime/src/test_support_tests.rs b/crates/worker-runtime/src/test_support_tests.rs index a4931def8..562ec2e8e 100644 --- a/crates/worker-runtime/src/test_support_tests.rs +++ b/crates/worker-runtime/src/test_support_tests.rs @@ -564,6 +564,7 @@ fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() { operation_id: WorkerLifecycleOperationId::new(), mode: PersistedWorkerRestoreMode::Explicit, commit: crate::runtime::RestoreCommitFact::NotCommitted, + source_repair_possible: false, request: request(), binding: None, last_settled_status: WorkerStatus::Stopped, diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index cf6bfe5d5..3d95f99ff 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -370,10 +370,23 @@ async fn prepare_restore_operation( factory: &F, request: WorkerExecutionRestoreRequest, ) -> Result, RuntimeRestoreFailure> { + let prior_effects = request.source_repair_possible; factory.preflight_restore(&request).await.map_err(|_| { - RuntimeRestoreFailure::refused(RuntimeRestoreStage::Preflight, "restore preflight refused") + let mut failure = RuntimeRestoreFailure::refused( + RuntimeRestoreStage::Preflight, + "restore preflight refused", + ); + failure.durable_effects_possible = prior_effects; + failure })?; - let repaired = factory.recover_restore_source(&request).await?; + let repaired = factory + .recover_restore_source(&request) + .await + .map_err(|mut failure| { + failure.durable_effects_possible |= prior_effects; + failure + })? + || prior_effects; let candidate = factory .prepare_restore(request) .await @@ -804,6 +817,7 @@ struct RestoreFaults { cleanup_fail: AtomicBool, socket_failure: AtomicBool, publication_failure: AtomicBool, + publication_attempts: std::sync::atomic::AtomicUsize, factory_gate: Mutex>>, factory_entered: Mutex>>, prepared_gate: Mutex>>, @@ -2626,6 +2640,11 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate { } let started = async { #[cfg(test)] + self.factory + .restore_faults + .publication_attempts + .fetch_add(1, Ordering::SeqCst); + #[cfg(test)] if self .factory .restore_faults @@ -3120,6 +3139,7 @@ struct RuntimeWorkerExecution { struct PendingRuntimeRestore { request: WorkerExecutionRestoreRequest, task: tokio::sync::Mutex>>, + task_in_flight: AtomicBool, result: Mutex, RuntimeRestoreFailure>>>, activated: Mutex>>, // Monotonic for this operation, not a claim that its current candidate has @@ -3131,7 +3151,24 @@ struct PendingRuntimeRestore { failure: Mutex>, } +struct RestoreTaskCompletion(Arc); + +impl Drop for RestoreTaskCompletion { + fn drop(&mut self) { + self.0.task_in_flight.store(false, Ordering::Release); + self.0 + .request + .context + .restore_progressed(self.0.request.operation_id); + } +} + impl PendingRuntimeRestore { + fn task_completion(self: &Arc) -> RestoreTaskCompletion { + self.task_in_flight.store(true, Ordering::Release); + RestoreTaskCompletion(self.clone()) + } + fn owns_prepared_candidate(&self) -> bool { matches!( self.result @@ -3455,7 +3492,9 @@ where if task.is_some() { return Err("restore task has not been joined".into()); } + let completion = pending.task_completion(); *task = Some(self.spawn_on_adapter_runtime(async move { + let _completion = completion; if resume_activation && !failure.needs_cleanup() && (failure.durable_effects_possible @@ -5340,6 +5379,7 @@ where let pending = Arc::new(PendingRuntimeRestore { request: request.clone(), task: tokio::sync::Mutex::new(None), + task_in_flight: AtomicBool::new(false), result: Mutex::new(None), activated: Mutex::new(None), activation_started: AtomicBool::new(false), @@ -5361,7 +5401,9 @@ where resources.insert(worker_ref.clone(), Arc::clone(&pending)); let factory = Arc::clone(&self.factory); let result_owner = Arc::clone(&pending); + let completion = pending.task_completion(); let task = self.spawn_on_adapter_runtime(async move { + let _completion = completion; let result = match prepare_restore_operation(factory.as_ref(), request.clone()) .await { @@ -5551,6 +5593,42 @@ where } } + fn restore_recovery_disposition( + &self, + operation_id: crate::execution::WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Option { + use crate::execution::RestoreRecoveryDisposition as D; + let pending = self.pending_restore(worker_ref).ok().flatten(); + Some(match pending { + Some(pending) if pending.request.operation_id == operation_id => { + if pending.task_in_flight.load(Ordering::Acquire) { + D::InFlight + } else if pending + .discard_result + .lock() + .ok() + .is_some_and(|r| matches!(*r, Some(Ok(())))) + { + // Cleanup is proven; prior repair effects may now require + // reconstruction, not repeated discard of a closed candidate. + D::Continue + } else if pending.discard_started.load(Ordering::Acquire) + || pending + .discard_result + .lock() + .ok() + .is_some_and(|r| r.is_some()) + { + D::Discard + } else { + D::Continue + } + } + _ => D::Continue, + }) + } + fn discard_uncommitted_restore( &self, operation_id: crate::execution::WorkerLifecycleOperationId, @@ -5582,8 +5660,10 @@ where self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; if !pending.discard_started.swap(true, Ordering::AcqRel) { let owner = pending.clone(); + let completion = pending.task_completion(); let task = self.spawn_on_adapter_runtime(async move { + let _completion = completion; let retained = owner .result .lock() @@ -5709,8 +5789,10 @@ where pending.activation_started.store(true, Ordering::Release); let owner = pending.clone(); let factory = self.factory.clone(); + let completion = pending.task_completion(); let task = self.spawn_on_adapter_runtime(async move { + let _completion = completion; let candidate = owner .result .lock() @@ -6683,6 +6765,7 @@ mod tests { }); store.write(&metadata).unwrap(); WorkerExecutionRestoreRequest { + source_repair_possible: false, operation_id: crate::execution::WorkerLifecycleOperationId::new(), worker_ref: worker_ref.clone(), request, @@ -7614,6 +7697,7 @@ mod tests { Arc::new(PendingRuntimeRestore { request: direct_restore_request(worker_ref.clone()), task: tokio::sync::Mutex::new(None), + task_in_flight: AtomicBool::new(false), result: Mutex::new(None), activated: Mutex::new(None), activation_started: AtomicBool::new(false), @@ -7657,6 +7741,7 @@ mod tests { let pending = Arc::new(PendingRuntimeRestore { request: request.clone(), task: tokio::sync::Mutex::new(Some(failed)), + task_in_flight: AtomicBool::new(false), result: Mutex::new(None), activated: Mutex::new(None), activation_started: AtomicBool::new(false), @@ -10156,6 +10241,7 @@ mod tests { fn direct_restore_request(worker_ref: WorkerRef) -> WorkerExecutionRestoreRequest { WorkerExecutionRestoreRequest { + source_repair_possible: false, operation_id: crate::execution::WorkerLifecycleOperationId::new(), request: create_request("pending operation"), workspace_scope: None, @@ -10729,6 +10815,7 @@ mod tests { .with_runtime_store_dir(&runtime_store_dir) .with_remote_worker_mutation_identity(identity) .restore_controller(WorkerExecutionRestoreRequest { + source_repair_possible: false, operation_id: crate::execution::WorkerLifecycleOperationId::new(), worker_ref: worker_ref.clone(), request, @@ -10826,6 +10913,7 @@ mod tests { .with_runtime_store_dir(&runtime_store_dir) .with_controller_transport(WorkerControllerTransport::InProcess) .restore_controller(WorkerExecutionRestoreRequest { + source_repair_possible: false, operation_id: crate::execution::WorkerLifecycleOperationId::new(), worker_ref: worker_ref.clone(), request: create_request("embedded restore"), @@ -10990,6 +11078,7 @@ mod tests { runtime.stop_worker(&worker.worker_ref, None).unwrap(); let operation_id = crate::execution::WorkerLifecycleOperationId::new(); let restore_request = WorkerExecutionRestoreRequest { + source_repair_possible: false, operation_id, worker_ref: worker.worker_ref.clone(), request, @@ -11383,6 +11472,7 @@ mod tests { let detail = runtime.create_worker(request.clone()).unwrap(); runtime.stop_worker(&detail.worker_ref, None).unwrap(); let restore = WorkerExecutionRestoreRequest { + source_repair_possible: false, operation_id: crate::execution::WorkerLifecycleOperationId::new(), worker_ref: detail.worker_ref.clone(), request, diff --git a/crates/worker-runtime/src/worker_backend/restore_recovery.rs b/crates/worker-runtime/src/worker_backend/restore_recovery.rs index 54feb63bd..4c751e872 100644 --- a/crates/worker-runtime/src/worker_backend/restore_recovery.rs +++ b/crates/worker-runtime/src/worker_backend/restore_recovery.rs @@ -33,6 +33,10 @@ impl ProfileRuntimeWorkerFactory { if !pending && !incomplete { return Ok(false); } + request + .context + .authorize_restore_source_repair(request.operation_id) + .map_err(|_| cause("canonical source repair authority is not durably confirmed"))?; // The accepted Runtime operation already owns this Session. Its stable // Session identity and sole committed seed, not a new restore request, // are used to reconcile an interrupted first-head publication. diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs index 620fd140d..d622a7462 100644 --- a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -688,6 +688,7 @@ fn profile_restore_prior_effects_survive_read_only_reprepare_failure() { for read_stage in [RuntimeRestoreStage::Metadata, RuntimeRestoreStage::Reconstruction] { let activation_stage = RuntimeRestoreStage::ActivationEffects; let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + let _pause = fixture.runtime.test_pause_restore_continuation(); let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload(); fixture.fail(activation_stage); let request = fixture @@ -882,6 +883,7 @@ fn profile_restore_prior_effects_from_pending_head_repair_survive_read_only_repr #[serial_test::serial(worker_allocation)] fn profile_restore_prior_effects_survive_rebuilt_candidate_pre_effect_activation_failure() { let fixture = ProfileFailureFixture::new(Duration::from_secs(5)); + let _pause = fixture.runtime.test_pause_restore_continuation(); let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload(); fixture.fail(RuntimeRestoreStage::ActivationEffects); let request = fixture diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs index 394fc64d9..3cc482aab 100644 --- a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -94,6 +94,141 @@ mod publication { (release, received) } + #[test] + #[serial_test::serial(worker_allocation)] + fn autonomous_late_prepare_and_publication_settle_without_another_request() { + for publication_failure in [false, true] { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend).unwrap().spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let request = runtime.test_restore_request(&worker_ref); + let (release, prepared) = prepared_gate(&fixture); + fixture.backend.factory.restore_faults.publication_failure.store(publication_failure, Ordering::SeqCst); + let first = runtime.restore_worker_operation_scoped(fixture.request.workspace_scope.as_ref().unwrap(), &worker_ref, request.clone()).unwrap(); + prepared.recv_timeout(WAIT).unwrap(); + assert_eq!(first.state, server_api::WorkerRestoreState::ReconciliationRequired); + let operation = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!(operation.commit, crate::runtime::RestoreCommitFact::NotCommitted); + fixture.assert_no_allocation(); + release.notify_one(); + runtime.test_wait_restore_continuation(&worker_ref, operation.operation_id, WAIT); + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + let receipt = serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap(); + assert_eq!(receipt["owners"][&request.request_id]["receipt"]["state"], "accepted"); + assert!(durable_operation(&runtime, &worker_ref).is_none()); + let transport = fixture.backend.clone().attach_worker_protocol(&worker_ref).unwrap(); + transport.validate().unwrap(); + drop(transport); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 1); + fixture.assert_quiet(); + runtime.stop_worker_scoped(fixture.request.workspace_scope.as_ref().unwrap(), &worker_ref, None).unwrap(); + fixture.assert_released(); + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn autonomous_cleanup_dispatch_settles_uncommitted_candidate_without_repost_or_stop() { + let fixture = Fixture::no_model(); + let runtime = Arc::new(stopped_runtime(&fixture)); + let pause = runtime.test_pause_restore_continuation(); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let before = fixture.canonical_files(); + let (release, prepared) = prepared_gate(&fixture); + let restoring = { + let runtime = runtime.clone(); + let worker_ref = worker_ref.clone(); + let scope = scope.clone(); + let request = request.clone(); + std::thread::spawn(move || runtime.restore_worker_operation_scoped(&scope, &worker_ref, request)) + }; + prepared.recv_timeout(WAIT).unwrap(); + runtime.test_runtime_store().fail_next_worker_write(AtomicWriteFault::BeforeRename); + fixture.backend.factory.restore_faults.cleanup_fail.store(true, Ordering::SeqCst); + release.notify_one(); + let first = restoring.join().unwrap().unwrap(); + assert_eq!(first.state, server_api::WorkerRestoreState::ReconciliationRequired); + let operation = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!(operation.commit, crate::runtime::RestoreCommitFact::NotCommitted); + let pending = fixture.backend.pending_restore(&worker_ref).unwrap().unwrap(); + assert!(matches!(pending.discard_result.lock().unwrap().as_ref(), Some(Err(_)))); + fixture.backend.factory.restore_faults.cleanup_fail.store(false, Ordering::SeqCst); + drop(pending); + drop(pause); + runtime.test_wait_restore_continuation(&worker_ref, operation.operation_id, WAIT); + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + let receipt = serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap(); + assert_eq!(receipt["owners"][&request.request_id]["receipt"]["state"], "rolled_back"); + assert_eq!(fixture.canonical_files(), before); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 1); + fixture.assert_released(); + fixture.assert_quiet(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn autonomous_unchanged_publication_obstruction_parks_without_false_terminal_outcome() { + let fixture = Fixture::no_model(); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + fixture.backend.factory.restore_faults.socket_failure.store(true, Ordering::SeqCst); + let request = runtime.test_restore_request(&worker_ref); + let first = runtime.restore_worker_operation_scoped(fixture.request.workspace_scope.as_ref().unwrap(), &worker_ref, request.clone()).unwrap(); + assert_eq!(first.state, server_api::WorkerRestoreState::ReconciliationRequired); + let pending = fixture.backend.pending_restore(&worker_ref).unwrap().unwrap(); + let reason = runtime.test_wait_restore_continuation_parked(&worker_ref, pending.request.operation_id, WAIT); + assert!(reason.contains("publication failed"), "{reason}"); + let durable = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed); + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + let guard = serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap(); + assert!(guard["owners"][&request.request_id]["receipt"].is_null()); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 1); + // Initial call plus at most four backed-off continuation attempts. The + // parked condition is observed directly, not guessed using a sleep. + assert!(fixture.backend.factory.restore_faults.publication_attempts.load(Ordering::SeqCst) <= 5); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!(fixture.backend.factory.observation_hub.get(&worker_ref).is_none()); + drop(pending); + // Permanent path-length obstruction: explicit fixture teardown only, + // not the successful completion path asserted by autonomous tests. + runtime.stop_worker_scoped(fixture.request.workspace_scope.as_ref().unwrap(), &worker_ref, None).unwrap(); + fixture.assert_released(); + fixture.assert_quiet(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn autonomous_late_failure_settles_without_another_request() { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend).unwrap().spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let request = runtime.test_restore_request(&worker_ref); + let release = Arc::new(tokio::sync::Notify::new()); + let (entered, prepared) = mpsc::channel(); + *fixture.backend.factory.restore_faults.factory_gate.lock().unwrap() = Some(release.clone()); + *fixture.backend.factory.restore_faults.factory_entered.lock().unwrap() = Some(entered); + *fixture.backend.factory.restore_faults.fail_at.lock().unwrap() = Some(RuntimeRestoreStage::Feature); + let before = fixture.canonical_files(); + let first = runtime.restore_worker_operation_scoped(fixture.request.workspace_scope.as_ref().unwrap(), &worker_ref, request.clone()).unwrap(); + prepared.recv_timeout(WAIT).unwrap(); + assert_eq!(first.state, server_api::WorkerRestoreState::ReconciliationRequired); + let operation = durable_operation(&runtime, &worker_ref).unwrap(); + release.notify_one(); + runtime.test_wait_restore_continuation(&worker_ref, operation.operation_id, WAIT); + let loaded = runtime.test_runtime_store().load_runtime_state().unwrap(); + let receipt = serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap(); + assert_eq!(receipt["owners"][&request.request_id]["receipt"]["state"], "rolled_back"); + assert_eq!(fixture.canonical_files(), before); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 1); + fixture.assert_released(); + fixture.assert_quiet(); + } + #[test] #[serial_test::serial(worker_allocation)] fn runtime_accepted_follows_real_prepared_snapshot_commit_attach_and_input() { @@ -288,6 +423,7 @@ mod publication { ] { let fixture = Fixture::no_model(); let runtime = Arc::new(stopped_runtime(&fixture)); + let _pause = runtime.test_pause_restore_continuation(); let worker_ref = fixture.request.worker_ref.clone(); let scope = fixture.request.workspace_scope.clone().unwrap(); let request = runtime.test_restore_request(&worker_ref); @@ -396,6 +532,7 @@ mod publication { fn uncommitted_cleanup_failure_fences_discarding_candidate_from_activation() { let fixture = Fixture::no_model(); let runtime = Arc::new(stopped_runtime(&fixture)); + let _pause = runtime.test_pause_restore_continuation(); let worker_ref = fixture.request.worker_ref.clone(); let scope = fixture.request.workspace_scope.clone().unwrap(); let request = runtime.test_restore_request(&worker_ref); @@ -443,6 +580,7 @@ mod publication { fn publication_failure_retains_same_committed_candidate_without_reconstruction() { let fixture = Fixture::no_model(); let runtime = stopped_runtime(&fixture); + let _pause = runtime.test_pause_restore_continuation(); let worker_ref = fixture.request.worker_ref.clone(); let scope = fixture.request.workspace_scope.clone().unwrap(); let request = runtime.test_restore_request(&worker_ref); From 9e3f87ac5237e48e61396c16a0ec9ae7896f6e3c Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 11:11:59 +0900 Subject: [PATCH 27/46] fix: runtime retain restore facts when Stop supersedes publication --- crates/runtime-api/src/lib.rs | 25 +++- crates/server-api/src/lib.rs | 12 +- crates/tui/src/backend_worker_picker.rs | 69 +++++---- .../src/http_server/runtime_management_api.rs | 6 + crates/worker-runtime/src/runtime.rs | 137 ++++++++++++++++-- .../src/runtime/restore_continuation.rs | 16 ++ ...orker_backend_restore_publication_tests.rs | 37 +++++ crates/workspace-server/src/hosts.rs | 3 + openapi/server-api.json | 27 ++-- .../src/lib/generated/companion-api.ts | 2 +- .../src/lib/generated/repository-api.ts | 2 +- .../src/lib/generated/runtime-api.ts | 4 +- web/workspace/src/lib/generated/ticket-api.ts | 2 +- .../src/lib/generated/worker-launch-api.ts | 2 +- .../src/lib/workspace/api/runtime-workers.ts | 9 +- .../workers/workers-page.browser.spec.ts | 34 ++++- .../test/sidebar/worker-actions.test.ts | 1 + web/workspace/tests/runtime-workers.test.ts | 3 +- 18 files changed, 314 insertions(+), 77 deletions(-) diff --git a/crates/runtime-api/src/lib.rs b/crates/runtime-api/src/lib.rs index 729243ba6..e2804d18c 100644 --- a/crates/runtime-api/src/lib.rs +++ b/crates/runtime-api/src/lib.rs @@ -86,6 +86,8 @@ pub enum WorkerRestoreState { Accepted, Rejected, RolledBack, + /// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted. + Superseded, ReconciliationRequired, } @@ -1540,13 +1542,24 @@ mod tests { ) .unwrap(); assert!(request.preparation.is_none()); - for state in [ - "accepted", - "rejected", - "rolled_back", - "reconciliation_required", + for (state, wire) in [ + (WorkerRestoreState::Accepted, "accepted"), + (WorkerRestoreState::Rejected, "rejected"), + (WorkerRestoreState::RolledBack, "rolled_back"), + (WorkerRestoreState::Superseded, "superseded"), + ( + WorkerRestoreState::ReconciliationRequired, + "reconciliation_required", + ), ] { - assert!(serde_json::from_value::(serde_json::json!(state)).is_ok()); + assert_eq!( + serde_json::from_value::(serde_json::json!(wire)).unwrap(), + state + ); + assert_eq!( + serde_json::to_value(state).unwrap(), + serde_json::json!(wire) + ); } assert!( serde_json::from_value::(serde_json::json!( diff --git a/crates/server-api/src/lib.rs b/crates/server-api/src/lib.rs index 1de9be8e8..91502a293 100644 --- a/crates/server-api/src/lib.rs +++ b/crates/server-api/src/lib.rs @@ -544,8 +544,9 @@ pub struct WorkspaceWorkerSessionHistoryResponse { /// comparisons do not replace admission checks under the Runtime operation lock; conflicts do /// not trigger automatic token replacement. /// [`WorkerRestoreState`] Accepted means Restore committed, not that the subsequent run succeeded. -/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects, and -/// ReconciliationRequired needing settled-state checks. Response loss does not imply rollback. +/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects, +/// Superseded by a later Stop, and ReconciliationRequired needing settled-state checks. +/// Response loss does not imply rollback. #[api(reqwest, axum, openapi)] pub trait ServerApi { #[get("/health", status = 200, error_status = 400)] @@ -10566,8 +10567,8 @@ pub enum WorkerOperationState { /// Cross-layer outcome of an explicit Worker restore operation. /// /// `Rejected` is reserved for read-only preflight failures. Once live restore -/// work starts, failure is either a confirmed `RolledBack` operation or a -/// `ReconciliationRequired` result whose commit state must be reread/retried. +/// work starts, distinguish confirmed `RolledBack`, unresolved +/// `ReconciliationRequired`, and a later Stop's `Superseded` outcome. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(rename_all = "snake_case")] @@ -10575,6 +10576,8 @@ pub enum WorkerRestoreState { Accepted, Rejected, RolledBack, + /// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted. + Superseded, ReconciliationRequired, } @@ -12668,6 +12671,7 @@ mod tests { (WorkerRestoreState::Accepted, "accepted"), (WorkerRestoreState::Rejected, "rejected"), (WorkerRestoreState::RolledBack, "rolled_back"), + (WorkerRestoreState::Superseded, "superseded"), ( WorkerRestoreState::ReconciliationRequired, "reconciliation_required", diff --git a/crates/tui/src/backend_worker_picker.rs b/crates/tui/src/backend_worker_picker.rs index 13c77901f..7812f85e3 100644 --- a/crates/tui/src/backend_worker_picker.rs +++ b/crates/tui/src/backend_worker_picker.rs @@ -1119,39 +1119,45 @@ mod tests { } #[tokio::test] - async fn rejected_resume_does_not_observe_or_open_retained_state() { - let lifecycle = FakeBackendWorkerLifecycle { - restore_response: restore_response(BackendWorkerRestoreState::Rejected), - observation: session_observation(serde_json::json!({ - "availability": "unavailable", - "reason": "retention_missing", - "message": "must not be opened" - })), - calls: std::sync::Mutex::new(Vec::new()), - }; - let target = client::BackendRuntimeTarget::new( - "http://127.0.0.1:3000", - "workspace-a", - "runtime-a", - "worker-a", - ); + async fn settled_nonaccepted_resume_does_not_retry_observe_or_open_retained_state() { + for state in [ + BackendWorkerRestoreState::Rejected, + BackendWorkerRestoreState::RolledBack, + BackendWorkerRestoreState::Superseded, + ] { + let lifecycle = FakeBackendWorkerLifecycle { + restore_response: restore_response(state), + observation: session_observation(serde_json::json!({ + "availability": "unavailable", + "reason": "retention_missing", + "message": "must not be opened" + })), + calls: std::sync::Mutex::new(Vec::new()), + }; + let target = client::BackendRuntimeTarget::new( + "http://127.0.0.1:3000", + "workspace-a", + "runtime-a", + "worker-a", + ); - let error = prepare_selected_worker( - &lifecycle, - target, - BackendWorkerPickerIntent::Resume, - Some(client::BackendWorkerRestoreRequest { - expected_observation_token: "observed-worker-state".into(), - request_id: "resume-operation".into(), - }), - |_| Ok(false), - ) - .await - .unwrap_err() - .to_string(); + let error = prepare_selected_worker( + &lifecycle, + target, + BackendWorkerPickerIntent::Resume, + Some(client::BackendWorkerRestoreRequest { + expected_observation_token: "observed-worker-state".into(), + request_id: "resume-operation".into(), + }), + |_| panic!("settled nonaccepted restore must not offer recovery retry"), + ) + .await + .unwrap_err() + .to_string(); - assert!(error.contains("Rejected")); - assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]); + assert!(error.contains(&format!("{state:?}"))); + assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]); + } } #[test] @@ -1176,6 +1182,7 @@ mod tests { for state in [ BackendWorkerRestoreState::Rejected, BackendWorkerRestoreState::RolledBack, + BackendWorkerRestoreState::Superseded, BackendWorkerRestoreState::ReconciliationRequired, ] { let mut response = restore_response(state); diff --git a/crates/worker-runtime/src/http_server/runtime_management_api.rs b/crates/worker-runtime/src/http_server/runtime_management_api.rs index 8bcdb123c..7db666b82 100644 --- a/crates/worker-runtime/src/http_server/runtime_management_api.rs +++ b/crates/worker-runtime/src/http_server/runtime_management_api.rs @@ -694,6 +694,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { server_api::WorkerRestoreState::RolledBack => { runtime_api::WorkerRestoreState::RolledBack } + server_api::WorkerRestoreState::Superseded => { + runtime_api::WorkerRestoreState::Superseded + } server_api::WorkerRestoreState::ReconciliationRequired => { runtime_api::WorkerRestoreState::ReconciliationRequired } @@ -744,6 +747,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { server_api::WorkerRestoreState::RolledBack => { runtime_api::WorkerRestoreState::RolledBack } + server_api::WorkerRestoreState::Superseded => { + runtime_api::WorkerRestoreState::Superseded + } server_api::WorkerRestoreState::ReconciliationRequired => { runtime_api::WorkerRestoreState::ReconciliationRequired } diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 6c396e6b4..ee9d0468e 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -225,6 +225,28 @@ impl RuntimeWorkerRestoreResult { } } + fn superseded_by_stop(pending: Option) -> Self { + let (code, message) = match pending { + Some(pending) if pending.commit == RestoreCommitFact::Committed => ( + "worker_restore_committed_then_stopped", + "Restore committed, then its execution was stopped; prior durable effects are not rolled back", + ), + Some(pending) if pending.commit == RestoreCommitFact::LegacyUnknown => ( + "worker_restore_unknown_decision_then_stopped", + "Restore execution was stopped; its earlier commit decision remains unknown", + ), + Some(pending) if pending.source_repair_possible => ( + "worker_restore_source_repair_then_stopped", + "Uncommitted Restore was stopped; prior source repair effects may remain", + ), + _ => ( + "worker_restore_superseded_by_stop", + "Worker Restore was superseded by confirmed Stop", + ), + }; + Self::failed(WorkerRestoreState::Superseded, code, message) + } + fn into_legacy_result(self) -> Result { match (self.state, self.worker) { (WorkerRestoreState::Accepted, Some(worker)) => Ok(worker), @@ -3684,11 +3706,7 @@ impl Runtime { worker.restore_guard.rotate(); worker .restore_guard - .finish(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::RolledBack, - "worker_restore_superseded_by_stop", - "Worker Restore was superseded by Stop", - )); + .finish(RuntimeWorkerRestoreResult::superseded_by_stop(None)); state.persist_worker_record(&worker)?; let ack = WorkerLifecycleAck { worker_ref: worker_ref.clone(), @@ -4250,6 +4268,7 @@ impl Runtime { WorkerRestoreState::Accepted => return Ok(()), WorkerRestoreState::Rejected | WorkerRestoreState::RolledBack + | WorkerRestoreState::Superseded | WorkerRestoreState::ReconciliationRequired => {} } let mut state = self.lock()?; @@ -4363,6 +4382,8 @@ impl Runtime { worker_ref.worker_id ))); } + let restore_receipt = + RuntimeWorkerRestoreResult::superseded_by_stop(candidate.pending_restore); candidate.pending_stop = None; candidate.pending_restore = None; if let Some(context) = candidate.restore_candidate_context.take() { @@ -4373,13 +4394,7 @@ impl Runtime { candidate.worker_state = None; candidate.restore_intent = WorkerRestoreIntent::Explicit; - candidate - .restore_guard - .finish(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::RolledBack, - "worker_restore_superseded_by_stop", - "Worker Restore was superseded by Stop", - )); + candidate.restore_guard.finish(restore_receipt); candidate.internal_workers.clear(); state.persist_worker_record(&candidate)?; let ack = WorkerLifecycleAck { @@ -15138,6 +15153,104 @@ mod tests { } } + #[cfg(feature = "fs-store")] + #[test] + fn stop_retains_restore_decision_evidence_in_superseded_receipt_across_restart() { + // Component scope: real Runtime admission and durable store; execution + // is substituted. Controller shutdown races are covered separately. + for (commit, repaired, code) in [ + ( + RestoreCommitFact::NotCommitted, + false, + "worker_restore_superseded_by_stop", + ), + ( + RestoreCommitFact::NotCommitted, + true, + "worker_restore_source_repair_then_stopped", + ), + ( + RestoreCommitFact::Committed, + false, + "worker_restore_committed_then_stopped", + ), + ( + RestoreCommitFact::LegacyUnknown, + true, + "worker_restore_unknown_decision_then_stopped", + ), + ] { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("stop-restore-evidence"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("stop restore evidence")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + *backend.restore_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::Errored( + WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "publication not confirmed", + ), + )); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + { + // Fixture represents each canonical checkpoint. Stop must keep + // its classification, rather than infer rollback from cleanup. + let mut state = runtime.lock().unwrap(); + let record = state.worker_mut(&worker.worker_ref).unwrap(); + let pending = record.pending_restore.as_mut().unwrap(); + pending.commit = commit; + pending.source_repair_possible = repaired; + state.persist_worker(&worker.worker_id).unwrap(); + } + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let receipt = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Superseded); + assert_eq!(receipt.reason_code.as_deref(), Some(code)); + assert!(receipt.worker.is_none()); + drop(runtime); + let restarted_backend = Arc::new(TestExecutionBackend::default()); + let restarted = + Runtime::with_fs_store_and_execution_backend(options, restarted_backend.clone()) + .unwrap(); + assert_eq!( + restarted + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(), + receipt + ); + assert_eq!(*restarted_backend.restore_count.lock().unwrap(), 0); + assert_eq!( + restarted.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + let mut input = WorkerInput::user("must not revive"); + input.kind = WorkerInputKind::UserIfIdle; + assert!(restarted.send_input(&worker.worker_ref, input).is_err()); + let mut stale = request; + stale.request_id = "new-intent-with-old-token".into(); + assert!(matches!( + restarted.restore_worker_operation(&worker.worker_ref, stale), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + } + } + #[cfg(feature = "fs-store")] #[test] fn restore_decision_reread_preserves_commit_across_uncertainty_stop_and_restart() { diff --git a/crates/worker-runtime/src/runtime/restore_continuation.rs b/crates/worker-runtime/src/runtime/restore_continuation.rs index 78cda5fdc..3de663219 100644 --- a/crates/worker-runtime/src/runtime/restore_continuation.rs +++ b/crates/worker-runtime/src/runtime/restore_continuation.rs @@ -330,6 +330,22 @@ impl Runtime { Err(std::sync::TryLockError::WouldBlock) => return Ok(Step::Contended), Err(_) => return Err(RuntimeError::StatePoisoned), }; + let stop_is_owned = { + let mut state = self.lock()?; + state.ensure_running()?; + state.refresh_uncertain_worker_write(worker_ref)?; + let worker = state.worker(worker_ref)?; + worker.pending_stop.is_some() + && worker + .pending_restore + .is_some_and(|pending| pending.operation_id == operation) + }; + if stop_is_owned { + // Stop supersedes publication, not ownership of late resources. + // Retain this same continuation until its cleanup is confirmed. + self.reconcile_pending_stop_under_lock(worker_ref)?; + return Ok(Step::Finished); + } let (backend, pending) = { let mut state = self.lock()?; state.ensure_running()?; diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs index 3cc482aab..28323570f 100644 --- a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -94,6 +94,43 @@ mod publication { (release, received) } + #[test] + #[serial_test::serial(worker_allocation)] + fn stop_timeout_retains_autonomous_late_cleanup_and_blocks_remove_until_confirmed() { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend).unwrap().spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let pause = runtime.test_pause_restore_continuation(); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let before = fixture.canonical_files(); + let (release, prepared) = prepared_gate(&fixture); + let result = runtime.restore_worker_operation_scoped(scope, &worker_ref, request.clone()).unwrap(); + prepared.recv_timeout(WAIT).unwrap(); + assert_eq!(result.state, server_api::WorkerRestoreState::ReconciliationRequired); + let operation = durable_operation(&runtime, &worker_ref).unwrap(); + assert!(runtime.stop_worker_scoped(scope, &worker_ref, None).is_err()); + assert!(runtime.delete_worker_scoped(scope, &worker_ref).is_err()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + release.notify_one(); + drop(pause); + runtime.test_wait_restore_continuation(&worker_ref, operation.operation_id, WAIT); + assert_eq!(runtime.worker_detail(&worker_ref).unwrap().status, crate::catalog::WorkerStatus::Stopped); + assert!(durable_operation(&runtime, &worker_ref).is_none()); + assert!(fixture.backend.pending_restore(&worker_ref).unwrap().is_none()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!(fixture.backend.factory.observation_hub.get(&worker_ref).is_none()); + assert!(fixture.backend.clone().attach_worker_protocol(&worker_ref).is_err()); + let receipt = runtime.restore_worker_operation_scoped(scope, &worker_ref, request).unwrap(); + assert_eq!(receipt.state, server_api::WorkerRestoreState::Superseded); + assert_eq!(receipt.reason_code.as_deref(), Some("worker_restore_superseded_by_stop")); + assert_eq!(fixture.canonical_files(), before); + fixture.assert_released(); + fixture.assert_quiet(); + assert!(runtime.delete_worker_scoped(scope, &worker_ref).unwrap().deleted); + } + #[test] #[serial_test::serial(worker_allocation)] fn autonomous_late_prepare_and_publication_settle_without_another_request() { diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 4027b221a..f687c4bf8 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -5063,6 +5063,9 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { runtime_api::WorkerRestoreState::RolledBack => { server_api::WorkerRestoreState::RolledBack } + runtime_api::WorkerRestoreState::Superseded => { + server_api::WorkerRestoreState::Superseded + } runtime_api::WorkerRestoreState::ReconciliationRequired => { server_api::WorkerRestoreState::ReconciliationRequired } diff --git a/openapi/server-api.json b/openapi/server-api.json index 720c13a72..7966ef8e5 100644 --- a/openapi/server-api.json +++ b/openapi/server-api.json @@ -15697,14 +15697,23 @@ "type": "object" }, "WorkerRestoreState": { - "description": "Cross-layer outcome of an explicit Worker restore operation.\n\n`Rejected` is reserved for read-only preflight failures. Once live restore\nwork starts, failure is either a confirmed `RolledBack` operation or a\n`ReconciliationRequired` result whose commit state must be reread/retried.", - "enum": [ - "accepted", - "rejected", - "rolled_back", - "reconciliation_required" - ], - "type": "string" + "description": "Cross-layer outcome of an explicit Worker restore operation.\n\n`Rejected` is reserved for read-only preflight failures. Once live restore\nwork starts, distinguish confirmed `RolledBack`, unresolved\n`ReconciliationRequired`, and a later Stop's `Superseded` outcome.", + "oneOf": [ + { + "enum": [ + "accepted", + "rejected", + "rolled_back", + "reconciliation_required" + ], + "type": "string" + }, + { + "const": "superseded", + "description": "A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.", + "type": "string" + } + ] }, "WorkerRetentionResponse": { "additionalProperties": false, @@ -18155,7 +18164,7 @@ "info": { "title": "Yoi Server API", "version": "0.1.0", - "x-yoi-source-digest": "sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7" + "x-yoi-source-digest": "sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e" }, "openapi": "3.1.0", "paths": { diff --git a/web/workspace/src/lib/generated/companion-api.ts b/web/workspace/src/lib/generated/companion-api.ts index ecd2b3f5d..ec620e61a 100644 --- a/web/workspace/src/lib/generated/companion-api.ts +++ b/web/workspace/src/lib/generated/companion-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-companion-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7) +// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e) // Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/companion-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check diff --git a/web/workspace/src/lib/generated/repository-api.ts b/web/workspace/src/lib/generated/repository-api.ts index bc61338f5..455ae30e0 100644 --- a/web/workspace/src/lib/generated/repository-api.ts +++ b/web/workspace/src/lib/generated/repository-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-repository-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7) +// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e) // Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output // Output: web/workspace/src/lib/generated/repository-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check diff --git a/web/workspace/src/lib/generated/runtime-api.ts b/web/workspace/src/lib/generated/runtime-api.ts index 72647613a..4ca1126ec 100644 --- a/web/workspace/src/lib/generated/runtime-api.ts +++ b/web/workspace/src/lib/generated/runtime-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-runtime-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7) +// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e) // Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/runtime-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check @@ -181,7 +181,7 @@ export type WorkerRestoreResponse = { result: WorkerRestoreResult; runtime_id: s export type WorkerRestoreResult = { diagnostics?: Array; state: WorkerRestoreState; worker?: WorkerSummary | null; }; -export type WorkerRestoreState = "accepted" | "rejected" | "rolled_back" | "reconciliation_required"; +export type WorkerRestoreState = "accepted" | "rejected" | "rolled_back" | "reconciliation_required" | "superseded"; export type WorkerRetentionResponse = { pinned: boolean; retention_state: string; runtime_id: string; worker_id: string; workspace_id: string; }; diff --git a/web/workspace/src/lib/generated/ticket-api.ts b/web/workspace/src/lib/generated/ticket-api.ts index 18b0a6804..fa858f1d9 100644 --- a/web/workspace/src/lib/generated/ticket-api.ts +++ b/web/workspace/src/lib/generated/ticket-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7) +// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e) // Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/ticket-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check diff --git a/web/workspace/src/lib/generated/worker-launch-api.ts b/web/workspace/src/lib/generated/worker-launch-api.ts index 8dd4a964b..d25719a4b 100644 --- a/web/workspace/src/lib/generated/worker-launch-api.ts +++ b/web/workspace/src/lib/generated/worker-launch-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-worker-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7) +// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e) // Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/worker-launch-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check diff --git a/web/workspace/src/lib/workspace/api/runtime-workers.ts b/web/workspace/src/lib/workspace/api/runtime-workers.ts index e00b83485..fcd88470c 100644 --- a/web/workspace/src/lib/workspace/api/runtime-workers.ts +++ b/web/workspace/src/lib/workspace/api/runtime-workers.ts @@ -363,8 +363,13 @@ export function parseWorkerRestoreResponse( exactKeys(result, ["state"], ["diagnostics", "worker"], `${path}.result`); const state = boundedString(result.state, `${path}.result.state`); if ( - !["accepted", "rejected", "rolled_back", "reconciliation_required"] - .includes(state) + ![ + "accepted", + "rejected", + "rolled_back", + "superseded", + "reconciliation_required", + ].includes(state) ) { fail(`${path}.result.state`, "has an unknown value"); } diff --git a/web/workspace/src/routes/w/[workspaceId]/workers/workers-page.browser.spec.ts b/web/workspace/src/routes/w/[workspaceId]/workers/workers-page.browser.spec.ts index a72afa7cb..255879fe9 100644 --- a/web/workspace/src/routes/w/[workspaceId]/workers/workers-page.browser.spec.ts +++ b/web/workspace/src/routes/w/[workspaceId]/workers/workers-page.browser.spec.ts @@ -427,7 +427,12 @@ test.each(["switch", "dispose"] as const)( ); function restoreResponse( - state: "accepted" | "rejected" | "rolled_back" | "reconciliation_required", + state: + | "accepted" + | "rejected" + | "rolled_back" + | "superseded" + | "reconciliation_required", extra = {}, ) { return Response.json({ @@ -461,7 +466,13 @@ test.each(["stopped", "idle", "running", "paused", "unavailable"] as const)( ); test.each( - ["accepted", "rejected", "rolled_back", "reconciliation_required"] as const, + [ + "accepted", + "rejected", + "rolled_back", + "superseded", + "reconciliation_required", + ] as const, )( "Restore classifies %s through the shared helper, without treating HTTP 200 as success", async (state) => { @@ -470,10 +481,11 @@ test.each( const unsubscribe = workspaceAlerts.subscribe((value) => { alerts = value; }); - const request = vi.fn(async (input: RequestInfo | URL) => - String(input).endsWith("/restore") - ? restoreResponse(state) - : Response.json(plan()) + const request = vi.fn( + async (input: RequestInfo | URL, _init?: RequestInit) => + String(input).endsWith("/restore") + ? restoreResponse(state) + : Response.json(plan()), ); vi.stubGlobal("fetch", request); render(WorkersPage, { data: data() }); @@ -498,6 +510,16 @@ test.each( screen.queryByRole("button", { name: "Retry Restore Catalog worker" }), ), ).toBe(state === "reconciliation_required"); + expect( + request.mock.calls.filter(([url]) => String(url).endsWith("/restore")), + ).toHaveLength(1); + expect( + request.mock.calls + .filter(([, init]) => init?.method && init.method !== "GET") + .map(([url]) => String(url)), + ).toEqual([ + "/api/w/page-test/runtimes/runtime-a/workers/worker-a/restore", + ]); // The response is never projected into the catalog by the action. expect(screen.getByRole("cell", { name: "stopped" })).toBeTruthy(); unsubscribe(); diff --git a/web/workspace/test/sidebar/worker-actions.test.ts b/web/workspace/test/sidebar/worker-actions.test.ts index 730618f12..a4d8d8792 100644 --- a/web/workspace/test/sidebar/worker-actions.test.ts +++ b/web/workspace/test/sidebar/worker-actions.test.ts @@ -305,6 +305,7 @@ Deno.test("Restore classification never promotes nonaccepted results and preserv "accepted", "rejected", "rolled_back", + "superseded", "reconciliation_required", ] as const ) { diff --git a/web/workspace/tests/runtime-workers.test.ts b/web/workspace/tests/runtime-workers.test.ts index 2de3059f3..a868cf508 100644 --- a/web/workspace/tests/runtime-workers.test.ts +++ b/web/workspace/tests/runtime-workers.test.ts @@ -173,12 +173,13 @@ function restoredWorker() { restore_observation_token: "generation-b", }; } -Deno.test("Restore parser preserves all four states and optional Workspace summary", () => { +Deno.test("Restore parser preserves typed outcomes and optional Workspace summary", () => { for ( const state of [ "accepted", "rejected", "rolled_back", + "superseded", "reconciliation_required", ] ) { From 6be4e8388bb8448a6477c03fa72b72f7f5fe7a6e Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 11:49:15 +0900 Subject: [PATCH 28/46] feat: recover saved restore operations by their durable decision --- crates/worker-runtime/src/execution.rs | 44 +- crates/worker-runtime/src/runtime.rs | 274 ++++++++- .../src/runtime/restore_continuation.rs | 11 +- crates/worker-runtime/src/worker_backend.rs | 533 +++++++++++++++++- .../worker_backend_restore_failure_tests.rs | 12 + ...orker_backend_restore_publication_tests.rs | 127 +++++ 6 files changed, 945 insertions(+), 56 deletions(-) diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index 39faadab5..ffa6e807f 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -621,9 +621,13 @@ impl fmt::Debug for WorkerProtocolTransport { /// authority: Runtime must reread its decision before choosing any action. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum RestoreRecoveryDisposition { + /// No live or pending owner for this exact operation. Not cleanup evidence. + Absent, InFlight, Continue, Discard, + /// A retained cleanup-only owner; never reconstruct or activate it. + Reclaim, } /// The connection selects this backend; WorkerRef identifies an execution, @@ -704,8 +708,15 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { None } - /// Opt in to Runtime-owned continuation. Waiting is signaled through the - /// original execution context; no callback may wait for Runtime locks. + /// Opt in to Runtime-owned autonomous restore continuation. Pending tasks + /// signal completion through the original execution context; callbacks must + /// not wait for Runtime locks. Independent of resource disposition support. + fn supports_restore_continuation(&self) -> bool { + false + } + + /// Describe exact operation resource ownership for warm/cold recovery + /// dispatch. This does not opt in to callbacks or autonomous continuation. fn restore_recovery_disposition( &self, _operation_id: WorkerLifecycleOperationId, @@ -721,6 +732,24 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { self.restore_worker(request) } + /// Reclaim an uncommitted operation after restart without preparing, + /// repairing, or activating a replacement candidate. Runtime must establish + /// the durable uncommitted decision before calling this seam. Unsupported + /// cleanup is uncertainty, never a successful rollback. + fn reclaim_uncommitted_restore( + &self, + _request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::unsupported( + WorkerExecutionOperation::Restore, + "backend cannot reclaim an uncommitted restore after restart", + ), + worker_state: None, + workdir_attachments: Vec::new(), + } + } + /// Discard only the exact unpublished operation after Runtime has proved /// its durable decision uncommitted. This is not Stop and must not affect a /// connected execution. RolledBack requires confirmed cleanup and no repair. @@ -1009,6 +1038,10 @@ impl WorkerExecutionBackendRef { .completed_restore_failure(operation_id, worker_ref) } + pub(crate) fn supports_restore_continuation(&self) -> bool { + self.backend.supports_restore_continuation() + } + pub(crate) fn restore_recovery_disposition( &self, operation_id: WorkerLifecycleOperationId, @@ -1025,6 +1058,13 @@ impl WorkerExecutionBackendRef { self.backend.reconcile_restore(request) } + pub(crate) fn reclaim_uncommitted_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + self.backend.reclaim_uncommitted_restore(request) + } + pub(crate) fn discard_uncommitted_restore( &self, operation_id: WorkerLifecycleOperationId, diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index ee9d0468e..f55d359f8 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -171,6 +171,12 @@ enum WorkerRestoreMode { Automatic, } +#[derive(Clone, Copy, Eq, PartialEq)] +enum RestoreRecoveryAction { + Continue, + ReclaimUncommitted, +} + #[cfg(feature = "fs-store")] impl From for PersistedWorkerRestoreMode { fn from(mode: WorkerRestoreMode) -> Self { @@ -2332,6 +2338,56 @@ impl Runtime { } } + // Seed durable crash checkpoints without launching a second Controller. + // Tests then reopen the real store with empty backend resource maps. + #[cfg(all(test, feature = "fs-store"))] + pub(crate) fn test_stage_restore_checkpoint( + &self, + worker_ref: &WorkerRef, + commit: RestoreCommitFact, + source_repair_possible: bool, + ) -> WorkerLifecycleOperationId { + let lease = self.worker_operation_lock(worker_ref.worker_id).unwrap(); + let _guard = lease.lock().unwrap(); + let operation = self + .lock() + .unwrap() + .worker(worker_ref) + .unwrap() + .pending_restore + .unwrap(); + self.begin_restore_operation( + worker_ref, + operation.operation_id, + operation.mode, + self.restore_candidate_execution_context(worker_ref.clone()), + ) + .unwrap(); + if source_repair_possible { + self.restore_repair_authorizer(worker_ref.clone())(operation.operation_id).unwrap(); + } + match commit { + RestoreCommitFact::Committed => { + self.commit_restored_worker_execution( + worker_ref, + protocol::WorkerStatus::Idle.into(), + WorkerStatus::Idle, + Vec::new(), + ) + .unwrap(); + } + RestoreCommitFact::LegacyUnknown => { + let mut state = self.lock().unwrap(); + let mut worker = state.worker(worker_ref).unwrap().clone(); + worker.pending_restore.as_mut().unwrap().commit = commit; + state.persist_restore_worker_record(&worker).unwrap(); + state.workers.insert(worker_ref.worker_id, worker); + } + RestoreCommitFact::NotCommitted => {} + } + operation.operation_id + } + #[cfg(test)] pub(crate) fn test_restore_request( &self, @@ -2357,6 +2413,19 @@ impl Runtime { &self, worker_ref: &WorkerRef, mode: WorkerRestoreMode, + ) -> Result { + self.restore_admitted_with_recovery_under_lock( + worker_ref, + mode, + RestoreRecoveryAction::Continue, + ) + } + + fn restore_admitted_with_recovery_under_lock( + &self, + worker_ref: &WorkerRef, + mode: WorkerRestoreMode, + recovery: RestoreRecoveryAction, ) -> Result { self.lock()? .ensure_subject_not_reserved(worker_ref, false)?; @@ -2407,14 +2476,28 @@ impl Runtime { } }; if let Some(detail) = committed_detail { - // Recover a receipt gap without launching execution again. + // A historical binding is not current readiness. The exact + // execution must still attest its operation and endpoint. + let backend = self.lock()?.execution_backend.clone(); + let readiness = backend + .ok_or_else(|| "Runtime has no Worker execution backend".to_owned()) + .and_then(|backend| { + backend.activate_restored_worker(owner.operation_id, worker_ref) + }); + if let Err(message) = readiness { + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_saved_binding_not_ready", + sanitize_worker_create_failure_message(&message), + )); + } let result = RuntimeWorkerRestoreResult::accepted(detail); self.finish_restore_receipt(worker_ref, &result)?; return Ok(result); } } } - let result = self.restore_worker_under_lock(worker_ref, mode)?; + let result = self.restore_worker_with_recovery_under_lock(worker_ref, mode, recovery)?; self.finish_restore_receipt(worker_ref, &result)?; Ok(result) } @@ -2451,6 +2534,19 @@ impl Runtime { &self, worker_ref: &WorkerRef, mode: WorkerRestoreMode, + ) -> Result { + self.restore_worker_with_recovery_under_lock( + worker_ref, + mode, + RestoreRecoveryAction::Continue, + ) + } + + fn restore_worker_with_recovery_under_lock( + &self, + worker_ref: &WorkerRef, + mode: WorkerRestoreMode, + recovery: RestoreRecoveryAction, ) -> Result { self.lock()? .ensure_subject_not_reserved(worker_ref, false)?; @@ -2562,8 +2658,12 @@ impl Runtime { .cloned() }); let request = WorkerExecutionRestoreRequest { - source_repair_possible: pending_restore - .is_some_and(|pending| pending.source_repair_possible), + // A committed decision also permits canonical activation effects. + // Cold reconstruction must retain that possibility through even a + // read-only failure, not cache it as an uncommitted rollback. + source_repair_possible: pending_restore.is_some_and(|pending| { + pending.source_repair_possible || pending.commit == RestoreCommitFact::Committed + }), operation_id, worker_ref: worker_ref.clone(), request: worker_request, @@ -2585,6 +2685,27 @@ impl Runtime { ) }; + let uncommitted = self + .lock()? + .worker(worker_ref)? + .pending_restore + .is_some_and(|pending| { + pending.operation_id == operation_id + && pending.commit == RestoreCommitFact::NotCommitted + }); + let reclaim = uncommitted + && (recovery == RestoreRecoveryAction::ReclaimUncommitted + || (reconcile + && match backend.restore_recovery_disposition(operation_id, worker_ref) { + Some(crate::execution::RestoreRecoveryDisposition::Reclaim) | None => true, + Some(crate::execution::RestoreRecoveryDisposition::Absent) => self + .lock()? + .worker(worker_ref)? + .restore_candidate_context + .is_none(), + Some(_) => false, + })); + let reconcile = reconcile || reclaim; if !reconcile && let Err(result) = backend.preflight_restore(&request) { // A preflight may attest an already-connected Controller snapshot. // Repeated restore is read-only; do not replace its operation owner. @@ -2643,6 +2764,8 @@ impl Runtime { let has_completed_failure = completed_failure.is_some(); let restore_result = if let Some(result) = completed_failure { result + } else if reclaim { + backend.reclaim_uncommitted_restore(request) } else if reconcile { backend.reconcile_restore(request) } else { @@ -4205,6 +4328,7 @@ impl Runtime { .lock() .map_err(|_| RuntimeError::StatePoisoned)?; if let Err(error) = self.reconcile_pending_stop_under_lock(&worker_ref) { + self.ensure_restore_continuation(&worker_ref)?; tracing::warn!( worker_id = %worker_ref.worker_id, error = %error, @@ -4242,8 +4366,34 @@ impl Runtime { let _operation_guard = operation_lock .lock() .map_err(|_| RuntimeError::StatePoisoned)?; - let result = self.restore_admitted_worker_under_lock(&worker_ref, mode)?; - self.record_startup_restore_result(&worker_ref, &result)?; + let recovery = { + let mut state = self.lock()?; + state.refresh_uncertain_worker_write(&worker_ref)?; + let pending = state.worker(&worker_ref)?.pending_restore; + let retained = pending.is_some_and(|pending| { + state.execution_backend.as_ref().is_some_and(|backend| { + matches!( + backend.restore_recovery_disposition(pending.operation_id, &worker_ref), + Some( + crate::execution::RestoreRecoveryDisposition::Continue + | crate::execution::RestoreRecoveryDisposition::InFlight + | crate::execution::RestoreRecoveryDisposition::Discard + ) + ) + }) + }); + if pending.is_some_and(|pending| pending.commit == RestoreCommitFact::NotCommitted) + && !retained + { + RestoreRecoveryAction::ReclaimUncommitted + } else { + RestoreRecoveryAction::Continue + } + }; + let result = + self.restore_admitted_with_recovery_under_lock(&worker_ref, mode, recovery); + self.ensure_restore_continuation(&worker_ref)?; + self.record_startup_restore_result(&worker_ref, &result?)?; } for worker_ref in candidates { @@ -4252,8 +4402,9 @@ impl Runtime { .lock() .map_err(|_| RuntimeError::StatePoisoned)?; let result = - self.restore_admitted_worker_under_lock(&worker_ref, WorkerRestoreMode::Automatic)?; - self.record_startup_restore_result(&worker_ref, &result)?; + self.restore_admitted_worker_under_lock(&worker_ref, WorkerRestoreMode::Automatic); + self.ensure_restore_continuation(&worker_ref)?; + self.record_startup_restore_result(&worker_ref, &result?)?; } Ok(()) } @@ -8411,6 +8562,7 @@ mod tests { restore_gate: Mutex>>, restore_count: Mutex, restore_operation_ids: Mutex>, + reclaim_operation_ids: Mutex>, activation_result: Mutex>>, activation_gate: Mutex>>, activation_operation_ids: Mutex>, @@ -8708,6 +8860,57 @@ mod tests { } } + fn restore_recovery_disposition( + &self, + operation_id: WorkerLifecycleOperationId, + _worker_ref: &WorkerRef, + ) -> Option { + Some( + if self + .restore_operation_ids + .lock() + .unwrap() + .contains(&operation_id) + { + crate::execution::RestoreRecoveryDisposition::Continue + } else { + crate::execution::RestoreRecoveryDisposition::Absent + }, + ) + } + + fn reclaim_uncommitted_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + self.reclaim_operation_ids + .lock() + .unwrap() + .push(request.operation_id); + assert!( + !self + .contexts + .lock() + .unwrap() + .contains_key(&request.worker_ref.worker_id) + ); + if request.source_repair_possible { + WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "prior repair may remain", + ), + worker_state: None, + workdir_attachments: Vec::new(), + } + } else { + WorkerExecutionSpawnResult::RolledBack(WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "cold request resources confirmed absent", + )) + } + } + fn reconcile_restore( &self, request: WorkerExecutionRestoreRequest, @@ -9242,7 +9445,9 @@ mod tests { Some(&owner), ) .unwrap(); - assert_eq!(result.unwrap().state, WorkerRestoreState::Accepted); + assert_eq!(result.unwrap().state, WorkerRestoreState::RolledBack); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + assert_eq!(backend.reclaim_operation_ids.lock().unwrap().len(), 1); assert!(preparation.is_none()); assert!( runtime @@ -11212,7 +11417,7 @@ mod tests { #[cfg(feature = "fs-store")] #[test] - fn restore_guard_interrupted_preparation_replays_original_pending_operation() { + fn restore_guard_interrupted_preparation_reclaims_original_pending_operation() { let dir = tempfile::tempdir().unwrap(); let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) .with_runtime_id("guard-partial-preparation"); @@ -11280,10 +11485,10 @@ mod tests { let receipt = resumed .restore_worker_operation(&worker.worker_ref, request) .unwrap(); - assert_eq!(receipt.state, WorkerRestoreState::Accepted); - assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!(receipt.state, WorkerRestoreState::RolledBack); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); assert_eq!( - backend.restore_operation_ids.lock().unwrap().as_slice(), + backend.reclaim_operation_ids.lock().unwrap().as_slice(), &[operation_id] ); assert!( @@ -11422,7 +11627,7 @@ mod tests { #[cfg(feature = "fs-store")] #[test] - fn restore_guard_unknown_preparation_write_retries_same_pending_operation() { + fn restore_guard_unknown_preparation_write_reclaims_same_pending_operation() { let dir = tempfile::tempdir().unwrap(); let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) .with_runtime_id("guard-unknown-preparation"); @@ -11484,12 +11689,16 @@ mod tests { .restore_worker_operation(&worker.worker_ref, request) .unwrap() .state, - WorkerRestoreState::Accepted + WorkerRestoreState::RolledBack + ); + assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 0); + assert_eq!( + resumed_backend.reclaim_operation_ids.lock().unwrap().len(), + 1 ); - assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 1); assert_eq!( resumed.worker_detail(&worker.worker_ref).unwrap().status, - WorkerStatus::Idle + WorkerStatus::Stopped ); } @@ -11667,7 +11876,7 @@ mod tests { Runtime::with_fs_store_and_execution_backend(options, resumed_backend.clone()).unwrap(); assert_eq!( resumed_backend - .restore_operation_ids + .reclaim_operation_ids .lock() .unwrap() .as_slice(), @@ -11676,8 +11885,8 @@ mod tests { let receipt = resumed .restore_worker_operation(&worker.worker_ref, request) .unwrap(); - assert_eq!(receipt.state, WorkerRestoreState::Accepted); - assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 1); + assert_eq!(receipt.state, WorkerRestoreState::RolledBack); + assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 0); } #[test] @@ -15521,6 +15730,21 @@ mod tests { restarted_backend.clone(), ) .unwrap(); + assert_eq!( + restarted_backend + .reclaim_operation_ids + .lock() + .unwrap() + .as_slice(), + &[operation_id] + ); + assert_eq!(*restarted_backend.restore_count.lock().unwrap(), 0); + assert_eq!( + restarted.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + // A deliberate fresh intent may restore after the old uncommitted + // request was reclaimed. It must allocate a different operation. let retry = restarted .restore_worker_operation( &worker.worker_ref, @@ -15529,13 +15753,9 @@ mod tests { .unwrap(); assert_eq!(retry.state, WorkerRestoreState::Accepted); assert_eq!(*restarted_backend.restore_count.lock().unwrap(), 1); - assert_eq!( - restarted_backend - .restore_operation_ids - .lock() - .unwrap() - .as_slice(), - &[operation_id] + assert_ne!( + restarted_backend.restore_operation_ids.lock().unwrap()[0], + operation_id ); assert!( restarted diff --git a/crates/worker-runtime/src/runtime/restore_continuation.rs b/crates/worker-runtime/src/runtime/restore_continuation.rs index 3de663219..ad50b4d2d 100644 --- a/crates/worker-runtime/src/runtime/restore_continuation.rs +++ b/crates/worker-runtime/src/runtime/restore_continuation.rs @@ -266,9 +266,10 @@ impl Runtime { let Some(backend) = &state.execution_backend else { return Ok(()); }; - if backend - .restore_recovery_disposition(operation, &worker_ref) - .is_none() + if !backend.supports_restore_continuation() + || backend + .restore_recovery_disposition(operation, &worker_ref) + .is_none() { return Ok(()); } @@ -396,6 +397,10 @@ impl Runtime { "worker_restore_rolled_back", result.message_or_default(), ); + #[cfg(feature = "fs-store")] + self.lock()? + .record_restore_failure(worker_ref, &result, &receipt)?; + #[cfg(not(feature = "fs-store"))] self.settle_restore_rollback(worker_ref, &receipt)?; return Ok(Step::Finished); } diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 3d95f99ff..7e1288701 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -2455,25 +2455,27 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .await .map_err(|_| RuntimeRestoreCleanupFailure::Workdir)?; } - if let Some(resources) = retained { - if let Some(allocation) = resources.allocation { - allocation - .release() - .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; - } else { - // Reconstruction may fail before returning its allocation guard. - // Absence is checked at the allocation authority, not inferred from Err. - let path = worker::runtime::worker_allocation::default_allocation_path() - .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; - let table = worker::runtime::worker_allocation::LockFileGuard::open(&path) - .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; - if table - .data() - .find(&Self::runtime_worker_name_for_ref(&request.worker_ref)) - .is_some() - { - return Err(RuntimeRestoreCleanupFailure::Allocation); - } + if let Some(allocation) = retained + .as_ref() + .and_then(|resources| resources.allocation.as_ref()) + { + allocation + .release() + .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; + } else { + // A cold restart loses the retained map as well as allocation guards. + // Prove absence at the allocation authority even with an empty map; + // never release a possibly different owner by Worker name. + let path = worker::runtime::worker_allocation::default_allocation_path() + .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; + let table = worker::runtime::worker_allocation::LockFileGuard::open(&path) + .map_err(|_| RuntimeRestoreCleanupFailure::Allocation)?; + if table + .data() + .find(&Self::runtime_worker_name_for_ref(&request.worker_ref)) + .is_some() + { + return Err(RuntimeRestoreCleanupFailure::Allocation); } } // restore_controller starts the Controller only as its final fallible @@ -2486,10 +2488,18 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .map_err(|_| RuntimeRestoreCleanupFailure::Artifacts)?; Self::remove_restore_run_artifacts(&run_dir) .map_err(|_| RuntimeRestoreCleanupFailure::Artifacts)?; - self.failed_restore_resources + let mut resources = self + .failed_restore_resources .lock() - .map_err(|_| RuntimeRestoreCleanupFailure::Ownership)? - .remove(&request.worker_ref); + .map_err(|_| RuntimeRestoreCleanupFailure::Ownership)?; + if resources + .get(&request.worker_ref) + .is_some_and(|owner| owner.operation_id != request.operation_id) + { + return Err(RuntimeRestoreCleanupFailure::Ownership); + } + resources.remove(&request.worker_ref); + drop(resources); self.observation_hub.get(&request.worker_ref); Ok(()) } @@ -2763,6 +2773,29 @@ impl ProfileRuntimeWorkerFactory { let run_dir = self .worker_restore_run_dir(&request.worker_ref, request.operation_id) .map_err(|_| "restore run directory unavailable")?; + // A process restart loses the candidate, not its exact operation's + // disposable directory. Never infer writer absence from empty maps. + match std::fs::symlink_metadata(&run_dir) { + Ok(_) => { + self.stopped_restore_run_dirs(&request.worker_ref) + .map_err(|_| "restore run artifacts cannot be validated")?; + let path = worker::runtime::worker_allocation::default_allocation_path() + .map_err(|_| "restore allocation authority unavailable")?; + let table = worker::runtime::worker_allocation::LockFileGuard::open(&path) + .map_err(|_| "restore allocation authority unavailable")?; + if table + .data() + .find(&Self::runtime_worker_name_for_ref(&request.worker_ref)) + .is_some() + { + return Err("restore run directory still has an allocation owner"); + } + Self::remove_restore_run_artifacts(&run_dir) + .map_err(|_| "restore run artifacts cannot be reclaimed")?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Err("restore run artifacts cannot be inspected"), + } let transport = self.controller_transport; #[cfg(test)] let transport = if self.restore_faults.socket_failure.load(Ordering::SeqCst) { @@ -3138,6 +3171,9 @@ struct RuntimeWorkerExecution { // marker. Timeout never drops the factory future or a late Controller result. struct PendingRuntimeRestore { request: WorkerExecutionRestoreRequest, + // Transient ownership mode, not a durable phase. Cold restart reclamation + // must never enter preparation, canonical repair, or activation. + cleanup_only: bool, task: tokio::sync::Mutex>>, task_in_flight: AtomicBool, result: Mutex, RuntimeRestoreFailure>>>, @@ -3469,6 +3505,7 @@ where pending: &Arc, resume_activation: bool, ) -> Result<(), String> { + let resume_activation = resume_activation && !pending.cleanup_only; let failure = pending .result .lock() @@ -5317,6 +5354,16 @@ where } }; if let Some(pending) = admitted_pending.as_ref() { + if pending.cleanup_only { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "cleanup-only restore cannot prepare or reconstruct execution", + ), + worker_state: None, + workdir_attachments: pending.request.previous_workdir_attachments.clone(), + }; + } if request.request != pending.request.request || request.workspace_scope != pending.request.workspace_scope || request.logical_workdir_attachments @@ -5378,6 +5425,7 @@ where .remove(&worker_ref); let pending = Arc::new(PendingRuntimeRestore { request: request.clone(), + cleanup_only: false, task: tokio::sync::Mutex::new(None), task_in_flight: AtomicBool::new(false), result: Mutex::new(None), @@ -5593,17 +5641,23 @@ where } } + fn supports_restore_continuation(&self) -> bool { + true + } + fn restore_recovery_disposition( &self, operation_id: crate::execution::WorkerLifecycleOperationId, worker_ref: &WorkerRef, ) -> Option { use crate::execution::RestoreRecoveryDisposition as D; - let pending = self.pending_restore(worker_ref).ok().flatten(); + let pending = self.pending_restore(worker_ref).ok()?; Some(match pending { Some(pending) if pending.request.operation_id == operation_id => { if pending.task_in_flight.load(Ordering::Acquire) { D::InFlight + } else if pending.cleanup_only { + D::Reclaim } else if pending .discard_result .lock() @@ -5625,10 +5679,132 @@ where D::Continue } } - _ => D::Continue, + _ => { + let workers = self.workers.lock().ok()?; + if workers + .get(worker_ref) + .is_some_and(|execution| execution.restore_operation_id == Some(operation_id)) + { + D::Continue + } else { + D::Absent + } + } }) } + fn reclaim_uncommitted_restore( + &self, + request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + let reclaimed = (|| -> Result { + let worker_ref = &request.worker_ref; + let operation_id = request.operation_id; + let operation_lock = self.worker_lock(worker_ref)?; + let _guard = operation_lock + .lock() + .map_err(|_| "worker operation lock is poisoned")?; + if request.context.worker_ref() != worker_ref { + return Err("restore cleanup context does not match Worker authority".into()); + } + if self + .workers + .lock() + .map_err(|_| "worker registry lock is poisoned")? + .contains_key(worker_ref) + { + return Err("uncommitted reclaim cannot target a connected execution".into()); + } + let pending = match self.pending_restore(worker_ref)? { + Some(pending) => { + if pending.request.operation_id != operation_id || !pending.cleanup_only { + return Err( + "uncommitted reclaim cannot replace an in-memory restore owner".into(), + ); + } + if request.request != pending.request.request + || request.workspace_scope != pending.request.workspace_scope + || request.logical_workdir_attachments + != pending.request.logical_workdir_attachments + || request.previous_workdir_attachments + != pending.request.previous_workdir_attachments + || request.source_repair_possible != pending.request.source_repair_possible + { + return Err( + "current Worker authority differs from the owned restore cleanup" + .into(), + ); + } + pending + } + None => { + if let Some(result) = self.completed_restore_failure(operation_id, worker_ref) { + return Ok(result); + } + let mut failure = RuntimeRestoreFailure::unproven( + RuntimeRestoreStage::Controller, + "restore decision was not committed before restart", + ); + failure.durable_effects_possible = request.source_repair_possible; + let pending = Arc::new(PendingRuntimeRestore { + request: request.clone(), + cleanup_only: true, + task: tokio::sync::Mutex::new(None), + task_in_flight: AtomicBool::new(false), + result: Mutex::new(Some(Err(failure))), + activated: Mutex::new(None), + activation_started: AtomicBool::new(false), + discard_started: AtomicBool::new(false), + discard_result: Mutex::new(None), + failure: Mutex::new(None), + }); + self.pending_restores + .lock() + .map_err(|_| "pending restore resources lock is poisoned")? + .insert(worker_ref.clone(), pending.clone()); + pending + } + }; + // Join the prior task before retrying. A timeout retains this owner, + // its original context notifier, and its exact task/JoinHandle. + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + self.retry_pending_restore_cleanup(&pending, false)?; + self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; + let failure = pending + .result + .lock() + .map_err(|_| "restore result lock is poisoned")? + .as_ref() + .and_then(|result| result.as_ref().err()) + .cloned() + .ok_or("restore cleanup has no retained failure result")?; + if failure.needs_cleanup() { + return Err(failure.to_string()); + } + if failure.durable_effects_possible { + return Err("Restore resource cleanup is confirmed; possible earlier canonical effects still require reconciliation".into()); + } + self.completed_restore_failures + .lock() + .map_err(|_| "restore failure receipts lock is poisoned")? + .insert(worker_ref.clone(), (operation_id, failure.clone())); + self.remove_pending_restore(worker_ref)?; + Ok(WorkerExecutionSpawnResult::RolledBack( + WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + failure.to_string(), + ), + )) + })(); + reclaimed.unwrap_or_else( + |message| WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message), + worker_state: None, + workdir_attachments: request.previous_workdir_attachments.clone(), + }, + ) + } + fn discard_uncommitted_restore( &self, operation_id: crate::execution::WorkerLifecycleOperationId, @@ -5643,6 +5819,7 @@ where .pending_restore(worker_ref)? .ok_or("unpublished restore owner is unavailable")?; if pending.request.operation_id != operation_id + || pending.cleanup_only || pending.activation_started.load(Ordering::Acquire) { return Err( @@ -5784,6 +5961,9 @@ where if pending.request.operation_id != operation_id { return Err("restored Worker operation identity does not match".into()); } + if pending.cleanup_only { + return Err("cleanup-only restore cannot activate execution".into()); + } self.wait_pending_restore(&pending, self.spawn_restore_timeout)?; if pending.owns_prepared_candidate() { pending.activation_started.store(true, Ordering::Release); @@ -7696,6 +7876,7 @@ mod tests { worker_ref.clone(), Arc::new(PendingRuntimeRestore { request: direct_restore_request(worker_ref.clone()), + cleanup_only: false, task: tokio::sync::Mutex::new(None), task_in_flight: AtomicBool::new(false), result: Mutex::new(None), @@ -7740,6 +7921,7 @@ mod tests { .unwrap(); let pending = Arc::new(PendingRuntimeRestore { request: request.clone(), + cleanup_only: false, task: tokio::sync::Mutex::new(Some(failed)), task_in_flight: AtomicBool::new(false), result: Mutex::new(None), @@ -10255,6 +10437,309 @@ mod tests { } } + // Backend component seam only: no saved Worker, model, or Controller is + // constructed. Any accidental preparation/recovery is an immediate failure. + struct CleanupOnlyFactory { + cleanup_calls: Arc, + cleanup_allowed: Arc, + release: Option>, + } + + #[async_trait] + impl RuntimeWorkerFactory for CleanupOnlyFactory { + async fn spawn_controller( + &self, + _: WorkerExecutionSpawnRequest, + ) -> Result { + panic!("reclaim must not spawn") + } + + async fn preflight_restore(&self, _: &WorkerExecutionRestoreRequest) -> Result<(), String> { + panic!("reclaim must not preflight preparation") + } + + async fn recover_restore_source( + &self, + _: &WorkerExecutionRestoreRequest, + ) -> Result { + panic!("reclaim must not repair canonical source") + } + + async fn prepare_restore( + &self, + _: WorkerExecutionRestoreRequest, + ) -> Result, RuntimeRestoreFailure> { + panic!("reclaim must not prepare") + } + + async fn cleanup_failed_restore( + &self, + _: &WorkerExecutionRestoreRequest, + ) -> Result<(), RuntimeRestoreCleanupFailure> { + if self.cleanup_calls.fetch_add(1, Ordering::SeqCst) == 0 { + if let Some(release) = &self.release { + release.notified().await; + } + } + if self.cleanup_allowed.load(Ordering::SeqCst) { + Ok(()) + } else { + Err(RuntimeRestoreCleanupFailure::Workdir) + } + } + + async fn reconcile_stopped_worker(&self, _: &WorkerRef) -> Result<(), String> { + panic!("reclaim must not clean all Worker operations through Stop") + } + } + + #[test] + fn cold_reclaim_retries_cleanup_only_and_keeps_cleaned_receipt_idempotent() { + use crate::execution::RestoreRecoveryDisposition as D; + let cleanup_calls = Arc::new(AtomicUsize::new(0)); + let cleanup_allowed = Arc::new(AtomicBool::new(false)); + let backend = WorkerRuntimeExecutionBackend::new(CleanupOnlyFactory { + cleanup_calls: cleanup_calls.clone(), + cleanup_allowed: cleanup_allowed.clone(), + release: None, + }) + .unwrap(); + let request = direct_restore_request(WorkerRef::new(WorkerId::now_v7())); + let worker_ref = &request.worker_ref; + assert_eq!( + backend.restore_recovery_disposition(request.operation_id, worker_ref), + Some(D::Absent) + ); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + let owner = backend.pending_restore(worker_ref).unwrap().unwrap(); + assert!(owner.cleanup_only); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 1); + assert_eq!( + backend.restore_recovery_disposition(request.operation_id, worker_ref), + Some(D::Reclaim) + ); + // Wrong operation and changed authority must not replace or retry it. + let mut different = request.clone(); + different.operation_id = crate::execution::WorkerLifecycleOperationId::new(); + assert!(matches!( + backend.reclaim_uncommitted_restore(different), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + let mut mismatched = request.clone(); + mismatched.request = create_request("changed cleanup authority"); + assert!(matches!( + backend.reclaim_uncommitted_restore(mismatched), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert!(Arc::ptr_eq( + &owner, + &backend.pending_restore(worker_ref).unwrap().unwrap() + )); + // Erroneous generic callers cannot reconstruct, activate, or substitute + // in-process candidate discard for cold cleanup. + assert!(matches!( + backend.restore_worker(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert!( + backend + .activate_restored_worker(request.operation_id, worker_ref) + .is_err() + ); + assert!(matches!( + backend.discard_uncommitted_restore(request.operation_id, worker_ref), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 1); + cleanup_allowed.store(true, Ordering::SeqCst); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::RolledBack(_) + )); + assert!(backend.pending_restore(worker_ref).unwrap().is_none()); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::RolledBack(_) + )); + assert!(matches!( + backend.restore_worker(request), + WorkerExecutionSpawnResult::RolledBack(_) + )); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 2); + assert!(backend.workers.lock().unwrap().is_empty()); + } + + #[test] + fn cold_reclaim_preserves_possible_effects_without_repreparing() { + use crate::execution::RestoreRecoveryDisposition as D; + let cleanup_calls = Arc::new(AtomicUsize::new(0)); + let backend = WorkerRuntimeExecutionBackend::new(CleanupOnlyFactory { + cleanup_calls: cleanup_calls.clone(), + cleanup_allowed: Arc::new(AtomicBool::new(true)), + release: None, + }) + .unwrap(); + let mut request = direct_restore_request(WorkerRef::new(WorkerId::now_v7())); + request.source_repair_possible = true; + for _ in 0..2 { + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + } + let owner = backend + .pending_restore(&request.worker_ref) + .unwrap() + .unwrap(); + assert!(owner.cleanup_only); + let retained = owner.result.lock().unwrap(); + let failure = retained.as_ref().unwrap().as_ref().err().unwrap(); + assert!(failure.durable_effects_possible); + assert_eq!(failure.cleanup, RuntimeRestoreCleanup::Confirmed); + drop(retained); + // Even an internal retry with the wrong resume flag cannot prepare. + backend.retry_pending_restore_cleanup(&owner, true).unwrap(); + assert!(owner.task.try_lock().unwrap().is_none()); + assert!( + backend + .completed_restore_failure(request.operation_id, &request.worker_ref) + .is_none() + ); + assert_eq!( + backend.restore_recovery_disposition(request.operation_id, &request.worker_ref), + Some(D::Reclaim) + ); + assert!(matches!( + backend.reconcile_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert!( + backend + .activate_restored_worker(request.operation_id, &request.worker_ref) + .is_err() + ); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 1); + } + + #[test] + fn cold_reclaim_timeout_retains_same_task_and_notifies_before_cleanup_retry() { + use crate::execution::RestoreRecoveryDisposition as D; + let cleanup_calls = Arc::new(AtomicUsize::new(0)); + let cleanup_allowed = Arc::new(AtomicBool::new(false)); + let release = Arc::new(tokio::sync::Notify::new()); + let mut backend = WorkerRuntimeExecutionBackend::new(CleanupOnlyFactory { + cleanup_calls: cleanup_calls.clone(), + cleanup_allowed: cleanup_allowed.clone(), + release: Some(release.clone()), + }) + .unwrap() + .with_spawn_restore_timeout(Duration::from_millis(1)); + let mut request = direct_restore_request(WorkerRef::new(WorkerId::now_v7())); + let (progress, progressed) = mpsc::channel(); + request.context = request + .context + .with_restore_progress(Arc::new(move |operation| { + let _ = progress.send(operation); + })); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + let owner = backend + .pending_restore(&request.worker_ref) + .unwrap() + .unwrap(); + let task_id = owner.task.try_lock().unwrap().as_ref().unwrap().id(); + assert_eq!( + backend.restore_recovery_disposition(request.operation_id, &request.worker_ref), + Some(D::InFlight) + ); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert!(Arc::ptr_eq( + &owner, + &backend + .pending_restore(&request.worker_ref) + .unwrap() + .unwrap() + )); + assert_eq!( + owner.task.try_lock().unwrap().as_ref().unwrap().id(), + task_id + ); + release.notify_one(); + assert_eq!( + progressed.recv_timeout(Duration::from_secs(5)).unwrap(), + request.operation_id + ); + backend + .wait_pending_restore(&owner, Duration::from_secs(5)) + .unwrap(); + assert!(owner.task.try_lock().unwrap().is_none()); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 1); + assert_eq!( + backend.restore_recovery_disposition(request.operation_id, &request.worker_ref), + Some(D::Reclaim) + ); + backend.spawn_restore_timeout = Duration::from_secs(5); + cleanup_allowed.store(true, Ordering::SeqCst); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::RolledBack(_) + )); + assert_eq!( + progressed.recv_timeout(Duration::from_secs(5)).unwrap(), + request.operation_id + ); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 2); + assert!(backend.workers.lock().unwrap().is_empty()); + } + + #[test] + fn cold_reclaim_refuses_existing_candidate_owner_even_for_same_operation() { + let backend = WorkerRuntimeExecutionBackend::new(CleanupOnlyFactory { + cleanup_calls: Arc::new(AtomicUsize::new(0)), + cleanup_allowed: Arc::new(AtomicBool::new(true)), + release: None, + }) + .unwrap(); + let request = direct_restore_request(WorkerRef::new(WorkerId::now_v7())); + let owner = Arc::new(PendingRuntimeRestore { + request: request.clone(), + cleanup_only: false, + task: tokio::sync::Mutex::new(None), + task_in_flight: AtomicBool::new(false), + result: Mutex::new(None), + activated: Mutex::new(None), + activation_started: AtomicBool::new(false), + discard_started: AtomicBool::new(false), + discard_result: Mutex::new(None), + failure: Mutex::new(None), + }); + backend + .pending_restores + .lock() + .unwrap() + .insert(request.worker_ref.clone(), owner.clone()); + assert!(matches!( + backend.reclaim_uncommitted_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert!(Arc::ptr_eq( + &owner, + &backend + .pending_restore(&request.worker_ref) + .unwrap() + .unwrap() + )); + assert_eq!(backend.factory.cleanup_calls.load(Ordering::SeqCst), 0); + } + #[test] fn absent_execution_stop_reconciles_pending_restore_cleanup_without_launching() { let restore_calls = Arc::new(AtomicUsize::new(0)); diff --git a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs index d622a7462..99823169c 100644 --- a/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_failure_tests.rs @@ -193,6 +193,18 @@ max_tokens = 100 self.inner.restore_worker(request) } + fn restore_recovery_disposition( + &self, + operation_id: crate::execution::WorkerLifecycleOperationId, + worker_ref: &WorkerRef, + ) -> Option { + self.inner.restore_recovery_disposition(operation_id, worker_ref) + } + + fn reclaim_uncommitted_restore(&self, request: WorkerExecutionRestoreRequest) -> WorkerExecutionSpawnResult { + self.inner.reclaim_uncommitted_restore(request) + } + fn reconcile_restore( &self, request: WorkerExecutionRestoreRequest, diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs index 28323570f..d7518319b 100644 --- a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -94,6 +94,133 @@ mod publication { (release, received) } + #[test] + #[serial_test::serial(worker_allocation)] + fn startup_fault_recovery_retains_cleanup_or_committed_reconstruction_without_repost() { + use crate::runtime::RestoreCommitFact as F; + for committed in [false, true] { + let fixture = Fixture::no_model(); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let request = runtime.test_restore_request(&worker_ref); + runtime.coordinate_worker_restore_operation(&worker_ref, runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), preparation: request.preparation.clone(), + }, fixture.request.workspace_scope.as_ref()).unwrap(); + let operation = runtime.test_stage_restore_checkpoint(&worker_ref, if committed { F::Committed } else { F::NotCommitted }, false); + let before = fixture.canonical_files(); + drop(runtime); + if committed { + *fixture.backend.factory.restore_faults.fail_at.lock().unwrap() = Some(RuntimeRestoreStage::Metadata); + } else { + fixture.backend.factory.restore_faults.cleanup_fail.store(true, Ordering::SeqCst); + } + let options = FsRuntimeStoreOptions::new(fixture._root.path().join("catalog")).with_runtime_id("candidate-runtime"); + let restarted = EmbeddedRuntime::with_fs_store_and_execution_backend(options, fixture.backend.clone()).unwrap(); + let pause = restarted.test_pause_restore_continuation(); + assert_eq!(durable_operation(&restarted, &worker_ref).unwrap().operation_id, operation); + assert!(fixture.backend.completed_restore_failure(operation, &worker_ref).is_none()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + *fixture.backend.factory.restore_faults.fail_at.lock().unwrap() = None; + fixture.backend.factory.restore_faults.cleanup_fail.store(false, Ordering::SeqCst); + drop(pause); + restarted.test_wait_restore_continuation(&worker_ref, operation, WAIT); + let receipt = restarted.restore_worker_operation(&worker_ref, request).unwrap(); + assert_eq!(receipt.state, if committed { server_api::WorkerRestoreState::Accepted } else { server_api::WorkerRestoreState::RolledBack }); + if committed { + let transport = fixture.backend.clone().attach_worker_protocol(&worker_ref).unwrap(); + transport.validate().unwrap(); + drop(transport); + restarted.stop_worker(&worker_ref, None).unwrap(); + } else { + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 0); + assert_eq!(fixture.canonical_files(), before); + } + fixture.assert_released(); + fixture.assert_quiet(); + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn startup_dispatches_saved_decisions_without_replaying_uncommitted_execution() { + use crate::runtime::RestoreCommitFact as F; + for (commit, repaired, expected) in [ + (F::NotCommitted, false, server_api::WorkerRestoreState::RolledBack), + (F::NotCommitted, true, server_api::WorkerRestoreState::ReconciliationRequired), + (F::Committed, false, server_api::WorkerRestoreState::Accepted), + (F::LegacyUnknown, true, server_api::WorkerRestoreState::ReconciliationRequired), + ] { + let fixture = Fixture::no_model(); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + assert!(runtime.coordinate_worker_restore_operation(&worker_ref, runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: request.preparation.clone(), + }, Some(scope)).unwrap().0.is_none()); + let operation = runtime.test_stage_restore_checkpoint(&worker_ref, commit, repaired); + let before = fixture.canonical_files(); + let run = fixture.backend.factory.worker_restore_run_dir(&worker_ref, operation).unwrap(); + std::fs::create_dir_all(&run).unwrap(); + std::fs::write(run.join("before-restart.marker"), "disposable").unwrap(); + if commit == F::NotCommitted && !repaired { + // Fixture for an interrupted automatic restore. A rollback must + // not leave an active/Automatic projection that restores again. + let store = runtime.test_runtime_store(); + let mut record = store.read_worker_record(&worker_ref).unwrap(); + record.status = crate::catalog::WorkerStatus::Idle; + let PersistedWorkerExecutionState::ReconciliationRequired( + crate::fs_store::PersistedWorkerLifecycleOperation::Restore(saved), + ) = &mut record.execution_state else { panic!("pending fixture") }; + saved.mode = crate::fs_store::PersistedWorkerRestoreMode::Automatic; + saved.last_settled_status = crate::catalog::WorkerStatus::Idle; + saved.binding = Some(crate::fs_store::PersistedWorkerExecutionBinding {}); + store.write_worker_record(&record).unwrap(); + } + drop(runtime); + assert!(fixture.backend.pending_restore(&worker_ref).unwrap().is_none()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + let options = FsRuntimeStoreOptions::new(fixture._root.path().join("catalog")).with_runtime_id("candidate-runtime"); + let restarted = EmbeddedRuntime::with_fs_store_and_execution_backend(options.clone(), fixture.backend.clone()).unwrap(); + if expected == server_api::WorkerRestoreState::ReconciliationRequired { + let reason = restarted.test_wait_restore_continuation_parked(&worker_ref, operation, WAIT); + assert!(!reason.is_empty()); + let pending = durable_operation(&restarted, &worker_ref).unwrap(); + assert_eq!(pending.operation_id, operation); + assert_eq!(pending.commit, commit); + assert_eq!(pending.source_repair_possible, repaired); + } + assert!(restarted.worker_detail(&worker_ref).is_ok(), "{commit:?}: {:?}", restarted.diagnostics()); + let receipt = restarted.restore_worker_operation(&worker_ref, request.clone()).unwrap(); + assert_eq!(receipt.state, expected, "{commit:?}: {receipt:?}; backend failure: {:?}", fixture.backend.completed_restore_failure(operation, &worker_ref)); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), usize::from(commit == F::Committed)); + if commit == F::Committed { + let transport = fixture.backend.clone().attach_worker_protocol(&worker_ref).unwrap(); + transport.validate().unwrap(); + drop(transport); + } else { + assert_eq!(fixture.canonical_files(), before); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!(fixture.backend.factory.observation_hub.get(&worker_ref).is_none()); + assert_eq!(run.exists(), commit == F::LegacyUnknown); + } + if expected == server_api::WorkerRestoreState::RolledBack { + assert_eq!(restarted.worker_detail(&worker_ref).unwrap().status, crate::catalog::WorkerStatus::Stopped); + drop(restarted); + let second = EmbeddedRuntime::with_fs_store_and_execution_backend(options, fixture.backend.clone()).unwrap(); + assert_eq!(second.restore_worker_operation_scoped(scope, &worker_ref, request).unwrap(), receipt); + assert_eq!(fixture.backend.factory.restore_faults.executions.load(Ordering::SeqCst), 0); + } else { + restarted.stop_worker_scoped(scope, &worker_ref, None).unwrap(); + } + fixture.assert_released(); + fixture.assert_quiet(); + } + } + #[test] #[serial_test::serial(worker_allocation)] fn stop_timeout_retains_autonomous_late_cleanup_and_blocks_remove_until_confirmed() { From f5ebae1134d7ce4f3f174a4dc68aa397a994b81f Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 12:27:33 +0900 Subject: [PATCH 29/46] test: verify restore response loss and owned runtime restart --- tests/e2e/README.md | 8 +- tests/e2e/src/worker_environment.rs | 84 +++++++- .../src/worker_environment/response_loss.rs | 198 ++++++++++++++++++ tests/e2e/tests/worker_restore.rs | 180 ++++++++++++++++ 4 files changed, 464 insertions(+), 6 deletions(-) create mode 100644 tests/e2e/src/worker_environment/response_loss.rs diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 5766c9471..a54eb8517 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -22,7 +22,7 @@ 所有crateの`test-support`依存はテスト側の非default `fixture-support` featureだけで有効にする。製品binary providerは別Cargo invocationと専用`target/e2e-products`へ`yoi`・`yoi-server`・`yoi-runtime`を追加featureなしでビルドする。support test自身をこのtargetへビルドしてはならない。`YOI_E2E_BIN`等の明示binary overrideを使用する場合も、呼出し側が同じ分離を保証する。既存TUI用`e2e-test` featureは別の`target/e2e-tui-products`へビルドし、restore製品binaryへ混入しない。 -固定資産の正常restore・並行隔離E2E: +固定資産の正常restore・並行隔離・応答喪失回復E2E: ```sh cargo test -p yoi-e2e --features e2e,fixture-support --test worker_restore @@ -33,6 +33,12 @@ YOI_E2E_RESTORE_FIXTURE=target/fixture-candidate-v1 \ 標準資産は`tests/e2e/fixtures/worker-codex-v1`の`manifest.json`・`worker-template.json`・`expected.json`・`workspace-config.dcdl`。manifestの各asset hash/size、schemaと固定ID、生成元commit、modelを検証する。Workspace configのimport参照を黙って最新へ置換せず、Gitで生成元commitに対する`resources`と`crates/builtin-source`の一致を確認する（Git履歴がない場合も失敗）。移植したSessionの正規公開projectionを供給された会話行と照合し、その全snapshotを実executionのWebSocketと比較する。欠落fixture・hash不一致・未実行は成功扱いしない。 +応答喪失ケースは同じ`WorkerE2eEnvironment`内の単発loopback relayで公開restore要求をそのまま転送し、完全なupstream応答を受け取った後に呼出元へ0byteで切断する。呼出元のtransport失敗とrelay側のAccepted観測は別の事実として記録する。回収は元のrequest ID・観測tokenをそのまま使い、fresh intentへの古いtokenはHTTP409を要求する。 + +process終了ケースはそのAccepted receipt確認後に環境が所有するRuntime childだけをkill/reapし、同じroot・Runtime identity・trustから再起動する。seedの再実行・tokenの差替えはしない。再起動前後のcanonical owner全体のhash・operation ID・owner数が一致し、公開APIの保存結果とfull Session snapshotを回収できることを確認する。公開Worker投影の観測tokenは再起動で変わるため、そのfieldだけはcurrent detailとの一致を別に確認する。canonical診断はboundedな読取りのみで、要求・receiptの内容や認証情報はartifactへ保存しない。 + +これは**commit済みかつterminal receipt保存後**のprocess終了であり、未commit候補やcommit直後・receipt前の終了証明ではない。固定Codex資産へSubmit/Notifyを送らないため、通常入力・Paused admissionの製品E2Eの証明でもない。検証jobはこの明示targetを選択する必要があり、通常suiteだけでは代替しない。 + 通常のharness回帰試験（製品processは起動しない）: ```sh diff --git a/tests/e2e/src/worker_environment.rs b/tests/e2e/src/worker_environment.rs index 1fbdc4b0b..749ef4c83 100644 --- a/tests/e2e/src/worker_environment.rs +++ b/tests/e2e/src/worker_environment.rs @@ -1,3 +1,6 @@ +#[path = "worker_environment/response_loss.rs"] +mod response_loss; + use std::fs::{self, File, OpenOptions}; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; @@ -571,12 +574,7 @@ impl WorkerE2eEnvironment { if self.runtime_public_identity.is_none() { self.initialize_runtime_identity(self.runtime_id.clone())?; } - let runtime_identity = self - .runtime_public_identity - .clone() - .expect("initialized identity"); let bundle_path = self.root.join("workspace-public-identity.json"); - let server_url = self.required_server_url()?.to_string(); let workspace_id = self.required_workspace_id()?.to_string(); let runtime_binary = self.binaries.binary("yoi-runtime")?; self.run_command( @@ -616,6 +614,20 @@ impl WorkerE2eEnvironment { seed(self)?; self.write_metadata("runtime-seeded")?; + self.start_runtime_process() + } + + fn start_runtime_process(&mut self) -> Result<()> { + if self.runtime.is_some() { + return Err(HarnessError::Protocol("Runtime is already running".into())); + } + let runtime_binary = self.binaries.binary("yoi-runtime")?; + let runtime_identity = self + .runtime_public_identity + .clone() + .ok_or_else(|| HarnessError::Protocol("Runtime identity is not initialized".into()))?; + let server_url = self.required_server_url()?.to_string(); + let workspace_id = self.required_workspace_id()?.to_string(); let mut command = self.isolated_command(&runtime_binary); command.args([ "--bind", @@ -669,6 +681,36 @@ impl WorkerE2eEnvironment { Ok(()) } + /// Kill/reap only the Runtime child owned by this environment, retaining roots + /// and identities. Restart neither seeds storage nor rotates restore requests. + pub fn crash_and_restart_runtime(&mut self) -> Result<()> { + let runtime = self + .runtime + .as_mut() + .ok_or_else(|| HarnessError::Protocol("Runtime is not running".into()))?; + let pid = runtime.id(); + let status = runtime.kill_and_reap(EXIT_WAIT)?; + runtime.join_readers_until(Instant::now() + EXIT_WAIT)?; + fs::write( + self.artifacts_dir.join("runtime-crash.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "owned_pid": pid, "reaped": true, "exit_status": status.to_string(), + "same_runtime_id": self.runtime_id, + }))?, + )?; + self.runtime.take(); + for stream in ["stdout", "stderr"] { + fs::rename( + self.artifacts_dir.join(format!("runtime.{stream}.log")), + self.artifacts_dir + .join(format!("runtime-before-crash.{stream}.log")), + )?; + } + self.runtime_url = None; + self.runtime_shutdown = WorkerProcessShutdownOutcome::NotStarted; + self.start_runtime_process() + } + pub fn assert_connected(&self) -> Result<()> { let server_url = self.required_server_url()?; let workspace_id = self.required_workspace_id()?; @@ -752,6 +794,38 @@ impl WorkerE2eEnvironment { ) } + /// Forward one unchanged public restore request through an owned loopback + /// relay, consume its complete upstream response, and close without delivering + /// any response bytes. The returned JSON is relay evidence, not caller success. + pub fn restore_worker_losing_response( + &self, + worker_id: &str, + request_id: &str, + expected_observation_token: &str, + ) -> Result { + let token = self.access_token.as_deref().ok_or_else(|| { + HarnessError::Protocol("Backend access token is not initialized".into()) + })?; + let request = self + .client + .post(self.worker_url(worker_id, "restore")?) + .bearer_auth(token) + .header(reqwest::header::CONNECTION, "close") + .timeout(STARTUP_WAIT + EXIT_WAIT) + .json(&serde_json::json!({ + "request_id": request_id, + "expected_observation_token": expected_observation_token, + })) + .build() + .map_err(|_| HarnessError::Protocol("build response-loss request".into()))?; + let evidence = response_loss::drop_response(&self.client, request)?; + self.append_operation( + "restore response consumed by relay, caller transport failed", + 0, + )?; + Ok(evidence) + } + pub fn stop_worker(&self, worker_id: &str) -> Result { self.request_json( self.client diff --git a/tests/e2e/src/worker_environment/response_loss.rs b/tests/e2e/src/worker_environment/response_loss.rs new file mode 100644 index 000000000..ee6654c85 --- /dev/null +++ b/tests/e2e/src/worker_environment/response_loss.rs @@ -0,0 +1,198 @@ +//! A single bounded response-loss injection in the shared product harness. +use std::io::{Read, Write}; +use std::net::{IpAddr, TcpListener, TcpStream}; +use std::time::Instant; + +use reqwest::blocking::{Client, Request}; +use serde_json::Value; + +use super::{EXIT_WAIT, HTTP_WAIT_STEP, STARTUP_WAIT}; +use crate::{HarnessError, Result}; + +const MAX_HEADER: usize = 16 * 1024; +const MAX_MESSAGE: usize = 2 * 1024 * 1024; + +fn failure(message: &str) -> HarnessError { + HarnessError::Protocol(message.into()) +} + +pub(super) fn drop_response(client: &Client, mut request: Request) -> Result { + let host: IpAddr = request + .url() + .host_str() + .ok_or_else(|| failure("missing relay host"))? + .parse() + .map_err(|_| failure("relay target must be a literal loopback address"))?; + if !host.is_loopback() || request.url().scheme() != "http" { + return Err(failure( + "response-loss relay only permits owned loopback HTTP", + )); + } + let port = request + .url() + .port() + .ok_or_else(|| failure("missing relay port"))?; + let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))?; + listener.set_nonblocking(true)?; + let relay_address = listener.local_addr()?; + request + .url_mut() + .set_port(Some(relay_address.port())) + .map_err(|_| failure("set relay port"))?; + request + .url_mut() + .set_host(Some("127.0.0.1")) + .map_err(|_| failure("set relay host"))?; + std::thread::scope(|scope| { + let relay = scope.spawn(move || -> Result { + let deadline = Instant::now() + STARTUP_WAIT; + let mut downstream = loop { + match listener.accept() { + Ok((stream, peer)) if peer.ip().is_loopback() => break stream, + Ok(_) => return Err(failure("non-loopback relay peer")), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if Instant::now() >= deadline { + return Err(failure("response-loss relay accept deadline exceeded")); + } + std::thread::sleep(HTTP_WAIT_STEP); + } + Err(error) => return Err(error.into()), + } + }; + let request_bytes = read_message(&mut DeadlineStream { + stream: &mut downstream, + deadline, + })?; + let mut upstream = TcpStream::connect_timeout(&(host, port).into(), EXIT_WAIT)?; + upstream.set_write_timeout(Some(EXIT_WAIT))?; + // The body, request identity and authorization bytes are forwarded + // unchanged. Neither request nor response headers enter diagnostics. + upstream.write_all(&request_bytes)?; + let response = read_message(&mut DeadlineStream { + stream: &mut upstream, + deadline, + })?; + let boundary = + header_end(&response).ok_or_else(|| failure("missing upstream header"))?; + let status = std::str::from_utf8(&response[..boundary]) + .ok() + .and_then(|header| header.lines().next()) + .and_then(|line| line.split_whitespace().nth(1)); + if status != Some("200") { + return Err(failure("restore relay upstream did not return HTTP 200")); + } + let value = serde_json::from_slice(&response[boundary..])?; + // Intentionally send zero bytes back; the caller must observe loss. + drop(downstream); + Ok(value) + }); + let caller_failed = client.execute(request).is_err(); + let evidence = relay + .join() + .map_err(|_| failure("response-loss relay panicked"))??; + if !caller_failed { + return Err(failure("caller unexpectedly received a response")); + } + Ok(evidence) + }) +} + +struct DeadlineStream<'a> { + stream: &'a mut TcpStream, + deadline: Instant, +} + +impl Read for DeadlineStream<'_> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let remaining = self.deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "relay deadline exceeded", + )); + } + self.stream.set_read_timeout(Some(remaining))?; + self.stream.read(buffer) + } +} + +fn header_end(bytes: &[u8]) -> Option { + bytes + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|i| i + 4) +} + +fn read_message(stream: &mut impl Read) -> Result> { + let mut bytes = Vec::new(); + let mut buffer = [0; 4096]; + let mut expected = None; + loop { + let count = stream.read(&mut buffer)?; + if count == 0 { + return Err(failure("relay peer closed before complete HTTP message")); + } + bytes.extend_from_slice(&buffer[..count]); + if bytes.len() > MAX_MESSAGE { + return Err(failure("relay HTTP byte budget exceeded")); + } + if expected.is_none() { + if let Some(boundary) = header_end(&bytes) { + if boundary > MAX_HEADER { + return Err(failure("relay HTTP header budget exceeded")); + } + let header = std::str::from_utf8(&bytes[..boundary]) + .map_err(|_| failure("invalid relay HTTP header"))?; + let mut lengths = header + .lines() + .filter_map(|line| line.split_once(':')) + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .map(|(_, value)| value.trim().parse::()); + let length = lengths + .next() + .ok_or_else(|| failure("relay requires Content-Length"))? + .map_err(|_| failure("invalid relay Content-Length"))?; + if lengths.next().is_some() + || length > MAX_MESSAGE - boundary + || header + .lines() + .filter_map(|line| line.split_once(':')) + .any(|(name, _)| name.eq_ignore_ascii_case("transfer-encoding")) + { + return Err(failure("unsupported relay HTTP framing")); + } + expected = Some(boundary + length); + } else if bytes.len() > MAX_HEADER { + return Err(failure("relay HTTP header budget exceeded")); + } + } + if let Some(expected) = expected { + if bytes.len() == expected { + return Ok(bytes); + } + if bytes.len() > expected { + return Err(failure("unexpected relay trailing bytes")); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn framing_rejects_incomplete_ambiguous_or_unbounded_messages() { + for bytes in [ + b"HTTP/1.1 200 OK\r\n\r\n".as_slice(), + b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nshort", + b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nContent-Length: 0\r\n\r\n", + b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nTransfer-Encoding: chunked\r\n\r\n", + b"HTTP/1.1 200 OK\r\nContent-Length: 999999999\r\n\r\n", + ] { + assert!(read_message(&mut &bytes[..]).is_err()); + } + let valid = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\n{}"; + assert_eq!(read_message(&mut &valid[..]).unwrap(), valid); + } +} diff --git a/tests/e2e/tests/worker_restore.rs b/tests/e2e/tests/worker_restore.rs index 3d1619847..75ef29cd2 100644 --- a/tests/e2e/tests/worker_restore.rs +++ b/tests/e2e/tests/worker_restore.rs @@ -143,6 +143,186 @@ fn concurrent_fixed_id_restores_keep_sessions_storage_and_cleanup_isolated() -> result } +#[test] +fn lost_http_response_recovers_exact_request_and_rejects_fresh_stale_intent() -> Result<()> { + response_loss_recovery(false) +} + +#[test] +fn committed_response_loss_survives_owned_runtime_process_kill_and_restart() -> Result<()> { + response_loss_recovery(true) +} + +fn response_loss_recovery(crash_runtime: bool) -> Result<()> { + let fixture = Fixture::load()?; + let result = (|| { + let mut installed = seed::start(&fixture)?; + let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?; + let token = stopped + .restore_observation_token + .as_deref() + .filter(|token| !token.is_empty()) + .ok_or_else(|| failure("missing stopped observation token"))?; + let request_id = "fixed-fixture-response-loss"; + phase( + &installed.environment, + "drop-http-response-after-complete-upstream", + )?; + let observed = installed.environment.restore_worker_losing_response( + &fixture.manifest.worker_id, + request_id, + token, + )?; + let receipt: WorkerRestoreResponse = serde_json::from_value(observed.clone())?; + if receipt.result.state != WorkerRestoreState::Accepted { + return Err(failure( + "upstream did not reach Accepted before response loss", + )); + } + fs::write( + installed + .environment + .artifacts_dir + .join("response-loss.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "caller_received_response": false, + "relay_upstream_state": receipt.result.state, + "request_id": request_id, + "crash_after_committed_terminal_receipt": crash_runtime, + }))?, + )?; + let owner_before = restore_owner_evidence(&installed.environment, &fixture, request_id)?; + if crash_runtime { + phase( + &installed.environment, + "kill-owned-runtime-after-committed-receipt", + )?; + installed.environment.crash_and_restart_runtime()?; + } + phase( + &installed.environment, + "same-request-original-token-recovery", + )?; + let recovered = + installed + .environment + .restore_worker(&fixture.manifest.worker_id, request_id, token)?; + if crash_runtime { + // The public response projects the current observation fence. Runtime + // startup invalidates prior active observations; this is not a change + // to the original request token or its saved outcome. + let current = installed + .environment + .worker_detail(&fixture.manifest.worker_id)?; + assert_eq!( + recovered.pointer("/result/worker/restore_observation_token"), + current.get("restore_observation_token") + ); + let mut expected = observed.clone(); + expected["result"]["worker"]["restore_observation_token"] = + recovered["result"]["worker"]["restore_observation_token"].clone(); + assert_eq!( + recovered, expected, + "restart may change only the public observation fence, not the saved outcome" + ); + } else { + assert_eq!( + recovered, observed, + "exact request must recover the saved receipt" + ); + } + assert_single_state(&installed.environment, &fixture, "idle")?; + let mut subscription = WorkerSessionSubscription::connect( + &installed.environment, + &fixture.manifest.worker_id, + &installed.snapshot, + )?; + subscription.assert_alive()?; + let stale = installed.environment.restore_worker( + &fixture.manifest.worker_id, + "fixed-fixture-fresh-stale-intent", + token, + ); + match stale { + Err(error) if error.to_string().contains("HTTP 409") => {} + _ => { + return Err(failure( + "fresh intent with old observation was not HTTP 409", + )); + } + } + let owner_after = restore_owner_evidence(&installed.environment, &fixture, request_id)?; + assert_eq!( + owner_after, owner_before, + "saved owner/operation/receipt must not change" + ); + fs::write( + installed + .environment + .artifacts_dir + .join("restore-owner-evidence.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "before": owner_before, "after": owner_after, + "request_token_replaced": false, + }))?, + )?; + installed.environment.assert_connected()?; + subscription.assert_alive()?; + stop_and_cleanup(installed, vec![subscription], &fixture) + })(); + fixture.source.assert_unchanged()?; + result +} + +// Read-only diagnostic of the product-written aggregate; never seeds or edits it. +// Persist only a digest, operation ID and owner count, not prompts or credentials. +fn restore_owner_evidence( + environment: &WorkerE2eEnvironment, + fixture: &Fixture, + request_id: &str, +) -> Result { + use sha2::{Digest, Sha256}; + use std::io::Read; + let path = environment + .runtime_data_dir + .join("runtime/workers") + .join(&fixture.manifest.worker_id) + .join("worker.json"); + let mut bytes = Vec::new(); + fs::File::open(path)? + .take(8 * 1024 * 1024 + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > 8 * 1024 * 1024 { + return Err(failure("Worker aggregate byte budget exceeded")); + } + let aggregate: serde_json::Value = serde_json::from_slice(&bytes)?; + let owners = aggregate + .pointer("/restore_guard/owners") + .and_then(serde_json::Value::as_object) + .ok_or_else(|| failure("canonical Worker omitted restore owners"))?; + let owner = owners + .get(request_id) + .ok_or_else(|| failure("canonical owner missing"))?; + if owner + .pointer("/receipt/state") + .and_then(serde_json::Value::as_str) + != Some("accepted") + { + return Err(failure("canonical owner has no Accepted receipt")); + } + let operation = owner + .get("operation_id") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| failure("canonical owner omitted operation ID"))?; + let digest: String = Sha256::digest(serde_json::to_vec(owner)?) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + Ok( + serde_json::json!({"operation_id": operation, "owner_sha256": digest, "owner_count": owners.len()}), + ) +} + fn restore( installed: &Installed, fixture: &Fixture, From f35dcef4d076d57451a17d62d95cc1d30deddfd0 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 13:10:54 +0900 Subject: [PATCH 30/46] test: synchronize owned runtime crashes at restore decision checkpoints --- crates/worker-runtime/Cargo.toml | 2 + crates/worker-runtime/README.md | 27 + crates/worker-runtime/src/lib.rs | 2 + crates/worker-runtime/src/main.rs | 72 +++ .../worker-runtime/src/restore_checkpoints.rs | 143 +++++ .../src/restore_checkpoints/tests.rs | 139 +++++ crates/worker-runtime/src/runtime.rs | 26 + .../src/runtime/restore_checkpoint.rs | 111 ++++ .../src/runtime/restore_checkpoint_tests.rs | 296 +++++++++++ tests/e2e/Cargo.toml | 1 + tests/e2e/README.md | 18 + tests/e2e/src/worker_environment.rs | 93 ++++ tests/e2e/tests/worker_restore.rs | 3 + tests/e2e/tests/worker_restore/checkpoint.rs | 496 ++++++++++++++++++ tests/e2e/tests/worker_restore/seed.rs | 5 +- 15 files changed, 1433 insertions(+), 1 deletion(-) create mode 100644 crates/worker-runtime/src/restore_checkpoints.rs create mode 100644 crates/worker-runtime/src/restore_checkpoints/tests.rs create mode 100644 crates/worker-runtime/src/runtime/restore_checkpoint.rs create mode 100644 crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs create mode 100644 tests/e2e/tests/worker_restore/checkpoint.rs diff --git a/crates/worker-runtime/Cargo.toml b/crates/worker-runtime/Cargo.toml index e61b4fe4f..c57cfa6a8 100644 --- a/crates/worker-runtime/Cargo.toml +++ b/crates/worker-runtime/Cargo.toml @@ -21,6 +21,8 @@ default = ["ws-server", "fs-store"] fs-store = ["dep:fs4"] # Offline fixture tooling only; never enabled by the product defaults. test-support = ["fs-store", "ws-server"] +# Explicit process-test synchronization; independent of offline fixture tooling. +restore-checkpoints = ["fs-store"] http-server = ["dep:axum", "dep:tower", "dep:reqwest"] ws-server = ["http-server", "axum/ws", "tokio/sync"] diff --git a/crates/worker-runtime/README.md b/crates/worker-runtime/README.md index e4f041745..37c4db1f0 100644 --- a/crates/worker-runtime/README.md +++ b/crates/worker-runtime/README.md @@ -39,3 +39,30 @@ not stripped or rewritten. Portability/secret-marker checks do not replace review of the supplied artificial conversation or its independently verified hashes. The installer refuses any destination content other than initialized `auth.toml`; interrupted publication is fail-closed, not an overwrite/recovery facility. + +## Owned-process Restore checkpoints + +The independent, nondefault `restore-checkpoints` feature exists only for explicit +process tests in the shared `tests/e2e` harness. Offline `test-support` does not enable +it. Normal binaries reject `--test-restore-checkpoint`; no environment variable or +HTTP endpoint activates this facility. Do not deploy a checkpoint-enabled binary +as a replacement for the default product. + +A test-enabled Runtime accepts `--test-restore-checkpoint ` with a +bounded regular JSON file selecting a loopback socket, Worker ID, one stage and a +diagnostic nonce. The nonce is only a handshake label, never execution identity, +request authority or routing state. The one-shot stages are real prepared candidate +with `NotCommitted`, and durable `Committed` before activation/terminal receipt. +Before acknowledging, the existing bounded canonical reader validates the exact +Worker/operation/decision, syncs its file/directories and requires no saved receipt. +No pending JSON is fabricated and no lifecycle work is skipped. + +The Runtime sends bounded newline-delimited JSON evidence, then waits at most eight +seconds for byte `R`. EOF, invalid reply, unavailable peer or deadline releases the +wait into the original operation. The Runtime-state mutex is not held across that +wait; existing per-Worker ownership still covers the actual candidate and cleanup. +The harness verifies the ack, kills/reaps only its owned child and inspects the +canonical aggregate offline before restarting the same roots/identity without the +checkpoint flag or another fixture install. These tests are a separate cfg/binary +lane from featureless product regression; they do not certify arbitrary OS failure, +external power loss or a real model call. diff --git a/crates/worker-runtime/src/lib.rs b/crates/worker-runtime/src/lib.rs index bb72c71e3..c00c5fe77 100644 --- a/crates/worker-runtime/src/lib.rs +++ b/crates/worker-runtime/src/lib.rs @@ -16,6 +16,8 @@ pub mod management; pub mod observation; pub mod profile_archive; pub mod resource; +#[cfg(feature = "restore-checkpoints")] +pub mod restore_checkpoints; #[cfg(test)] #[path = "../../../tests/support/restore_environment.rs"] mod restore_test_environment; diff --git a/crates/worker-runtime/src/main.rs b/crates/worker-runtime/src/main.rs index d8b985f43..e40577695 100644 --- a/crates/worker-runtime/src/main.rs +++ b/crates/worker-runtime/src/main.rs @@ -117,6 +117,12 @@ fn run() -> Result<(), ProcessError> { let listener = tokio::net::TcpListener::bind(config.http.bind_addr).await?; let local_addr = listener.local_addr()?; let worker_runtime = build_runtime(&config)?; + #[cfg(feature = "restore-checkpoints")] + if let Some(checkpoint) = &config.test_restore_checkpoint { + worker_runtime + .install_test_restore_checkpoint(checkpoint.clone()) + .map_err(ProcessError::Runtime)?; + } eprintln!( "yoi-runtime listening on {local_addr}; intended client is a trusted backend/proxy, not a browser" ); @@ -391,6 +397,21 @@ where let (flag, inline_value) = split_flag_value(arg)?; match flag.as_str() { "--help" | "-h" => return Ok(None), + #[cfg(feature = "restore-checkpoints")] + "--test-restore-checkpoint" => { + let path = take_value(&flag, inline_value, &mut args)?; + if config.test_restore_checkpoint.is_some() { + return Err(ProcessError::usage( + "--test-restore-checkpoint may only be specified once".into(), + )); + } + config.test_restore_checkpoint = Some( + worker_runtime::restore_checkpoints::RestoreCheckpointConfig::from_path( + Path::new(&path), + ) + .map_err(ProcessError::Runtime)?, + ); + } "--bind" => { let value = take_value(&flag, inline_value, &mut args)?; config.http.bind_addr = value.parse::().map_err(|error| { @@ -465,6 +486,12 @@ where } apply_store_selection(&mut config); + #[cfg(feature = "restore-checkpoints")] + if config.test_restore_checkpoint.is_some() && config.no_store { + return Err(ProcessError::usage( + "--test-restore-checkpoint requires an fs Runtime store".into(), + )); + } Ok(Some(config)) } @@ -521,6 +548,8 @@ struct ProcessConfig { no_store: bool, backend_resource_endpoint: Option, backend_resource_token: Option, + #[cfg(feature = "restore-checkpoints")] + test_restore_checkpoint: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -542,6 +571,8 @@ impl ProcessConfig { no_store: false, backend_resource_endpoint: None, backend_resource_token: None, + #[cfg(feature = "restore-checkpoints")] + test_restore_checkpoint: None, }) } @@ -1227,6 +1258,47 @@ Auth commands: mod tests { use super::*; + #[test] + fn checkpoint_cli_is_absent_by_default() { + let config = parse_args([] as [&str; 0]).unwrap().unwrap(); + #[cfg(feature = "restore-checkpoints")] + assert!(config.test_restore_checkpoint.is_none()); + #[cfg(not(feature = "restore-checkpoints"))] + { + let _ = config; + let error = parse_args(["--test-restore-checkpoint", "never-opened.json"]).unwrap_err(); + assert!(error.to_string().contains("unknown argument")); + } + } + + #[cfg(feature = "restore-checkpoints")] + #[test] + fn checkpoint_cli_requires_valid_explicit_file_and_fs_store() { + use worker_runtime::restore_checkpoints::{ + RestoreCheckpointConfig, RestoreCheckpointStage, + }; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("checkpoint.json"); + let checkpoint = RestoreCheckpointConfig { + endpoint: "127.0.0.1:1234".parse().unwrap(), + worker_id: protocol::WorkerId::now_v7(), + stage: RestoreCheckpointStage::CommittedPreterminal, + nonce: "cli-test".into(), + }; + std::fs::write(&path, serde_json::to_vec(&checkpoint).unwrap()).unwrap(); + let args = ["--test-restore-checkpoint", path.to_str().unwrap()]; + let config = parse_args(args).unwrap().unwrap(); + assert_eq!(config.test_restore_checkpoint, Some(checkpoint)); + assert!(parse_args([args[0], args[1], "--no-store"]).is_err()); + assert!(parse_args([args[0], args[1], args[0], args[1]]).is_err()); + assert!(parse_args([args[0]]).is_err()); + std::fs::write(&path, b"private-invalid-body").unwrap(); + let error = parse_args(args).unwrap_err().to_string(); + assert!(!error.contains("private-invalid-body")); + std::fs::remove_file(&path).unwrap(); + assert!(parse_args(args).is_err()); + } + #[test] fn help_does_not_start_server() { assert!(parse_args(["--help"]).unwrap().is_none()); diff --git a/crates/worker-runtime/src/restore_checkpoints.rs b/crates/worker-runtime/src/restore_checkpoints.rs new file mode 100644 index 000000000..91321afa6 --- /dev/null +++ b/crates/worker-runtime/src/restore_checkpoints.rs @@ -0,0 +1,143 @@ +//! Explicit, ephemeral synchronization for owned Runtime process tests only. +//! +//! The nonce authenticates no Runtime action and is never a routing identity. +//! Socket failure, EOF and the fixed deadline only release the gate; the lifecycle +//! owner continues its original operation and remains responsible for cleanup. + +use std::fs::OpenOptions; +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpStream}; +use std::path::Path; +use std::time::{Duration, Instant}; + +use protocol::WorkerId; +use serde::{Deserialize, Serialize}; + +use crate::error::RuntimeError; +use crate::execution::WorkerLifecycleOperationId; + +pub const RESTORE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(8); +pub const MAX_RESTORE_CHECKPOINT_CONFIG_BYTES: usize = 4096; +pub const MAX_RESTORE_CHECKPOINT_ACK_BYTES: usize = 4096; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RestoreCheckpointStage { + PreparedNotCommitted, + CommittedPreterminal, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RestoreCheckpointCommit { + NotCommitted, + Committed, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RestoreCheckpointConfig { + pub endpoint: SocketAddr, + pub worker_id: WorkerId, + pub stage: RestoreCheckpointStage, + pub nonce: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RestoreCheckpointAck { + pub nonce: String, + pub stage: RestoreCheckpointStage, + pub runtime_id: String, + pub worker_id: WorkerId, + pub operation_id: WorkerLifecycleOperationId, + pub commit: RestoreCheckpointCommit, + pub receipt_absent: bool, + pub pid: u32, +} + +fn invalid_config() -> RuntimeError { + // Do not include serde errors, paths or input contents in process diagnostics. + RuntimeError::InvalidRequest("invalid test restore checkpoint configuration".into()) +} + +impl RestoreCheckpointConfig { + pub fn validate(&self) -> Result<(), RuntimeError> { + if !self.endpoint.ip().is_loopback() + || self.endpoint.port() == 0 + || self.nonce.is_empty() + || self.nonce.len() > 256 + || self.nonce.chars().any(char::is_control) + { + return Err(invalid_config()); + } + Ok(()) + } + + pub fn from_path(path: &Path) -> Result { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + // A FIFO must not stall startup before its regular-file check. + options.custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW); + } + let file = options.open(path).map_err(|_| invalid_config())?; + let metadata = file.metadata().map_err(|_| invalid_config())?; + if !metadata.is_file() || metadata.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 { + return Err(invalid_config()); + } + let mut bytes = Vec::new(); + file.take(MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 + 1) + .read_to_end(&mut bytes) + .map_err(|_| invalid_config())?; + if bytes.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES { + return Err(invalid_config()); + } + let config: Self = serde_json::from_slice(&bytes).map_err(|_| invalid_config())?; + config.validate()?; + Ok(config) + } +} + +pub(crate) fn wait_for_release(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) { + let _ = exchange(config, ack); +} + +fn remaining(deadline: Instant) -> std::io::Result { + deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or_else(|| std::io::Error::from(std::io::ErrorKind::TimedOut)) +} + +fn exchange(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) -> std::io::Result<()> { + let deadline = Instant::now() + RESTORE_CHECKPOINT_TIMEOUT; + let mut bytes = serde_json::to_vec(ack)?; + bytes.push(b'\n'); + if bytes.len() > MAX_RESTORE_CHECKPOINT_ACK_BYTES { + return Err(std::io::Error::from(std::io::ErrorKind::InvalidData)); + } + let mut stream = TcpStream::connect_timeout(&config.endpoint, remaining(deadline)?)?; + // Each syscall shares one budget; partial writes cannot reset the deadline. + let mut pending = bytes.as_slice(); + while !pending.is_empty() { + stream.set_write_timeout(Some(remaining(deadline)?))?; + let written = stream.write(pending)?; + if written == 0 { + return Err(std::io::Error::from(std::io::ErrorKind::WriteZero)); + } + pending = &pending[written..]; + } + stream.set_read_timeout(Some(remaining(deadline)?))?; + let mut release = [0]; + stream.read_exact(&mut release)?; + if release != [b'R'] { + return Err(std::io::Error::from(std::io::ErrorKind::InvalidData)); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/worker-runtime/src/restore_checkpoints/tests.rs b/crates/worker-runtime/src/restore_checkpoints/tests.rs new file mode 100644 index 000000000..16735224d --- /dev/null +++ b/crates/worker-runtime/src/restore_checkpoints/tests.rs @@ -0,0 +1,139 @@ +use super::*; +use std::io::{BufRead, BufReader}; +use std::net::TcpListener; +use std::thread; + +fn config(endpoint: SocketAddr) -> RestoreCheckpointConfig { + RestoreCheckpointConfig { + endpoint, + worker_id: WorkerId::now_v7(), + stage: RestoreCheckpointStage::PreparedNotCommitted, + nonce: "checkpoint-test-handshake".into(), + } +} + +fn ack(config: &RestoreCheckpointConfig) -> RestoreCheckpointAck { + RestoreCheckpointAck { + nonce: config.nonce.clone(), + stage: config.stage, + runtime_id: "checkpoint-runtime".into(), + worker_id: config.worker_id, + operation_id: WorkerLifecycleOperationId::new(), + commit: RestoreCheckpointCommit::NotCommitted, + receipt_absent: true, + pid: std::process::id(), + } +} + +#[test] +fn config_rejects_nonliteral_nonloopback_bad_stage_unknown_fields_and_invalid_worker() { + let valid = serde_json::to_value(config("127.0.0.1:1234".parse().unwrap())).unwrap(); + for (field, value) in [ + ("endpoint", "localhost:1234"), + ("endpoint", "192.0.2.1:1234"), + ("endpoint", "0.0.0.0:1234"), + ("endpoint", "127.0.0.1:0"), + ("endpoint", "[::ffff:127.0.0.1]:1234"), + ("worker_id", "not-a-worker-id"), + ("stage", "after_terminal"), + ("nonce", ""), + ("nonce", "secret\nbody"), + ("timeout", "unbounded"), + ] { + let mut invalid = valid.clone(); + invalid[field] = value.into(); + let parsed = serde_json::from_value::(invalid); + assert!( + parsed.is_err() || parsed.unwrap().validate().is_err(), + "{field}" + ); + } + for endpoint in ["127.0.0.1:1234", "[::1]:1234"] { + config(endpoint.parse().unwrap()).validate().unwrap(); + } +} + +#[test] +fn config_file_bounds_and_errors_do_not_disclose_contents() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.json"); + assert!(RestoreCheckpointConfig::from_path(&path).is_err()); + std::fs::write(&path, vec![b'x'; MAX_RESTORE_CHECKPOINT_CONFIG_BYTES + 1]).unwrap(); + assert!(RestoreCheckpointConfig::from_path(&path).is_err()); + std::fs::write( + &path, + br#"{"nonce":"private-config-body","stage":"secret"}"#, + ) + .unwrap(); + let error = RestoreCheckpointConfig::from_path(&path) + .unwrap_err() + .to_string(); + assert!(!error.contains("private-config-body")); + assert!(!error.contains("secret")); + let mut valid = config("127.0.0.1:1234".parse().unwrap()); + valid.nonce = "x".repeat(257); + assert!(valid.validate().is_err()); + valid.nonce = "valid".into(); + std::fs::write(&path, serde_json::to_vec(&valid).unwrap()).unwrap(); + assert_eq!(RestoreCheckpointConfig::from_path(&path).unwrap(), valid); + assert!(RestoreCheckpointConfig::from_path(dir.path()).is_err()); +} + +#[test] +fn explicit_release_eof_and_invalid_release_close_the_owned_stream() { + for release in [Some(b'R'), None, Some(b'X')] { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let config = config(listener.local_addr().unwrap()); + let expected = ack(&config); + let sent = expected.clone(); + let task = thread::spawn(move || exchange(&config, &sent)); + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut line = String::new(); + BufReader::new(&mut stream).read_line(&mut line).unwrap(); + assert!(line.len() <= MAX_RESTORE_CHECKPOINT_ACK_BYTES); + assert_eq!( + serde_json::from_str::(&line).unwrap(), + expected + ); + if let Some(byte) = release { + stream.write_all(&[byte]).unwrap(); + } else { + stream.shutdown(std::net::Shutdown::Write).unwrap(); + } + assert_eq!(task.join().unwrap().is_ok(), release == Some(b'R')); + let mut byte = [0]; + assert_eq!(stream.read(&mut byte).unwrap(), 0); + } +} + +#[test] +fn absent_listener_resumes_without_waiting_or_retrying() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let config = config(listener.local_addr().unwrap()); + drop(listener); + let start = Instant::now(); + wait_for_release(&config, &ack(&config)); + assert!(start.elapsed() < Duration::from_secs(2)); +} + +#[test] +fn silent_peer_hits_fixed_deadline_and_stream_is_dropped() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let config = config(listener.local_addr().unwrap()); + let start = Instant::now(); + let task = thread::spawn(move || exchange(&config, &ack(&config))); + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(12))) + .unwrap(); + let mut reader = BufReader::new(&mut stream); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + assert!(task.join().unwrap().is_err()); + assert!(start.elapsed() >= RESTORE_CHECKPOINT_TIMEOUT - Duration::from_millis(100)); + assert!(start.elapsed() < RESTORE_CHECKPOINT_TIMEOUT + Duration::from_secs(3)); + assert_eq!(reader.read(&mut [0]).unwrap(), 0); +} diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index f55d359f8..4f9141cc1 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "restore-checkpoints")] +mod restore_checkpoint; mod restore_continuation; use crate::catalog::{ ConfigBundleRef, CreateWorkerRequest, LogicalWorkdirAttachment, ProfileSelector, @@ -2776,6 +2778,12 @@ impl Runtime { worker_state, workdir_attachments, } => { + #[cfg(feature = "restore-checkpoints")] + self.test_restore_checkpoint( + worker_ref, + operation_id, + crate::restore_checkpoints::RestoreCheckpointStage::PreparedNotCommitted, + ); let commit = self.commit_restored_worker_execution( worker_ref, worker_state.clone(), @@ -2784,6 +2792,12 @@ impl Runtime { ); match commit { Ok(_) => { + #[cfg(feature = "restore-checkpoints")] + self.test_restore_checkpoint( + worker_ref, + operation_id, + crate::restore_checkpoints::RestoreCheckpointStage::CommittedPreterminal, + ); if let Err(error) = backend.activate_restored_worker(operation_id, worker_ref) { @@ -5419,6 +5433,8 @@ impl std::fmt::Debug for BackendResourceClientRef { #[derive(Debug)] struct RuntimeState { + #[cfg(feature = "restore-checkpoints")] + restore_checkpoint: Option, display_name: Option, backend: RuntimeBackendKind, /// Backend-bound stable identity used for cross-boundary retention evidence. @@ -5466,6 +5482,8 @@ impl RuntimeState { display_name, backend: RuntimeBackendKind::Memory, runtime_identity: None, + #[cfg(feature = "restore-checkpoints")] + restore_checkpoint: None, persistence: RuntimePersistence::Memory, status: RuntimeStatus::Running, execution_backend: None, @@ -5500,6 +5518,8 @@ impl RuntimeState { display_name, backend: RuntimeBackendKind::FsStore, runtime_identity: None, + #[cfg(feature = "restore-checkpoints")] + restore_checkpoint: None, persistence: RuntimePersistence::Fs(store), status: RuntimeStatus::Running, execution_backend: None, @@ -5550,6 +5570,8 @@ impl RuntimeState { display_name: persisted.display_name, backend: RuntimeBackendKind::FsStore, runtime_identity: None, + #[cfg(feature = "restore-checkpoints")] + restore_checkpoint: None, persistence: RuntimePersistence::Fs(store), status: persisted.status, execution_backend: None, @@ -7553,6 +7575,10 @@ fn subscription_worker_state(status: WorkerStatus) -> SubscriptionWorkerState { #[cfg(test)] mod tests { + #[cfg(feature = "restore-checkpoints")] + mod restore_checkpoint_tests { + include!("runtime/restore_checkpoint_tests.rs"); + } use super::*; use crate::catalog::{ ConfigBundleRef, MaterializerKind, ProfileSelector, RepositoryMaterializationContext, diff --git a/crates/worker-runtime/src/runtime/restore_checkpoint.rs b/crates/worker-runtime/src/runtime/restore_checkpoint.rs new file mode 100644 index 000000000..96da2eeb9 --- /dev/null +++ b/crates/worker-runtime/src/runtime/restore_checkpoint.rs @@ -0,0 +1,111 @@ +use super::*; +use crate::restore_checkpoints::{ + RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage, + wait_for_release, +}; + +impl Runtime { + /// Install an explicitly selected, one-shot process-test gate after normal startup. + /// It is not persisted or consulted by admission, routing or recovery authority. + pub fn install_test_restore_checkpoint( + &self, + config: RestoreCheckpointConfig, + ) -> Result<(), RuntimeError> { + config.validate()?; + let mut state = self.lock()?; + if !matches!(state.persistence, RuntimePersistence::Fs(_)) { + return Err(RuntimeError::InvalidRequest( + "test restore checkpoint requires an fs Runtime store".into(), + )); + } + state.restore_checkpoint = Some(config); + Ok(()) + } + + // The lifecycle caller holds the per-Worker operation lock. No Runtime-state + // mutex is retained while waiting; the real backend candidate remains owned by + // the same operation throughout this synchronous call. + pub(super) fn test_restore_checkpoint( + &self, + worker_ref: &WorkerRef, + operation_id: WorkerLifecycleOperationId, + stage: RestoreCheckpointStage, + ) { + let config = { + let Ok(mut state) = self.lock() else { + return; + }; + let slot = &mut state.restore_checkpoint; + if !slot.as_ref().is_some_and(|config| { + config.worker_id == worker_ref.worker_id && config.stage == stage + }) { + return; + } + // Consume even if canonical verification or the handshake fails. + slot.take().expect("matched checkpoint") + }; + let Ok(ack) = self.restore_checkpoint_ack(&config, worker_ref, operation_id) else { + return; + }; + wait_for_release(&config, &ack); + } + + pub(super) fn restore_checkpoint_ack( + &self, + config: &RestoreCheckpointConfig, + worker_ref: &WorkerRef, + operation_id: WorkerLifecycleOperationId, + ) -> Result { + let invalid = || { + RuntimeError::InvalidRequest("test restore checkpoint authority did not match".into()) + }; + let state = self.lock()?; + let RuntimePersistence::Fs(store) = &state.persistence else { + return Err(invalid()); + }; + // The existing helper validates the exact bounded canonical aggregate and + // syncs that same file plus its owning directories. Never manufacture a + // journal marker or infer commit from an in-memory projection. + let record = store.read_durable_worker_record(worker_ref)?; + let PersistedWorkerExecutionState::ReconciliationRequired( + PersistedWorkerLifecycleOperation::Restore(pending), + ) = &record.execution_state + else { + return Err(invalid()); + }; + let expected = match config.stage { + RestoreCheckpointStage::PreparedNotCommitted => RestoreCommitFact::NotCommitted, + RestoreCheckpointStage::CommittedPreterminal => RestoreCommitFact::Committed, + }; + if config.worker_id != worker_ref.worker_id + || pending.operation_id != operation_id + || pending.commit != expected + { + return Err(invalid()); + } + let guard = &record.restore_guard; + let id = guard.active_request_id.as_ref().ok_or_else(invalid)?; + let owner = guard.owners.get(id).ok_or_else(invalid)?; + if owner.operation_id != operation_id + || owner.request.request_id != *id + || owner.receipt.is_some() + { + return Err(invalid()); + } + let runtime_id = state.runtime_identity.clone().ok_or_else(invalid)?; + Ok(RestoreCheckpointAck { + nonce: config.nonce.clone(), + stage: config.stage, + runtime_id, + worker_id: record.worker_id, + operation_id, + commit: match expected { + RestoreCommitFact::NotCommitted => RestoreCheckpointCommit::NotCommitted, + RestoreCommitFact::Committed => RestoreCheckpointCommit::Committed, + RestoreCommitFact::LegacyUnknown => unreachable!(), + }, + receipt_absent: true, + pid: std::process::id(), + }) + } +} diff --git a/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs b/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs new file mode 100644 index 000000000..7369d6c6b --- /dev/null +++ b/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs @@ -0,0 +1,296 @@ +use super::*; +use crate::restore_checkpoints::{ + RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage, +}; +use std::io::{BufRead, BufReader, Write}; +use std::net::TcpListener; +use std::time::Duration; + +#[test] +fn checkpoint_stages_ack_canonical_decision_and_eof_resumes_same_operation_once() { + // Component scope: durable Runtime with a substituted execution backend. + // Real prepared Controllers and process death are owned by the E2E suite. + for (stage, release) in [ + (RestoreCheckpointStage::PreparedNotCommitted, "release"), + (RestoreCheckpointStage::CommittedPreterminal, "eof"), + (RestoreCheckpointStage::CommittedPreterminal, "timeout"), + ] { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("checkpoint-runtime"), + backend.clone(), + ) + .unwrap(); + runtime.bind_runtime_identity("checkpoint-runtime").unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("checkpoint original operation")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + runtime + .install_test_restore_checkpoint(RestoreCheckpointConfig { + endpoint: listener.local_addr().unwrap(), + worker_id: worker.worker_id, + stage, + nonce: "runtime-stage-test".into(), + }) + .unwrap(); + let cloned = runtime.clone(); + let worker_ref = worker.worker_ref.clone(); + let sent_request = request.clone(); + let (sent, received) = std::sync::mpsc::channel(); + let task = std::thread::spawn(move || { + sent.send(cloned.restore_worker_operation(&worker_ref, sent_request)) + .unwrap(); + }); + listener.set_nonblocking(true).unwrap(); + let stream = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + let listener = tokio::net::TcpListener::from_std(listener).unwrap(); + let (stream, _) = tokio::time::timeout(Duration::from_secs(5), listener.accept()) + .await + .expect("checkpoint did not connect") + .unwrap(); + stream.into_std().unwrap() + }); + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(3))) + .unwrap(); + let mut reader = BufReader::new(stream); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + let ack: RestoreCheckpointAck = serde_json::from_str(&line).unwrap(); + assert_eq!(ack.stage, stage); + assert_eq!(ack.nonce, "runtime-stage-test"); + assert_eq!(ack.runtime_id, "checkpoint-runtime"); + assert_eq!(ack.worker_id, worker.worker_id); + assert_eq!(ack.pid, std::process::id()); + assert!(ack.receipt_absent); + assert_eq!( + ack.commit, + match stage { + RestoreCheckpointStage::PreparedNotCommitted => + RestoreCheckpointCommit::NotCommitted, + RestoreCheckpointStage::CommittedPreterminal => RestoreCheckpointCommit::Committed, + } + ); + assert_eq!( + backend.restore_operation_ids.lock().unwrap().as_slice(), + &[ack.operation_id] + ); + assert!(backend.activation_operation_ids.lock().unwrap().is_empty()); + assert!(received.try_recv().is_err()); + let canonical = runtime_store(&runtime) + .read_durable_worker_record(&worker.worker_ref) + .unwrap(); + let id = canonical.restore_guard.active_request_id.as_ref().unwrap(); + assert_eq!(id, &request.request_id); + assert!(canonical.restore_guard.owners[id].receipt.is_none()); + // Release, EOF and timeout all resume the original lifecycle owner. + let start = std::time::Instant::now(); + if release == "release" { + reader.get_mut().write_all(b"R").unwrap(); + } else if release == "eof" { + reader + .get_mut() + .shutdown(std::net::Shutdown::Write) + .unwrap(); + } + let result = received + .recv_timeout(Duration::from_secs(12)) + .unwrap() + .unwrap(); + assert!(start.elapsed() < Duration::from_secs(11)); + task.join().unwrap(); + assert_eq!(result.state, WorkerRestoreState::Accepted); + assert_eq!( + backend.activation_operation_ids.lock().unwrap().as_slice(), + &[ack.operation_id] + ); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + result + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert!(runtime.lock().unwrap().restore_checkpoint.is_none()); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let next = runtime.test_restore_request(&worker.worker_ref); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, next) + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 2); + } +} + +#[test] +fn checkpoint_disabled_mismatched_or_unreachable_does_not_change_restore_result() { + for mode in ["disabled", "mismatch", "unreachable"] { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("checkpoint-runtime"), + backend.clone(), + ) + .unwrap(); + runtime.bind_runtime_identity("checkpoint-runtime").unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("checkpoint absent")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + if mode != "disabled" { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + drop(listener); + runtime + .install_test_restore_checkpoint(RestoreCheckpointConfig { + endpoint, + worker_id: if mode == "mismatch" { + WorkerId::now_v7() + } else { + worker.worker_id + }, + stage: RestoreCheckpointStage::PreparedNotCommitted, + nonce: "absent-test".into(), + }) + .unwrap(); + } + let result = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(result.state, WorkerRestoreState::Accepted, "{mode}"); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + result + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!( + runtime.lock().unwrap().restore_checkpoint.is_some(), + mode == "mismatch" + ); + } +} + +#[test] +fn checkpoint_verification_requires_canonical_worker_operation_commit_and_absent_receipt() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("runtime"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(&root).with_runtime_id("checkpoint-runtime"), + backend.clone(), + ) + .unwrap(); + runtime.bind_runtime_identity("checkpoint-runtime").unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("checkpoint validation")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + *backend.restore_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::Errored( + WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, "not yet connected"), + )); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + let operation = backend.restore_operation_ids.lock().unwrap()[0]; + let mut config = RestoreCheckpointConfig { + endpoint: "127.0.0.1:1234".parse().unwrap(), + worker_id: worker.worker_id, + stage: RestoreCheckpointStage::PreparedNotCommitted, + nonce: "verification-test".into(), + }; + let path = root + .join("workers") + .join(worker.worker_id.to_string()) + .join("worker.json"); + let before = std::fs::read(&path).unwrap(); + assert!( + runtime + .restore_checkpoint_ack(&config, &worker.worker_ref, operation) + .is_ok() + ); + assert!( + runtime + .restore_checkpoint_ack( + &config, + &worker.worker_ref, + WorkerLifecycleOperationId::new() + ) + .is_err() + ); + config.worker_id = WorkerId::now_v7(); + assert!( + runtime + .restore_checkpoint_ack(&config, &worker.worker_ref, operation) + .is_err() + ); + config.worker_id = worker.worker_id; + config.stage = RestoreCheckpointStage::CommittedPreterminal; + assert!( + runtime + .restore_checkpoint_ack(&config, &worker.worker_ref, operation) + .is_err() + ); + config.stage = RestoreCheckpointStage::PreparedNotCommitted; + runtime_store(&runtime).fail_next_worker_reconcile_sync(); + assert!( + runtime + .restore_checkpoint_ack(&config, &worker.worker_ref, operation) + .is_err() + ); + assert_eq!( + std::fs::read(&path).unwrap(), + before, + "verification must not write authority" + ); + // In-memory equality must not hide an unreadable canonical aggregate. + std::fs::write(&path, b"unreadable canonical aggregate").unwrap(); + assert!( + runtime + .restore_checkpoint_ack(&config, &worker.worker_ref, operation) + .is_err() + ); + assert_eq!( + std::fs::read(&path).unwrap(), + b"unreadable canonical aggregate" + ); + std::fs::write(&path, before).unwrap(); + backend.restore_result.lock().unwrap().take(); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + assert!( + runtime + .restore_checkpoint_ack(&config, &worker.worker_ref, operation) + .is_err(), + "terminal operation cannot ack a preterminal checkpoint" + ); +} diff --git a/tests/e2e/Cargo.toml b/tests/e2e/Cargo.toml index 53b60d5e3..000155fe2 100644 --- a/tests/e2e/Cargo.toml +++ b/tests/e2e/Cargo.toml @@ -10,6 +10,7 @@ autotests = false default = [] e2e = [] fixture-support = ["yoi-workspace-server/test-support", "worker-runtime/test-support"] +restore-checkpoints = ["fixture-support", "worker-runtime/restore-checkpoints"] [dependencies] libc.workspace = true diff --git a/tests/e2e/README.md b/tests/e2e/README.md index a54eb8517..4622758ab 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -39,6 +39,24 @@ process終了ケースはそのAccepted receipt確認後に環境が所有する これは**commit済みかつterminal receipt保存後**のprocess終了であり、未commit候補やcommit直後・receipt前の終了証明ではない。固定Codex資産へSubmit/Notifyを送らないため、通常入力・Paused admissionの製品E2Eの証明でもない。検証jobはこの明示targetを選択する必要があり、通常suiteだけでは代替しない。 +## Opt-in Runtime checkpoint process試験 + +上のfeatureless製品4case（target内の既存seed projection試験1件とは別）に加え、`restore-checkpoints`を明示した実Runtimeで2つのlive windowを試験する。このfeatureはoffline fixture-supportと独立しており、通常RuntimeではCLI入口も環境変数による入口も無い。 + +```sh +# 同期を有効にしたRuntimeだけを専用targetへ。通常製品binaryを上書きしない。 +cargo build --locked -p worker-runtime --bin yoi-runtime \ + --features restore-checkpoints --target-dir target/e2e-checkpoint-products +cargo test --locked -p yoi-e2e --features e2e,restore-checkpoints \ + --test worker_restore checkpoint:: -- --test-threads=1 +``` + +`WorkerE2eEnvironment::use_checkpoint_runtime`はその環境のRuntime binary/hashと、隔離root内の明示configだけを選択する。global provider・環境変数・通常起動への暗黙設定は変更しない。configは一度だけCLIへ渡し、再起動時は渡さない。test側のlink featureをfeatureless製品binaryへ混入させない。 + +共有harnessはloopback ackのstage/Worker/operation/PID/nonceを確認し、直ちに所有childをkill/reapする。**offline正本を確認してから**同roots/identity/trustを再起動するため、ack後のdeadlineとの競合を成功扱いしない。NotCommittedの実私有候補は同operationのcleanup-only→RolledBack/Stopped、Committed/preterminalは同じdecision→実protocol/full Session snapshot→Acceptedへ回収する。元のrequest/token、owner数1、同operation、stale fresh intent拒否、保存source hash、restore run path数、最終cleanupを検証する。JSONでpending状態を作らず、seedは初回のみ。 + +明示解除・peer EOF・observer scopeの意図的panicも試験する。gateの8秒deadlineとscoped HTTP callの15秒deadlineが待機を制限し、通常operationの継続後に環境のRAIIがprocess/rootを回収する。これはtest scope失敗の回収証明であり、OSによるharness強制終了や電源断の一般的な保証ではない。原fixtureへのSubmit/Notifyや実モデル呼出しは行わない。checkpoint ack/at-death/recovery、kill/reap、binary/hashとcleanup artifactはfeaturelessログと分けて報告する。 + 通常のharness回帰試験（製品processは起動しない）: ```sh diff --git a/tests/e2e/src/worker_environment.rs b/tests/e2e/src/worker_environment.rs index 749ef4c83..6ff2c40cb 100644 --- a/tests/e2e/src/worker_environment.rs +++ b/tests/e2e/src/worker_environment.rs @@ -266,6 +266,8 @@ pub struct WorkerE2eEnvironment { access_token: Option, client: Client, binaries: WorkerBinaryProviderInfo, + #[cfg(feature = "restore-checkpoints")] + restore_checkpoint_config: Option, } impl WorkerE2eEnvironment { @@ -331,11 +333,49 @@ impl WorkerE2eEnvironment { access_token: None, client, binaries, + #[cfg(feature = "restore-checkpoints")] + restore_checkpoint_config: None, }; environment.write_metadata("created")?; Ok(environment) } + /// Select the separately built test-enabled Runtime for this environment + /// only. The checkpoint file must already belong to its isolated root. + #[cfg(feature = "restore-checkpoints")] + pub fn use_checkpoint_runtime(&mut self, binary: PathBuf, config: &Path) -> Result<()> { + if self.runtime.is_some() || self.server.is_some() { + return Err(HarnessError::Protocol( + "checkpoint selection requires an unstarted environment".into(), + )); + } + let config = config.canonicalize()?; + if !config.starts_with(self.root.canonicalize()?) || !config.is_file() { + return Err(HarnessError::Protocol( + "checkpoint config is outside the owned environment".into(), + )); + } + let record = binary_record( + "yoi-runtime", + binary.canonicalize()?, + "restore-checkpoints-opt-in".into(), + None, + )?; + let runtime = self + .binaries + .binaries + .iter_mut() + .find(|record| record.name == "yoi-runtime") + .ok_or_else(|| HarnessError::Protocol("missing Runtime binary record".into()))?; + *runtime = record; + self.restore_checkpoint_config = Some(config); + fs::write( + self.artifacts_dir.join("binary-provider.json"), + serde_json::to_vec_pretty(&self.binaries)?, + )?; + Ok(()) + } + pub fn start(&mut self) -> Result<()> { self.start_backend()?; self.start_runtime() @@ -639,6 +679,10 @@ impl WorkerE2eEnvironment { "--fs-root", self.runtime_data_dir.to_string_lossy().as_ref(), ]); + #[cfg(feature = "restore-checkpoints")] + if let Some(config) = self.restore_checkpoint_config.take() { + command.arg("--test-restore-checkpoint").arg(config); + } let runtime = ManagedProcess::spawn( "runtime", command, @@ -684,6 +728,13 @@ impl WorkerE2eEnvironment { /// Kill/reap only the Runtime child owned by this environment, retaining roots /// and identities. Restart neither seeds storage nor rotates restore requests. pub fn crash_and_restart_runtime(&mut self) -> Result<()> { + self.crash_runtime()?; + self.start_runtime_process() + } + + /// Leave the owned Runtime reaped and offline for read-only canonical crash + /// inspection before restart. No root, identity or request is recreated. + pub fn crash_runtime(&mut self) -> Result<()> { let runtime = self .runtime .as_mut() @@ -708,6 +759,17 @@ impl WorkerE2eEnvironment { } self.runtime_url = None; self.runtime_shutdown = WorkerProcessShutdownOutcome::NotStarted; + Ok(()) + } + + /// Restart the previously initialized identity without seed or checkpoint flags. + #[cfg(feature = "restore-checkpoints")] + pub fn restart_runtime_after_crash(&mut self) -> Result<()> { + if self.runtime.is_some() || self.runtime_public_identity.is_none() { + return Err(HarnessError::Protocol( + "Runtime is not an offline initialized child".into(), + )); + } self.start_runtime_process() } @@ -794,6 +856,37 @@ impl WorkerE2eEnvironment { ) } + /// Build one authenticated call that may run in a scoped test thread while + /// the owner observes a checkpoint. The HTTP request deadline bounds joining + /// that thread even on test failure; credentials never enter artifacts. + #[cfg(feature = "restore-checkpoints")] + pub fn pending_restore_call( + &self, + worker_id: &str, + request_id: &str, + expected_observation_token: &str, + ) -> Result Result + Send + 'static + use<>> { + let token = self.access_token.as_deref().ok_or_else(|| { + HarnessError::Protocol("Backend access token is not initialized".into()) + })?; + let request = self + .client + .post(self.worker_url(worker_id, "restore")?) + .bearer_auth(token) + .timeout(STARTUP_WAIT) + .json(&serde_json::json!({"request_id": request_id, + "expected_observation_token": expected_observation_token})) + .build() + .map_err(|_| HarnessError::Protocol("build pending restore request".into()))?; + let client = self.client.clone(); + Ok(move || { + let response = client + .execute(request) + .map_err(|_| HarnessError::Protocol("pending restore transport failed".into()))?; + decode_api_response(response, "pending restore") + }) + } + /// Forward one unchanged public restore request through an owned loopback /// relay, consume its complete upstream response, and close without delivering /// any response bytes. The returned JSON is relay evidence, not caller success. diff --git a/tests/e2e/tests/worker_restore.rs b/tests/e2e/tests/worker_restore.rs index 75ef29cd2..accb62ed8 100644 --- a/tests/e2e/tests/worker_restore.rs +++ b/tests/e2e/tests/worker_restore.rs @@ -1,4 +1,7 @@ //! Fixed, externally supplied Worker restore. Never generates a fixture or sends model input. +#[cfg(feature = "restore-checkpoints")] +#[path = "worker_restore/checkpoint.rs"] +mod checkpoint; #[path = "worker_restore/fixture.rs"] mod fixture; #[path = "worker_restore/seed.rs"] diff --git a/tests/e2e/tests/worker_restore/checkpoint.rs b/tests/e2e/tests/worker_restore/checkpoint.rs new file mode 100644 index 000000000..e55e815be --- /dev/null +++ b/tests/e2e/tests/worker_restore/checkpoint.rs @@ -0,0 +1,496 @@ +//! Opt-in real-process checkpoint cases; never substitute the featureless lane. +use super::*; +use std::collections::BTreeMap; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::path::{Path, PathBuf}; +use worker_runtime::restore_checkpoints::{ + MAX_RESTORE_CHECKPOINT_ACK_BYTES, RestoreCheckpointAck, RestoreCheckpointCommit, + RestoreCheckpointConfig, RestoreCheckpointStage, +}; + +struct Peer { + listener: TcpListener, + config: RestoreCheckpointConfig, + connection: Option, +} + +impl Peer { + fn configure( + environment: &mut WorkerE2eEnvironment, + fixture: &Fixture, + stage: RestoreCheckpointStage, + ) -> Result { + let listener = TcpListener::bind("127.0.0.1:0")?; + listener.set_nonblocking(true)?; + let config = RestoreCheckpointConfig { + endpoint: listener.local_addr()?, + worker_id: fixture::support(fixture.manifest.worker_id.parse())?, + stage, + nonce: format!( + "owned-checkpoint-{}-{}", + std::process::id(), + listener.local_addr()?.port() + ), + }; + let path = environment.root.join("restore-checkpoint.json"); + fs::write(&path, serde_json::to_vec(&config)?)?; + let provider = yoi_e2e::worker_binary_provider_info()?; + let binary = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../target/e2e-checkpoint-products") + .join(provider.profile) + .join("yoi-runtime"); + environment.use_checkpoint_runtime(binary, &path)?; + Ok(Self { + listener, + config, + connection: None, + }) + } + + fn acknowledge(&mut self, environment: &WorkerE2eEnvironment) -> Result { + let deadline = Instant::now() + Duration::from_secs(15); + let (mut stream, address) = loop { + match self.listener.accept() { + Ok(value) => break value, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if Instant::now() >= deadline { + return Err(failure("checkpoint ack deadline exceeded")); + } + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => return Err(error.into()), + } + }; + if !address.ip().is_loopback() { + return Err(failure("non-loopback checkpoint peer")); + } + let mut bytes = Vec::new(); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(failure("checkpoint ack read deadline exceeded")); + } + stream.set_read_timeout(Some(remaining))?; + let mut byte = [0]; + stream.read_exact(&mut byte)?; + if byte == [b'\n'] { + break; + } + bytes.push(byte[0]); + if bytes.len() > MAX_RESTORE_CHECKPOINT_ACK_BYTES { + return Err(failure("checkpoint ack oversized")); + } + } + let ack: RestoreCheckpointAck = serde_json::from_slice(&bytes)?; + if ack.nonce != self.config.nonce + || ack.stage != self.config.stage + || ack.worker_id != self.config.worker_id + || ack.runtime_id != environment.runtime_id + || Some(ack.pid) != environment.runtime_pid() + || !ack.receipt_absent + { + return Err(failure( + "checkpoint ack does not match owned process and stage", + )); + } + fs::write( + environment.artifacts_dir.join("checkpoint-ack.json"), + serde_json::to_vec_pretty(&ack)?, + )?; + self.connection = Some(stream); + Ok(ack) + } + + fn release(&mut self) -> Result<()> { + let mut stream = self + .connection + .take() + .ok_or_else(|| failure("checkpoint not acknowledged"))?; + stream.set_write_timeout(Some(Duration::from_secs(1)))?; + stream.write_all(b"R")?; + Ok(()) + } +} + +fn aggregate_path(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> PathBuf { + environment + .runtime_data_dir + .join("runtime/workers") + .join(&fixture.manifest.worker_id) +} + +fn canonical(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> Result { + let mut bytes = Vec::new(); + fs::File::open(aggregate_path(environment, fixture).join("worker.json"))? + .take(8 * 1024 * 1024 + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > 8 * 1024 * 1024 { + return Err(failure("canonical aggregate oversized")); + } + Ok(serde_json::from_slice(&bytes)?) +} + +fn saved_source_hashes( + environment: &WorkerE2eEnvironment, + fixture: &Fixture, +) -> Result> { + fn visit(root: &Path, path: &Path, files: &mut BTreeMap) -> Result<()> { + if files.len() > 128 { + return Err(failure("saved source file budget exceeded")); + } + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() { + return Err(failure("unexpected saved source symlink")); + } + if metadata.is_dir() { + for entry in fs::read_dir(path)? { + visit(root, &entry?.path(), files)?; + } + } else if metadata.is_file() { + if metadata.len() > 8 * 1024 * 1024 { + return Err(failure("saved source byte budget exceeded")); + } + use sha2::{Digest, Sha256}; + let digest: String = Sha256::digest(fs::read(path)?) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + files.insert( + path.strip_prefix(root) + .unwrap() + .to_string_lossy() + .into_owned(), + digest, + ); + } else { + return Err(failure("unexpected saved source entry type")); + } + Ok(()) + } + let root = aggregate_path(environment, fixture); + let mut files = BTreeMap::new(); + for name in ["metadata.json", "session"] { + visit(&root, &root.join(name), &mut files)?; + } + if files.len() < 2 { + return Err(failure("saved source inventory is incomplete")); + } + Ok(files) +} + +fn restore_runs(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> Result> { + let runs = aggregate_path(environment, fixture).join("runs"); + if !runs.exists() { + return Ok(Vec::new()); + } + let mut names = Vec::new(); + for entry in fs::read_dir(runs)? { + let entry = entry?; + if entry.file_name().to_string_lossy().starts_with("restore-") { + if !entry.file_type()?.is_dir() { + return Err(failure("restore run is not a directory")); + } + names.push(entry.file_name().to_string_lossy().into_owned()); + } + } + names.sort(); + Ok(names) +} + +#[test] +fn acknowledged_not_committed_candidate_process_death_recovers_same_request_as_rollback() +-> Result<()> { + crash_window(RestoreCheckpointStage::PreparedNotCommitted) +} + +#[test] +fn acknowledged_committed_preterminal_process_death_reconstructs_same_decision() -> Result<()> { + crash_window(RestoreCheckpointStage::CommittedPreterminal) +} + +fn crash_window(stage: RestoreCheckpointStage) -> Result<()> { + let fixture = Fixture::load()?; + let result = (|| { + let mut environment = WorkerE2eEnvironment::new()?; + let mut peer = Peer::configure(&mut environment, &fixture, stage)?; + let mut installed = seed::start_in(&fixture, environment)?; + let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?; + let token = stopped + .restore_observation_token + .ok_or_else(|| failure("missing original observation"))?; + let original_sources = saved_source_hashes(&installed.environment, &fixture)?; + let request_id = "owned-process-checkpoint-request"; + let call = installed.environment.pending_restore_call( + &fixture.manifest.worker_id, + request_id, + &token, + )?; + let ack = std::thread::scope(|scope| -> Result { + let request = scope.spawn(call); + let ack = peer.acknowledge(&installed.environment)?; + installed.environment.crash_runtime()?; + // With the actual child reaped there can be no deadline race while + // inspecting the exact decision and nonterminal owner at process death. + let saved = canonical(&installed.environment, &fixture)?; + let op = saved + .pointer("/execution_state/execution/intent") + .ok_or_else(|| failure("checkpoint lacks canonical pending operation"))?; + let owner = &saved["restore_guard"]["owners"][request_id]; + assert_eq!(op["operation_id"], serde_json::to_value(ack.operation_id)?); + assert_eq!(owner["operation_id"], op["operation_id"]); + assert!(owner["receipt"].is_null()); + assert_eq!(owner["request"]["expected_observation_token"], token); + assert_eq!(op["source_repair_possible"], false); + let committed = stage == RestoreCheckpointStage::CommittedPreterminal; + assert_eq!( + op["commit"], + if committed { + "committed" + } else { + "not_committed" + } + ); + assert_eq!( + ack.commit, + if committed { + RestoreCheckpointCommit::Committed + } else { + RestoreCheckpointCommit::NotCommitted + } + ); + assert_eq!( + saved_source_hashes(&installed.environment, &fixture)?, + original_sources + ); + assert_eq!( + restore_runs(&installed.environment, &fixture)?, + vec![format!("restore-{}", ack.operation_id)] + ); + fs::write( + installed + .environment + .artifacts_dir + .join("checkpoint-at-death.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "runtime_reaped_before_inspection": installed.environment.runtime_pid().is_none(), + "operation_id": op["operation_id"], "commit": op["commit"], + "receipt_absent": owner["receipt"].is_null(), + "source_repair_possible": op["source_repair_possible"], + "same_original_request_and_token": true, + "restore_runs": restore_runs(&installed.environment, &fixture)?, + "saved_source_hashes": saved_source_hashes(&installed.environment, &fixture)?, + }))?, + )?; + let outcome = request + .join() + .map_err(|_| failure("public restore caller panicked"))?; + if let Ok(value) = outcome { + let receipt: WorkerRestoreResponse = serde_json::from_value(value)?; + assert_ne!( + receipt.result.state, + WorkerRestoreState::Accepted, + "no caller Accepted may precede either checkpoint kill" + ); + } + installed.environment.restart_runtime_after_crash()?; + assert_ne!(installed.environment.runtime_pid(), Some(ack.pid)); + Ok(ack) + })?; + drop(peer); + let recovered = installed.environment.restore_worker( + &fixture.manifest.worker_id, + request_id, + &token, + )?; + let receipt: WorkerRestoreResponse = serde_json::from_value(recovered.clone())?; + let committed = stage == RestoreCheckpointStage::CommittedPreterminal; + assert_eq!( + receipt.result.state, + if committed { + WorkerRestoreState::Accepted + } else { + WorkerRestoreState::RolledBack + } + ); + assert_eq!( + installed.environment.restore_worker( + &fixture.manifest.worker_id, + request_id, + &token + )?, + recovered + ); + let final_saved = canonical(&installed.environment, &fixture)?; + let owners = final_saved["restore_guard"]["owners"] + .as_object() + .ok_or_else(|| failure("missing canonical owners"))?; + assert_eq!(owners.len(), 1); + assert_eq!( + owners[request_id]["operation_id"], + serde_json::to_value(ack.operation_id)? + ); + assert_eq!( + owners[request_id]["request"]["expected_observation_token"], + token + ); + assert_eq!( + owners[request_id]["receipt"]["state"], + if committed { "accepted" } else { "rolled_back" } + ); + let stale = installed.environment.restore_worker( + &fixture.manifest.worker_id, + "new-stale-checkpoint-intent", + &token, + ); + if !matches!(stale, Err(error) if error.to_string().contains("HTTP 409")) { + return Err(failure("fresh stale intent was not rejected")); + } + fs::write( + installed + .environment + .artifacts_dir + .join("checkpoint-recovery.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "ack": ack, "recovered_state": receipt.result.state, + "canonical_operation_id": owners[request_id]["operation_id"], + "owner_count": owners.len(), "same_original_request_and_token": true, + "saved_source_hashes_before": original_sources, + "saved_source_hashes_after": saved_source_hashes(&installed.environment, &fixture)?, + "restore_runs_after": restore_runs(&installed.environment, &fixture)?, + }))?, + )?; + if committed { + assert_single_state(&installed.environment, &fixture, "idle")?; + assert_eq!( + restore_runs(&installed.environment, &fixture)?, + vec![format!("restore-{}", ack.operation_id)] + ); + let mut subscription = WorkerSessionSubscription::connect( + &installed.environment, + &fixture.manifest.worker_id, + &installed.snapshot, + )?; + subscription.assert_alive()?; + stop_and_cleanup(installed, vec![subscription], &fixture) + } else { + assert_single_state(&installed.environment, &fixture, "stopped")?; + assert!(restore_runs(&installed.environment, &fixture)?.is_empty()); + assert_eq!( + saved_source_hashes(&installed.environment, &fixture)?, + original_sources + ); + let report = installed.environment.cleanup()?; + assert!(report.cleanup_success && !report.fixture_root_exists_after); + Ok(()) + } + })(); + fixture.source.assert_unchanged()?; + result +} + +#[test] +fn acknowledged_checkpoint_peer_disconnect_resumes_original_operation_and_tears_down() -> Result<()> +{ + observer_completion(false) +} + +#[test] +fn acknowledged_checkpoint_explicit_release_resumes_original_operation_and_tears_down() -> Result<()> +{ + observer_completion(true) +} + +fn observer_completion(release: bool) -> Result<()> { + let fixture = Fixture::load()?; + let result = (|| { + let mut environment = WorkerE2eEnvironment::new()?; + let mut peer = Peer::configure( + &mut environment, + &fixture, + RestoreCheckpointStage::PreparedNotCommitted, + )?; + let installed = seed::start_in(&fixture, environment)?; + let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?; + let token = stopped + .restore_observation_token + .ok_or_else(|| failure("missing observation"))?; + let call = installed.environment.pending_restore_call( + &fixture.manifest.worker_id, + "peer-disconnect-request", + &token, + )?; + std::thread::scope(|scope| -> Result<()> { + let request = scope.spawn(call); + peer.acknowledge(&installed.environment)?; + // EOF models an abandoned observer, not a successful restore result. + if release { + peer.release()?; + } else { + drop(peer); + } + let value = request + .join() + .map_err(|_| failure("public caller panicked"))??; + let response: WorkerRestoreResponse = serde_json::from_value(value)?; + assert_eq!(response.result.state, WorkerRestoreState::Accepted); + Ok(()) + })?; + let mut subscription = WorkerSessionSubscription::connect( + &installed.environment, + &fixture.manifest.worker_id, + &installed.snapshot, + )?; + subscription.assert_alive()?; + stop_and_cleanup(installed, vec![subscription], &fixture) + })(); + fixture.source.assert_unchanged()?; + result +} + +#[test] +fn acknowledged_checkpoint_failed_observer_scope_has_bounded_owned_cleanup() -> Result<()> { + let fixture = Fixture::load()?; + let environment = WorkerE2eEnvironment::new()?; + let root = environment.root.clone(); + let artifacts = environment.artifacts_dir.clone(); + let started = Instant::now(); + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<()> { + let mut environment = environment; + let mut peer = Peer::configure( + &mut environment, + &fixture, + RestoreCheckpointStage::PreparedNotCommitted, + )?; + let installed = seed::start_in(&fixture, environment)?; + let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?; + let token = stopped + .restore_observation_token + .ok_or_else(|| failure("missing observation"))?; + let call = installed.environment.pending_restore_call( + &fixture.manifest.worker_id, + "failed-observer-request", + &token, + )?; + std::thread::scope(|scope| -> Result<()> { + scope.spawn(call); + peer.acknowledge(&installed.environment)?; + // Leave the peer connected while unwinding the scoped HTTP caller. + // The Runtime's own deadline must unblock it, then RAII must reap + // both owned processes and remove the environment, not wait forever. + panic!("intentional checkpoint observer failure"); + }) + })); + assert!(outcome.is_err(), "the observer failure was not exercised"); + assert!( + started.elapsed() < Duration::from_secs(45), + "owned cleanup exceeded scenario budget" + ); + assert!(!root.exists(), "failed observer left its isolated root"); + let cleanup: serde_json::Value = + serde_json::from_slice(&fs::read(artifacts.join("cleanup.json"))?)?; + assert_eq!(cleanup["cleanup_success"], true); + assert_eq!(cleanup["fixture_root_exists_after"], false); + fixture.source.assert_unchanged()?; + Ok(()) +} diff --git a/tests/e2e/tests/worker_restore/seed.rs b/tests/e2e/tests/worker_restore/seed.rs index afc7804e4..f95085b06 100644 --- a/tests/e2e/tests/worker_restore/seed.rs +++ b/tests/e2e/tests/worker_restore/seed.rs @@ -42,7 +42,10 @@ pub struct Installed { } pub fn start(fixture: &Fixture) -> Result { - let mut environment = WorkerE2eEnvironment::new()?; + start_in(fixture, WorkerE2eEnvironment::new()?) +} + +pub fn start_in(fixture: &Fixture, mut environment: WorkerE2eEnvironment) -> Result { super::phase(&environment, "fixture-copy-verify")?; let copy = environment.root.join("provided-fixture"); fixture.source.copy_to(©)?; From a10875d68e8cf554e8a7916511adc78d12303d9f Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 13:36:37 +0900 Subject: [PATCH 31/46] fix: retire the exact stopped Controller from runtime observation --- crates/worker-runtime/src/worker_backend.rs | 41 +++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 7e1288701..4d879bb71 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -539,6 +539,16 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { ) -> Result<(), String> { Ok(()) } + + /// Retire observation of this exact Controller after confirmed cleanup. + /// Called under the same per-Worker lock as publication and replacement. + fn deactivate_stopped_controller( + &self, + _worker_ref: &WorkerRef, + _handle: &WorkerHandle, + ) -> Result<(), String> { + Ok(()) + } } /// Production factory that resolves a normal Worker profile and spawns it under @@ -577,6 +587,21 @@ impl RuntimeWorkerObservationHub { Ok(()) } + fn unregister(&self, worker_ref: &WorkerRef, handle: &WorkerHandle) -> Result<(), String> { + let mut workers = self + .workers + .lock() + .map_err(|_| "Worker observation registry lock is poisoned".to_string())?; + if workers.get(worker_ref).is_some_and(|entry| { + entry + .shared_state + .ptr_eq(&Arc::downgrade(&handle.shared_state)) + }) { + workers.remove(worker_ref); + } + Ok(()) + } + fn get( &self, worker_ref: &WorkerRef, @@ -2560,6 +2585,14 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { self.observation_hub .register(worker_ref.clone(), workspace_id.map(str::to_string), handle) } + + fn deactivate_stopped_controller( + &self, + worker_ref: &WorkerRef, + handle: &WorkerHandle, + ) -> Result<(), String> { + self.observation_hub.unregister(worker_ref, handle) + } } type ProfileRestoreStore = CombinedStore; @@ -4463,6 +4496,14 @@ where if let Err(message) = self.cleanup_execution_resources(&execution) { return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } + // External consumers may retain a closed handle indefinitely. Weak + // shared-state liveness is not evidence of a published execution. + if let Err(message) = self + .factory + .deactivate_stopped_controller(worker_ref, &execution.handle) + { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } match self.workers.lock() { Ok(mut workers) => { From e1945d977c96c7e179dfbadedd5d3fe457276e19 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 13:40:51 +0900 Subject: [PATCH 32/46] test: prove real postcommit activation timeout ownership and Stop fencing --- crates/worker-runtime/src/worker_backend.rs | 83 +++ ...orker_backend_restore_publication_tests.rs | 641 ++++++++++++++++++ 2 files changed, 724 insertions(+) diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 4d879bb71..d8eb3cd15 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -847,10 +847,43 @@ struct RestoreFaults { factory_entered: Mutex>>, prepared_gate: Mutex>>, prepared_entered: Mutex>>, + activation_effects_gate: Mutex>, + activation_started_gate: Mutex>, cleanup_gate: Mutex>>, cleanup_entered: Mutex>>, } +// Test-only synchronization inside the real postcommit candidate, not around +// factory preparation. The receiver is moved into the operation-owned task; +// dropping the one-shot sender releases it even during test unwinding. +#[cfg(test)] +struct RestoreActivationGate { + entered: mpsc::Sender, + release: tokio::sync::oneshot::Receiver<()>, +} + +#[cfg(test)] +struct RestoreActivationEvidence { + operation_id: crate::execution::WorkerLifecycleOperationId, + candidate: usize, + snapshot: Event, + handle: Option, + allocation_retained: bool, +} + +#[cfg(test)] +impl RestoreActivationGate { + async fn wait(self, evidence: RestoreActivationEvidence) { + let _ = self.entered.send(evidence); + assert!( + tokio::time::timeout(Duration::from_secs(30), self.release) + .await + .is_ok(), + "test did not release the internal restore activation gate" + ); + } +} + impl ProfileRuntimeWorkerFactory { fn retain_construction_failure( &self, @@ -2631,6 +2664,8 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate { } async fn activate(&mut self) -> Result { + #[cfg(test)] + let candidate_identity = self as *const Self as usize; let prepared = self.prepared.as_mut().ok_or_else(|| { RuntimeRestoreFailure::unproven( RuntimeRestoreStage::Controller, @@ -2668,6 +2703,30 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate { .expect("retained restore owner") .allocation = prepared.allocation_cleanup(); #[cfg(test)] + { + let gate = self + .factory + .restore_faults + .activation_effects_gate + .lock() + .unwrap() + .take(); + if let Some(gate) = gate { + let allocation_retained = self.factory.failed_restore_resources.lock().unwrap() + [&self.request.worker_ref] + .allocation + .is_some(); + gate.wait(RestoreActivationEvidence { + operation_id: self.request.operation_id, + candidate: candidate_identity, + snapshot: prepared.snapshot().clone(), + handle: None, + allocation_retained, + }) + .await; + } + } + #[cfg(test)] if self .factory .restore_checkpoint(RuntimeRestoreStage::ActivationEffects) @@ -2703,6 +2762,30 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate { } .await .map_err(map_error)?; + #[cfg(test)] + { + let gate = self + .factory + .restore_faults + .activation_started_gate + .lock() + .unwrap() + .take(); + if let Some(gate) = gate { + let allocation_retained = self.factory.failed_restore_resources.lock().unwrap() + [&self.request.worker_ref] + .allocation + .is_some(); + gate.wait(RestoreActivationEvidence { + operation_id: self.request.operation_id, + candidate: candidate_identity, + snapshot: started.streams.snapshot_event.clone(), + handle: Some(started.controller.handle.clone()), + allocation_retained, + }) + .await; + } + } self.prepared = None; self.factory .failed_restore_resources diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs index d7518319b..c27401d6c 100644 --- a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -94,6 +94,647 @@ mod publication { (release, received) } + // One-shot sender Drop is the RAII release, including a panic before the + // explicit release. The task also has a 30s fail-safe independent of Drop. + struct ActivationRelease(tokio::sync::oneshot::Sender<()>); + + impl ActivationRelease { + fn release(self) { + let _ = self.0.send(()); + } + } + + fn activation_gate( + fixture: &Fixture, + started: bool, + ) -> (ActivationRelease, mpsc::Receiver) { + let (release, receiver) = tokio::sync::oneshot::channel(); + let (entered, evidence) = mpsc::channel(); + let faults = &fixture.backend.factory.restore_faults; + let slot = if started { + &faults.activation_started_gate + } else { + &faults.activation_effects_gate + }; + assert!( + slot.lock() + .unwrap() + .replace(RestoreActivationGate { + entered, + release: receiver + }) + .is_none() + ); + (ActivationRelease(release), evidence) + } + + fn restore_owner( + runtime: &EmbeddedRuntime, + worker_ref: &WorkerRef, + request_id: &str, + ) -> serde_json::Value { + let record = runtime + .test_runtime_store() + .read_worker_record(worker_ref) + .unwrap(); + let guard = serde_json::to_value(&record.restore_guard).unwrap(); + assert_eq!(guard["owners"].as_object().unwrap().len(), 1); + assert!( + guard["owners"] + .as_object() + .unwrap() + .contains_key(request_id) + ); + guard["owners"][request_id].clone() + } + + fn assert_activation_owned( + fixture: &Fixture, + runtime: &EmbeddedRuntime, + request_id: &str, + evidence: &RestoreActivationEvidence, + ) -> Arc { + let worker_ref = &fixture.request.worker_ref; + let durable = durable_operation(runtime, worker_ref).unwrap(); + assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed); + assert_eq!(durable.operation_id, evidence.operation_id); + let owner = restore_owner(runtime, worker_ref, request_id); + assert_eq!( + owner["operation_id"], + serde_json::to_value(evidence.operation_id).unwrap() + ); + assert!( + owner["receipt"].is_null(), + "committed activation cannot have a terminal receipt yet" + ); + let pending = fixture + .backend + .pending_restore(worker_ref) + .unwrap() + .unwrap(); + assert_eq!(pending.request.operation_id, evidence.operation_id); + assert!(pending.activation_started.load(Ordering::Acquire)); + assert!(pending.task_in_flight.load(Ordering::Acquire)); + assert!(pending.task.try_lock().unwrap().is_some()); + assert!( + pending.result.lock().unwrap().is_none(), + "the task, not another caller, owns the candidate" + ); + assert!(pending.activated.lock().unwrap().is_none()); + assert!(evidence.allocation_retained); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!( + fixture + .backend + .factory + .observation_hub + .get(worker_ref) + .is_none() + ); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + fixture.assert_quiet(); + pending + } + + fn alert_marker(handle: &WorkerHandle, marker: &str) { + handle.alerter.alert( + protocol::AlertLevel::Warn, + protocol::AlertSource::Worker, + marker.into(), + ); + } + + fn is_marker(event: &Event, marker: &str) -> bool { + matches!(event, Event::Alert(alert) if alert.message == marker) + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn postcommit_activation_timeout_autonomously_publishes_same_controller_and_paired_streams() { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let (effects_release, effects_entered) = activation_gate(&fixture, false); + let (started_release, started_entered) = activation_gate(&fixture, true); + let observations = runtime.subscribe_worker_observation().unwrap(); + let first = runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let effects = effects_entered.recv_timeout(WAIT).unwrap(); + assert!(effects.handle.is_none()); + let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &effects); + let canonical = fixture.canonical_files(); + let owner = restore_owner(&runtime, &worker_ref, &request.request_id); + effects_release.release(); + let started = started_entered.recv_timeout(WAIT).unwrap(); + let same_pending = + assert_activation_owned(&fixture, &runtime, &request.request_id, &started); + assert!(Arc::ptr_eq(&pending, &same_pending)); + assert_eq!(effects.candidate, started.candidate); + assert_eq!( + serde_json::to_value(&effects.snapshot).unwrap(), + serde_json::to_value(&started.snapshot).unwrap() + ); + assert_eq!( + restore_owner(&runtime, &worker_ref, &request.request_id), + owner + ); + let old = started.handle.unwrap(); + assert!(!old.protocol_is_closed()); + // This live event is queued on the precommit paired receiver. A newly + // subscribed bridge would miss it; it is not in its original snapshot. + alert_marker(&old, "activation-paired-before-publication"); + started_release.release(); + runtime.test_wait_restore_continuation(&worker_ref, started.operation_id, WAIT); + assert!(durable_operation(&runtime, &worker_ref).is_none()); + let settled = restore_owner(&runtime, &worker_ref, &request.request_id); + assert_eq!(settled["operation_id"], owner["operation_id"]); + assert_eq!(settled["receipt"]["state"], "accepted"); + let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(old.same_controller(&execution.handle)); + assert_eq!(execution.restore_operation_id, Some(started.operation_id)); + assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .publication_attempts + .load(Ordering::SeqCst), + 1 + ); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + transport.validate().unwrap(); + assert_eq!( + serde_json::to_value(&transport.snapshot).unwrap(), + serde_json::to_value(&effects.snapshot).unwrap() + ); + fixture + .backend + .run_on_adapter_runtime(async move { + let mut observations = observations; + tokio::time::timeout(WAIT, async { + loop { + let observed = observations.recv().await.unwrap(); + if is_marker(&observed.payload, "activation-paired-before-publication") { + break; + } + } + }) + .await + .map_err(|_| "paired receiver lost its prepublication event".to_string())?; + Ok(()) + }) + .unwrap(); + assert!( + fixture.canonical_files() == canonical, + "canonical files changed unexpectedly" + ); + drop(transport); + drop(execution); + drop(pending); + drop(same_pending); + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + assert!(old.protocol_is_closed()); + fixture.assert_released(); + fixture.assert_quiet(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn postcommit_activation_timeout_late_nonretryable_failure_never_fakes_rollback() { + for cleanup_fails in [false, true] { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let (release, entered) = activation_gate(&fixture, false); + let faults = &fixture.backend.factory.restore_faults; + *faults.fail_at.lock().unwrap() = Some(RuntimeRestoreStage::ActivationEffects); + faults.cleanup_fail.store(cleanup_fails, Ordering::SeqCst); + let first = runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let effects = entered.recv_timeout(WAIT).unwrap(); + let pending = + assert_activation_owned(&fixture, &runtime, &request.request_id, &effects); + let canonical = fixture.canonical_files(); + release.release(); + if cleanup_fails { + let reason = runtime.test_wait_restore_continuation_parked( + &worker_ref, + effects.operation_id, + WAIT, + ); + assert!(!reason.is_empty()); + let failure = pending + .activated + .lock() + .unwrap() + .as_ref() + .unwrap() + .as_ref() + .err() + .unwrap() + .clone(); + assert_eq!(failure.stage, RuntimeRestoreStage::ActivationEffects); + assert!(failure.durable_effects_possible); + assert!(failure.needs_cleanup()); + assert!(matches!(failure.cleanup, RuntimeRestoreCleanup::Failed(_))); + let durable = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!(durable.operation_id, effects.operation_id); + assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed); + assert!( + restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null() + ); + assert!(Arc::ptr_eq( + &pending, + &fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .unwrap() + )); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!( + fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .is_none() + ); + assert!( + fixture + .backend + .completed_restore_failure(effects.operation_id, &worker_ref) + .is_none() + ); + assert_eq!(faults.executions.load(Ordering::SeqCst), 1); + // Obstruction proof ends here. Stop below is fixture teardown, + // not an autonomous terminal-failure/success claim. + faults.cleanup_fail.store(false, Ordering::SeqCst); + } else { + // A nonretryable candidate must be discarded, not reactivated. + // Reconstruction belongs to the same committed operation. + runtime.test_wait_restore_continuation(&worker_ref, effects.operation_id, WAIT); + let settled = restore_owner(&runtime, &worker_ref, &request.request_id); + assert_eq!(settled["receipt"]["state"], "accepted"); + assert_eq!( + fixture.backend.workers.lock().unwrap()[&worker_ref].restore_operation_id, + Some(effects.operation_id) + ); + assert_eq!(faults.executions.load(Ordering::SeqCst), 2); + assert_eq!(faults.publication_attempts.load(Ordering::SeqCst), 1); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + transport.validate().unwrap(); + drop(transport); + } + assert!( + fixture.canonical_files() == canonical, + "canonical files changed unexpectedly" + ); + drop(pending); + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + fixture.assert_quiet(); + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn postcommit_activation_timeout_stop_remove_fence_late_cleanup_then_fresh_replacement() { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let pause = runtime.test_pause_restore_continuation(); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let (release, entered) = activation_gate(&fixture, true); + let first = runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let started = entered.recv_timeout(WAIT).unwrap(); + let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started); + let old = started.handle.unwrap(); + let canonical = fixture.canonical_files(); + alert_marker(&old, "superseded-old-buffered"); + let stop_error = runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap_err(); + assert!( + stop_error + .to_string() + .contains("restore factory is still pending"), + "Stop must time out joining the activation task" + ); + let remove_error = runtime + .delete_worker_scoped(scope, &worker_ref) + .unwrap_err(); + assert!( + remove_error + .to_string() + .contains("pending stop reconciliation"), + "Remove must be fenced by the unresolved Stop" + ); + assert!(Arc::ptr_eq( + &pending, + &fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .unwrap() + )); + assert_ne!( + runtime.worker_detail(&worker_ref).unwrap().status, + crate::catalog::WorkerStatus::Stopped + ); + assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null()); + assert!(pending.task_in_flight.load(Ordering::Acquire)); + assert!(!old.protocol_is_closed()); + assert!(fixture.backend.workers.lock().unwrap().is_empty()); + assert!( + fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .is_none() + ); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 1 + ); + release.release(); + drop(pause); + runtime.test_wait_restore_continuation( + &worker_ref, + started.operation_id, + Duration::from_secs(10), + ); + assert_eq!( + runtime.worker_detail(&worker_ref).unwrap().status, + crate::catalog::WorkerStatus::Stopped + ); + assert!(old.protocol_is_closed()); + assert!(!pending.task_in_flight.load(Ordering::Acquire)); + assert!( + pending.task.try_lock().unwrap().is_none(), + "late task must have been joined" + ); + fixture.assert_released(); + // Confirmed Stop deletes the fixture orphan upload, not its Session, + // metadata, or committed entries. Keep the comparison content-private. + let mut stopped_canonical = canonical.clone(); + let orphans = canonical + .keys() + .filter(|path| { + path.parent() == Some(std::path::Path::new("session/artifacts/paste")) + && path.to_string_lossy().ends_with(".file.json") + }) + .cloned() + .collect::>(); + assert_eq!(orphans.len(), 1); + stopped_canonical.remove(&orphans[0]); + assert!( + fixture.canonical_files() == stopped_canonical, + "Stop must remove only the disposable orphan upload" + ); + let superseded = restore_owner(&runtime, &worker_ref, &request.request_id); + assert_eq!(superseded["receipt"]["state"], "superseded"); + assert!( + runtime + .read_worker_observation_events( + &worker_ref, + crate::observation::WorkerObservationCursor::zero() + ) + .unwrap() + .is_empty() + ); + // Only a fresh request from a fresh observation may create a replacement. + let replacement_request = runtime.test_restore_request(&worker_ref); + assert_ne!(replacement_request.request_id, request.request_id); + let accepted = runtime + .restore_worker_operation_scoped(scope, &worker_ref, replacement_request) + .unwrap(); + assert_eq!(accepted.state, server_api::WorkerRestoreState::Accepted); + let replacement = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(!old.same_controller(&replacement.handle)); + { + // Even a late retirement callback identifies the actual old + // Controller, never whichever execution now has this WorkerRef. + let lock = fixture.backend.worker_lock(&worker_ref).unwrap(); + let _guard = lock.lock().unwrap(); + fixture + .backend + .factory + .deactivate_stopped_controller(&worker_ref, &old) + .unwrap(); + let (_, observed_state, _) = fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .unwrap(); + assert!(Arc::ptr_eq( + &observed_state, + &replacement.handle.shared_state + )); + } + assert_ne!(replacement.restore_operation_id, Some(started.operation_id)); + assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); + assert_eq!( + fixture + .backend + .factory + .restore_faults + .executions + .load(Ordering::SeqCst), + 2 + ); + // A diagnostic event on the retained old endpoint must never enter the + // new bridge, and a command on that closed endpoint must not be rerouted. + alert_marker(&old, "superseded-old-after-replacement"); + let old_command = old.clone(); + assert!( + fixture + .backend + .run_on_adapter_runtime(async move { + old_command + .send(Method::Pause { + command: protocol::WorkerCommandEnvelope::new(77111), + }) + .await + .map_err(|error| error.to_string()) + }) + .is_err() + ); + assert_eq!( + replacement.handle.shared_state.snapshot().state, + protocol::WorkerState::Idle + ); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + transport.validate().unwrap(); + let mut observations = runtime.subscribe_worker_observation().unwrap(); + alert_marker(&replacement.handle, "replacement-live-barrier"); + fixture + .backend + .run_on_adapter_runtime(async move { + tokio::time::timeout(WAIT, async { + loop { + let observed = observations.recv().await.unwrap(); + assert!(!is_marker(&observed.payload, "superseded-old-buffered")); + assert!(!is_marker( + &observed.payload, + "superseded-old-after-replacement" + )); + if is_marker(&observed.payload, "replacement-live-barrier") { + break; + } + } + }) + .await + .map_err(|_| "replacement bridge did not progress".to_string())?; + Ok(()) + }) + .unwrap(); + let events = runtime + .read_worker_observation_events( + &worker_ref, + crate::observation::WorkerObservationCursor::zero(), + ) + .unwrap(); + assert!( + !events + .iter() + .any(|event| is_marker(&event.payload, "superseded-old-buffered") + || is_marker(&event.payload, "superseded-old-after-replacement")) + ); + assert!( + fixture.canonical_files() == stopped_canonical, + "replacement changed canonical Session content" + ); + drop(transport); + drop(replacement); + drop(pending); + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); + fixture.assert_quiet(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn postcommit_activation_timeout_gate_release_on_panic_keeps_owned_task_bounded() { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let (release, entered) = activation_gate(&fixture, true); + let first = runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let started = entered.recv_timeout(WAIT).unwrap(); + let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started); + let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || { + let _release_on_unwind = release; + panic!("intentional test-owner unwind at activation gate"); + })); + assert!(panic.is_err()); + runtime.test_wait_restore_continuation(&worker_ref, started.operation_id, WAIT); + assert_eq!( + restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"], + "accepted" + ); + assert!(!pending.task_in_flight.load(Ordering::Acquire)); + assert!(pending.task.try_lock().unwrap().is_none()); + assert!( + started + .handle + .as_ref() + .unwrap() + .same_controller(&fixture.backend.workers.lock().unwrap()[&worker_ref].handle) + ); + drop(pending); + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + assert!(started.handle.unwrap().protocol_is_closed()); + fixture.assert_released(); + fixture.assert_quiet(); + } + #[test] #[serial_test::serial(worker_allocation)] fn startup_fault_recovery_retains_cleanup_or_committed_reconstruction_without_repost() { From 2b24db5a15b50d886d321402cdd953bfe4a7b18a Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 13:49:48 +0900 Subject: [PATCH 33/46] test: fence superseded restore replay and old Shutdown after replacement --- ...orker_backend_restore_publication_tests.rs | 52 ++++++++++++++----- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs index c27401d6c..9a905c4bd 100644 --- a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -600,6 +600,23 @@ mod publication { )); } assert_ne!(replacement.restore_operation_id, Some(started.operation_id)); + // Recovering the superseded intent must not settle or mutate the new + // execution, even though both requests address the same WorkerRef. + let replacement_owner = replacement.restore_operation_id; + let replay = runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!(replay.state, server_api::WorkerRestoreState::Superseded); + let saved = runtime + .test_runtime_store() + .read_worker_record(&worker_ref) + .unwrap(); + let guard = serde_json::to_value(&saved.restore_guard).unwrap(); + assert_eq!(guard["owners"][&request.request_id], superseded); + assert_eq!( + fixture.backend.workers.lock().unwrap()[&worker_ref].restore_operation_id, + replacement_owner + ); assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); assert_eq!( fixture @@ -613,20 +630,27 @@ mod publication { // A diagnostic event on the retained old endpoint must never enter the // new bridge, and a command on that closed endpoint must not be rerouted. alert_marker(&old, "superseded-old-after-replacement"); - let old_command = old.clone(); - assert!( - fixture - .backend - .run_on_adapter_runtime(async move { - old_command - .send(Method::Pause { - command: protocol::WorkerCommandEnvelope::new(77111), - }) - .await - .map_err(|error| error.to_string()) - }) - .is_err() - ); + for method in [ + Method::Pause { + command: protocol::WorkerCommandEnvelope::new(77111), + }, + Method::Shutdown { + command: protocol::WorkerCommandEnvelope::new(77112), + }, + ] { + let old_command = old.clone(); + assert!( + fixture + .backend + .run_on_adapter_runtime(async move { + old_command + .send(method) + .await + .map_err(|error| error.to_string()) + }) + .is_err() + ); + } assert_eq!( replacement.handle.shared_state.snapshot().state, protocol::WorkerState::Idle From f85d8311042653b85df03c8d3cf750d8127c0ce4 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 14:10:02 +0900 Subject: [PATCH 34/46] fix: retain committed controller paired streams across bridge admission failure --- crates/worker-runtime/src/worker_backend.rs | 302 ++++++++++++-------- 1 file changed, 190 insertions(+), 112 deletions(-) diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index d8eb3cd15..80d8118d6 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -2974,6 +2974,8 @@ impl ProfileRuntimeWorkerFactory { #[derive(Clone)] struct RuntimeExecutionTaskScope { + #[cfg(feature = "ws-server")] + protocol_bridge: Arc>>, tasks: Arc>>>, shutdown_admission: Arc>>, // Actual unconnected Controllers whose cleanup failed, owned by this @@ -2981,6 +2983,15 @@ struct RuntimeExecutionTaskScope { cleanup_candidates: Arc>>>, } +// The structured scope owns the original paired subscription until task admission +// succeeds. A failed spawn drops a future, not the committed Controller's streams. +#[cfg(feature = "ws-server")] +struct RuntimeProtocolBridge { + context: crate::execution::WorkerExecutionContext, + streams: Arc>>, + snapshot_pending: bool, +} + // Evidence for one exact command on this actual Controller. Keep the send task // and receiver across bounded waits: unknown admission never authorizes resend. struct RuntimeShutdownAdmission { @@ -3062,6 +3073,8 @@ struct RuntimeExecutionTaskCompletion { impl RuntimeExecutionTaskScope { fn new(controller_task: tokio::task::JoinHandle<()>) -> Self { let scope = Self { + #[cfg(feature = "ws-server")] + protocol_bridge: Arc::default(), tasks: Arc::new(Mutex::new(Vec::new())), shutdown_admission: Arc::new(tokio::sync::Mutex::new(None)), cleanup_candidates: Arc::new(Mutex::new(Vec::new())), @@ -4117,6 +4130,108 @@ where ) } + #[cfg(feature = "ws-server")] + fn start_protocol_bridge( + &self, + handle: &WorkerHandle, + worker_state: &Arc>, + tasks: &RuntimeExecutionTaskScope, + ) -> Result<(), String> { + let mut retained = tasks + .protocol_bridge + .lock() + .map_err(|_| "protocol bridge owner lock is poisoned".to_string())?; + let Some(bridge) = retained.as_mut() else { + return Ok(()); + }; + if bridge.snapshot_pending { + let streams = bridge + .streams + .lock() + .map_err(|_| "protocol streams lock is poisoned")?; + bridge + .context + .publish_protocol_event( + streams + .as_ref() + .expect("unadmitted bridge owns streams") + .snapshot_event + .clone(), + ) + .map_err(|error| error.to_string())?; + bridge.snapshot_pending = false; + } + let streams = bridge.streams.clone(); + let bridge_context = bridge.context.clone(); + let bridge_worker_state = worker_state.clone(); + let bridge_handle = handle.clone(); + let task = self.spawn_on_adapter_runtime(async move { + let streams = streams.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + .take().expect("one bridge consumes the paired subscription"); + let mut events = streams.events; + let mut entry_events = streams.log_entries; + loop { + tokio::select! { + event = events.recv() => { + match event { + Ok(event) => { + let mut event = match bridge_handle.alerter.freshen_event(event) { + Ok(event) => event, + Err(_) => { + // This shared bridge also carries Worker state: do + // not terminate it when a bounded Alert was evicted. + // Replace both receivers with the paired snapshot. + let replacement = subscribe_worker_protocol_session(&bridge_handle); + events = replacement.events; + entry_events = replacement.log_entries; + let mut snapshot = replacement.snapshot_event; + if apply_protocol_worker_state(&bridge_worker_state, &mut snapshot).is_err() { + break; + } + let _ = bridge_context.publish_protocol_event(snapshot); + for alert in replacement.alert_snapshot { + let _ = bridge_context.publish_protocol_event(Event::Alert(alert)); + } + continue; + } + }; + match apply_protocol_worker_state(&bridge_worker_state, &mut event) { + Ok(true) => { + let _ = bridge_context.publish_protocol_event(event); + } + Ok(false) => {} + Err(message) => { + let _ = bridge_context.publish_protocol_event(Event::Error { + code: protocol::ErrorCode::Internal, + message: format!("worker state stream rejected: {message}"), + }); + break; + } + } + } + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(broadcast::error::RecvError::Closed) => break, + } + } + entry = entry_events.recv() => { + match entry { + Ok(entry) => { + if let Some(event) = live_log_entry_event(entry) { + let _ = bridge_context.publish_protocol_event(event); + } + } + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(broadcast::error::RecvError::Closed) => break, + } + } + } + } + })?; + tasks.push("protocol bridge", task, true); + *retained = None; + Ok(()) + } + // Own the actual Controller scope before doing any fallible connection work. // The split also permits testing duplicate cleanup with an already-pending // real admission waiter, rather than relying on a scheduler race. @@ -4189,121 +4304,60 @@ where }; } #[cfg(feature = "ws-server")] + let bridge_failure = { + *tasks + .protocol_bridge + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(RuntimeProtocolBridge { + context: bridge_context, + streams: Arc::new(Mutex::new(Some(streams))), + snapshot_pending: restore_operation_id.is_some(), + }); + self.start_protocol_bridge(&handle, &worker_state, &tasks) + .err() + }; + #[cfg(not(feature = "ws-server"))] + let bridge_failure: Option = None; + if let Some(message) = bridge_failure + .as_ref() + .filter(|_| restore_operation_id.is_none()) { - let mut events = streams.events; - let mut entry_events = streams.log_entries; - let bridge_worker_state = worker_state.clone(); - let bridge_handle = handle.clone(); - let initial_publication = if restore_operation_id.is_some() { - bridge_context - .publish_protocol_event(streams.snapshot_event.clone()) - .map(|_| ()) - .map_err(|error| error.to_string()) - } else { - Ok(()) - }; - let bridge_task = match initial_publication.and_then(|()| self.spawn_on_adapter_runtime(async move { - loop { - tokio::select! { - event = events.recv() => { - match event { - Ok(event) => { - let mut event = match bridge_handle.alerter.freshen_event(event) { - Ok(event) => event, - Err(_) => { - // This shared bridge also carries Worker state: do - // not terminate it when a bounded Alert was evicted. - // Replace both receivers with the paired snapshot. - let replacement = subscribe_worker_protocol_session(&bridge_handle); - events = replacement.events; - entry_events = replacement.log_entries; - let mut snapshot = replacement.snapshot_event; - if apply_protocol_worker_state(&bridge_worker_state, &mut snapshot).is_err() { - break; - } - let _ = bridge_context.publish_protocol_event(snapshot); - for alert in replacement.alert_snapshot { - let _ = bridge_context.publish_protocol_event(Event::Alert(alert)); - } - continue; - } - }; - match apply_protocol_worker_state(&bridge_worker_state, &mut event) { - Ok(true) => { - let _ = bridge_context.publish_protocol_event(event); - } - Ok(false) => {} - Err(message) => { - let _ = bridge_context.publish_protocol_event(Event::Error { - code: protocol::ErrorCode::Internal, - message: format!("worker state stream rejected: {message}"), - }); - break; - } - } - } - Err(broadcast::error::RecvError::Lagged(_)) => continue, - Err(broadcast::error::RecvError::Closed) => break, - } - } - entry = entry_events.recv() => { - match entry { - Ok(entry) => { - if let Some(event) = live_log_entry_event(entry) { - let _ = bridge_context.publish_protocol_event(event); - } - } - Err(broadcast::error::RecvError::Lagged(_)) => continue, - Err(broadcast::error::RecvError::Closed) => break, - } - } + drop(workers); + let cleanup = self.cleanup_unconnected_controller( + &handle, + &shutdown, + &tasks, + &shutdown_requested, + &worker_state, + ); + let result = WorkerExecutionResult::errored( + operation, + match &cleanup { + Ok(()) => message.clone(), + Err(cleanup) => format!("{message}; controller cleanup failed: {cleanup}"), + }, + ); + return match cleanup { + Ok(()) => WorkerExecutionSpawnResult::RolledBack(result), + Err(_) => { + self.retain_uncertain_unconnected_controller( + &worker_ref, + handle, + shutdown, + shutdown_requested, + tasks, + worker_state, + workspace_client, + restore_operation_id, + workdir_attachment_statuses.clone(), + ); + WorkerExecutionSpawnResult::ReconciliationRequired { + result, + worker_state: None, + workdir_attachments: workdir_attachment_statuses, } } - })) { - Ok(task) => task, - Err(message) => { - drop(workers); - let cleanup = self.cleanup_unconnected_controller( - &handle, - &shutdown, - &tasks, - &shutdown_requested, - &worker_state, - ); - let result = WorkerExecutionResult::errored( - operation, - match &cleanup { - Ok(()) => message, - Err(cleanup) => { - format!("{message}; controller cleanup failed: {cleanup}") - } - }, - ); - return match cleanup { - Ok(()) => WorkerExecutionSpawnResult::RolledBack(result), - Err(_) => { - let retained_state = worker_state.read().ok().map(|state| state.clone()); - self.retain_uncertain_unconnected_controller( - &worker_ref, - handle, - shutdown, - shutdown_requested, - tasks, - worker_state, - workspace_client, - restore_operation_id, - workdir_attachment_statuses.clone(), - ); - WorkerExecutionSpawnResult::ReconciliationRequired { - result, - worker_state: retained_state, - workdir_attachments: workdir_attachment_statuses, - } - } - }; - } }; - tasks.push("protocol bridge", bridge_task, true); } #[cfg(not(feature = "ws-server"))] { @@ -4332,6 +4386,15 @@ where }, ); + if let Some(message) = bridge_failure { + // A committed restore retains this exact execution and its unadmitted + // paired bridge in the structured scope. Cleanup is not rollback. + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored(operation, message), + worker_state: Some(connected_worker_state), + workdir_attachments: workdir_attachment_statuses, + }; + } WorkerExecutionSpawnResult::Connected { worker_state: connected_worker_state, workdir_attachments: workdir_attachment_statuses, @@ -6188,7 +6251,20 @@ where WorkerExecutionSpawnResult::Connected { .. } => { self.remove_pending_restore(worker_ref)? } - other => return Err(format!("restore activation connection failed: {other:?}")), + other => { + if self + .workers + .lock() + .map_err(|_| "worker registry lock is poisoned")? + .get(worker_ref) + .is_some_and(|execution| { + execution.restore_operation_id == Some(operation_id) + }) + { + self.remove_pending_restore(worker_ref)?; + } + return Err(format!("restore activation connection failed: {other:?}")); + } } } let workers = self @@ -6206,6 +6282,8 @@ where { return Err("restored Worker protocol endpoint is not attachable".into()); } + #[cfg(feature = "ws-server")] + self.start_protocol_bridge(&execution.handle, &execution.worker_state, &execution.tasks)?; self.factory.activate_restored_controller( worker_ref, execution.workspace_id.as_deref(), From 1fd2e099715344fea3f35c07f23c4de58ef3bdd7 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 14:28:37 +0900 Subject: [PATCH 35/46] test: cover real partial publication and terminal receipt durability faults --- crates/worker-runtime/src/execution.rs | 87 +- crates/worker-runtime/src/fs_store.rs | 52 + crates/worker-runtime/src/runtime.rs | 60 ++ .../src/runtime/restore_continuation.rs | 3 + crates/worker-runtime/src/worker_backend.rs | 37 + ...orker_backend_restore_publication_tests.rs | 2 + ...rker_backend_restore_registration_tests.rs | 889 ++++++++++++++++++ 7 files changed, 1128 insertions(+), 2 deletions(-) create mode 100644 crates/worker-runtime/src/worker_backend_restore_registration_tests.rs diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index ffa6e807f..f4c7d85e9 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -215,6 +215,38 @@ impl WorkerExecutionResult { } } +// Private one-shot synchronization at real publication boundaries. Disconnect +// releases the waiter during unwinding; timeout bounds a forgotten test owner. +#[cfg(test)] +#[derive(Debug)] +pub(crate) struct RestoreTestGate { + pub(crate) entered: std::sync::mpsc::Sender, + pub(crate) release: std::sync::mpsc::Receiver<()>, +} + +#[cfg(test)] +impl RestoreTestGate { + pub(crate) fn wait(self, evidence: T) { + let _ = self.entered.send(evidence); + assert!( + !matches!( + self.release + .recv_timeout(std::time::Duration::from_secs(30)), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) + ), + "publication test gate timed out" + ); + } +} + +#[cfg(all(test, feature = "ws-server"))] +#[derive(Debug)] +pub(crate) struct RestoreFlushFault { + pub(crate) remaining_successes: usize, + pub(crate) gate: RestoreTestGate, + pub(crate) restored: std::sync::mpsc::Sender>, +} + /// Runtime hooks available to an execution backend for one Worker. #[cfg(feature = "ws-server")] type WorkerObservationPublisher = Arc< @@ -249,6 +281,8 @@ pub struct WorkerExecutionContext { observation_publisher: WorkerObservationPublisher, #[cfg(feature = "ws-server")] publication_state: Arc>, + #[cfg(all(test, feature = "ws-server"))] + flush_fault: Arc>>, } impl WorkerExecutionContext { @@ -262,6 +296,8 @@ impl WorkerExecutionContext { restore_progress: None, restore_repair_authorizer: None, observation_publisher, + #[cfg(test)] + flush_fault: Arc::default(), publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Active)), } } @@ -276,6 +312,8 @@ impl WorkerExecutionContext { restore_progress: None, restore_repair_authorizer: None, observation_publisher, + #[cfg(test)] + flush_fault: Arc::default(), publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Pending( VecDeque::new(), ))), @@ -386,20 +424,65 @@ impl WorkerExecutionContext { let Some(event) = event else { return Ok(()); }; - if let Err(error) = (self.observation_publisher)(self.worker_ref.clone(), event.clone()) - { + #[cfg(test)] + let fault = { + let mut fault = self.flush_fault.lock().unwrap(); + if fault.as_ref().is_some_and(|f| f.remaining_successes == 0) { + fault.take() + } else { + if let Some(fault) = fault.as_mut() { + fault.remaining_successes -= 1; + } + None + } + }; + #[cfg(test)] + let mut restored = None; + #[cfg(test)] + let publication = if let Some(fault) = fault { + restored = Some(fault.restored); + fault.gate.wait(event.clone()); + Err(RuntimeError::InvalidRequest( + "injected buffered publication failure".into(), + )) + } else { + (self.observation_publisher)(self.worker_ref.clone(), event.clone()) + }; + #[cfg(not(test))] + let publication = (self.observation_publisher)(self.worker_ref.clone(), event.clone()); + if let Err(error) = publication { let mut state = self .publication_state .lock() .map_err(|_| RuntimeError::StatePoisoned)?; if let WorkerExecutionPublicationState::Activating(events) = &mut *state { events.push_front(event); + #[cfg(test)] + if let Some(restored) = restored { + let _ = restored.send(events.iter().cloned().collect()); + } } return Err(error); } } } + #[cfg(all(test, feature = "ws-server"))] + pub(crate) fn test_fail_flush(&self, fault: RestoreFlushFault) { + assert!(self.flush_fault.lock().unwrap().replace(fault).is_none()); + } + + #[cfg(all(test, feature = "ws-server"))] + pub(crate) fn test_buffered_events(&self) -> Vec { + match &*self.publication_state.lock().unwrap() { + WorkerExecutionPublicationState::Pending(events) + | WorkerExecutionPublicationState::Activating(events) => { + events.iter().cloned().collect() + } + _ => Vec::new(), + } + } + #[cfg(not(feature = "ws-server"))] pub(crate) fn activate_candidate(&self) -> Result<(), RuntimeError> { Ok(()) diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index d2b9c86de..1278f89f7 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -128,6 +128,13 @@ pub struct FsRuntimeStore { worker_write_fault: Arc, #[cfg(test)] worker_reconcile_sync_fault: Arc, + #[cfg(test)] + worker_reconcile_read_fault: Arc, + #[cfg(test)] + worker_reconcile_counts: Arc<( + std::sync::atomic::AtomicUsize, + std::sync::atomic::AtomicUsize, + )>, } impl PartialEq for FsRuntimeStore { @@ -174,6 +181,20 @@ impl FsRuntimeStore { .store(true, Ordering::Release); } + #[cfg(test)] + pub(crate) fn fail_next_worker_reconcile_read(&self) { + self.worker_reconcile_read_fault + .store(true, Ordering::Release); + } + + #[cfg(test)] + pub(crate) fn test_worker_reconcile_counts(&self) -> (usize, usize) { + ( + self.worker_reconcile_counts.0.load(Ordering::Acquire), + self.worker_reconcile_counts.1.load(Ordering::Acquire), + ) + } + pub(crate) fn open_or_create( root: PathBuf, runtime_id: &str, @@ -235,6 +256,10 @@ impl FsRuntimeStore { worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), #[cfg(test)] worker_reconcile_sync_fault: Default::default(), + #[cfg(test)] + worker_reconcile_read_fault: Default::default(), + #[cfg(test)] + worker_reconcile_counts: Default::default(), }; let state = if snapshot_exists { Some(store.load_runtime_state()?) @@ -298,8 +323,16 @@ impl FsRuntimeStore { &self, worker_ref: &WorkerRef, ) -> Result { + #[cfg(test)] + self.worker_reconcile_counts + .0 + .fetch_add(1, Ordering::AcqRel); self.read_durable_worker_record_with_sync(worker_ref, |file| { #[cfg(test)] + self.worker_reconcile_counts + .1 + .fetch_add(1, Ordering::AcqRel); + #[cfg(test)] if self .worker_reconcile_sync_fault .swap(false, Ordering::AcqRel) @@ -337,6 +370,17 @@ impl FsRuntimeStore { "Worker aggregate reconciliation requires the Runtime store owner lock".to_string(), )); } + #[cfg(test)] + if self + .worker_reconcile_read_fault + .swap(false, Ordering::AcqRel) + { + return Err(RuntimeError::StoreCommitOutcomeUnknown { + operation: "read reconciled Worker aggregate", + path: self.worker_dir(&worker_ref.worker_id).join(WORKER_FILE), + source: std::io::Error::other("injected reconciliation read failure"), + }); + } let (record, file) = self.read_worker_record_file(worker_ref)?; let worker_dir = self.worker_dir(&worker_ref.worker_id); let path = worker_dir.join(WORKER_FILE); @@ -1724,6 +1768,10 @@ fn validate_current_runtime_store(root: &Path) -> Result<(), RuntimeError> { worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), #[cfg(test)] worker_reconcile_sync_fault: Default::default(), + #[cfg(test)] + worker_reconcile_read_fault: Default::default(), + #[cfg(test)] + worker_reconcile_counts: Default::default(), }; store.load_runtime_state().map(|_| ()) } @@ -1735,6 +1783,10 @@ fn validate_completed_migration_staging(root: &Path) -> Result<(), RuntimeError> worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), #[cfg(test)] worker_reconcile_sync_fault: Default::default(), + #[cfg(test)] + worker_reconcile_read_fault: Default::default(), + #[cfg(test)] + worker_reconcile_counts: Default::default(), }; let state = store.load_runtime_state()?; if let Some(worker) = state.workers.values().find(|worker| { diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 4f9141cc1..755509f7b 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -2340,6 +2340,48 @@ impl Runtime { } } + #[cfg(test)] + pub(crate) fn test_terminal_receipt_gate( + &self, + gate: crate::execution::RestoreTestGate, + ) { + assert!( + self.restore_continuations + .terminal_receipt_gate + .lock() + .unwrap() + .replace(gate) + .is_none() + ); + } + + #[cfg(test)] + pub(crate) fn test_pending_restore_context( + &self, + worker_ref: &WorkerRef, + ) -> ( + WorkerLifecycleOperationId, + crate::execution::WorkerExecutionContext, + bool, + ) { + let state = self.lock().unwrap(); + let worker = state.worker(worker_ref).unwrap(); + assert!( + worker + .restore_guard + .owners + .values() + .all(|owner| owner.receipt.is_none()) + ); + ( + worker.pending_restore.unwrap().operation_id, + worker.restore_candidate_context.clone().unwrap(), + state + .unreconciled_worker_writes + .contains(&worker_ref.worker_id), + ) + } + // Seed durable crash checkpoints without launching a second Controller. // Tests then reopen the real store with empty backend resource maps. #[cfg(all(test, feature = "fs-store"))] @@ -2512,6 +2554,24 @@ impl Runtime { if result.state == WorkerRestoreState::ReconciliationRequired { return Ok(()); } + #[cfg(test)] + if result.state == WorkerRestoreState::Accepted { + let gate = self + .restore_continuations + .terminal_receipt_gate + .lock() + .unwrap() + .take(); + if let Some(gate) = gate { + let operation = self + .lock()? + .worker(worker_ref)? + .pending_restore + .expect("terminal receipt follows committed operation") + .operation_id; + gate.wait(operation); + } + } let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); if candidate.restore_guard.active_request_id.is_some() diff --git a/crates/worker-runtime/src/runtime/restore_continuation.rs b/crates/worker-runtime/src/runtime/restore_continuation.rs index ad50b4d2d..ba8e884c7 100644 --- a/crates/worker-runtime/src/runtime/restore_continuation.rs +++ b/crates/worker-runtime/src/runtime/restore_continuation.rs @@ -20,6 +20,9 @@ pub(super) struct RestoreContinuations { owners: Mutex>>, #[cfg(test)] paused: std::sync::atomic::AtomicBool, + #[cfg(test)] + pub(super) terminal_receipt_gate: + Mutex>>, } #[derive(Debug, Default)] diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 80d8118d6..fe7a65e7e 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -555,9 +555,17 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { /// `WorkerController`. #[derive(Default)] struct RuntimeWorkerObservationHub { + #[cfg(test)] + registration_gate: Mutex>, workers: Mutex>, } +#[cfg(test)] +struct RestoreHubRegistrationGate { + gate: crate::execution::RestoreTestGate, + fail: bool, +} + #[derive(Clone)] struct RuntimeObservedWorker { workspace_id: Option, @@ -584,6 +592,17 @@ impl RuntimeWorkerObservationHub { sink: handle.sink.clone(), }, ); + drop(workers); + #[cfg(test)] + { + let gate = self.registration_gate.lock().unwrap().take(); + if let Some(gate) = gate { + gate.gate.wait(handle.clone()); + if gate.fail { + return Err("injected post-insert hub registration failure".into()); + } + } + } Ok(()) } @@ -837,6 +856,7 @@ struct FailedRestoreResources { #[cfg(test)] #[derive(Default)] struct RestoreFaults { + effects_calls: std::sync::atomic::AtomicUsize, fail_at: Mutex>, executions: std::sync::atomic::AtomicUsize, cleanup_fail: AtomicBool, @@ -2694,6 +2714,11 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate { failure.io_kind = io_kind; failure }; + #[cfg(test)] + self.factory + .restore_faults + .effects_calls + .fetch_add(1, Ordering::SeqCst); prepared.apply_effects().await.map_err(&map_error)?; self.factory .failed_restore_resources @@ -3392,6 +3417,8 @@ struct SubjectControllerReservation { /// Worker execution backend backed by real worker crate Workers. pub struct WorkerRuntimeExecutionBackend { + #[cfg(test)] + bridge_fault: Mutex>>, backend_id: String, factory: Arc, working_directory_materializer: Option>, @@ -3427,6 +3454,8 @@ where .build() .map_err(|err| format!("failed to build worker adapter runtime: {err}"))?; Ok(Self { + #[cfg(test)] + bridge_fault: Mutex::default(), backend_id: DEFAULT_BACKEND_ID.to_string(), factory: Arc::new(factory), working_directory_materializer: None, @@ -4161,6 +4190,14 @@ where .map_err(|error| error.to_string())?; bridge.snapshot_pending = false; } + #[cfg(test)] + { + let fault = self.bridge_fault.lock().unwrap().take(); + if let Some(fault) = fault { + fault.wait(handle.clone()); + return Err("injected bridge task admission failure after snapshot".into()); + } + } let streams = bridge.streams.clone(); let bridge_context = bridge.context.clone(); let bridge_worker_state = worker_state.clone(); diff --git a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs index 9a905c4bd..42c1c9153 100644 --- a/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_publication_tests.rs @@ -1499,4 +1499,6 @@ mod publication { fixture.assert_released(); fixture.assert_quiet(); } + + include!("worker_backend_restore_registration_tests.rs"); } diff --git a/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs b/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs new file mode 100644 index 000000000..a57d80dcf --- /dev/null +++ b/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs @@ -0,0 +1,889 @@ +// Included by the production Profile publication suite. Each fault is at a +// distinct partial-publication boundary; no factory result is substituted. + +fn publication_gate() -> ( + mpsc::Sender<()>, + mpsc::Receiver, + crate::execution::RestoreTestGate, +) { + let (release, receiver) = mpsc::channel(); + let (entered, evidence) = mpsc::channel(); + ( + release, + evidence, + crate::execution::RestoreTestGate { + entered, + release: receiver, + }, + ) +} + +fn assert_single_publication(fixture: &Fixture, expected: usize) { + let faults = &fixture.backend.factory.restore_faults; + assert_eq!( + faults.executions.load(Ordering::SeqCst), + expected, + "construction reran" + ); + assert_eq!( + faults.effects_calls.load(Ordering::SeqCst), + expected, + "restore effects reran" + ); + assert_eq!( + faults.publication_attempts.load(Ordering::SeqCst), + expected, + "Controller start reran" + ); + fixture.assert_quiet(); +} + +fn assert_real_attach( + fixture: &Fixture, + handle: &WorkerHandle, +) -> crate::execution::WorkerProtocolTransport { + let worker_ref = &fixture.request.worker_ref; + assert!(matches!( + fixture + .backend + .worker_session(crate::execution::WorkerSessionObservationRequest { + worker_ref: worker_ref.clone(), + }), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + let transport = fixture + .backend + .clone() + .attach_worker_protocol(worker_ref) + .unwrap(); + transport.validate().unwrap(); + let paired = subscribe_worker_protocol_session(handle); + assert_eq!( + serde_json::to_value(&transport.snapshot).unwrap(), + serde_json::to_value(paired.snapshot_event).unwrap() + ); + assert!( + matches!(&transport.snapshot, Event::Snapshot { session, greeting, state, in_flight, .. } + if session == &session_store::public_snapshot::project_current_session_snapshot(&fixture.entries()) + && !greeting.tools.is_empty() && state.state == protocol::WorkerState::Idle && in_flight.is_empty()) + ); + transport +} + +fn observe_publication_markers( + fixture: &Fixture, + observations: tokio::sync::broadcast::Receiver, + markers: &'static [&'static str], +) -> tokio::sync::broadcast::Receiver { + fixture + .backend + .run_on_adapter_runtime(async move { + let mut observations = observations; + let mut counts = vec![0; markers.len()]; + let mut snapshots = 0; + tokio::time::timeout(WAIT, async { + loop { + let observed = observations.recv().await.unwrap(); + if matches!(observed.payload, Event::Snapshot { .. }) { + snapshots += 1; + } + for (index, marker) in markers.iter().enumerate() { + if is_marker(&observed.payload, marker) { + counts[index] += 1; + } + } + if counts.last() == Some(&1) { + break; + } + } + }) + .await + .map_err(|_| format!("lost original subscription markers: {counts:?}"))?; + assert_eq!( + counts, + vec![1; markers.len()], + "lost or duplicated prefix on original receiver" + ); + assert_eq!(snapshots, 1, "initial full snapshot was lost or replayed"); + while let Ok(event) = observations.try_recv() { + assert!( + !markers + .iter() + .any(|marker| is_marker(&event.payload, marker)), + "duplicate after tail fence" + ); + } + Ok(observations) + }) + .unwrap() +} + +fn partial_registration_recovers_same_controller(hub: bool) { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let pause = runtime.test_pause_restore_continuation(); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let observations = runtime.subscribe_worker_observation().unwrap(); + let (started_release, started_entered) = activation_gate(&fixture, true); + let first = runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!( + first.state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let started = started_entered.recv_timeout(WAIT).unwrap(); + let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started); + let old = started.handle.as_ref().unwrap().clone(); + let canonical = fixture.canonical_files(); + alert_marker(&old, "registration-original-paired-prefix"); + let (release, entered, fault) = publication_gate(); + if hub { + *fixture + .backend + .factory + .observation_hub + .registration_gate + .lock() + .unwrap() = Some(RestoreHubRegistrationGate { + gate: fault, + fail: true, + }); + } else { + *fixture.backend.bridge_fault.lock().unwrap() = Some(fault); + } + started_release.release(); + drop(pause); + let boundary_handle = entered.recv_timeout(WAIT).unwrap(); + assert!(boundary_handle.same_controller(&old)); + let durable = durable_operation(&runtime, &worker_ref).unwrap(); + assert_eq!(durable.operation_id, started.operation_id); + assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed); + assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null()); + if hub { + let (_, shared, _) = fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .unwrap(); + assert!( + Arc::ptr_eq(&shared, &old.shared_state), + "real hub insert preceded fault" + ); + } else { + assert!( + fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .is_none() + ); + } + let (operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref); + assert_eq!(operation, started.operation_id); + assert!(!uncertain); + assert!( + context + .test_buffered_events() + .iter() + .any(|event| matches!(event, Event::Snapshot { .. })) + ); + alert_marker(&old, "registration-during-fault"); + let pause = runtime.test_pause_restore_continuation(); + release.send(()).unwrap(); + // This lock also observes the end of the actual registration attempt, not + // merely the test's entered notification. Its retained scope owns cleanup. + let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(old.same_controller(&execution.handle)); + assert_eq!(execution.restore_operation_id, Some(operation)); + assert_eq!( + execution.tasks.tasks.lock().unwrap().len(), + if hub { 2 } else { 1 } + ); + let bridge = execution.tasks.protocol_bridge.lock().unwrap(); + if hub { + assert!(bridge.is_none()); + } else { + let bridge = bridge.as_ref().unwrap(); + assert!( + !bridge.snapshot_pending, + "snapshot already registered, do not replay it" + ); + let streams = bridge.streams.lock().unwrap(); + assert!( + streams.is_some(), + "original paired streams survived failed task admission" + ); + assert_eq!( + serde_json::to_value(&streams.as_ref().unwrap().snapshot_event).unwrap(), + serde_json::to_value(&started.snapshot).unwrap() + ); + } + drop(bridge); + assert!( + !old.protocol_is_closed(), + "publication failure must not clean a committed Controller" + ); + assert_single_publication(&fixture, 1); + drop(pause); + runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT); + assert!(durable_operation(&runtime, &worker_ref).is_none()); + let owner = restore_owner(&runtime, &worker_ref, &request.request_id); + assert_eq!( + owner["operation_id"], + serde_json::to_value(operation).unwrap() + ); + assert_eq!(owner["receipt"]["state"], "accepted"); + assert!( + fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .is_none() + ); + assert!(!pending.task_in_flight.load(Ordering::Acquire)); + assert!(pending.activated.lock().unwrap().is_none()); + assert!(pending.result.lock().unwrap().is_none()); + assert!(pending.task.try_lock().unwrap().is_none()); + assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); + assert!(old.same_controller(&fixture.backend.workers.lock().unwrap()[&worker_ref].handle)); + let transport = assert_real_attach(&fixture, &old); + assert_eq!(fixture.canonical_files(), canonical); + assert_single_publication(&fixture, 1); + alert_marker(&old, "registration-tail-fence"); + let observations = observe_publication_markers( + &fixture, + observations, + &[ + "registration-original-paired-prefix", + "registration-during-fault", + "registration-tail-fence", + ], + ); + assert_eq!( + runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_single_publication(&fixture, 1); + // Retain the original transport/handle and scope through Stop/replacement. + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + assert!(old.protocol_is_closed()); + assert!(transport.validate().is_err()); + let replacement_request = runtime.test_restore_request(&worker_ref); + assert_eq!( + runtime + .restore_worker_operation_scoped(scope, &worker_ref, replacement_request.clone()) + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + let replacement = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(!old.same_controller(&replacement.handle)); + assert_ne!(replacement.restore_operation_id, Some(operation)); + assert!( + transport + .dispatch(Method::Pause { + command: protocol::WorkerCommandEnvelope::new(801) + }) + .is_err() + ); + assert!( + transport + .dispatch(Method::Shutdown { + command: protocol::WorkerCommandEnvelope::new(802) + }) + .is_err() + ); + for method in [ + Method::Pause { + command: protocol::WorkerCommandEnvelope::new(803), + }, + Method::Shutdown { + command: protocol::WorkerCommandEnvelope::new(804), + }, + ] { + let old_command = old.clone(); + assert!( + fixture + .backend + .run_on_adapter_runtime(async move { + old_command + .send(method) + .await + .map_err(|error| error.to_string()) + }) + .is_err(), + "closed original Controller command must not route to replacement" + ); + } + fixture + .backend + .factory + .deactivate_stopped_controller(&worker_ref, &old) + .unwrap(); + let (_, shared, _) = fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .unwrap(); + assert!(Arc::ptr_eq(&shared, &replacement.handle.shared_state)); + alert_marker(&old, "registration-stopped-event-must-not-leak"); + alert_marker(&replacement.handle, "registration-replacement-fence"); + fixture + .backend + .run_on_adapter_runtime(async move { + let mut observations = observations; + tokio::time::timeout(WAIT, async { + loop { + let event = observations.recv().await.unwrap(); + assert!(!is_marker( + &event.payload, + "registration-stopped-event-must-not-leak" + )); + if is_marker(&event.payload, "registration-replacement-fence") { + break; + } + } + }) + .await + .map_err(|_| "replacement observation fence missing".to_string())?; + drop(transport); + Ok(()) + }) + .unwrap(); + assert_single_publication(&fixture, 2); + assert_eq!( + runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!(restore_owner_count(&runtime, &worker_ref), 2); + let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(current.handle.same_controller(&replacement.handle)); + assert_eq!( + current.restore_operation_id, + replacement.restore_operation_id + ); + assert_eq!( + replacement.handle.shared_state.snapshot().state, + protocol::WorkerState::Idle + ); + drop(current); + drop(execution); + drop(replacement); + drop(pending); + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); +} + +fn restore_owner_count(runtime: &EmbeddedRuntime, worker_ref: &WorkerRef) -> usize { + let saved = runtime + .test_runtime_store() + .read_worker_record(worker_ref) + .unwrap(); + serde_json::to_value(saved.restore_guard).unwrap()["owners"] + .as_object() + .unwrap() + .len() +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn bridge_partial_snapshot_registration_retains_original_streams_and_autonomously_recovers() { + partial_registration_recovers_same_controller(false); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn hub_partial_insert_registration_retains_original_controller_and_autonomously_recovers() { + partial_registration_recovers_same_controller(true); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn context_partial_flush_retains_only_undelivered_queue_and_recovers_without_repost() { + let mut fixture = Fixture::no_model(); + Arc::get_mut(&mut fixture.backend) + .unwrap() + .spawn_restore_timeout = Duration::from_millis(250); + let runtime = stopped_runtime(&fixture); + let pause = runtime.test_pause_restore_continuation(); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.as_ref().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let observations = runtime.subscribe_worker_observation().unwrap(); + let (started_release, started_entered) = activation_gate(&fixture, true); + assert_eq!( + runtime + .restore_worker_operation_scoped(scope, &worker_ref, request.clone()) + .unwrap() + .state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let started = started_entered.recv_timeout(WAIT).unwrap(); + let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started); + let old = started.handle.unwrap(); + let (operation, context, _) = runtime.test_pending_restore_context(&worker_ref); + // Stop after the actual bridge and hub registration, without failing that + // boundary, so the flush begins with its real original full snapshot. + let (hub_release, hub_entered, hub_gate) = publication_gate(); + *fixture + .backend + .factory + .observation_hub + .registration_gate + .lock() + .unwrap() = Some(RestoreHubRegistrationGate { + gate: hub_gate, + fail: false, + }); + started_release.release(); + drop(pause); + assert!( + hub_entered + .recv_timeout(WAIT) + .unwrap() + .same_controller(&old) + ); + let buffered_prefix = context.test_buffered_events(); + assert!(matches!( + buffered_prefix.first(), + Some(Event::Snapshot { .. }) + )); + let prefix_length = buffered_prefix.len(); + let marker = |text: &str| Event::Error { + code: protocol::ErrorCode::Internal, + message: text.into(), + }; + context + .publish_protocol_event(marker("flush-delivered")) + .unwrap(); + context + .publish_protocol_event(marker("flush-failed")) + .unwrap(); + context + .publish_protocol_event(marker("flush-pending")) + .unwrap(); + let (release, entered, gate) = publication_gate(); + let (restored, restored_queue) = mpsc::channel(); + context.test_fail_flush(crate::execution::RestoreFlushFault { + remaining_successes: prefix_length + 1, + gate, + restored, + }); + drop(hub_release); + assert!( + matches!(entered.recv_timeout(WAIT).unwrap(), Event::Error { message, .. } if message == "flush-failed") + ); + let partial_execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(old.same_controller(&partial_execution.handle)); + assert_eq!(partial_execution.restore_operation_id, Some(operation)); + assert_eq!(partial_execution.tasks.tasks.lock().unwrap().len(), 2); + let (_, shared, _) = fixture + .backend + .factory + .observation_hub + .get(&worker_ref) + .unwrap(); + assert!(Arc::ptr_eq(&shared, &old.shared_state)); + let pause = runtime.test_pause_restore_continuation(); + context + .publish_protocol_event(marker("flush-arrived-during-error")) + .unwrap(); + release.send(()).unwrap(); + let restored = restored_queue.recv_timeout(WAIT).unwrap(); + let undelivered: Vec<_> = restored + .iter() + .filter_map(|event| match event { + Event::Error { message, .. } if message.starts_with("flush-") => Some(message.as_str()), + _ => None, + }) + .collect(); + assert_eq!( + undelivered, + [ + "flush-failed", + "flush-pending", + "flush-arrived-during-error" + ] + ); + assert!( + !restored + .iter() + .any(|event| matches!(event, Event::Snapshot { .. })), + "delivered snapshot must not rejoin the failed suffix" + ); + assert_eq!( + serde_json::to_value(context.test_buffered_events()).unwrap(), + serde_json::to_value(&restored).unwrap() + ); + assert_eq!( + durable_operation(&runtime, &worker_ref) + .unwrap() + .operation_id, + operation + ); + assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null()); + // The next terminal gate observes successful retry only after the same + // queued owner has repaired its failed callback and remaining suffix. + let (terminal_release, terminal_entered, gate) = publication_gate(); + runtime.test_terminal_receipt_gate(gate); + drop(pause); + assert_eq!(terminal_entered.recv_timeout(WAIT).unwrap(), operation); + assert!(context.test_buffered_events().is_empty()); + assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null()); + assert_eq!( + durable_operation(&runtime, &worker_ref).unwrap().commit, + crate::runtime::RestoreCommitFact::Committed + ); + let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(old.same_controller(&execution.handle)); + assert_eq!(execution.restore_operation_id, Some(operation)); + drop(terminal_release); // RAII disconnect, not a second request. + runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT); + assert_eq!( + restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"], + "accepted" + ); + assert_real_attach(&fixture, &old); + fixture + .backend + .run_on_adapter_runtime(async move { + let mut observations = observations; + let mut messages = Vec::new(); + let mut snapshots = 0; + while let Ok(event) = observations.try_recv() { + match event.payload { + Event::Error { message, .. } if message.starts_with("flush-") => { + messages.push(message) + } + Event::Snapshot { .. } => snapshots += 1, + _ => {} + } + } + assert_eq!( + messages, + [ + "flush-delivered", + "flush-failed", + "flush-pending", + "flush-arrived-during-error" + ] + ); + assert_eq!(snapshots, 1); + Ok(()) + }) + .unwrap(); + assert_single_publication(&fixture, 1); + drop(partial_execution); + drop(execution); + drop(pending); + runtime + .stop_worker_scoped(scope, &worker_ref, None) + .unwrap(); + fixture.assert_released(); +} + +#[derive(Clone, Copy, Debug)] +enum TerminalFault { + BeforeRename, + AfterRename, + SyncUnknown, + ReadUnknown, +} + +fn published_terminal_receipt_fault(fault: TerminalFault) { + let mut fixture = Fixture::no_model(); + // Reuse the existing ordinary non-network model boundary, not the product + // Codex profile. All fault/convergence assertions precede the sole input. + let script = MockClient::sequential(vec![MockResponse::Complete(simple_text_events())]); + let injected = script.clone(); + let factory = fixture + .backend + .factory + .as_ref() + .clone() + .with_spawn_model_client_factory(move || Box::new(injected.clone())); + fixture.backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap()); + let runtime = Arc::new(stopped_runtime(&fixture)); + let pause = runtime.test_pause_restore_continuation(); + let worker_ref = fixture.request.worker_ref.clone(); + let scope = fixture.request.workspace_scope.clone().unwrap(); + let request = runtime.test_restore_request(&worker_ref); + let observations = runtime.subscribe_worker_observation().unwrap(); + let (release, entered, gate) = publication_gate(); + runtime.test_terminal_receipt_gate(gate); + let restoring = { + let runtime = runtime.clone(); + let worker_ref = worker_ref.clone(); + let scope = scope.clone(); + let request = request.clone(); + std::thread::spawn(move || { + runtime.restore_worker_operation_scoped(&scope, &worker_ref, request) + }) + }; + let operation = entered.recv_timeout(WAIT).unwrap(); + // Restore legitimately repairs derived metadata before publication. Receipt + // failures must not rerun those effects; compare from this actual boundary. + let canonical = fixture.canonical_files(); + let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + let old = execution.handle.clone(); + assert_eq!(execution.restore_operation_id, Some(operation)); + assert!( + fixture + .backend + .pending_restore(&worker_ref) + .unwrap() + .is_none() + ); + assert_eq!(execution.tasks.tasks.lock().unwrap().len(), 2); + assert!(execution.tasks.protocol_bridge.lock().unwrap().is_none()); + assert_eq!( + durable_operation(&runtime, &worker_ref) + .unwrap() + .operation_id, + operation + ); + assert_eq!( + durable_operation(&runtime, &worker_ref).unwrap().commit, + crate::runtime::RestoreCommitFact::Committed + ); + let owner = restore_owner(&runtime, &worker_ref, &request.request_id); + assert!(owner["receipt"].is_null()); + let (same_operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref); + assert_eq!(same_operation, operation); + assert!(!uncertain); + assert!(context.test_buffered_events().is_empty()); + let transport = assert_real_attach(&fixture, &old); + assert_single_publication(&fixture, 1); + alert_marker(&old, "terminal-before-receipt"); + let store = runtime.test_runtime_store(); + let reconcile_before = store.test_worker_reconcile_counts(); + store.fail_next_worker_write(if matches!(fault, TerminalFault::BeforeRename) { + AtomicWriteFault::BeforeRename + } else { + AtomicWriteFault::AfterRename + }); + match fault { + TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(), + TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(), + _ => {} + } + release.send(()).unwrap(); + let first = restoring.join().unwrap().unwrap(); + let reconcile_after = store.test_worker_reconcile_counts(); + let reconciliation = ( + reconcile_after.0 - reconcile_before.0, + reconcile_after.1 - reconcile_before.1, + ); + assert_eq!( + reconciliation, + match fault { + TerminalFault::BeforeRename => (0, 0), + TerminalFault::AfterRename => (1, 4), // canonical file plus Worker/workers/root directories + TerminalFault::SyncUnknown => (1, 1), + TerminalFault::ReadUnknown => (1, 0), + }, + "{fault:?}: actual canonical read/sync attempts" + ); + assert_eq!( + first.state, + if matches!(fault, TerminalFault::AfterRename) { + server_api::WorkerRestoreState::Accepted + } else { + server_api::WorkerRestoreState::ReconciliationRequired + }, + "{fault:?}: {first:?}" + ); + assert_eq!(fixture.canonical_files(), canonical); + assert!(!old.protocol_is_closed()); + transport.validate().unwrap(); + assert_single_publication(&fixture, 1); + let disk_owner = restore_owner(&runtime, &worker_ref, &request.request_id); + assert_eq!(disk_owner["operation_id"], owner["operation_id"]); + match fault { + TerminalFault::BeforeRename => { + assert!(disk_owner["receipt"].is_null()); + assert!(!runtime.test_pending_restore_context(&worker_ref).2); + } + TerminalFault::AfterRename => { + assert_eq!(disk_owner["receipt"]["state"], "accepted"); + assert!(durable_operation(&runtime, &worker_ref).is_none()); + } + TerminalFault::SyncUnknown | TerminalFault::ReadUnknown => { + assert_eq!( + disk_owner["receipt"]["state"], "accepted", + "visible rename alone is not durable proof" + ); + let (same_operation, _, fenced) = runtime.test_pending_restore_context(&worker_ref); + assert_eq!(same_operation, operation); + assert!( + fenced, + "unknown canonical outcome must retain the memory owner/write fence" + ); + // Keep durability unproved for one competing Stop as well: it must + // fail before any Shutdown, never manufacture a Stopped judgment. + match fault { + TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(), + TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(), + _ => unreachable!(), + } + assert!(matches!( + runtime.stop_worker_scoped(&scope, &worker_ref, None), + Err(crate::error::RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); + assert!(!old.protocol_is_closed()); + assert!(!execution.shutdown_requested.load(Ordering::Acquire)); + assert!(runtime.test_pending_restore_context(&worker_ref).2); + assert_single_publication(&fixture, 1); + } + } + // Recovery is autonomous after the failed write / confirmed unknown fence. + // No POST, fresh request, or input is needed to make it converge. + drop(pause); + if !matches!(fault, TerminalFault::AfterRename) { + runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT); + } + assert!(durable_operation(&runtime, &worker_ref).is_none()); + assert_eq!( + restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"], + "accepted" + ); + let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone(); + assert!(old.same_controller(¤t.handle)); + assert_eq!(current.restore_operation_id, Some(operation)); + assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); + assert_real_attach(&fixture, &old); + assert_single_publication(&fixture, 1); + alert_marker(&old, "terminal-after-recovery"); + let _observations = observe_publication_markers( + &fixture, + observations, + &["terminal-before-receipt", "terminal-after-recovery"], + ); + let accepted = runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request.clone()) + .unwrap(); + assert_eq!(accepted.state, server_api::WorkerRestoreState::Accepted); + assert_eq!( + runtime + .restore_worker_operation_scoped(&scope, &worker_ref, request.clone()) + .unwrap(), + accepted + ); + assert_eq!(restore_owner_count(&runtime, &worker_ref), 1); + let mut stale = request; + stale.request_id = "terminal-fresh-stale-intent".into(); + assert!(matches!( + runtime.restore_worker_operation_scoped(&scope, &worker_ref, stale), + Err(crate::error::RuntimeError::RestoreObservationConflict { .. }) + )); + assert_single_publication(&fixture, 1); + assert_eq!( + script.call_count.load(Ordering::SeqCst), + 0, + "receipt recovery cannot trigger input or a model call" + ); + // This transport was paired before the failed receipt write. Exercise its + // normal log-entry receiver after autonomous settlement, not a new attach. + transport + .dispatch(Method::submit_text( + "terminal-ordinary-input", + "terminal-ordinary-input", + )) + .unwrap(); + fixture + .backend + .run_on_adapter_runtime(async move { + let mut transport = transport; + let (mut admitted, mut committed, mut text, mut ended) = (false, false, false, false); + tokio::time::timeout(WAIT, async { + while !(admitted && committed && text && ended) { + match transport.events.recv().await.unwrap() { + Event::SubmissionAccepted { + submission_request_id, + .. + } if submission_request_id == "terminal-ordinary-input" => admitted = true, + Event::UserMessage { segments, .. } => { + assert_eq!( + segments, + vec![protocol::Segment::text("terminal-ordinary-input")] + ); + assert!(!committed, "paired log receiver duplicated input"); + committed = true; + } + Event::TextDone { text: output } => { + assert_eq!(output, "hello from worker"); + text = true; + } + Event::RunEnd { .. } => ended = true, + Event::SubmissionRejected { .. } | Event::Error { .. } => { + panic!("ordinary input failed after receipt recovery") + } + _ => {} + } + } + }) + .await + .map_err(|_| { + "ordinary input failed to settle on original paired transport".to_string() + })?; + drop(transport); + Ok(()) + }) + .unwrap(); + assert_eq!(script.call_count.load(Ordering::SeqCst), 1); + assert_eq!(fixture.entries().iter().filter(|entry| matches!(entry, + LogEntry::AnnotatedUserInput { segments, .. } if segments == &vec![protocol::Segment::text("terminal-ordinary-input")] + )).count(), 1); + assert_single_publication(&fixture, 1); + drop(execution); + drop(current); + runtime + .stop_worker_scoped(&scope, &worker_ref, None) + .unwrap(); + assert!(old.protocol_is_closed()); + fixture.assert_released(); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn published_controller_terminal_receipt_before_rename_autonomously_settles_without_reconstruction() +{ + published_terminal_receipt_fault(TerminalFault::BeforeRename); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn published_controller_terminal_receipt_after_rename_is_canonically_reread_and_synced() { + published_terminal_receipt_fault(TerminalFault::AfterRename); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn published_controller_terminal_receipt_sync_unknown_retains_owner_and_write_fence() { + published_terminal_receipt_fault(TerminalFault::SyncUnknown); +} + +#[test] +#[serial_test::serial(worker_allocation)] +fn published_controller_terminal_receipt_read_unknown_retains_owner_and_write_fence() { + published_terminal_receipt_fault(TerminalFault::ReadUnknown); +} From 1e88893ef92574a69be665b29718b30ccb8b67d9 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 14:31:45 +0900 Subject: [PATCH 36/46] fix: preserve non-restore cleanup evidence through bridge refactoring --- crates/worker-runtime/src/worker_backend.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index fe7a65e7e..da734f13a 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -4377,6 +4377,7 @@ where return match cleanup { Ok(()) => WorkerExecutionSpawnResult::RolledBack(result), Err(_) => { + let retained_state = worker_state.read().ok().map(|state| state.clone()); self.retain_uncertain_unconnected_controller( &worker_ref, handle, @@ -4390,7 +4391,7 @@ where ); WorkerExecutionSpawnResult::ReconciliationRequired { result, - worker_state: None, + worker_state: retained_state, workdir_attachments: workdir_attachment_statuses, } } From b19de45a8237ce0f75347a832235d293f6808d43 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 14:40:58 +0900 Subject: [PATCH 37/46] test: verify original catalog projection after receipt reconciliation --- ...rker_backend_restore_registration_tests.rs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs b/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs index a57d80dcf..b6767d951 100644 --- a/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_registration_tests.rs @@ -628,6 +628,12 @@ fn published_terminal_receipt_fault(fault: TerminalFault) { let scope = fixture.request.workspace_scope.clone().unwrap(); let request = runtime.test_restore_request(&worker_ref); let observations = runtime.subscribe_worker_observation().unwrap(); + let mut catalog = runtime + .subscribe_event_selector_scoped( + &scope, + protocol::subscription::EventSubscriptionSelector::RuntimeWorkers, + ) + .unwrap(); let (release, entered, gate) = publication_gate(); runtime.test_terminal_receipt_gate(gate); let restoring = { @@ -772,6 +778,38 @@ fn published_terminal_receipt_fault(fault: TerminalFault) { assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1); assert_real_attach(&fixture, &old); assert_single_publication(&fixture, 1); + // A canonical terminal reread may bypass the final redundant upsert. The + // original subscriber must still have the current state and observation token. + let fresh_catalog = runtime + .subscribe_event_selector_scoped( + &scope, + protocol::subscription::EventSubscriptionSelector::RuntimeWorkers, + ) + .unwrap(); + let protocol::subscription::SubscriptionSnapshot::Workers { workers } = + fresh_catalog.snapshot() + else { + panic!("Runtime catalog must return Workers"); + }; + assert_eq!(workers.len(), 1); + let mut last_applied = None; + use futures::FutureExt; + while let Some(update) = catalog.recv().now_or_never() { + match update.unwrap().payload { + protocol::subscription::SubscriptionEventPayload::WorkerUpserted { worker } => { + last_applied = Some(worker); + } + payload => panic!("unexpected catalog update: {payload:?}"), + } + } + assert!( + serde_json::to_value(last_applied.expect("restore must publish its catalog state")) + .unwrap() + == serde_json::to_value(&workers[0]).unwrap(), + "original catalog subscriber diverged after {fault:?}" + ); + drop(fresh_catalog); + drop(catalog); alert_marker(&old, "terminal-after-recovery"); let _observations = observe_publication_markers( &fixture, From 8d47d2670bba4999c15ddea3a57d650383364f14 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 14:43:38 +0900 Subject: [PATCH 38/46] fix: initialize reconciliation fault evidence in offline store construction --- crates/worker-runtime/src/test_support.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/worker-runtime/src/test_support.rs b/crates/worker-runtime/src/test_support.rs index ca9192f8f..229815241 100644 --- a/crates/worker-runtime/src/test_support.rs +++ b/crates/worker-runtime/src/test_support.rs @@ -378,6 +378,10 @@ impl StoppedWorkerTemplate { worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)), #[cfg(test)] worker_reconcile_sync_fault: Default::default(), + #[cfg(test)] + worker_reconcile_read_fault: Default::default(), + #[cfg(test)] + worker_reconcile_counts: Default::default(), }; let result = (|| { let worker = PersistedWorkerRecord { From 6f5a42466137868b3032f7ba4e7a50efef49e96c Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 19:37:26 +0900 Subject: [PATCH 39/46] fix: reclaim stale allocation authority before restoring committed run artifacts --- crates/worker-runtime/src/worker_backend.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index da734f13a..927e26b1a 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -2922,8 +2922,15 @@ impl ProfileRuntimeWorkerFactory { .map_err(|_| "restore run artifacts cannot be validated")?; let path = worker::runtime::worker_allocation::default_allocation_path() .map_err(|_| "restore allocation authority unavailable")?; - let table = worker::runtime::worker_allocation::LockFileGuard::open(&path) + let mut table = worker::runtime::worker_allocation::LockFileGuard::open(&path) .map_err(|_| "restore allocation authority unavailable")?; + // Activated process death leaves a durable allocation entry. + // Use the same stale-owner authority as normal registration; + // a live or uninspectable owner must still fence reclamation. + worker::runtime::worker_allocation::reclaim_stale(&mut table); + table + .save() + .map_err(|_| "restore stale allocation cleanup unconfirmed")?; if table .data() .find(&Self::runtime_worker_name_for_ref(&request.worker_ref)) From b3209c2012f07a33cd87f436f78df41c6e88cefa Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 19:38:24 +0900 Subject: [PATCH 40/46] test: verify real orphan pin repair across owned restore process death --- .../worker-runtime/src/restore_checkpoints.rs | 1 + crates/worker-runtime/src/runtime.rs | 6 + .../src/runtime/restore_checkpoint.rs | 3 +- .../src/runtime/restore_checkpoint_tests.rs | 12 +- .../worker_backend_restore_candidate_tests.rs | 34 +++ tests/e2e/README.md | 4 +- tests/e2e/tests/worker_restore/checkpoint.rs | 2 + tests/e2e/tests/worker_restore/repair.rs | 246 ++++++++++++++++++ tests/e2e/tests/worker_restore/seed.rs | 18 +- 9 files changed, 321 insertions(+), 5 deletions(-) create mode 100644 tests/e2e/tests/worker_restore/repair.rs diff --git a/crates/worker-runtime/src/restore_checkpoints.rs b/crates/worker-runtime/src/restore_checkpoints.rs index 91321afa6..0232eb976 100644 --- a/crates/worker-runtime/src/restore_checkpoints.rs +++ b/crates/worker-runtime/src/restore_checkpoints.rs @@ -25,6 +25,7 @@ pub const MAX_RESTORE_CHECKPOINT_ACK_BYTES: usize = 4096; pub enum RestoreCheckpointStage { PreparedNotCommitted, CommittedPreterminal, + ActivatedPreterminal, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 755509f7b..8eef2a877 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -2871,6 +2871,12 @@ impl Runtime { sanitize_worker_create_failure_message(&error), )); } + #[cfg(feature = "restore-checkpoints")] + self.test_restore_checkpoint( + worker_ref, + operation_id, + crate::restore_checkpoints::RestoreCheckpointStage::ActivatedPreterminal, + ); if let Err(error) = self.activate_restore_candidate_context(worker_ref, &candidate_context) { diff --git a/crates/worker-runtime/src/runtime/restore_checkpoint.rs b/crates/worker-runtime/src/runtime/restore_checkpoint.rs index 96da2eeb9..d94aa4fa1 100644 --- a/crates/worker-runtime/src/runtime/restore_checkpoint.rs +++ b/crates/worker-runtime/src/runtime/restore_checkpoint.rs @@ -75,7 +75,8 @@ impl Runtime { }; let expected = match config.stage { RestoreCheckpointStage::PreparedNotCommitted => RestoreCommitFact::NotCommitted, - RestoreCheckpointStage::CommittedPreterminal => RestoreCommitFact::Committed, + RestoreCheckpointStage::CommittedPreterminal + | RestoreCheckpointStage::ActivatedPreterminal => RestoreCommitFact::Committed, }; if config.worker_id != worker_ref.worker_id || pending.operation_id != operation_id diff --git a/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs b/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs index 7369d6c6b..8103c2fe0 100644 --- a/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs +++ b/crates/worker-runtime/src/runtime/restore_checkpoint_tests.rs @@ -14,6 +14,7 @@ fn checkpoint_stages_ack_canonical_decision_and_eof_resumes_same_operation_once( (RestoreCheckpointStage::PreparedNotCommitted, "release"), (RestoreCheckpointStage::CommittedPreterminal, "eof"), (RestoreCheckpointStage::CommittedPreterminal, "timeout"), + (RestoreCheckpointStage::ActivatedPreterminal, "release"), ] { let dir = tempfile::tempdir().unwrap(); let backend = Arc::new(TestExecutionBackend::default()); @@ -79,14 +80,19 @@ fn checkpoint_stages_ack_canonical_decision_and_eof_resumes_same_operation_once( match stage { RestoreCheckpointStage::PreparedNotCommitted => RestoreCheckpointCommit::NotCommitted, - RestoreCheckpointStage::CommittedPreterminal => RestoreCheckpointCommit::Committed, + RestoreCheckpointStage::CommittedPreterminal + | RestoreCheckpointStage::ActivatedPreterminal => RestoreCheckpointCommit::Committed, } ); assert_eq!( backend.restore_operation_ids.lock().unwrap().as_slice(), &[ack.operation_id] ); - assert!(backend.activation_operation_ids.lock().unwrap().is_empty()); + if stage == RestoreCheckpointStage::ActivatedPreterminal { + assert_eq!(backend.activation_operation_ids.lock().unwrap().as_slice(), &[ack.operation_id]); + } else { + assert!(backend.activation_operation_ids.lock().unwrap().is_empty()); + } assert!(received.try_recv().is_err()); let canonical = runtime_store(&runtime) .read_durable_worker_record(&worker.worker_ref) @@ -255,6 +261,8 @@ fn checkpoint_verification_requires_canonical_worker_operation_commit_and_absent .restore_checkpoint_ack(&config, &worker.worker_ref, operation) .is_err() ); + config.stage = RestoreCheckpointStage::ActivatedPreterminal; + assert!(runtime.restore_checkpoint_ack(&config, &worker.worker_ref, operation).is_err()); config.stage = RestoreCheckpointStage::PreparedNotCommitted; runtime_store(&runtime).fail_next_worker_reconcile_sync(); assert!( diff --git a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs index 690a4025f..894beee9b 100644 --- a/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs +++ b/crates/worker-runtime/src/worker_backend_restore_candidate_tests.rs @@ -557,6 +557,40 @@ mod restore_candidate { } } + #[test] + #[serial_test::serial(worker_allocation)] + fn restore_run_reclamation_preserves_live_allocation_owner() { + let fixture = Fixture::no_model(); + let factory = fixture.backend.factory.clone(); + let request = fixture.request.clone(); + let run = factory + .worker_restore_run_dir(&request.worker_ref, request.operation_id) + .unwrap(); + fs::create_dir_all(&run).unwrap(); + fs::write(run.join("retained-owner"), b"live allocation").unwrap(); + let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&request.worker_ref); + let allocation = worker::runtime::worker_allocation::install_top_level( + name.clone(), std::process::id(), run.join("worker.sock"), + Vec::new(), fixture.segment_id, + ).unwrap(); + let failure = fixture.backend.run_on_adapter_runtime(async move { + match factory.prepare_restore(request).await { + Err(failure) => Ok(failure), + Ok(_) => Err("live allocation unexpectedly reclaimed".into()), + } + }).unwrap(); + assert_eq!(failure.stage, RuntimeRestoreStage::Controller); + assert_eq!(failure.cause, "restore run directory still has an allocation owner"); + let table = worker::runtime::worker_allocation::LockFileGuard::open( + &worker::runtime::worker_allocation::default_allocation_path().unwrap(), + ).unwrap(); + assert_eq!(table.data().find(&name).unwrap().pid, std::process::id()); + assert_eq!(fs::read(run.join("retained-owner")).unwrap(), b"live allocation"); + drop(table); + drop(allocation); + fixture.assert_quiet(); + } + #[test] #[serial_test::serial(worker_allocation)] fn session_availability_requires_attachable_current_controller() { diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 4622758ab..857a28a09 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -41,7 +41,7 @@ process終了ケースはそのAccepted receipt確認後に環境が所有する ## Opt-in Runtime checkpoint process試験 -上のfeatureless製品4case（target内の既存seed projection試験1件とは別）に加え、`restore-checkpoints`を明示した実Runtimeで2つのlive windowを試験する。このfeatureはoffline fixture-supportと独立しており、通常RuntimeではCLI入口も環境変数による入口も無い。 +上のfeatureless製品4case（target内の既存seed projection試験1件とは別）に加え、`restore-checkpoints`を明示した実Runtimeでdecision前後とactivation後・terminal前のlive windowを試験する。このfeatureはoffline fixture-supportと独立しており、通常RuntimeではCLI入口も環境変数による入口も無い。 ```sh # 同期を有効にしたRuntimeだけを専用targetへ。通常製品binaryを上書きしない。 @@ -55,6 +55,8 @@ cargo test --locked -p yoi-e2e --features e2e,restore-checkpoints \ 共有harnessはloopback ackのstage/Worker/operation/PID/nonceを確認し、直ちに所有childをkill/reapする。**offline正本を確認してから**同roots/identity/trustを再起動するため、ack後のdeadlineとの競合を成功扱いしない。NotCommittedの実私有候補は同operationのcleanup-only→RolledBack/Stopped、Committed/preterminalは同じdecision→実protocol/full Session snapshot→Acceptedへ回収する。元のrequest/token、owner数1、同operation、stale fresh intent拒否、保存source hash、restore run path数、最終cleanupを検証する。JSONでpending状態を作らず、seedは初回のみ。 +orphan upload pinの代表repairは、隔離copyへ正規Session storeのupload/pin APIで作った未結付けpinを、実Profileのcommit後activationが解放する経路を使う。Prepared窓は未修復のsource保持、Activated窓は実修復bytes/hashの保持を区別し、kill/reap後に正本を読んでから再起動する。materialized headの固定資産を巻き戻さないため、この試験は先行repair済みNotCommittedのprocess証明ではない。Activated後のstale allocationは通常登録と同じallocation authorityで回収し、live ownerが残る場合はrun artifactsを回収しない。 + 明示解除・peer EOF・observer scopeの意図的panicも試験する。gateの8秒deadlineとscoped HTTP callの15秒deadlineが待機を制限し、通常operationの継続後に環境のRAIIがprocess/rootを回収する。これはtest scope失敗の回収証明であり、OSによるharness強制終了や電源断の一般的な保証ではない。原fixtureへのSubmit/Notifyや実モデル呼出しは行わない。checkpoint ack/at-death/recovery、kill/reap、binary/hashとcleanup artifactはfeaturelessログと分けて報告する。 通常のharness回帰試験（製品processは起動しない）: diff --git a/tests/e2e/tests/worker_restore/checkpoint.rs b/tests/e2e/tests/worker_restore/checkpoint.rs index e55e815be..312c5beaa 100644 --- a/tests/e2e/tests/worker_restore/checkpoint.rs +++ b/tests/e2e/tests/worker_restore/checkpoint.rs @@ -1,4 +1,6 @@ //! Opt-in real-process checkpoint cases; never substitute the featureless lane. +#[path = "repair.rs"] +mod repair; use super::*; use std::collections::BTreeMap; use std::io::{Read, Write}; diff --git a/tests/e2e/tests/worker_restore/repair.rs b/tests/e2e/tests/worker_restore/repair.rs new file mode 100644 index 000000000..9e502d642 --- /dev/null +++ b/tests/e2e/tests/worker_restore/repair.rs @@ -0,0 +1,246 @@ +//! Real orphan-pin repair on an installed fixed-asset copy, not a synthetic +//! pending head. Only the postcommit activation path can reconcile these pins. +use super::*; +use session_store::{Store, UploadedFileLimits, WorkerSessionStore}; + +const PIN_OWNER: &str = "interrupted-pin-before-queue-checkpoint"; +const UPLOAD: &[u8] = b"isolated orphan upload; never model input"; + +#[test] +fn orphan_pin_not_committed_process_death_preserves_unrepaired_source() -> Result<()> { + repair_crash(RestoreCheckpointStage::PreparedNotCommitted) +} + +#[test] +fn orphan_pin_activated_process_death_preserves_real_repair_and_committed_decision() -> Result<()> { + repair_crash(RestoreCheckpointStage::ActivatedPreterminal) +} + +fn repair_crash(stage: RestoreCheckpointStage) -> Result<()> { + let fixture = Fixture::load()?; + let result = (|| { + let mut environment = WorkerE2eEnvironment::new()?; + let mut peer = Peer::configure(&mut environment, &fixture, stage)?; + let session_id = fixture::support(fixture.manifest.session_id.parse())?; + let mut upload = None; + let mut installed = seed::start_in_with_source_setup(&fixture, environment, |aggregate| { + let store = fixture::support(WorkerSessionStore::new(aggregate.join("session")))?; + let reference = fixture::support(store.write_uploaded_file( + session_id, + "orphan.txt", + "text/plain", + UPLOAD, + UploadedFileLimits::default(), + ))?; + fixture::support(store.pin_uploaded_file(session_id, &reference, PIN_OWNER))?; + upload = Some(reference); + Ok(()) + })?; + let upload = upload.ok_or_else(|| failure("offline upload setup not called"))?; + let relative = format!("session/artifacts/paste/{}.file.json", upload.artifact_id); + let pin_path = aggregate_path(&installed.environment, &fixture).join(&relative); + let pin_before: serde_json::Value = serde_json::from_slice(&fs::read(&pin_path)?)?; + assert_eq!(pin_before["pending_owner_id"], PIN_OWNER); + let before = saved_source_hashes(&installed.environment, &fixture)?; + let initial = canonical(&installed.environment, &fixture)?; + let token = assert_single_state(&installed.environment, &fixture, "stopped")? + .restore_observation_token + .ok_or_else(|| failure("missing original token"))?; + let request_id = "real-orphan-repair-crash-request"; + let committed = stage == RestoreCheckpointStage::ActivatedPreterminal; + let call = installed.environment.pending_restore_call( + &fixture.manifest.worker_id, + request_id, + &token, + )?; + let (ack, at_death) = std::thread::scope(|scope| -> Result<_> { + let request = scope.spawn(call); + let ack = peer.acknowledge(&installed.environment)?; + installed.environment.crash_runtime()?; + let saved = canonical(&installed.environment, &fixture)?; + let op = &saved["execution_state"]["execution"]["intent"]; + let owner = &saved["restore_guard"]["owners"][request_id]; + assert_eq!(saved["restore_guard"]["active_request_id"], request_id); + assert_eq!(op["operation_id"], serde_json::to_value(ack.operation_id)?); + assert_eq!(owner["operation_id"], op["operation_id"]); + assert_eq!(owner["request"]["expected_observation_token"], token); + assert!(owner["receipt"].is_null()); + assert_eq!( + op["commit"], + if committed { + "committed" + } else { + "not_committed" + } + ); + assert_eq!( + ack.commit, + if committed { + RestoreCheckpointCommit::Committed + } else { + RestoreCheckpointCommit::NotCommitted + } + ); + // The durable Committed decision authorizes activation effects. The + // precommit repair callback is not used for a materialized head. + assert_eq!(op["source_repair_possible"], false); + let at_death = saved_source_hashes(&installed.environment, &fixture)?; + let pin_after: serde_json::Value = serde_json::from_slice(&fs::read(&pin_path)?)?; + let mut expected_pin = pin_before.clone(); + if committed { + expected_pin + .as_object_mut() + .unwrap() + .remove("pending_owner_id"); + } + assert_eq!( + pin_after, expected_pin, + "real repair changes only orphan ownership" + ); + if committed { + assert_ne!( + at_death[&relative], before[&relative], + "actual pin bytes must change" + ); + } else { + assert_eq!( + at_death, before, + "preparation cannot repair pin or metadata" + ); + } + // In both windows the original conversation is immutable. + for (path, hash) in &before { + if path == "session/session.json" || path.starts_with("session/segments/") { + assert_eq!(at_death.get(path), Some(hash), "{path}"); + } + } + let store = fixture::support(WorkerSessionStore::open_read_only( + aggregate_path(&installed.environment, &fixture).join("session"), + ))?; + assert_eq!( + fixture::support(store.read_uploaded_file(session_id, &upload))?, + UPLOAD + ); + assert_eq!( + restore_runs(&installed.environment, &fixture)?, + vec![format!("restore-{}", ack.operation_id)] + ); + fs::write( + installed + .environment + .artifacts_dir + .join("repair-at-death.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "ack":ack, "initial_execution":initial["execution_state"], + "canonical_commit":op["commit"], "source_repair_possible":op["source_repair_possible"], + "operation_id":op["operation_id"], "owner_operation_id":owner["operation_id"], + "receipt_absent":owner["receipt"].is_null(), "original_request_and_token":true, + "pin_pending_owner_before":pin_before["pending_owner_id"], + "pin_pending_owner_after":pin_after["pending_owner_id"], + "hashes_before":before, "hashes_at_death":at_death, + "restore_runs":restore_runs(&installed.environment, &fixture)?, + }))?, + )?; + let outcome = request + .join() + .map_err(|_| failure("restore caller panicked"))?; + if let Ok(value) = outcome { + let receipt: WorkerRestoreResponse = serde_json::from_value(value)?; + assert_ne!(receipt.result.state, WorkerRestoreState::Accepted); + } + installed.environment.restart_runtime_after_crash()?; + assert_ne!(installed.environment.runtime_pid(), Some(ack.pid)); + Ok((ack, at_death)) + })?; + drop(peer); + let recovered = installed.environment.restore_worker( + &fixture.manifest.worker_id, + request_id, + &token, + )?; + let receipt: WorkerRestoreResponse = serde_json::from_value(recovered.clone())?; + fs::write( + installed + .environment + .artifacts_dir + .join("repair-first-recovery.json"), + serde_json::to_vec_pretty(&recovered)?, + )?; + assert_eq!( + receipt.result.state, + if committed { + WorkerRestoreState::Accepted + } else { + WorkerRestoreState::RolledBack + } + ); + assert_eq!( + installed.environment.restore_worker( + &fixture.manifest.worker_id, + request_id, + &token + )?, + recovered + ); + let final_saved = canonical(&installed.environment, &fixture)?; + let owners = final_saved["restore_guard"]["owners"] + .as_object() + .ok_or_else(|| failure("missing owners"))?; + assert_eq!(owners.len(), 1); + let owner = &owners[request_id]; + assert_eq!( + owner["operation_id"], + serde_json::to_value(ack.operation_id)? + ); + assert_eq!(owner["request"]["expected_observation_token"], token); + assert_eq!( + owner["receipt"]["state"], + if committed { "accepted" } else { "rolled_back" } + ); + let after = saved_source_hashes(&installed.environment, &fixture)?; + assert_eq!( + after, at_death, + "restart/replay cannot restore the orphan pin or change saved source" + ); + let stale = installed.environment.restore_worker( + &fixture.manifest.worker_id, + "fresh-stale-repair-intent", + &token, + ); + assert!(matches!(stale, Err(error) if error.to_string().contains("HTTP 409"))); + fs::write( + installed + .environment + .artifacts_dir + .join("repair-recovery.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "ack":ack, "state":receipt.result.state, "diagnostics":receipt.result.diagnostics, + "operation_id":owner["operation_id"], "owner_count":owners.len(), + "hashes_after_recovery":after, "restore_runs":restore_runs(&installed.environment, &fixture)?, + "same_original_request_and_token":true, "fresh_stale_rejected":true, + }))?, + )?; + if committed { + assert_single_state(&installed.environment, &fixture, "idle")?; + assert_eq!( + restore_runs(&installed.environment, &fixture)?, + vec![format!("restore-{}", ack.operation_id)] + ); + let mut subscription = WorkerSessionSubscription::connect( + &installed.environment, + &fixture.manifest.worker_id, + &installed.snapshot, + )?; + subscription.assert_alive()?; + stop_and_cleanup(installed, vec![subscription], &fixture) + } else { + assert_single_state(&installed.environment, &fixture, "stopped")?; + assert!(restore_runs(&installed.environment, &fixture)?.is_empty()); + let report = installed.environment.cleanup()?; + assert!(report.cleanup_success && !report.fixture_root_exists_after); + Ok(()) + } + })(); + fixture.source.assert_unchanged()?; + result +} diff --git a/tests/e2e/tests/worker_restore/seed.rs b/tests/e2e/tests/worker_restore/seed.rs index f95085b06..3640db189 100644 --- a/tests/e2e/tests/worker_restore/seed.rs +++ b/tests/e2e/tests/worker_restore/seed.rs @@ -45,7 +45,17 @@ pub fn start(fixture: &Fixture) -> Result { start_in(fixture, WorkerE2eEnvironment::new()?) } -pub fn start_in(fixture: &Fixture, mut environment: WorkerE2eEnvironment) -> Result { +pub fn start_in(fixture: &Fixture, environment: WorkerE2eEnvironment) -> Result { + start_in_with_source_setup(fixture, environment, |_| Ok(())) +} + +// The callback runs only on the installed isolation copy, while Runtime is offline. +// Restart never calls the seed or this setup again. +pub fn start_in_with_source_setup( + fixture: &Fixture, + mut environment: WorkerE2eEnvironment, + setup: impl FnOnce(&Path) -> Result<()>, +) -> Result { super::phase(&environment, "fixture-copy-verify")?; let copy = environment.root.join("provided-fixture"); fixture.source.copy_to(©)?; @@ -114,6 +124,12 @@ pub fn start_in(fixture: &Fixture, mut environment: WorkerE2eEnvironment) -> Res backend_job: None, }; support(template.install(&options, request, &bootstrap.manifest))?; + setup( + &options + .root + .join("workers") + .join(template.worker_id().to_string()), + )?; let sessions = support(session_store::WorkerSessionStore::open_read_only( options .root From 15c3006d09029ccdc9e437d9bc4fd55a576334b6 Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 19:59:56 +0900 Subject: [PATCH 41/46] fix: propagate fallible restore publication through server factory adapters --- .../src/server/tests/restored_parent_control_tests.rs | 4 ++-- .../src/server/tests/subjektiv_journal_connected_tests.rs | 4 ++-- .../workspace-server/src/server_workspace_config_wip_tests.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs index d1a4d6c61..e8249cf8b 100644 --- a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs +++ b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs @@ -138,9 +138,9 @@ impl RuntimeWorkerFactory for CapturingFactory { worker_ref: &worker_runtime::identity::WorkerRef, workspace_id: Option<&str>, handle: &worker::WorkerHandle, - ) { + ) -> std::result::Result<(), String> { self.inner - .activate_restored_controller(worker_ref, workspace_id, handle); + .activate_restored_controller(worker_ref, workspace_id, handle) } } diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs index 3b9215113..7c5fe0442 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs @@ -1084,9 +1084,9 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for RestoreAuthFactory worker_ref: &worker_runtime::identity::WorkerRef, workspace_id: Option<&str>, handle: &worker::WorkerHandle, - ) { + ) -> std::result::Result<(), String> { self.inner - .activate_restored_controller(worker_ref, workspace_id, handle); + .activate_restored_controller(worker_ref, workspace_id, handle) } } diff --git a/crates/workspace-server/src/server_workspace_config_wip_tests.rs b/crates/workspace-server/src/server_workspace_config_wip_tests.rs index 9e970e17d..7c719a648 100644 --- a/crates/workspace-server/src/server_workspace_config_wip_tests.rs +++ b/crates/workspace-server/src/server_workspace_config_wip_tests.rs @@ -636,9 +636,9 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto worker_ref: &worker_runtime::identity::WorkerRef, workspace_id: Option<&str>, handle: &worker::WorkerHandle, - ) { + ) -> std::result::Result<(), String> { self.inner - .activate_restored_controller(worker_ref, workspace_id, handle); + .activate_restored_controller(worker_ref, workspace_id, handle) } } From e84057f402649f7c4f2316db396fd96ea940d63e Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 20:39:28 +0900 Subject: [PATCH 42/46] fix: retain exact restore ownership in test execution backends --- crates/tui/src/backend_worker_picker.rs | 25 ++++++++++++++++++++++++ crates/workspace-server/src/server.rs | 26 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/crates/tui/src/backend_worker_picker.rs b/crates/tui/src/backend_worker_picker.rs index 7812f85e3..f8fa5336c 100644 --- a/crates/tui/src/backend_worker_picker.rs +++ b/crates/tui/src/backend_worker_picker.rs @@ -785,6 +785,12 @@ mod tests { #[derive(Default)] struct PendingRuntimeBackend { operations: std::sync::Mutex>, + owners: std::sync::Mutex< + std::collections::HashMap< + worker_runtime::execution::WorkerLifecycleOperationId, + worker_runtime::identity::WorkerRef, + >, + >, } impl worker_runtime::execution::WorkerExecutionBackend for PendingRuntimeBackend { @@ -806,6 +812,21 @@ mod tests { ) } + fn restore_recovery_disposition( + &self, + operation_id: worker_runtime::execution::WorkerLifecycleOperationId, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> Option { + use worker_runtime::execution::RestoreRecoveryDisposition as D; + Some( + if self.owners.lock().unwrap().get(&operation_id) == Some(worker_ref) { + D::Continue + } else { + D::Absent + }, + ) + } + fn restore_worker( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, @@ -813,6 +834,10 @@ mod tests { use worker_runtime::execution::*; let mut operations = self.operations.lock().unwrap(); operations.push(request.operation_id); + self.owners + .lock() + .unwrap() + .insert(request.operation_id, request.worker_ref.clone()); if operations.len() == 1 { WorkerExecutionSpawnResult::ReconciliationRequired { result: WorkerExecutionResult::errored( diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index a8ddde657..175feb3e8 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -48392,6 +48392,12 @@ mod tests { std::sync::Mutex>, reconcile_operations: std::sync::Mutex>, + restore_owners: std::sync::Mutex< + std::collections::HashMap< + worker_runtime::execution::WorkerLifecycleOperationId, + worker_runtime::identity::WorkerRef, + >, + >, inputs: std::sync::Mutex>, protocol_methods: Arc>>, @@ -48425,6 +48431,7 @@ mod tests { restore_failure: std::sync::Mutex::new(None), restore_operations: std::sync::Mutex::new(Vec::new()), reconcile_operations: std::sync::Mutex::new(Vec::new()), + restore_owners: Default::default(), inputs: std::sync::Mutex::new(Vec::new()), protocol_methods: Arc::new(std::sync::Mutex::new(Vec::new())), stops: std::sync::Mutex::new(Vec::new()), @@ -48629,6 +48636,10 @@ mod tests { .lock() .unwrap() .push(request.operation_id); + self.restore_owners + .lock() + .unwrap() + .insert(request.operation_id, request.worker_ref.clone()); if let Some(result) = self.restore_failure.lock().unwrap().take() { return result; } @@ -48664,6 +48675,21 @@ mod tests { } } + fn restore_recovery_disposition( + &self, + operation_id: worker_runtime::execution::WorkerLifecycleOperationId, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> Option { + use worker_runtime::execution::RestoreRecoveryDisposition as D; + Some( + if self.restore_owners.lock().unwrap().get(&operation_id) == Some(worker_ref) { + D::Continue + } else { + D::Absent + }, + ) + } + fn reconcile_restore( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, From eb24cb04dca26c9b1bded604fca4afd8459e8d0c Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 20:39:28 +0900 Subject: [PATCH 43/46] fix: revalidate Host authority when publishing prepared restore controllers --- crates/worker/src/bootstrap.rs | 7 ++ .../subjektiv_journal_connected_tests.rs | 64 +++++++++++++++++-- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/crates/worker/src/bootstrap.rs b/crates/worker/src/bootstrap.rs index 2a591cb3f..cb712e11d 100644 --- a/crates/worker/src/bootstrap.rs +++ b/crates/worker/src/bootstrap.rs @@ -465,6 +465,13 @@ where WorkerError::RestoreCandidateClosing, )); } + // Preparation's read-only authorization is not a publication grant. + // Revalidate current Host authority after commit, before effects/input, + // including a publication retry on this same private Controller. + initialized + .worker + .validate_subjektiv_body_publication() + .map_err(WorkerBootstrapError::Worker)?; if !self.effects_applied { self.activation_failed = true; self.activation diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs index 7c5fe0442..cc8895777 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs @@ -1088,6 +1088,14 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for RestoreAuthFactory self.inner .activate_restored_controller(worker_ref, workspace_id, handle) } + + fn deactivate_stopped_controller( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + handle: &worker::WorkerHandle, + ) -> std::result::Result<(), String> { + self.inner.deactivate_stopped_controller(worker_ref, handle) + } } struct RestoreAuthCandidate { @@ -1101,6 +1109,14 @@ impl worker_runtime::worker_backend::RuntimeRestoreCandidate for RestoreAuthCand self.inner.initial_state() } + fn has_durable_effects(&self) -> bool { + self.inner.has_durable_effects() + } + + fn can_retry_publication(&self) -> bool { + self.inner.can_retry_publication() + } + async fn activate( &mut self, ) -> std::result::Result< @@ -1346,6 +1362,32 @@ enabled = false ) -> worker_runtime::execution::WorkerExecutionSpawnResult { self.inner.restore_worker(request) } + fn restore_recovery_disposition( + &self, + id: worker_runtime::execution::WorkerLifecycleOperationId, + worker: &worker_runtime::identity::WorkerRef, + ) -> Option { + self.inner.restore_recovery_disposition(id, worker) + } + // This caller-replay fixture gates repair/prepare to inspect the durable + // lost-response state before its next explicit call. Runtime continuation + // itself is covered with the unwrapped backend in the Runtime suite. + fn supports_restore_continuation(&self) -> bool { + false + } + fn reclaim_uncommitted_restore( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.inner.reclaim_uncommitted_restore(request) + } + fn discard_uncommitted_restore( + &self, + id: worker_runtime::execution::WorkerLifecycleOperationId, + worker: &worker_runtime::identity::WorkerRef, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.inner.discard_uncommitted_restore(id, worker) + } fn reconcile_restore( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, @@ -1798,6 +1840,7 @@ async fn restore_after_real_prepare_rejects_revoked_host_trust_without_publicati "the same production signer must be accepted before trust revocation" ); requests.lock().unwrap().clear(); + fixture.callbacks.lock().unwrap().clear(); let runtime = api.runtime.clone(); let target = worker.clone(); let intent = request.clone(); @@ -1816,13 +1859,20 @@ async fn restore_after_real_prepare_rejects_revoked_host_trust_without_publicati "preparation must not consume saved input or append history" ); assert_eq!(metadata.read_by_name(&name).unwrap().unwrap(), saved); - assert!( - requests - .lock() - .unwrap() - .iter() - .all(|(path, _)| !path.ends_with("/subjektiv/sessions")), - "publication authorization belongs after candidate preparation" + { + let callbacks = fixture.callbacks.lock().unwrap(); + assert_eq!( + callbacks.len(), + 1, + "prepare verifies current Host authority" + ); + assert_eq!(callbacks[0]["create_if_missing"], false); + assert_eq!(callbacks[0]["session_id"], active.session_id.to_string()); + } + assert_eq!(probe.publications.load(SeqCst), 0); + assert_eq!( + store.subject_session_attributions(subject_id).unwrap(), + attributions ); let prepared_record: Value = serde_json::from_slice(&std::fs::read(aggregate.join("worker.json")).unwrap()).unwrap(); From e8d4e964cb1943402671f9e10834efd0cf983b1c Mon Sep 17 00:00:00 2001 From: Hare Date: Mon, 26 Oct 2026 21:25:46 +0900 Subject: [PATCH 44/46] fix: refresh Workdir API generated contract digest --- web/workspace/src/lib/generated/workdir-api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/workspace/src/lib/generated/workdir-api.ts b/web/workspace/src/lib/generated/workdir-api.ts index 99db0b0a5..04bfe3967 100644 --- a/web/workspace/src/lib/generated/workdir-api.ts +++ b/web/workspace/src/lib/generated/workdir-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-workdir-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7) +// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e) // Options: workdir-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/workdir-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_workdir_api_types -- --check From dddb21bd70f4097073712c1e5cb47bb4048b8361 Mon Sep 17 00:00:00 2001 From: Hare Date: Tue, 27 Oct 2026 01:24:50 +0900 Subject: [PATCH 45/46] fix: preserve possible repair during v9 Restore migration --- crates/worker-runtime/src/fs_store.rs | 131 ++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index 1278f89f7..cbc549383 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -1245,6 +1245,7 @@ fn migrate_worker_record( migrate_legacy_request_evidence(&mut identity_document) .map_err(|message| runtime_store_corrupt(identity_path, message.into()))?; migrate_restore_commit_fact(&mut identity_document, identity_path)?; + migrate_restore_repair_fact(&mut identity_document, identity_path)?; identity_document["schema_version"] = serde_json::Value::from(SCHEMA_VERSION); let aggregate: WorkerAggregateRecord = serde_json::from_value(identity_document).map_err(|error| { @@ -4097,6 +4098,136 @@ mod tests { (temp, opened.store, worker, path) } + #[test] + fn schema_v9_pending_restore_migrates_conservatively_and_survives_reopen() { + assert_v9_pending_restore_migration(false); + } + + #[test] + fn schema_v9_pending_stop_restore_migrates_conservatively_and_survives_reopen() { + assert_v9_pending_restore_migration(true); + } + + fn assert_v9_pending_restore_migration(under_stop: bool) { + // Real store migration/open/writer/reopen, with frozen v9 operation and + // admission shapes. No execution backend or recovery is substituted. + let (_temp, store, worker, path) = canonical_worker_store(); + let root = store.root.clone(); + let mut legacy = + serde_json::to_value(WorkerAggregateRecord::from_persisted(&worker)).unwrap(); + let execution = legacy["execution_state"]["execution"].clone(); + let restore_id = WorkerLifecycleOperationId::new(); + let restore = serde_json::json!({ + "operation_id": restore_id, + "mode": "explicit", + "request": execution["request"], + "binding": execution["binding"], + "last_settled_status": legacy["status"] + }); + let operation = if under_stop { + serde_json::json!({"operation": "stop", "intent": { + "operation_id": WorkerLifecycleOperationId::new(), + "pending_restore": restore, + "request": execution["request"], + "binding": execution["binding"], + "restore_intent": execution["restore_intent"], + "last_settled_status": legacy["status"] + }}) + } else { + serde_json::json!({"operation": "restore", "intent": restore}) + }; + legacy["execution_state"] = serde_json::json!({ + "state": "reconciliation_required", "execution": operation + }); + let token = legacy["restore_guard"]["observation_token"].clone(); + let owner = |request_id: &str, operation_id, receipt| { + serde_json::json!({ + "fingerprint": format!("intent-{request_id}"), + "operation_id": operation_id, + "request": {"expected_observation_token": token, "request_id": request_id}, + "preparation_applied": true, + "execution_started": true, + "receipt": receipt + }) + }; + legacy["restore_guard"] = serde_json::json!({ + "generation": token, + "active_request_id": "pending-restore", + "owners": { + "pending-restore": owner("pending-restore", serde_json::to_value(restore_id).unwrap(), serde_json::Value::Null), + "earlier-restore": owner("earlier-restore", serde_json::to_value(WorkerLifecycleOperationId::new()).unwrap(), + serde_json::json!({"state": "rolled_back", "worker": null, + "reason_code": "worker_restore_failed", "message": "retained failure"})) + } + }); + legacy["schema_version"] = serde_json::json!(AGGREGATE_SCHEMA_VERSION); + fs::write(&path, serde_json::to_vec(&legacy).unwrap()).unwrap(); + write_empty_schema_v8_store(&root); + let runtime_path = root.join(RUNTIME_FILE); + let mut runtime: serde_json::Value = read_json(&runtime_path, "read fixture").unwrap(); + runtime["schema_version"] = serde_json::json!(AGGREGATE_SCHEMA_VERSION); + fs::write(&runtime_path, serde_json::to_vec(&runtime).unwrap()).unwrap(); + let metadata_path = path.with_file_name(WORKER_METADATA_FILE); + let history_path = path.parent().unwrap().join("session/history.json"); + fs::create_dir_all(history_path.parent().unwrap()).unwrap(); + let metadata = serde_json::to_vec(&serde_json::json!({ + "worker_name": runtime_worker_name(worker.worker_id), + "retained_extension": {"source": "unchanged"} + })) + .unwrap(); + let history = b"[{\"role\":\"user\",\"content\":\"retained source\"}]\n"; + fs::write(&metadata_path, &metadata).unwrap(); + fs::write(&history_path, history).unwrap(); + drop(store); + + let mut expected = legacy; + expected["schema_version"] = serde_json::json!(SCHEMA_VERSION); + let guard = expected["restore_guard"].as_object_mut().unwrap(); + let token = guard.remove("generation").unwrap(); + guard.insert("observation_token".into(), token); + let intent = &mut expected["execution_state"]["execution"]["intent"]; + let restore = if under_stop { + &mut intent["pending_restore"] + } else { + intent + }; + restore["commit"] = serde_json::json!("legacy_unknown"); + restore["source_repair_possible"] = serde_json::json!(true); + + let opened = FsRuntimeStore::open_or_create(root.clone(), "test-runtime").unwrap(); + let persisted = opened.store.read_worker_record(&worker.worker_ref).unwrap(); + // Compare the persisted aggregate, including both operation IDs, full + // request/source references, token, pending owner and terminal receipt. + assert_eq!( + read_json::(&path, "read migrated Worker").unwrap(), + expected + ); + opened.store.write_worker_record(&persisted).unwrap(); + assert_eq!( + read_json::(&path, "read written Worker").unwrap(), + expected + ); + drop(opened); + let bytes = fs::read(&path).unwrap(); + let reopened = FsRuntimeStore::open_or_create(root, "test-runtime").unwrap(); + reopened + .store + .read_worker_record(&worker.worker_ref) + .unwrap(); + assert_eq!( + read_json::(&path, "read reopened Worker").unwrap(), + expected + ); + assert_eq!(fs::read(&path).unwrap(), bytes); + // v9 metadata is canonically reformatted; its retained values, unlike + // whitespace, are the preservation contract. + assert_eq!( + read_json::(&metadata_path, "read retained metadata").unwrap(), + serde_json::from_slice::(&metadata).unwrap() + ); + assert_eq!(fs::read(&history_path).unwrap(), history); + } + #[test] fn legacy_pending_restore_migrates_as_unknown_and_preserves_stop_ownership() { use crate::runtime::RestoreCommitFact; From 897c025b85c0c0ef418d9239e7ae5b4b913f9e66 Mon Sep 17 00:00:00 2001 From: Hare Date: Tue, 27 Oct 2026 08:03:43 +0900 Subject: [PATCH 46/46] feat(journal): execute frozen body inputs with durable delivery and erasure fences --- Cargo.lock | 3 + crates/runtime-api/Cargo.toml | 1 + crates/runtime-api/src/lib.rs | 45 + crates/session-store/src/capture.rs | 15 +- crates/session-store/src/capture/body.rs | 121 +- crates/session-store/src/capture/recording.rs | 425 ++++ crates/session-store/src/committed_replay.rs | 1346 +++++++++++++ crates/session-store/src/lib.rs | 1 + crates/session-store/src/public_material.rs | 1118 +---------- .../src/public_material/tests.rs | 1015 ++++++++++ crates/session-store/src/retention.rs | 12 +- crates/standalone/src/job_store.rs | 549 +++++- .../src/job_store/legacy_migrations.rs | 36 + crates/standalone/src/jobs.rs | 161 +- crates/standalone/src/jobs/tests.rs | 101 + crates/standalone/src/subjektiv.rs | 12 +- crates/standalone/src/subjektiv/journal.rs | 729 ++++--- .../standalone/src/subjektiv/journal/tests.rs | 859 ++++++-- crates/standalone/tests/host.rs | 12 +- crates/standalone/tests/subjektiv.rs | 52 +- crates/subjektiv/Cargo.toml | 2 + crates/subjektiv/src/journal/recording.rs | 213 ++ crates/subjektiv/src/journal/service.rs | 151 +- crates/subjektiv/src/journal/store.rs | 56 +- crates/subjektiv/src/journal/tests.rs | 1566 +++++++++++++++ crates/subjektiv/src/journal/transport.rs | 13 +- crates/subjektiv/src/lib.rs | 5 + .../src/http_server/runtime_management_api.rs | 25 + .../tests/session_material_tests.rs | 304 ++- crates/worker-runtime/src/runtime.rs | 127 ++ crates/worker-runtime/src/worker_backend.rs | 2 +- crates/worker/src/job/tests.rs | 10 +- .../src/assignment_tracking_tests.rs | 3 +- crates/workspace-server/src/backend_job.rs | 80 + .../src/frozen_content_state_tests.rs | 62 +- crates/workspace-server/src/hosts.rs | 47 + .../src/implementation_result_tests.rs | 14 +- .../src/job_notification_routing.rs | 76 +- crates/workspace-server/src/latest_schema.sql | 14 + crates/workspace-server/src/server.rs | 81 +- .../src/server/subjektiv_journal.rs | 733 +++---- .../subjektiv_journal_connected_tests.rs | 105 +- .../server/tests/subjektiv_journal_tests.rs | 1752 ++++++++++++++++- crates/workspace-server/src/store.rs | 452 ++++- .../src/ticket_item_checker.rs | 32 +- .../internal/subjektiv_journal_system.md | 10 +- 46 files changed, 10368 insertions(+), 2180 deletions(-) create mode 100644 crates/session-store/src/capture/recording.rs create mode 100644 crates/session-store/src/committed_replay.rs diff --git a/Cargo.lock b/Cargo.lock index fb65b50ba..a674c0ee9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3990,6 +3990,7 @@ dependencies = [ "protocol", "serde", "serde_json", + "session-store", "tokio", "workdir", ] @@ -4783,12 +4784,14 @@ version = "0.1.0" dependencies = [ "chrono", "feature-storage", + "job", "manifest", "memory", "rusqlite", "serde", "serde_json", "server-api", + "session-store", "sha2 0.11.0", "tempfile", "thiserror 2.0.18", diff --git a/crates/runtime-api/Cargo.toml b/crates/runtime-api/Cargo.toml index 4f250aad5..cc544d3e9 100644 --- a/crates/runtime-api/Cargo.toml +++ b/crates/runtime-api/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true api-macros = { workspace = true, features = ["reqwest", "axum"] } manifest.workspace = true protocol.workspace = true +session-store.workspace = true serde = { workspace = true, features = ["derive"] } workdir.workspace = true diff --git a/crates/runtime-api/src/lib.rs b/crates/runtime-api/src/lib.rs index 729243ba6..e0e3502ea 100644 --- a/crates/runtime-api/src/lib.rs +++ b/crates/runtime-api/src/lib.rs @@ -836,6 +836,34 @@ pub struct SessionMaterialPinResponse { pub pin_id: String, } +/// One body-bearing immutable slice. The byte budget includes slice metadata, +/// but not the enclosing availability enum or the caller's durable Job envelope. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SessionRecordingCaptureRequest { + pub workspace_id: String, + pub source: SessionPublicSource, + pub expected_session_id: String, + pub start: session_store::capture::RecordingPosition, + pub max_entries: usize, + pub max_serialized_bytes: usize, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "availability", rename_all = "snake_case")] +pub enum SessionRecordingCaptureAvailability { + NotMaterialized { + session_id: String, + }, + Captured { + slice: session_store::capture::RecordingCaptureSlice, + }, + Unavailable { + reason: SessionPublicUnavailableReason, + message: String, + }, +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SessionMaterialCatalogRequest { @@ -1246,6 +1274,16 @@ pub trait RuntimeApi { #[body] request: SessionMaterialPinRequest, ) -> Result; + #[post( + "/v1/session-public/recording-capture", + status = 200, + error_status = 400 + )] + async fn session_recording_capture( + &self, + #[body] request: SessionRecordingCaptureRequest, + ) -> Result; + #[post( "/v1/session-public/material/catalog", status = 200, @@ -1730,6 +1768,13 @@ mod tests { Err(test_error(501)) } + async fn session_recording_capture( + &self, + _request: SessionRecordingCaptureRequest, + ) -> Result { + Err(test_error(501)) + } + async fn session_material_catalog( &self, _request: SessionMaterialCatalogRequest, diff --git a/crates/session-store/src/capture.rs b/crates/session-store/src/capture.rs index 07da4c1f7..b010e3381 100644 --- a/crates/session-store/src/capture.rs +++ b/crates/session-store/src/capture.rs @@ -13,6 +13,14 @@ //! observation share only this domain; they do not enable or modify each other's Features. //! The provider owns authorization to recapture a current Worker. A capture itself proves neither //! completion nor approval and grants no business-mutation authority. +//! +//! Recording Hosts use the common committed replay through the guarded APIs below. +//! [`recording_catalog_worker_guarded`] and [`recording_catalog_fs_guarded`] expose +//! descriptors only to an admission callback under the caller's existing publication/ +//! erasure guard and persisted active selector, without allocating a full body DTO. +//! [`capture_recording_slice_worker_guarded`] and [`capture_recording_slice_fs_guarded`] +//! separately construct bounded body-bearing inputs. Admission descriptors are not +//! sufficient Job inputs; saved recording DTOs are read/sliced without live storage. use std::sync::Arc; @@ -21,12 +29,14 @@ use crate::{ LoggedSessionHistoryOrigin as HistoryOrigin, }; mod body; +mod recording; use agen::{HistoryEntry, Item, Role}; pub use body::*; use protocol::{ SessionContentPart, SessionEntryProvenance, SessionMessageRole, SessionSnapshot, SessionSnapshotEntryData, }; +pub use recording::*; use serde::{Deserialize, Serialize}; const DEFAULT_SEARCH_LIMIT: usize = 20; @@ -50,7 +60,8 @@ impl SessionEntryRef { pub fn parse(value: &str) -> Option { let suffix = value.strip_prefix('E')?; if suffix.is_empty() - || suffix.len() > 64 + // Legacy Host boundary identities contain both Session and Segment UUIDs. + || suffix.len() > 256 || !suffix .bytes() .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') @@ -822,7 +833,7 @@ impl SessionCapture { } } -fn public_provenance(origin: &HistoryOrigin) -> SessionEntryProvenance { +pub(crate) fn public_provenance(origin: &HistoryOrigin) -> SessionEntryProvenance { match origin { HistoryOrigin::HumanInput { .. } | HistoryOrigin::LocalHumanInput => { SessionEntryProvenance::HumanInput diff --git a/crates/session-store/src/capture/body.rs b/crates/session-store/src/capture/body.rs index eb7768622..d1ca3dfc1 100644 --- a/crates/session-store/src/capture/body.rs +++ b/crates/session-store/src/capture/body.rs @@ -5,7 +5,7 @@ use super::*; use serde::de::Error as _; use sha2::{Digest, Sha256}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::io::Write; pub const CAPTURE_VERSION: u32 = 1; @@ -81,6 +81,17 @@ impl CaptureContentPart { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum CaptureBody { + /// Canonical recording bytes retain the historical Journal descriptor contract. + /// A nonzero offset or short text is an explicit fragment, never a full body. + /// History entries retain their original raw segment-history row, including + /// hidden rows in the count. A typed Host boundary has no history row: its + /// `source.entry_index` is the next-history gap at the original boundary. + CanonicalPublic { + descriptor: crate::public_material::PublicMaterialDescriptor, + byte_offset: u64, + text: String, + boundary: Option, + }, Message { role: CaptureRole, content: Vec, @@ -175,6 +186,7 @@ impl CaptureBody { pub(super) fn to_item(&self) -> Item { match self { + Self::CanonicalPublic { text, .. } => Item::system_message(text.clone()), Self::Message { role, content } => Item::Message { id: None, status: None, @@ -214,6 +226,7 @@ impl CaptureBody { fn fields(&self) -> Vec<(RawField, &str)> { match self { + Self::CanonicalPublic { text, .. } => vec![(RawField::CanonicalPublic, text)], Self::Message { content, .. } => content .iter() .enumerate() @@ -235,6 +248,16 @@ impl CaptureBody { fn metadata(&self, field: RawField) -> CaptureBodyMetadata { match self { + Self::CanonicalPublic { + descriptor, + byte_offset, + boundary, + .. + } => CaptureBodyMetadata::CanonicalPublic { + descriptor: descriptor.clone(), + byte_offset: *byte_offset, + boundary: *boundary, + }, Self::Message { role, content } => CaptureBodyMetadata::Message { role: *role, refusal: matches!(field, RawField::MessagePart(i) if matches!(content.get(i), Some(CaptureContentPart::Refusal { .. }))), @@ -264,6 +287,9 @@ impl CaptureBody { #[serde(deny_unknown_fields)] pub struct CaptureSource { pub segment_id: String, + /// Original raw segment-history position, not captured/public view order. + /// For `CanonicalPublic` Host boundaries only, this is the next-history gap; + /// several typed Host events may share that gap with the following history row. pub entry_index: u64, pub provenance: SessionEntryProvenance, /// Full original origin when supplied by the committed-history adapter. @@ -375,7 +401,7 @@ impl Write for ByteCounter { } } -fn json_size(value: &T, limit: usize) -> Result { +pub(super) fn json_size(value: &T, limit: usize) -> Result { let mut counter = ByteCounter { bytes: 0, limit }; serde_json::to_writer(&mut counter, value).map_err(|_| CaptureError::ResourceLimit)?; Ok(counter.bytes) @@ -403,6 +429,7 @@ impl CaptureDto { identity(&self.segment_id, limits)?; let mut ids = HashSet::new(); let mut sources = HashSet::new(); + let mut recording_source_floor = HashMap::<&str, u64>::new(); let mut previous = None; for entry in &self.entries { identity(&entry.entry_id, limits)?; @@ -419,7 +446,16 @@ impl CaptureDto { } previous = Some(entry.order); identity(&entry.source.segment_id, limits)?; - if !sources.insert((&entry.source.segment_id, entry.source.entry_index)) { + let host_boundary = matches!( + &entry.body, + CaptureBody::CanonicalPublic { + boundary: Some(_), + .. + } + ); + if !host_boundary + && !sources.insert((&entry.source.segment_id, entry.source.entry_index)) + { return Err(CaptureError::Invalid("duplicate source position")); } if !entry.policy.any() { @@ -443,6 +479,79 @@ impl CaptureDto { } } match &entry.body { + CaptureBody::CanonicalPublic { + descriptor, + byte_offset, + text, + boundary, + } => { + if descriptor.identity.entry_ref != format!("E{}", entry.entry_id) + || descriptor.identity.segment_id.to_string() != entry.source.segment_id + || descriptor.canonical_order != entry.order + || text.is_empty() + || descriptor.total_bytes == 0 + || byte_offset + .checked_add(text.len() as u64) + .is_none_or(|end| end > descriptor.total_bytes) + || descriptor.sha256.len() != 64 + || !descriptor + .sha256 + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + || !entry.policy.recording_material + || entry.policy.memory_evidence + || entry.policy.observation_readable + { + return Err(CaptureError::Invalid("canonical descriptor")); + } + match (&descriptor.origin, &entry.source.history_origin, boundary) { + ( + crate::public_material::PublicMaterialOrigin::History { origin }, + Some(stored), + None, + ) if origin == stored => {} + ( + crate::public_material::PublicMaterialOrigin::HostRunBoundary, + None, + Some(_), + ) => {} + _ => return Err(CaptureError::Invalid("canonical origin")), + } + // A partial DTO cannot prove the hidden history extent, but can + // reject aliases and reversals of original per-Segment positions. + // Host events anchor a gap, not a fabricated history row. + let floor = recording_source_floor + .entry(&entry.source.segment_id) + .or_default(); + if entry.source.entry_index < *floor { + return Err(CaptureError::Invalid("recording source position order")); + } + *floor = if boundary.is_some() { + entry.source.entry_index + } else { + entry + .source + .entry_index + .checked_add(1) + .ok_or(CaptureError::Invalid("source position overflow"))? + }; + if *byte_offset == 0 && text.len() as u64 == descriptor.total_bytes { + let digest: String = Sha256::digest(text.as_bytes()) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + if digest != descriptor.sha256 { + return Err(CaptureError::Invalid("canonical digest")); + } + if let Some(boundary) = boundary { + let expected = + serde_json::json!({"kind":"run_boundary", "boundary":boundary}); + if serde_json::from_str::(text)? != expected { + return Err(CaptureError::Invalid("run boundary body")); + } + } + } + } CaptureBody::Message { content, .. } if content.is_empty() => { return Err(CaptureError::Invalid("empty message parts")); } @@ -493,6 +602,7 @@ pub struct ScopedSessionEntryRef { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum RawField { + CanonicalPublic, MessagePart(usize), PublicSystem, ToolArguments, @@ -515,6 +625,11 @@ pub struct RawRead { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum CaptureBodyMetadata { + CanonicalPublic { + descriptor: crate::public_material::PublicMaterialDescriptor, + byte_offset: u64, + boundary: Option, + }, Message { role: CaptureRole, refusal: bool, diff --git a/crates/session-store/src/capture/recording.rs b/crates/session-store/src/capture/recording.rs new file mode 100644 index 000000000..5a767da8c --- /dev/null +++ b/crates/session-store/src/capture/recording.rs @@ -0,0 +1,425 @@ +//! Host capture and pure canonical slicing. No Job state or live-reader fallback lives here. + +use super::*; +use crate::committed_replay::CommittedPublicReplay; +use crate::public_material::{ + PublicMaterialDescriptor, PublicMaterialError, PublicMaterialLimits, PublicMaterialUse, +}; +use crate::retention::SessionRetentionStore; +use crate::{SegmentId, SessionId}; +use std::path::Path; + +/// Continuation in canonical Journal order, not an observation/Memory view cursor. +/// A Host must reconcile the descriptor at this position with its saved checkpoint. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecordingPosition { + pub canonical_order: u64, + pub byte_offset: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecordingCaptureSlice { + pub capture: CaptureDto, + pub next: Option, +} + +/// Inspect recording admission descriptors without constructing a body-bearing DTO. +/// +/// The Host must authorize the source and hold the **same** Session publication/erasure +/// guard from before reading its persisted active pointer until this function returns +/// (including the callback). Pass that pointer's Segment, not a filename/mtime-derived +/// selector. This function verifies the guard and stored Session identities but does +/// not authorize the caller or establish that the supplied guard fences this root. +/// No lock is recursively acquired, no active pointer is reread, and no replay handle +/// or body-read capability is exposed to `inspect`. +/// +/// The common committed recording replay fixes per-file ends before decoding. It +/// retains only descriptors/locations, decoding one record at a time and streaming +/// canonical bodies into their digests. `replay_limits` bound record allocation, +/// aggregate storage extent, entries and Segments; there is no full-capture text/JSON +/// allocation or `CaptureLimits` ceiling. The callback receives only descriptors in +/// canonical order and may return a Host-specific mapping (including a `Result`). It +/// runs once on success and not at all on replay/identity/resource errors. +/// +/// This catalog is for Host admission, **not** a descriptor-only Job input. Construct +/// the Job's bounded body input with the guarded slice API at its own authorized +/// boundary; after saving it, execute only that immutable value. Neither descriptors +/// nor a completed callback authorize delayed adoption after erasure. +/// +/// ```no_run +/// # use std::path::Path; +/// # use session_store::{SessionId, SegmentId}; +/// # use session_store::retention::SessionRetentionGuard; +/// # use session_store::public_material::PublicMaterialError; +/// # fn admit(root: &Path, session: SessionId, active: SegmentId, +/// # guard: &SessionRetentionGuard) -> Result, PublicMaterialError> { +/// // `active` was resolved from persisted Host metadata while holding `guard`. +/// session_store::capture::recording_catalog_worker_guarded( +/// root, session, active, Default::default(), guard, +/// |descriptors| descriptors.iter().map(|d| d.total_bytes).collect(), +/// ) +/// # } +/// ``` +pub fn recording_catalog_worker_guarded( + session_root: &Path, + expected_session: SessionId, + active_segment: SegmentId, + replay_limits: PublicMaterialLimits, + guard: &crate::retention::SessionRetentionGuard, + inspect: impl FnOnce(&[PublicMaterialDescriptor]) -> T, +) -> Result { + if guard.session_id() != expected_session { + return Err(PublicMaterialError::Inconsistent); + } + let replay = CommittedPublicReplay::open_worker_guarded( + session_root, + active_segment, + replay_limits, + None, + PublicMaterialUse::Recording, + )?; + if replay.session_id() != expected_session { + return Err(PublicMaterialError::Inconsistent); + } + Ok(inspect(&replay.into_recording_descriptors())) +} + +/// FsStore-layout counterpart of [`recording_catalog_worker_guarded`]. +/// +/// `sessions_root` is the directory containing per-Session directories. Invoke this +/// inside the Host's `with_recording_boundary` callback, passing **its existing guard +/// and persisted active Segment**. That same external retention control store must +/// fence publishers, pointer changes, archive moves and erasure. Do not reacquire the +/// guard or use a separately opened legacy public reader inside the boundary. +/// All descriptor-only, allocation and callback guarantees of the Worker variant +/// apply; this function does not construct an unbounded transient `CaptureDto`. +pub fn recording_catalog_fs_guarded( + sessions_root: &Path, + session: SessionId, + active_segment: SegmentId, + replay_limits: PublicMaterialLimits, + guard: &crate::retention::SessionRetentionGuard, + inspect: impl FnOnce(&[PublicMaterialDescriptor]) -> T, +) -> Result { + if guard.session_id() != session { + return Err(PublicMaterialError::Inconsistent); + } + let replay = CommittedPublicReplay::open_fs_guarded( + sessions_root, + session, + active_segment, + replay_limits, + )?; + Ok(inspect(&replay.into_recording_descriptors())) +} + +/// Capture the selected original lineage, including typed Host run boundaries. +/// The caller must hold the Session publication/erasure guard while selecting the +/// active pointer and through this call. Runtime publishers use this same lock +/// for appends and Segment creation. No lock is recursively acquired here. +pub fn capture_recording_worker_guarded( + session_root: &Path, + expected_session: SessionId, + active_segment: SegmentId, + replay_limits: PublicMaterialLimits, + capture_limits: CaptureLimits, + guard: &crate::retention::SessionRetentionGuard, +) -> Result { + if guard.session_id() != expected_session { + return Err(PublicMaterialError::Inconsistent); + } + let replay = CommittedPublicReplay::open_worker_guarded( + session_root, + active_segment, + replay_limits, + None, + PublicMaterialUse::Recording, + )?; + if replay.session_id() != expected_session { + return Err(PublicMaterialError::Inconsistent); + } + replay.into_recording_dto(capture_limits) +} + +/// Standalone convenience entry; Hosts resolving a mutable active pointer should +/// instead hold the fence while resolving it and call the guarded entry. +pub fn capture_recording_worker( + session_root: &Path, + expected_session: SessionId, + active_segment: SegmentId, + retention: &SessionRetentionStore, + replay_limits: PublicMaterialLimits, + capture_limits: CaptureLimits, +) -> Result { + let guard = retention + .lock_live(expected_session) + .map_err(|error| match error { + crate::StoreError::SessionErased(_) => PublicMaterialError::Erased, + _ => PublicMaterialError::Unavailable, + })?; + capture_recording_worker_guarded( + session_root, + expected_session, + active_segment, + replay_limits, + capture_limits, + &guard, + ) +} + +/// Bounded construction for one Job input: descriptor replay is bounded separately; +/// only at most the requested body budget is materialized from the fixed files. +pub fn capture_recording_slice_worker_guarded( + session_root: &Path, + expected_session: SessionId, + active_segment: SegmentId, + replay_limits: PublicMaterialLimits, + start: RecordingPosition, + max_entries: usize, + max_serialized_bytes: usize, + guard: &crate::retention::SessionRetentionGuard, +) -> Result { + if guard.session_id() != expected_session { + return Err(PublicMaterialError::Inconsistent); + } + let replay = CommittedPublicReplay::open_worker_guarded( + session_root, + active_segment, + replay_limits, + None, + PublicMaterialUse::Recording, + )?; + if replay.session_id() != expected_session { + return Err(PublicMaterialError::Inconsistent); + } + replay.recording_slice(start, max_entries, max_serialized_bytes) +} + +/// The standalone FsStore layout uses the same committed replay and fixed values. +pub fn capture_recording_fs_guarded( + sessions_root: &Path, + session: SessionId, + active_segment: SegmentId, + replay_limits: PublicMaterialLimits, + capture_limits: CaptureLimits, + guard: &crate::retention::SessionRetentionGuard, +) -> Result { + if guard.session_id() != session { + return Err(PublicMaterialError::Inconsistent); + } + CommittedPublicReplay::open_fs_guarded(sessions_root, session, active_segment, replay_limits)? + .into_recording_dto(capture_limits) +} + +/// Bounded standalone capture; the caller holds its FsStore retention fence. +pub fn capture_recording_slice_fs_guarded( + sessions_root: &Path, + session: SessionId, + active_segment: SegmentId, + replay_limits: PublicMaterialLimits, + start: RecordingPosition, + max_entries: usize, + max_serialized_bytes: usize, + guard: &crate::retention::SessionRetentionGuard, +) -> Result { + if guard.session_id() != session { + return Err(PublicMaterialError::Inconsistent); + } + CommittedPublicReplay::open_fs_guarded(sessions_root, session, active_segment, replay_limits)? + .recording_slice(start, max_entries, max_serialized_bytes) +} + +impl CaptureDto { + /// Select the oldest canonical prefix fitting both budgets, including JSON + /// escaping, metadata and continuation. Only immutable owned bytes are read. + /// A later Job can recapture at `next`; never combine live pages into this value. + pub fn recording_slice( + &self, + start: RecordingPosition, + max_entries: usize, + max_serialized_bytes: usize, + ) -> Result { + if max_entries == 0 { + return Err(CaptureError::BudgetTooSmall); + } + if start.canonical_order > self.view_entry_count { + return Err(CaptureError::InvalidRange); + } + let mut result = RecordingCaptureSlice { + capture: CaptureDto { + version: self.version, + session_id: self.session_id.clone(), + segment_id: self.segment_id.clone(), + view_entry_count: self.view_entry_count, + entries: Vec::new(), + }, + next: (start.canonical_order < self.view_entry_count).then_some(start), + }; + if body::json_size(&result, max_serialized_bytes).is_err() { + return Err(CaptureError::BudgetTooSmall); + } + if start.canonical_order == self.view_entry_count { + if start.byte_offset != 0 { + return Err(CaptureError::InvalidRange); + } + result.capture.validate(CaptureLimits { + max_bytes: max_serialized_bytes, + max_text_bytes: max_serialized_bytes, + max_entries, + ..Default::default() + })?; + return Ok(result); + } + let first = self + .entries + .iter() + .position(|entry| entry.order == start.canonical_order) + .ok_or(CaptureError::InvalidRange)?; + let mut expected_order = start.canonical_order; + for entry in self.entries.iter().skip(first).take(max_entries) { + if entry.order != expected_order { + return Err(CaptureError::InvalidRange); + } + let CaptureBody::CanonicalPublic { + descriptor, + byte_offset, + text, + boundary, + } = &entry.body + else { + return Err(CaptureError::Invalid("not a recording capture")); + }; + if !entry.policy.recording_material { + return Err(CaptureError::PolicyDenied); + } + let offset = if entry.order == start.canonical_order { + start.byte_offset + } else { + *byte_offset + }; + let local = offset + .checked_sub(*byte_offset) + .and_then(|o| usize::try_from(o).ok()) + .ok_or(CaptureError::InvalidRange)?; + if local > text.len() + || !text.is_char_boundary(local) + || offset >= descriptor.total_bytes + { + return Err(CaptureError::InvalidRange); + } + let item = CaptureEntryDto { + entry_id: entry.entry_id.clone(), + order: entry.order, + source: entry.source.clone(), + policy: entry.policy, + body: CaptureBody::CanonicalPublic { + descriptor: descriptor.clone(), + byte_offset: offset, + text: String::new(), + boundary: *boundary, + }, + }; + result.capture.entries.push(item); + // At most budget bytes are considered before allocating a trial fragment. + let mut low = 0; + let mut high = text.len().saturating_sub(local).min(max_serialized_bytes); + let prior_next = result.next; + while low < high { + let mid = low + (high - low).div_ceil(2); + let mut end = local + mid; + while !text.is_char_boundary(end) { + end -= 1; + } + if let CaptureBody::CanonicalPublic { text: fragment, .. } = + &mut result.capture.entries.last_mut().unwrap().body + { + *fragment = text[local..end].to_owned(); + } + let next_offset = offset + (end - local) as u64; + result.next = if next_offset < descriptor.total_bytes { + Some(RecordingPosition { + canonical_order: entry.order, + byte_offset: next_offset, + }) + } else { + (entry.order + 1 < self.view_entry_count).then_some(RecordingPosition { + canonical_order: entry.order + 1, + byte_offset: 0, + }) + }; + if body::json_size(&result, max_serialized_bytes).is_ok() { + low = mid; + } else { + high = mid - 1; + } + } + let mut end = local + low; + while !text.is_char_boundary(end) { + end -= 1; + } + if end == local { + result.capture.entries.pop(); + result.next = prior_next; + if result.capture.entries.is_empty() { + return Err(CaptureError::BudgetTooSmall); + } + break; + } + if let CaptureBody::CanonicalPublic { text: fragment, .. } = + &mut result.capture.entries.last_mut().unwrap().body + { + *fragment = text[local..end].to_owned(); + } + let next_offset = offset + (end - local) as u64; + result.next = if next_offset < descriptor.total_bytes { + Some(RecordingPosition { + canonical_order: entry.order, + byte_offset: next_offset, + }) + } else { + (entry.order + 1 < self.view_entry_count).then_some(RecordingPosition { + canonical_order: entry.order + 1, + byte_offset: 0, + }) + }; + if next_offset < descriptor.total_bytes { + break; + } + expected_order = entry.order + 1; + } + body::json_size(&result, max_serialized_bytes).map_err(|_| CaptureError::BudgetTooSmall)?; + result.capture.validate(CaptureLimits { + max_bytes: max_serialized_bytes, + max_text_bytes: max_serialized_bytes, + max_entries, + ..Default::default() + })?; + Ok(result) + } + + /// Resolve canonical text already in the value. No original storage is consulted. + pub fn recording_text( + &self, + order: u64, + ) -> Result<(&crate::public_material::PublicMaterialDescriptor, u64, &str), CaptureError> { + let entry = self + .entries + .iter() + .find(|entry| entry.order == order) + .ok_or(CaptureError::UnknownReference)?; + if !entry.policy.recording_material { + return Err(CaptureError::PolicyDenied); + } + match &entry.body { + CaptureBody::CanonicalPublic { + descriptor, + byte_offset, + text, + .. + } => Ok((descriptor, *byte_offset, text)), + _ => Err(CaptureError::Invalid("not canonical recording text")), + } + } +} diff --git a/crates/session-store/src/committed_replay.rs b/crates/session-store/src/committed_replay.rs new file mode 100644 index 000000000..051096cca --- /dev/null +++ b/crates/session-store/src/committed_replay.rs @@ -0,0 +1,1346 @@ +//! Original public material from a saved, rollback-ineligible Session prefix. +//! +//! Unlike the Memory evidence index, this reader includes saved output from +//! interrupted runs and active runs with materialized AI activity. It does not +//! interpret domain-specific recording or completion policy. Callers authorize +//! the Session root and explicitly select its active Segment. +//! +//! Catalogs retain only descriptors and record locations, never original bodies. +//! Records are decoded one at a time under an explicit allocation ceiling, not +//! under a whole-run scan budget. Canonical bodies are streamed to SHA-256 and +//! bounded UTF-8 windows. A single JSONL record (including a Segment seed) must +//! fit `max_record_bytes`; exceeding that ceiling is an error, not a preview. + +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::{ + LogEntry, LoggedHistoryEntry, LoggedItem, LoggedRole, LoggedSessionHistoryOrigin, SegmentId, + SegmentOrigin, SessionId, WorkerSessionStore, +}; + +#[derive(Debug, thiserror::Error)] +pub enum PublicMaterialError { + #[error("public material storage: {0}")] + Storage(#[from] io::Error), + #[error("public material record: {0}")] + Record(#[from] serde_json::Error), + #[error("public material source is missing")] + Missing, + #[error("public material source was permanently erased")] + Erased, + #[error("public material source requires migration or is unavailable")] + Unavailable, + #[error("public material lineage or identity is inconsistent")] + Inconsistent, + #[error("public material resource ceiling exceeded")] + ResourceLimit, + #[error("public material changed since the catalog was read")] + Changed, + #[error("public material offset is not a UTF-8 boundary or is outside the body")] + InvalidOffset, +} + +#[derive(Debug, Clone, Copy)] +pub struct PublicMaterialLimits { + pub max_segments: usize, + /// Total history identities examined, including inherited and private items. + pub max_entries: usize, + /// Allocation ceiling for one persisted JSONL record, not an entire run. + pub max_record_bytes: u64, + /// Aggregate fixed storage extent, bounding replay metadata as well as body decoding. + pub max_storage_bytes: u64, +} + +impl Default for PublicMaterialLimits { + fn default() -> Self { + Self { + max_segments: 256, + max_entries: 1_000_000, + max_record_bytes: 64 * 1024 * 1024, + max_storage_bytes: 256 * 1024 * 1024, + } + } +} + +/// Original identity, never a body-based deduplication key. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicMaterialIdentity { + pub segment_id: SegmentId, + pub entry_ref: String, +} + +/// Public provenance distinguishes saved conversation from Host control facts. Run boundaries +/// are not model output, user intent, or evidence that the requested work succeeded. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum PublicMaterialOrigin { + History { origin: LoggedSessionHistoryOrigin }, + HostRunBoundary, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PublicRunOutcome { + Finished, + Paused, + Yielded, + LimitReached, +} + +/// Only whitelisted execution facts cross this boundary; raw error/provider messages do not. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum PublicRunBoundary { + Completed { + outcome: PublicRunOutcome, + interrupted: bool, + }, + Yielded, + Resumed, + Cancelled, + Errored, + Abandoned, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicMaterialDescriptor { + pub identity: PublicMaterialIdentity, + /// Zero-based position on this explicitly selected active lineage. Forks + /// may remove a suffix; callers must reconcile identities, not assume that + /// positions from a previous active lineage are interchangeable. + pub canonical_order: u64, + pub origin: PublicMaterialOrigin, + /// Lowercase hexadecimal SHA-256 of the complete canonical UTF-8 body. + pub sha256: String, + pub total_bytes: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublicMaterialContent { + pub content: String, + pub next_offset: u64, + pub total_bytes: u64, + pub sha256: String, +} + +#[derive(Clone)] +struct Location { + segment: SegmentId, + offset: u64, + // Index within this physical record (e.g. a Segment seed), used only for I/O. + history_index: usize, +} + +struct Material { + descriptor: PublicMaterialDescriptor, + location: Location, + // Original segment-history row, or the next-history gap for a Host boundary. + // Never overwritten when a correction moves the physical body location. + source_history_position: u64, + is_tool_result: bool, + boundary: Option, +} + +struct SegmentHeader { + id: SegmentId, + parent: Option, + compact: bool, + // Compaction's exact source-history IDs disambiguate mid-turn boundaries. + source_ids: Option>, + file_len: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PublicMaterialUse { + Recording, + MemoryEvidence, +} + +/// A fixed material catalog. Opening a new catalog observes newly appended +/// records; reading a body through this catalog verifies its saved digest and +/// byte length again. Missing/corrected/deleted sources are explicit failures. +/// Evidence eligibility never publishes private model text or Feature payload: +/// System inputs also require an explicit public projection for every public use. +pub struct CommittedPublicReplay { + session_id: SessionId, + active_segment_id: SegmentId, + layout: Layout, + limits: PublicMaterialLimits, + materials: Vec, + purpose: PublicMaterialUse, + file_ends: HashMap, + retention: Option, +} + +enum Layout { + Worker(PathBuf), + Fs(PathBuf), +} + +impl Layout { + fn path(&self, segment: SegmentId) -> PathBuf { + match self { + Self::Worker(root) => root.join("segments").join(format!("{segment}.jsonl")), + Self::Fs(root) => root.join(format!("{segment}.jsonl")), + } + } +} + +impl CommittedPublicReplay { + pub fn open_worker( + session_root: &Path, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + ) -> Result { + Self::open_worker_guarded( + session_root, + active_segment_id, + limits, + None, + PublicMaterialUse::Recording, + ) + } + + /// Uses the Host's aggregate-external control root for the whole scan and every body read. + /// The same store must fence publishers, archive transfer, and physical deletion. + pub fn open_worker_with_retention( + session_root: &Path, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + retention: crate::retention::SessionRetentionStore, + ) -> Result { + Self::open_worker_for_use( + session_root, + active_segment_id, + limits, + retention, + PublicMaterialUse::Recording, + ) + } + + /// Selects the evidence purpose independently of public display and observation. + pub fn open_worker_for_use( + session_root: &Path, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + retention: crate::retention::SessionRetentionStore, + purpose: PublicMaterialUse, + ) -> Result { + Self::open_worker_guarded( + session_root, + active_segment_id, + limits, + Some(retention), + purpose, + ) + } + + pub(crate) fn open_worker_guarded( + session_root: &Path, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + retention: Option, + purpose: PublicMaterialUse, + ) -> Result { + let store = WorkerSessionStore::open_read_only(session_root) + .map_err(|_| PublicMaterialError::Unavailable)?; + let session_id = store + .session_id() + .map_err(|_| PublicMaterialError::Unavailable)? + .ok_or(PublicMaterialError::Missing)?; + Self::open( + session_id, + active_segment_id, + Layout::Worker(session_root.into()), + limits, + retention, + purpose, + ) + } + + pub fn open_fs( + sessions_root: &Path, + session_id: SessionId, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + ) -> Result { + Self::open_fs_with_retention( + sessions_root, + session_id, + active_segment_id, + limits, + crate::retention::fs_retention(sessions_root), + ) + } + + /// Keeps the same erasure fence when a Host relocates originals to retained storage. + pub fn open_fs_with_retention( + sessions_root: &Path, + session_id: SessionId, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + retention: crate::retention::SessionRetentionStore, + ) -> Result { + Self::open_fs_for_use( + sessions_root, + session_id, + active_segment_id, + limits, + retention, + PublicMaterialUse::Recording, + ) + } + + /// Uses retained originals with the independently selected evidence policy. + pub fn open_fs_for_use( + sessions_root: &Path, + session_id: SessionId, + active_segment_id: SegmentId, + limits: PublicMaterialLimits, + retention: crate::retention::SessionRetentionStore, + purpose: PublicMaterialUse, + ) -> Result { + Self::open( + session_id, + active_segment_id, + Layout::Fs(sessions_root.join(session_id.to_string())), + limits, + Some(retention), + purpose, + ) + } + + pub(crate) fn open_fs_guarded( + sessions_root: &Path, + session: SessionId, + active: SegmentId, + limits: PublicMaterialLimits, + ) -> Result { + Self::open( + session, + active, + Layout::Fs(sessions_root.join(session.to_string())), + limits, + None, + PublicMaterialUse::Recording, + ) + } + + pub fn session_id(&self) -> SessionId { + self.session_id + } + pub fn active_segment_id(&self) -> SegmentId { + self.active_segment_id + } + pub fn descriptors(&self) -> impl ExactSizeIterator { + self.materials.iter().map(|value| &value.descriptor) + } + + /// Consume replay metadata for guarded recording admission, without reading or + /// collecting original bodies. The public capture facade exposes only these + /// descriptors to its callback, never this replay's body-read capability. + pub(crate) fn into_recording_descriptors(self) -> Vec { + self.materials + .into_iter() + .map(|value| value.descriptor) + .collect() + } + + fn open( + session_id: SessionId, + active_segment_id: SegmentId, + layout: Layout, + limits: PublicMaterialLimits, + retention: Option, + purpose: PublicMaterialUse, + ) -> Result { + let _guard = retention + .as_ref() + .map(|store| store.lock_live(session_id).map_err(retention_error)) + .transpose()?; + if limits.max_segments == 0 || limits.max_entries == 0 || limits.max_record_bytes == 0 { + return Err(PublicMaterialError::ResourceLimit); + } + // Freeze every file end before decoding even a Segment seed. Publishers + // hold the same Session publication lock; replay never refreshes this map. + let directory = match &layout { + Layout::Worker(root) => root.join("segments"), + Layout::Fs(root) => root.clone(), + }; + let mut file_ends = HashMap::new(); + let mut fixed_bytes = 0u64; + for entry in std::fs::read_dir(directory)? { + let entry = entry?; + let path = entry.path(); + if path.extension().and_then(|s| s.to_str()) != Some("jsonl") { + continue; + } + let Some(id) = path + .file_stem() + .and_then(|s| s.to_str()) + .and_then(|s| s.parse::().ok()) + else { + continue; + }; + if file_ends.len() >= limits.max_segments { + return Err(PublicMaterialError::ResourceLimit); + } + let reader = RecordReader::open(&path, limits.max_record_bytes)?; + // This also bounds retained descriptor/identity allocations during replay, + // not just the final capture. Oversized histories fail explicitly. + fixed_bytes = fixed_bytes + .checked_add(reader.end) + .ok_or(PublicMaterialError::ResourceLimit)?; + if fixed_bytes > limits.max_storage_bytes { + return Err(PublicMaterialError::ResourceLimit); + } + file_ends.insert(id, reader.end); + } + let mut chain = Vec::new(); + let mut visited = HashSet::new(); + let mut current = active_segment_id; + let mut header_entries = 0usize; + loop { + if !visited.insert(current) { + return Err(PublicMaterialError::Inconsistent); + } + if visited.len() > limits.max_segments { + return Err(PublicMaterialError::ResourceLimit); + } + let mut reader = RecordReader::open(&layout.path(current), limits.max_record_bytes)?; + let file_len = *file_ends + .get(¤t) + .ok_or(PublicMaterialError::Missing)?; + reader.end = file_len; + let Some(( + _, + LogEntry::AnnotatedSegmentStart { + session_id: stored, + forked_from, + compacted_from, + history, + .. + }, + )) = reader.next()? + else { + return Err(PublicMaterialError::Inconsistent); + }; + if stored != session_id || (forked_from.is_some() && compacted_from.is_some()) { + return Err(PublicMaterialError::Inconsistent); + } + header_entries = header_entries + .checked_add(history.len()) + .ok_or(PublicMaterialError::ResourceLimit)?; + if header_entries > limits.max_entries { + return Err(PublicMaterialError::ResourceLimit); + } + let compact = compacted_from.is_some(); + let source_ids: Option> = if compact { + let summaries: Vec<_> = history + .iter() + .filter(|entry| { + entry.metadata.input_metadata.as_ref().is_some_and(|input| { + input.usage().compaction == crate::InputCompactionUse::OpeningSummary + }) + }) + .collect(); + if summaries.len() != 1 { + return Err(PublicMaterialError::Inconsistent); + } + Some( + summaries[0] + .metadata + .derivation + .as_ref() + .ok_or(PublicMaterialError::Inconsistent)? + .sources + .iter() + .map(|id| id.0.clone()) + .collect(), + ) + } else { + None + }; + header_entries = header_entries + .checked_add(source_ids.as_ref().map_or(0, |ids| ids.len())) + .ok_or(PublicMaterialError::ResourceLimit)?; + if header_entries > limits.max_entries { + return Err(PublicMaterialError::ResourceLimit); + } + let parent = forked_from.or(compacted_from); + let next = parent.as_ref().map(|p| p.segment_id); + chain.push(SegmentHeader { + id: current, + parent, + compact, + source_ids, + file_len, + }); + match next { + Some(id) => current = id, + None => break, + } + } + chain.reverse(); + let mut materials = Vec::::new(); + let mut identities = HashMap::::new(); + let mut examined = 0usize; + let mut expected_fork_seed: Option> = None; + for (segment_index, header) in chain.iter().enumerate() { + let child = chain.get(segment_index + 1); + let mut reader = RecordReader::open(&layout.path(header.id), limits.max_record_bytes)?; + reader.end = header.file_len; + let mut history_ids = Vec::::new(); + let mut history_seen = HashSet::::new(); + let mut inherited_order = None; + let mut pending = Vec::::new(); + let mut in_run = false; + let mut safe = true; + let mut turn = 0usize; + let mut reached_cut = child.is_none(); + let mut compact_cut_turn_seen = false; + while let Some((offset, record)) = reader.next()? { + // Once a compaction's exact source history is reached, consume + // only non-history boundaries/corrections. A later history + // addition belongs to the parent's unadopted suffix. + if reached_cut + && child.is_some_and(|c| c.compact) + && matches!( + record, + LogEntry::AnnotatedUserInput { .. } + | LogEntry::AnnotatedAssistantItem { .. } + | LogEntry::AnnotatedToolResult { .. } + | LogEntry::AnnotatedSystemItem { .. } + ) + { + break; + } + let seed = matches!(record, LogEntry::AnnotatedSegmentStart { .. }); + if seed { + if let LogEntry::AnnotatedSegmentStart { + history, + session_id: stored, + .. + } = &record + { + if *stored != session_id { + return Err(PublicMaterialError::Inconsistent); + } + if let Some(expected) = expected_fork_seed.take() { + if history + .iter() + .map(|e| &e.metadata.entry_id.0) + .ne(expected.iter()) + { + return Err(PublicMaterialError::Inconsistent); + } + } + } + } + if offset != 0 && seed { + return Err(PublicMaterialError::Inconsistent); + } + match &record { + LogEntry::Invoke { .. } | LogEntry::RunResumed { .. } => { + if !pending.is_empty() { + return Err(PublicMaterialError::Inconsistent); + } + in_run = true; + safe = false; + } + LogEntry::RunCompleted { .. } + | LogEntry::RunYielded { .. } + | LogEntry::RunCancelled { .. } + | LogEntry::RunErrored { .. } + | LogEntry::PausedTurnAbandoned { .. } => { + safe = true; + in_run = false; + } + LogEntry::TurnEnd { turn_count, .. } => turn = *turn_count, + LogEntry::ActiveRunCheckpoint { + total_turn_count, .. + } => turn = *total_turn_count, + _ => {} + } + let system_history; + let entries: &[LoggedHistoryEntry] = match &record { + LogEntry::AnnotatedSegmentStart { history, .. } + | LogEntry::AnnotatedUserInput { history, .. } => history, + LogEntry::AnnotatedAssistantItem { entry, .. } + | LogEntry::AnnotatedToolResult { entry, .. } + | LogEntry::ToolResultCorrected { entry, .. } => std::slice::from_ref(entry), + LogEntry::AnnotatedSystemItem { entry, .. } => { + system_history = entry.to_history_entry(); + std::slice::from_ref(&system_history) + } + _ => &[], + }; + for (history_index, entry) in entries.iter().enumerate() { + charge(&mut examined, limits.max_entries)?; + let id = &entry.metadata.entry_id.0; + if id.is_empty() { + return Err(PublicMaterialError::Inconsistent); + } + let corrected = matches!(record, LogEntry::ToolResultCorrected { .. }); + if !corrected { + if !history_seen.insert(id.clone()) { + return Err(PublicMaterialError::Inconsistent); + } + history_ids.push(id.clone()); + } + if in_run && ai_materialized(&entry.item) { + safe = true; + } + let Some((sha256, total_bytes)) = canonical_metadata(entry, purpose)? else { + continue; + }; + let location = Location { + segment: header.id, + offset, + history_index, + }; + let descriptor = PublicMaterialDescriptor { + identity: PublicMaterialIdentity { + segment_id: header.id, + entry_ref: format!("E{id}"), + }, + canonical_order: 0, + origin: PublicMaterialOrigin::History { + origin: entry.metadata.origin.clone(), + }, + sha256, + total_bytes, + }; + if seed && header.parent.is_some() { + // Every inherited original must already exist on the + // selected ancestor prefix; compare identity + bytes. + let index = identities + .get(id) + .ok_or(PublicMaterialError::Inconsistent)?; + if inherited_order.is_some_and(|previous| *index <= previous) { + return Err(PublicMaterialError::Inconsistent); + } + inherited_order = Some(*index); + let original = &materials[*index].descriptor; + if original.sha256 != descriptor.sha256 + || original.total_bytes != descriptor.total_bytes + || original.origin != descriptor.origin + { + return Err(PublicMaterialError::Inconsistent); + } + } else if corrected { + let index = identities + .get(id) + .ok_or(PublicMaterialError::Inconsistent)?; + let original = &mut materials[*index]; + if !original.is_tool_result + || original.descriptor.origin != descriptor.origin + || !matches!(entry.item, LoggedItem::ToolResult { .. }) + { + return Err(PublicMaterialError::Inconsistent); + } + original.descriptor.sha256 = descriptor.sha256; + original.descriptor.total_bytes = descriptor.total_bytes; + original.location = location; + } else { + if identities.contains_key(id) + || pending.iter().any(|m| { + m.descriptor.identity.entry_ref == descriptor.identity.entry_ref + }) + { + return Err(PublicMaterialError::Inconsistent); + } + pending.push(Material { + descriptor, + location, + source_history_position: (history_ids.len() - 1) as u64, + is_tool_result: matches!(entry.item, LoggedItem::ToolResult { .. }), + boundary: None, + }); + } + } + if let Some(boundary) = run_boundary(&record) { + charge(&mut examined, limits.max_entries)?; + let reference = boundary_reference(&record, session_id, header.id, offset); + if identities.contains_key(&reference[1..]) + || pending + .iter() + .any(|m| m.descriptor.identity.entry_ref == reference) + { + return Err(PublicMaterialError::Inconsistent); + } + let mut sink = BodySink::new(0, 0); + serde_json::to_writer(&mut sink, &CanonicalBody::RunBoundary { boundary })?; + let (sha256, total_bytes, _) = sink.finish()?; + pending.push(Material { + descriptor: PublicMaterialDescriptor { + identity: PublicMaterialIdentity { + segment_id: header.id, + entry_ref: reference, + }, + canonical_order: 0, + origin: PublicMaterialOrigin::HostRunBoundary, + sha256, + total_bytes, + }, + location: Location { + segment: header.id, + offset, + history_index: 0, + }, + source_history_position: history_ids.len() as u64, + is_tool_result: false, + boundary: Some(boundary), + }); + } + if safe { + flush(&mut materials, &mut identities, &mut pending); + } + if let Some(child) = child { + if child.compact { + let sources = child + .source_ids + .as_ref() + .ok_or(PublicMaterialError::Inconsistent)?; + if &history_ids == sources { + // Do not stop at an earlier non-history record when + // compaction's source-history is empty. + if !sources.is_empty() || seed { + reached_cut = true; + if child + .parent + .as_ref() + .is_some_and(|p| p.at_turn_index == turn) + { + compact_cut_turn_seen = true; + } + } + } + } else if (seed && child.parent.as_ref().is_some_and(|p| p.at_turn_index == 0)) + || (turn + == child + .parent + .as_ref() + .ok_or(PublicMaterialError::Inconsistent)? + .at_turn_index + && matches!(record, LogEntry::TurnEnd { .. })) + { + reached_cut = true; + break; + } + } + } + if !reached_cut || (child.is_some_and(|c| c.compact) && !compact_cut_turn_seen) { + return Err(PublicMaterialError::Inconsistent); + } + if child.is_some_and(|c| !c.compact) { + expected_fork_seed = Some(history_ids); + } + // A compaction seed proves its selected input is durably retained; + // still require materialized AI or a terminal boundary for a run. + if !pending.is_empty() && child.is_some() { + return Err(PublicMaterialError::Inconsistent); + } + } + Ok(Self { + session_id, + active_segment_id, + layout, + limits, + materials, + purpose, + file_ends, + retention: retention.clone(), + }) + } + + /// Build a portable immutable value under the caller's publication/erasure fence. + /// Preflight the aggregate original-body allocation; never truncate an over-limit history. + pub(crate) fn into_recording_dto( + self, + limits: crate::capture::CaptureLimits, + ) -> Result { + use crate::capture::*; + let mut total = 0usize; + if self.materials.len() > limits.max_entries { + return Err(PublicMaterialError::ResourceLimit); + } + for material in &self.materials { + let len = usize::try_from(material.descriptor.total_bytes) + .map_err(|_| PublicMaterialError::ResourceLimit)?; + total = total + .checked_add(len) + .ok_or(PublicMaterialError::ResourceLimit)?; + if len > limits.max_text_bytes || total > limits.max_bytes { + return Err(PublicMaterialError::ResourceLimit); + } + } + let entries = self + .materials + .iter() + .map(|material| { + self.recording_entry( + material, + 0, + usize::try_from(material.descriptor.total_bytes).unwrap_or(usize::MAX), + ) + }) + .collect::, _>>()?; + let dto = CaptureDto { + version: CAPTURE_VERSION, + session_id: self.session_id.to_string(), + segment_id: self.active_segment_id.to_string(), + view_entry_count: self.materials.len() as u64, + entries, + }; + dto.validate(limits) + .map_err(|_| PublicMaterialError::ResourceLimit)?; + Ok(dto) + } + + fn recording_entry( + &self, + material: &Material, + offset: u64, + max_bytes: usize, + ) -> Result { + use crate::capture::*; + let d = &material.descriptor; + let content = self.read_content(d, offset, max_bytes.max(4))?; + let history_origin = match &d.origin { + PublicMaterialOrigin::History { origin } => Some(origin.clone()), + PublicMaterialOrigin::HostRunBoundary => None, + }; + Ok(CaptureEntryDto { + entry_id: d + .identity + .entry_ref + .strip_prefix('E') + .ok_or(PublicMaterialError::Inconsistent)? + .to_owned(), + order: d.canonical_order, + source: CaptureSource { + segment_id: d.identity.segment_id.to_string(), + entry_index: material.source_history_position, + provenance: history_origin + .as_ref() + .map(crate::capture::public_provenance) + .unwrap_or(protocol::SessionEntryProvenance::LegacyUnknown), + history_origin, + derived_from: Vec::new(), + }, + // Recording selection is independent of observation and Memory evidence. + policy: CaptureUsePolicy { + recording_material: true, + memory_evidence: false, + observation_readable: false, + }, + body: CaptureBody::CanonicalPublic { + descriptor: d.clone(), + byte_offset: offset, + text: content.content, + boundary: material.boundary, + }, + }) + } + + /// Capture only a bounded body prefix, after the same full-lineage committed replay. + /// A long Tool result never requires allocating its full canonical rendering. + pub(crate) fn recording_slice( + &self, + start: crate::capture::RecordingPosition, + max_entries: usize, + max_bytes: usize, + ) -> Result { + use crate::capture::*; + if max_entries == 0 || max_bytes < 4 { + return Err(PublicMaterialError::ResourceLimit); + } + let first = usize::try_from(start.canonical_order) + .map_err(|_| PublicMaterialError::InvalidOffset)?; + if first > self.materials.len() || (first == self.materials.len() && start.byte_offset != 0) + { + return Err(PublicMaterialError::InvalidOffset); + } + let mut entries = Vec::new(); + let mut remaining = max_bytes; + for material in self.materials.iter().skip(first).take(max_entries) { + if remaining < 4 { + break; + } + let offset = if material.descriptor.canonical_order == start.canonical_order { + start.byte_offset + } else { + 0 + }; + let entry = self.recording_entry(material, offset, remaining)?; + let CaptureBody::CanonicalPublic { text, .. } = &entry.body else { + unreachable!() + }; + remaining -= text.len(); + let partial = offset + (text.len() as u64) < material.descriptor.total_bytes; + entries.push(entry); + if partial { + break; + } + } + let dto = CaptureDto { + version: CAPTURE_VERSION, + session_id: self.session_id.to_string(), + segment_id: self.active_segment_id.to_string(), + view_entry_count: self.materials.len() as u64, + entries, + }; + dto.recording_slice(start, max_entries, max_bytes) + .map_err(|error| match error { + CaptureError::InvalidRange => PublicMaterialError::InvalidOffset, + _ => PublicMaterialError::ResourceLimit, + }) + } + + /// Read a canonical body window without allocating its full rendered text. + /// `max_content_bytes >= 4` guarantees progress for every UTF-8 scalar. + pub fn read_content( + &self, + descriptor: &PublicMaterialDescriptor, + offset: u64, + max_content_bytes: usize, + ) -> Result { + let _guard = self + .retention + .as_ref() + .map(|store| store.lock_live(self.session_id).map_err(retention_error)) + .transpose()?; + if max_content_bytes < 4 { + return Err(PublicMaterialError::ResourceLimit); + } + if offset > descriptor.total_bytes { + return Err(PublicMaterialError::InvalidOffset); + } + let material = self + .materials + .get( + usize::try_from(descriptor.canonical_order) + .map_err(|_| PublicMaterialError::Changed)?, + ) + .filter(|m| &m.descriptor == descriptor) + .ok_or(PublicMaterialError::Changed)?; + let mut reader = RecordReader::open( + &self.layout.path(material.location.segment), + self.limits.max_record_bytes, + )?; + reader.end = *self + .file_ends + .get(&material.location.segment) + .ok_or(PublicMaterialError::Missing)?; + reader.seek(material.location.offset)?; + let Some((_, record)) = reader.next()? else { + return Err(PublicMaterialError::Missing); + }; + let system_history; + let (body, origin) = if let Some(boundary) = run_boundary(&record) { + if boundary_reference( + &record, + self.session_id, + material.location.segment, + material.location.offset, + ) != descriptor.identity.entry_ref + { + return Err(PublicMaterialError::Changed); + } + ( + CanonicalBody::RunBoundary { boundary }, + PublicMaterialOrigin::HostRunBoundary, + ) + } else { + let entry = match &record { + LogEntry::AnnotatedSegmentStart { history, .. } + | LogEntry::AnnotatedUserInput { history, .. } => { + history.get(material.location.history_index) + } + LogEntry::AnnotatedAssistantItem { entry, .. } + | LogEntry::AnnotatedToolResult { entry, .. } + | LogEntry::ToolResultCorrected { entry, .. } => Some(entry), + LogEntry::AnnotatedSystemItem { entry, .. } => { + system_history = entry.to_history_entry(); + Some(&system_history) + } + _ => None, + } + .filter(|e| format!("E{}", e.metadata.entry_id.0) == descriptor.identity.entry_ref) + .ok_or(PublicMaterialError::Changed)?; + ( + canonical_body(entry, self.purpose).ok_or(PublicMaterialError::Changed)?, + PublicMaterialOrigin::History { + origin: entry.metadata.origin.clone(), + }, + ) + }; + let mut sink = BodySink::new(offset, max_content_bytes); + serde_json::to_writer(&mut sink, &body)?; + let (sha256, total_bytes, content) = sink.finish()?; + if sha256 != descriptor.sha256 + || total_bytes != descriptor.total_bytes + || origin != descriptor.origin + { + return Err(PublicMaterialError::Changed); + } + Ok(PublicMaterialContent { + next_offset: offset + content.len() as u64, + content, + total_bytes, + sha256, + }) + } +} + +fn retention_error(error: crate::StoreError) -> PublicMaterialError { + match error { + crate::StoreError::SessionErased(_) => PublicMaterialError::Erased, + crate::StoreError::Io(error) => PublicMaterialError::Storage(error), + crate::StoreError::Serde(error) => PublicMaterialError::Record(error), + _ => PublicMaterialError::Unavailable, + } +} + +fn charge(examined: &mut usize, max: usize) -> Result<(), PublicMaterialError> { + *examined = examined + .checked_add(1) + .ok_or(PublicMaterialError::ResourceLimit)?; + if *examined > max { + return Err(PublicMaterialError::ResourceLimit); + } + Ok(()) +} + +fn flush( + materials: &mut Vec, + identities: &mut HashMap, + pending: &mut Vec, +) { + for mut material in pending.drain(..) { + let index = materials.len(); + material.descriptor.canonical_order = index as u64; + identities.insert(material.descriptor.identity.entry_ref[1..].into(), index); + materials.push(material); + } +} + +fn ai_materialized(item: &LoggedItem) -> bool { + !matches!( + item, + LoggedItem::Message { + role: LoggedRole::User | LoggedRole::System, + .. + } + ) +} + +/// Versioned canonical JSON framing. This is original text, not a display +/// preview: both result summary and optional full content remain distinguishable. +#[derive(Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum CanonicalBody<'a> { + RunBoundary { + boundary: PublicRunBoundary, + }, + Message { + origin: &'a LoggedSessionHistoryOrigin, + role: LoggedRole, + content: std::borrow::Cow<'a, [crate::LoggedContentPart]>, + }, + ToolCall { + origin: &'a LoggedSessionHistoryOrigin, + call_id: &'a str, + name: &'a str, + arguments: &'a str, + }, + ToolResult { + origin: &'a LoggedSessionHistoryOrigin, + call_id: &'a str, + summary: &'a str, + content: &'a Option, + disposition: &'a agen::tool::ToolResultDisposition, + is_error: bool, + }, +} + +fn run_boundary(record: &LogEntry) -> Option { + Some(match record { + LogEntry::RunCompleted { + result, + interrupted, + .. + } => PublicRunBoundary::Completed { + outcome: match result { + agen::EngineResult::Finished => PublicRunOutcome::Finished, + agen::EngineResult::Paused => PublicRunOutcome::Paused, + agen::EngineResult::Yielded => PublicRunOutcome::Yielded, + agen::EngineResult::LimitReached => PublicRunOutcome::LimitReached, + }, + interrupted: *interrupted, + }, + LogEntry::RunYielded { .. } => PublicRunBoundary::Yielded, + LogEntry::RunResumed { .. } => PublicRunBoundary::Resumed, + LogEntry::RunCancelled { .. } => PublicRunBoundary::Cancelled, + LogEntry::RunErrored { .. } => PublicRunBoundary::Errored, + LogEntry::PausedTurnAbandoned { .. } => PublicRunBoundary::Abandoned, + _ => return None, + }) +} + +fn boundary_reference( + record: &LogEntry, + session: SessionId, + segment: SegmentId, + offset: u64, +) -> String { + let id = match record { + LogEntry::RunCompleted { entry_id, .. } + | LogEntry::PausedTurnAbandoned { entry_id, .. } + | LogEntry::RunYielded { entry_id, .. } + | LogEntry::RunResumed { entry_id, .. } + | LogEntry::RunCancelled { entry_id, .. } + | LogEntry::RunErrored { entry_id, .. } => entry_id.as_ref(), + _ => None, + }; + // Reuse the storage convention for legacy transitions, including the exact physical + // record position. New normal completions have no persisted transition ID either. + id.map(|id| format!("E{}", id.0)) + .unwrap_or_else(|| format!("El-run-{session}-{segment}-{offset}")) +} + +fn canonical_body( + entry: &LoggedHistoryEntry, + purpose: PublicMaterialUse, +) -> Option> { + if let Some(input) = &entry.metadata.input_metadata { + let usage = input.usage(); + let allowed = match purpose { + PublicMaterialUse::Recording => usage.recording_material, + PublicMaterialUse::MemoryEvidence => usage.memory_evidence, + }; + if allowed == crate::InputEvidenceUse::Exclude { + return None; + } + if let LoggedItem::Message { + role: LoggedRole::System, + .. + } = &entry.item + { + let public = input.public_item()?; + return Some(CanonicalBody::Message { + origin: &entry.metadata.origin, + role: LoggedRole::System, + content: std::borrow::Cow::Owned(vec![crate::LoggedContentPart::Text { + text: public.body.clone(), + }]), + }); + } + } + match &entry.item { + LoggedItem::Message { + role: LoggedRole::System, + .. + } + | LoggedItem::Reasoning { .. } + | LoggedItem::AssistantResponseBoundary { .. } => None, + LoggedItem::Message { role, content } => Some(CanonicalBody::Message { + origin: &entry.metadata.origin, + role: *role, + content: std::borrow::Cow::Borrowed(content), + }), + LoggedItem::ToolCall { + call_id, + name, + arguments, + .. + } => Some(CanonicalBody::ToolCall { + origin: &entry.metadata.origin, + call_id, + name, + arguments, + }), + LoggedItem::ToolResult { + call_id, + summary, + content, + disposition, + is_error, + .. + } => Some(CanonicalBody::ToolResult { + origin: &entry.metadata.origin, + call_id, + summary, + content, + disposition, + is_error: *is_error, + }), + } +} + +fn canonical_metadata( + entry: &LoggedHistoryEntry, + purpose: PublicMaterialUse, +) -> Result, PublicMaterialError> { + let Some(body) = canonical_body(entry, purpose) else { + return Ok(None); + }; + let mut sink = BodySink::new(0, 0); + serde_json::to_writer(&mut sink, &body)?; + Ok(Some((hex_digest(&sink.digest.finalize()), sink.total))) +} + +fn hex_digest(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +struct BodySink { + digest: Sha256, + total: u64, + offset: u64, + max: usize, + bytes: Vec, +} + +impl BodySink { + fn new(offset: u64, max: usize) -> Self { + Self { + digest: Sha256::new(), + total: 0, + offset, + max, + bytes: Vec::new(), + } + } + fn finish(mut self) -> Result<(String, u64, String), PublicMaterialError> { + // The final byte may land inside a UTF-8 scalar. Never include a + // partial scalar and never advance the caller over its missing bytes. + loop { + match std::str::from_utf8(&self.bytes) { + Ok(_) => break, + Err(e) if e.error_len().is_none() => { + self.bytes.truncate(e.valid_up_to()); + } + Err(_) => return Err(PublicMaterialError::InvalidOffset), + } + } + Ok(( + hex_digest(&self.digest.finalize()), + self.total, + String::from_utf8(self.bytes).map_err(|_| PublicMaterialError::InvalidOffset)?, + )) + } +} + +impl Write for BodySink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.digest.update(bytes); + let start = self + .offset + .saturating_sub(self.total) + .min(bytes.len() as u64) as usize; + let available = self.max.saturating_sub(self.bytes.len()); + let end = start.saturating_add(available).min(bytes.len()); + if end > start { + self.bytes.extend_from_slice(&bytes[start..end]); + } + self.total = self + .total + .checked_add(bytes.len() as u64) + .ok_or_else(|| io::Error::other("canonical body length overflow"))?; + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +struct RecordReader { + reader: BufReader, + offset: u64, + end: u64, + max: u64, +} + +impl RecordReader { + fn open(path: &Path, max: u64) -> Result { + // Do not follow symlinks in a caller-selected storage root or a log. + let mut prefix = PathBuf::new(); + for component in path.components() { + prefix.push(component); + let metadata = std::fs::symlink_metadata(&prefix)?; + if metadata.file_type().is_symlink() { + return Err(PublicMaterialError::Unavailable); + } + } + let file = open_noatime(path)?; + if !file.metadata()?.is_file() { + return Err(PublicMaterialError::Unavailable); + } + let end = file.metadata()?.len(); + Ok(Self { + reader: BufReader::new(file), + offset: 0, + end, + max, + }) + } + fn seek(&mut self, offset: u64) -> Result<(), PublicMaterialError> { + self.reader.seek(SeekFrom::Start(offset))?; + self.offset = offset; + Ok(()) + } + fn next(&mut self) -> Result, PublicMaterialError> { + if self.offset >= self.end { + return Ok(None); + } + let mut bytes = Vec::new(); + let read = (&mut self.reader) + .take(self.max.saturating_add(1).min(self.end - self.offset)) + .read_until(b'\n', &mut bytes)? as u64; + if read > self.max { + return Err(PublicMaterialError::ResourceLimit); + } + if bytes.last() != Some(&b'\n') { + return Ok(None); + } // Uncommitted tail. + let offset = self.offset; + self.offset += read; + Ok(Some((offset, serde_json::from_slice(&bytes)?))) + } +} + +fn open_noatime(path: &Path) -> io::Result { + #[cfg(any(target_os = "linux", target_os = "android"))] + { + use std::os::unix::fs::OpenOptionsExt; + std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOATIME | libc::O_NOFOLLOW) + .open(path) + } + #[cfg(not(any(target_os = "linux", target_os = "android")))] + { + let _ = path; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "no-atime reads unavailable", + )) + } +} + +#[cfg(test)] +#[path = "public_material/tests.rs"] +mod tests; + +/// Legacy descriptor-reader facade over the shared committed replay. +pub type SessionPublicMaterialReader = CommittedPublicReplay; diff --git a/crates/session-store/src/lib.rs b/crates/session-store/src/lib.rs index 15abc32dd..0f057d242 100644 --- a/crates/session-store/src/lib.rs +++ b/crates/session-store/src/lib.rs @@ -39,6 +39,7 @@ pub(crate) fn read_without_atime_bounded(path: &Path, max_bytes: u64) -> io::Res } pub mod capture; +mod committed_replay; pub mod event_trace; pub mod fs_store; pub mod history; diff --git a/crates/session-store/src/public_material.rs b/crates/session-store/src/public_material.rs index 150852174..7a0975148 100644 --- a/crates/session-store/src/public_material.rs +++ b/crates/session-store/src/public_material.rs @@ -1,1114 +1,8 @@ -//! Original public material from a saved, rollback-ineligible Session prefix. -//! -//! Unlike the Memory evidence index, this reader includes saved output from -//! interrupted runs and active runs with materialized AI activity. It does not -//! interpret domain-specific recording or completion policy. Callers authorize -//! the Session root and explicitly select its active Segment. -//! -//! Catalogs retain only descriptors and record locations, never original bodies. -//! Records are decoded one at a time under an explicit allocation ceiling, not -//! under a whole-run scan budget. Canonical bodies are streamed to SHA-256 and -//! bounded UTF-8 windows. A single JSONL record (including a Segment seed) must -//! fit `max_record_bytes`; exceeding that ceiling is an error, not a preview. +//! Legacy descriptor reads backed by the common committed public replay. +//! Recording capture uses the same replay, but freezes bodies into `capture::CaptureDto`. -use std::collections::{HashMap, HashSet}; -use std::fs::File; -use std::io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; - -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; - -use crate::{ - LogEntry, LoggedHistoryEntry, LoggedItem, LoggedRole, LoggedSessionHistoryOrigin, SegmentId, - SegmentOrigin, SessionId, WorkerSessionStore, +pub use crate::committed_replay::{ + PublicMaterialContent, PublicMaterialDescriptor, PublicMaterialError, PublicMaterialIdentity, + PublicMaterialLimits, PublicMaterialOrigin, PublicMaterialUse, PublicRunBoundary, + PublicRunOutcome, SessionPublicMaterialReader, }; - -#[derive(Debug, thiserror::Error)] -pub enum PublicMaterialError { - #[error("public material storage: {0}")] - Storage(#[from] io::Error), - #[error("public material record: {0}")] - Record(#[from] serde_json::Error), - #[error("public material source is missing")] - Missing, - #[error("public material source was permanently erased")] - Erased, - #[error("public material source requires migration or is unavailable")] - Unavailable, - #[error("public material lineage or identity is inconsistent")] - Inconsistent, - #[error("public material resource ceiling exceeded")] - ResourceLimit, - #[error("public material changed since the catalog was read")] - Changed, - #[error("public material offset is not a UTF-8 boundary or is outside the body")] - InvalidOffset, -} - -#[derive(Debug, Clone, Copy)] -pub struct PublicMaterialLimits { - pub max_segments: usize, - /// Total history identities examined, including inherited and private items. - pub max_entries: usize, - /// Allocation ceiling for one persisted JSONL record, not an entire run. - pub max_record_bytes: u64, -} - -impl Default for PublicMaterialLimits { - fn default() -> Self { - Self { - max_segments: 256, - max_entries: 1_000_000, - max_record_bytes: 64 * 1024 * 1024, - } - } -} - -/// Original identity, never a body-based deduplication key. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct PublicMaterialIdentity { - pub segment_id: SegmentId, - pub entry_ref: String, -} - -/// Public provenance distinguishes saved conversation from Host control facts. Run boundaries -/// are not model output, user intent, or evidence that the requested work succeeded. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum PublicMaterialOrigin { - History { origin: LoggedSessionHistoryOrigin }, - HostRunBoundary, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum PublicRunOutcome { - Finished, - Paused, - Yielded, - LimitReached, -} - -/// Only whitelisted execution facts cross this boundary; raw error/provider messages do not. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum PublicRunBoundary { - Completed { - outcome: PublicRunOutcome, - interrupted: bool, - }, - Yielded, - Resumed, - Cancelled, - Errored, - Abandoned, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct PublicMaterialDescriptor { - pub identity: PublicMaterialIdentity, - /// Zero-based position on this explicitly selected active lineage. Forks - /// may remove a suffix; callers must reconcile identities, not assume that - /// positions from a previous active lineage are interchangeable. - pub canonical_order: u64, - pub origin: PublicMaterialOrigin, - /// Lowercase hexadecimal SHA-256 of the complete canonical UTF-8 body. - pub sha256: String, - pub total_bytes: u64, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PublicMaterialContent { - pub content: String, - pub next_offset: u64, - pub total_bytes: u64, - pub sha256: String, -} - -#[derive(Clone)] -struct Location { - segment: SegmentId, - offset: u64, - history_index: usize, -} - -struct Material { - descriptor: PublicMaterialDescriptor, - location: Location, - is_tool_result: bool, -} - -struct SegmentHeader { - id: SegmentId, - parent: Option, - compact: bool, - // Compaction's exact source-history IDs disambiguate mid-turn boundaries. - source_ids: Option>, - file_len: u64, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PublicMaterialUse { - Recording, - MemoryEvidence, -} - -/// A fixed material catalog. Opening a new catalog observes newly appended -/// records; reading a body through this catalog verifies its saved digest and -/// byte length again. Missing/corrected/deleted sources are explicit failures. -/// Evidence eligibility never publishes private model text or Feature payload: -/// System inputs also require an explicit public projection for every public use. -pub struct SessionPublicMaterialReader { - session_id: SessionId, - active_segment_id: SegmentId, - layout: Layout, - limits: PublicMaterialLimits, - materials: Vec, - purpose: PublicMaterialUse, - retention: Option, -} - -enum Layout { - Worker(PathBuf), - Fs(PathBuf), -} - -impl Layout { - fn path(&self, segment: SegmentId) -> PathBuf { - match self { - Self::Worker(root) => root.join("segments").join(format!("{segment}.jsonl")), - Self::Fs(root) => root.join(format!("{segment}.jsonl")), - } - } -} - -impl SessionPublicMaterialReader { - pub fn open_worker( - session_root: &Path, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - ) -> Result { - Self::open_worker_guarded( - session_root, - active_segment_id, - limits, - None, - PublicMaterialUse::Recording, - ) - } - - /// Uses the Host's aggregate-external control root for the whole scan and every body read. - /// The same store must fence publishers, archive transfer, and physical deletion. - pub fn open_worker_with_retention( - session_root: &Path, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - retention: crate::retention::SessionRetentionStore, - ) -> Result { - Self::open_worker_for_use( - session_root, - active_segment_id, - limits, - retention, - PublicMaterialUse::Recording, - ) - } - - /// Selects the evidence purpose independently of public display and observation. - pub fn open_worker_for_use( - session_root: &Path, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - retention: crate::retention::SessionRetentionStore, - purpose: PublicMaterialUse, - ) -> Result { - Self::open_worker_guarded( - session_root, - active_segment_id, - limits, - Some(retention), - purpose, - ) - } - - fn open_worker_guarded( - session_root: &Path, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - retention: Option, - purpose: PublicMaterialUse, - ) -> Result { - let store = WorkerSessionStore::open_read_only(session_root) - .map_err(|_| PublicMaterialError::Unavailable)?; - let session_id = store - .session_id() - .map_err(|_| PublicMaterialError::Unavailable)? - .ok_or(PublicMaterialError::Missing)?; - Self::open( - session_id, - active_segment_id, - Layout::Worker(session_root.into()), - limits, - retention, - purpose, - ) - } - - pub fn open_fs( - sessions_root: &Path, - session_id: SessionId, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - ) -> Result { - Self::open_fs_with_retention( - sessions_root, - session_id, - active_segment_id, - limits, - crate::retention::fs_retention(sessions_root), - ) - } - - /// Keeps the same erasure fence when a Host relocates originals to retained storage. - pub fn open_fs_with_retention( - sessions_root: &Path, - session_id: SessionId, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - retention: crate::retention::SessionRetentionStore, - ) -> Result { - Self::open_fs_for_use( - sessions_root, - session_id, - active_segment_id, - limits, - retention, - PublicMaterialUse::Recording, - ) - } - - /// Uses retained originals with the independently selected evidence policy. - pub fn open_fs_for_use( - sessions_root: &Path, - session_id: SessionId, - active_segment_id: SegmentId, - limits: PublicMaterialLimits, - retention: crate::retention::SessionRetentionStore, - purpose: PublicMaterialUse, - ) -> Result { - Self::open( - session_id, - active_segment_id, - Layout::Fs(sessions_root.join(session_id.to_string())), - limits, - Some(retention), - purpose, - ) - } - - pub fn session_id(&self) -> SessionId { - self.session_id - } - pub fn active_segment_id(&self) -> SegmentId { - self.active_segment_id - } - pub fn descriptors(&self) -> impl ExactSizeIterator { - self.materials.iter().map(|value| &value.descriptor) - } - - fn open( - session_id: SessionId, - active_segment_id: SegmentId, - layout: Layout, - limits: PublicMaterialLimits, - retention: Option, - purpose: PublicMaterialUse, - ) -> Result { - let _guard = retention - .as_ref() - .map(|store| store.lock_live(session_id).map_err(retention_error)) - .transpose()?; - if limits.max_segments == 0 || limits.max_entries == 0 || limits.max_record_bytes == 0 { - return Err(PublicMaterialError::ResourceLimit); - } - let mut chain = Vec::new(); - let mut visited = HashSet::new(); - let mut current = active_segment_id; - let mut header_entries = 0usize; - loop { - if !visited.insert(current) { - return Err(PublicMaterialError::Inconsistent); - } - if visited.len() > limits.max_segments { - return Err(PublicMaterialError::ResourceLimit); - } - let mut reader = RecordReader::open(&layout.path(current), limits.max_record_bytes)?; - let file_len = reader.end; - let Some(( - _, - LogEntry::AnnotatedSegmentStart { - session_id: stored, - forked_from, - compacted_from, - history, - .. - }, - )) = reader.next()? - else { - return Err(PublicMaterialError::Inconsistent); - }; - if stored != session_id || (forked_from.is_some() && compacted_from.is_some()) { - return Err(PublicMaterialError::Inconsistent); - } - header_entries = header_entries - .checked_add(history.len()) - .ok_or(PublicMaterialError::ResourceLimit)?; - if header_entries > limits.max_entries { - return Err(PublicMaterialError::ResourceLimit); - } - let compact = compacted_from.is_some(); - let source_ids: Option> = if compact { - let summaries: Vec<_> = history - .iter() - .filter(|entry| { - entry.metadata.input_metadata.as_ref().is_some_and(|input| { - input.usage().compaction == crate::InputCompactionUse::OpeningSummary - }) - }) - .collect(); - if summaries.len() != 1 { - return Err(PublicMaterialError::Inconsistent); - } - Some( - summaries[0] - .metadata - .derivation - .as_ref() - .ok_or(PublicMaterialError::Inconsistent)? - .sources - .iter() - .map(|id| id.0.clone()) - .collect(), - ) - } else { - None - }; - header_entries = header_entries - .checked_add(source_ids.as_ref().map_or(0, |ids| ids.len())) - .ok_or(PublicMaterialError::ResourceLimit)?; - if header_entries > limits.max_entries { - return Err(PublicMaterialError::ResourceLimit); - } - let parent = forked_from.or(compacted_from); - let next = parent.as_ref().map(|p| p.segment_id); - chain.push(SegmentHeader { - id: current, - parent, - compact, - source_ids, - file_len, - }); - match next { - Some(id) => current = id, - None => break, - } - } - chain.reverse(); - let mut materials = Vec::::new(); - let mut identities = HashMap::::new(); - let mut examined = 0usize; - let mut expected_fork_seed: Option> = None; - for (segment_index, header) in chain.iter().enumerate() { - let child = chain.get(segment_index + 1); - let mut reader = RecordReader::open(&layout.path(header.id), limits.max_record_bytes)?; - reader.end = header.file_len; - let mut history_ids = Vec::::new(); - let mut history_seen = HashSet::::new(); - let mut inherited_order = None; - let mut pending = Vec::::new(); - let mut in_run = false; - let mut safe = true; - let mut turn = 0usize; - let mut reached_cut = child.is_none(); - let mut compact_cut_turn_seen = false; - while let Some((offset, record)) = reader.next()? { - // Once a compaction's exact source history is reached, consume - // only non-history boundaries/corrections. A later history - // addition belongs to the parent's unadopted suffix. - if reached_cut - && child.is_some_and(|c| c.compact) - && matches!( - record, - LogEntry::AnnotatedUserInput { .. } - | LogEntry::AnnotatedAssistantItem { .. } - | LogEntry::AnnotatedToolResult { .. } - | LogEntry::AnnotatedSystemItem { .. } - ) - { - break; - } - let seed = matches!(record, LogEntry::AnnotatedSegmentStart { .. }); - if seed { - if let LogEntry::AnnotatedSegmentStart { - history, - session_id: stored, - .. - } = &record - { - if *stored != session_id { - return Err(PublicMaterialError::Inconsistent); - } - if let Some(expected) = expected_fork_seed.take() { - if history - .iter() - .map(|e| &e.metadata.entry_id.0) - .ne(expected.iter()) - { - return Err(PublicMaterialError::Inconsistent); - } - } - } - } - if offset != 0 && seed { - return Err(PublicMaterialError::Inconsistent); - } - match &record { - LogEntry::Invoke { .. } | LogEntry::RunResumed { .. } => { - if !pending.is_empty() { - return Err(PublicMaterialError::Inconsistent); - } - in_run = true; - safe = false; - } - LogEntry::RunCompleted { .. } - | LogEntry::RunYielded { .. } - | LogEntry::RunCancelled { .. } - | LogEntry::RunErrored { .. } - | LogEntry::PausedTurnAbandoned { .. } => { - safe = true; - in_run = false; - } - LogEntry::TurnEnd { turn_count, .. } => turn = *turn_count, - LogEntry::ActiveRunCheckpoint { - total_turn_count, .. - } => turn = *total_turn_count, - _ => {} - } - let system_history; - let entries: &[LoggedHistoryEntry] = match &record { - LogEntry::AnnotatedSegmentStart { history, .. } - | LogEntry::AnnotatedUserInput { history, .. } => history, - LogEntry::AnnotatedAssistantItem { entry, .. } - | LogEntry::AnnotatedToolResult { entry, .. } - | LogEntry::ToolResultCorrected { entry, .. } => std::slice::from_ref(entry), - LogEntry::AnnotatedSystemItem { entry, .. } => { - system_history = entry.to_history_entry(); - std::slice::from_ref(&system_history) - } - _ => &[], - }; - for (history_index, entry) in entries.iter().enumerate() { - charge(&mut examined, limits.max_entries)?; - let id = &entry.metadata.entry_id.0; - if id.is_empty() { - return Err(PublicMaterialError::Inconsistent); - } - let corrected = matches!(record, LogEntry::ToolResultCorrected { .. }); - if !corrected { - if !history_seen.insert(id.clone()) { - return Err(PublicMaterialError::Inconsistent); - } - history_ids.push(id.clone()); - } - if in_run && ai_materialized(&entry.item) { - safe = true; - } - let Some((sha256, total_bytes)) = canonical_metadata(entry, purpose)? else { - continue; - }; - let location = Location { - segment: header.id, - offset, - history_index, - }; - let descriptor = PublicMaterialDescriptor { - identity: PublicMaterialIdentity { - segment_id: header.id, - entry_ref: format!("E{id}"), - }, - canonical_order: 0, - origin: PublicMaterialOrigin::History { - origin: entry.metadata.origin.clone(), - }, - sha256, - total_bytes, - }; - if seed && header.parent.is_some() { - // Every inherited original must already exist on the - // selected ancestor prefix; compare identity + bytes. - let index = identities - .get(id) - .ok_or(PublicMaterialError::Inconsistent)?; - if inherited_order.is_some_and(|previous| *index <= previous) { - return Err(PublicMaterialError::Inconsistent); - } - inherited_order = Some(*index); - let original = &materials[*index].descriptor; - if original.sha256 != descriptor.sha256 - || original.total_bytes != descriptor.total_bytes - || original.origin != descriptor.origin - { - return Err(PublicMaterialError::Inconsistent); - } - } else if corrected { - let index = identities - .get(id) - .ok_or(PublicMaterialError::Inconsistent)?; - let original = &mut materials[*index]; - if !original.is_tool_result - || original.descriptor.origin != descriptor.origin - || !matches!(entry.item, LoggedItem::ToolResult { .. }) - { - return Err(PublicMaterialError::Inconsistent); - } - original.descriptor.sha256 = descriptor.sha256; - original.descriptor.total_bytes = descriptor.total_bytes; - original.location = location; - } else { - if identities.contains_key(id) - || pending.iter().any(|m| { - m.descriptor.identity.entry_ref == descriptor.identity.entry_ref - }) - { - return Err(PublicMaterialError::Inconsistent); - } - pending.push(Material { - descriptor, - location, - is_tool_result: matches!(entry.item, LoggedItem::ToolResult { .. }), - }); - } - } - if let Some(boundary) = run_boundary(&record) { - charge(&mut examined, limits.max_entries)?; - let reference = boundary_reference(&record, session_id, header.id, offset); - if identities.contains_key(&reference[1..]) - || pending - .iter() - .any(|m| m.descriptor.identity.entry_ref == reference) - { - return Err(PublicMaterialError::Inconsistent); - } - let mut sink = BodySink::new(0, 0); - serde_json::to_writer(&mut sink, &CanonicalBody::RunBoundary { boundary })?; - let (sha256, total_bytes, _) = sink.finish()?; - pending.push(Material { - descriptor: PublicMaterialDescriptor { - identity: PublicMaterialIdentity { - segment_id: header.id, - entry_ref: reference, - }, - canonical_order: 0, - origin: PublicMaterialOrigin::HostRunBoundary, - sha256, - total_bytes, - }, - location: Location { - segment: header.id, - offset, - history_index: 0, - }, - is_tool_result: false, - }); - } - if safe { - flush(&mut materials, &mut identities, &mut pending); - } - if let Some(child) = child { - if child.compact { - let sources = child - .source_ids - .as_ref() - .ok_or(PublicMaterialError::Inconsistent)?; - if &history_ids == sources { - // Do not stop at an earlier non-history record when - // compaction's source-history is empty. - if !sources.is_empty() || seed { - reached_cut = true; - if child - .parent - .as_ref() - .is_some_and(|p| p.at_turn_index == turn) - { - compact_cut_turn_seen = true; - } - } - } - } else if (seed && child.parent.as_ref().is_some_and(|p| p.at_turn_index == 0)) - || (turn - == child - .parent - .as_ref() - .ok_or(PublicMaterialError::Inconsistent)? - .at_turn_index - && matches!(record, LogEntry::TurnEnd { .. })) - { - reached_cut = true; - break; - } - } - } - if !reached_cut || (child.is_some_and(|c| c.compact) && !compact_cut_turn_seen) { - return Err(PublicMaterialError::Inconsistent); - } - if child.is_some_and(|c| !c.compact) { - expected_fork_seed = Some(history_ids); - } - // A compaction seed proves its selected input is durably retained; - // still require materialized AI or a terminal boundary for a run. - if !pending.is_empty() && child.is_some() { - return Err(PublicMaterialError::Inconsistent); - } - } - Ok(Self { - session_id, - active_segment_id, - layout, - limits, - materials, - purpose, - retention: retention.clone(), - }) - } - - /// Read a canonical body window without allocating its full rendered text. - /// `max_content_bytes >= 4` guarantees progress for every UTF-8 scalar. - pub fn read_content( - &self, - descriptor: &PublicMaterialDescriptor, - offset: u64, - max_content_bytes: usize, - ) -> Result { - let _guard = self - .retention - .as_ref() - .map(|store| store.lock_live(self.session_id).map_err(retention_error)) - .transpose()?; - if max_content_bytes < 4 { - return Err(PublicMaterialError::ResourceLimit); - } - if offset > descriptor.total_bytes { - return Err(PublicMaterialError::InvalidOffset); - } - let material = self - .materials - .get( - usize::try_from(descriptor.canonical_order) - .map_err(|_| PublicMaterialError::Changed)?, - ) - .filter(|m| &m.descriptor == descriptor) - .ok_or(PublicMaterialError::Changed)?; - let mut reader = RecordReader::open( - &self.layout.path(material.location.segment), - self.limits.max_record_bytes, - )?; - reader.seek(material.location.offset)?; - let Some((_, record)) = reader.next()? else { - return Err(PublicMaterialError::Missing); - }; - let system_history; - let (body, origin) = if let Some(boundary) = run_boundary(&record) { - if boundary_reference( - &record, - self.session_id, - material.location.segment, - material.location.offset, - ) != descriptor.identity.entry_ref - { - return Err(PublicMaterialError::Changed); - } - ( - CanonicalBody::RunBoundary { boundary }, - PublicMaterialOrigin::HostRunBoundary, - ) - } else { - let entry = match &record { - LogEntry::AnnotatedSegmentStart { history, .. } - | LogEntry::AnnotatedUserInput { history, .. } => { - history.get(material.location.history_index) - } - LogEntry::AnnotatedAssistantItem { entry, .. } - | LogEntry::AnnotatedToolResult { entry, .. } - | LogEntry::ToolResultCorrected { entry, .. } => Some(entry), - LogEntry::AnnotatedSystemItem { entry, .. } => { - system_history = entry.to_history_entry(); - Some(&system_history) - } - _ => None, - } - .filter(|e| format!("E{}", e.metadata.entry_id.0) == descriptor.identity.entry_ref) - .ok_or(PublicMaterialError::Changed)?; - ( - canonical_body(entry, self.purpose).ok_or(PublicMaterialError::Changed)?, - PublicMaterialOrigin::History { - origin: entry.metadata.origin.clone(), - }, - ) - }; - let mut sink = BodySink::new(offset, max_content_bytes); - serde_json::to_writer(&mut sink, &body)?; - let (sha256, total_bytes, content) = sink.finish()?; - if sha256 != descriptor.sha256 - || total_bytes != descriptor.total_bytes - || origin != descriptor.origin - { - return Err(PublicMaterialError::Changed); - } - Ok(PublicMaterialContent { - next_offset: offset + content.len() as u64, - content, - total_bytes, - sha256, - }) - } -} - -fn retention_error(error: crate::StoreError) -> PublicMaterialError { - match error { - crate::StoreError::SessionErased(_) => PublicMaterialError::Erased, - crate::StoreError::Io(error) => PublicMaterialError::Storage(error), - crate::StoreError::Serde(error) => PublicMaterialError::Record(error), - _ => PublicMaterialError::Unavailable, - } -} - -fn charge(examined: &mut usize, max: usize) -> Result<(), PublicMaterialError> { - *examined = examined - .checked_add(1) - .ok_or(PublicMaterialError::ResourceLimit)?; - if *examined > max { - return Err(PublicMaterialError::ResourceLimit); - } - Ok(()) -} - -fn flush( - materials: &mut Vec, - identities: &mut HashMap, - pending: &mut Vec, -) { - for mut material in pending.drain(..) { - let index = materials.len(); - material.descriptor.canonical_order = index as u64; - identities.insert(material.descriptor.identity.entry_ref[1..].into(), index); - materials.push(material); - } -} - -fn ai_materialized(item: &LoggedItem) -> bool { - !matches!( - item, - LoggedItem::Message { - role: LoggedRole::User | LoggedRole::System, - .. - } - ) -} - -/// Versioned canonical JSON framing. This is original text, not a display -/// preview: both result summary and optional full content remain distinguishable. -#[derive(Serialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -enum CanonicalBody<'a> { - RunBoundary { - boundary: PublicRunBoundary, - }, - Message { - origin: &'a LoggedSessionHistoryOrigin, - role: LoggedRole, - content: std::borrow::Cow<'a, [crate::LoggedContentPart]>, - }, - ToolCall { - origin: &'a LoggedSessionHistoryOrigin, - call_id: &'a str, - name: &'a str, - arguments: &'a str, - }, - ToolResult { - origin: &'a LoggedSessionHistoryOrigin, - call_id: &'a str, - summary: &'a str, - content: &'a Option, - disposition: &'a agen::tool::ToolResultDisposition, - is_error: bool, - }, -} - -fn run_boundary(record: &LogEntry) -> Option { - Some(match record { - LogEntry::RunCompleted { - result, - interrupted, - .. - } => PublicRunBoundary::Completed { - outcome: match result { - agen::EngineResult::Finished => PublicRunOutcome::Finished, - agen::EngineResult::Paused => PublicRunOutcome::Paused, - agen::EngineResult::Yielded => PublicRunOutcome::Yielded, - agen::EngineResult::LimitReached => PublicRunOutcome::LimitReached, - }, - interrupted: *interrupted, - }, - LogEntry::RunYielded { .. } => PublicRunBoundary::Yielded, - LogEntry::RunResumed { .. } => PublicRunBoundary::Resumed, - LogEntry::RunCancelled { .. } => PublicRunBoundary::Cancelled, - LogEntry::RunErrored { .. } => PublicRunBoundary::Errored, - LogEntry::PausedTurnAbandoned { .. } => PublicRunBoundary::Abandoned, - _ => return None, - }) -} - -fn boundary_reference( - record: &LogEntry, - session: SessionId, - segment: SegmentId, - offset: u64, -) -> String { - let id = match record { - LogEntry::RunCompleted { entry_id, .. } - | LogEntry::PausedTurnAbandoned { entry_id, .. } - | LogEntry::RunYielded { entry_id, .. } - | LogEntry::RunResumed { entry_id, .. } - | LogEntry::RunCancelled { entry_id, .. } - | LogEntry::RunErrored { entry_id, .. } => entry_id.as_ref(), - _ => None, - }; - // Reuse the storage convention for legacy transitions, including the exact physical - // record position. New normal completions have no persisted transition ID either. - id.map(|id| format!("E{}", id.0)) - .unwrap_or_else(|| format!("El-run-{session}-{segment}-{offset}")) -} - -fn canonical_body( - entry: &LoggedHistoryEntry, - purpose: PublicMaterialUse, -) -> Option> { - if let Some(input) = &entry.metadata.input_metadata { - let usage = input.usage(); - let allowed = match purpose { - PublicMaterialUse::Recording => usage.recording_material, - PublicMaterialUse::MemoryEvidence => usage.memory_evidence, - }; - if allowed == crate::InputEvidenceUse::Exclude { - return None; - } - if let LoggedItem::Message { - role: LoggedRole::System, - .. - } = &entry.item - { - let public = input.public_item()?; - return Some(CanonicalBody::Message { - origin: &entry.metadata.origin, - role: LoggedRole::System, - content: std::borrow::Cow::Owned(vec![crate::LoggedContentPart::Text { - text: public.body.clone(), - }]), - }); - } - } - match &entry.item { - LoggedItem::Message { - role: LoggedRole::System, - .. - } - | LoggedItem::Reasoning { .. } - | LoggedItem::AssistantResponseBoundary { .. } => None, - LoggedItem::Message { role, content } => Some(CanonicalBody::Message { - origin: &entry.metadata.origin, - role: *role, - content: std::borrow::Cow::Borrowed(content), - }), - LoggedItem::ToolCall { - call_id, - name, - arguments, - .. - } => Some(CanonicalBody::ToolCall { - origin: &entry.metadata.origin, - call_id, - name, - arguments, - }), - LoggedItem::ToolResult { - call_id, - summary, - content, - disposition, - is_error, - .. - } => Some(CanonicalBody::ToolResult { - origin: &entry.metadata.origin, - call_id, - summary, - content, - disposition, - is_error: *is_error, - }), - } -} - -fn canonical_metadata( - entry: &LoggedHistoryEntry, - purpose: PublicMaterialUse, -) -> Result, PublicMaterialError> { - let Some(body) = canonical_body(entry, purpose) else { - return Ok(None); - }; - let mut sink = BodySink::new(0, 0); - serde_json::to_writer(&mut sink, &body)?; - Ok(Some((hex_digest(&sink.digest.finalize()), sink.total))) -} - -fn hex_digest(bytes: &[u8]) -> String { - bytes.iter().map(|b| format!("{b:02x}")).collect() -} - -struct BodySink { - digest: Sha256, - total: u64, - offset: u64, - max: usize, - bytes: Vec, -} - -impl BodySink { - fn new(offset: u64, max: usize) -> Self { - Self { - digest: Sha256::new(), - total: 0, - offset, - max, - bytes: Vec::new(), - } - } - fn finish(mut self) -> Result<(String, u64, String), PublicMaterialError> { - // The final byte may land inside a UTF-8 scalar. Never include a - // partial scalar and never advance the caller over its missing bytes. - loop { - match std::str::from_utf8(&self.bytes) { - Ok(_) => break, - Err(e) if e.error_len().is_none() => { - self.bytes.truncate(e.valid_up_to()); - } - Err(_) => return Err(PublicMaterialError::InvalidOffset), - } - } - Ok(( - hex_digest(&self.digest.finalize()), - self.total, - String::from_utf8(self.bytes).map_err(|_| PublicMaterialError::InvalidOffset)?, - )) - } -} - -impl Write for BodySink { - fn write(&mut self, bytes: &[u8]) -> io::Result { - self.digest.update(bytes); - let start = self - .offset - .saturating_sub(self.total) - .min(bytes.len() as u64) as usize; - let available = self.max.saturating_sub(self.bytes.len()); - let end = start.saturating_add(available).min(bytes.len()); - if end > start { - self.bytes.extend_from_slice(&bytes[start..end]); - } - self.total = self - .total - .checked_add(bytes.len() as u64) - .ok_or_else(|| io::Error::other("canonical body length overflow"))?; - Ok(bytes.len()) - } - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -struct RecordReader { - reader: BufReader, - offset: u64, - end: u64, - max: u64, -} - -impl RecordReader { - fn open(path: &Path, max: u64) -> Result { - // Do not follow symlinks in a caller-selected storage root or a log. - let mut prefix = PathBuf::new(); - for component in path.components() { - prefix.push(component); - let metadata = std::fs::symlink_metadata(&prefix)?; - if metadata.file_type().is_symlink() { - return Err(PublicMaterialError::Unavailable); - } - } - let file = open_noatime(path)?; - if !file.metadata()?.is_file() { - return Err(PublicMaterialError::Unavailable); - } - let end = file.metadata()?.len(); - Ok(Self { - reader: BufReader::new(file), - offset: 0, - end, - max, - }) - } - fn seek(&mut self, offset: u64) -> Result<(), PublicMaterialError> { - self.reader.seek(SeekFrom::Start(offset))?; - self.offset = offset; - Ok(()) - } - fn next(&mut self) -> Result, PublicMaterialError> { - if self.offset >= self.end { - return Ok(None); - } - let mut bytes = Vec::new(); - let read = (&mut self.reader) - .take(self.max.saturating_add(1).min(self.end - self.offset)) - .read_until(b'\n', &mut bytes)? as u64; - if read > self.max { - return Err(PublicMaterialError::ResourceLimit); - } - if bytes.last() != Some(&b'\n') { - return Ok(None); - } // Uncommitted tail. - let offset = self.offset; - self.offset += read; - Ok(Some((offset, serde_json::from_slice(&bytes)?))) - } -} - -fn open_noatime(path: &Path) -> io::Result { - #[cfg(any(target_os = "linux", target_os = "android"))] - { - use std::os::unix::fs::OpenOptionsExt; - std::fs::OpenOptions::new() - .read(true) - .custom_flags(libc::O_NOATIME | libc::O_NOFOLLOW) - .open(path) - } - #[cfg(not(any(target_os = "linux", target_os = "android")))] - { - let _ = path; - Err(io::Error::new( - io::ErrorKind::Unsupported, - "no-atime reads unavailable", - )) - } -} - -#[cfg(test)] -mod tests; diff --git a/crates/session-store/src/public_material/tests.rs b/crates/session-store/src/public_material/tests.rs index 230b81055..b7f4a8e7c 100644 --- a/crates/session-store/src/public_material/tests.rs +++ b/crates/session-store/src/public_material/tests.rs @@ -897,8 +897,28 @@ fn midturn_compaction_uses_exact_source_ids_and_preserves_discarded_originals() store .create_segment(session, compact, &[start, output("new", "new")]) .unwrap(); + let retention = crate::retention::fs_retention(temp.path()); + let guard = retention.lock_live(session).unwrap(); + let captured = crate::capture::capture_recording_fs_guarded( + temp.path(), + session, + compact, + Default::default(), + Default::default(), + &guard, + ) + .unwrap(); + drop(guard); + let saved_bytes = captured.encode(Default::default()).unwrap(); let reader = read(temp.path(), session, compact); assert_eq!(ids(&reader), ["Eu", "Ekept", "Enew"]); + for (entry, descriptor) in captured.entries.iter().zip(reader.descriptors()) { + assert_eq!(captured.recording_text(entry.order).unwrap().0, descriptor); + } + store + .append(session, compact, &output("after-snapshot", "next Job only")) + .unwrap(); + assert_eq!(captured.encode(Default::default()).unwrap(), saved_bytes); let d: Vec<_> = reader.descriptors().collect(); assert_eq!(d[0].identity.segment_id, root); assert_eq!(d[1].identity.segment_id, root); @@ -913,6 +933,16 @@ fn midturn_compaction_uses_exact_source_ids_and_preserves_discarded_originals() .is_err() ); assert!(reader.read_content(d[0], 0, 4096).is_err()); + assert_eq!( + captured + .recording_slice(Default::default(), 100, 8192) + .unwrap() + .capture + .entries + .len(), + 3 + ); + assert_eq!(captured.encode(Default::default()).unwrap(), saved_bytes); } #[test] @@ -1484,6 +1514,991 @@ fn generic_system_material_uses_explicit_evidence_purpose_not_producer_or_body() assert!(!body.contains("not-public")); assert!(!body.contains("private model text")); assert!(!body.contains("unknown:future")); + if purpose == PublicMaterialUse::Recording { + let retention = crate::retention::fs_retention(temp.path()); + let guard = retention.lock_live(session).unwrap(); + let capture = if worker_layout { + crate::capture::capture_recording_worker_guarded( + &root, + session, + fork, + Default::default(), + Default::default(), + &guard, + ) + } else { + crate::capture::capture_recording_fs_guarded( + temp.path(), + session, + fork, + Default::default(), + Default::default(), + &guard, + ) + } + .unwrap(); + assert_eq!(capture.entries.len(), 1); + let (saved, _, text) = capture.recording_text(0).unwrap(); + assert_eq!(saved, descriptor); + assert_eq!(text, body); + let view = + crate::capture::SessionCapture::try_from_dto(capture, Default::default()) + .unwrap(); + assert!(view.search(&Default::default()).is_empty()); + assert!(view.overview().is_empty()); + } + } + } +} + +#[test] +fn recording_capture_slices_canonical_utf8_with_metadata_and_legacy_descriptors() { + use crate::capture::*; + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("session"); + let retention = crate::retention::SessionRetentionStore::new(temp.path().join("control")); + let session = crate::new_session_id(); + let segment = crate::new_segment_id(); + let store = WorkerSessionStore::new_with_retention(&root, retention.clone()).unwrap(); + let mut records = vec![seed(session, vec![]), invoke(), input("u")]; + for i in 0..150 { + records.push(output(&format!("a-{i}"), "saved text")); + } + records.push(LogEntry::AnnotatedToolResult { + ts: 5, + entry: entry( + "tool", + LoggedItem::ToolResult { + call_id: "call".into(), + summary: "summary".into(), + content: Some("漢字🍎\n\"".repeat(6000)), + disposition: agen::tool::ToolResultDisposition::default(), + is_error: false, + attachments: vec![], + }, + ), + }); + records.push(cancelled()); + store.create_segment(session, segment, &records).unwrap(); + let guard = retention.lock_live(session).unwrap(); + let capture = capture_recording_worker_guarded( + &root, + session, + segment, + Default::default(), + Default::default(), + &guard, + ) + .unwrap(); + drop(guard); + assert_eq!(capture.entries.len(), 153); + let reader = SessionPublicMaterialReader::open_worker_with_retention( + &root, + segment, + Default::default(), + retention.clone(), + ) + .unwrap(); + let descriptors: Vec<_> = reader.descriptors().cloned().collect(); + for (entry, descriptor) in capture.entries.iter().zip(&descriptors) { + let (saved, offset, text) = capture.recording_text(entry.order).unwrap(); + assert_eq!(saved, descriptor); + assert_eq!(offset, 0); + assert_eq!( + reader + .read_content(descriptor, 0, text.len().max(4)) + .unwrap() + .content, + text + ); + } + let bytes = capture.encode(Default::default()).unwrap(); + let restored = CaptureDto::decode(&bytes, Default::default()).unwrap(); + let mut position = RecordingPosition::default(); + let mut combined = vec![String::new(); descriptors.len()]; + let mut slices = 0; + loop { + let slice = restored.recording_slice(position, 29, 4096).unwrap(); + assert!(serde_json::to_vec(&slice).unwrap().len() <= 4096); + assert!(!slice.capture.entries.is_empty()); + slice.capture.validate(Default::default()).unwrap(); + for entry in &slice.capture.entries { + let (descriptor, offset, text) = slice.capture.recording_text(entry.order).unwrap(); + let body = &mut combined[entry.order as usize]; + assert_eq!(offset, body.len() as u64); + assert_eq!(descriptor, &descriptors[entry.order as usize]); + body.push_str(text); + } + slices += 1; + let Some(next) = slice.next else { + break; + }; + assert_ne!(next, position); + position = next; + } + assert!(slices > 10); + for (order, text) in combined.iter().enumerate() { + assert_eq!(text, capture.recording_text(order as u64).unwrap().2); + } + assert!(matches!( + capture.entries.last().unwrap().body, + CaptureBody::CanonicalPublic { + boundary: Some(PublicRunBoundary::Cancelled), + .. + } + )); + let mut corrupt = restored.clone(); + if let CaptureBody::CanonicalPublic { text, .. } = &mut corrupt.entries[0].body { + text.replace_range(0..1, "["); + } + assert!(corrupt.validate(Default::default()).is_err()); + let mut unknown: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + unknown["entries"][0]["body"]["descriptor"]["unexpected"] = true.into(); + assert!( + CaptureDto::decode(&serde_json::to_vec(&unknown).unwrap(), Default::default()).is_err() + ); + let mut missing = restored.clone(); + missing.entries.remove(1); + assert!(matches!( + missing.recording_slice(Default::default(), 3, 8192), + Err(CaptureError::InvalidRange) + )); + let tool_order = 151; + let tool_text = capture.recording_text(tool_order).unwrap().2; + let bad = tool_text.find('漢').unwrap() as u64 + 1; + assert!(matches!( + capture.recording_slice( + RecordingPosition { + canonical_order: tool_order, + byte_offset: bad + }, + 1, + 4096 + ), + Err(CaptureError::InvalidRange) + )); + assert!(matches!( + capture.recording_slice(Default::default(), 1, 10), + Err(CaptureError::BudgetTooSmall) + )); + // The full-value ceiling does not constrain bounded construction of a long Tool result. + let guard = retention.lock_live(session).unwrap(); + assert!(matches!( + capture_recording_worker_guarded( + &root, + session, + segment, + Default::default(), + CaptureLimits { + max_bytes: 2048, + ..Default::default() + }, + &guard + ), + Err(PublicMaterialError::ResourceLimit) + )); + let bounded = capture_recording_slice_worker_guarded( + &root, + session, + segment, + Default::default(), + RecordingPosition { + canonical_order: tool_order, + byte_offset: 0, + }, + 1, + 4096, + &guard, + ) + .unwrap(); + assert!(serde_json::to_vec(&bounded).unwrap().len() <= 4096); + assert!(bounded.next.is_some()); +} + +#[test] +fn recording_capture_fence_blocks_publication_and_value_survives_later_append_and_erasure() { + use crate::capture::*; + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("session"); + let retention = crate::retention::SessionRetentionStore::new(temp.path().join("control")); + let session = crate::new_session_id(); + let segment = crate::new_segment_id(); + let store = WorkerSessionStore::new_with_retention(&root, retention.clone()).unwrap(); + store + .create_segment( + session, + segment, + &[ + seed(session, vec![]), + invoke(), + input("u"), + output("a", "saved"), + ], + ) + .unwrap(); + let guard = retention.lock_live(session).unwrap(); + let (started, receiver) = std::sync::mpsc::channel(); + let (done, completed) = std::sync::mpsc::channel(); + let writer = store.clone(); + let thread = std::thread::spawn(move || { + started.send(()).unwrap(); + writer + .append(session, segment, &output("later", "not in snapshot")) + .unwrap(); + done.send(()).unwrap(); + }); + receiver + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + let capture = capture_recording_worker_guarded( + &root, + session, + segment, + Default::default(), + Default::default(), + &guard, + ) + .unwrap(); + assert!(completed.try_recv().is_err()); + let bytes = capture.encode(Default::default()).unwrap(); + drop(guard); + completed + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + thread.join().unwrap(); + assert_eq!(capture.encode(Default::default()).unwrap(), bytes); + assert_eq!(capture.entries.len(), 2); + retention + .delete( + session, + crate::retention::SessionDeletionIntent::ErasePermanently, + || { + std::fs::remove_dir_all(&root)?; + Ok(()) + }, + ) + .unwrap(); + assert!(matches!( + capture_recording_worker( + &root, + session, + segment, + &retention, + Default::default(), + Default::default() + ), + Err(PublicMaterialError::Erased) + )); + // Pure reads do not authorize adoption after erase; Host consumer revocation is separate. + assert_eq!( + capture + .recording_slice(Default::default(), 2, 8192) + .unwrap() + .capture + .entries + .len(), + 2 + ); +} + +#[test] +fn recording_capture_rejects_foreign_guard_and_storage_or_body_allocation_excess() { + use crate::capture::*; + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("session"); + let retention = crate::retention::SessionRetentionStore::new(temp.path().join("control")); + let session = crate::new_session_id(); + let segment = crate::new_segment_id(); + let store = WorkerSessionStore::new_with_retention(&root, retention.clone()).unwrap(); + store + .create_segment( + session, + segment, + &[seed( + session, + vec![message("original", LoggedRole::User, "text")], + )], + ) + .unwrap(); + let wrong_guard = retention.lock_live(crate::new_session_id()).unwrap(); + assert!(matches!( + capture_recording_worker_guarded( + &root, + session, + segment, + Default::default(), + Default::default(), + &wrong_guard + ), + Err(PublicMaterialError::Inconsistent) + )); + let guard = retention.lock_live(session).unwrap(); + let limits = PublicMaterialLimits { + max_storage_bytes: 1, + ..Default::default() + }; + assert!(matches!( + capture_recording_worker_guarded( + &root, + session, + segment, + limits, + Default::default(), + &guard + ), + Err(PublicMaterialError::ResourceLimit) + )); + let limits = CaptureLimits { + max_text_bytes: 1, + ..Default::default() + }; + assert!(matches!( + capture_recording_worker_guarded( + &root, + session, + segment, + Default::default(), + limits, + &guard + ), + Err(PublicMaterialError::ResourceLimit) + )); +} + +#[test] +fn fixed_replay_ends_do_not_refresh_for_appends_between_replay_and_body_snapshot() { + use crate::capture::*; + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("session"); + let retention = crate::retention::SessionRetentionStore::new(temp.path().join("control")); + let session = crate::new_session_id(); + let segment = crate::new_segment_id(); + let store = WorkerSessionStore::new_with_retention(&root, retention.clone()).unwrap(); + store + .create_segment( + session, + segment, + &[ + seed(session, vec![]), + invoke(), + input("u"), + output("a", "saved"), + ], + ) + .unwrap(); + let guard = retention.lock_live(session).unwrap(); + let replay = CommittedPublicReplay::open_worker_guarded( + &root, + segment, + Default::default(), + None, + PublicMaterialUse::Recording, + ) + .unwrap(); + let before = replay + .recording_slice(Default::default(), 100, 8192) + .unwrap(); + drop(guard); + // Production keeps its fence through construction. Releasing it deliberately + // here also verifies that the lower-level replay never refreshes file ends. + store + .append(session, segment, &output("later", "not in fixed files")) + .unwrap(); + let other = crate::new_segment_id(); + store + .create_segment( + session, + other, + &[ + seed(session, vec![]), + output("other", "different active segment"), + ], + ) + .unwrap(); + let after = replay + .recording_slice(RecordingPosition::default(), 100, 8192) + .unwrap(); + assert_eq!(after, before); + assert_eq!(after.capture.entries.len(), 2); +} + +#[test] +fn recording_source_coordinates_count_hidden_rows_and_preserve_originals_across_compaction() { + use crate::capture::*; + let temp = tempfile::tempdir().unwrap(); + let retention = crate::retention::fs_retention(temp.path()); + let store = FsStore::new_with_retention(temp.path(), retention.clone()).unwrap(); + let session = crate::new_session_id(); + let root = crate::new_segment_id(); + let compact = crate::new_segment_id(); + let hidden = + |id: &str| system_context(id, "private omitted row", crate::InputCompactionUse::Retain); + let root_user = message("root-user", LoggedRole::User, "first public row"); + let tool = entry( + "root-tool", + LoggedItem::ToolResult { + call_id: "call".into(), + summary: "original".into(), + content: Some("full original".into()), + disposition: Default::default(), + is_error: false, + attachments: vec![], + }, + ); + let mut corrected = tool.clone(); + if let LoggedItem::ToolResult { content, .. } = &mut corrected.item { + *content = Some("corrected full original".into()); + } + let source_ids = [ + "hidden-0", + "root-user", + "hidden-2", + "hidden-3", + "batch-user", + "reasoning", + "root-tool", + "root-answer", + ]; + store + .create_segment( + session, + root, + &[ + seed( + session, + vec![hidden("hidden-0"), root_user.clone(), hidden("hidden-2")], + ), + invoke(), + LogEntry::AnnotatedUserInput { + ts: 2, + segments: vec![], + extensions: vec![], + history: vec![ + hidden("hidden-3"), + message("batch-user", LoggedRole::User, "second public row"), + ], + }, + LogEntry::AnnotatedAssistantItem { + ts: 3, + entry: entry( + "reasoning", + LoggedItem::Reasoning { + text: "private reasoning".into(), + summary: vec![], + encrypted_content: None, + signature: None, + }, + ), + }, + LogEntry::AnnotatedToolResult { + ts: 4, + entry: tool.clone(), + }, + LogEntry::RunYielded { + ts: 6, + entry_id: Some(LoggedSessionHistoryEntryId("yield-gap".into())), + reason: protocol::RunYieldReason::Compaction, + active_run_turn_count: 1, + }, + LogEntry::RunResumed { + ts: 7, + entry_id: Some(LoggedSessionHistoryEntryId("resume-gap".into())), + source: protocol::RunResumeSource::Compaction, + active_run_turn_count: 1, + }, + output("root-answer", "public answer"), + LogEntry::TurnEnd { + ts: 8, + turn_count: 1, + }, + ], + ) + .unwrap(); + let mut summary = system_context( + "summary", + "not an original", + crate::InputCompactionUse::OpeningSummary, + ); + summary.metadata.origin = LoggedSessionHistoryOrigin::CompactionSummary; + summary.metadata.derivation = Some(crate::LoggedHistoryDerivation { + sources: source_ids + .iter() + .map(|id| LoggedSessionHistoryEntryId((*id).into())) + .collect(), + }); + let mut start = seed( + session, + vec![summary, root_user, tool, hidden("child-hidden-seed")], + ); + if let LogEntry::AnnotatedSegmentStart { compacted_from, .. } = &mut start { + *compacted_from = Some(SegmentOrigin { + segment_id: root, + at_turn_index: 1, + }); + } + store + .create_segment( + session, + compact, + &[ + start, + LogEntry::ToolResultCorrected { + ts: 9, + entry: corrected, + }, + output("child-answer", "new child original"), + LogEntry::AnnotatedUserInput { + ts: 10, + segments: vec![], + extensions: vec![], + history: vec![ + hidden("child-hidden-input"), + message("child-user", LoggedRole::User, "new child user"), + ], + }, + cancelled(), + ], + ) + .unwrap(); + let guard = retention.lock_live(session).unwrap(); + let capture = capture_recording_fs_guarded( + temp.path(), + session, + compact, + Default::default(), + Default::default(), + &guard, + ) + .unwrap(); + let catalog = recording_catalog_fs_guarded( + temp.path(), + session, + compact, + Default::default(), + &guard, + |descriptors| descriptors.to_vec(), + ) + .unwrap(); + assert_eq!( + catalog, + capture + .entries + .iter() + .map(|entry| capture.recording_text(entry.order).unwrap().0.clone()) + .collect::>() + ); + drop(guard); + let expected = [ + ("Eroot-user", root, 1), + ("Ebatch-user", root, 4), + ("Eroot-tool", root, 6), + ("Eyield-gap", root, 7), + ("Eresume-gap", root, 7), + ("Eroot-answer", root, 7), + ("Echild-answer", compact, 4), + ("Echild-user", compact, 6), + ]; + for ((reference, segment, index), entry) in expected.iter().zip(&capture.entries) { + assert_eq!(format!("E{}", entry.entry_id), *reference); + assert_eq!(entry.source.segment_id, segment.to_string()); + assert_eq!(entry.source.entry_index, *index); + } + assert_eq!(capture.entries.len(), expected.len() + 1); // final cancelled boundary at child gap 7 + assert_eq!(capture.entries.last().unwrap().source.entry_index, 7); + assert!( + capture + .recording_text(2) + .unwrap() + .2 + .contains("corrected full original") + ); + let reader = read(temp.path(), session, compact); + for (entry, descriptor) in capture.entries.iter().zip(reader.descriptors()) { + assert_eq!(capture.recording_text(entry.order).unwrap().0, descriptor); + } + let decoded = CaptureDto::decode( + &capture.encode(Default::default()).unwrap(), + Default::default(), + ) + .unwrap(); + assert_eq!(decoded, capture); + let slice = decoded + .recording_slice( + RecordingPosition { + canonical_order: 2, + byte_offset: 0, + }, + 5, + 16 * 1024, + ) + .unwrap(); + assert_eq!( + slice + .capture + .entries + .iter() + .map(|e| e.source.entry_index) + .collect::>(), + vec![6, 7, 7, 7, 4] + ); + slice.capture.validate(Default::default()).unwrap(); + let mut alias = capture.clone(); + alias.entries[1].source.entry_index = 1; + assert!(matches!( + alias.validate(Default::default()), + Err(CaptureError::Invalid("duplicate source position")) + )); + let mut reverse = capture.clone(); + reverse.entries[2].source.entry_index = 2; + assert!(matches!( + reverse.validate(Default::default()), + Err(CaptureError::Invalid("recording source position order")) + )); + let mut reversed_gap = capture; + reversed_gap.entries[3].source.entry_index = 5; + assert!(matches!( + reversed_gap.validate(Default::default()), + Err(CaptureError::Invalid("recording source position order")) + )); +} + +#[test] +fn recording_catalog_guarded_is_descriptor_only_beyond_body_capture_budget_in_both_layouts() { + use crate::capture::*; + for worker_layout in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("source"); + let retention = crate::retention::SessionRetentionStore::new(temp.path().join("control")); + let session = crate::new_session_id(); + let segment = crate::new_segment_id(); + let mut records = vec![seed( + session, + vec![ + system_context("hidden", "private", crate::InputCompactionUse::Retain), + message("u", LoggedRole::User, "request"), + ], + )]; + records.push(invoke()); + for i in 0..130 { + records.push(output(&format!("a-{i}"), &"漢字🍎\n\"".repeat(400))); + } + records.push(cancelled()); + if worker_layout { + WorkerSessionStore::new_with_retention(&root, retention.clone()) + .unwrap() + .create_segment(session, segment, &records) + .unwrap(); + } else { + FsStore::new_with_retention(&root, retention.clone()) + .unwrap() + .create_segment(session, segment, &records) + .unwrap(); + } + let guard = retention.lock_live(session).unwrap(); + let inspect = |descriptors: &[PublicMaterialDescriptor]| descriptors.to_vec(); + let catalog = if worker_layout { + recording_catalog_worker_guarded( + &root, + session, + segment, + Default::default(), + &guard, + inspect, + ) + } else { + recording_catalog_fs_guarded( + &root, + session, + segment, + Default::default(), + &guard, + inspect, + ) + } + .unwrap(); + assert_eq!(catalog.len(), 132); // user, 130 answers, typed Host boundary + assert_eq!(catalog[0].identity.entry_ref, "Eu"); + assert_eq!(catalog[131].origin, PublicMaterialOrigin::HostRunBoundary); + assert!(catalog.iter().map(|d| d.total_bytes).sum::() > 128 * 1024); + assert!( + catalog + .iter() + .enumerate() + .all(|(i, d)| d.canonical_order == i as u64) + ); + // This is an admission scan, not a full body DTO with a huge transient ceiling. + let small = CaptureLimits { + max_bytes: 8192, + ..Default::default() + }; + let full = if worker_layout { + capture_recording_worker_guarded( + &root, + session, + segment, + Default::default(), + small, + &guard, + ) + } else { + capture_recording_fs_guarded(&root, session, segment, Default::default(), small, &guard) + }; + assert!(matches!(full, Err(PublicMaterialError::ResourceLimit))); + // Descriptor identities/order/digests are exactly those of the bounded body path. + for order in [0, 1, 130, 131] { + let start = RecordingPosition { + canonical_order: order, + byte_offset: 0, + }; + let slice = if worker_layout { + capture_recording_slice_worker_guarded( + &root, + session, + segment, + Default::default(), + start, + 1, + 4096, + &guard, + ) + } else { + capture_recording_slice_fs_guarded( + &root, + session, + segment, + Default::default(), + start, + 1, + 4096, + &guard, + ) + } + .unwrap(); + assert_eq!( + slice.capture.recording_text(order).unwrap().0, + &catalog[order as usize] + ); + assert!(serde_json::to_vec(&slice).unwrap().len() <= 4096); + } + } +} + +#[test] +fn recording_catalog_callback_keeps_same_external_guard_and_persisted_selector() { + use crate::capture::*; + use crate::{ + WorkerActiveSegmentRef, WorkerAggregateStore, WorkerMetadata, WorkerMetadataStore, + }; + for worker_layout in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("source"); + let control = temp.path().join("control"); + let retention = crate::retention::SessionRetentionStore::new(&control); + let session = crate::new_session_id(); + let selected = crate::new_segment_id(); + let unrelated_newer = crate::new_segment_id(); + for (segment, id) in [(selected, "selected"), (unrelated_newer, "not-active")] { + let records = [seed(session, vec![message(id, LoggedRole::User, id)])]; + if worker_layout { + WorkerSessionStore::new_with_retention(&root, retention.clone()) + .unwrap() + .create_segment(session, segment, &records) + .unwrap(); + } else { + FsStore::new_with_retention(&root, retention.clone()) + .unwrap() + .create_segment(session, segment, &records) + .unwrap(); + } + } + let metadata_root = temp.path().join("worker-metadata"); + let metadata = WorkerAggregateStore::new(&metadata_root, "test-worker").unwrap(); + let guard = retention.lock_live(session).unwrap(); + metadata + .write(&WorkerMetadata::new( + "test-worker", + Some(WorkerActiveSegmentRef::active_segment(session, selected)), + )) + .unwrap(); + // Simulate with_recording_boundary: persisted pointer resolution and descriptor + // mapping share exactly the same guard. Neither call selects the newest file. + let active = WorkerAggregateStore::open_read_only(&metadata_root, "test-worker") + .unwrap() + .read_read_only() + .unwrap() + .unwrap() + .active + .unwrap(); + assert_eq!(active.session_id, session); + let inspect = |descriptors: &[PublicMaterialDescriptor]| -> Result, &'static str> { + // Direct non-blocking check of the exact external fence, during callback. + // Publishers, active-pointer changes and erasure all use this same lock. + let probe = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(control.join(format!("{session}.lock"))) + .unwrap(); + assert!(!fs4::fs_std::FileExt::try_lock_exclusive(&probe).unwrap()); + assert_eq!(descriptors.len(), 1); + assert_eq!(descriptors[0].identity.segment_id, selected); + assert_eq!(descriptors[0].identity.entry_ref, "Eselected"); + Ok(descriptors.to_vec()) + }; + let catalog = if worker_layout { + recording_catalog_worker_guarded( + &root, + session, + active.segment_id.unwrap(), + Default::default(), + &guard, + inspect, + ) + } else { + recording_catalog_fs_guarded( + &root, + session, + active.segment_id.unwrap(), + Default::default(), + &guard, + inspect, + ) + } + .unwrap() + .unwrap(); + let callback_error = if worker_layout { + recording_catalog_worker_guarded( + &root, + session, + selected, + Default::default(), + &guard, + |_| Err::<(), _>("host-error"), + ) + } else { + recording_catalog_fs_guarded( + &root, + session, + selected, + Default::default(), + &guard, + |_| Err::<(), _>("host-error"), + ) + }; + assert_eq!(callback_error.unwrap(), Err("host-error")); + // The API borrows, not replaces/drops, the caller's guard. + let probe = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(control.join(format!("{session}.lock"))) + .unwrap(); + assert!(!fs4::fs_std::FileExt::try_lock_exclusive(&probe).unwrap()); + drop(guard); + assert!(fs4::fs_std::FileExt::try_lock_exclusive(&probe).unwrap()); + fs4::fs_std::FileExt::unlock(&probe).unwrap(); + retention + .delete( + session, + crate::retention::SessionDeletionIntent::ErasePermanently, + || { + std::fs::remove_dir_all(&root)?; + Ok(()) + }, + ) + .unwrap(); + assert!(matches!( + retention.lock_live(session), + Err(crate::StoreError::SessionErased(_)) + )); + // Descriptor results contain no source body and do not authorize post-erase reads/adoption. + assert_eq!(catalog[0].identity.entry_ref, "Eselected"); + } +} + +#[test] +fn recording_catalog_errors_do_not_invoke_callback_or_relax_resource_and_identity_checks() { + use crate::capture::*; + for worker_layout in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("source"); + let retention = crate::retention::SessionRetentionStore::new(temp.path().join("control")); + let session = crate::new_session_id(); + let segment = crate::new_segment_id(); + let records = [seed( + session, + vec![message("u", LoggedRole::User, "request")], + )]; + if worker_layout { + WorkerSessionStore::new_with_retention(&root, retention.clone()) + .unwrap() + .create_segment(session, segment, &records) + .unwrap(); + } else { + FsStore::new_with_retention(&root, retention.clone()) + .unwrap() + .create_segment(session, segment, &records) + .unwrap(); + } + let guard = retention.lock_live(session).unwrap(); + let other_session = crate::new_session_id(); + let other_guard = retention.lock_live(other_session).unwrap(); + let open = |session, segment, limits, guard| { + if worker_layout { + recording_catalog_worker_guarded(&root, session, segment, limits, guard, |_| { + panic!("callback must not run for rejected catalog") + }) + } else { + recording_catalog_fs_guarded(&root, session, segment, limits, guard, |_| { + panic!("callback must not run for rejected catalog") + }) + } + }; + assert!(matches!( + open(session, segment, Default::default(), &other_guard), + Err(PublicMaterialError::Inconsistent) + )); + for limits in [ + PublicMaterialLimits { + max_storage_bytes: 1, + ..Default::default() + }, + PublicMaterialLimits { + max_record_bytes: 1, + ..Default::default() + }, + PublicMaterialLimits { + max_entries: 0, + ..Default::default() + }, + PublicMaterialLimits { + max_segments: 0, + ..Default::default() + }, + ] { + assert!(matches!( + open(session, segment, limits, &guard), + Err(PublicMaterialError::ResourceLimit) + )); + } + assert!(open(session, crate::new_segment_id(), Default::default(), &guard).is_err()); + // Even a guard matching a claimed Session cannot bless another stored Session. + if worker_layout { + assert!(matches!( + open(other_session, segment, Default::default(), &other_guard), + Err(PublicMaterialError::Inconsistent) + )); + } else { + let impostor = root.join(other_session.to_string()); + std::fs::create_dir(&impostor).unwrap(); + std::fs::copy( + root.join(session.to_string()) + .join(format!("{segment}.jsonl")), + impostor.join(format!("{segment}.jsonl")), + ) + .unwrap(); + assert!(matches!( + open(other_session, segment, Default::default(), &other_guard), + Err(PublicMaterialError::Inconsistent) + )); } } } diff --git a/crates/session-store/src/retention.rs b/crates/session-store/src/retention.rs index 4acecbc96..28870c0eb 100644 --- a/crates/session-store/src/retention.rs +++ b/crates/session-store/src/retention.rs @@ -46,6 +46,13 @@ pub struct SessionRetentionStore { /// Do not recursively acquire the same Session lock inside a guarded callback. pub struct SessionRetentionGuard { _file: File, + session_id: SessionId, +} + +impl SessionRetentionGuard { + pub fn session_id(&self) -> SessionId { + self.session_id + } } impl SessionRetentionStore { @@ -71,7 +78,10 @@ impl SessionRetentionStore { } let file = options.open(self.root.join(format!("{session}.lock")))?; FileExt::lock_exclusive(&file)?; - Ok(SessionRetentionGuard { _file: file }) + Ok(SessionRetentionGuard { + _file: file, + session_id: session, + }) } fn read(&self, session: SessionId) -> Result { diff --git a/crates/standalone/src/job_store.rs b/crates/standalone/src/job_store.rs index 7721dc429..dd69c1568 100644 --- a/crates/standalone/src/job_store.rs +++ b/crates/standalone/src/job_store.rs @@ -25,7 +25,46 @@ use thiserror::Error; #[path = "job_store/legacy_migrations.rs"] mod legacy_migrations; -const SCHEMA_VERSION: i64 = 4; +const SCHEMA_VERSION: i64 = 5; + +/// Availability of the immutable input body, independent of execution outcome. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum JobInputDisposition { + Available, + Reclaimed, + Erased, + Invalidated, +} + +impl JobInputDisposition { + pub fn as_str(self) -> &'static str { + match self { + Self::Available => "available", + Self::Reclaimed => "reclaimed", + Self::Erased => "erased", + Self::Invalidated => "invalidated", + } + } +} + +/// Host-authorized retirement; erasure and legacy invalidation do not change outcomes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JobInputRetirementReason { + Completed, + ExplicitErasure, + LegacyInvalidation, +} + +impl JobInputRetirementReason { + pub fn disposition(self) -> JobInputDisposition { + match self { + Self::Completed => JobInputDisposition::Reclaimed, + Self::ExplicitErasure => JobInputDisposition::Erased, + Self::LegacyInvalidation => JobInputDisposition::Invalidated, + } + } +} #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -67,7 +106,11 @@ impl JobAttemptState { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct JobSnapshot { + /// Receipt metadata. After retirement `input` is null; use the verified loader to execute. pub request: JobRequest, + pub input_digest: String, + pub intent_fingerprint: String, + pub input_disposition: JobInputDisposition, /// Durable Host-issued domain delegation, not model input or Profile identity. pub domain_grant: Option, pub state: JobState, @@ -77,14 +120,27 @@ pub struct JobSnapshot { } impl JobSnapshot { + /// Load only the original available body, checking both content and immutable intent. + pub fn load_executable_request(&self) -> Result<&JobRequest, JobStoreError> { + if self.input_disposition != JobInputDisposition::Available { + return Err(JobStoreError::InvalidTransition("Job input is retired")); + } + self.request.validate()?; + if self.request.input_digest()? != self.input_digest + || self.request.fingerprint()? != self.intent_fingerprint + { + return Err(JobStoreError::Corrupt( + "immutable input or intent fingerprint mismatch".into(), + )); + } + Ok(&self.request) + } + /// Shared Feature-facing read contract; no Workspace or execution identity leaks. pub fn outcome(&self) -> job::JobOutcome { job::JobOutcome { job_id: self.request.job_id.clone(), - input_digest: self - .request - .input_digest() - .expect("validated immutable Job input"), + input_digest: self.input_digest.clone(), attempt_id: self.attempt.attempt_id.clone(), attempt: self.attempt.number, state: match self.state { @@ -204,9 +260,23 @@ impl JobStore { tx.execute_batch(RECONCILIATION_SCHEMA)?; tx.pragma_update(None, "user_version", SCHEMA_VERSION)?; } - SCHEMA_VERSION => {} + 4 | SCHEMA_VERSION => {} other => return Err(JobStoreError::UnsupportedSchema(other)), } + if version < 5 { + tx.execute_batch(INPUT_RETENTION_SCHEMA)?; + let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?; + if has_legacy_archive { + tx.execute_batch("DROP TRIGGER legacy_job_archive_immutable; + CREATE TRIGGER legacy_job_archive_immutable BEFORE UPDATE OF job_id,fingerprint,attempts_json ON legacy_job_archive + BEGIN SELECT RAISE(ABORT, 'immutable legacy Job archive'); END; + CREATE TRIGGER legacy_job_archive_input_retirement BEFORE UPDATE OF request_json ON legacy_job_archive + WHEN NOT EXISTS (SELECT 1 FROM job_intents WHERE job_id=OLD.job_id AND input_disposition<>'available') + OR (json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') AND json_type(NEW.request_json,'$.input')='null') IS NOT TRUE + BEGIN SELECT RAISE(ABORT, 'immutable legacy Job input'); END;")?; + } + tx.pragma_update(None, "user_version", SCHEMA_VERSION)?; + } // An accepted result is the only evidence of success. Reserved attempts // were never started and remain pending; cancelled attempts never resume. tx.execute_batch( @@ -249,10 +319,7 @@ impl JobStore { [&request.job_id], |row| row.get(0), )?; - if stored != fingerprint - || snapshot.request != request - || snapshot.domain_grant != grant - { + if stored != fingerprint || snapshot.domain_grant != grant { return Err(JobStoreError::IntentConflict(request.job_id)); } tx.commit()?; @@ -334,6 +401,7 @@ impl JobStore { .conn .transaction_with_behavior(TransactionBehavior::Immediate)?; let previous = require_snapshot(&tx, job_id)?; + previous.load_executable_request()?; if previous.state != JobState::Failed || previous.attempt.state != JobAttemptState::Failed { return Err(JobStoreError::InvalidTransition( "retry requires known failure", @@ -354,6 +422,81 @@ impl JobStore { Ok(snapshot) } + /// Historical snapshots for one domain Session, including terminal and retired Jobs. + pub(crate) fn list_session_snapshots( + &self, + session_id: &str, + after: Option<&str>, + limit: usize, + ) -> Result, JobStoreError> { + let mut stmt = self.conn.prepare("SELECT i.job_id FROM job_intents i JOIN job_domain_grants g ON g.job_id=i.job_id WHERE json_extract(g.grant_json,'$.session_id')=?1 AND (?2 IS NULL OR i.job_id>?2) ORDER BY i.job_id LIMIT ?3")?; + let ids = stmt + .query_map( + params![session_id, after, limit.clamp(1, 100) as i64], + |r| r.get::<_, String>(0), + )? + .collect::>>()?; + ids.into_iter() + .map(|id| require_snapshot(&self.conn, &id)) + .collect() + } + + /// The Host must fence domain mutation and delivery before erasure/invalidation. + /// This atomically prevents dispatch/retry/result acceptance, without rewriting outcome. + pub(crate) fn reclaim_input( + &mut self, + job_id: &str, + expected_input_digest: &str, + reason: JobInputRetirementReason, + ) -> Result { + let tx = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate)?; + let snapshot = require_snapshot(&tx, job_id)?; + if snapshot.input_digest != expected_input_digest { + return Err(JobStoreError::DigestMismatch); + } + let desired = reason.disposition(); + if snapshot.input_disposition == desired { + tx.commit()?; + reclaim_retired_input_storage(&self.conn)?; + return Ok(snapshot); + } + if reason == JobInputRetirementReason::Completed + && (snapshot.state != JobState::Completed + || snapshot.attempt.state != JobAttemptState::Completed + || snapshot.result.is_none()) + { + return Err(JobStoreError::InvalidTransition( + "reclaim requires known completed result", + )); + } + if snapshot.input_disposition != JobInputDisposition::Available + && reason != JobInputRetirementReason::ExplicitErasure + { + return Err(JobStoreError::InvalidTransition( + "retired input cannot be restored or reclassified", + )); + } + if reason == JobInputRetirementReason::Completed { + snapshot.load_executable_request()?; + } + let mut metadata = snapshot.request.clone(); + metadata.input = Value::Null; + tx.execute( + "UPDATE job_intents SET request_json=?2, input_disposition=?3 WHERE job_id=?1", + params![job_id, serde_json::to_string(&metadata)?, desired.as_str()], + )?; + let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?; + if has_legacy_archive { + tx.execute("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input',json('null')) WHERE job_id=?1", [job_id])?; + } + let snapshot = require_snapshot(&tx, job_id)?; + tx.commit()?; + reclaim_retired_input_storage(&self.conn)?; + Ok(snapshot) + } + pub(crate) fn get(&self, job_id: &str) -> Result { require_snapshot(&self.conn, job_id) } @@ -383,7 +526,7 @@ impl JobStore { let mut statement = self.conn.prepare( "SELECT i.job_id FROM job_intents i JOIN job_attempts a ON a.job_id = i.job_id AND a.number = i.current_attempt - WHERE i.state = 'pending' AND a.state = 'reserved' ORDER BY i.rowid", + WHERE i.state = 'pending' AND i.input_disposition='available' AND a.state = 'reserved' ORDER BY i.rowid", )?; let ids = statement .query_map([], |row| row.get::<_, String>(0))? @@ -406,6 +549,7 @@ impl JobStore { .transaction_with_behavior(TransactionBehavior::Immediate)?; let snapshot = require_snapshot(&tx, job_id)?; check_attempt(&snapshot, attempt_id)?; + snapshot.load_executable_request()?; if snapshot.state != JobState::Pending || snapshot.attempt.state != JobAttemptState::Reserved { @@ -447,7 +591,7 @@ impl JobStore { .transaction_with_behavior(TransactionBehavior::Immediate)?; let snapshot = require_snapshot(&tx, &bound.job_id)?; check_attempt(&snapshot, &bound.attempt_id)?; - if snapshot.request.input_digest()? != bound.input_digest { + if snapshot.input_digest != bound.input_digest { return Err(JobStoreError::DigestMismatch); } let (digest, encoded) = @@ -466,6 +610,7 @@ impl JobStore { tx.commit()?; return Ok(snapshot); } + snapshot.load_executable_request()?; if snapshot.state != JobState::Pending || snapshot.attempt.state != JobAttemptState::Dispatched { @@ -649,7 +794,7 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result, let row = conn .query_row( "SELECT i.request_json, i.state, i.acknowledged, a.attempt_id, a.number, - a.state, a.failure, a.result_json FROM job_intents i JOIN job_attempts a + a.state, a.failure, a.result_json, a.input_digest, i.fingerprint, i.input_disposition FROM job_intents i JOIN job_attempts a ON a.job_id = i.job_id AND a.number = i.current_attempt WHERE i.job_id = ?1", [job_id], |row| { @@ -662,12 +807,27 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result, row.get::<_, String>(5)?, row.get::<_, Option>(6)?, row.get::<_, Option>(7)?, + row.get::<_, String>(8)?, + row.get::<_, String>(9)?, + row.get::<_, String>(10)?, )) }, ) .optional()?; row.map( - |(request, state, acknowledged, attempt_id, number, attempt_state, failure, result)| { + |( + request, + state, + acknowledged, + attempt_id, + number, + attempt_state, + failure, + result, + input_digest, + intent_fingerprint, + disposition, + )| { let state = match state.as_str() { "pending" => JobState::Pending, "completed" => JobState::Completed, @@ -693,8 +853,17 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result, ))); } }; + let request_value: Value = serde_json::from_str(&request)?; + if request_value.get("input").is_none() { + return Err(JobStoreError::Corrupt( + "stored Job input field is missing".into(), + )); + } Ok(JobSnapshot { - request: serde_json::from_str(&request)?, + request: serde_json::from_value(request_value)?, + input_digest, + intent_fingerprint, + input_disposition: serde_json::from_value(Value::String(disposition))?, domain_grant: conn .query_row( "SELECT grant_json FROM job_domain_grants WHERE job_id = ?1", @@ -719,6 +888,37 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result, .transpose() } +// Retirement is already durable before this maintenance step. Failure must be +// retried with the same Job/digest/reason; it must never restore input. VACUUM +// removes historical free-page copies, including stores written before secure_delete. +fn reclaim_retired_input_storage(conn: &Connection) -> Result<(), JobStoreError> { + conn.execute_batch("PRAGMA secure_delete=ON; VACUUM;")?; + let busy: i64 = conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| r.get(0))?; + if busy != 0 { + return Err(JobStoreError::InvalidTransition( + "input retired; SQLite body cleanup blocked by a reader, retry retirement", + )); + } + Ok(()) +} + +const INPUT_RETENTION_SCHEMA: &str = " +ALTER TABLE job_intents ADD COLUMN input_disposition TEXT NOT NULL DEFAULT 'available' + CHECK (input_disposition IN ('available','reclaimed','erased','invalidated')); +DROP TRIGGER job_intent_immutable; +CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF + job_id, fingerprint, serialization_key, max_concurrent ON job_intents +BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END; +CREATE TRIGGER job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON job_intents +WHEN ( + ((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated')) + OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased')) + AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') + AND json_type(NEW.request_json,'$.input') = 'null' +) IS NOT TRUE +BEGIN SELECT RAISE(ABORT, 'immutable Job input retirement'); END; +"; + const RECONCILIATION_SCHEMA: &str = " DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) @@ -885,7 +1085,11 @@ mod tests { Some("unconfirmed"), ) .unwrap(); - store.conn.execute_batch("DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap(); + store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable; + ALTER TABLE job_intents DROP COLUMN input_disposition; + CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents + BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END; + DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap(); drop(store); let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap(); assert_eq!(reopened.get("first").unwrap(), unknown); @@ -911,10 +1115,7 @@ mod tests { JobResultSubmission { job_id: snapshot.request.job_id.clone(), attempt_id: snapshot.attempt.attempt_id.clone(), - input_digest: snapshot - .request - .input_digest() - .expect("validated immutable Job input"), + input_digest: snapshot.input_digest.clone(), result, } } @@ -926,6 +1127,314 @@ mod tests { .unwrap() } + #[test] + fn input_reclaim_reopens_as_receipt_and_exact_replay_never_rehydrates() { + let (dir, mut store) = fixture(); + let original = request("retained"); + let grant = Some(json!({"session_id":"session-a", "consumer":"fixture"})); + let reserved = store + .reserve_granted(original.clone(), grant.clone()) + .unwrap(); + let running = store + .dispatch("retained", &reserved.attempt.attempt_id, 8) + .unwrap(); + let result = submission(&running, json!({"saved":true})); + let completed = store.accept(&result).unwrap(); + store.ack("retained").unwrap(); + let retired = store + .reclaim_input( + "retained", + &completed.input_digest, + JobInputRetirementReason::Completed, + ) + .unwrap(); + assert_eq!(retired.input_disposition, JobInputDisposition::Reclaimed); + assert!(retired.request.input.is_null()); + assert!(retired.load_executable_request().is_err()); + assert_eq!( + retired.outcome().input_digest, + original.input_digest().unwrap() + ); + assert_eq!(retired.result, completed.result); + drop(store); + let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap(); + assert_eq!( + reopened + .reserve_granted(original.clone(), grant.clone()) + .unwrap(), + retired + ); + assert_eq!(reopened.accept(&result).unwrap(), retired); + assert_eq!( + reopened + .list_session_snapshots("session-a", None, 1) + .unwrap(), + vec![retired.clone()] + ); + assert!( + reopened + .list_session_snapshots("session-b", None, 100) + .unwrap() + .is_empty() + ); + assert!( + reopened + .list_session_snapshots("session-a", Some("retained"), 100) + .unwrap() + .is_empty() + ); + let mut changed = original.clone(); + changed.input = json!({"changed":true}); + assert!(matches!( + reopened.reserve_granted(changed, grant), + Err(JobStoreError::IntentConflict(_)) + )); + assert!(reopened.retry("retained").is_err()); + assert!( + reopened + .dispatch("retained", &retired.attempt.attempt_id, 8) + .is_err() + ); + assert!(reopened.conn.execute("UPDATE job_intents SET request_json=?1,input_disposition='available' WHERE job_id='retained'", [serde_json::to_string(&original).unwrap()]).is_err()); + let erased = reopened + .reclaim_input( + "retained", + &retired.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert_eq!(erased.input_disposition, JobInputDisposition::Erased); + assert_eq!(erased.state, JobState::Completed); + assert!(erased.acknowledged); + assert_eq!( + reopened + .reserve_granted(original, erased.domain_grant.clone()) + .unwrap(), + erased + ); + } + + #[test] + fn unfinished_unknown_and_failed_inputs_are_kept_until_explicit_invalidation() { + let (_dir, mut store) = fixture(); + for (id, state) in [ + ("pending", None), + ("unknown", Some(JobAttemptState::Unknown)), + ("failed", Some(JobAttemptState::Failed)), + ] { + let original = request(id); + let snapshot = store.reserve(original.clone()).unwrap(); + if let Some(state) = state { + store.dispatch(id, &snapshot.attempt.attempt_id, 8).unwrap(); + store + .finish( + id, + &snapshot.attempt.attempt_id, + state, + Some("not complete"), + ) + .unwrap(); + } + let before = store.get(id).unwrap(); + assert!( + store + .reclaim_input( + id, + &before.input_digest, + JobInputRetirementReason::Completed + ) + .is_err() + ); + assert_eq!(store.get(id).unwrap(), before); + assert_eq!(before.load_executable_request().unwrap(), &original); + assert!( + store + .reclaim_input(id, "wrong", JobInputRetirementReason::ExplicitErasure) + .is_err() + ); + let retired = store + .reclaim_input( + id, + &before.input_digest, + JobInputRetirementReason::LegacyInvalidation, + ) + .unwrap(); + assert_eq!(retired.state, before.state); + assert_eq!(retired.attempt, before.attempt); + assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated); + assert!(store.retry(id).is_err()); + assert!(store.dispatch(id, &before.attempt.attempt_id, 8).is_err()); + assert_eq!(store.reserve(original).unwrap(), retired); + } + } + + #[test] + fn erased_dispatched_input_refuses_delayed_result_without_rewriting_unknown() { + let (dir, mut store) = fixture(); + let running = dispatched(&mut store, request("erase-running")); + let result = submission(&running, json!({"late":true})); + let erased = store + .reclaim_input( + "erase-running", + &running.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert!(store.accept(&result).is_err()); + assert_eq!(erased.state, JobState::Pending); + drop(store); + let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap(); + let recovered = reopened.get("erase-running").unwrap(); + assert_eq!(recovered.state, JobState::Unknown); + assert_eq!(recovered.input_disposition, JobInputDisposition::Erased); + assert_eq!(recovered.input_digest, running.input_digest); + assert!(reopened.accept(&result).is_err()); + assert!(reopened.retry("erase-running").is_err()); + } + + #[test] + fn available_input_loader_refuses_tamper_missing_body_and_envelope_changes() { + for mutation in [ + "json_set(request_json,'$.input',json('{\"tampered\":true}'))", + "json_remove(request_json,'$.input')", + "json_set(request_json,'$.instruction','changed')", + ] { + let (_dir, mut store) = fixture(); + store.reserve(request("tamper")).unwrap(); + // Simulate external corruption bypassing SQLite's normal write guard. + store + .conn + .execute_batch("DROP TRIGGER job_input_retirement") + .unwrap(); + store + .conn + .execute_batch(&format!("UPDATE job_intents SET request_json={mutation}")) + .unwrap(); + match store.get("tamper") { + Ok(snapshot) => { + assert!(snapshot.load_executable_request().is_err()); + assert!( + store + .dispatch("tamper", &snapshot.attempt.attempt_id, 8) + .is_err() + ); + } + Err(JobStoreError::Json(_) | JobStoreError::Corrupt(_)) => {} + other => panic!("unexpected corrupt input read: {other:?}"), + } + } + } + + #[test] + fn input_retirement_cleanup_failure_is_durable_and_exact_retry_finishes_without_rehydration() { + let (dir, mut store) = fixture(); + let mut original = request("physical-erasure"); + let private_body = "private-input-retirement-original-bytes".repeat(1000); + original.input = json!({"body":private_body}); + let reserved = store.reserve(original.clone()).unwrap(); + // A reader retaining the old SQLite snapshot prevents truncation of the old WAL. + let reader = Connection::open(dir.path().join("jobs.sqlite3")).unwrap(); + reader + .execute_batch("BEGIN; SELECT request_json FROM job_intents;") + .unwrap(); + store.conn.busy_timeout(std::time::Duration::ZERO).unwrap(); + assert!( + store + .reclaim_input( + "physical-erasure", + &reserved.input_digest, + JobInputRetirementReason::ExplicitErasure + ) + .is_err() + ); + let retired = store.get("physical-erasure").unwrap(); + assert_eq!(retired.input_disposition, JobInputDisposition::Erased); + assert!(retired.load_executable_request().is_err()); + reader.execute_batch("ROLLBACK").unwrap(); + drop(reader); + let replay = store + .reclaim_input( + "physical-erasure", + &reserved.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert_eq!(replay, retired); + assert_eq!(store.reserve(original).unwrap(), retired); + let needle = b"private-input-retirement-original-bytes"; + for entry in std::fs::read_dir(dir.path()).unwrap() { + let path = entry.unwrap().path(); + if path.is_file() { + let bytes = std::fs::read(&path).unwrap(); + assert!( + !bytes.windows(needle.len()).any(|bytes| bytes == needle), + "retired body remained in {}", + path.display() + ); + } + } + } + + #[test] + fn missing_original_null_input_is_not_treated_as_available_null() { + let (_dir, mut store) = fixture(); + let mut original = request("null-original"); + original.input = Value::Null; + let reserved = store.reserve(original).unwrap(); + assert!(reserved.load_executable_request().is_ok()); + store.conn.execute_batch("DROP TRIGGER job_input_retirement; UPDATE job_intents SET request_json=json_remove(request_json,'$.input')").unwrap(); + assert!(store.get("null-original").is_err()); + } + + #[test] + fn input_retirement_migration_preserves_receipts_and_rolls_back_failed_ddl() { + let (dir, mut store) = fixture(); + let original = store.reserve(request("upgrade")).unwrap(); + store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable; + ALTER TABLE job_intents DROP COLUMN input_disposition; + CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents + BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END; + PRAGMA user_version=4; + CREATE TRIGGER job_input_retirement BEFORE UPDATE ON job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;").unwrap(); + drop(store); + let path = dir.path().join("jobs.sqlite3"); + assert!(JobStore::open(&path).is_err()); + let conn = Connection::open(&path).unwrap(); + let version: i64 = conn + .pragma_query_value(None, "user_version", |r| r.get(0)) + .unwrap(); + assert_eq!(version, 4); + let present: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM pragma_table_info('job_intents') WHERE name='input_disposition')",[],|r|r.get(0)).unwrap(); + assert!(!present); + conn.execute_batch("DROP TRIGGER job_input_retirement") + .unwrap(); + drop(conn); + let mut reopened = JobStore::open(&path).unwrap(); + assert_eq!(reopened.get("upgrade").unwrap(), original); + assert_eq!( + reopened + .get("upgrade") + .unwrap() + .load_executable_request() + .unwrap(), + &request("upgrade") + ); + let retired = reopened + .reclaim_input( + "upgrade", + &original.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert_eq!(retired.state, JobState::Pending); + assert!( + reopened + .conn + .execute_batch("UPDATE job_intents SET input_disposition='available'") + .is_err() + ); + } + #[test] fn database_configuration_schema_and_identity_constraints() { let (_dir, mut store) = fixture(); diff --git a/crates/standalone/src/job_store/legacy_migrations.rs b/crates/standalone/src/job_store/legacy_migrations.rs index ec05286cf..f1b8cb539 100644 --- a/crates/standalone/src/job_store/legacy_migrations.rs +++ b/crates/standalone/src/job_store/legacy_migrations.rs @@ -143,6 +143,42 @@ mod tests { (original, receipt, grant) } + #[test] + fn legacy_archive_input_erasure_preserves_fingerprints_and_attempt_receipts() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.sqlite3"); + seed(&path, 2); + let mut store = JobStore::open(&path).unwrap(); + let before: (String, String) = store + .conn + .query_row( + "SELECT fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + let snapshot = store.get("done").unwrap(); + store + .reclaim_input( + "done", + &snapshot.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + let (body, fingerprint, attempts): (String,String,String) = store.conn.query_row("SELECT request_json,fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap(); + assert!(serde_json::from_str::(&body).unwrap()["input"].is_null()); + assert_eq!((fingerprint, attempts), before); + assert!(store.conn.execute_batch("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input','restored') WHERE job_id='done'").is_err()); + drop(store); + let mut reopened = JobStore::open(&path).unwrap(); + let receipt = reopened + .reserve_granted(request("done"), snapshot.domain_grant) + .unwrap(); + assert_eq!(receipt.state, JobState::Completed); + assert_eq!(receipt.input_disposition, JobInputDisposition::Erased); + assert!(receipt.load_executable_request().is_err()); + } + #[test] fn legacy_versions_preserve_intent_receipts_attempts_grants_and_replay() { for version in [1, 2] { diff --git a/crates/standalone/src/jobs.rs b/crates/standalone/src/jobs.rs index 890eed041..bc375e005 100644 --- a/crates/standalone/src/jobs.rs +++ b/crates/standalone/src/jobs.rs @@ -26,7 +26,10 @@ use job::{JobRequest, JobResultSubmission}; use tokio::sync::watch; use crate::job_store::JobStore; -pub use crate::job_store::{JobAttempt, JobAttemptState, JobSnapshot, JobState, JobStoreError}; +pub use crate::job_store::{ + JobAttempt, JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot, + JobState, JobStoreError, +}; #[derive(Debug, thiserror::Error)] pub enum JobServiceError { @@ -40,6 +43,8 @@ pub enum JobServiceError { Capacity, #[error("standalone Job cleanup failed: {0}")] Cleanup(String), + #[error("standalone Job shutdown is incomplete; execution resources are still owned")] + ShutdownIncomplete, } pub(crate) trait JobAttemptFence: Send + Sync { @@ -173,6 +178,36 @@ impl StandaloneJobs { Ok(self.with_store(|store| store.reserve(request))?) } + /// Enumerate historical domain snapshots for a Session, including retired input receipts. + pub fn list_session_snapshots( + &self, + session_id: &str, + after: Option<&str>, + limit: usize, + ) -> Result, JobServiceError> { + self.with_store(|store| store.list_session_snapshots(session_id, after, limit)) + } + + /// Retire body only. The caller fences domain effects/delivered snapshots before erasure. + /// Normal reclamation also requires that execution cleanup has finished. + pub fn reclaim_input( + &self, + job_id: &str, + expected_input_digest: &str, + reason: JobInputRetirementReason, + ) -> Result { + let executions = self.executions.lock().expect("Job executions poisoned"); + if reason == JobInputRetirementReason::Completed + && executions + .jobs + .get(job_id) + .is_some_and(|execution| !execution.task.is_finished()) + { + return Err(JobServiceError::Capacity); + } + self.with_store(|store| store.reclaim_input(job_id, expected_input_digest, reason)) + } + pub fn get(&self, job_id: &str) -> Result { Ok(self.with_store(|store| store.get(job_id))?) } @@ -274,6 +309,12 @@ impl StandaloneJobs { job_id: &str, client: Option>, ) -> Result<(), JobServiceError> { + // Preparation owns copies of the immutable input and domain adapter too. Exclude + // owner shutdown before creating them, not only before publishing the task. + let mut executions = self.executions.lock().expect("Job executions poisoned"); + if executions.closed { + return Err(JobServiceError::Closed); + } let client = client.or_else(|| { self.model_client .lock() @@ -282,16 +323,14 @@ impl StandaloneJobs { .map(|c| c.clone_boxed()) }); let snapshot = self.get(job_id)?; + snapshot.load_executable_request()?; let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds)); let deadline = tokio::time::Instant::now() + timeout; let fence: Arc = Arc::new(BoundAttemptFence { store: self.store.clone(), job_id: job_id.into(), attempt_id: snapshot.attempt.attempt_id.clone(), - input_digest: snapshot - .request - .input_digest() - .map_err(JobStoreError::from)?, + input_digest: snapshot.input_digest.clone(), deadline, }); let grant = match ( @@ -311,10 +350,6 @@ impl StandaloneJobs { let prepared = worker::job::prepare_job_with_grant(&snapshot.request, &self.cwd, client, grant) .map_err(|error| JobServiceError::Configuration(error.to_string()))?; - let mut executions = self.executions.lock().expect("Job executions poisoned"); - if executions.closed { - return Err(JobServiceError::Closed); - } executions.jobs.retain(|_, run| { !(run.task.is_finished() && matches!(run.complete.borrow().as_ref(), Some(Ok(())))) }); @@ -367,10 +402,7 @@ impl StandaloneJobs { store: store.clone(), job_id: id.clone(), attempt_id: attempt_id.clone(), - input_digest: snapshot - .request - .input_digest() - .map_err(JobStoreError::from)?, + input_digest: snapshot.input_digest.clone(), deadline, }); let (cancel, mut cancellation) = watch::channel(false); @@ -446,23 +478,25 @@ impl StandaloneJobs { .clone() .expect("completion"); result.map_err(JobServiceError::Cleanup)?; - let task = { - let mut executions = self.executions.lock().expect("Job executions poisoned"); - if executions - .jobs - .get(job_id) - .is_some_and(|run| run.attempt_id == attempt) - { - executions.jobs.remove(job_id) - } else { - None - } - }; - if let Some(execution) = task { - execution - .task - .await - .map_err(|error| JobServiceError::Cleanup(error.to_string()))??; + // Completion is published immediately before the task returns. Do not remove + // its ownership record until the runtime confirms that return (and future drop). + while self + .executions + .lock() + .expect("Job executions poisoned") + .jobs + .get(job_id) + .is_some_and(|run| run.attempt_id == attempt && !run.task.is_finished()) + { + tokio::task::yield_now().await; + } + let mut executions = self.executions.lock().expect("Job executions poisoned"); + if executions + .jobs + .get(job_id) + .is_some_and(|run| run.attempt_id == attempt) + { + executions.jobs.remove(job_id); } } self.get(job_id) @@ -476,33 +510,63 @@ impl StandaloneJobs { } } - pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> { + /// Initiate owner shutdown without blocking a synchronous caller. Success confirms all + /// local executions have returned, including request/history and Feature cleanup. An + /// incomplete/error result retains the service resources and must retain the Host lease. + pub(crate) fn try_shutdown(&self) -> Result<(), JobServiceError> { self.close(); - let tasks = { - let mut executions = self.executions.lock().expect("Job executions poisoned"); - std::mem::take(&mut executions.jobs) - }; - let mut error = None; - for execution in tasks.into_values() { - match execution.task.await { - Ok(Ok(())) => {} - Ok(Err(failure)) => { - error = Some(failure); - } - Err(failure) => { - error = Some(JobServiceError::Cleanup(failure.to_string())); + let mut executions = self.executions.lock().expect("Job executions poisoned"); + for execution in executions.jobs.values() { + if !execution.task.is_finished() { + return Err(JobServiceError::ShutdownIncomplete); + } + match execution.complete.borrow().as_ref() { + Some(Ok(())) => {} + Some(Err(error)) => return Err(JobServiceError::Cleanup(error.clone())), + None => { + return Err(JobServiceError::Cleanup( + "Internal Worker ended without cleanup confirmation".into(), + )); } } } - // Close SQLite before the Host releases its lease. Cloned handles may - // outlive the Host but cannot mutate or retain the live connection. + executions.jobs.clear(); + // Retained external handles share these slots; they cannot retain a live store, + // provider or model client after successful owner shutdown. self.store.lock().expect("Job store poisoned").take(); self.domain.lock().expect("Job domain poisoned").take(); self.model_client .lock() .expect("Job model transport poisoned") .take(); - error.map_or(Ok(()), Err) + Ok(()) + } + + pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> { + self.close(); + let completions: Vec<_> = self + .executions + .lock() + .expect("Job executions poisoned") + .jobs + .values() + .map(|execution| execution.complete.clone()) + .collect(); + // Never take execution ownership out of the shared service across an await: a + // cancelled shutdown future or a concurrent close must still see the barrier. + for mut completion in completions { + completion + .wait_for(Option::is_some) + .await + .map_err(|error| JobServiceError::Cleanup(error.to_string()))?; + } + loop { + match self.try_shutdown() { + // Only the task's final return follows its completion notification. + Err(JobServiceError::ShutdownIncomplete) => tokio::task::yield_now().await, + result => return result, + } + } } } @@ -574,7 +638,8 @@ impl JobAttemptFence for BoundAttemptFence { if snapshot.state != JobState::Pending || snapshot.attempt.state != JobAttemptState::Dispatched || snapshot.attempt.attempt_id != self.attempt_id - || snapshot.request.input_digest().map_err(|e| e.to_string())? != self.input_digest + || snapshot.input_digest != self.input_digest + || snapshot.input_disposition != JobInputDisposition::Available { return Err("Job domain grant is not bound to the current live attempt".into()); } diff --git a/crates/standalone/src/jobs/tests.rs b/crates/standalone/src/jobs/tests.rs index cc5d00f7b..06325c054 100644 --- a/crates/standalone/src/jobs/tests.rs +++ b/crates/standalone/src/jobs/tests.rs @@ -468,3 +468,104 @@ async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_han assert!(matches!(operation, Err(JobServiceError::Closed))); } } + +#[tokio::test] +async fn cancelled_shutdown_future_keeps_execution_barrier_and_unknown_input() { + // Actual generic runner + SQLite; only the provider stream is scripted/blocked. + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("jobs.sqlite3"); + let jobs = StandaloneJobs::open(&path, temp.path().into()).unwrap(); + let reserved = jobs.request(request("cancel-shutdown-wait")).unwrap(); + let model = DomainTestClient { + stall: true, + ..Default::default() + }; + jobs.start_with_model_client(&reserved.request.job_id, model.clone()) + .await + .unwrap(); + await_model(&model).await; + { + let shutdown = jobs.shutdown(); + tokio::pin!(shutdown); + assert!(futures::poll!(shutdown.as_mut()).is_pending()); + // Drop the caller's waiter while cancellation/cleanup is still pending. + } + assert!(matches!( + jobs.try_shutdown(), + Err(JobServiceError::ShutdownIncomplete) + )); + assert!(jobs.store.lock().unwrap().is_some()); + assert!(matches!( + jobs.request(request("after-shutdown")), + Err(JobServiceError::Closed) + )); + tokio::time::timeout(Duration::from_secs(5), jobs.shutdown()) + .await + .unwrap() + .unwrap(); + assert!(jobs.store.lock().unwrap().is_none()); + let reopened = StandaloneJobs::open(&path, temp.path().into()).unwrap(); + let unknown = reopened.get(&reserved.request.job_id).unwrap(); + assert_eq!(unknown.attempt.state, JobAttemptState::Unknown); + assert_eq!(unknown.input_disposition, JobInputDisposition::Available); + assert_eq!(unknown.request, reserved.request); + assert_eq!(unknown.input_digest, reserved.input_digest); + assert!(reopened.retry(&unknown.request.job_id).is_err()); + reopened.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn input_retirement_service_fences_live_attempt_and_preserves_session_receipts() { + // Production service + SQLite + attempt/result fences, without a model or Worker fixture. + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("jobs.sqlite3"); + let jobs = StandaloneJobs::open(&path, temp.path().to_path_buf()).unwrap(); + let original = request("input-retirement"); + let grant = json!({"session_id":"session-a"}); + let reserved = jobs + .request_granted(original.clone(), grant.clone()) + .unwrap(); + jobs.with_store(|store| store.dispatch("input-retirement", &reserved.attempt.attempt_id, 8)) + .unwrap(); + let fence = BoundAttemptFence { + store: jobs.store.clone(), + job_id: "input-retirement".into(), + attempt_id: reserved.attempt.attempt_id.clone(), + input_digest: reserved.input_digest.clone(), + deadline: tokio::time::Instant::now() + Duration::from_secs(10), + }; + assert!(fence.ensure_live().is_ok()); + assert!( + jobs.reclaim_input( + "input-retirement", + &reserved.input_digest, + JobInputRetirementReason::Completed + ) + .is_err() + ); + let erased = jobs + .reclaim_input( + "input-retirement", + &reserved.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert!(fence.ensure_live().is_err()); + assert!( + sink(&jobs, "input-retirement", &reserved.attempt.attempt_id) + .submit(result("input-retirement", &reserved.attempt.attempt_id)) + .is_err() + ); + assert!(jobs.start("input-retirement").await.is_err()); + assert_eq!( + jobs.list_session_snapshots("session-a", None, 100).unwrap(), + vec![erased.clone()] + ); + assert!( + jobs.list_session_snapshots("session-b", None, 100) + .unwrap() + .is_empty() + ); + assert_eq!(jobs.request_granted(original, grant).unwrap(), erased); + jobs.shutdown().await.unwrap(); +} diff --git a/crates/standalone/src/subjektiv.rs b/crates/standalone/src/subjektiv.rs index a5cea4379..353538072 100644 --- a/crates/standalone/src/subjektiv.rs +++ b/crates/standalone/src/subjektiv.rs @@ -54,6 +54,8 @@ pub enum SubjectError { Journal(#[from] subjektiv::journal::JournalError), #[error(transparent)] Storage(#[from] feature_storage::FeatureStorageError), + #[error("local Subject Job shutdown: {0}")] + JobShutdown(#[from] crate::jobs::JobServiceError), #[error("local Subject storage metadata: {0}")] Json(#[from] serde_json::Error), #[error("local Subject storage permission or identity is invalid: {0}")] @@ -338,8 +340,16 @@ impl SubjectConnection { } } pub fn close(&mut self) -> Result<(), SubjectError> { - self.host.jobs.lock().expect("Subject Jobs poisoned").take(); self.host.live.store(false, Ordering::Release); + { + let mut jobs = self.host.jobs.lock().expect("Subject Jobs poisoned"); + if let Some(service) = jobs.as_ref() { + // Signal cancellation, but retain the handle and exclusive lease until + // the generic owner confirms local request/history/resource teardown. + service.try_shutdown()?; + } + jobs.take(); + } self.host.catalog.manager.shutdown()?; if let Some(file) = &self.lease { file.unlock()?; diff --git a/crates/standalone/src/subjektiv/journal.rs b/crates/standalone/src/subjektiv/journal.rs index 5b2c880a4..85bbe26f1 100644 --- a/crates/standalone/src/subjektiv/journal.rs +++ b/crates/standalone/src/subjektiv/journal.rs @@ -5,9 +5,12 @@ //! recovery authority; shutdown does not delete either original Sessions or pending requests. use super::*; -use crate::jobs::{JobAttemptState, JobSnapshot, JobState}; +use crate::jobs::{ + JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot, JobState, +}; +use session_store::capture::{CaptureDto, CaptureLimits, RecordingPosition}; use subjektiv::api::JobAttemptBinding; -use subjektiv::journal::recording::JournalMaterial; +use subjektiv::journal::recording::JournalInput; use subjektiv::journal::transport::*; use subjektiv::journal::*; use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeature}; @@ -16,7 +19,6 @@ use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeat mod tests; pub(super) const PURPOSE: &str = "subjektiv_journal"; -const INPUT_BUDGET: usize = 48 * 1024; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -27,6 +29,7 @@ struct LocalJournalGrant { session_id: String, journal_id: String, ranges: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] origins: Vec, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -35,11 +38,6 @@ struct SourceOrigin { pointer: JournalSourcePointer, origin: session_store::public_material::PublicMaterialOrigin, } -impl LocalJournalGrant { - fn input(&self) -> serde_json::Value { - serde_json::json!({"selected_ranges": self.ranges, "origins": self.origins}) - } -} impl StandaloneSubjects { /// Explicit operator erasure, separate from ordinary Worker removal. Requires the @@ -84,32 +82,29 @@ impl StandaloneSubjects { self.state_root.clone(), ) .map_err(|error| SubjectError::InvalidScope(error.to_string()))?; - for snapshot in jobs - .pending() - .map_err(|e| SubjectError::InvalidScope(e.to_string()))? - .into_iter() - .chain( - jobs.unknown_granted() - .map_err(|e| SubjectError::InvalidScope(e.to_string()))?, - ) - { - if snapshot.domain_grant.as_ref().is_some_and(|grant| { - grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id) - }) { + let mut after = None; + loop { + let page = jobs + .list_session_snapshots(session_id, after.as_deref(), 100) + .map_err(|e| SubjectError::InvalidScope(e.to_string()))?; + if page.is_empty() { + break; + } + for snapshot in &page { jobs.cancel(&snapshot.request.job_id) .map_err(|e| SubjectError::InvalidScope(e.to_string()))?; + jobs.reclaim_input( + &snapshot.request.job_id, + &snapshot.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .map_err(|e| SubjectError::InvalidScope(e.to_string()))?; + if snapshot.state == JobState::Completed && !snapshot.acknowledged { + jobs.acknowledge(&snapshot.request.job_id) + .map_err(|e| SubjectError::InvalidScope(e.to_string()))?; + } } - } - for snapshot in jobs - .unacknowledged_results() - .map_err(|e| SubjectError::InvalidScope(e.to_string()))? - { - if snapshot.domain_grant.as_ref().is_some_and(|grant| { - grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id) - }) { - jobs.acknowledge(&snapshot.request.job_id) - .map_err(|e| SubjectError::InvalidScope(e.to_string()))?; - } + after = page.last().map(|job| job.request.job_id.clone()); } session_store::FsStore::new_with_retention( path, @@ -590,17 +585,28 @@ impl LocalSubjectHost { self.ensure_live()?; let jobs = self.jobs()?; for saved in jobs.unacknowledged_results().map_err(service_error)? { - if self.journal_grant(&saved).is_ok() { + if self.journal_grant_scope(&saved).is_ok() { self.complete_journal_job(&jobs, &saved)?; } } for pending in jobs.pending().map_err(service_error)? { - if self.journal_grant(&pending).is_ok() { + if pending.request.purpose != PURPOSE { + continue; + } + self.journal_grant_scope(&pending)?; + if pending.request.input.get("capture").is_some() { + self.journal_grant(&pending)?; return Ok(Some((jobs, pending.request.job_id))); } + // Old descriptor jobs never acquire new execution authority. Their immutable + // identity/input is retired, not upgraded in place; saved checkpoint wins. + let replacement = self.replace_legacy_journal_job(&jobs, &pending)?; + if let Some(id) = replacement { + return Ok(Some((jobs, id))); + } } for unknown in jobs.unknown_granted().map_err(service_error)? { - let Ok(grant) = self.journal_grant(&unknown) else { + let Ok(grant) = self.journal_grant_scope(&unknown) else { continue; }; let context = authority(self, &grant.session_id)?; @@ -685,32 +691,96 @@ impl LocalSubjectHost { } } } - let materials = self.select_journal_materials(&context, &journal.id)?; - if materials.is_empty() { + let mut retained = None; + // Saved body snapshots always win over descriptor migration. In particular, + // a failed continuation must not recapture its original text even if an + // invalidated legacy grant still bounds a larger unprocessed suffix. + for legacy_pass in [false, true] { + let mut after_job = None; + loop { + let page = jobs + .list_session_snapshots( + &journal.source_session_id, + after_job.as_deref(), + 100, + ) + .map_err(service_error)?; + if page.is_empty() { + break; + } + for saved in &page { + if saved.request.purpose != PURPOSE { + continue; + } + let legacy = saved.input_disposition + == JobInputDisposition::Invalidated + || (saved.input_disposition == JobInputDisposition::Available + && saved.request.input.get("capture").is_none()); + if legacy != legacy_pass + || (!legacy + && saved.input_disposition != JobInputDisposition::Available) + { + continue; + } + let saved_grant = self.journal_grant_scope(saved)?; + if saved_grant.journal_id != journal.id { + continue; + } + if legacy { + if let Some(id) = self.replace_legacy_journal_job(&jobs, saved)? { + return Ok(Some((jobs, id))); + } + continue; + } + self.journal_grant(saved)?; + let original = JournalInput::decode(&saved.request.input) + .map_err(journal_error)?; + let continuation = self.build_journal_input( + &context, + &journal.id, + Some(&original.capture), + )?; + if !continuation.selected_ranges.is_empty() { + retained = Some(continuation); + break; + } + if saved.state == JobState::Completed && saved.acknowledged { + // Recover the acknowledgement-before-reclamation gap without + // turning a retained receipt into an executable empty input. + jobs.reclaim_input( + &saved.request.job_id, + &saved.input_digest, + JobInputRetirementReason::Completed, + ) + .map_err(service_error)?; + } + } + if retained.is_some() { + break; + } + after_job = page.last().map(|job| job.request.job_id.clone()); + } + if retained.is_some() { + break; + } + } + let input = match retained { + Some(input) => input, + None => self.build_journal_input(&context, &journal.id, None)?, + }; + + if input.selected_ranges.is_empty() { continue; } - let ranges = materials - .into_iter() - .map(|material| material.range) - .collect::>(); - let origins = self.journal_origins(&ranges)?; let grant = LocalJournalGrant { writer_binding_id: Some(self.writer.binding_id.clone()), binding: self.binding.clone(), session_id: journal.source_session_id, journal_id: journal.id.clone(), - ranges, - origins, - }; - let request = job::JobRequest { - job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()), - purpose: PURPOSE.into(), input_ref: format!("subjektiv://{}/journal/{}", self.binding.subject_id, journal.id), - input: grant.input(), - instruction: "ReadJournalState first. Input descriptors are the frozen canonical public ranges authorized for this Job, not already read material. Read every selected source byte with bounded ReadJournalText before recording; then read initial/current position and open text in bounded pages. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(), - profile: "builtin:subjektiv-journal-recording".into(), - serialization_key: Some(format!("subjektiv-journal:{}", journal.id)), - limits: job::JobLimits { timeout_seconds: 600, ..Default::default() }, + ranges: input.selected_ranges.clone(), + origins: vec![], }; + let request = self.journal_request(&journal.id, &input)?; let saved = jobs .request_granted( request, @@ -727,6 +797,138 @@ impl LocalSubjectHost { } } + fn replace_legacy_journal_job( + &self, + jobs: &crate::jobs::StandaloneJobs, + old: &JobSnapshot, + ) -> Result, SubjektivHostError> { + let grant = self.journal_grant_scope(old)?; + if old.input_disposition == JobInputDisposition::Available { + old.load_executable_request() + .map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?; + if old.request.input + != serde_json::json!({"selected_ranges":grant.ranges,"origins":grant.origins}) + { + return Err(SubjektivHostError::InvalidArgument( + "invalid legacy Journal descriptor input".into(), + )); + } + } else if old.input_disposition != JobInputDisposition::Invalidated { + return Err(SubjektivHostError::InvalidArgument( + "legacy Journal input is unavailable".into(), + )); + } + let context = authority(self, &grant.session_id)?; + if let Some(active) = self + .catalog + .store + .active_journal_generation(&context, &grant.journal_id) + .map_err(journal_error)? + { + if active.job.job_id != old.request.job_id { + return Err(SubjektivHostError::InvalidArgument( + "another generation owns legacy migration".into(), + )); + } + self.catalog + .store + .close_journal_generation(&context, &active.id, &active.job) + .map_err(journal_error)?; + } + jobs.cancel(&old.request.job_id).map_err(service_error)?; + let current = self + .catalog + .store + .read_journal(&context, &grant.journal_id) + .map_err(journal_error)?; + let end = grant + .ranges + .last() + .ok_or_else(|| SubjektivHostError::InvalidArgument("empty legacy selection".into()))?; + let end_position = if end.bytes.end == end.source.total_bytes { + (end.source.order + 1, 0) + } else { + (end.source.order, end.bytes.end) + }; + let replacement = if (current.process_order, current.process_byte) >= end_position { + None + } else { + let mut capture = self.recording_capture(&grant.session_id)?; + capture + .entries + .retain(|entry| grant.ranges.iter().any(|r| r.source.order == entry.order)); + for entry in &mut capture.entries { + let range = grant + .ranges + .iter() + .find(|r| r.source.order == entry.order) + .unwrap(); + let session_store::capture::CaptureBody::CanonicalPublic { + descriptor, + byte_offset, + text, + .. + } = &mut entry.body + else { + unreachable!() + }; + if source_descriptor(&grant.session_id, descriptor) != range.source + || !grant.origins.iter().any(|origin| { + origin.pointer == range.source.pointer && origin.origin == descriptor.origin + }) + || !text.is_char_boundary(range.bytes.start as usize) + || !text.is_char_boundary(range.bytes.end as usize) + { + return Err(SubjektivHostError::InvalidArgument( + "legacy original source identity/range changed".into(), + )); + } + *text = text[range.bytes.start as usize..range.bytes.end as usize].to_owned(); + *byte_offset = range.bytes.start; + } + let input = self.build_journal_input(&context, &grant.journal_id, Some(&capture))?; + let request = self.journal_request(&grant.journal_id, &input)?; + let next_grant = LocalJournalGrant { + ranges: input.selected_ranges.clone(), + origins: vec![], + ..grant + }; + Some( + jobs.request_granted( + request, + serde_json::to_value(next_grant) + .map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?, + ) + .map_err(service_error)? + .request + .job_id, + ) + }; + jobs.reclaim_input( + &old.request.job_id, + &old.input_digest, + JobInputRetirementReason::LegacyInvalidation, + ) + .map_err(service_error)?; + Ok(replacement) + } + + fn journal_request( + &self, + journal_id: &str, + input: &JournalInput, + ) -> Result { + Ok(job::JobRequest { + job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()), purpose: PURPOSE.into(), + input_ref: format!("subjektiv://{}/journal/{journal_id}", self.binding.subject_id), + input: serde_json::to_value(input).map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?, + instruction: "Input contains the frozen canonical public source bodies, provenance and byte ranges. Use only this saved input, never re-fetch Source. ReadJournalState first and read initial/current Journal position/open text in bounded pages if needed. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(), + profile: "builtin:subjektiv-journal-recording".into(), + serialization_key: Some(format!("subjektiv-journal:{journal_id}")), + limits: job::JobLimits { timeout_seconds: 600, ..Default::default() }, + }) + } + fn pin_journal_session(&self, session: &str) -> Result<(), SubjektivHostError> { let path = crate::subjektiv_sessions::authorized_sessions_path( &self.catalog.state_root, @@ -869,11 +1071,16 @@ impl LocalSubjectHost { Ok(()) } - fn material_reader( + fn with_recording_boundary( &self, session: &str, - ) -> Result - { + read: impl FnOnce( + &Path, + session_store::SessionId, + session_store::SegmentId, + &session_store::retention::SessionRetentionGuard, + ) -> Result, + ) -> Result { let sessions = crate::subjektiv_sessions::authorized_sessions_path( &self.catalog.state_root, &self.binding.scope_id, @@ -906,6 +1113,11 @@ impl LocalSubjectHost { .ok_or_else(|| SubjektivHostError::InvalidArgument("invalid Session storage".into()))? .join("worker") .join(&record.storage_key); + let session_id = session + .parse() + .map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?; + let retention = crate::store::session_retention(&self.catalog.state_root); + let guard = retention.lock_live(session_id).map_err(source_error)?; // This is the persisted active selector, not the newest filename/mtime or a model claim. let metadata = session_store::WorkerAggregateStore::open_read_only(metadata_root, &record.storage_key) @@ -933,154 +1145,203 @@ impl LocalSubjectHost { "no attested active lineage for the retained Session".into(), ) })?; - session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention( - &sessions, - session - .parse() - .map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?, - active, - session_store::public_material::PublicMaterialLimits { - max_segments: 256, - max_entries: 100_000, - max_record_bytes: 64 * 1024 * 1024, - }, - crate::store::session_retention(&self.catalog.state_root), - ) - .map_err(|error| SubjektivHostError::Unavailable(error.to_string())) + read(&sessions, session_id, active, &guard) + } + + fn recording_capture(&self, session: &str) -> Result { + self.with_recording_boundary(session, |sessions, session_id, active, guard| { + session_store::capture::capture_recording_fs_guarded( + sessions, + session_id, + active, + session_store::public_material::PublicMaterialLimits::default(), + CaptureLimits { + max_bytes: 128 * 1024 * 1024, + max_entries: 100_000, + max_text_bytes: 64 * 1024 * 1024, + ..Default::default() + }, + guard, + ) + .map_err(|e| SubjektivHostError::Unavailable(e.to_string())) + }) } fn journal_sources( &self, session: &str, ) -> Result, SubjektivHostError> { - let reader = self.material_reader(session)?; - Ok(reader - .descriptors() - .map(|descriptor| JournalSourceRange { - source: source_descriptor(session, descriptor), - bytes: JournalByteRange { - start: 0, - end: descriptor.total_bytes, + self.with_recording_boundary(session, |sessions, session_id, active, guard| { + session_store::capture::recording_catalog_fs_guarded( + sessions, + session_id, + active, + session_store::public_material::PublicMaterialLimits::default(), + guard, + |descriptors| { + descriptors + .iter() + .map(|descriptor| JournalSourceRange { + source: source_descriptor(session, descriptor), + bytes: JournalByteRange { + start: 0, + end: descriptor.total_bytes, + }, + }) + .collect() }, - }) - .collect()) - } - - fn source_reader( - &self, - source: &JournalSource, - ) -> Result - { - let sessions = crate::subjektiv_sessions::authorized_sessions_path( - &self.catalog.state_root, - &self.binding.scope_id, - &self.binding.subject_id, - &self.catalog.store, - &source.session_id, - )?; - // Frozen, already admitted original pointers do not follow the Worker's later active - // Session/Segment. This reads that original lineage and still checks descriptor identity, - // canonical order and digest below; the shared service enforces the frozen byte range. - session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention( - &sessions, - source - .session_id - .parse() - .map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?, - source.pointer.segment_id, - session_store::public_material::PublicMaterialLimits { - max_segments: 256, - max_entries: 100_000, - max_record_bytes: 64 * 1024 * 1024, - }, - crate::store::session_retention(&self.catalog.state_root), - ) - .map_err(|error| SubjektivHostError::Unavailable(error.to_string())) - } - - fn journal_origins( - &self, - ranges: &[JournalSourceRange], - ) -> Result, SubjektivHostError> { - if ranges.is_empty() { - return Err(SubjektivHostError::InvalidArgument( - "empty material selection".into(), - )); - } - ranges - .iter() - .map(|range| { - let reader = self.source_reader(&range.source)?; - reader - .descriptors() - .find(|descriptor| { - source_descriptor(&range.source.session_id, descriptor) == range.source - }) - .map(|descriptor| SourceOrigin { - pointer: range.source.pointer.clone(), - origin: descriptor.origin.clone(), - }) - .ok_or_else(|| { - SubjektivHostError::InvalidArgument( - "source provenance/identity changed".into(), - ) - }) - }) - .collect() - } - - fn journal_source_window( - &self, - source: &JournalSource, - offset: u64, - max_bytes: usize, - ) -> Result { - let reader = self.source_reader(source)?; - let descriptor = reader.descriptors().find(|descriptor| source_descriptor(&source.session_id, descriptor) == *source) - .ok_or_else(|| SubjektivHostError::InvalidArgument("canonical source identity/order/digest changed or is outside the authorized Session".into()))?; - reader - .read_content(descriptor, offset, max_bytes) - .map(|window| window.content) - .map_err(|error| SubjektivHostError::Unavailable(error.to_string())) - } - - fn journal_source_material( - &self, - source: &JournalSource, - start: u64, - limit: usize, - ) -> JournalResult { - let text = self - .journal_source_window(source, start, limit.max(4)) - .map_err(|error| JournalError::Conflict(error.to_string()))?; - Ok(JournalMaterial { - range: JournalSourceRange { - source: source.clone(), - bytes: JournalByteRange { - start, - end: start + text.len() as u64, - }, - }, - text, + ) + .map_err(|e| SubjektivHostError::Unavailable(e.to_string())) }) } - fn select_journal_materials( + fn build_journal_input( &self, context: &JournalHostContext, id: &str, - ) -> Result, SubjektivHostError> { - subjektiv::journal::recording::select_materials_bounded( - &self.catalog.store, - context, - id, - INPUT_BUDGET, - |source, start, limit| self.journal_source_material(source, start, limit), - ) + saved: Option<&CaptureDto>, + ) -> Result { + let session = self + .catalog + .store + .read_journal(context, id) + .map_err(journal_error)? + .source_session_id; + JournalInput::build(&self.catalog.store, context, id, |order, offset, budget| { + let capture = match saved { + Some(capture) => capture.clone(), + None => { + let capture = self + .with_recording_boundary(&session, |sessions, session_id, active, guard| { + session_store::capture::capture_recording_slice_fs_guarded( + sessions, + session_id, + active, + session_store::public_material::PublicMaterialLimits::default(), + RecordingPosition { + canonical_order: order, + byte_offset: offset, + }, + MAX_JOURNAL_RANGES, + budget, + guard, + ) + .map(|slice| slice.capture) + .map_err(|e| SubjektivHostError::Unavailable(e.to_string())) + }) + .map_err(|e| JournalError::Conflict(e.to_string()))?; + // Admission and capture are separate saves. An append between them + // may be in replay but has not yet been accepted for this Journal. + return self.accepted_capture_prefix(context, id, capture); + } + }; + let end = capture + .entries + .last() + .map(|entry| { + capture + .recording_text(entry.order) + .map(|(descriptor, start, text)| { + if start + text.len() as u64 == descriptor.total_bytes { + RecordingPosition { + canonical_order: entry.order + 1, + byte_offset: 0, + } + } else { + RecordingPosition { + canonical_order: entry.order, + byte_offset: start + text.len() as u64, + } + } + }) + }) + .transpose() + .map_err(|e| JournalError::Conflict(e.to_string()))?; + if end.is_none_or(|end| (order, offset) >= (end.canonical_order, end.byte_offset)) { + return Ok(CaptureDto { + entries: vec![], + ..capture + }); + } + capture + .recording_slice( + RecordingPosition { + canonical_order: order, + byte_offset: offset, + }, + MAX_JOURNAL_RANGES, + budget, + ) + .map(|slice| slice.capture) + .map_err(|e| JournalError::Conflict(e.to_string())) + }) .map_err(journal_error) } - fn journal_grant( + fn accepted_capture_prefix( + &self, + context: &JournalHostContext, + id: &str, + mut capture: CaptureDto, + ) -> JournalResult { + let mut selected = Vec::new(); + for mut entry in capture.entries { + let status = match self + .catalog + .store + .read_journal_source(context, id, entry.order) + { + Ok(status) => status, + Err(JournalError::NotFound(_)) => break, + Err(error) => return Err(error), + }; + let session_store::capture::CaptureBody::CanonicalPublic { + descriptor, + byte_offset, + text, + .. + } = &mut entry.body + else { + return Err(JournalError::InvalidInput("non-recording capture".into())); + }; + if source_descriptor(&capture.session_id, descriptor) != status.source { + return Err(JournalError::Conflict( + "captured source differs from admitted identity".into(), + )); + } + let Some(accepted) = status + .accepted + .iter() + .find(|range| range.start <= *byte_offset && *byte_offset < range.end) + else { + break; + }; + let end = status + .unavailable + .iter() + .filter(|range| range.end > *byte_offset) + .fold(accepted.end, |end, range| { + end.min(range.start.max(*byte_offset)) + }); + let mut length = text.len().min(end.saturating_sub(*byte_offset) as usize); + while !text.is_char_boundary(length) { + length -= 1; + } + if length == 0 { + break; + } + text.truncate(length); + let partial = *byte_offset + (text.len() as u64) < descriptor.total_bytes; + selected.push(entry); + if partial { + break; + } + } + capture.entries = selected; + Ok(capture) + } + + fn journal_grant_scope( &self, snapshot: &JobSnapshot, ) -> Result { @@ -1097,7 +1358,6 @@ impl LocalSubjectHost { || snapshot.request.purpose != PURPOSE || snapshot.request.serialization_key.as_deref() != Some(&format!("subjektiv-journal:{}", grant.journal_id)) - || snapshot.request.input != grant.input() { return Err(SubjektivHostError::InvalidArgument( "Job input/grant does not match this Journal capability".into(), @@ -1135,12 +1395,33 @@ impl LocalSubjectHost { Ok(grant) } + fn journal_grant( + &self, + snapshot: &JobSnapshot, + ) -> Result { + snapshot + .load_executable_request() + .map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?; + let grant = self.journal_grant_scope(snapshot)?; + let input = JournalInput::decode(&snapshot.request.input).map_err(journal_error)?; + if input.subject_id != self.binding.subject_id + || input.journal_id != grant.journal_id + || input.capture.session_id != grant.session_id + || input.selected_ranges != grant.ranges + { + return Err(SubjektivHostError::InvalidArgument( + "saved Journal body identity/ranges differ from grant".into(), + )); + } + Ok(grant) + } + fn complete_journal_job( &self, jobs: &crate::jobs::StandaloneJobs, snapshot: &JobSnapshot, ) -> Result<(), SubjektivHostError> { - let grant = self.journal_grant(snapshot)?; + let grant = self.journal_grant_scope(snapshot)?; let context = authority(self, &grant.session_id)?; let completion: JournalJobCompletion = serde_json::from_value(snapshot.result.clone().ok_or_else(|| { @@ -1174,6 +1455,12 @@ impl LocalSubjectHost { } jobs.acknowledge(&snapshot.request.job_id) .map_err(service_error)?; + jobs.reclaim_input( + &snapshot.request.job_id, + &snapshot.input_digest, + JobInputRetirementReason::Completed, + ) + .map_err(service_error)?; if let Err(error) = self.prune_retained_workers() { // Pruning is independent of a successful checkpoint/result. Keep the originals // and continue draining other saved prefixes; a retention error is not model failure. @@ -1192,47 +1479,18 @@ pub(super) fn grant( .journal_grant(snapshot) .map_err(|error| error.to_string())?; let context = authority(&host, &grant.session_id).map_err(|error| error.to_string())?; - // Re-fetch selected identities before opening mutation authority. Only descriptors are - // retained in Job storage; the model must fetch actual source bytes with bounded tools. - for range in &grant.ranges { - let length = usize::try_from( - range - .bytes - .end - .checked_sub(range.bytes.start) - .ok_or("invalid source range")?, - ) - .map_err(|_| "source range overflow")?; - if length > INPUT_BUDGET { - return Err("source range exceeds frozen input budget".into()); - } - let text = host - .journal_source_window(&range.source, range.bytes.start, length.max(4)) - .map_err(|error| error.to_string())?; - if text.len() != length { - return Err("frozen Journal source byte range changed".into()); - } - } - if host - .journal_origins(&grant.ranges) - .map_err(|error| error.to_string())? - != grant.origins - { - return Err( - "frozen Journal provenance does not match the authorized original Session".into(), - ); - } + let input = JournalInput::decode(&snapshot.request.input).map_err(|e| e.to_string())?; + let input_digest = snapshot.input_digest.clone(); let generation = host .catalog .store - .prepare_journal_generation( + .prepare_journal_input( &context, - &grant.journal_id, + &input, JobAttemptBinding { job_id: snapshot.request.job_id.clone(), attempt_id: snapshot.attempt.attempt_id.clone(), }, - grant.ranges.clone(), ) .map_err(|error| error.to_string())?; let job_context = JournalJobContext::validated( @@ -1242,12 +1500,17 @@ pub(super) fn grant( generation.job.clone(), ) .map_err(|error| error.to_string())?; + input + .validate_generation(&generation) + .map_err(|e| e.to_string())?; let adapter = Arc::new(LocalJournalHost { host, grant, context, job_context, + #[cfg(test)] generation, + input_digest, fence, }); Ok(worker::job::JobFeatureGrant { @@ -1263,7 +1526,9 @@ struct LocalJournalHost { grant: LocalJournalGrant, context: JournalHostContext, job_context: JournalJobContext, + #[cfg(test)] generation: JournalGeneration, + input_digest: String, fence: Arc, } impl LocalJournalHost { @@ -1296,18 +1561,20 @@ impl JournalJobHost for LocalJournalHost { request: JournalJobRequest, ) -> Result { let context = self.live()?; - subjektiv::journal::service::execute_bounded( + // The generic runner has supplied this exact immutable input before it can + // invoke this bound tool. Preparation alone is not evidence of delivery. + subjektiv::journal::service::attest_input_delivery( &self.host.catalog.store, &context, + &self.input_digest, + &self.grant.ranges, + ) + .map_err(journal_error)?; + subjektiv::journal::service::execute_snapshot( + &self.host.catalog.store, + &context, + &self.input_digest, request, - |source, start, limit| { - if source.session_id != self.generation.initial_journal.source_session_id { - return Err(JournalError::PermissionDenied( - "foreign original Session".into(), - )); - } - self.host.journal_source_material(source, start, limit) - }, ) .map_err(journal_error) } diff --git a/crates/standalone/src/subjektiv/journal/tests.rs b/crates/standalone/src/subjektiv/journal/tests.rs index c488b4879..0e4d76f5f 100644 --- a/crates/standalone/src/subjektiv/journal/tests.rs +++ b/crates/standalone/src/subjektiv/journal/tests.rs @@ -1,6 +1,6 @@ //! Integration of standalone attribution, original FsStore material, shared Journal storage //! and the local Job adapter. Only the outer model/execution lease is replaced in these cases. -//! The generic Job runner is exercised separately by the standalone Jobs integration suite. +//! The scripted model also exercises saved direct input through the actual generic Job runner. use super::*; use crate::store::{StaleLeasePolicy, StandaloneWorkerAllocation, StandaloneWorkerStore}; @@ -144,8 +144,8 @@ impl Fixture { .unwrap() .contains(sentinel) ); - // Include SQLite/WAL/journal files: immutable request/grant storage must never hold - // original text, even in freed pages left behind by explicit source erasure. + // Include SQLite/WAL/journal files: erasure must scrub retired original bodies, + // including any freed pages or journal left by the immutable request store. for entry in fs::read_dir(self.connection.jobs_path.parent().unwrap()).unwrap() { let entry = entry.unwrap(); if entry.file_type().unwrap().is_file() { @@ -194,9 +194,9 @@ impl Fixture { context, job_context, generation, + input_digest: snapshot.input_digest.clone(), fence: fence.clone(), }; - adapter.read_all_sources(); (adapter, fence, snapshot) } } @@ -220,37 +220,6 @@ fn message(id: &str, text: &str) -> session_store::LoggedHistoryEntry { } } impl LocalJournalHost { - fn read_all_sources(&self) { - for range in &self.generation.selected_ranges { - let mut offset = range.bytes.start as usize; - let mut content_token = None; - loop { - let response = self - .execute(JournalJobRequest::ReadText { - read: JournalTextRead { - target: JournalTextTarget::Source { - pointer: range.source.pointer.clone(), - }, - offset, - limit: MAX_JOURNAL_READ_BYTES, - content_token: content_token.clone(), - }, - }) - .unwrap(); - let JournalJobResponse::Text(text) = response else { - panic!() - }; - match text.next_byte { - Some(next) => { - assert!(next > offset); - offset = next; - content_token = Some(text.content_token); - } - None => break, - } - } - } - } fn state(&self) -> Result { match self.execute(JournalJobRequest::ReadState)? { JournalJobResponse::State(state) => Ok(state), @@ -362,8 +331,8 @@ fn duplicate_admission_reserves_one_job_and_excludes_private_prompt() { assert_eq!(jobs.pending().unwrap().len(), 1); let input = jobs.get(&id).unwrap().request.input.to_string(); assert!( - !input.contains("short request"), - "Job storage retains descriptors, never a second copy of original text" + input.contains("short request"), + "Job storage supplies the canonical body directly" ); assert!(!input.contains("PRIVATE PROMPT")); let foreign = SubjektivHostContext { @@ -498,14 +467,16 @@ fn generation_input_stays_frozen_and_next_prefix_includes_last_input() { &adapter.generation.job, ) .unwrap(); - let materials = fixture + let next = fixture .host() - .select_journal_materials(&adapter.context, &adapter.grant.journal_id) + .build_journal_input(&adapter.context, &adapter.grant.journal_id, None) .unwrap(); - assert_eq!(materials[0].range.source.pointer.entry_ref, "Elast"); + assert_eq!(next.selected_ranges[0].source.pointer.entry_ref, "Elast"); assert!( - materials[0] - .text + next.capture + .recording_text(next.selected_ranges[0].source.order) + .unwrap() + .2 .contains("last input while generation runs") ); } @@ -547,7 +518,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() { let (adapter, _, _) = fixture.adapter(); let range = &adapter.grant.ranges[0]; assert!(range.bytes.end < range.source.total_bytes); - assert!(range.bytes.end - range.bytes.start < INPUT_BUDGET as u64); + assert!(range.bytes.end - range.bytes.start < job::MAX_JOB_INPUT_BYTES as u64); let snapshot = fixture .host() .jobs() @@ -557,7 +528,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() { .pop() .unwrap(); assert!(serde_json::to_vec(&snapshot.request.input).unwrap().len() < 128 * 1024); - assert!(!snapshot.request.input.to_string().contains("漢字")); + assert!(snapshot.request.input.to_string().contains("漢字")); assert!(adapter.finish().is_err()); } @@ -628,43 +599,35 @@ impl JournalScript { ) } else if let Some(state) = result("state") { let state = &state["value"]; - for (index, range) in state["selected_ranges"] - .as_array() - .unwrap() + let input: serde_json::Value = request + .items .iter() - .enumerate() - { - let mut offset = range["bytes"]["start"].as_u64().unwrap(); - let mut token = Value::Null; - loop { - let id = format!("source-{index}-{offset}"); - if let Some(page) = result(&id) { - let page = &page["value"]; - let key = format!("{}:{id}", state["generation_id"].as_str().unwrap()); - let text = page["text"].as_str().unwrap().to_owned(); - if let Some(previous) = - self.source_reads.lock().unwrap().insert(key, text.clone()) - { - assert_eq!( - previous, text, - "frozen source read changed within a generation" - ); - } - if let Some(next) = page["next_byte"].as_u64() { - assert!(next > offset); - offset = next; - token = page["content_token"].clone(); - } else { - break; - } - } else { - let input = json!({"target":{"target":"source","pointer":range["source"]["pointer"]},"offset":offset,"limit":MAX_JOURNAL_READ_BYTES,"content_token":token}); - return vec![ - Event::tool_use_start(0, &id, "ReadJournalText"), - Event::tool_input_delta(0, input.to_string()), - Event::tool_use_stop(0), - ]; - } + .filter_map(Item::as_text) + .find_map(|text| { + text.lines() + .find_map(|line| line.strip_prefix("input_json: ")) + }) + .map(|json| serde_json::from_str(json).unwrap()) + .expect("direct saved Journal input"); + let input = JournalInput::decode(&input).unwrap(); + assert_eq!( + serde_json::to_value(&input.selected_ranges).unwrap(), + state["selected_ranges"] + ); + for entry in &input.capture.entries { + let (_, offset, text) = input.capture.recording_text(entry.order).unwrap(); + let key = format!( + "{}:{}:{offset}", + state["generation_id"].as_str().unwrap(), + entry.order + ); + if let Some(previous) = self + .source_reads + .lock() + .unwrap() + .insert(key, text.to_owned()) + { + assert_eq!(previous, text, "saved source changed within generation"); } } let (name, action) = if state["current_open"].is_null() { @@ -1166,6 +1129,11 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c .items; assert_eq!(entries.len(), 1); let saved_entry = entries[0].id.clone(); + let original_input = jobs.get(&id).unwrap().request.input; + fixture.append( + "newer-than-snapshot", + "NEW INPUT MUST NOT MIX INTO RESUMED SNAPSHOT", + ); jobs.shutdown().await.unwrap(); fixture.connection.close().unwrap(); let mut reopened = StandaloneSubjects::open_existing(&state) @@ -1176,8 +1144,29 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c .unwrap(); reopened.host.bind_jobs(jobs.clone()).unwrap(); assert_eq!(jobs.get(&id).unwrap().state, JobState::Unknown); - // Domain reconciliation inspects the now-fenced prior attempt and preserves its receipts. + // Deny every original-source read while preparing and running the replacement. + let source_path = fixture + .sessions + .root_dir() + .join(&fixture.context.session_id); + let denied = fixture + ._temp + .path() + .join("temporarily-unavailable-original"); + fs::rename(&source_path, &denied).unwrap(); let (_, replacement) = reopened.host.next_journal_job().unwrap().unwrap(); + let replacement_input = jobs.get(&replacement).unwrap().request.input; + assert_eq!(replacement_input["capture"], original_input["capture"]); + assert!( + !replacement_input + .to_string() + .contains("NEW INPUT MUST NOT MIX") + ); + assert_eq!(jobs.get(&id).unwrap().request.input, original_input); + assert_eq!( + jobs.get(&id).unwrap().input_disposition, + JobInputDisposition::Available + ); assert_ne!( replacement, id, "unknown external execution is not blindly replayed" @@ -1205,8 +1194,26 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c .items; assert_eq!(entries.len(), 1); assert_eq!(entries[0].id, saved_entry); + fs::rename(&denied, &source_path).unwrap(); + let journal = reopened + .host + .catalog + .store + .read_journal(&context, &journal_id) + .unwrap(); + let original = JournalInput::decode(&original_input).unwrap(); + let end = original.selected_ranges.last().unwrap(); + assert_eq!( + (journal.process_order, journal.process_byte), + if end.bytes.end == end.source.total_bytes { + (end.source.order + 1, 0) + } else { + (end.source.order, end.bytes.end) + } + ); + assert!(reopened.host.admit_journal(&fixture.context).unwrap()); assert!( - reopened + !reopened .host .session_fully_reflected(&fixture.context.session_id) .unwrap() @@ -1302,9 +1309,9 @@ fn missing_original_material_blocks_worker_removal_without_erasing_state() { } #[test] -fn frozen_original_remains_readable_after_worker_switches_its_active_session() { +fn frozen_input_survives_active_session_change_and_source_tool_remains_refused() { let fixture = Fixture::new("old Session original"); - let (adapter, _, _) = fixture.adapter(); + let (adapter, _, snapshot) = fixture.adapter(); let workers = StandaloneWorkerStore::open_existing(&fixture.host().catalog.state_root).unwrap(); let worker = fixture._allocation.as_ref().unwrap().worker_id(); let record = workers.load(worker).unwrap(); @@ -1318,22 +1325,30 @@ fn frozen_original_remains_readable_after_worker_switches_its_active_session() { )), )) .unwrap(); - let response = adapter - .execute(JournalJobRequest::ReadText { - read: JournalTextRead { - target: JournalTextTarget::Source { - pointer: adapter.generation.selected_ranges[0].source.pointer.clone(), - }, - offset: 0, - limit: MAX_JOURNAL_READ_BYTES, - content_token: None, - }, - }) - .unwrap(); - let JournalJobResponse::Text(page) = response else { - panic!() - }; - assert!(page.text.contains("old Session original")); + assert!( + adapter + .execute(JournalJobRequest::ReadText { + read: JournalTextRead { + target: JournalTextTarget::Source { + pointer: adapter.generation.selected_ranges[0].source.pointer.clone() + }, + offset: 0, + limit: MAX_JOURNAL_READ_BYTES, + content_token: None + } + }) + .is_err(), + "original Source lookup is explicitly unavailable" + ); + let input = JournalInput::decode(&snapshot.request.input).unwrap(); + assert!( + input + .capture + .recording_text(0) + .unwrap() + .2 + .contains("old Session original") + ); assert!( fixture .host() @@ -1512,14 +1527,18 @@ fn relocated_erasure_fences_old_and_retained_readers_publishers_and_late_pins() const SENTINEL: &str = "original-plaintext-sentinel-must-never-enter-immutable-job-storage"; let mut fixture = Fixture::new(SENTINEL); let (_, _, snapshot) = fixture.adapter(); - fixture.assert_job_storage_excludes(&snapshot, SENTINEL); + assert!(snapshot.request.input.to_string().contains(SENTINEL)); let state = fixture.host().catalog.state_root.clone(); let binding = fixture.connection.binding.clone(); let session: session_store::SessionId = fixture.context.session_id.parse().unwrap(); - let old_reader = fixture - .host() - .material_reader(&fixture.context.session_id) - .unwrap(); + let old_reader = SessionPublicMaterialReader::open_fs_with_retention( + fixture.sessions.root_dir(), + session, + fixture.segment, + PublicMaterialLimits::default(), + crate::store::session_retention(&state), + ) + .unwrap(); let descriptor = old_reader.descriptors().next().unwrap().clone(); let (worker, workers) = fixture.stop_worker(); assert!(workers.delete(worker).is_err()); @@ -1759,14 +1778,12 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job( let snapshot = jobs.get(&next).unwrap(); let grant = fixture.host().journal_grant(&snapshot).unwrap(); assert_eq!(grant.ranges.len(), 1); - assert_eq!( - grant.origins[0].origin, - PublicMaterialOrigin::HostRunBoundary - ); - let content = fixture - .host() - .journal_source_window(&grant.ranges[0].source, 0, 4096) + let input = JournalInput::decode(&snapshot.request.input).unwrap(); + let (descriptor, _, content) = input + .capture + .recording_text(grant.ranges[0].source.order) .unwrap(); + assert_eq!(descriptor.origin, PublicMaterialOrigin::HostRunBoundary); let value: serde_json::Value = serde_json::from_str(&content).unwrap(); assert_eq!(value["kind"], "run_boundary"); assert_eq!(value["boundary"]["kind"], "errored"); @@ -1791,3 +1808,621 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job( assert!(fixture.host().next_journal_job().unwrap().is_none()); jobs.shutdown().await.unwrap(); } + +/// Original FsStore -> bounded common capture -> persisted Job -> actual generic runner -> +/// sequential Journal tools. Only the LLM transport is scripted. Source files are deliberately +/// unavailable after persistence, and input bodies are inspected in the provider request. +#[tokio::test] +async fn saved_body_jobs_process_over_a_hundred_entries_and_utf8_without_live_source_reads() { + let fixture = Fixture::new("oldest original"); + for index in 0..115 { + fixture.append( + &format!("entry-{index}"), + &format!("public original {index}"), + ); + } + fixture.append("long-utf8", &"漢字🙂\n\"".repeat(24_000)); + fixture.host().admit_journal(&fixture.context).unwrap(); + let expected = fixture + .host() + .recording_capture(&fixture.context.session_id) + .unwrap(); + let source_path = fixture + .sessions + .root_dir() + .join(&fixture.context.session_id); + let denied = fixture._temp.path().join("denied-original-source"); + let mut seen: std::collections::BTreeMap = Default::default(); + let mut jobs_processed = 0; + loop { + let Some((jobs, id)) = fixture.host().next_journal_job().unwrap() else { + break; + }; + let saved = jobs.get(&id).unwrap(); + let input = JournalInput::decode(&saved.request.input).unwrap(); + assert!( + serde_json::to_vec(&saved.request.input).unwrap().len() <= job::MAX_JOB_INPUT_BYTES + ); + assert!(input.capture.entries.len() <= MAX_JOURNAL_RANGES); + for entry in &input.capture.entries { + let (_, offset, text) = input.capture.recording_text(entry.order).unwrap(); + let previous = seen.entry(entry.order).or_default(); + assert_eq!( + previous.len() as u64, + offset, + "no gap, preview or duplicate prefix" + ); + previous.push_str(text); + } + // Deny all catalogs/reads, including grant preparation. The model still receives + // exactly the persisted body, and the Host must not follow the active pointer. + fs::rename(&source_path, &denied).unwrap(); + let model = JournalScript::default(); + jobs.start_with_model_client(&id, model.clone()) + .await + .unwrap(); + let result = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id)) + .await + .unwrap() + .unwrap(); + assert_eq!(result.state, JobState::Completed); + assert!( + !model.source_reads.lock().unwrap().is_empty(), + "provider saw saved source bodies" + ); + fixture.host().complete_journal_job(&jobs, &result).unwrap(); + assert_eq!( + jobs.get(&id).unwrap().input_disposition, + JobInputDisposition::Reclaimed + ); + fs::rename(&denied, &source_path).unwrap(); + jobs_processed += 1; + assert!(jobs_processed < 20, "bounded prefix drain must progress"); + } + assert!( + jobs_processed > 2, + "entry count and long text both require continuation" + ); + assert_eq!(seen.len(), expected.entries.len()); + for entry in &expected.entries { + assert_eq!( + seen[&entry.order], + expected.recording_text(entry.order).unwrap().2 + ); + } + fixture.host().jobs().unwrap().shutdown().await.unwrap(); +} + +#[tokio::test] +async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful_records() { + let fixture = Fixture::new("original descriptor-only input"); + let (adapter, _, modern) = fixture.adapter(); + let jobs = fixture.host().jobs().unwrap(); + fixture + .host() + .catalog + .store + .close_journal_generation( + &adapter.context, + &adapter.generation.id, + &adapter.generation.job, + ) + .unwrap(); + jobs.cancel(&modern.request.job_id).unwrap(); + let input = JournalInput::decode(&modern.request.input).unwrap(); + let origins: Vec<_> = input + .capture + .entries + .iter() + .map(|entry| { + let (descriptor, _, _) = input.capture.recording_text(entry.order).unwrap(); + SourceOrigin { + pointer: source_descriptor(&fixture.context.session_id, descriptor).pointer, + origin: descriptor.origin.clone(), + } + }) + .collect(); + let mut old_grant = adapter.grant.clone(); + old_grant.origins = origins; + old_grant.writer_binding_id = None; + let mut request = modern.request.clone(); + request.job_id = format!("legacy-descriptor:{}", uuid::Uuid::now_v7()); + request.input = + serde_json::json!({"selected_ranges":old_grant.ranges,"origins":old_grant.origins}); + let old = jobs + .request_granted(request, serde_json::to_value(&old_grant).unwrap()) + .unwrap(); + assert!( + fixture.host().journal_grant(&old).is_err(), + "descriptor jobs must not execute as snapshots" + ); + let generation = fixture + .host() + .catalog + .store + .prepare_journal_generation( + &adapter.context, + &old_grant.journal_id, + JobAttemptBinding { + job_id: old.request.job_id.clone(), + attempt_id: old.attempt.attempt_id.clone(), + }, + old_grant.ranges.clone(), + ) + .unwrap(); + let context = JournalJobContext::validated( + &fixture.host().catalog.store, + &adapter.context, + &generation.id, + generation.job.clone(), + ) + .unwrap(); + let saved = fixture + .host() + .catalog + .store + .execute_journal( + &context, + JournalOperation { + request_id: "legacy-record-success-response-lost".into(), + expected_state: generation.initial_journal.state_token.clone(), + action: JournalAction::Create { + title: "already saved".into(), + body: "legacy successful operation".into(), + ongoing: "unfinished".into(), + state: JournalEntryState::Open, + source_end: None, + correction_refs: vec![], + }, + covered_ranges: generation.selected_ranges.clone(), + }, + ) + .unwrap(); + fixture.append( + "new-not-selected", + "LATEST SOURCE MUST WAIT UNTIL OLD INPUT FINISHES", + ); + fixture.host().admit_journal(&fixture.context).unwrap(); + let (_, next) = fixture.host().next_journal_job().unwrap().unwrap(); + assert_ne!(next, old.request.job_id); + let replacement = jobs.get(&next).unwrap(); + let replacement_input = JournalInput::decode(&replacement.request.input).unwrap(); + assert!( + !replacement + .request + .input + .to_string() + .contains("LATEST SOURCE MUST WAIT") + ); + assert_eq!(replacement_input.selected_ranges, old_grant.ranges); + let retired = jobs.get(&old.request.job_id).unwrap(); + assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated); + assert_eq!(retired.input_digest, old.input_digest); + assert_eq!(retired.intent_fingerprint, old.intent_fingerprint); + assert!(retired.request.input.is_null()); + assert!( + fixture + .host() + .catalog + .store + .execute_journal( + &context, + JournalOperation { + request_id: "legacy-delayed".into(), + expected_state: saved.state_token, + action: JournalAction::ConfirmUnchanged, + covered_ranges: old_grant.ranges, + } + ) + .is_err() + ); + jobs.start_with_model_client(&next, JournalScript::default()) + .await + .unwrap(); + let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&next)) + .await + .unwrap() + .unwrap(); + assert_eq!(completed.state, JobState::Completed); + fixture + .host() + .complete_journal_job(&jobs, &completed) + .unwrap(); + let entries = fixture + .host() + .catalog + .store + .list_journal_entries(&adapter.context, &adapter.grant.journal_id, None, 32) + .unwrap(); + assert_eq!(entries.items.len(), 1); + assert_eq!(entries.items[0].id, saved.entry_id.unwrap()); + jobs.shutdown().await.unwrap(); +} + +#[test] +fn explicit_erase_retires_every_historical_body_across_job_history_pages() { + const SENTINEL: &str = "historical-input-erasure-sentinel-781"; + let mut fixture = Fixture::new("ordinary original"); + let (adapter, _, first) = fixture.adapter(); + let jobs = fixture.host().jobs().unwrap(); + for index in 0..105 { + let mut request = first.request.clone(); + request.job_id = format!("historical-{index:03}"); + request.purpose = "historical-other-domain-job".into(); + request.serialization_key = None; + request.input = serde_json::json!({"body":SENTINEL, "index":index}); + let saved = jobs + .request_granted( + request, + serde_json::json!({"session_id":fixture.context.session_id}), + ) + .unwrap(); + jobs.cancel(&saved.request.job_id).unwrap(); + } + let state = fixture.host().catalog.state_root.clone(); + let subject = fixture.connection.binding.subject_id.clone(); + fixture.connection.close().unwrap(); + StandaloneSubjects::open_existing(&state) + .unwrap() + .erase_session_completely(&subject, &fixture.context.session_id, "erase-all-history") + .unwrap(); + let jobs = crate::jobs::StandaloneJobs::open( + &fixture.connection.jobs_path, + fixture._temp.path().into(), + ) + .unwrap(); + let mut after = None; + let mut count = 0; + loop { + let page = jobs + .list_session_snapshots(&fixture.context.session_id, after.as_deref(), 100) + .unwrap(); + if page.is_empty() { + break; + } + for saved in &page { + assert_eq!(saved.input_disposition, JobInputDisposition::Erased); + assert!(saved.request.input.is_null()); + assert!(saved.load_executable_request().is_err()); + } + count += page.len(); + after = page.last().map(|job| job.request.job_id.clone()); + } + assert_eq!(count, 106); + assert!( + adapter.execute(JournalJobRequest::ReadState).is_err(), + "prepared local adapter is invalidated" + ); + fixture.assert_job_storage_excludes(&jobs.get(&first.request.job_id).unwrap(), SENTINEL); +} + +#[test] +fn saved_snapshot_missing_tampered_foreign_or_changed_initial_state_is_not_recaptured() { + let fixture = Fixture::new("saved immutable original"); + fixture.host().admit_journal(&fixture.context).unwrap(); + let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap(); + let saved = jobs.get(&id).unwrap(); + for changed in [serde_json::Value::Null, serde_json::json!({"capture":{}})] { + let mut malformed = saved.clone(); + malformed.request.input = changed; + assert!(fixture.host().journal_grant(&malformed).is_err()); + } + let original = JournalInput::decode(&saved.request.input).unwrap(); + for field in ["subject_id", "journal_id", "initial_state_token"] { + let mut changed = saved.request.clone(); + changed.job_id = format!("changed-{field}"); + changed.serialization_key = None; + changed.input[field] = serde_json::json!("foreign-or-altered"); + // Even a separately reserved immutable Job may not adopt the original capability. + let changed = jobs + .request_granted(changed, saved.domain_grant.clone().unwrap()) + .unwrap(); + assert!( + grant( + fixture.host().clone(), + &changed, + Arc::new(TestFence(AtomicBool::new(true))) + ) + .is_err() + ); + } + let mut generation = fixture + .host() + .catalog + .store + .prepare_journal_generation( + &authority(fixture.host(), &fixture.context.session_id).unwrap(), + &original.journal_id, + JobAttemptBinding { + job_id: id, + attempt_id: saved.attempt.attempt_id.clone(), + }, + original.selected_ranges.clone(), + ) + .unwrap(); + generation.initial_journal.state_token = "changed-after-snapshot".into(); + assert!(original.validate_generation(&generation).is_err()); + assert_eq!( + saved.request.input, + jobs.get(&saved.request.job_id).unwrap().request.input + ); +} + +#[tokio::test] +async fn append_between_admission_and_body_capture_does_not_publish_unaccepted_input() { + let fixture = Fixture::new("accepted oldest prefix"); + fixture.host().admit_journal(&fixture.context).unwrap(); + fixture.append("not-yet-admitted", "newly saved but not accepted"); + let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap(); + let input = jobs.get(&id).unwrap().request.input; + assert!(input.to_string().contains("accepted oldest prefix")); + assert!(!input.to_string().contains("newly saved but not accepted")); + jobs.start_with_model_client(&id, JournalScript::default()) + .await + .unwrap(); + let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id)) + .await + .unwrap() + .unwrap(); + assert_eq!(completed.state, JobState::Completed); + fixture + .host() + .complete_journal_job(&jobs, &completed) + .unwrap(); + fixture.host().admit_journal(&fixture.context).unwrap(); + let (_, next) = fixture.host().next_journal_job().unwrap().unwrap(); + let remaining = jobs.get(&next).unwrap().request.input; + assert!( + remaining + .to_string() + .contains("newly saved but not accepted") + ); + assert!(!remaining.to_string().contains("accepted oldest prefix")); + jobs.shutdown().await.unwrap(); +} + +/// Generic Job preparation must not attest delivery. The saved body is supplied by the +/// runner before a bound Journal tool may execute; no model-provided digest is trusted. +#[tokio::test] +async fn actual_bound_tool_attests_delivery_but_prepared_blocked_job_does_not() { + let fixture = Fixture::new("body delivered through generic runner"); + fixture.host().admit_journal(&fixture.context).unwrap(); + let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap(); + let saved = jobs.get(&id).unwrap(); + let model = JournalScript { + gate_first: true, + pause_after_record: true, + ..Default::default() + }; + jobs.start_with_model_client(&id, model.clone()) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified()) + .await + .unwrap(); + let host_context = authority(fixture.host(), &fixture.context.session_id).unwrap(); + let journal_id = fixture + .host() + .catalog + .store + .journal_for_session(&host_context) + .unwrap() + .unwrap(); + let generation = fixture + .host() + .catalog + .store + .active_journal_generation(&host_context, &journal_id) + .unwrap() + .unwrap(); + let context = JournalJobContext::validated( + &fixture.host().catalog.store, + &host_context, + &generation.id, + generation.job.clone(), + ) + .unwrap(); + let undelivered = subjektiv::journal::service::execute_snapshot( + &fixture.host().catalog.store, + &context, + &saved.input_digest, + JournalJobRequest::ReadState, + ) + .unwrap_err(); + assert!( + undelivered.to_string().contains("delivery"), + "prepared grant must not satisfy delivery gate: {undelivered}" + ); + let before = fixture + .host() + .catalog + .store + .read_journal(&host_context, &journal_id) + .unwrap(); + assert_eq!((before.process_order, before.process_byte), (0, 0)); + model.release.notify_one(); + tokio::time::timeout( + std::time::Duration::from_secs(20), + model.record_saved.notified(), + ) + .await + .unwrap(); + assert!( + subjektiv::journal::service::execute_snapshot( + &fixture.host().catalog.store, + &context, + &saved.input_digest, + JournalJobRequest::ReadState + ) + .is_ok(), + "actual bound tool execution attests exact saved input" + ); + model.release.notify_one(); + let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id)) + .await + .unwrap() + .unwrap(); + assert_eq!(completed.state, JobState::Completed); + fixture + .host() + .complete_journal_job(&jobs, &completed) + .unwrap(); + jobs.shutdown().await.unwrap(); +} + +#[derive(Clone, Default)] +struct BlockedJournalRequest { + entered: Arc, + dropped: Arc, + request_dropped: Arc, +} +struct OwnedBlockedJournalRequest { + // A local request copy, including the saved input, must be dropped before erasure + // can acquire the lease. This does not claim remote/provider bytes are retractable. + _request: agen::llm_client::Request, + model: BlockedJournalRequest, +} +impl Drop for OwnedBlockedJournalRequest { + fn drop(&mut self) { + self.model.request_dropped.store(true, Ordering::Release); + self.model.dropped.notify_one(); + } +} +#[async_trait::async_trait] +impl agen::llm_client::LlmClient for BlockedJournalRequest { + async fn stream( + &self, + request: agen::llm_client::Request, + ) -> Result< + std::pin::Pin< + Box< + dyn futures::Stream< + Item = Result< + agen::llm_client::event::Event, + agen::llm_client::error::ClientError, + >, + > + Send, + >, + >, + agen::llm_client::error::ClientError, + > { + let owned = OwnedBlockedJournalRequest { + _request: request, + model: self.clone(), + }; + assert!( + owned + ._request + .items + .iter() + .filter_map(agen::llm_client::types::Item::as_text) + .any(|text| text.contains("local blocked source body")) + ); + self.entered.notify_one(); + std::future::pending::<()>().await; + drop(owned); + unreachable!() + } + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } +} + +/// Real generic Jobs and a retained public handle, with only the provider transport blocked. +/// Connection close must initiate cancellation and preserve the exclusive lease until local +/// request/history ownership is gone (or return an explicit incomplete barrier). +#[tokio::test] +async fn close_cancels_retained_local_job_before_erase_can_acquire_exclusive_lease() { + let mut fixture = Fixture::new("local blocked source body"); + fixture.host().admit_journal(&fixture.context).unwrap(); + let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap(); + let retained = jobs.clone(); + let model = BlockedJournalRequest::default(); + retained + .start_with_model_client(&id, model.clone()) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified()) + .await + .unwrap(); + let state = fixture.host().catalog.state_root.clone(); + let subject = fixture.connection.binding.subject_id.clone(); + let close_error = fixture.connection.close().unwrap_err(); + assert!(matches!( + close_error, + SubjectError::JobShutdown(crate::jobs::JobServiceError::ShutdownIncomplete) + )); + assert!(fixture.connection.lease.is_some()); + assert!(fixture.host().jobs.lock().unwrap().is_some()); + assert!(matches!( + retained.start(&id).await, + Err(crate::jobs::JobServiceError::Closed) + )); + let locally_dropped_before_erase = model.request_dropped.load(Ordering::Acquire); + let early_erasure = StandaloneSubjects::open_existing(&state) + .unwrap() + .erase_session_completely( + &subject, + &fixture.context.session_id, + "erase-blocked-local-run", + ); + let early_lease_was_retained = matches!(early_erasure, Err(SubjectError::Active(_))); + let teardown_from_close = model.request_dropped.load(Ordering::Acquire) + || tokio::time::timeout(std::time::Duration::from_secs(5), model.dropped.notified()) + .await + .is_ok(); + // Always tear down before asserting the observed violation, so a failed regression + // cannot leak the blocked model future into another test. + let _ = retained.shutdown().await; + assert!(model.request_dropped.load(Ordering::Acquire)); + fixture.connection.close().unwrap(); + assert!(fixture.connection.lease.is_none()); + assert!(fixture.host().jobs.lock().unwrap().is_none()); + // Cancellation is not completion/erasure. Reopening retains the same unknown + // attempt's immutable body until the explicit erasure transaction retires it. + let reopened = crate::jobs::StandaloneJobs::open( + &fixture.connection.jobs_path, + fixture._temp.path().into(), + ) + .unwrap(); + let unknown = reopened.get(&id).unwrap(); + assert_eq!(unknown.attempt.state, crate::jobs::JobAttemptState::Unknown); + assert_eq!(unknown.input_disposition, JobInputDisposition::Available); + assert!( + unknown + .request + .input + .to_string() + .contains("local blocked source body") + ); + reopened.shutdown().await.unwrap(); + StandaloneSubjects::open_existing(&state) + .unwrap() + .erase_session_completely( + &subject, + &fixture.context.session_id, + "erase-blocked-local-run", + ) + .unwrap(); + let erased_store = crate::jobs::StandaloneJobs::open( + &fixture.connection.jobs_path, + fixture._temp.path().into(), + ) + .unwrap(); + let erased = erased_store.get(&id).unwrap(); + assert_eq!(erased.input_disposition, JobInputDisposition::Erased); + fixture.assert_job_storage_excludes(&erased, "local blocked source body"); + erased_store.shutdown().await.unwrap(); + assert!( + locally_dropped_before_erase || early_lease_was_retained, + "close released exclusive lease while a retained generic Job still owned local input/history" + ); + assert!( + teardown_from_close, + "connection close did not initiate cancellation of the externally retained local Job" + ); + assert!( + retained.get(&id).is_err(), + "retained handles are closed after teardown" + ); +} diff --git a/crates/standalone/tests/host.rs b/crates/standalone/tests/host.rs index 4670410eb..64882761f 100644 --- a/crates/standalone/tests/host.rs +++ b/crates/standalone/tests/host.rs @@ -615,10 +615,14 @@ async fn headless_unix_client_uses_production_history_lease_and_local_authority( use tokio::io::AsyncWriteExt; let temp = tempfile::tempdir()?; - // Only the transport endpoint needs a short private pathname. Persistence remains the - // same production store exercised by the in-process Host tests above. - std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o700))?; - let endpoint_path = temp.path().join("operator.sock"); + // Shorten only the owned endpoint's components, preserving the harness TMPDIR + // even when it is checkout-local. No process-global cwd or external /tmp fallback. + let socket_dir = tempfile::Builder::new() + .prefix("") + .rand_bytes(4) + .tempdir()?; + std::fs::set_permissions(socket_dir.path(), std::fs::Permissions::from_mode(0o700))?; + let endpoint_path = socket_dir.path().join("s"); let endpoint = standalone::UnixEndpoint::bind(&endpoint_path)?; let state_dir = temp.path().join("state"); let launch = StandaloneLaunchConfig::new( diff --git a/crates/standalone/tests/subjektiv.rs b/crates/standalone/tests/subjektiv.rs index 0e587d22b..28edfb5be 100644 --- a/crates/standalone/tests/subjektiv.rs +++ b/crates/standalone/tests/subjektiv.rs @@ -479,38 +479,28 @@ fn journal_response(request: &Request) -> Vec { }), ); } - for (index, range) in state["selected_ranges"] - .as_array() - .unwrap() + let input: Value = request + .items .iter() - .enumerate() - { - let mut offset = range["bytes"]["start"].as_u64().unwrap(); - let mut token = Value::Null; - loop { - let id = format!("journal-source-{index}-{offset}"); - match result(&id) { - Some(page) => match page["value"]["next_byte"].as_u64() { - Some(next) => { - assert!(next > offset); - offset = next; - token = page["value"]["content_token"].clone(); - } - None => break, - }, - None => { - return tool( - &id, - "ReadJournalText", - json!({ - "target":{"target":"source","pointer":range["source"]["pointer"]}, - "offset":offset,"limit":subjektiv::journal::MAX_JOURNAL_READ_BYTES,"content_token":token - }), - ); - } - } - } - } + .filter_map(Item::as_text) + .find_map(|text| { + text.lines() + .find_map(|line| line.strip_prefix("input_json: ")) + }) + .map(|json| serde_json::from_str(json).unwrap()) + .expect("saved direct Journal input"); + let input = subjektiv::journal::recording::JournalInput::decode(&input).unwrap(); + assert_eq!( + serde_json::to_value(&input.selected_ranges).unwrap(), + state["selected_ranges"] + ); + assert!( + input + .capture + .entries + .iter() + .all(|entry| input.capture.recording_text(entry.order).is_ok()) + ); let (name, action) = if state["current_open"].is_null() { ( "CreateJournalEntry", diff --git a/crates/subjektiv/Cargo.toml b/crates/subjektiv/Cargo.toml index 9480041cf..fc1c00ee2 100644 --- a/crates/subjektiv/Cargo.toml +++ b/crates/subjektiv/Cargo.toml @@ -15,6 +15,8 @@ serde = { workspace = true, features = ["derive"] } serde_json.workspace = true sha2.workspace = true server-api.workspace = true +session-store.workspace = true +job.workspace = true thiserror.workspace = true uuid = { workspace = true, features = ["v7"] } diff --git a/crates/subjektiv/src/journal/recording.rs b/crates/subjektiv/src/journal/recording.rs index aaf81a29e..63947b90d 100644 --- a/crates/subjektiv/src/journal/recording.rs +++ b/crates/subjektiv/src/journal/recording.rs @@ -5,8 +5,221 @@ //! original canonical identity while streaming and return only the requested bounded fragment. use super::*; +use session_store::capture::{CaptureDto, CaptureLimits}; use sha2::{Digest, Sha256}; +/// A Job owns only this bounded original-body prefix. The initial content condition binds +/// the separate generation snapshot (whose potentially large Journal text remains paged). +/// Neither possessing these bytes nor delivering them is a processing checkpoint. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct JournalInput { + pub version: u32, + pub subject_id: String, + pub journal_id: String, + pub initial_state_token: String, + pub initial_open_id: Option, + pub previous_fixed_end: Option, + pub selected_ranges: Vec, + pub capture: CaptureDto, +} + +impl JournalInput { + /// Capture callback runs once for this immutable Job, never once per page. Fitting uses + /// only that captured value and measures the complete serialized input, including the + /// initial Journal condition and ranges duplicated beside the capture's descriptors. + pub fn build( + store: &SubjektivStore, + host: &JournalHostContext, + journal_id: &str, + capture_once: impl FnOnce(u64, u64, usize) -> JournalResult, + ) -> JournalResult { + let (journal, open, previous_fixed_end) = store.database.try_with_connection(|tx| { + super::store::check_host(store, tx, host, true)?; + Ok::<_, JournalError>(( + super::store::journal(tx, host, journal_id, true)?, + super::store::open(tx, journal_id)?, + super::store::previous_fixed(tx, journal_id)?.map(|p| p.0), + )) + })?; + let mut input = Self { + version: 1, + subject_id: journal.subject_id.clone(), + journal_id: journal_id.into(), + initial_state_token: journal.state_token, + initial_open_id: open.map(|e| e.id), + previous_fixed_end, + selected_ranges: Vec::new(), + capture: CaptureDto { + version: 1, + session_id: journal.source_session_id, + segment_id: "pending".into(), + view_entry_count: 0, + entries: Vec::new(), + }, + }; + let envelope = serde_json::to_vec(&input)?.len(); + // Capture once at a fixed storage boundary. Any adjustment for the Journal envelope + // below operates on this value only, never by making another live capture request. + let budget = job::MAX_JOB_INPUT_BYTES.saturating_sub(envelope); + input.capture = capture_once(journal.process_order, journal.process_byte, budget)?; + input.selected_ranges = capture_ranges(&input.capture)?; + if input.selected_ranges.is_empty() { + return Ok(input); + } + input.fit()?; + let value = serde_json::to_value(&input)?; + Self::decode(&value) + } + + fn fit(&mut self) -> JournalResult<()> { + let original = self.capture.clone(); + let first = self + .selected_ranges + .first() + .ok_or_else(|| invalid("empty recording selection"))?; + let start = session_store::capture::RecordingPosition { + canonical_order: first.source.order, + byte_offset: first.bytes.start, + }; + let mut budget = serde_json::to_vec(&original)?.len(); + loop { + let bytes = serde_json::to_vec(self)?.len(); + if bytes <= job::MAX_JOB_INPUT_BYTES { + return Ok(()); + } + budget = budget + .checked_sub(bytes - job::MAX_JOB_INPUT_BYTES) + .ok_or_else(|| invalid("Journal envelope cannot fit source bytes"))?; + self.capture = original + .recording_slice(start, MAX_JOURNAL_RANGES, budget) + .map_err(|e| invalid(&e.to_string()))? + .capture; + self.selected_ranges = capture_ranges(&self.capture)?; + if self.selected_ranges.is_empty() { + return Err(invalid("Journal envelope cannot fit one source scalar")); + } + } + } + + pub fn decode(value: &serde_json::Value) -> JournalResult { + if serde_json::to_vec(value)?.len() > job::MAX_JOB_INPUT_BYTES { + return Err(invalid( + "Journal input exceeds whole serialized Job input limit", + )); + } + let input: Self = serde_json::from_value(value.clone())?; + input + .capture + .validate(CaptureLimits { + max_bytes: job::MAX_JOB_INPUT_BYTES, + max_entries: MAX_JOURNAL_RANGES, + max_text_bytes: job::MAX_JOB_INPUT_BYTES, + ..CaptureLimits::default() + }) + .map_err(|e| invalid(&e.to_string()))?; + if input.version != 1 + || input.subject_id.is_empty() + || input.journal_id.is_empty() + || input.initial_state_token.is_empty() + || input.selected_ranges.is_empty() + || input.selected_ranges.len() > MAX_JOURNAL_RANGES + || input + .selected_ranges + .iter() + .any(|r| r.source.session_id != input.capture.session_id) + { + return Err(invalid("invalid Journal input identity or selection")); + } + if capture_ranges(&input.capture)? != input.selected_ranges { + return Err(conflict( + "Journal body snapshot does not match selected canonical ranges", + )); + } + Ok(input) + } + + pub fn validate_generation(&self, generation: &JournalGeneration) -> JournalResult<()> { + if generation.journal_id != self.journal_id + || generation.initial_journal.subject_id != self.subject_id + || generation.initial_journal.source_session_id != self.capture.session_id + || generation.initial_journal.state_token != self.initial_state_token + || generation.initial_open.as_ref().map(|e| &e.id) != self.initial_open_id.as_ref() + || generation.previous_fixed_end != self.previous_fixed_end + || generation.selected_ranges != self.selected_ranges + { + return Err(conflict( + "saved Journal input differs from its initial generation snapshot", + )); + } + Ok(()) + } +} + +/// Converts the shared recording projection into Journal's coverage language without +/// interpreting model text or constructing a second capture representation. +pub fn capture_ranges(capture: &CaptureDto) -> JournalResult> { + use session_store::capture::CaptureBody; + let mut ranges = Vec::new(); + for entry in &capture.entries { + let CaptureBody::CanonicalPublic { + descriptor, + byte_offset, + text, + .. + } = &entry.body + else { + return Err(invalid( + "Journal requires canonical original recording capture", + )); + }; + if !entry.policy.recording_material + || text.is_empty() + || descriptor.identity.segment_id.to_string() != entry.source.segment_id + || descriptor.identity.entry_ref != format!("E{}", entry.entry_id) + || descriptor.canonical_order != entry.order + || byte_offset + .checked_add(text.len() as u64) + .is_none_or(|end| end > descriptor.total_bytes) + { + return Err(conflict( + "recording body identity, policy or range mismatch", + )); + } + let source = JournalSource { + session_id: capture.session_id.clone(), + pointer: JournalSourcePointer { + segment_id: descriptor.identity.segment_id, + entry_ref: descriptor.identity.entry_ref.clone(), + }, + order: descriptor.canonical_order, + content_sha256: descriptor.sha256.clone(), + total_bytes: descriptor.total_bytes, + }; + if *byte_offset == 0 && text.len() as u64 == descriptor.total_bytes { + verify_canonical_text(&source, text)?; + } + ranges.push(JournalSourceRange { + source, + bytes: JournalByteRange { + start: *byte_offset, + end: *byte_offset + text.len() as u64, + }, + }); + } + for pair in ranges.windows(2) { + if pair[0].source.order.checked_add(1) != Some(pair[1].source.order) + || pair[0].bytes.end != pair[0].source.total_bytes + || pair[1].bytes.start != 0 + { + return Err(conflict( + "recording capture is not an oldest contiguous prefix", + )); + } + } + Ok(ranges) +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct JournalMaterial { diff --git a/crates/subjektiv/src/journal/service.rs b/crates/subjektiv/src/journal/service.rs index 085d20904..1829a0e18 100644 --- a/crates/subjektiv/src/journal/service.rs +++ b/crates/subjektiv/src/journal/service.rs @@ -66,8 +66,8 @@ fn text_page( }) } -/// Adapter for Host captures bounded by their storage contract. Production source reads should -/// use [`execute_bounded`] to avoid allocating an entire source for one small read. +/// Legacy adapter for Host captures bounded by their storage contract. +/// Production Hosts use [`attest_input_delivery`] and [`execute_snapshot`] instead. pub fn execute( store: &SubjektivStore, context: &JournalJobContext, @@ -101,23 +101,149 @@ pub fn execute( }) } -/// Reads are delivered through this service before mutation authority is enabled. The durable -/// delivery ledger stores byte ranges only: Jobs may retain immutable descriptors without -/// accumulating a permanent second copy of the original Session in their request ledgers. -/// The Host verifies original provenance/digest/length and returns a bounded UTF-8 fragment. +pub(crate) fn migrate_input_delivery( + tx: &rusqlite::Transaction<'_>, +) -> feature_storage::Result<()> { + tx.execute_batch( + "CREATE TABLE journal_generation_input_deliveries ( + generation_id TEXT PRIMARY KEY NOT NULL REFERENCES journal_generations(id) ON DELETE CASCADE, + input_digest TEXT NOT NULL CHECK(length(input_digest)=71 AND substr(input_digest,1,7)='sha256:' AND substr(input_digest,8) NOT GLOB '*[^0-9a-f]*') + ); + CREATE TRIGGER journal_generation_input_delivery_no_update + BEFORE UPDATE ON journal_generation_input_deliveries + BEGIN SELECT RAISE(ABORT, 'Journal input delivery bindings are immutable'); END;", + )?; + Ok(()) +} + +fn validate_input_digest(input_digest: &str) -> JournalResult<()> { + // Preserve JobRequest::input_digest() verbatim, including its algorithm prefix. + if input_digest.strip_prefix("sha256:").is_none_or(|hex| { + hex.len() != 64 + || !hex + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) { + return Err(JournalError::InvalidInput( + "immutable Job input digest must be sha256: followed by 64 lowercase hex digits".into(), + )); + } + Ok(()) +} + +/// Trusted Host-only delivery evidence, never a model tool or a claim of understanding. +/// Before calling, the Host must validate the saved, immutable body-bearing input against the +/// bound Job's input digest, its Session/provenance/ranges, and the actual dispatch plan that +/// supplies that exact input. The service cannot verify bytes it does not own. This transaction +/// stores only the digest bound to the generation's immutable selection; it neither copies the +/// body nor changes records, position, reflection or checkpoint. Exact replay is allowed only +/// while the same generation and writer binding remain active; changed digests/ranges conflict. +pub fn attest_input_delivery( + store: &SubjektivStore, + context: &JournalJobContext, + input_digest: &str, + selected_ranges: &[JournalSourceRange], +) -> JournalResult<()> { + validate_input_digest(input_digest)?; + store.database.try_transaction(|tx| { + super::lifecycle::reject_erased(tx, &context.host)?; + let generation = super::store::job(store, tx, context, true)?; + if generation.selected_ranges != selected_ranges { + return Err(JournalError::Conflict( + "immutable Job input selection differs from generation".into(), + )); + } + let saved: Option = tx.query_row( + "SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1", + [&generation.id], |row| row.get(0), + ).optional()?; + if let Some(saved) = saved { + if saved != input_digest { + return Err(JournalError::Conflict( + "generation is bound to a different immutable Job input digest".into(), + )); + } + } else { + tx.execute( + "INSERT INTO journal_generation_input_deliveries(generation_id,input_digest) VALUES(?1,?2)", + params![generation.id, input_digest], + )?; + } + Ok(()) + }) +} + +fn require_input_delivery( + tx: &rusqlite::Connection, + context: &JournalJobContext, + input_digest: &str, +) -> JournalResult<()> { + validate_input_digest(input_digest)?; + let saved: Option = tx + .query_row( + "SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1", + [&context.generation_id], + |row| row.get(0), + ) + .optional()?; + if saved.as_deref() != Some(input_digest) { + return Err(JournalError::Conflict( + "trusted Host delivery of this immutable Job input is required".into(), + )); + } + Ok(()) +} + +/// Production sequential execution over Journal state and the Host-attested immutable input. +/// The digest is taken from the bound saved Job by the Host, not from model arguments. Every +/// request requires the matching delivery binding and an active generation/writer. Source text +/// is already in immutable input: Source targets fail explicitly, with no Session reader or +/// fallback. Receipt-backed record, position and checkpoint constraints remain unchanged. +pub fn execute_snapshot( + store: &SubjektivStore, + context: &JournalJobContext, + input_digest: &str, + request: JournalJobRequest, +) -> JournalResult { + execute_inner(store, context, request, Some(input_digest), |_, _, _| { + Err(JournalError::PermissionDenied( + "Source reads are unavailable for immutable Journal input".into(), + )) + }) +} + +/// Legacy/test source-read adapter. Production Hosts must use [`execute_snapshot`]. +/// This ledger attests bounded Source responses only and cannot satisfy the input-digest gate. pub fn execute_bounded( store: &SubjektivStore, context: &JournalJobContext, request: JournalJobRequest, + read_source: impl FnMut(&JournalSource, u64, usize) -> JournalResult, +) -> JournalResult { + execute_inner(store, context, request, None, read_source) +} + +fn execute_inner( + store: &SubjektivStore, + context: &JournalJobContext, + request: JournalJobRequest, + input_digest: Option<&str>, mut read_source: impl FnMut(&JournalSource, u64, usize) -> JournalResult, ) -> JournalResult { - let generation = store - .database - .try_with_connection(|tx| super::store::job(store, tx, context, true))?; + let generation = store.database.try_with_connection(|tx| { + super::lifecycle::reject_erased(tx, &context.host)?; + let generation = super::store::job(store, tx, context, true)?; + if let Some(input_digest) = input_digest { + require_input_delivery(tx, context, input_digest)?; + } + Ok::<_, JournalError>(generation) + })?; let mut delivered = None; let response = match request { JournalJobRequest::Execute { operation } => { - require_delivery(store, context, &generation.selected_ranges)?; + if input_digest.is_none() { + require_delivery(store, context, &generation.selected_ranges)?; + } let receipt = if matches!(operation.action, JournalAction::Checkpoint { .. }) { crate::api::checkpoint_journal(store, context, operation)? } else { @@ -197,6 +323,11 @@ pub fn execute_bounded( JournalJobRequest::ReadText { read } => { let text = match &read.target { JournalTextTarget::Source { pointer } => { + if input_digest.is_some() { + return Err(JournalError::PermissionDenied( + "Source target is unavailable: raw source bodies are in immutable Job input".into(), + )); + } let range = generation .selected_ranges .iter() diff --git a/crates/subjektiv/src/journal/store.rs b/crates/subjektiv/src/journal/store.rs index 373c06996..72008f1b1 100644 --- a/crates/subjektiv/src/journal/store.rs +++ b/crates/subjektiv/src/journal/store.rs @@ -202,7 +202,7 @@ pub(super) fn open(tx: &Connection, journal_id: &str) -> JournalResult JournalResult> { @@ -569,6 +569,60 @@ impl SubjektivStore { Ok(snapshot) }) } + /// Rebind a saved body input to a new attempt without resampling its initial state. + /// The first binding freezes the full initial Journal snapshot; retries reuse it and + /// retain operation receipts. Different Jobs cannot borrow that snapshot as authority. + pub fn prepare_journal_input( + &self, + host: &JournalHostContext, + input: &super::recording::JournalInput, + binding: JobAttemptBinding, + ) -> JournalResult { + label(&binding.job_id)?; + label(&binding.attempt_id)?; + self.database.try_transaction(|tx| { + check_host(self, tx, host, false)?; + check_session_host(self, tx, host)?; + let current = journal(tx, host, &input.journal_id, true)?; + let existing: Option = tx.query_row( + "SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 AND attempt_id=?3", + params![input.journal_id, binding.job_id, binding.attempt_id], |r| r.get(0)).optional()?; + if let Some(raw) = existing { + let g = serde_json::from_str(&raw)?; + input.validate_generation(&g)?; + return Ok(g); + } + // Retained receipt lookup does not reacquire writer authority. Only a new + // generation requires the current active Subject/Session writer lifetime. + check_host(self, tx, host, true)?; + if tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_generations WHERE journal_id=?1 AND active=1)", [&input.journal_id], |r| r.get::<_, bool>(0))? { + return Err(conflict("another active generation owns this Journal")); + } + let prior: Option = tx.query_row( + "SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 ORDER BY rowid LIMIT 1", + params![input.journal_id, binding.job_id], |r| r.get(0)).optional()?; + let mut g = if let Some(raw) = prior { + serde_json::from_str::(&raw)? + } else { + JournalGeneration { id: String::new(), journal_id: input.journal_id.clone(), job: binding.clone(), + selected_ranges: input.selected_ranges.clone(), initial_journal: current.clone(), + initial_open: open(tx, &input.journal_id)?, previous_fixed_end: previous_fixed(tx, &input.journal_id)?.map(|p| p.0) } + }; + input.validate_generation(&g)?; + check_ranges(tx, ¤t, &input.selected_ranges, true)?; + let end = prefix(&g.initial_journal, &input.selected_ranges)?; + let cursor = (current.process_order, current.process_byte); + if cursor < (g.initial_journal.process_order, g.initial_journal.process_byte) || cursor > end { + return Err(conflict("saved input no longer owns the current processing cursor")); + } + g.id = issued_id("journal-generation"); + g.job = binding; + tx.execute("INSERT INTO journal_generations(id,journal_id,job_id,attempt_id,request_json,snapshot_json,active) VALUES(?1,?2,?3,?4,?5,?6,1)", + params![g.id,g.journal_id,g.job.job_id,g.job.attempt_id,serde_json::to_string(&g.selected_ranges)?,serde_json::to_string(&g)?])?; + Ok(g) + }) + } + pub(super) fn validate_journal_job(&self, ctx: &JournalJobContext) -> JournalResult<()> { self.database.try_with_connection(|tx| { job(self, tx, ctx, true)?; diff --git a/crates/subjektiv/src/journal/tests.rs b/crates/subjektiv/src/journal/tests.rs index 7fae087db..2457fdea7 100644 --- a/crates/subjektiv/src/journal/tests.rs +++ b/crates/subjektiv/src/journal/tests.rs @@ -3,6 +3,1572 @@ //! runtime or provider is started. Temporary storage owns cleanup. Receipt insertion faults are //! deterministic SQLite triggers, not a claim of arbitrary process-crash durability. +mod input_delivery { + use super::*; + use transport::{JournalJobRequest as Request, JournalJobResponse as Response}; + + fn job_request(body: &str) -> ::job::JobRequest { + ::job::JobRequest { + job_id: "immutable-body-job".into(), + purpose: "Journal input delivery gate".into(), + input_ref: "journal:immutable-input".into(), + input: serde_json::json!({ "raw_body": body }), + instruction: "Record the immutable input body".into(), + profile: "builtin:subjektiv-journal-recording".into(), + serialization_key: None, + limits: Default::default(), + } + } + + fn digest() -> String { + job_request("保存済みの公開原文本文と経緯") + .input_digest() + .unwrap() + } + + fn malformed_digests() -> Vec { + vec![ + String::new(), + "sha256:not-a-digest".into(), + "a".repeat(64), + format!("sha256:{}", "A".repeat(64)), + format!("sha256:{}", "g".repeat(64)), + format!("sha256:{}", "a".repeat(63)), + format!("sha256:{}", "a".repeat(65)), + format!("SHA256:{}", "a".repeat(64)), + ] + } + + fn run(f: &Fixture, ctx: &JournalJobContext, request: Request) -> JournalResult { + service::execute_snapshot(f.store(), ctx, &digest(), request) + } + + fn attest(f: &Fixture, ctx: &JournalJobContext) { + service::attest_input_delivery(f.store(), ctx, &digest(), &f.ranges).unwrap(); + } + + #[test] + fn input_attestation_does_not_advance_journal_progress_and_replays_after_reopen() { + let mut f = Fixture::new(); + let (generation, ctx) = f.generation("immutable-input", f.ranges.clone()); + let before = f.state(); + let sources = f.sources(); + attest(&f, &ctx); + attest(&f, &ctx); + assert_eq!(f.state(), before); + assert_eq!(f.sources(), sources); + assert!(f.entries().is_empty()); + assert!(matches!( + service::finish(f.store(), &ctx), + Err(JournalError::Conflict(_)) + )); + assert!(f.store().journal_retention_required(&f.host).unwrap()); + f.reopen(); + attest(&f, &ctx); + assert_eq!(f.state(), before); + assert_eq!(f.sources(), sources); + f.store() + .database + .try_with_connection(|tx| { + let saved: (String, String) = tx.query_row( + "SELECT generation_id,input_digest FROM journal_generation_input_deliveries", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + )?; + assert_eq!(saved, (generation.id.clone(), digest())); + let source_deliveries: u64 = tx.query_row( + "SELECT COUNT(*) FROM journal_generation_deliveries", + [], + |r| r.get(0), + )?; + let receipts: u64 = + tx.query_row("SELECT COUNT(*) FROM journal_operations", [], |r| r.get(0))?; + assert_eq!((source_deliveries, receipts), (0, 0)); + Ok::<_, JournalError>(()) + }) + .unwrap(); + } + + #[test] + fn snapshot_gate_requires_matching_digest_and_exact_generation_selection() { + let mut f = Fixture::new(); + let (generation, ctx) = f.generation("digest-binding", f.ranges.clone()); + let record = f.operation("record", open_action("ongoing"), f.ranges.clone()); + for request in [ + Request::ReadState, + Request::Execute { + operation: record.clone(), + }, + ] { + assert!(matches!( + run(&f, &ctx, request), + Err(JournalError::Conflict(_)) + )); + } + let mut changed = f.ranges.clone(); + changed[0].bytes.start += 1; + for selection in [ + changed, + f.ranges[1..].to_vec(), + f.ranges.iter().rev().cloned().collect(), + ] { + assert!(matches!( + service::attest_input_delivery(f.store(), &ctx, &digest(), &selection), + Err(JournalError::Conflict(_)) + )); + } + for malformed in malformed_digests() { + assert!(matches!( + service::attest_input_delivery(f.store(), &ctx, &malformed, &f.ranges), + Err(JournalError::InvalidInput(_)) + )); + } + attest(&f, &ctx); + f.reopen(); + let different = job_request("改変された公開原文本文") + .input_digest() + .unwrap(); + assert_ne!(different, digest()); + assert!(matches!( + service::attest_input_delivery(f.store(), &ctx, &different, &f.ranges), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + service::execute_snapshot(f.store(), &ctx, &different, Request::ReadState), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + service::execute_snapshot( + f.store(), + &ctx, + &different, + Request::Execute { + operation: record.clone() + } + ), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + run(&f, &ctx, Request::Execute { operation: record }), + Ok(Response::Saved(_)) + )); + f.close(&generation); + let (_, replacement) = f.generation("replacement-input", f.ranges.clone()); + assert!(matches!( + run(&f, &replacement, Request::ReadState), + Err(JournalError::Conflict(_)) + )); + attest(&f, &replacement); + assert!(matches!( + run(&f, &replacement, Request::ReadState), + Ok(Response::State(_)) + )); + } + + #[test] + fn snapshot_execution_reads_journal_only_and_keeps_receipt_checkpoint_and_position_constraints() + { + let mut f = Fixture::new(); + let (_, ctx) = f.generation("source-free", f.ranges.clone()); + attest(&f, &ctx); + let source = Request::ReadText { + read: transport::JournalTextRead { + target: transport::JournalTextTarget::Source { + pointer: f.ranges[0].source.pointer.clone(), + }, + offset: 0, + limit: MAX_JOURNAL_READ_BYTES, + content_token: None, + }, + }; + let error = run(&f, &ctx, source).unwrap_err(); + assert!( + matches!(error, JournalError::PermissionDenied(ref message) if message.contains("Source target") && message.contains("immutable Job input")) + ); + let record = f.operation("saved-record", open_action("ongoing"), f.ranges.clone()); + let request = Request::Execute { + operation: record.clone(), + }; + let receipt = run(&f, &ctx, request.clone()).unwrap(); + // The lost response is recovered from the same operation after closing/reopening SQLite. + f.reopen(); + assert_eq!(run(&f, &ctx, request).unwrap(), receipt); + assert_eq!(f.entries().len(), 1); + assert_eq!(f.state().0.process_order, 0); + assert!(matches!( + service::finish(f.store(), &ctx), + Err(JournalError::Conflict(_)) + )); + let position = f.operation( + "saved-position", + JournalAction::UpdatePosition { + current_position: "現在地".into(), + }, + vec![], + ); + run( + &f, + &ctx, + Request::Execute { + operation: position, + }, + ) + .unwrap(); + for evidence in [ + vec![], + vec!["missing-record".into()], + vec!["saved-position".into()], + ] { + let operation = f.operation( + "invalid-checkpoint", + JournalAction::Checkpoint { + record_operation_ids: evidence, + }, + f.ranges.clone(), + ); + assert!(run(&f, &ctx, Request::Execute { operation }).is_err()); + assert_eq!( + (f.state().0.process_order, f.state().0.process_byte), + (0, 0) + ); + } + for (snapshot, expected) in [ + (transport::JournalSnapshot::Initial, ""), + (transport::JournalSnapshot::Current, "現在地"), + ] { + let response = run( + &f, + &ctx, + Request::ReadText { + read: transport::JournalTextRead { + target: transport::JournalTextTarget::Position { snapshot }, + offset: 0, + limit: 100, + content_token: None, + }, + }, + ) + .unwrap(); + assert!(matches!(response, Response::Text(page) if page.text == expected)); + } + for target in [ + transport::JournalTextTarget::OpenBody { + snapshot: transport::JournalSnapshot::Current, + }, + transport::JournalTextTarget::OpenOngoing { + snapshot: transport::JournalSnapshot::Current, + }, + ] { + assert!(matches!( + run( + &f, + &ctx, + Request::ReadText { + read: transport::JournalTextRead { + target, + offset: 0, + limit: 100, + content_token: None + } + } + ), + Ok(Response::Text(_)) + )); + } + assert!(matches!( + run( + &f, + &ctx, + Request::QueryOperation { + request_id: record.request_id.clone() + } + ), + Ok(Response::Operation(Some(_))) + )); + assert!( + matches!(run(&f, &ctx, Request::ListOperations { after: None, limit: 32 }), Ok(Response::Operations(page)) if page.items.len() == 2) + ); + assert!( + matches!(run(&f, &ctx, Request::ReadOperationCoverage { request_id: record.request_id.clone(), offset: 0, limit: 32 }), Ok(Response::Coverage(page)) if page.items == f.ranges) + ); + let checkpoint = f.operation( + "checkpoint", + JournalAction::Checkpoint { + record_operation_ids: vec![record.request_id], + }, + f.ranges.clone(), + ); + run( + &f, + &ctx, + Request::Execute { + operation: checkpoint, + }, + ) + .unwrap(); + f.reopen(); + assert_eq!( + (f.state().0.process_order, f.state().0.process_byte), + (2, 0) + ); + assert!(!f.store().journal_retention_required(&f.host).unwrap()); + let completion = service::finish(f.store(), &ctx).unwrap(); + assert_eq!((completion.process_order, completion.process_byte), (2, 0)); + assert!(matches!( + run(&f, &ctx, Request::ReadState), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + service::attest_input_delivery(f.store(), &ctx, &digest(), &f.ranges), + Err(JournalError::Conflict(_)) + )); + assert_eq!(service::finish(f.store(), &ctx).unwrap(), completion); + let attr = f.host.attribution.as_ref().unwrap(); + let binding = f + .store() + .claim_journal_writer(&f.host.subject_id, &attr.runtime_id, &attr.worker_id) + .unwrap(); + f.store().release_journal_writer(&binding).unwrap(); + assert!(matches!( + service::attest_input_delivery(f.store(), &ctx, &digest(), &f.ranges), + Err(JournalError::PermissionDenied(_)) + )); + assert!(matches!( + run(&f, &ctx, Request::ReadState), + Err(JournalError::PermissionDenied(_)) + )); + } + + #[test] + fn retired_subject_rejects_late_attestation_and_snapshot_execution() { + let mut f = Fixture::new(); + let (_, ctx) = f.generation("retired-input", f.ranges.clone()); + attest(&f, &ctx); + let operation = f.operation( + "late-retired-record", + open_action("must not save"), + f.ranges.clone(), + ); + f.store().retire_subject(&f.host.subject_id).unwrap(); + f.reopen(); + assert!(matches!( + service::attest_input_delivery(f.store(), &ctx, &digest(), &f.ranges), + Err(JournalError::PermissionDenied(_)) + )); + assert!(matches!( + run(&f, &ctx, Request::ReadState), + Err(JournalError::PermissionDenied(_)) + )); + assert!(matches!( + run(&f, &ctx, Request::Execute { operation }), + Err(JournalError::PermissionDenied(_)) + )); + assert!(f.entries().is_empty()); + } + + #[test] + fn legacy_source_delivery_cannot_satisfy_immutable_input_gate() { + let f = Fixture::with_sizes(&[8]); + let (_, ctx) = f.generation("legacy-read", f.ranges.clone()); + service::execute_bounded( + f.store(), + &ctx, + Request::ReadText { + read: transport::JournalTextRead { + target: transport::JournalTextTarget::Source { + pointer: f.ranges[0].source.pointer.clone(), + }, + offset: 0, + limit: 8, + content_token: None, + }, + }, + |source, _, _| { + Ok(recording::JournalMaterial { + range: JournalSourceRange { + source: source.clone(), + bytes: JournalByteRange { start: 0, end: 8 }, + }, + text: "12345678".into(), + }) + }, + ) + .unwrap(); + assert!(matches!( + run(&f, &ctx, Request::ReadState), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + run( + &f, + &ctx, + Request::Execute { + operation: f.operation("new-record", open_action("ongoing"), f.ranges.clone()) + } + ), + Err(JournalError::Conflict(_)) + )); + } + + #[test] + fn input_delivery_migration_preserves_existing_subjects_and_enforces_binding_constraints() { + // Exercise the oldest Journal baseline, the immediate predecessor and fresh creation. + for baseline in [ + &crate::MIGRATIONS[..7], + &crate::MIGRATIONS[..12], + crate::MIGRATIONS, + ] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("migration"); + let manager = FeatureStorage::new(&root); + let scope = manager.scope("input-delivery-upgrade").unwrap(); + let registration = scope + .register(feature_storage::FeatureRegistration::new( + crate::SUBJEKTIV_FEATURE_ID, + baseline, + )) + .unwrap(); + let store = SubjektivStore::open(&scope, ®istration).unwrap(); + let subject = store + .create_subject_with_behavior( + SubjectRole::new("preserved").unwrap(), + "Existing behavior".into(), + ) + .unwrap(); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + drop(manager); + + let manager = FeatureStorage::new(&root); + let scope = manager.scope("input-delivery-upgrade").unwrap(); + let registration = SubjektivStore::register(&scope).unwrap(); + let store = SubjektivStore::open(&scope, ®istration).unwrap(); + assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone())); + let host = session_host(&store, &subject.id, "migration-session"); + let ranges = vec![source_range("migration-session", 0, 8)]; + let id = store + .accept_journal_sources(&host, "Migration", &ranges) + .unwrap(); + let binding = JobAttemptBinding { + job_id: "migration-job".into(), + attempt_id: "attempt".into(), + }; + let generation = store + .prepare_journal_generation(&host, &id, binding.clone(), ranges.clone()) + .unwrap(); + let ctx = JournalJobContext::validated(&store, &host, &generation.id, binding).unwrap(); + service::attest_input_delivery(&store, &ctx, &digest(), &ranges).unwrap(); + store + .close_journal_generation(&host, &generation.id, &generation.job) + .unwrap(); + let second = store + .prepare_journal_generation( + &host, + &id, + JobAttemptBinding { + job_id: "next-migration-job".into(), + attempt_id: "attempt".into(), + }, + ranges.clone(), + ) + .unwrap(); + store.database.try_transaction(|tx| { + let columns = tx.prepare("PRAGMA table_info(journal_generation_input_deliveries)")?.query_map([], |r| r.get::<_, String>(1))?.collect::, _>>()?; + assert_eq!(columns, ["generation_id", "input_digest"]); + assert!(tx.execute("INSERT INTO journal_generation_input_deliveries VALUES(?1,?2)", rusqlite::params![generation.id, digest()]).is_err()); + assert!(tx.execute("INSERT INTO journal_generation_input_deliveries VALUES('missing-generation',?1)", [digest()]).is_err()); + assert!(tx.execute("UPDATE journal_generation_input_deliveries SET input_digest=?1", [job_request("changed body").input_digest().unwrap()]).is_err()); + // Digest validation is a schema constraint as well as an API boundary check. + for invalid in malformed_digests() { + assert!(tx.execute("INSERT INTO journal_generation_input_deliveries VALUES(?1,?2)", rusqlite::params![second.id, invalid]).is_err()); + } + let saved: String = tx.query_row("SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1", [&generation.id], |r| r.get(0))?; + assert_eq!(saved, digest()); + Ok::<_, JournalError>(()) + }).unwrap(); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + } + } + + #[test] + fn input_delivery_migration_failure_rolls_back_table_creation_and_retries() { + // A colliding trigger is created before migration so failure occurs after its new table DDL. + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("rollback"); + let manager = FeatureStorage::new(&root); + let scope = manager.scope("input-delivery-rollback").unwrap(); + let registration = scope + .register(feature_storage::FeatureRegistration::new( + crate::SUBJEKTIV_FEATURE_ID, + &crate::MIGRATIONS[..12], + )) + .unwrap(); + let store = SubjektivStore::open(&scope, ®istration).unwrap(); + let subject = store + .create_subject_with_behavior( + SubjectRole::new("preserved").unwrap(), + "Retain on failure".into(), + ) + .unwrap(); + store.database.try_transaction(|tx| { + tx.execute_batch("CREATE TRIGGER journal_generation_input_delivery_no_update BEFORE UPDATE ON subjects BEGIN SELECT RAISE(ABORT, 'injected migration collision'); END;")?; + Ok::<_, JournalError>(()) + }).unwrap(); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + drop(manager); + + let manager = FeatureStorage::new(&root); + let scope = manager.scope("input-delivery-rollback").unwrap(); + let registration = SubjektivStore::register(&scope).unwrap(); + assert!(matches!( + SubjektivStore::open(&scope, ®istration), + Err(crate::SubjektivError::Storage( + feature_storage::FeatureStorageError::MigrationFailed { .. } + )) + )); + drop(scope); + manager.shutdown().unwrap(); + drop(manager); + + let manager = FeatureStorage::new(&root); + let scope = manager.scope("input-delivery-rollback").unwrap(); + let registration = scope + .register(feature_storage::FeatureRegistration::new( + crate::SUBJEKTIV_FEATURE_ID, + &crate::MIGRATIONS[..12], + )) + .unwrap(); + let store = SubjektivStore::open(&scope, ®istration).unwrap(); + assert_eq!(store.subject(&subject.id).unwrap(), Some(subject.clone())); + store.database.try_transaction(|tx| { + assert!(!tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='journal_generation_input_deliveries')", [], |r| r.get::<_, bool>(0))?); + tx.execute_batch("DROP TRIGGER journal_generation_input_delivery_no_update")?; + Ok::<_, JournalError>(()) + }).unwrap(); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + drop(manager); + + let manager = FeatureStorage::new(&root); + let scope = manager.scope("input-delivery-rollback").unwrap(); + let registration = SubjektivStore::register(&scope).unwrap(); + let store = SubjektivStore::open(&scope, ®istration).unwrap(); + assert_eq!(store.subject(&subject.id).unwrap(), Some(subject)); + store.database.try_with_connection(|tx| { + assert!(tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name='journal_generation_input_deliveries')", [], |r| r.get::<_, bool>(0))?); + Ok::<_, JournalError>(()) + }).unwrap(); + drop(store); + drop(scope); + manager.shutdown().unwrap(); + } + + #[test] + fn late_input_attestation_and_execution_cannot_revive_cancelled_or_erased_generations() { + for erased in [false, true] { + let mut f = Fixture::new(); + let (generation, ctx) = f.generation("late-input", f.ranges.clone()); + attest(&f, &ctx); + let record = f.operation( + "late-record", + open_action("must not save"), + f.ranges.clone(), + ); + if erased { + f.store() + .erase_journal_session(&f.host, "erase-input") + .unwrap(); + } else { + f.close(&generation); + f.generation("replacement", f.ranges.clone()); + } + f.reopen(); + assert!(service::attest_input_delivery(f.store(), &ctx, &digest(), &f.ranges).is_err()); + assert!(run(&f, &ctx, Request::ReadState).is_err()); + assert!(run(&f, &ctx, Request::Execute { operation: record }).is_err()); + f.store() + .database + .try_with_connection(|tx| { + let count: u64 = tx.query_row( + "SELECT COUNT(*) FROM journal_generation_input_deliveries", + [], + |r| r.get(0), + )?; + assert_eq!(count, u64::from(!erased)); + let records: u64 = + tx.query_row("SELECT COUNT(*) FROM journal_operations", [], |r| r.get(0))?; + assert_eq!(records, 0); + Ok::<_, JournalError>(()) + }) + .unwrap(); + } + } +} + +mod snapshot_inputs { + //! Component coverage joins shared canonical values, Journal input construction, SQLite + //! generations, and the production snapshot service. Synthetic Host-attested captures and + //! a saved Job file replace only capture/dispatch ownership; no Worker or Session reader runs. + use super::*; + use recording::JournalInput; + use session_store::capture::{ + CaptureBody, CaptureDto, CaptureEntryDto, CaptureLimits, CaptureSource, CaptureUsePolicy, + RecordingPosition, + }; + use session_store::public_material::{ + PublicMaterialDescriptor, PublicMaterialIdentity, PublicMaterialOrigin, + }; + use transport::{JournalJobRequest as Request, JournalJobResponse as Response}; + + fn canonical_capture(session: &str, bodies: &[String]) -> CaptureDto { + let capture = CaptureDto { + version: 1, + session_id: session.into(), + segment_id: Uuid::from_u128(3).to_string(), + view_entry_count: bodies.len() as u64, + entries: bodies + .iter() + .enumerate() + .map(|(index, body)| { + let segment_id = Uuid::from_u128(1 + (index / 50) as u128); + let entry_id = format!("synthetic-{index}"); + let origin = session_store::LoggedSessionHistoryOrigin::LocalHumanInput; + CaptureEntryDto { + entry_id: entry_id.clone(), + order: index as u64, + source: CaptureSource { + segment_id: segment_id.to_string(), + entry_index: index as u64, + provenance: serde_json::from_value(serde_json::json!("human_input")) + .unwrap(), + history_origin: Some(origin.clone()), + derived_from: vec![], + }, + policy: CaptureUsePolicy { + recording_material: true, + memory_evidence: false, + observation_readable: false, + }, + body: CaptureBody::CanonicalPublic { + descriptor: PublicMaterialDescriptor { + identity: PublicMaterialIdentity { + segment_id, + entry_ref: format!("E{entry_id}"), + }, + canonical_order: index as u64, + origin: PublicMaterialOrigin::History { origin }, + sha256: Sha256::digest(body.as_bytes()) + .iter() + .map(|b| format!("{b:02x}")) + .collect(), + total_bytes: body.len() as u64, + }, + byte_offset: 0, + text: body.clone(), + boundary: None, + }, + } + }) + .collect(), + }; + capture.validate(CaptureLimits::default()).unwrap(); + capture + } + + fn admitted_capture( + f: &Fixture, + bodies: &[String], + ) -> (JournalHostContext, String, CaptureDto) { + let host = session_host(f.store(), &f.host.subject_id, "canonical-input-session"); + let capture = canonical_capture("canonical-input-session", bodies); + let mut journal_id = None; + for ranges in recording::capture_ranges(&capture) + .unwrap() + .chunks(MAX_JOURNAL_RANGES) + { + let id = f + .store() + .accept_journal_sources(&host, "Canonical input", ranges) + .unwrap(); + if let Some(prior) = &journal_id { + assert_eq!(prior, &id); + } + journal_id = Some(id); + } + (host, journal_id.unwrap(), capture) + } + + fn operation( + f: &Fixture, + host: &JournalHostContext, + id: &str, + request_id: &str, + action: JournalAction, + ranges: Vec, + ) -> JournalOperation { + JournalOperation { + request_id: request_id.into(), + expected_state: f.store().read_journal(host, id).unwrap().state_token, + action, + covered_ranges: ranges, + } + } + + fn seed_initial_state( + f: &Fixture, + host: &JournalHostContext, + id: &str, + ranges: Vec, + ) { + let binding = JobAttemptBinding { + job_id: "seed-initial-state".into(), + attempt_id: "seed-attempt".into(), + }; + let generation = f + .store() + .prepare_journal_generation(host, id, binding.clone(), ranges) + .unwrap(); + let ctx = JournalJobContext::validated(f.store(), host, &generation.id, binding).unwrap(); + f.store() + .execute_journal( + &ctx, + operation( + f, + host, + id, + "seed-open", + open_action(&"Original full open body 日本語\n\"\\".repeat(300)), + vec![], + ), + ) + .unwrap(); + f.store() + .execute_journal( + &ctx, + operation( + f, + host, + id, + "seed-position", + JournalAction::UpdatePosition { + current_position: "Initial position 日本語\n\"\\".repeat(200), + }, + vec![], + ), + ) + .unwrap(); + f.store() + .close_journal_generation(host, &generation.id, &generation.job) + .unwrap(); + } + + fn job_request(job_id: &str, input: &JournalInput) -> ::job::JobRequest { + let request = ::job::JobRequest { + job_id: job_id.into(), + purpose: "Journal canonical input".into(), + input_ref: format!("journal:{}", input.journal_id), + input: serde_json::to_value(input).unwrap(), + instruction: "Process saved immutable bodies only".into(), + profile: "builtin:subjektiv-journal-recording".into(), + serialization_key: None, + limits: Default::default(), + }; + request.validate().unwrap(); + request + } + + fn run( + f: &Fixture, + ctx: &JournalJobContext, + request: &::job::JobRequest, + action: Request, + ) -> JournalResult { + service::execute_snapshot(f.store(), ctx, &request.input_digest().unwrap(), action) + } + + #[test] + fn complete_input_budget_and_successive_canonical_prefixes_preserve_all_escaped_utf8_bytes() { + let f = Fixture::new(); + let mut bodies: Vec<_> = (0..131) + .map(|i| format!("[user {i}] 公開原文\n\"\\\u{1}\t").repeat(8)) + .collect(); + bodies[60] = "[long public result] 日本語🦀\n\"\\\u{1}\t".repeat(14_000); + let (host, id, capture) = admitted_capture(&f, &bodies); + let all_ranges = recording::capture_ranges(&capture).unwrap(); + seed_initial_state(&f, &host, &id, all_ranges[..1].to_vec()); + let mut reconstructed = vec![String::new(); bodies.len()]; + let mut jobs = 0; + let mut partial = false; + for step in 0..20 { + let before = f.store().journal_resume_state(&host, &id).unwrap(); + if before.0.process_order == bodies.len() as u64 { + break; + } + let callback_count = std::cell::Cell::new(0); + let input = JournalInput::build(f.store(), &host, &id, |order, byte, budget| { + callback_count.set(callback_count.get() + 1); + assert_eq!( + (order, byte), + (before.0.process_order, before.0.process_byte) + ); + Ok(capture + .recording_slice( + RecordingPosition { + canonical_order: order, + byte_offset: byte, + }, + MAX_JOURNAL_RANGES, + budget, + ) + .unwrap() + .capture) + }) + .unwrap(); + assert_eq!(callback_count.get(), 1); + let request = job_request(&format!("bounded-{step}"), &input); + assert!( + serde_json::to_vec(&request.input).unwrap().len() <= ::job::MAX_JOB_INPUT_BYTES + ); + assert_eq!(JournalInput::decode(&request.input).unwrap(), input); + assert_eq!(input.initial_state_token, before.0.state_token); + assert_eq!( + input.initial_open_id, + before.1.as_ref().map(|e| e.id.clone()) + ); + let binding = JobAttemptBinding { + job_id: request.job_id.clone(), + attempt_id: "attempt".into(), + }; + let generation = f + .store() + .prepare_journal_input(&host, &input, binding.clone()) + .unwrap(); + input.validate_generation(&generation).unwrap(); + let ctx = + JournalJobContext::validated(f.store(), &host, &generation.id, binding).unwrap(); + service::attest_input_delivery( + f.store(), + &ctx, + &request.input_digest().unwrap(), + &input.selected_ranges, + ) + .unwrap(); + // Capture, input construction, generation preparation and delivery cannot reflect bytes. + assert_eq!(f.store().journal_resume_state(&host, &id).unwrap(), before); + for range in &input.selected_ranges { + let (_, offset, text) = input.capture.recording_text(range.source.order).unwrap(); + let target = &mut reconstructed[range.source.order as usize]; + assert_eq!(offset, target.len() as u64); + assert_eq!(range.bytes.start, offset); + assert_eq!(range.bytes.end, offset + text.len() as u64); + assert_eq!( + text, + &bodies[range.source.order as usize] + [range.bytes.start as usize..range.bytes.end as usize] + ); + target.push_str(text); + partial |= range.bytes.end < range.source.total_bytes; + } + let record_id = format!("bounded-record-{step}"); + run( + &f, + &ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + &record_id, + JournalAction::ConfirmUnchanged, + input.selected_ranges.clone(), + ), + }, + ) + .unwrap(); + run( + &f, + &ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + &format!("bounded-position-{step}"), + JournalAction::UpdatePosition { + current_position: format!("Processed prefix {step}"), + }, + vec![], + ), + }, + ) + .unwrap(); + run( + &f, + &ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + &format!("bounded-checkpoint-{step}"), + JournalAction::Checkpoint { + record_operation_ids: vec![record_id], + }, + input.selected_ranges, + ), + }, + ) + .unwrap(); + service::finish(f.store(), &ctx).unwrap(); + jobs += 1; + } + assert!( + jobs > 2, + "the long escaped input and 131 entries must require multiple prefixes" + ); + assert!( + partial, + "a long UTF-8 body must be split, not previewed or dropped" + ); + assert_eq!(reconstructed, bodies); + let journal = f.store().read_journal(&host, &id).unwrap(); + assert_eq!((journal.process_order, journal.process_byte), (131, 0)); + for range in all_ranges { + assert_eq!( + f.store() + .read_journal_source(&host, &id, range.source.order) + .unwrap() + .reflected_bytes, + range.source.total_bytes + ); + } + } + + #[test] + fn journal_input_decode_rejects_mismatched_ranges_policy_session_tampered_text_and_missing_body() + { + let f = Fixture::new(); + let (host, id, capture) = admitted_capture( + &f, + &["[user] 公開原文\n".into(), "[user] 次の原文\n".into()], + ); + let input = + JournalInput::build(f.store(), &host, &id, |_, _, _| Ok(capture.clone())).unwrap(); + let original = serde_json::to_value(input).unwrap(); + let cases: &[(&str, fn(&mut serde_json::Value))] = &[ + ("range", |v| { + v["selected_ranges"][0]["bytes"]["end"] = serde_json::json!(1) + }), + ("recording policy", |v| { + v["capture"]["entries"][0]["policy"]["recording_material"] = + serde_json::json!(false) + }), + ("cross-purpose policy", |v| { + v["capture"]["entries"][0]["policy"]["memory_evidence"] = serde_json::json!(true) + }), + ("session", |v| { + v["capture"]["session_id"] = serde_json::json!("different-session") + }), + ("full text digest", |v| { + let text = &mut v["capture"]["entries"][0]["body"]["text"]; + *text = serde_json::json!("X".repeat(text.as_str().unwrap().len())); + }), + ("Journal version", |v| v["version"] = serde_json::json!(2)), + ("capture version", |v| { + v["capture"]["version"] = serde_json::json!(2) + }), + ("missing body", |v| { + v["capture"]["entries"][0] + .as_object_mut() + .unwrap() + .remove("body"); + }), + ("missing text", |v| { + v["capture"]["entries"][0]["body"] + .as_object_mut() + .unwrap() + .remove("text"); + }), + ]; + for (name, corrupt) in cases { + let mut value = original.clone(); + corrupt(&mut value); + assert!( + JournalInput::decode(&value).is_err(), + "accepted corrupt input: {name}" + ); + } + } + + #[test] + fn released_writer_recovers_exact_saved_input_and_completion_without_renewing_mutation_authority() + { + let mut f = Fixture::new(); + // This Subject has only the tested Session; unrelated fixture history must not + // participate in the writer's pending-source release protection. + let subject = f + .store() + .create_subject_with_behavior( + SubjectRole::new("completion-recovery").unwrap(), + String::new(), + ) + .unwrap(); + let host = session_host(f.store(), &subject.id, "released-completion-session"); + let writer = f + .store() + .claim_journal_writer(&subject.id, "runtime", "worker") + .unwrap(); + let capture = canonical_capture( + "released-completion-session", + &["[user] 記録済みの公開本文\n".into()], + ); + let ranges = recording::capture_ranges(&capture).unwrap(); + let id = f + .store() + .accept_journal_sources(&host, "Retained completion", &ranges) + .unwrap(); + seed_initial_state(&f, &host, &id, ranges.clone()); + let input = JournalInput::build(f.store(), &host, &id, |_, _, _| Ok(capture)).unwrap(); + let request = job_request("retained-completion-job", &input); + let saved_request = serde_json::to_vec(&request).unwrap(); + let binding = JobAttemptBinding { + job_id: request.job_id.clone(), + attempt_id: "original-attempt".into(), + }; + let generation = f + .store() + .prepare_journal_input(&host, &input, binding.clone()) + .unwrap(); + let old_ctx = + JournalJobContext::validated(f.store(), &host, &generation.id, binding.clone()) + .unwrap(); + service::attest_input_delivery( + f.store(), + &old_ctx, + &request.input_digest().unwrap(), + &input.selected_ranges, + ) + .unwrap(); + run( + &f, + &old_ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "completed-record", + JournalAction::ConfirmUnchanged, + ranges.clone(), + ), + }, + ) + .unwrap(); + run( + &f, + &old_ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "completed-position", + JournalAction::UpdatePosition { + current_position: "Saved position after all selected input".into(), + }, + vec![], + ), + }, + ) + .unwrap(); + run( + &f, + &old_ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "completed-checkpoint", + JournalAction::Checkpoint { + record_operation_ids: vec!["completed-record".into()], + }, + ranges, + ), + }, + ) + .unwrap(); + let saved_completion = service::finish(f.store(), &old_ctx).unwrap(); + let finished_journal = f.store().read_journal(&host, &id).unwrap(); + assert_eq!( + ( + saved_completion.process_order, + saved_completion.process_byte + ), + (1, 0) + ); + f.store().release_journal_writer(&writer).unwrap(); + assert!( + f.store() + .journal_worker_was_released(&subject.id, "runtime", "worker") + .unwrap() + ); + f.reopen(); + + // Reconciliation reads the same immutable Job input, not a new capture or the + // current Journal projection. Release did not erase this retained completion. + let request: ::job::JobRequest = serde_json::from_slice(&saved_request).unwrap(); + let input = JournalInput::decode(&request.input).unwrap(); + let retained = f + .store() + .prepare_journal_input(&host, &input, binding.clone()) + .unwrap(); + assert_eq!(retained, generation); + let recovery = + JournalJobContext::validated_recovery(f.store(), &host, &retained.id, binding.clone()) + .unwrap(); + assert_eq!( + service::finish(f.store(), &recovery).unwrap(), + saved_completion + ); + assert_eq!( + f.store().read_journal(&host, &id).unwrap(), + finished_journal + ); + assert!(matches!( + JournalJobContext::validated(f.store(), &host, &retained.id, binding.clone()), + Err(JournalError::PermissionDenied(_)) + )); + assert!(matches!( + f.store().prepare_journal_input( + &host, + &input, + JobAttemptBinding { + job_id: binding.job_id.clone(), + attempt_id: "new-attempt".into() + } + ), + Err(JournalError::PermissionDenied(_)) + )); + let mut changed = input.clone(); + changed.initial_state_token.push('x'); + assert!(matches!( + f.store().prepare_journal_input(&host, &changed, binding), + Err(JournalError::Conflict(_)) + )); + let late_mutation = operation( + &f, + &host, + &id, + "late-released-position", + JournalAction::UpdatePosition { + current_position: "Must not replace saved position".into(), + }, + vec![], + ); + assert!(matches!( + run( + &f, + &old_ctx, + &request, + Request::Execute { + operation: late_mutation.clone() + } + ), + Err(JournalError::PermissionDenied(_)) + )); + assert!(matches!( + f.store().execute_journal(&recovery, late_mutation), + Err(JournalError::PermissionDenied(_)) + )); + assert!(matches!( + service::attest_input_delivery( + f.store(), + &old_ctx, + &request.input_digest().unwrap(), + &input.selected_ranges + ), + Err(JournalError::PermissionDenied(_)) + )); + assert_eq!( + f.store().read_journal(&host, &id).unwrap(), + finished_journal + ); + assert_eq!( + service::finish(f.store(), &recovery).unwrap(), + saved_completion + ); + } + + #[test] + fn journal_input_generation_validation_rejects_foreign_scope_and_changed_initial_proof() { + let f = Fixture::new(); + let (host, id, capture) = admitted_capture(&f, &["[user] original canonical body".into()]); + let ranges = recording::capture_ranges(&capture).unwrap(); + seed_initial_state(&f, &host, &id, ranges); + let input = JournalInput::build(f.store(), &host, &id, |_, _, _| Ok(capture)).unwrap(); + let generation = f + .store() + .prepare_journal_input( + &host, + &input, + JobAttemptBinding { + job_id: "generation-proof".into(), + attempt_id: "attempt".into(), + }, + ) + .unwrap(); + input.validate_generation(&generation).unwrap(); + let cases: &[(&str, fn(&mut JournalGeneration))] = &[ + ("Journal", |g| g.journal_id.push('x')), + ("Subject", |g| g.initial_journal.subject_id.push('x')), + ("Session", |g| g.initial_journal.source_session_id.push('x')), + ("initial content condition", |g| { + g.initial_journal.state_token.push('x') + }), + ("initial open entry", |g| { + g.initial_open.as_mut().unwrap().id.push('x') + }), + ("previous fixed boundary", |g| { + g.previous_fixed_end = Some(g.selected_ranges[0].source.pointer.clone()) + }), + ("selection", |g| g.selected_ranges[0].bytes.end -= 1), + ]; + for (name, change) in cases { + let mut changed = generation.clone(); + change(&mut changed); + assert!( + matches!( + input.validate_generation(&changed), + Err(JournalError::Conflict(_)) + ), + "accepted changed generation proof: {name}" + ); + } + } + + #[test] + fn same_saved_job_input_rebinds_full_initial_snapshot_and_reuses_partial_record_receipts_after_reopen() + { + let mut f = Fixture::new(); + let bodies = vec![ + "[user] 最初の原文は長いUTF8\n".repeat(30), + "[user] 続きの確定原文\n".repeat(20), + ]; + let (host, id, capture) = admitted_capture(&f, &bodies); + let ranges = recording::capture_ranges(&capture).unwrap(); + seed_initial_state(&f, &host, &id, ranges.clone()); + let input = JournalInput::build(f.store(), &host, &id, |_, _, _| Ok(capture)).unwrap(); + let request = job_request("durable-saved-input", &input); + let saved_path = f.temp.path().join("immutable-job.json"); + std::fs::write(&saved_path, serde_json::to_vec(&request).unwrap()).unwrap(); + let first_binding = JobAttemptBinding { + job_id: request.job_id.clone(), + attempt_id: "first".into(), + }; + let first = f + .store() + .prepare_journal_input(&host, &input, first_binding.clone()) + .unwrap(); + assert_eq!( + f.store() + .prepare_journal_input(&host, &input, first_binding.clone()) + .unwrap(), + first + ); + input.validate_generation(&first).unwrap(); + let first_ctx = + JournalJobContext::validated(f.store(), &host, &first.id, first_binding).unwrap(); + service::attest_input_delivery( + f.store(), + &first_ctx, + &request.input_digest().unwrap(), + &input.selected_ranges, + ) + .unwrap(); + let record = operation( + &f, + &host, + &id, + "durable-record", + JournalAction::Rewrite { + entry_id: first.initial_open.as_ref().unwrap().id.clone(), + title: "Saved activity".into(), + body: "Both original ranges reflected in this record".into(), + ongoing: "Continue work".into(), + }, + ranges.clone(), + ); + let saved_record = run( + &f, + &first_ctx, + &request, + Request::Execute { + operation: record.clone(), + }, + ) + .unwrap(); + let saved_position = run( + &f, + &first_ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "first-position", + JournalAction::UpdatePosition { + current_position: "Position after the successful record".into(), + }, + vec![], + ), + }, + ) + .unwrap(); + let mut cut = bodies[0].len() / 2; + while !bodies[0].is_char_boundary(cut) { + cut -= 1; + } + let saved_checkpoint = run( + &f, + &first_ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "partial-checkpoint", + JournalAction::Checkpoint { + record_operation_ids: vec![record.request_id.clone()], + }, + vec![fragment(&ranges[0], 0, cut as u64)], + ), + }, + ) + .unwrap(); + assert_eq!( + ( + f.store().read_journal(&host, &id).unwrap().process_order, + f.store().read_journal(&host, &id).unwrap().process_byte + ), + (0, cut as u64) + ); + f.store() + .close_journal_generation(&host, &first.id, &first.job) + .unwrap(); + f.reopen(); + // Only the saved input and SQLite are reopened. There is no latest-source callback. + let retry_request: ::job::JobRequest = + serde_json::from_slice(&std::fs::read(saved_path).unwrap()).unwrap(); + assert_eq!(retry_request, request); + let retry_input = JournalInput::decode(&retry_request.input).unwrap(); + let retry_binding = JobAttemptBinding { + job_id: retry_request.job_id.clone(), + attempt_id: "second".into(), + }; + let mut changed_initial = retry_input.clone(); + changed_initial.initial_state_token.push('x'); + assert!(matches!( + changed_initial.validate_generation(&first), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + f.store() + .prepare_journal_input(&host, &changed_initial, retry_binding.clone()), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + f.store().prepare_journal_input( + &host, + &retry_input, + JobAttemptBinding { + job_id: "different-job-cannot-borrow-initial-proof".into(), + attempt_id: "attempt".into() + } + ), + Err(JournalError::Conflict(_)) + )); + let retry = f + .store() + .prepare_journal_input(&host, &retry_input, retry_binding.clone()) + .unwrap(); + assert_ne!(retry.id, first.id); + assert_eq!(retry.initial_journal, first.initial_journal); + assert_eq!(retry.initial_open, first.initial_open); + assert_eq!(retry.previous_fixed_end, first.previous_fixed_end); + assert_eq!(retry.selected_ranges, first.selected_ranges); + assert_ne!( + retry.initial_journal.state_token, + f.store().read_journal(&host, &id).unwrap().state_token + ); + retry_input.validate_generation(&retry).unwrap(); + assert_eq!( + f.store() + .prepare_journal_input(&host, &retry_input, retry_binding.clone()) + .unwrap(), + retry + ); + let retry_ctx = + JournalJobContext::validated(f.store(), &host, &retry.id, retry_binding).unwrap(); + service::attest_input_delivery( + f.store(), + &retry_ctx, + &retry_request.input_digest().unwrap(), + &retry_input.selected_ranges, + ) + .unwrap(); + assert!(matches!( + run(&f, &first_ctx, &request, Request::ReadState), + Err(JournalError::Conflict(_)) + )); + assert!(matches!( + run( + &f, + &first_ctx, + &request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "late-first-attempt", + JournalAction::ConfirmUnchanged, + ranges.clone() + ) + } + ), + Err(JournalError::Conflict(_)) + )); + for (request_id, saved) in [ + (record.request_id.as_str(), saved_record), + ("first-position", saved_position), + ("partial-checkpoint", saved_checkpoint), + ] { + let Response::Saved(receipt) = saved else { + panic!("saved operation receipt expected") + }; + assert_eq!( + f.store() + .query_journal_operation(&host, &id, request_id) + .unwrap(), + Some(receipt) + ); + } + assert!( + matches!(run(&f, &retry_ctx, &retry_request, Request::ReadOperationCoverage { request_id: record.request_id.clone(), offset: 0, limit: 32 }), Ok(Response::Coverage(page)) if page.items == ranges) + ); + for (target, expected) in [ + ( + transport::JournalTextTarget::Position { + snapshot: transport::JournalSnapshot::Initial, + }, + first.initial_journal.current_position.clone(), + ), + ( + transport::JournalTextTarget::OpenBody { + snapshot: transport::JournalSnapshot::Initial, + }, + first.initial_open.as_ref().unwrap().body.clone(), + ), + ( + transport::JournalTextTarget::OpenOngoing { + snapshot: transport::JournalSnapshot::Initial, + }, + first.initial_open.as_ref().unwrap().ongoing.clone(), + ), + ] { + assert!( + matches!(run(&f, &retry_ctx, &retry_request, Request::ReadText { read: transport::JournalTextRead { target, offset: 0, limit: MAX_JOURNAL_READ_BYTES, content_token: None } }), Ok(Response::Text(page)) if page.text == expected) + ); + } + let remainder = vec![ + fragment(&ranges[0], cut as u64, ranges[0].bytes.end), + ranges[1].clone(), + ]; + let missing_position = operation( + &f, + &host, + &id, + "retry-checkpoint-needs-position", + JournalAction::Checkpoint { + record_operation_ids: vec![record.request_id.clone()], + }, + remainder.clone(), + ); + assert!(matches!( + run( + &f, + &retry_ctx, + &retry_request, + Request::Execute { + operation: missing_position + } + ), + Err(JournalError::Conflict(_)) + )); + run( + &f, + &retry_ctx, + &retry_request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "retry-position", + JournalAction::UpdatePosition { + current_position: "Retry confirmed saved activity".into(), + }, + vec![], + ), + }, + ) + .unwrap(); + run( + &f, + &retry_ctx, + &retry_request, + Request::Execute { + operation: operation( + &f, + &host, + &id, + "remainder-checkpoint", + JournalAction::Checkpoint { + record_operation_ids: vec![record.request_id], + }, + remainder, + ), + }, + ) + .unwrap(); + let completion = service::finish(f.store(), &retry_ctx).unwrap(); + assert_eq!((completion.process_order, completion.process_byte), (2, 0)); + assert_eq!( + f.store() + .list_journal_entries(&host, &id, None, 32) + .unwrap() + .items + .len(), + 1 + ); + } +} + use super::*; use crate::SubjectRole; use feature_storage::{FeatureStorage, ScopedFeatureStorage}; diff --git a/crates/subjektiv/src/journal/transport.rs b/crates/subjektiv/src/journal/transport.rs index cb43bd9b0..4981f185b 100644 --- a/crates/subjektiv/src/journal/transport.rs +++ b/crates/subjektiv/src/journal/transport.rs @@ -1,8 +1,10 @@ //! Bounded, Host-bound Journal recording and sequential Job transport. //! //! Subject, Journal and generation authority never come from these model inputs. -//! Hosts validate live execution and public source bytes before selection; reading a -//! preview must not select the enclosing entry. Operation retries retain the entire +//! Hosts validate live execution and public source bytes before selection, save body-bearing +//! immutable Job input, and attest its exact digest and dispatch plan through the service. +//! Model requests cannot attest input delivery; Source targets are rejected by snapshot execution. +//! Initial/current Journal reads remain separate from input delivery. Operation retries retain the entire //! JournalOperation, including its original expected_state and covered_ranges. use serde::{Deserialize, Serialize}; @@ -50,8 +52,8 @@ pub struct JournalJobState { pub current_journal: JournalMetadata, pub current_open: Option, pub previous_fixed_end: Option, - /// Frozen permission to fetch this prefix, not evidence of delivery or reflection. - /// The sequential service gates mutations on its durable Source-read delivery ledger. + /// Frozen source selection delivered in immutable Job input, not evidence of reflection. + /// Production execution requires the trusted Host's matching input-digest attestation. pub selected_ranges: Vec, } @@ -83,7 +85,8 @@ pub enum JournalTextTarget { OpenOngoing { snapshot: JournalSnapshot, }, - /// The Host restricts this pointer to this Job's frozen public source input. + /// Legacy source-read adapter only. Production snapshot execution rejects this target; + /// raw source bodies must already be present in the immutable Job input. Source { pointer: JournalSourcePointer, }, diff --git a/crates/subjektiv/src/lib.rs b/crates/subjektiv/src/lib.rs index 86ba70a6e..43bd67689 100644 --- a/crates/subjektiv/src/lib.rs +++ b/crates/subjektiv/src/lib.rs @@ -639,6 +639,11 @@ static MIGRATIONS: &[FeatureMigration] = &[ "add explicit Subject permission sources and immutable child execution grants", permissions::migrate, ), + FeatureMigration::new( + 13, + "bind Journal generation delivery to immutable Job input digest", + journal::service::migrate_input_delivery, + ), ]; pub const REGISTRATION: FeatureRegistration = diff --git a/crates/worker-runtime/src/http_server/runtime_management_api.rs b/crates/worker-runtime/src/http_server/runtime_management_api.rs index e7daf4906..425bd0fb5 100644 --- a/crates/worker-runtime/src/http_server/runtime_management_api.rs +++ b/crates/worker-runtime/src/http_server/runtime_management_api.rs @@ -444,6 +444,31 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { material_pin(&self.state, request, true) } + async fn session_recording_capture( + &self, + request: runtime_api::SessionRecordingCaptureRequest, + ) -> Result + { + let scope = material_scope(&self.state, &request.workspace_id)?; + #[cfg(feature = "fs-store")] + { + Ok(self + .state + .runtime + .session_recording_capture_scoped(&scope, &request)) + } + #[cfg(not(feature = "fs-store"))] + { + let _ = scope; + Ok( + runtime_api::SessionRecordingCaptureAvailability::Unavailable { + reason: runtime_api::SessionPublicUnavailableReason::StorageUnavailable, + message: "Committed Session storage unavailable".into(), + }, + ) + } + } + async fn session_material_catalog( &self, request: runtime_api::SessionMaterialCatalogRequest, diff --git a/crates/worker-runtime/src/http_server/tests/session_material_tests.rs b/crates/worker-runtime/src/http_server/tests/session_material_tests.rs index 733f658ac..f47845f92 100644 --- a/crates/worker-runtime/src/http_server/tests/session_material_tests.rs +++ b/crates/worker-runtime/src/http_server/tests/session_material_tests.rs @@ -150,6 +150,27 @@ impl Fixture { limit: 1, } } + fn recording_request(&self) -> runtime_api::SessionRecordingCaptureRequest { + runtime_api::SessionRecordingCaptureRequest { + workspace_id: self.scope.workspace_id.clone(), + source: self.source(), + expected_session_id: self.session.to_string(), + start: Default::default(), + max_entries: 1000, + max_serialized_bytes: 128 * 1024, + } + } + fn capture(&self, source: Source) -> session_store::capture::RecordingCaptureSlice { + let mut request = self.recording_request(); + request.source = source; + match self + .runtime + .session_recording_capture_scoped(&self.scope, &request) + { + runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => slice, + other => panic!("{other:?}"), + } + } fn catalog(&self) -> runtime_api::SessionMaterialCatalogPage { match self .runtime @@ -226,6 +247,9 @@ fn pending_session_proof_is_not_missing_material_and_pin_release_allows_empty_ar .is_err(), "retention intent is not source availability" ); + assert!( + matches!(f.runtime.session_recording_capture_scoped(&f.scope, &f.recording_request()), runtime_api::SessionRecordingCaptureAvailability::NotMaterialized { session_id } if session_id == f.session.to_string()) + ); // A missing committed source is never the same fact as a pending pointer. metadata .write( @@ -479,6 +503,7 @@ fn source_scope_session_and_active_pointer_mismatches_never_return_material() { #[test] fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source() { let f = Fixture::new(); + let frozen = f.capture(f.source()); let original: Vec<_> = f.reader(&f.source()).descriptors().cloned().collect(); let pin = SessionRetentionPin(uuid::Uuid::now_v7()); f.runtime @@ -516,6 +541,7 @@ fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source() let archived = Source::Archived { archive_id: "canonical-archive".into(), }; + assert_eq!(f.capture(archived.clone()), frozen); let reader = f.reader(&archived); assert_eq!(reader.descriptors().cloned().collect::>(), original); let body = reader.read_content(&original[0], 0, 16384).unwrap(); @@ -951,11 +977,52 @@ async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router .authorizer(SignedAuthorizer { identity, trust }) .build() .unwrap(); + let frozen = match client + .session_recording_capture(f.recording_request()) + .await + .unwrap() + { + runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => slice, + other => panic!("{other:?}"), + }; + frozen.capture.validate(Default::default()).unwrap(); + assert!(frozen.next.is_none()); + assert!( + serde_json::to_vec(&frozen).unwrap().len() + <= f.recording_request().max_serialized_bytes + ); + let snapshot = serde_json::to_vec(&frozen).unwrap(); + assert!(!String::from_utf8_lossy(&snapshot).contains("private-system-prompt")); + f.store + .append( + f.session, + f.segment, + &LogEntry::AnnotatedAssistantItem { + ts: 6, + entry: message("late-answer", LoggedRole::Assistant, "not in frozen input"), + }, + ) + .unwrap(); + assert_eq!(serde_json::to_vec(&frozen).unwrap(), snapshot); let page = match client.session_material_catalog(f.request()).await.unwrap() { Catalog::Page { page } => page, other => panic!("{other:?}"), }; let descriptor = page.descriptors[0].clone(); + assert_eq!( + frozen + .capture + .recording_text(0) + .unwrap() + .0 + .identity + .entry_ref, + descriptor.entry_ref + ); + assert_eq!( + frozen.capture.recording_text(0).unwrap().0.sha256, + descriptor.sha256 + ); let read = client .session_material_read(runtime_api::SessionMaterialReadRequest { workspace_id: "workspace-a".into(), @@ -1019,6 +1086,13 @@ async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router ); } assert!(!f.root().exists()); + assert!(matches!( + client + .session_recording_capture(f.recording_request()) + .await + .unwrap(), + runtime_api::SessionRecordingCaptureAvailability::Unavailable { .. } + )); assert!(matches!( client.session_material_catalog(f.request()).await.unwrap(), Catalog::Unavailable { .. } @@ -1050,41 +1124,54 @@ async fn canonical_http_rejects_replay_tampered_body_and_workspace_mismatch() { None, workspace_test_auth(vec![trust.clone()]), ); - let path = "/v1/session-public/material/catalog"; - let body = serde_json::to_vec(&f.request()).unwrap(); - let claims = signed_claims(&trust, "POST", path, &body); - let first = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await; - assert_eq!(first.status(), StatusCode::OK); - let replay = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await; - assert_eq!(replay.status(), StatusCode::UNAUTHORIZED); - let claims = signed_claims(&trust, "POST", path, &body); - let changed = signed_request( - app.clone(), - &identity, - &claims, - b"{}".to_vec(), - "workspace-a", - ) - .await; - assert_eq!(changed.status(), StatusCode::UNAUTHORIZED); - let mut wrong_scope = f.request(); - wrong_scope.workspace_id = "other-workspace".into(); - let wrong_body = serde_json::to_vec(&wrong_scope).unwrap(); - let claims = signed_claims(&trust, "POST", path, &wrong_body); - let response = signed_request(app.clone(), &identity, &claims, wrong_body, "workspace-a").await; - assert_eq!(response.status(), StatusCode::FORBIDDEN); - let unsigned = app - .oneshot( - Request::builder() - .method("POST") - .uri(path) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(body)) - .unwrap(), + for (path, body) in [ + ( + "/v1/session-public/material/catalog", + serde_json::to_vec(&f.request()).unwrap(), + ), + ( + "/v1/session-public/recording-capture", + serde_json::to_vec(&f.recording_request()).unwrap(), + ), + ] { + let claims = signed_claims(&trust, "POST", path, &body); + let first = + signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await; + assert_eq!(first.status(), StatusCode::OK); + let replay = + signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await; + assert_eq!(replay.status(), StatusCode::UNAUTHORIZED); + let claims = signed_claims(&trust, "POST", path, &body); + let changed = signed_request( + app.clone(), + &identity, + &claims, + b"{}".to_vec(), + "workspace-a", ) - .await - .unwrap(); - assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED); + .await; + assert_eq!(changed.status(), StatusCode::UNAUTHORIZED); + let mut wrong_scope: serde_json::Value = serde_json::from_slice(&body).unwrap(); + wrong_scope["workspace_id"] = "other-workspace".into(); + let wrong_body = serde_json::to_vec(&wrong_scope).unwrap(); + let claims = signed_claims(&trust, "POST", path, &wrong_body); + let response = + signed_request(app.clone(), &identity, &claims, wrong_body, "workspace-a").await; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let unsigned = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri(path) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED); + } } #[tokio::test] @@ -1170,3 +1257,152 @@ async fn explicit_material_erasure_requires_delete_capability_and_matching_scope ); assert!(f.root().exists()); } + +#[test] +fn recording_capture_rejects_scope_session_cursor_and_budget_without_returning_bodies() { + use runtime_api::SessionRecordingCaptureAvailability as Capture; + let f = Fixture::new(); + let request = f.recording_request(); + let frozen = f.capture(f.source()); + assert_eq!(frozen.capture.entries.len(), 3); + let mut request = request; + for invalid in 0..5 { + let mut bad = request.clone(); + match invalid { + 0 => bad.workspace_id = "other".into(), + 1 => bad.expected_session_id = session_store::new_session_id().to_string(), + 2 => bad.start.canonical_order = 100, + 3 => bad.max_entries = 0, + 4 => bad.max_serialized_bytes = 4 * 1024 * 1024 + 1, + _ => unreachable!(), + } + assert!(matches!( + f.runtime.session_recording_capture_scoped(&f.scope, &bad), + Capture::Unavailable { .. } + )); + } + let text = frozen.capture.recording_text(1).unwrap().2; + request.start = session_store::capture::RecordingPosition { + canonical_order: 1, + byte_offset: text.find('🍎').unwrap() as u64 + 1, + }; + assert!(matches!( + f.runtime + .session_recording_capture_scoped(&f.scope, &request), + Capture::Unavailable { + reason: Reason::InvalidCursor, + .. + } + )); + request.start = Default::default(); + request.max_entries = 1; + let Capture::Captured { slice } = f + .runtime + .session_recording_capture_scoped(&f.scope, &request) + else { + panic!() + }; + assert_eq!(slice.capture.entries.len(), 1); + assert_eq!(slice.next.unwrap().canonical_order, 1); +} + +#[test] +fn recording_capture_compaction_preserves_original_descriptors_and_excludes_summary_and_new_tail() { + let f = Fixture::new(); + let old = f.capture(f.source()); + let compact = session_store::new_segment_id(); + let mut summary = message( + "summary", + LoggedRole::System, + "derived summary is not original", + ); + summary.metadata.input_metadata = Some( + session_store::HistoryInputMetadata::new( + session_store::HistoryInputUsePolicy { + recording_material: session_store::InputEvidenceUse::Exclude, + memory_evidence: session_store::InputEvidenceUse::Exclude, + observation: session_store::InputObservationUse::Hidden, + compaction: session_store::InputCompactionUse::OpeningSummary, + }, + None, + None, + ) + .unwrap(), + ); + summary.metadata.derivation = Some(session_store::LoggedHistoryDerivation { + sources: vec![ + LoggedSessionHistoryEntryId("user-1".into()), + LoggedSessionHistoryEntryId("answer-1".into()), + ], + }); + f.store + .create_segment( + f.session, + compact, + &[ + LogEntry::AnnotatedSegmentStart { + ts: 6, + session_id: f.session, + system_prompt: Some("private".into()), + config: Default::default(), + history: vec![summary], + forked_from: None, + compacted_from: Some(session_store::SegmentOrigin { + segment_id: f.segment, + at_turn_index: 0, + }), + }, + LogEntry::AnnotatedAssistantItem { + ts: 7, + entry: message( + "new-answer", + LoggedRole::Assistant, + "new original after compaction", + ), + }, + ], + ) + .unwrap(); + WorkerAggregateStore::new( + f.root().parent().unwrap(), + format!("worker-runtime-{}", f.worker.worker_id), + ) + .unwrap() + .set_active( + &format!("worker-runtime-{}", f.worker.worker_id), + Some(session_store::WorkerActiveSegmentRef::active_segment( + f.session, compact, + )), + None, + ) + .unwrap(); + let frozen = f.capture(f.source()); + assert_eq!(frozen.capture.entries.len(), old.capture.entries.len() + 1); + for entry in &old.capture.entries { + assert_eq!( + frozen.capture.recording_text(entry.order).unwrap(), + old.capture.recording_text(entry.order).unwrap() + ); + } + assert!( + !serde_json::to_string(&frozen) + .unwrap() + .contains("derived summary is not original") + ); + let bytes = serde_json::to_vec(&frozen).unwrap(); + f.store + .append( + f.session, + compact, + &LogEntry::AnnotatedAssistantItem { + ts: 8, + entry: message("tail", LoggedRole::Assistant, "next Job only"), + }, + ) + .unwrap(); + assert_eq!(serde_json::to_vec(&frozen).unwrap(), bytes); + assert_eq!( + f.capture(f.source()).capture.entries.len(), + frozen.capture.entries.len() + 1 + ); +} diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 086969155..38dc5cb9f 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -4949,6 +4949,133 @@ impl Runtime { .map_err(crate::retention::retention_error) } + /// Freeze committed original public bodies once; no catalog/read pagination + /// or restored Worker is involved. Publication and erasure share this fence. + #[cfg(feature = "fs-store")] + pub fn session_recording_capture_scoped( + &self, + scope: &RuntimeWorkspaceScope, + request: &runtime_api::SessionRecordingCaptureRequest, + ) -> runtime_api::SessionRecordingCaptureAvailability { + use crate::worker_backend::SessionMaterialSourceError as Error; + use runtime_api::{ + SessionPublicUnavailableReason as R, SessionRecordingCaptureAvailability as A, + }; + use session_store::public_material::{PublicMaterialError as E, PublicMaterialLimits}; + let result = (|| -> Result<_, Error> { + if scope.workspace_id != request.workspace_id { + return Err("Session Workspace scope mismatch".into()); + } + if !(1..=4096).contains(&request.max_entries) + || !(1024..=4 * 1024 * 1024).contains(&request.max_serialized_bytes) + { + return Err(E::ResourceLimit.into()); + } + let session = request + .expected_session_id + .parse::() + .map_err(|_| "Invalid Session identity")?; + // Resolve attribution without reading archive bodies. Do not acquire + // the Runtime mutex under the Session lock: archive execution takes + // those locks in the opposite order. + let (root, archive) = self + .resolve_session_public_source_with_validation(scope, &request.source, false) + .map_err(|e| e.to_string())?; + let runtime_dir = { + let state = self.lock().map_err(|e| e.to_string())?; + state + .fs_store() + .ok_or("Session storage unavailable")? + .runtime_dir() + .to_path_buf() + }; + // This is the storage append fence used by live Runtime publishers, + // also fencing archive transfer and erase. Resolve the pointer under it. + let retention = crate::retention::runtime_session_retention(&runtime_dir); + let guard = retention.lock_live(session).map_err(|error| match error { + session_store::StoreError::SessionErased(_) => E::Erased, + _ => E::Unavailable, + })?; + let mut archive_to_verify = None; + let active = match (&request.source, archive) { + (runtime_api::SessionPublicSource::Retained { worker_id }, None) => { + let metadata = session_store::worker_metadata::WorkerAggregateStore::open_read_only( + root.parent().ok_or("Session root is unavailable")?, + crate::worker_backend::ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&WorkerRef::new(*worker_id)), + ).and_then(|store| store.read_read_only()).map_err(|e| e.to_string())?.ok_or("Session metadata missing")?; + if metadata.workspace_id.as_deref() != Some(scope.workspace_id.as_str()) { + return Err("Session Workspace mismatch".into()); + } + let active = metadata.active.ok_or("Session active pointer missing")?; + if active.session_id != session { + return Err("Session identity mismatch".into()); + } + match active.segment_id { + Some(segment) => segment, + None if request.start + == session_store::capture::RecordingPosition::default() + && crate::retention::unmaterialized_session( + root.parent().ok_or("Session root unavailable")?, + &scope.workspace_id, + ) + .map_err(|e| e.to_string())? + == Some(session) => + { + return Ok(A::NotMaterialized { + session_id: session.to_string(), + }); + } + None => return Err("Session active Segment missing".into()), + } + } + (runtime_api::SessionPublicSource::Archived { archive_id }, Some(manifest)) => { + archive_to_verify = Some((archive_id, manifest.clone())); + if manifest.source_session_id != request.expected_session_id { + return Err("Archive Session identity mismatch".into()); + } + manifest + .source_active_segment_id + .ok_or("Archive active Segment missing")? + } + _ => return Err("Session source unavailable".into()), + }; + let slice = session_store::capture::capture_recording_slice_worker_guarded( + &root, + session, + active, + PublicMaterialLimits::default(), + request.start, + request.max_entries, + request.max_serialized_bytes, + &guard, + )?; + if let Some((archive_id, manifest)) = archive_to_verify { + // Validate archive checksums only after capture froze every file + // end, still under the same publication/erase fence. + let (verified, verified_root) = FsWorkerRetentionProvider::new(&runtime_dir) + .session_archive(archive_id) + .map_err(|e| e.to_string())?; + if verified != manifest || verified_root != root { + return Err(E::Changed.into()); + } + } + Ok(A::Captured { slice }) + })(); + match result { + Ok(availability) => availability, + Err(error) => A::Unavailable { + reason: match error { + Error::Material(E::ResourceLimit) => R::ResourceLimit, + Error::Material(E::InvalidOffset | E::Changed) => R::InvalidCursor, + Error::Material(E::Missing | E::Erased) => R::RetentionMissing, + Error::Material(E::Inconsistent | E::Record(_)) => R::CorruptLog, + _ => R::StorageUnavailable, + }, + message: "Committed Session recording capture unavailable".into(), + }, + } + } + #[cfg(feature = "fs-store")] pub fn session_material_catalog_scoped( &self, diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index a67fd9d6a..83dd338a7 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -730,7 +730,7 @@ impl ProfileRuntimeWorkerFactory { .map_err(|error| format!("failed to clean restore run artifacts: {error}")) } - fn runtime_worker_name_for_ref(worker_ref: &crate::identity::WorkerRef) -> String { + pub(crate) fn runtime_worker_name_for_ref(worker_ref: &crate::identity::WorkerRef) -> String { format!("worker-runtime-{}", worker_ref.worker_id) } diff --git a/crates/worker/src/job/tests.rs b/crates/worker/src/job/tests.rs index f1eaefddf..4c867bd61 100644 --- a/crates/worker/src/job/tests.rs +++ b/crates/worker/src/job/tests.rs @@ -179,11 +179,11 @@ async fn builtin_journal_profile_installs_only_bound_tools_and_finalizes_saved_s call(SUBMIT_JOB_RESULT_TOOL, json!({"result":{}})), prose(), ]); - // This runner/Feature fixture has no selected sources; descriptor delivery gates - // for nonempty ranges are owned and tested by the Subjektiv service. + // This runner/Feature fixture has no selected sources; actual body-bearing input, + // trusted delivery and checkpoint gates are covered by the Host integration suites. let mut journal_request = request("builtin:subjektiv-journal-recording"); - journal_request.input = json!({"selected_ranges":[],"origins":[]}); - journal_request.instruction = "Read immutable selected_ranges through ReadJournalText Source before recording; descriptors are not source bodies.".into(); + journal_request.input = json!({"selected_ranges":[],"capture":{"entries":[]}}); + journal_request.instruction = "Record only the immutable canonical bodies in this Job input. Do not read original Sessions through ReadJournalText Source.".into(); let job = prepare_job_from_registry_with_grant( &journal_request, root.path(), @@ -249,7 +249,7 @@ async fn builtin_journal_profile_installs_only_bound_tools_and_finalizes_saved_s "ReadJournalText", "Source", "content_token", - "Conflict", + "input digest", ] { assert!( prompt.contains(contract), diff --git a/crates/workspace-server/src/assignment_tracking_tests.rs b/crates/workspace-server/src/assignment_tracking_tests.rs index 2e9085438..152b48f14 100644 --- a/crates/workspace-server/src/assignment_tracking_tests.rs +++ b/crates/workspace-server/src/assignment_tracking_tests.rs @@ -476,7 +476,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip (99, "Explicit hosting administrators and self-registration policy".into()), (100, "Provisional users and scoped passkey activation".into()), (101, "Durable explicit Subject body release".into()), - (102, "Durable failed Subject creation recovery".into()) + (102, "Durable failed Subject creation recovery".into()), + (103, "Job input retirement independent of immutable receipts".into()) ] ); for (table, rows) in retained.iter().zip(original.iter()) { diff --git a/crates/workspace-server/src/backend_job.rs b/crates/workspace-server/src/backend_job.rs index 6d804bc97..e49d552c5 100644 --- a/crates/workspace-server/src/backend_job.rs +++ b/crates/workspace-server/src/backend_job.rs @@ -463,10 +463,52 @@ impl BackendJobRequest { } } +/// Availability of the immutable input body, independent of execution outcome. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum JobInputDisposition { + Available, + Reclaimed, + Erased, + Invalidated, +} + +impl JobInputDisposition { + pub fn as_str(self) -> &'static str { + match self { + Self::Available => "available", + Self::Reclaimed => "reclaimed", + Self::Erased => "erased", + Self::Invalidated => "invalidated", + } + } +} + +/// Host-authorized retirement; erasure and legacy invalidation do not change outcomes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JobInputRetirementReason { + Completed, + ExplicitErasure, + LegacyInvalidation, +} + +impl JobInputRetirementReason { + pub fn disposition(self) -> JobInputDisposition { + match self { + Self::Completed => JobInputDisposition::Reclaimed, + Self::ExplicitErasure => JobInputDisposition::Erased, + Self::LegacyInvalidation => JobInputDisposition::Invalidated, + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct BackendJobRecord { pub workspace_id: String, + /// Receipt metadata. After retirement `input` is null; use the verified loader to execute. pub request: BackendJobRequest, + pub input_digest: String, + pub input_disposition: JobInputDisposition, pub intent_fingerprint: String, pub state: BackendJobState, pub current_attempt: u8, @@ -479,6 +521,28 @@ pub struct BackendJobRecord { pub completed_at: Option, } +impl BackendJobRecord { + /// Verify the original immutable body and envelope before any execution or retry. + /// No Session read or replay request may repair unavailable/corrupt input. + pub fn load_executable_request(&self) -> Result<&BackendJobRequest> { + if self.input_disposition != JobInputDisposition::Available { + return Err(Error::InvalidInput(format!( + "Backend Job input is {}", + self.input_disposition.as_str() + ))); + } + self.request.validate()?; + if self.request.input_digest()? != self.input_digest + || self.request.fingerprint()? != self.intent_fingerprint + { + return Err(Error::Store( + "Backend Job immutable input or intent fingerprint mismatch".into(), + )); + } + Ok(&self.request) + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum BackendJobWorkerCleanupState { @@ -665,6 +729,8 @@ mod tests { "workspace_id": "workspace-a", "request": request(), "intent_fingerprint": request().fingerprint().unwrap(), + "input_digest": request().input_digest().unwrap(), + "input_disposition": "available", "state": "pending", "current_attempt": 2, "created_at": "created", @@ -1124,3 +1190,17 @@ mod tests { } } } + +pub(crate) const INPUT_RETENTION_TRIGGERS: &str = r#" +CREATE TRIGGER backend_job_identity_immutable BEFORE UPDATE OF + workspace_id, job_id, purpose, input_digest, input_ref, resource_key, intent_fingerprint ON backend_jobs +BEGIN SELECT RAISE(ABORT, 'immutable Backend Job identity'); END; +CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON backend_jobs +WHEN ( + ((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated')) + OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased')) + AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') + AND json_type(NEW.request_json,'$.input') = 'null' +) IS NOT TRUE +BEGIN SELECT RAISE(ABORT, 'immutable Backend Job input retirement'); END; +"#; diff --git a/crates/workspace-server/src/frozen_content_state_tests.rs b/crates/workspace-server/src/frozen_content_state_tests.rs index 954d42ccf..b2003d2ca 100644 --- a/crates/workspace-server/src/frozen_content_state_tests.rs +++ b/crates/workspace-server/src/frozen_content_state_tests.rs @@ -71,6 +71,11 @@ fn legacy_content_database(path: &Path) -> (Connection, String) { conn.execute("INSERT INTO workspace_config_tree_revisions VALUES('space',?1,?2,'toolchain','projection',?3,?4,'{}')", params![ordinal,snapshot.digest,manifest,created]).unwrap(); } + seed_legacy_content_job(&conn); + (conn, snapshot.digest) +} + +fn seed_legacy_content_job(conn: &Connection) { let request = BackendJobRequest { job_id: "legacy-job".into(), purpose: "migration_test".into(), @@ -93,7 +98,19 @@ fn legacy_content_database(path: &Path) -> (Connection, String) { VALUES('space','legacy-job','legacy-job:attempt:1',1,'old-number','dispatched','runtime','worker','2026-08-14T00:00:00Z','2026-08-13T00:00:00Z','2026-08-13T00:00:00Z'); INSERT INTO backend_job_deliveries(workspace_id,delivery_id,job_id,attempt_id,target_runtime_id,target_worker_id,state,created_at,updated_at) VALUES('space','delivery','legacy-job','legacy-job:attempt:1','runtime','target','pending','created','updated');") .unwrap(); - (conn, snapshot.digest) +} + +// Replay/resource checks use the current Store only after the full retained chain. +// Keep the focused partial DDL fixture above for the cutover's schema assertions. +fn legacy_content_job_database(path: &Path) -> Connection { + super::tests::prepare_retained_schema(path, 85); + let conn = Connection::open(path).unwrap(); + configure_sqlite(&conn).unwrap(); + conn.execute_batch("INSERT INTO accounts VALUES('owner','user','owner','Owner','created','updated'); + INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at) + VALUES('space','owner','Space','active','created','updated');").unwrap(); + seed_legacy_content_job(&conn); + conn } #[test] @@ -432,7 +449,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_ ] { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("content.db"); - let (conn, _) = legacy_content_database(&path); + let conn = legacy_content_job_database(&path); conn.execute( "UPDATE backend_job_attempts SET state=?1, runtime_id=CASE WHEN ?1='reserved' THEN NULL ELSE runtime_id END, @@ -441,11 +458,22 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_ ) .unwrap(); migrate_content_state_v85_to_v86(&conn).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 86); + let cutover: (String, String, Option) = conn.query_row( + "SELECT j.state,a.state,a.worker_cleanup_state FROM backend_jobs j JOIN backend_job_attempts a USING(workspace_id,job_id)", + [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ).unwrap(); + assert_eq!( + cutover, + if state == "reserved" { + ("failed".into(), "failed".into(), Some("completed".into())) + } else { + ("unknown".into(), "unknown".into(), Some("pending".into())) + } + ); + apply_migrations(&conn).unwrap(); drop(conn); - let store = SqliteWorkspaceStore { - db_executor: crate::db_executor::DbExecutor::shared(), - conn: Arc::new(Mutex::new(Connection::open(path).unwrap())), - }; + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let job = store .get_backend_job("space", "legacy-job") .unwrap() @@ -478,7 +506,7 @@ fn content_cutover_reserved_only_releases_resource_but_uncertain_dispatch_keeps_ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("content.db"); - let (conn, _) = legacy_content_database(&path); + let conn = legacy_content_job_database(&path); let result = serde_json::json!({"finding":"preserved"}); let (digest, encoded) = job::result_digest(&result, 4096).unwrap(); conn.execute("UPDATE backend_jobs SET state='completed',result_json=?1,result_digest=?2,completed_at='finished'",params![encoded,digest]).unwrap(); @@ -490,11 +518,23 @@ fn content_cutover_completed_result_cleanup_and_replay_remain_unchanged() { ) .unwrap(); migrate_content_state_v85_to_v86(&conn).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 86); + let cutover: (String, String, String, String) = conn.query_row( + "SELECT j.state,j.result_json,a.worker_cleanup_state,a.worker_cleanup_completed_at FROM backend_jobs j JOIN backend_job_attempts a USING(workspace_id,job_id)", + [], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), + ).unwrap(); + assert_eq!( + cutover, + ( + "completed".into(), + encoded, + "completed".into(), + "cleaned".into() + ) + ); + apply_migrations(&conn).unwrap(); drop(conn); - let store = SqliteWorkspaceStore { - db_executor: crate::db_executor::DbExecutor::shared(), - conn: Arc::new(Mutex::new(Connection::open(path).unwrap())), - }; + let store = SqliteWorkspaceStore::open_for_test(&path).unwrap(); let job = store .get_backend_job("space", "legacy-job") .unwrap() diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 0f9cd24a1..f657bac06 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -1326,6 +1326,13 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { Err("Session source permanent erasure is unavailable on this Runtime".into()) } + fn session_recording_capture( + &self, + _request: runtime_api::SessionRecordingCaptureRequest, + ) -> Result { + Err("committed recording capture is unavailable on this Runtime".into()) + } + fn session_material_catalog( &self, _request: runtime_api::SessionMaterialCatalogRequest, @@ -2336,6 +2343,21 @@ impl RuntimeRegistry { }) } + pub fn session_recording_capture( + &self, + runtime_id: &str, + request: runtime_api::SessionRecordingCaptureRequest, + ) -> Result { + validate_backend_identifier("runtime_id", runtime_id)?; + self.runtime(runtime_id)? + .session_recording_capture(request) + .map_err(|message| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: runtime_id.into(), + code: "session_recording_capture_failed".into(), + message, + }) + } + pub fn session_material_catalog( &self, runtime_id: &str, @@ -3441,6 +3463,19 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { }) } + fn session_recording_capture( + &self, + request: runtime_api::SessionRecordingCaptureRequest, + ) -> Result { + if request.workspace_id != self.workspace_id { + return Err("recording capture Workspace mismatch".into()); + } + let scope = RuntimeWorkspaceScope::new(&request.workspace_id, "embedded-backend"); + Ok(self + .runtime + .session_recording_capture_scoped(&scope, &request)) + } + fn session_material_catalog( &self, request: runtime_api::SessionMaterialCatalogRequest, @@ -5677,6 +5712,18 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { .map_err(|diagnostic| diagnostic.message) } + fn session_recording_capture( + &self, + request: runtime_api::SessionRecordingCaptureRequest, + ) -> Result { + self.run_runtime_api( + self.request_timeout, + MAX_REMOTE_RUNTIME_RESPONSE_BYTES, + move |client| async move { client.session_recording_capture(request).await }, + ) + .map_err(|diagnostic| diagnostic.message) + } + fn session_material_catalog( &self, request: runtime_api::SessionMaterialCatalogRequest, diff --git a/crates/workspace-server/src/implementation_result_tests.rs b/crates/workspace-server/src/implementation_result_tests.rs index c5a7cd7d2..a46555429 100644 --- a/crates/workspace-server/src/implementation_result_tests.rs +++ b/crates/workspace-server/src/implementation_result_tests.rs @@ -381,6 +381,10 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations; + DROP TRIGGER backend_job_input_retirement; + DROP TRIGGER backend_job_identity_immutable; + ALTER TABLE backend_jobs DROP COLUMN input_disposition; + UPDATE __yoi_schema_migrations SET version=102 WHERE version=103; DROP TABLE subject_create_recoveries; UPDATE __yoi_schema_migrations SET version=101 WHERE version=102; DROP TABLE subject_worker_releases; @@ -517,6 +521,10 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow // Reconstruct the prior composition before its new business capture existed. conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture; DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations; + DROP TRIGGER backend_job_input_retirement; + DROP TRIGGER backend_job_identity_immutable; + ALTER TABLE backend_jobs DROP COLUMN input_disposition; + UPDATE __yoi_schema_migrations SET version=102 WHERE version=103; DROP TABLE subject_create_recoveries; UPDATE __yoi_schema_migrations SET version=101 WHERE version=102; DROP TABLE subject_worker_releases; @@ -679,7 +687,11 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() { if historical { conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture; DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations; - DROP TABLE subject_create_recoveries; + DROP TRIGGER backend_job_input_retirement; + DROP TRIGGER backend_job_identity_immutable; + ALTER TABLE backend_jobs DROP COLUMN input_disposition; + UPDATE __yoi_schema_migrations SET version=102 WHERE version=103; + DROP TABLE subject_create_recoveries; UPDATE __yoi_schema_migrations SET version=101 WHERE version=102; DROP TABLE subject_worker_releases; UPDATE __yoi_schema_migrations SET version=100 WHERE version=101; diff --git a/crates/workspace-server/src/job_notification_routing.rs b/crates/workspace-server/src/job_notification_routing.rs index a5bf62efa..c3245a497 100644 --- a/crates/workspace-server/src/job_notification_routing.rs +++ b/crates/workspace-server/src/job_notification_routing.rs @@ -2,8 +2,8 @@ //! lifecycle, are the durable source. Fixed Worker recipients never follow leases. use super::*; use crate::backend_job::{ - BackendJobAttemptState, BackendJobDeliveryRecord, BackendJobDeliveryState, - BackendJobResultAcceptance, BackendJobState, + BackendJobAttemptState, BackendJobDeliveryRecord, BackendJobDeliveryState, BackendJobRequest, + BackendJobState, }; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -103,6 +103,32 @@ pub(in crate::store) fn install(conn: &Connection) -> Result<()> { Ok(()) } +struct NotificationResultSnapshot { + request: BackendJobRequest, + result: Option, + result_digest: Option, +} + +// The schema-96 import runs before later Job lifecycle migrations. Read only +// the retained result envelope, not the executable Job's current-schema columns. +fn read_result_snapshot( + conn: &Connection, + workspace: &str, + job: &str, +) -> Result { + let (request, result, result_digest): (String, Option, Option) = conn + .query_row( + "SELECT request_json, result_json, result_digest FROM backend_jobs WHERE workspace_id=?1 AND job_id=?2", + params![workspace, job], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + Ok(NotificationResultSnapshot { + request: decode(&request)?, + result: result.map(|value| decode(&value)).transpose()?, + result_digest, + }) +} + pub(in crate::store) fn capture( conn: &Connection, workspace: &str, @@ -115,8 +141,7 @@ pub(in crate::store) fn capture( attempt_id: attempt.into(), delivery_id: id.clone(), }; - let job = read_backend_job(conn, workspace, job)? - .ok_or_else(|| Error::Store("Job notification source missing".into()))?; + let job = read_result_snapshot(conn, workspace, job)?; let target = &job.request.notification_target; let facts = serde_json::json!({"principal_kind":if target.is_some(){Some("worker")}else{None}, "principal_id":null,"runtime_id":target.as_ref().map(|w|&w.runtime_id),"worker_id":target.as_ref().map(|w|&w.worker_id),"status":null,"case_state":"open","released":false,"backend_job":source,"recipient_kind":"fixed_worker","job_purpose":job.request.purpose,"result_digest":job.result_digest}); conn.execute("INSERT OR IGNORE INTO notification_sources(workspace_id,ticket_id,input_event_id,role,assignment_id,kind,facts_json) VALUES(?1,NULL,?2,'backend_job','','backend_job_result',?3)",params![workspace,id,json(&facts)?])?; @@ -290,22 +315,13 @@ fn import_legacy(conn: &Connection, workspace: &str, job_id: &str, attempt_id: & | "notification_too_large", ) => ("defer", NotificationReason::SourceUnavailable), _ => { - let job = read_backend_job(conn, workspace, job_id)? - .ok_or_else(|| Error::Store("Legacy Job result missing".into()))?; - let attempt = read_backend_job_attempt(conn, workspace, job_id, attempt_id)? - .ok_or_else(|| Error::Store("Legacy Job attempt missing".into()))?; - let acceptance = BackendJobResultAcceptance { - job, - attempt, - replayed: true, - }; - let facts = if acceptance.job.request.notification_target.as_ref() != Some(&old.target) - { + let snapshot = read_result_snapshot(conn, workspace, job_id)?; + let facts = if snapshot.request.notification_target.as_ref() != Some(&old.target) { JobNotificationFacts::Failed(NotificationFailureCode::PreparationFailed) } else { - legacy_result_facts(&acceptance) + legacy_result_facts(&snapshot, job_id, attempt_id) }; - let decision = policy(acceptance.job.request.notification_target.is_some(), &facts); + let decision = policy(snapshot.request.notification_target.is_some(), &facts); match facts { JobNotificationFacts::Result(body) => content = Some(body), JobNotificationFacts::Failed(code) => failure = Some(code), @@ -338,28 +354,26 @@ fn import_legacy(conn: &Connection, workspace: &str, job_id: &str, attempt_id: & Ok(()) } -fn legacy_result_facts(acceptance: &BackendJobResultAcceptance) -> JobNotificationFacts { - let content = if acceptance.job.request.purpose == crate::ticket_item_checker::PURPOSE { - let Some(result) = &acceptance.job.result else { +fn legacy_result_facts( + snapshot: &NotificationResultSnapshot, + job_id: &str, + attempt_id: &str, +) -> JobNotificationFacts { + let content = if snapshot.request.purpose == crate::ticket_item_checker::PURPOSE { + let Some(result) = &snapshot.result else { return JobNotificationFacts::Failed(NotificationFailureCode::InvalidResult); }; - match crate::ticket_item_checker::validate_result(&acceptance.job.request.input, result) { + match crate::ticket_item_checker::validate_result(&snapshot.request.input, result) { Ok(result) if result.findings.is_empty() => return JobNotificationFacts::NoFindings, Ok(_) => {} Err(_) => return JobNotificationFacts::Failed(NotificationFailureCode::InvalidResult), } - crate::ticket_item_checker::render_result(acceptance) + crate::ticket_item_checker::render_result(&snapshot.request.input, result) } else { Ok(format!( - "Backend Job `{}` completed attempt `{}` for `{}`. Durable result digest: {}.", - acceptance.job.request.job_id, - acceptance.attempt.attempt_id, - acceptance.job.request.purpose, - acceptance - .job - .result_digest - .as_deref() - .unwrap_or("unavailable") + "Backend Job `{job_id}` completed attempt `{attempt_id}` for `{}`. Durable result digest: {}.", + snapshot.request.purpose, + snapshot.result_digest.as_deref().unwrap_or("unavailable") )) }; match content { diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index 489e78728..30244ac7f 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -789,6 +789,7 @@ CREATE TABLE "worker_registry" ( FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE ); CREATE TABLE backend_jobs ( + input_disposition TEXT NOT NULL DEFAULT 'available' CHECK (input_disposition IN ('available','reclaimed','erased','invalidated')), workspace_id TEXT NOT NULL, job_id TEXT NOT NULL, purpose TEXT NOT NULL, @@ -1642,6 +1643,19 @@ CREATE TABLE IF NOT EXISTS ticket_assignment_work_releases ( FOREIGN KEY (workspace_id, assignment_id) REFERENCES ticket_worker_assignments(workspace_id, assignment_id) ON DELETE CASCADE ); + +CREATE TRIGGER backend_job_identity_immutable BEFORE UPDATE OF + workspace_id, job_id, purpose, input_digest, input_ref, resource_key, intent_fingerprint ON backend_jobs +BEGIN SELECT RAISE(ABORT, 'immutable Backend Job identity'); END; +CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON backend_jobs +WHEN ( + ((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated')) + OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased')) + AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') + AND json_type(NEW.request_json,'$.input') = 'null' +) IS NOT TRUE +BEGIN SELECT RAISE(ABORT, 'immutable Backend Job input retirement'); END; + -- Ticket case assignment work constraints (schema 91). DROP VIEW IF EXISTS ticket_active_worker_assignments; CREATE VIEW ticket_active_worker_assignments AS diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 809f7d203..6ca61c905 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -4635,11 +4635,79 @@ impl WorkspaceApi { &self, reservation: BackendJobReservation, ) -> Result { + // Reread even a terminal/stale reservation before deciding whether it can + // dispatch or replay: an explicit erasure may have retired its saved input. + let journal_gate = reservation + .job + .request + .grants + .subjektiv_journal + .as_ref() + .map(|grant| subjektiv_journal::lifecycle_lock(self, &grant.subject_id)) + .transpose()?; + let _journal_guard = journal_gate + .as_ref() + .map(|gate| { + gate.lock() + .map_err(|_| Error::Store("Journal lifecycle lock poisoned".into())) + }) + .transpose()?; + let current_job = self + .store + .get_backend_job(&self.config.workspace_id, &reservation.job.request.job_id)? + .ok_or_else(|| Error::Store("reserved Backend Job disappeared".into()))?; + let current_attempt = self + .store + .get_backend_job_attempt( + &self.config.workspace_id, + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + )? + .ok_or_else(|| Error::Store("reserved Backend attempt disappeared".into()))?; + let reservation = BackendJobReservation { + job: current_job, + attempt: current_attempt, + replayed: reservation.replayed, + resource_reused: reservation.resource_reused, + }; + if matches!( + reservation.job.input_disposition, + crate::backend_job::JobInputDisposition::Erased + | crate::backend_job::JobInputDisposition::Invalidated + ) { + return Err(Error::WorkspacePermissionDenied( + "erased or invalidated Backend Job input cannot dispatch".into(), + )); + } if reservation.job.state != BackendJobState::Pending || reservation.attempt.state != BackendJobAttemptState::Reserved || reservation.attempt.worker.is_some() { - return Ok(reservation); + return Ok(BackendJobReservation { + replayed: true, + ..reservation + }); + } + reservation.job.load_executable_request()?; + if subjektiv_journal::migrate_reserved_legacy(self, &reservation)? { + let job = self + .store + .get_backend_job(self.workspace_id(), &reservation.job.request.job_id)? + .ok_or_else(|| Error::Store("retired legacy Job missing".into()))?; + let attempt = self + .store + .get_backend_job_attempt( + self.workspace_id(), + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + )? + .ok_or_else(|| Error::Store("retired legacy attempt missing".into()))?; + return Ok(BackendJobReservation { + job, + attempt, + replayed: reservation.replayed, + resource_reused: reservation.resource_reused, + }); } subjektiv_journal::prepare(self, &reservation)?; let (reservation, claimed) = self.store.claim_backend_job_attempt_dispatch( @@ -4844,6 +4912,7 @@ impl WorkspaceApi { .to_string(), }); } + subjektiv_journal::attest_delivery(self, &reservation)?; let job = self .store .get_backend_job(&self.config.workspace_id, &request.job_id)? @@ -5220,6 +5289,16 @@ impl WorkspaceApi { subjektiv_journal::close(self, &job.request, ¤t)?; if current.state == BackendJobAttemptState::Completed { subjektiv_journal::drain_after_completion(self, &job.request); + if job.input_disposition + == crate::backend_job::JobInputDisposition::Available + { + self.store.reclaim_backend_job_input( + self.workspace_id(), + job_id, + &job.input_digest, + crate::backend_job::JobInputRetirementReason::Completed, + )?; + } } } } diff --git a/crates/workspace-server/src/server/subjektiv_journal.rs b/crates/workspace-server/src/server/subjektiv_journal.rs index 5abfd49b2..fccee17a6 100644 --- a/crates/workspace-server/src/server/subjektiv_journal.rs +++ b/crates/workspace-server/src/server/subjektiv_journal.rs @@ -4,7 +4,7 @@ use super::*; use crate::backend_job::{ BackendJobGrants, BackendJobRecord, SubjektivJournalGrant, journal_serialization_key, }; -use subjektiv::journal::recording::{JournalMaterial, select_materials_bounded}; +use subjektiv::journal::recording::JournalInput; use subjektiv::journal::transport::*; use subjektiv::journal::{self, *}; @@ -92,20 +92,82 @@ pub(super) fn prepare(api: &WorkspaceApi, reservation: &BackendJobReservation) - }; let store = store(api)?; let host = host(api, &store, grant)?; - store - .prepare_journal_generation( + let input = saved_input(&reservation.job.request, grant)?; + let g = store + .prepare_journal_input( &host, - &grant.journal_id, + &input, subjektiv::api::JobAttemptBinding { job_id: reservation.job.request.job_id.clone(), attempt_id: reservation.attempt.attempt_id.clone(), }, - grant.selected_ranges.clone(), ) .map_err(domain)?; + input.validate_generation(&g).map_err(domain)?; Ok(()) } +fn legacy_descriptor(request: &BackendJobRequest) -> bool { + let Some(grant) = request.grants.subjektiv_journal.as_ref() else { + return false; + }; + let Some(object) = request.input.as_object() else { + return false; + }; + object.len() == 2 + && object.get("journal_id").and_then(|v| v.as_str()) == Some(&grant.journal_id) + && object.get("selected_ranges") + == serde_json::to_value(&grant.selected_ranges).ok().as_ref() +} + +/// Retire only the explicitly recognized descriptor wire. Malformed body snapshots are +/// failures, not migration requests and never a reason to reselect from current Sessions. +pub(super) fn migrate_reserved_legacy( + api: &WorkspaceApi, + reservation: &BackendJobReservation, +) -> Result { + if !legacy_descriptor(&reservation.job.request) { + return Ok(false); + } + reservation.job.load_executable_request()?; + close(api, &reservation.job.request, &reservation.attempt)?; + api.store.finish_backend_job_attempt( + api.workspace_id(), + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + BackendJobAttemptState::Failed, + "journal_descriptor_invalidated", + "Descriptor-only Journal input retired; saved checkpoints retained", + &Utc::now().to_rfc3339(), + )?; + api.store.reclaim_backend_job_input( + api.workspace_id(), + &reservation.job.request.job_id, + &reservation.job.input_digest, + crate::backend_job::JobInputRetirementReason::LegacyInvalidation, + )?; + let queued = api.clone(); + let request = reservation.job.request.clone(); + api.host_executors.host()?.dispatch(move || { + drain_after_completion(&queued, &request); + })?; + Ok(true) +} + +fn saved_input(request: &BackendJobRequest, grant: &SubjektivJournalGrant) -> Result { + let input = JournalInput::decode(&request.input).map_err(domain)?; + if input.subject_id != grant.subject_id + || input.journal_id != grant.journal_id + || input.capture.session_id != grant.session_id + || input.selected_ranges != grant.selected_ranges + { + return Err(Error::WorkspacePermissionDenied( + "Journal saved input/capability mismatch".into(), + )); + } + Ok(input) +} + fn live( api: &WorkspaceApi, worker: &RuntimeWorkerRef, @@ -125,6 +187,7 @@ fn live( .store .get_backend_job(api.workspace_id(), &binding.job_id)? .ok_or_else(|| Error::WorkspacePermissionDenied("Journal Job missing".into()))?; + job.load_executable_request()?; let attempt = api .store .get_backend_job_attempt(api.workspace_id(), &binding.job_id, &binding.attempt_id)? @@ -180,15 +243,44 @@ pub(super) fn execute( worker: &RuntimeWorkerRef, request: JournalJobRequest, ) -> Result { + let (grant, _, _) = live(api, worker)?; + let gate = lifecycle_lock(api, &grant.subject_id)?; + let _guard = gate + .lock() + .map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?; + // Dispatch holds this same gate until matching Runtime acceptance is attested. + // Revalidate after waiting: erasure, cancellation or replacement may have won. let (grant, job, attempt) = live(api, worker)?; let store = store(api)?; let host = host(api, &store, &grant)?; let g = generation(&store, &host, &grant, &job, &attempt)?; + let input = saved_input(&job.request, &grant)?; + input.validate_generation(&g).map_err(domain)?; let ctx = JournalJobContext::validated(&store, &host, &g.id, g.job.clone()).map_err(domain)?; - journal::service::execute_bounded(&store, &ctx, request, |source, start, limit| { - read_source(api, &grant, source, start, limit) - .map_err(|e| JournalError::Conflict(e.to_string())) - }) + journal::service::execute_snapshot(&store, &ctx, &attempt.input_digest, request).map_err(domain) +} + +/// Called only after matching tracked Runtime acceptance of the exact rendered saved input. +/// The durable request and Worker binding alone do not establish body delivery. +pub(super) fn attest_delivery( + api: &WorkspaceApi, + reservation: &BackendJobReservation, +) -> Result<()> { + let Some(grant) = &reservation.job.request.grants.subjektiv_journal else { + return Ok(()); + }; + let input = saved_input(&reservation.job.request, grant)?; + let store = store(api)?; + let host = host(api, &store, grant)?; + let g = generation(&store, &host, grant, &reservation.job, &reservation.attempt)?; + input.validate_generation(&g).map_err(domain)?; + let ctx = JournalJobContext::validated(&store, &host, &g.id, g.job.clone()).map_err(domain)?; + journal::service::attest_input_delivery( + &store, + &ctx, + &reservation.attempt.input_digest, + &input.selected_ranges, + ) .map_err(domain) } @@ -196,21 +288,29 @@ pub(super) fn finish( api: &WorkspaceApi, worker: &RuntimeWorkerRef, ) -> Result { + let (grant, _, _) = live(api, worker)?; + let gate = lifecycle_lock(api, &grant.subject_id)?; + let _guard = gate + .lock() + .map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?; + // Dispatch holds this same gate until matching Runtime acceptance is attested. + // Revalidate after waiting: erasure, cancellation or replacement may have won. let (grant, job, attempt) = live(api, worker)?; let store = store(api)?; let host = host(api, &store, &grant)?; + let input = saved_input(&job.request, &grant)?; // prepare is an exact binding lookup too, including the saved completion of a closed generation. let g = store - .prepare_journal_generation( + .prepare_journal_input( &host, - &grant.journal_id, + &input, subjektiv::api::JobAttemptBinding { job_id: job.request.job_id, attempt_id: attempt.attempt_id, }, - grant.selected_ranges.clone(), ) .map_err(domain)?; + input.validate_generation(&g).map_err(domain)?; let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone()) .map_err(domain)?; journal::service::finish(&store, &ctx).map_err(domain) @@ -277,21 +377,6 @@ pub(super) fn close( Ok(()) } -fn read_source( - api: &WorkspaceApi, - grant: &SubjektivJournalGrant, - source: &JournalSource, - start: u64, - limit: usize, -) -> Result { - let store = store(api)?; - let attr = store - .session_attribution(&grant.session_id) - .map_err(|e| Error::Store(e.to_string()))? - .ok_or_else(|| Error::WorkspacePermissionDenied("Session attribution absent".into()))?; - read_canonical(&material_reader(api, &attr)?, source, start, limit) -} - pub(super) fn bind_host_session( api: &WorkspaceApi, attr: crate::subjektiv::SubjectSessionAttribution, @@ -524,22 +609,63 @@ fn dispatch( .close_journal_generation(&host, &g.id, &g.job) .map_err(domain)?; } - let reader = material_reader(api, &attribution)?; - let materials = select_materials_bounded( - &store, - &host, - journal_id, - 96 * 1024, - |source, start, limit| { - read_canonical(&reader, source, start, limit) - .map_err(|e| JournalError::Conflict(e.to_string())) - }, - ) + let mut after = None; + loop { + let jobs = api.store.list_backend_journal_jobs( + api.workspace_id(), + &attribution.session_id, + after.as_deref(), + 100, + )?; + for job in &jobs { + if job + .request + .grants + .subjektiv_journal + .as_ref() + .is_none_or(|g| g.journal_id != journal_id) + || job.input_disposition != crate::backend_job::JobInputDisposition::Available + { + continue; + } + if job.state == BackendJobState::Pending || job.state == BackendJobState::Unknown { + return Ok(()); + } + if job.state == BackendJobState::Failed && !legacy_descriptor(&job.request) { + job.load_executable_request()?; + saved_input( + &job.request, + job.request + .grants + .subjektiv_journal + .as_ref() + .expect("checked Journal grant"), + )?; + let reservation = api.store.reserve_backend_job_retry( + api.workspace_id(), + &job.request.job_id, + &Utc::now().to_rfc3339(), + )?; + return queue_reservation(api, reservation); + } + } + if jobs.len() < 100 { + break; + } + after = jobs.last().map(|j| j.request.job_id.clone()); + } + let input = JournalInput::build(&store, &host, journal_id, |order, offset, budget| { + capture_input(api, &attribution, order, offset, budget) + .map_err(|e| JournalError::Conflict(e.to_string())) + }) .map_err(domain)?; - if materials.is_empty() { + if input.selected_ranges.is_empty() { return Ok(()); } - let request = recording_request(&host, attribution, journal_id, materials)?; + store + .accept_journal_sources(&host, "Session activity", &input.selected_ranges) + .map_err(domain)?; + let request = recording_request(&host, attribution, journal_id, input)?; // The intent is saved before generation binding or dispatch. If binding/dispatch fails, // Reserved recovery repeats this exact intent, never creates a replacement blindly. api.resolve_backend_job_profile(&request)?; @@ -548,6 +674,10 @@ fn dispatch( &request, &Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true), )?; + queue_reservation(api, reservation) +} + +fn queue_reservation(api: &WorkspaceApi, reservation: BackendJobReservation) -> Result<()> { let queued = api.clone(); if let Err(error)=api.host_executors.host()?.dispatch(move || { if let Err(error)=queued.dispatch_backend_job_reservation(reservation) { @@ -578,33 +708,21 @@ fn admit_sources( )); } let mut id = store.journal_for_session(&host).map_err(domain)?; - let reader = material_reader(api, &attribution)?; - if reader.not_materialized { - if store.journal_retention_required(&host).map_err(domain)? { - return Err(Error::RepositoryConflict( - "Admitted Journal original Session is not materialized".into(), - )); - } - retain_source( - api.runtime.as_ref(), - api.workspace_id(), - &attribution, - material_source(api, &attribution)?, - true, - )?; - return Ok(JournalRecordingResponse::NoNewSources); - } + let (order, offset) = if let Some(id) = id.as_deref() { + let journal = store.read_journal(&host, id).map_err(domain)?; + (journal.process_order, journal.process_byte) + } else { + (0, 0) + }; + let capture = capture_input(api, &attribution, order, offset, 48 * 1024)?; retain_source( api.runtime.as_ref(), api.workspace_id(), &attribution, material_source(api, &attribution)?, - false, + capture.segment_id == "not-materialized", )?; - let sources: Vec<_> = reader - .descriptors() - .map(|d| source_from_descriptor(&attribution.session_id, d)) - .collect(); + let sources = journal::recording::capture_ranges(&capture).map_err(domain)?; for batch in sources.chunks(MAX_JOURNAL_RANGES) { id = Some( store @@ -995,7 +1113,7 @@ pub(super) fn release_subject( { // Legacy saved-before-admission Sessions are not optional. Prove canonical // source identity first, then bind only this original Host-attributed Session. - let _canonical = material_reader(api, &attr)?; + let _canonical = capture_input(api, &attr, 0, 0, 48 * 1024)?; let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?; if host @@ -1281,80 +1399,6 @@ fn material_source( }, ) } -fn material_reader<'a>( - api: &'a WorkspaceApi, - attribution: &crate::subjektiv::SubjectSessionAttribution, -) -> Result> { - MaterialReader::open( - api.runtime.as_ref(), - api.workspace_id(), - &attribution.runtime_id, - material_source(api, attribution)?, - &attribution.session_id, - ) -} -fn source_from_descriptor( - session: &str, - d: &session_store::public_material::PublicMaterialDescriptor, -) -> JournalSourceRange { - JournalSourceRange { - source: JournalSource { - session_id: session.into(), - pointer: JournalSourcePointer { - segment_id: d.identity.segment_id, - entry_ref: d.identity.entry_ref.clone(), - }, - order: d.canonical_order, - content_sha256: d.sha256.clone(), - total_bytes: d.total_bytes, - }, - bytes: JournalByteRange { - start: 0, - end: d.total_bytes, - }, - } -} -fn read_canonical( - reader: &MaterialReader<'_>, - source: &JournalSource, - start: u64, - limit: usize, -) -> Result { - if reader.session_id != source.session_id { - return Err(Error::WorkspacePermissionDenied( - "Journal source Session mismatch".into(), - )); - } - let descriptor = reader - .descriptors() - .find(|d| source_from_descriptor(&source.session_id, d).source == *source) - .ok_or_else(|| { - Error::Store("Journal original public source changed or unavailable".into()) - })?; - // The reader streams the complete canonical digest while returning only this bounded window. - let page = reader - .read_content(descriptor, start, limit) - .map_err(|e| Error::Store(e.to_string()))?; - if page.sha256 != source.content_sha256 - || page.total_bytes != source.total_bytes - || page.next_offset != start.saturating_add(page.content.len() as u64) - { - return Err(Error::RepositoryConflict( - "Journal canonical source identity changed".into(), - )); - } - Ok(JournalMaterial { - range: JournalSourceRange { - source: source.clone(), - bytes: JournalByteRange { - start, - end: page.next_offset, - }, - }, - text: page.content, - }) -} - /// Close the first-run save/notification gap: Runtime can commit an initial turn before /// the create handler has finalized the Backend registry required by recording callbacks. /// Reconcile only after creation succeeds, using the ordinary current-owner/Feature checks. @@ -1475,7 +1519,7 @@ pub(super) fn recording_request( host: &JournalHostContext, attribution: crate::subjektiv::SubjectSessionAttribution, journal_id: &str, - materials: Vec, + input: JournalInput, ) -> Result { let binding_id = host .writer_binding_id() @@ -1488,18 +1532,88 @@ pub(super) fn recording_request( subject_id: attribution.subject_id, session_id: attribution.session_id, journal_id: journal_id.into(), - selected_ranges: materials.iter().map(|m| m.range.clone()).collect(), + selected_ranges: input.selected_ranges.clone(), }; - Ok(BackendJobRequest { - job_id:format!("journal-recording:{}",Uuid::new_v4()),purpose:"subjektiv_journal".into(), - input_ref:format!("subjektiv://{}/journal/{journal_id}",grant.subject_id), - input:serde_json::json!({"journal_id":journal_id,"selected_ranges":grant.selected_ranges}), - instruction:"Record ONLY this frozen original public selected_ranges prefix. The input contains descriptors, NOT source bodies. First ReadJournalState and use bounded ReadJournalText Source until every byte of every selected range is returned; record operations are refused until then. ReadJournalText also exposes initial/current position/open text. Sequentially save CreateJournalEntry/RewriteJournalEntry/FixJournalEntry, update current position and CheckpointJournal using retained record receipts. Never infer event completion from turn/task/job/compaction boundaries. Preserve exact operation ID/full input after unknown outcomes. SubmitBackendJobResult only after the full selected prefix has a saved checkpoint; no aggregate JournalRecordResult.".into(), - profile:"builtin:subjektiv-journal-recording".into(), - grants:BackendJobGrants {subjektiv_journal:Some(grant),..Default::default()}, - serialization_key:Some(journal_serialization_key(journal_id)),source_worker:None,notification_target:None, - limits:crate::backend_job::BackendJobLimits {timeout_seconds:600,..Default::default()}, - }) + let request = BackendJobRequest { + job_id: format!("journal-recording:{}",Uuid::new_v4()), purpose: "subjektiv_journal".into(), + input_ref: format!("subjektiv://{}/journal/{journal_id}",grant.subject_id), + input: serde_json::to_value(input).map_err(|e| Error::Store(e.to_string()))?, + instruction: "Record ONLY the immutable body-bearing capture supplied in this input. These exact canonical source bytes, identities and ranges are the whole input for this Job; do not fetch original Sessions or use ReadJournalText Source. First ReadJournalState; initial/current Journal position/open text remain separately paged by ReadJournalText. Sequentially save CreateJournalEntry/RewriteJournalEntry/FixJournalEntry, current position and receipt-backed CheckpointJournal. Input delivery is not understanding or reflection. Never infer event completion from turn/task/job/compaction boundaries. Preserve exact operation ID/full input after Unknown. SubmitBackendJobResult only after the complete selected prefix has a saved checkpoint; no aggregate JournalRecordResult.".into(), + profile: "builtin:subjektiv-journal-recording".into(), + grants: BackendJobGrants { subjektiv_journal: Some(grant), ..Default::default() }, + serialization_key: Some(journal_serialization_key(journal_id)), source_worker: None, notification_target: None, + limits: crate::backend_job::BackendJobLimits { timeout_seconds: 600, ..Default::default() }, + }; + saved_input( + &request, + request + .grants + .subjektiv_journal + .as_ref() + .expect("constructed grant"), + )?; + request.validate()?; + Ok(request) +} + +pub(super) fn capture_input( + api: &WorkspaceApi, + attr: &crate::subjektiv::SubjectSessionAttribution, + order: u64, + offset: u64, + budget: usize, +) -> Result { + match api + .runtime + .session_recording_capture( + &attr.runtime_id, + runtime_api::SessionRecordingCaptureRequest { + workspace_id: api.workspace_id().into(), + source: material_source(api, attr)?, + expected_session_id: attr.session_id.clone(), + start: session_store::capture::RecordingPosition { + canonical_order: order, + byte_offset: offset, + }, + max_entries: MAX_JOURNAL_RANGES, + max_serialized_bytes: budget, + }, + ) + .map_err(RuntimeRegistryError::into_error)? + { + runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => { + if slice.capture.session_id != attr.session_id + || serde_json::to_vec(&slice) + .map_err(|e| Error::Store(e.to_string()))? + .len() + > budget + { + return Err(Error::RepositoryConflict( + "recording capture identity/budget mismatch".into(), + )); + } + Ok(slice.capture) + } + runtime_api::SessionRecordingCaptureAvailability::NotMaterialized { session_id } + if session_id == attr.session_id && order == 0 && offset == 0 => + { + Ok(session_store::capture::CaptureDto { + version: 1, + session_id, + segment_id: "not-materialized".into(), + view_entry_count: 0, + entries: vec![], + }) + } + runtime_api::SessionRecordingCaptureAvailability::Unavailable { reason, .. } => { + Err(Error::Store(format!( + "committed Journal capture unavailable ({reason:?}); no latest-source fallback" + ))) + } + _ => Err(Error::Store( + "committed Journal capture identity mismatch; no latest-source fallback".into(), + )), + } } fn removal_attribution( @@ -1575,6 +1689,16 @@ fn finish_erased_release( "explicit erase tombstone missing".into(), )); } + for session in store + .subject_session_attributions(&attr.subject_id) + .map_err(|e| Error::Store(e.to_string()))? + .into_iter() + .filter(|session| { + session.runtime_id == attr.runtime_id && session.worker_id == attr.worker_id + }) + { + require_erased_job_copies(api, &session.session_id)?; + } // Pre-binding bodies have no write token. This stable receipt label is not a writer // grant: the domain erase operation records an inactive lifetime without binding Sessions. let erased_body_id = format!("erased-body:{}", attr.worker_id); @@ -1835,183 +1959,6 @@ pub(super) async fn finish_route( } /// A bounded Runtime transport view, not a second history index or a local-path fallback. -/// Reopening it observes a new catalog; each generation still compares its frozen source identity. -struct MaterialReader<'a> { - runtime: &'a RuntimeRegistry, - workspace_id: &'a str, - runtime_id: String, - source: runtime_api::SessionPublicSource, - session_id: String, - not_materialized: bool, - descriptors: Vec, -} -fn material_descriptor( - d: runtime_api::SessionMaterialDescriptor, -) -> Result { - Ok(session_store::public_material::PublicMaterialDescriptor { - identity: session_store::public_material::PublicMaterialIdentity { - segment_id: Uuid::parse_str(&d.segment_id).map_err(|e| Error::Store(e.to_string()))?, - entry_ref: d.entry_ref, - }, - canonical_order: d.canonical_order, - origin: serde_json::from_str(&d.origin_json).map_err(|e| Error::Store(e.to_string()))?, - sha256: d.sha256, - total_bytes: d.total_bytes, - }) -} -impl<'a> MaterialReader<'a> { - fn open( - runtime: &'a RuntimeRegistry, - workspace_id: &'a str, - runtime_id: &str, - source: runtime_api::SessionPublicSource, - session_id: &str, - ) -> Result { - let mut descriptors = Vec::new(); - let mut offset = 0; - let mut condition = None; - loop { - let page = match runtime - .session_material_catalog( - runtime_id, - runtime_api::SessionMaterialCatalogRequest { - workspace_id: workspace_id.into(), - source: source.clone(), - expected_session_id: session_id.into(), - expected_catalog_sha256: condition.clone(), - offset, - limit: 100, - }, - ) - .map_err(|e| e.into_error())? - { - runtime_api::SessionMaterialCatalogAvailability::NotMaterialized { - session_id: observed, - } => { - if observed != session_id || offset != 0 || condition.is_some() { - return Err(Error::RepositoryConflict( - "Journal Session materialization proof changed during catalog read" - .into(), - )); - } - return Ok(Self { - runtime, - workspace_id, - runtime_id: runtime_id.into(), - source, - session_id: session_id.into(), - not_materialized: true, - descriptors, - }); - } - runtime_api::SessionMaterialCatalogAvailability::Page { page } => page, - runtime_api::SessionMaterialCatalogAvailability::Unavailable { - reason, - message, - } => { - return Err(Error::Store(format!( - "Journal original material unavailable ({reason:?}): {message}" - ))); - } - }; - if page.session_id != session_id - || condition - .as_ref() - .is_some_and(|c| c != &page.catalog_sha256) - || page.descriptors.len() > 100 - || descriptors.len().saturating_add(page.descriptors.len()) > 1_000_000 - { - return Err(Error::RepositoryConflict( - "Journal canonical Session catalog changed or exceeded its bound".into(), - )); - } - condition = Some(page.catalog_sha256); - for d in page.descriptors { - let d = material_descriptor(d)?; - if d.canonical_order != descriptors.len() as u64 { - return Err(Error::RepositoryConflict( - "Journal canonical catalog order is discontinuous".into(), - )); - } - descriptors.push(d); - } - match page.next_offset { - Some(next) if next > offset && next == descriptors.len() as u64 => offset = next, - None => break, - _ => { - return Err(Error::RepositoryConflict( - "Journal catalog continuation did not advance".into(), - )); - } - } - } - Ok(Self { - runtime, - workspace_id, - runtime_id: runtime_id.into(), - source, - session_id: session_id.into(), - not_materialized: false, - descriptors, - }) - } - fn descriptors( - &self, - ) -> impl Iterator { - self.descriptors.iter() - } - fn read_content( - &self, - descriptor: &session_store::public_material::PublicMaterialDescriptor, - offset: u64, - limit: usize, - ) -> Result { - let page = match self - .runtime - .session_material_read( - &self.runtime_id, - runtime_api::SessionMaterialReadRequest { - workspace_id: self.workspace_id.into(), - source: self.source.clone(), - expected_session_id: self.session_id.clone(), - expected_descriptor: runtime_api::SessionMaterialDescriptor { - segment_id: descriptor.identity.segment_id.to_string(), - entry_ref: descriptor.identity.entry_ref.clone(), - canonical_order: descriptor.canonical_order, - origin_json: serde_json::to_string(&descriptor.origin) - .map_err(|e| Error::Store(e.to_string()))?, - sha256: descriptor.sha256.clone(), - total_bytes: descriptor.total_bytes, - }, - byte_offset: offset, - max_content_bytes: limit.clamp(4, 16 * 1024), - }, - ) - .map_err(|e| e.into_error())? - { - runtime_api::SessionMaterialReadAvailability::Page { page } => page, - runtime_api::SessionMaterialReadAvailability::Unavailable { reason, message } => { - return Err(Error::Store(format!( - "Journal original material read unavailable ({reason:?}): {message}" - ))); - } - }; - if material_descriptor(page.descriptor)? != *descriptor - || page.content.len() > limit.clamp(4, 16 * 1024) - { - return Err(Error::RepositoryConflict( - "Journal canonical material identity changed".into(), - )); - } - Ok(session_store::public_material::PublicMaterialContent { - content: page.content, - next_offset: page.next_offset, - total_bytes: descriptor.total_bytes, - sha256: descriptor.sha256.clone(), - }) - } -} - fn erase_source( runtime: &RuntimeRegistry, workspace: &str, @@ -2145,17 +2092,29 @@ pub(super) fn reconcile_unknown( let host = host(api, &store, grant)?; let erased = store.journal_session_erased(&host).map_err(domain)?; if !erased { - let g = store - .prepare_journal_generation( - &host, - &grant.journal_id, - subjektiv::api::JobAttemptBinding { - job_id: job.request.job_id.clone(), - attempt_id: attempt.attempt_id.clone(), - }, - grant.selected_ranges.clone(), - ) - .map_err(domain)?; + job.load_executable_request()?; + let binding = subjektiv::api::JobAttemptBinding { + job_id: job.request.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + }; + let g = if legacy_descriptor(&job.request) { + // Only reconcile already-saved legacy receipts; never redispatch descriptors. + store + .prepare_journal_generation( + &host, + &grant.journal_id, + binding, + grant.selected_ranges.clone(), + ) + .map_err(domain)? + } else { + let input = saved_input(&job.request, grant)?; + let generation = store + .prepare_journal_input(&host, &input, binding) + .map_err(domain)?; + input.validate_generation(&generation).map_err(domain)?; + generation + }; let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone()) .map_err(domain)?; match journal::service::finish(&store, &ctx) { @@ -2187,6 +2146,14 @@ pub(super) fn reconcile_unknown( ); if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed) { drain_after_completion(api, &job.request); + if job.input_disposition == crate::backend_job::JobInputDisposition::Available { + api.store.reclaim_backend_job_input( + api.workspace_id(), + &job.request.job_id, + &job.input_digest, + crate::backend_job::JobInputRetirementReason::Completed, + )?; + } } else { api.schedule_backend_job_worker_cleanup(¤t); } @@ -2226,8 +2193,8 @@ pub(super) fn reconcile_unknown( &Utc::now().to_rfc3339(), )?; if !erased { - // The old immutable input may include reflected bytes. Select the remaining - // prefix afresh only AFTER definitive lifecycle/domain fences; never replace Unknown. + // Resume the same saved body input only after canonical execution/domain fences. + // dispatch selects its retained intent before it can capture any newer source. let attr = store .session_attribution(&grant.session_id) .map_err(|e| Error::Store(e.to_string()))? @@ -2256,6 +2223,30 @@ fn reconcile_session_jobs( .store .get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)? .ok_or_else(|| Error::Store("Journal current attempt missing".into()))?; + if !erased + && legacy_descriptor(&job.request) + && job.input_disposition == crate::backend_job::JobInputDisposition::Available + { + job.load_executable_request()?; + close(api, &job.request, &attempt)?; + if !attempt.state.terminal() { + attempt = api.store.finish_backend_job_attempt( + api.workspace_id(), + &job.request.job_id, + &id, + BackendJobAttemptState::Failed, + "journal_descriptor_invalidated", + "Descriptor-only Journal input retired; saved checkpoints retained", + &Utc::now().to_rfc3339(), + )?; + } + api.store.reclaim_backend_job_input( + api.workspace_id(), + &job.request.job_id, + &job.input_digest, + crate::backend_job::JobInputRetirementReason::LegacyInvalidation, + )?; + } if erased && !attempt.state.terminal() { // The domain tombstone precedes this cancellation and also fences any // crash window before it. No erased Job can prepare or use tools again. @@ -2269,6 +2260,30 @@ fn reconcile_session_jobs( &Utc::now().to_rfc3339(), )?; } + if erased { + api.store.reclaim_backend_job_input( + api.workspace_id(), + &job.request.job_id, + &job.input_digest, + crate::backend_job::JobInputRetirementReason::ExplicitErasure, + )?; + // Every historical execution can own a delivered copy, not just the current + // attempt or an unacknowledged result. Cleanup is canonical Purge and retryable. + for number in 1..=job.current_attempt { + let historical_id = crate::backend_job::attempt_id(&job.request.job_id, number); + let historical = api + .store + .get_backend_job_attempt( + api.workspace_id(), + &job.request.job_id, + &historical_id, + )? + .ok_or_else(|| Error::Store("historical Journal attempt missing".into()))?; + if historical.state.terminal() { + api.schedule_backend_job_worker_cleanup(&historical); + } + } + } if attempt.state == BackendJobAttemptState::Unknown { reconcile_unknown(api, job, &attempt)?; } @@ -2284,6 +2299,41 @@ fn reconcile_session_jobs( Ok(()) } +fn require_erased_job_copies(api: &WorkspaceApi, session: &str) -> Result<()> { + let mut after = None; + loop { + let jobs = api.store.list_backend_journal_jobs( + api.workspace_id(), + session, + after.as_deref(), + 100, + )?; + for job in &jobs { + if job.input_disposition != crate::backend_job::JobInputDisposition::Erased { + return Err(Error::RepositoryConflict( + "Journal input erasure remains incomplete".into(), + )); + } + for number in 1..=job.current_attempt { + let id = crate::backend_job::attempt_id(&job.request.job_id, number); + let attempt = api + .store + .get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)? + .ok_or_else(|| Error::Store("erased Journal attempt missing".into()))?; + if attempt.worker.is_some() + && attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) + { + return Err(Error::RepositoryConflict("Journal delivered-copy Purge remains pending; retry the same erasure after cleanup".into())); + } + } + } + if jobs.len() < 100 { + return Ok(()); + } + after = jobs.last().map(|j| j.request.job_id.clone()); + } +} + impl WorkspaceApi { /// Host-only erasure saves the domain fence, cancels Jobs, then erases originals. /// Runtime requires a stopped source Worker. Physical failure leaves the domain @@ -2301,6 +2351,10 @@ impl WorkspaceApi { .ok_or_else(|| { Error::WorkspacePermissionDenied("Session attribution missing".into()) })?; + let gate = lifecycle_lock(self, &attr.subject_id)?; + let _guard = gate + .lock() + .map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?; let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?; store .erase_journal_session(&host, request_id) @@ -2311,7 +2365,8 @@ impl WorkspaceApi { self.workspace_id(), &attr, material_source(self, &attr)?, - ) + )?; + require_erased_job_copies(self, &attr.session_id) } } diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs index 4cc4b5c24..72ff4d8cc 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_connected_tests.rs @@ -14,10 +14,33 @@ use std::sync::{Arc, Mutex}; #[derive(Clone, Default)] struct ConnectedJournalModel { dialogs: Arc>>, + journal_inputs: Arc>>, } impl ConnectedJournalModel { fn journal_response(request: &Request) -> Vec { + // Consume the actual Controller-rendered input, not a fixture generation or + // a Source tool response. Decode also checks complete bodies/ranges/digests. + let envelope = request + .items + .iter() + .filter_map(Item::as_text) + .find(|text| text.lines().any(|line| line.starts_with("input_json: "))) + .expect("rendered immutable Backend Job envelope"); + let input_json = envelope + .lines() + .find_map(|line| line.strip_prefix("input_json: ")) + .unwrap(); + let input_value: Value = serde_json::from_str(input_json).unwrap(); + let frozen = subjektiv::journal::recording::JournalInput::decode(&input_value).unwrap(); + assert_eq!( + serde_json::to_string(&input_value).unwrap(), + input_json, + "render preserves the exact persisted JSON bytes" + ); + assert!(frozen.capture.entries.iter().all(|entry| matches!( + &entry.body, session_store::capture::CaptureBody::CanonicalPublic { text, .. } if !text.is_empty()))); + let ranges = serde_json::to_value(&frozen.selected_ranges).unwrap(); // Runtime Controllers deliberately retain alias-routed core schemas for their // lifetime even with zero attachments. Verify invocation authority, not schema // absence (standalone Internal Jobs use a different composition). @@ -89,7 +112,7 @@ impl ConnectedJournalModel { "request_id":format!("checkpoint:{}",state["value"]["generation_id"].as_str().unwrap()), "expected_state":position["value"]["state_token"], "action":{"action":"checkpoint","record_operation_ids":[format!("record:{}",state["value"]["generation_id"].as_str().unwrap())]}, - "covered_ranges":state["value"]["selected_ranges"], + "covered_ranges":ranges, }), ) } else if let Some(record) = result("record") { @@ -106,41 +129,14 @@ impl ConnectedJournalModel { ) } else if let Some(state) = result("state") { let state = &state["value"]; - // Deliver the actual frozen Runtime source through authenticated Job tools, rather - // than injecting a fixture Journal or claiming unseen source was processed. - for (index, range) in state["selected_ranges"] - .as_array() - .unwrap() - .iter() - .enumerate() - { - let mut offset = range["bytes"]["start"].as_u64().unwrap(); - let mut token = Value::Null; - loop { - let id = format!("source-{index}-{offset}"); - if let Some(page) = result(&id) { - let page = &page["value"]; - if let Some(next) = page["next_byte"].as_u64() { - assert!(next > offset); - offset = next; - token = page["content_token"].clone(); - } else { - break; - } - } else { - assert!( - request - .tools - .iter() - .any(|tool| tool.name == "ReadJournalText") - ); - return vec![Event::tool_use_start(0, &id, "ReadJournalText"), Event::tool_input_delta(0, json!({ - "target":{"target":"source","pointer":range["source"]["pointer"]}, - "offset":offset,"limit":MAX_JOURNAL_READ_BYTES,"content_token":token, - }).to_string()), Event::tool_use_stop(0)]; - } - } - } + assert_eq!( + state["selected_ranges"], ranges, + "mutation coverage must come from the rendered frozen input" + ); + assert_eq!( + state["initial_journal"]["state_token"], + input_value["initial_state_token"] + ); let (tool, action) = if state["current_open"].is_null() { ( "CreateJournalEntry", @@ -158,7 +154,7 @@ impl ConnectedJournalModel { json!({ "request_id":format!("record:{}",state["generation_id"].as_str().unwrap()), "expected_state":state["current_journal"]["state_token"], - "action":action,"covered_ranges":state["selected_ranges"], + "action":action,"covered_ranges":ranges, }), ) } else { @@ -209,6 +205,18 @@ impl LlmClient for ConnectedJournalModel { .iter() .any(|tool| tool.name == "ReadJournalState") { + let envelope = request + .items + .iter() + .filter_map(Item::as_text) + .find(|text| text.lines().any(|line| line.starts_with("input_json: "))) + .expect("actual Journal input in model render") + .to_owned(); + let mut inputs = self.journal_inputs.lock().unwrap(); + if !inputs.contains(&envelope) { + inputs.push(envelope); + } + drop(inputs); Self::journal_response(&request) } else if request.tools.iter().any(|tool| { matches!( @@ -458,6 +466,29 @@ async fn backend_real_runtime_journal_release_and_next_body_persisted_startup() .store .list_backend_journal_jobs(TEST_WORKSPACE_ID, &old_attr.session_id, None, 100) .unwrap(); + // Compare actual render with the durable identity, even if completion already + // reclaimed the input body. Reconstituting only for comparison is test-local. + for saved in &recorded { + let inputs = model.journal_inputs.lock().unwrap(); + let prefix = format!("job_id: {}", saved.request.job_id); + let rendered = inputs + .iter() + .find(|text| text.lines().any(|line| line == prefix)) + .expect("every saved Journal Job reached the real Controller"); + let actual = rendered + .lines() + .find_map(|line| line.strip_prefix("input_json: ")) + .unwrap(); + let mut original = saved.request.clone(); + original.input = serde_json::from_str(actual).unwrap(); + assert_eq!(original.input_digest().unwrap(), saved.input_digest); + let attempt = crate::backend_job::attempt_id(&saved.request.job_id, saved.current_attempt); + assert_eq!(*rendered, original.worker_input(&attempt).unwrap()); + let input = subjektiv::journal::recording::JournalInput::decode(&original.input).unwrap(); + assert_eq!(input.capture.session_id, old_attr.session_id); + assert!(actual.contains("User chose B rather than A")); + assert!(actual.contains("Partial B implementation")); + } let uncertain_job = &recorded[0].request.job_id; let conn = rusqlite::Connection::open(&api.config.database_path).unwrap(); let saved_result: (String, String) = conn.query_row( diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs index 81ff5817a..364438db2 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs @@ -188,6 +188,55 @@ struct Fixture { source: RuntimeWorkerRef, id: String, } +fn fixture_capture(range: &JournalSourceRange, text: &str) -> session_store::capture::CaptureDto { + use session_store::capture::*; + let descriptor = session_store::public_material::PublicMaterialDescriptor { + identity: session_store::public_material::PublicMaterialIdentity { + segment_id: range.source.pointer.segment_id, + entry_ref: range.source.pointer.entry_ref.clone(), + }, + canonical_order: range.source.order, + origin: session_store::public_material::PublicMaterialOrigin::History { + origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput, + }, + sha256: range.source.content_sha256.clone(), + total_bytes: range.source.total_bytes, + }; + CaptureDto { + version: 1, + session_id: range.source.session_id.clone(), + segment_id: range.source.pointer.segment_id.to_string(), + view_entry_count: range.source.order + 1, + entries: vec![CaptureEntryDto { + entry_id: range + .source + .pointer + .entry_ref + .trim_start_matches('E') + .into(), + order: range.source.order, + source: CaptureSource { + segment_id: range.source.pointer.segment_id.to_string(), + entry_index: range.source.order, + provenance: protocol::SessionEntryProvenance::HumanInput, + history_origin: Some(session_store::LoggedSessionHistoryOrigin::LocalHumanInput), + derived_from: vec![], + }, + policy: CaptureUsePolicy { + recording_material: true, + memory_evidence: false, + observation_readable: false, + }, + body: CaptureBody::CanonicalPublic { + descriptor, + byte_offset: range.bytes.start, + text: text.into(), + boundary: None, + }, + }], + } +} + impl Fixture { async fn new() -> Self { let dir = tempfile::tempdir().unwrap(); @@ -236,7 +285,16 @@ impl Fixture { job_id: "journal-job-test".into(), purpose: "subjektiv_journal".into(), input_ref: format!("journal://{id}"), - input: serde_json::json!({"selected_ranges":[range]}), + input: serde_json::to_value( + subjektiv::journal::recording::JournalInput::build( + &store, + &host, + &id, + |_, _, _| Ok(fixture_capture(&range, text)), + ) + .unwrap(), + ) + .unwrap(), instruction: "Record this public input with sequential tools".into(), profile: "builtin:subjektiv-journal-recording".into(), grants: BackendJobGrants { @@ -537,16 +595,23 @@ fn message(text: &str) -> session_store::LoggedItem { } impl RuntimeFixture { async fn new() -> Self { + Self::with_history(vec![session_store::LoggedHistoryEntry { + item: message(SOURCE_SENTINEL), + metadata: session_store::LoggedSessionHistoryMetadata { + input_metadata: None, + entry_id: session_store::LoggedSessionHistoryEntryId("read-source-original".into()), + origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput, + derivation: None, + }, + }]) + .await + } + async fn with_history(history: Vec) -> Self { let dir = tempfile::tempdir().unwrap(); let (api, execution) = test_api_with_recording_backend(dir.path()).await; let source = spawn_ticket_check_source(&api, "original-source"); let session = session_store::new_session_id(); - seed_session( - &session_root(dir.path(), &source), - session, - message(SOURCE_SENTINEL), - session_store::LoggedSessionHistoryOrigin::LocalHumanInput, - ); + seed_session_history(&session_root(dir.path(), &source), session, history); let store = open_subjektiv_store(&api).unwrap(); let subject = store .create_subject(crate::subjektiv::SubjectRole::new("source-reader-test").unwrap()) @@ -600,7 +665,7 @@ impl RuntimeFixture { else { panic!("canonical catalog required") }; - assert_eq!(page.descriptors.len(), 1); + assert!(!page.descriptors.is_empty()); let d = &page.descriptors[0]; let range = JournalSourceRange { source: JournalSource { @@ -621,20 +686,24 @@ impl RuntimeFixture { let id = store .accept_journal_sources(&host, "source", &[range.clone()]) .unwrap(); - let request = subjektiv_journal::recording_request( + let input = subjektiv::journal::recording::JournalInput::build( + &store, &host, - attr, &id, - vec![subjektiv::journal::recording::JournalMaterial { - range: range.clone(), - text: SOURCE_SENTINEL.into(), - }], + |order, offset, budget| { + subjektiv_journal::capture_input(&api, &attr, order, offset, budget) + .map_err(|e| JournalError::Conflict(e.to_string())) + }, ) .unwrap(); + store + .accept_journal_sources(&host, "bounded captured prefix", &input.selected_ranges) + .unwrap(); + let request = subjektiv_journal::recording_request(&host, attr, &id, input).unwrap(); assert!( - !serde_json::to_string(&request) - .unwrap() - .contains(SOURCE_SENTINEL) + request.input["capture"]["entries"] + .as_array() + .is_some_and(|entries| !entries.is_empty()) ); let reserved = api .store @@ -666,47 +735,53 @@ impl RuntimeFixture { ) .unwrap(); } - fn dispatch_and_read(&self) -> BackendJobReservation { + fn dispatch_input(&self) -> BackendJobReservation { let dispatched = self .api .dispatch_backend_job_reservation(self.reserved.clone()) .unwrap(); let worker = dispatched.attempt.worker.as_ref().unwrap(); - let response = subjektiv_journal::execute( - &self.api, - worker, - JournalJobRequest::ReadText { - read: JournalTextRead { - target: JournalTextTarget::Source { - pointer: self.range.source.pointer.clone(), - }, - offset: 0, - limit: 16384, - content_token: None, - }, - }, - ) - .unwrap(); - let encoded = serde_json::to_string(&response).unwrap(); + let encoded = self + .execution + .inputs + .lock() + .unwrap() + .last() + .unwrap() + .1 + .clone(); + assert_eq!( + encoded, + self.request + .worker_input(&dispatched.attempt.attempt_id) + .unwrap() + ); assert!(encoded.contains(SOURCE_SENTINEL)); + assert!( + subjektiv_journal::execute( + &self.api, + worker, + JournalJobRequest::ReadText { + read: JournalTextRead { + target: JournalTextTarget::Source { + pointer: self.range.source.pointer.clone() + }, + offset: 0, + limit: 16384, + content_token: None + } + } + ) + .is_err(), + "body-bearing Jobs cannot read their original Session" + ); + // Deterministic execution stands in for Controller persistence here; actual Runtime + // input storage is exercised separately by its signed capture/delivery integration. seed_session( &session_root(self.dir.path(), worker), session_store::new_session_id(), - session_store::LoggedItem::ToolResult { - call_id: "read-original".into(), - summary: "ReadJournalText".into(), - content: Some(encoded), - attachments: vec![], - disposition: agen::tool::ToolResultDisposition::Success, - is_error: false, - }, - session_store::LoggedSessionHistoryOrigin::ToolOutput { - worker: session_store::LoggedWorkerSubject { - workspace_id: Some(TEST_WORKSPACE_ID.into()), - runtime_id: Some(worker.runtime_id.clone()), - worker_id: worker.worker_id.clone(), - }, - }, + message(&encoded), + session_store::LoggedSessionHistoryOrigin::LocalHumanInput, ); dispatched } @@ -748,6 +823,60 @@ impl RuntimeFixture { ) .unwrap(); } + /// Pending is NOT erasure success: observe durable Purge and retry the exact key. + fn erase_pending_or_acknowledged(&self, key: &str) -> bool { + match self + .api + .erase_subjektiv_journal_session(&self.session.to_string(), key) + { + Ok(()) => false, + Err(Error::RepositoryConflict(message)) => { + assert_eq!( + message, + "Journal delivered-copy Purge remains pending; retry the same erasure after cleanup" + ); + true + } + Err(error) => panic!("unexpected erasure failure: {error}"), + } + } + fn assert_erasure_acknowledged(&self, key: &str) { + self.api + .erase_subjektiv_journal_session(&self.session.to_string(), key) + .unwrap(); + let saved = self + .api + .store + .get_backend_job(TEST_WORKSPACE_ID, &self.request.job_id) + .unwrap() + .unwrap(); + assert_eq!( + saved.input_disposition, + crate::backend_job::JobInputDisposition::Erased + ); + assert_eq!(saved.input_digest, self.request.input_digest().unwrap()); + assert!( + !serde_json::to_string(&saved.request) + .unwrap() + .contains(SOURCE_SENTINEL) + ); + assert!(saved.load_executable_request().is_err()); + self.assert_no_erased_resurrection(); + } + fn assert_no_erased_resurrection(&self) { + self.execution.take_inputs(); + // Internal dispatch must reread and reject retired input, including stale + // terminal reservations. Exact public API replay is a separate contract. + assert!( + self.api + .dispatch_backend_job_reservation(self.reserved.clone()) + .is_err() + ); + assert!( + self.execution.take_inputs().is_empty(), + "stale saved body must not resurrect a delivered copy" + ); + } fn assert_purged(&self, worker: &RuntimeWorkerRef) { assert!(!session_root(self.dir.path(), worker).exists()); let removal = self @@ -791,7 +920,7 @@ async fn terminal_journal_attempts_purge_real_tool_result_sessions_without_chang .worker_retention_policy(TEST_WORKSPACE_ID) .unwrap() .unwrap(); - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); assert!(session_root(f.dir.path(), worker).exists()); match outcome { @@ -823,12 +952,7 @@ async fn terminal_journal_attempts_purge_real_tool_result_sessions_without_chang } "erased" => { f.stop_source(); - f.api - .erase_subjektiv_journal_session( - &f.session.to_string(), - "erase-original-session", - ) - .unwrap(); + f.erase_pending_or_acknowledged("erase-original-session"); } _ => unreachable!(), } @@ -850,12 +974,9 @@ async fn terminal_journal_attempts_purge_real_tool_result_sessions_without_chang ); assert!(subjektiv_journal::execute(&f.api, worker, JournalJobRequest::ReadState).is_err()); if outcome == "erased" { + f.assert_erasure_acknowledged("erase-original-session"); assert!(f.store.journal_session_erased(&f.host).unwrap()); - assert!( - f.api - .dispatch_backend_job_reservation(f.reserved.clone()) - .is_err() - ); + f.assert_no_erased_resurrection(); } } } @@ -863,7 +984,7 @@ async fn terminal_journal_attempts_purge_real_tool_result_sessions_without_chang #[tokio::test] async fn unknown_saved_completion_reconciles_same_binding_without_redispatch_after_lost_result() { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); f.checkpoint(worker); let saved = subjektiv_journal::finish(&f.api, worker).unwrap(); // response/result delivery is lost @@ -900,7 +1021,7 @@ async fn unknown_saved_completion_reconciles_same_binding_without_redispatch_aft #[tokio::test] async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_cleanup_fails() { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); let g = f .store @@ -966,6 +1087,28 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_ ) .is_err() ); + let retained = f + .api + .store + .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) + .unwrap() + .unwrap(); + assert_eq!(retained.input_digest, f.request.input_digest().unwrap()); + assert_eq!( + retained.load_executable_request().unwrap().input, + f.request.input + ); + assert!( + f.api + .store + .reserve_backend_job_retry( + TEST_WORKSPACE_ID, + &f.request.job_id, + &Utc::now().to_rfc3339() + ) + .is_err(), + "unknown cannot be reissued as a new attempt before the fence" + ); assert!(f.store.journal_retention_required(&f.host).unwrap()); assert!(session_root(f.dir.path(), worker).exists()); assert_eq!( @@ -1044,6 +1187,42 @@ async fn unknown_partial_waits_for_durable_purge_fence_and_retains_sources_when_ 1, "source pin survives failed/partial Job cleanup" ); + f.execution.take_inputs(); + let retry = f + .api + .store + .reserve_backend_job_retry( + TEST_WORKSPACE_ID, + &f.request.job_id, + &Utc::now().to_rfc3339(), + ) + .unwrap(); + assert_eq!(retry.job.request.input, f.request.input); + assert_eq!(retry.job.input_digest, f.request.input_digest().unwrap()); + let retry = f.api.dispatch_backend_job_reservation(retry).unwrap(); + let resumed = retry.attempt.worker.as_ref().unwrap(); + let delivered = f.execution.take_inputs(); + assert_eq!(delivered.len(), 1); + assert_eq!( + delivered[0].1, + f.request.worker_input(&retry.attempt.attempt_id).unwrap() + ); + assert_eq!( + f.store + .query_journal_operation(&f.host, &f.id, "partial-record") + .unwrap(), + Some(recorded) + ); + assert_eq!( + f.store + .list_journal_entries(&f.host, &f.id, None, 16) + .unwrap() + .items + .len(), + 1 + ); + assert!(subjektiv_journal::execute(&f.api, resumed, JournalJobRequest::ReadState).is_ok()); + assert!(subjektiv_journal::execute(&f.api, worker, JournalJobRequest::ReadState).is_err()); } #[tokio::test] @@ -1126,7 +1305,7 @@ async fn journal_reserved_restart_dispatches_exact_intent_once_and_unbound_unkno #[tokio::test] async fn archived_original_session_remains_readable_then_releases_its_pin_after_saved_completion() { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); let root = f.dir.path().join(".test-embedded-runtime-store"); let retention = worker_runtime::retention::runtime_session_retention(&root); @@ -1158,8 +1337,23 @@ async fn archived_original_session_remains_readable_then_releases_its_pin_after_ .unwrap() .expect("durable archive binding"); assert!(!session_root(f.dir.path(), &f.source).exists()); - // Same frozen descriptor; material_source must now resolve through the durable - // archive mapping, not attempt a stale retained Worker path. + // Archive movement neither changes nor reclaims the saved immutable body. + // The Job cannot fall back to a retained or archived Session read. + let saved = f + .api + .store + .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) + .unwrap() + .unwrap(); + assert_eq!( + saved.load_executable_request().unwrap().input, + f.request.input + ); + assert!( + serde_json::to_string(&saved.request.input) + .unwrap() + .contains(SOURCE_SENTINEL) + ); let reread = subjektiv_journal::execute( &f.api, worker, @@ -1174,11 +1368,13 @@ async fn archived_original_session_remains_readable_then_releases_its_pin_after_ }, }, ) - .unwrap(); - assert!( - serde_json::to_string(&reread) - .unwrap() - .contains(SOURCE_SENTINEL) + .unwrap_err(); + assert!(matches!(reread, Error::WorkspacePermissionDenied(message) + if message.contains("raw source bodies are in immutable Job input"))); + assert_eq!( + retention.pins(f.session).unwrap().len(), + 1, + "delivery alone never releases original pin" ); f.checkpoint(worker); let completion = subjektiv_journal::finish(&f.api, worker).unwrap(); @@ -1251,7 +1447,7 @@ async fn journal_profile_name_without_host_job_binding_keeps_normal_archive_poli async fn saved_checkpoint_unknown_auto_resolves_without_waiting_for_failed_session_cleanup() { for recovery in ["transition", "restart"] { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); f.checkpoint(worker); // Deliberately do not call finish: checkpoint receipt is durable. f.execution.take_inputs(); @@ -1353,10 +1549,9 @@ async fn saved_checkpoint_unknown_auto_resolves_without_waiting_for_failed_sessi } #[tokio::test] -async fn stopped_partial_journal_recovery_fences_old_attempt_before_dispatching_only_remaining_work() - { +async fn stopped_partial_journal_recovery_fences_old_attempt_and_resumes_same_immutable_input() { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); let g = f .store @@ -1420,11 +1615,14 @@ async fn stopped_partial_journal_recovery_fences_old_attempt_before_dispatching_ .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) .unwrap() .unwrap(); - assert_eq!(old.state, BackendJobState::Failed); + assert_eq!(old.state, BackendJobState::Pending); + assert_eq!(old.request.job_id, f.request.job_id); + assert_eq!(old.input_digest, f.request.input_digest().unwrap()); assert_eq!( - old.failure_category.as_deref(), - Some("journal_attempt_fenced") + old.load_executable_request().unwrap().input, + f.request.input ); + assert_eq!(old.current_attempt, dispatched.attempt.attempt + 1); f.assert_purged(worker); assert_eq!( f.store @@ -1446,11 +1644,20 @@ async fn stopped_partial_journal_recovery_fences_old_attempt_before_dispatching_ .active_journal_generation(&f.host, &f.id) .unwrap() .unwrap(); - assert_ne!(next.job, g.job); - assert_eq!(next.initial_open.unwrap().title, "unfinished"); + assert_eq!(next.job.job_id, g.job.job_id); + assert_ne!(next.job.attempt_id, g.job.attempt_id); + assert_eq!( + next.initial_open, g.initial_open, + "original initial snapshot stays bound to same input" + ); + assert_eq!(next.selected_ranges, g.selected_ranges); let new_inputs = f.execution.take_inputs(); assert_eq!(new_inputs.len(), 1); - assert!(!new_inputs[0].1.contains(SOURCE_SENTINEL)); + assert!(new_inputs[0].1.contains(SOURCE_SENTINEL)); + assert_eq!( + new_inputs[0].1, + f.request.worker_input(&next.job.attempt_id).unwrap() + ); assert!(subjektiv_journal::execute(&f.api, worker, JournalJobRequest::ReadState).is_err()); f.api.recover_backend_jobs().unwrap(); assert!( @@ -1463,7 +1670,7 @@ async fn stopped_partial_journal_recovery_fences_old_attempt_before_dispatching_ async fn failed_cancelled_and_erased_job_sessions_retry_purge_after_cleanup_failure() { for outcome in ["failed", "cancelled", "erased"] { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); f.api .store @@ -1476,12 +1683,10 @@ async fn failed_cancelled_and_erased_job_sessions_retry_purge_after_cleanup_fail .unwrap(); if outcome == "erased" { f.stop_source(); - f.api - .erase_subjektiv_journal_session( - &f.session.to_string(), - "erase-with-blocked-cleanup", - ) - .unwrap(); + assert!( + f.erase_pending_or_acknowledged("erase-with-blocked-cleanup"), + "pinned delivered copy must prevent erasure acknowledgement" + ); } else { f.api .finish_backend_job_attempt( @@ -1534,12 +1739,9 @@ async fn failed_cancelled_and_erased_job_sessions_retry_purge_after_cleanup_fail .await; f.assert_purged(worker); if outcome == "erased" { + f.assert_erasure_acknowledged("erase-with-blocked-cleanup"); assert!(f.store.journal_session_erased(&f.host).unwrap()); - assert!( - f.api - .dispatch_backend_job_reservation(f.reserved.clone()) - .is_err() - ); + f.assert_no_erased_resurrection(); } } } @@ -1547,7 +1749,7 @@ async fn failed_cancelled_and_erased_job_sessions_retry_purge_after_cleanup_fail #[tokio::test] async fn explicit_erasure_interrupts_completed_journal_cleanup_waiting_for_run_teardown() { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); let worker = dispatched.attempt.worker.as_ref().unwrap(); f.checkpoint(worker); let completion = subjektiv_journal::finish(&f.api, worker).unwrap(); @@ -1585,9 +1787,7 @@ async fn explicit_erasure_interrupts_completed_journal_cleanup_waiting_for_run_t "normal cleanup still waits for tracked response teardown" ); f.stop_source(); - f.api - .erase_subjektiv_journal_session(&f.session.to_string(), "erase-while-completed-run-busy") - .unwrap(); + f.erase_pending_or_acknowledged("erase-while-completed-run-busy"); // No Idle/finish marker is published by this fixture. The erasure fence must // cause a real Runtime stop/Purge instead of leaving plaintext indefinitely. wait_for_backend_job_worker_cleanup( @@ -1598,6 +1798,7 @@ async fn explicit_erasure_interrupts_completed_journal_cleanup_waiting_for_run_t ) .await; f.assert_purged(worker); + f.assert_erasure_acknowledged("erase-while-completed-run-busy"); assert_eq!( f.api .store @@ -2189,7 +2390,7 @@ async fn erased_subject_removal_retries_interrupted_physical_erasure_under_the_p #[tokio::test] async fn physical_erase_failure_keeps_domain_tombstone_and_job_cancellation_for_exact_retry() { let f = RuntimeFixture::new().await; - let dispatched = f.dispatch_and_read(); + let dispatched = f.dispatch_input(); // Original source is not stopped: Runtime must reject physical erasure, after // the Backend has durably fenced all Journal adoption and cancelled its Job. assert!( @@ -2215,9 +2416,16 @@ async fn physical_erase_failure_keeps_domain_tombstone_and_job_cancellation_for_ .is_err() ); f.stop_source(); - f.api - .erase_subjektiv_journal_session(&f.session.to_string(), "erase-unsettled-source") - .unwrap(); + f.erase_pending_or_acknowledged("erase-unsettled-source"); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + f.assert_erasure_acknowledged("erase-unsettled-source"); + f.assert_purged(dispatched.attempt.worker.as_ref().unwrap()); assert!(!session_root(f.dir.path(), &f.source).exists()); assert_eq!( f.api @@ -2807,3 +3015,1345 @@ async fn legacy_journal_grant_preserves_old_json_and_only_recovers_original_curr "legacy grant must never map to a later writer" ); } + +/// Real embedded Runtime capture + SQLite + production dispatch; synthetic ordinary +/// Session entries only, no fixed Worker data, fixture generator or model calls. +mod immutable { + use super::*; + use session_store::{LogEntry, Store as _, WorkerSessionStore}; + use std::collections::BTreeMap; + use subjektiv::journal::recording::JournalInput; + + fn history(id: &str, item: session_store::LoggedItem) -> session_store::LoggedHistoryEntry { + session_store::LoggedHistoryEntry { + item, + metadata: session_store::LoggedSessionHistoryMetadata { + input_metadata: None, + entry_id: session_store::LoggedSessionHistoryEntryId(id.into()), + origin: session_store::LoggedSessionHistoryOrigin::LocalHumanInput, + derivation: None, + }, + } + } + fn append(f: &RuntimeFixture, segment: Uuid, id: &str) { + WorkerSessionStore::new(session_root(f.dir.path(), &f.source)) + .unwrap() + .append( + f.session, + segment, + &LogEntry::AnnotatedAssistantItem { + ts: 2, + entry: history(id, message(id)), + }, + ) + .unwrap(); + } + fn checkpoint_ranges(f: &RuntimeFixture, worker: &RuntimeWorkerRef, input: &JournalInput) { + let state = f.store.read_journal(&f.host, &f.id).unwrap(); + let record_id = format!("immutable-record:{}", input.initial_state_token); + let operation = JournalOperation { + request_id: record_id.clone(), + expected_state: state.state_token, + action: JournalAction::ConfirmUnchanged, + covered_ranges: input.selected_ranges.clone(), + }; + let response = subjektiv_journal::execute( + &f.api, + worker, + JournalJobRequest::Execute { + operation: operation.clone(), + }, + ) + .unwrap(); + // Exact operation replay after a lost response cannot duplicate or advance the cursor. + assert_eq!( + subjektiv_journal::execute(&f.api, worker, JournalJobRequest::Execute { operation }) + .unwrap(), + response + ); + let JournalJobResponse::Saved(record) = response else { + panic!("record receipt") + }; + assert_eq!( + f.store + .query_journal_operation(&f.host, &f.id, &record_id) + .unwrap(), + Some(record.clone()) + ); + assert_eq!( + f.store.read_journal(&f.host, &f.id).unwrap().process_order, + state.process_order + ); + let JournalJobResponse::Saved(position) = subjektiv_journal::execute( + &f.api, + worker, + JournalJobRequest::Execute { + operation: JournalOperation { + request_id: format!("immutable-position:{}", input.initial_state_token), + expected_state: record.state_token, + action: JournalAction::UpdatePosition { + current_position: "Original prefix saved; remaining work unconfirmed" + .into(), + }, + covered_ranges: vec![], + }, + }, + ) + .unwrap() else { + panic!("position receipt") + }; + subjektiv_journal::execute( + &f.api, + worker, + JournalJobRequest::Execute { + operation: JournalOperation { + request_id: format!("immutable-checkpoint:{}", input.initial_state_token), + expected_state: position.state_token, + action: JournalAction::Checkpoint { + record_operation_ids: vec![record_id], + }, + covered_ranges: input.selected_ranges.clone(), + }, + }, + ) + .unwrap(); + } + + #[tokio::test] + async fn backend_immutable_input_survives_missing_source_and_predispatch_host_reopen() { + let RuntimeFixture { + dir, + api, + execution, + store, + host, + source, + session, + request, + reserved, + range, + id, + } = RuntimeFixture::new().await; + let frozen = serde_json::to_vec(&request.input).unwrap(); + let digest = request.input_digest().unwrap(); + subjektiv_journal::prepare(&api, &reserved).unwrap(); + let generation = store + .active_journal_generation(&host, &id) + .unwrap() + .unwrap(); + let root = session_root(dir.path(), &source); + let unavailable = dir.path().join("source-unavailable-during-job"); + std::fs::rename(&root, &unavailable).unwrap(); + let config = api.config.clone(); + drop(api); + drop(store); + // Reopen after input save but before dispatch, with no readable original source. + let api = WorkspaceApi::new_with_execution_backend_and_broker( + config.clone(), + Arc::new(test_control_store(&config)), + execution.clone(), + BackendResourceBroker::default(), + HostExecutors::default(), + None, + ) + .await + .unwrap(); + let store = open_subjektiv_store(&api).unwrap(); + let f = RuntimeFixture { + dir, + api, + execution, + store, + host, + source, + session, + request, + reserved, + range, + id, + }; + let saved = f + .api + .store + .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) + .unwrap() + .unwrap(); + assert_eq!( + serde_json::to_vec(&saved.load_executable_request().unwrap().input).unwrap(), + frozen + ); + assert_eq!(saved.input_digest, digest); + let dispatched = f + .api + .dispatch_backend_job_reservation(f.reserved.clone()) + .unwrap(); + assert_eq!( + f.store + .active_journal_generation(&f.host, &f.id) + .unwrap() + .unwrap(), + generation + ); + let worker = dispatched.attempt.worker.as_ref().unwrap(); + assert!(!root.exists()); + assert_eq!( + f.execution.take_inputs().last().unwrap().1, + f.request + .worker_input(&dispatched.attempt.attempt_id) + .unwrap() + ); + f.checkpoint(worker); + let completion = subjektiv_journal::finish(&f.api, worker).unwrap(); + assert_eq!(completion.process_order, 1); + assert!( + !root.exists(), + "checkpoint completed with original capture/catalog/read unavailable" + ); + std::fs::rename(&unavailable, &root).unwrap(); + f.api + .accept_backend_job_result( + worker, + &BackendJobResultSubmission { + job_id: f.request.job_id.clone(), + attempt_id: dispatched.attempt.attempt_id.clone(), + input_digest: digest, + result: serde_json::to_value(completion).unwrap(), + }, + ) + .unwrap(); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + } + + #[tokio::test] + async fn backend_over_100_entries_long_utf8_cap_continuation_and_postcapture_compaction() { + let long = "原文🙂é".repeat(36_000); + let mut entries: Vec<_> = (0..130) + .map(|n| { + history( + &format!("ordinary-{n}"), + message(&format!("ordinary original {n:03}")), + ) + }) + .collect(); + entries.push(history( + "long-tool", + session_store::LoggedItem::ToolResult { + call_id: "ordinary-tool".into(), + summary: "NOT THE ORIGINAL BODY".into(), + content: Some(long.clone()), + attachments: vec![], + disposition: Default::default(), + is_error: false, + }, + )); + let f = RuntimeFixture::with_history(entries).await; + let attr = f + .store + .session_attribution(&f.session.to_string()) + .unwrap() + .unwrap(); + let original_segment = f.range.source.pointer.segment_id; + let frozen = serde_json::to_vec(&f.request.input).unwrap(); + let rebuilt = JournalInput::build(&f.store, &f.host, &f.id, |order, offset, budget| { + let captured = subjektiv_journal::capture_input(&f.api, &attr, order, offset, budget) + .map_err(|e| JournalError::Conflict(e.to_string()))?; + // Append after Host capture but before the enclosing Journal input is finalized. + append(&f, original_segment, "append-before-dispatch"); + Ok(captured) + }) + .unwrap(); + assert_eq!( + serde_json::to_vec(&serde_json::to_value(&rebuilt).unwrap()).unwrap(), + frozen + ); + assert!( + !serde_json::to_string(&rebuilt) + .unwrap() + .contains("append-before-dispatch") + ); + let first = f + .api + .dispatch_backend_job_reservation(f.reserved.clone()) + .unwrap(); + // A real compaction header carries exactly one excluded opening summary + // and its complete original lineage; an empty header is intentionally invalid. + let mut summary = history( + "compaction-summary", + session_store::LoggedItem::Message { + role: session_store::LoggedRole::System, + content: vec![session_store::LoggedContentPart::Text { + text: "PRIVATE_COMPACTION_SUMMARY_NEVER_CAPTURE".into(), + }], + }, + ); + summary.metadata.input_metadata = Some( + session_store::HistoryInputMetadata::new( + session_store::HistoryInputUsePolicy { + recording_material: session_store::InputEvidenceUse::Exclude, + memory_evidence: session_store::InputEvidenceUse::Exclude, + observation: session_store::InputObservationUse::Hidden, + compaction: session_store::InputCompactionUse::OpeningSummary, + }, + None, + None, + ) + .unwrap(), + ); + summary.metadata.origin = session_store::LoggedSessionHistoryOrigin::Provided { + producer: "ordinary-compaction".into(), + }; + summary.metadata.derivation = Some(session_store::LoggedHistoryDerivation { + sources: (0..130) + .map(|n| session_store::LoggedSessionHistoryEntryId(format!("ordinary-{n}"))) + .chain([ + session_store::LoggedSessionHistoryEntryId("long-tool".into()), + session_store::LoggedSessionHistoryEntryId("append-before-dispatch".into()), + ]) + .collect(), + }); + // Actual compacted Segment/pointer movement while the first Job is executing. + let compact = Uuid::new_v4(); + let root = session_root(f.dir.path(), &f.source); + WorkerSessionStore::new(&root) + .unwrap() + .create_segment( + f.session, + compact, + &[LogEntry::AnnotatedSegmentStart { + ts: 3, + session_id: f.session, + system_prompt: Some("PRIVATE_SYSTEM_NEVER_CAPTURE".into()), + config: Default::default(), + history: vec![summary], + forked_from: None, + compacted_from: Some(session_store::SegmentOrigin { + segment_id: original_segment, + at_turn_index: 0, + }), + }], + ) + .unwrap(); + use session_store::WorkerMetadataStore as _; + let aggregate = root.parent().unwrap(); + let name = format!("worker-runtime-{}", f.source.worker_id); + session_store::WorkerAggregateStore::new(aggregate, &name) + .unwrap() + .write( + &session_store::WorkerMetadata::new( + &name, + Some(session_store::WorkerActiveSegmentRef::active_segment( + f.session, compact, + )), + ) + .with_workspace_id(TEST_WORKSPACE_ID), + ) + .unwrap(); + append(&f, compact, "append-during-job"); + assert_eq!( + serde_json::to_vec( + &f.api + .store + .get_backend_job(TEST_WORKSPACE_ID, &f.request.job_id) + .unwrap() + .unwrap() + .request + .input + ) + .unwrap(), + frozen + ); + + let mut reconstructed: BTreeMap = BTreeMap::new(); + let mut current = first; + let mut generations = 0; + let mut split_tool = 0; + loop { + generations += 1; + assert!(generations < 20, "bounded continuation made no progress"); + let input = JournalInput::decode(¤t.job.request.input).unwrap(); + assert!( + serde_json::to_vec(¤t.job.request.input) + .unwrap() + .len() + <= job::MAX_JOB_INPUT_BYTES + ); + assert_eq!( + input.selected_ranges, + current + .job + .request + .grants + .subjektiv_journal + .as_ref() + .unwrap() + .selected_ranges + ); + let before = f.store.read_journal(&f.host, &f.id).unwrap(); + assert_eq!(input.selected_ranges[0].source.order, before.process_order); + assert_eq!(input.selected_ranges[0].bytes.start, before.process_byte); + for (entry, range) in input.capture.entries.iter().zip(&input.selected_ranges) { + let session_store::capture::CaptureBody::CanonicalPublic { + text, byte_offset, .. + } = &entry.body + else { + panic!("original canonical bytes") + }; + let slot = reconstructed + .entry(range.source.order) + .or_insert_with(|| (range.source.clone(), String::new())); + assert_eq!(slot.0, range.source); + assert_eq!( + *byte_offset, + slot.1.len() as u64, + "no overlap or skipped bytes across Jobs" + ); + assert_eq!(range.bytes.end - range.bytes.start, text.len() as u64); + slot.1.push_str(text); + if entry.entry_id == "long-tool" { + split_tool += 1; + } + assert!(!text.contains("PRIVATE_SYSTEM_NEVER_CAPTURE")); + assert!(!text.contains("PRIVATE_COMPACTION_SUMMARY_NEVER_CAPTURE")); + } + let worker = current.attempt.worker.as_ref().unwrap(); + let encoded = current + .job + .request + .worker_input(¤t.attempt.attempt_id) + .unwrap(); + assert!( + f.execution + .take_inputs() + .iter() + .any(|(_, text)| text == &encoded) + ); + // Host delivery is not a processing checkpoint. + assert_eq!( + f.store.read_journal(&f.host, &f.id).unwrap().process_order, + before.process_order + ); + checkpoint_ranges(&f, worker, &input); + let completion = subjektiv_journal::finish(&f.api, worker).unwrap(); + let done = completion.process_order == 133 && completion.process_byte == 0; + f.api + .accept_backend_job_result( + worker, + &BackendJobResultSubmission { + job_id: current.job.request.job_id.clone(), + attempt_id: current.attempt.attempt_id.clone(), + input_digest: current.job.input_digest.clone(), + result: serde_json::to_value(completion).unwrap(), + }, + ) + .unwrap(); + wait_for_backend_job_worker_cleanup( + &f.api, + ¤t.job.request.job_id, + ¤t.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + if done { + break; + } + current = tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let jobs = f + .api + .store + .list_backend_journal_jobs( + TEST_WORKSPACE_ID, + &f.session.to_string(), + None, + 100, + ) + .unwrap(); + for job in jobs + .into_iter() + .filter(|job| job.state == BackendJobState::Pending) + { + let attempt = f + .api + .store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &job.request.job_id, + &crate::backend_job::attempt_id( + &job.request.job_id, + job.current_attempt, + ), + ) + .unwrap() + .unwrap(); + if attempt.state == BackendJobAttemptState::Dispatched { + return BackendJobReservation { + job, + attempt, + replayed: true, + resource_reused: false, + }; + } + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap_or_else(|_| { + let state = f.store.read_journal(&f.host, &f.id).unwrap(); + let probe = JournalInput::build(&f.store, &f.host, &f.id, |order, offset, budget| { + subjektiv_journal::capture_input(&f.api, &attr, order, offset, budget) + .map_err(|e| JournalError::Conflict(e.to_string())) + }).map(|input| input.selected_ranges.iter().map(|range| (range.source.order, range.bytes.clone())).collect::>()); + panic!("production drain stalled after {generations} generations at {}/{}, next capture={probe:?}", state.process_order, state.process_byte); + }); + } + assert!(generations > 3); + assert!( + split_tool > 1, + "long UTF-8 source must continue across whole-envelope caps" + ); + assert_eq!(reconstructed.len(), 133); + for (order, (source, bytes)) in &reconstructed { + assert_eq!( + bytes.len() as u64, + source.total_bytes, + "complete canonical source {order}" + ); + let digest: String = Sha256::digest(bytes.as_bytes()) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + assert_eq!( + digest, source.content_sha256, + "exact original body, not summary or preview" + ); + } + assert!(reconstructed[&130].1.contains(&long)); + assert!(reconstructed[&131].1.contains("append-before-dispatch")); + assert!(reconstructed[&132].1.contains("append-during-job")); + } +} + +#[tokio::test] +async fn explicit_erasure_reclaims_over_100_historical_body_inputs_and_waits_for_delivered_copy_ack() + { + let mut f = RuntimeFixture::new().await; + f.api + .store + .finish_backend_job_attempt( + TEST_WORKSPACE_ID, + &f.request.job_id, + &f.reserved.attempt.attempt_id, + BackendJobAttemptState::Failed, + "ordinary_predispatch_failure", + "saved input remains resumable", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + // Synthetic ordinary historical Job intents: no execution/model fixture. Failed + // attempts deliberately keep resume bodies, so erasure must page beyond 100. + for n in 0..105 { + let mut request = f.request.clone(); + request.job_id = format!("historical-body-{n:03}"); + let reserved = f + .api + .store + .reserve_backend_job(TEST_WORKSPACE_ID, &request, &Utc::now().to_rfc3339()) + .unwrap(); + assert_eq!(reserved.job.request.input, f.request.input); + f.api + .store + .finish_backend_job_attempt( + TEST_WORKSPACE_ID, + &request.job_id, + &reserved.attempt.attempt_id, + BackendJobAttemptState::Failed, + "ordinary_historical_failure", + "saved input still required for resume", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + } + f.reserved = f + .api + .store + .reserve_backend_job_retry( + TEST_WORKSPACE_ID, + &f.request.job_id, + &Utc::now().to_rfc3339(), + ) + .unwrap(); + let delivered = f.dispatch_input(); + let worker = delivered.attempt.worker.as_ref().unwrap(); + f.api + .store + .update_worker_retention( + TEST_WORKSPACE_ID, + worker, + "pinned", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + f.stop_source(); + let key = "erase-all-historical-job-bodies"; + assert!( + f.erase_pending_or_acknowledged(key), + "blocked copy must never acknowledge erasure" + ); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &delivered.attempt.attempt_id, + BackendJobWorkerCleanupState::Failed, + ) + .await; + assert!( + session_root(f.dir.path(), worker).exists(), + "failure injection remains effective" + ); + let first = f + .api + .store + .list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.session.to_string(), None, 100) + .unwrap(); + assert_eq!(first.len(), 100); + let rest = f + .api + .store + .list_backend_journal_jobs( + TEST_WORKSPACE_ID, + &f.session.to_string(), + Some(&first.last().unwrap().request.job_id), + 100, + ) + .unwrap(); + assert_eq!(rest.len(), 6); + for saved in first.iter().chain(&rest) { + assert_eq!( + saved.input_disposition, + crate::backend_job::JobInputDisposition::Erased + ); + assert_eq!(saved.input_digest, f.request.input_digest().unwrap()); + assert!(saved.load_executable_request().is_err()); + assert!( + !serde_json::to_string(&saved.request) + .unwrap() + .contains(SOURCE_SENTINEL) + ); + } + // Domain and input reclamation alone are not a delivered-copy cleanup ack. + assert!(f.erase_pending_or_acknowledged(key)); + f.api + .store + .update_worker_retention( + TEST_WORKSPACE_ID, + worker, + "normal", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + f.api.recover_backend_jobs().unwrap(); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &delivered.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + f.assert_erasure_acknowledged(key); + f.assert_purged(worker); + assert!(subjektiv_journal::execute(&f.api, worker, JournalJobRequest::ReadState).is_err()); +} + +mod lifecycle_barrier { + use super::*; + use std::sync::mpsc::{self, Receiver, Sender}; + use std::time::Duration; + + const DEADLINE: Duration = Duration::from_secs(20); + // Bounded observation of an already signalled callback while the actual mutex + // is demonstrably held. This is a channel timeout, never a scheduling sleep. + const HELD_WINDOW: Duration = Duration::from_millis(100); + + pub(super) fn spawn( + work: impl FnOnce() -> T + Send + 'static, + ) -> (std::thread::JoinHandle<()>, Receiver) { + let runtime = tokio::runtime::Handle::current(); + let (tx, rx) = mpsc::channel(); + let join = std::thread::spawn(move || { + let _runtime = runtime.enter(); + let result = work(); + tx.send(result).unwrap(); + }); + (join, rx) + } + pub(super) fn joined(task: (std::thread::JoinHandle<()>, Receiver)) -> T { + let result = task + .1 + .recv_timeout(DEADLINE) + .expect("bounded callback progress; no gate/Runtime acknowledgement cycle"); + task.0.join().expect("callback thread joined"); + result + } + fn acceptance_barrier(f: &RuntimeFixture) -> (Receiver, Sender<()>) { + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let root = f.dir.path().to_path_buf(); + let encoded = f + .request + .worker_input(&f.reserved.attempt.attempt_id) + .unwrap(); + f.execution.before_input_returns(move |worker| { + let worker = + RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, worker.worker_id.to_string()); + seed_session( + &session_root(&root, &worker), + session_store::new_session_id(), + message(&encoded), + session_store::LoggedSessionHistoryOrigin::LocalHumanInput, + ); + entered_tx.send(worker).unwrap(); + // Model the Runtime's SubmissionAccepted boundary, NOT Tool completion. + // A real Controller can send the first Tool concurrently. Waiting for + // that Tool here would manufacture the very circular wait under test. + release_rx + .recv_timeout(DEADLINE) + .expect("test released Runtime acceptance"); + }); + (entered_rx, release_tx) + } + fn dispatch( + f: &RuntimeFixture, + ) -> ( + std::thread::JoinHandle<()>, + Receiver>, + ) { + let api = f.api.clone(); + let reserved = f.reserved.clone(); + spawn(move || api.dispatch_backend_job_reservation(reserved)) + } + fn operation(f: &RuntimeFixture) -> JournalOperation { + let generation = f + .store + .active_journal_generation(&f.host, &f.id) + .unwrap() + .unwrap(); + JournalOperation { + request_id: "first-tool-after-acceptance".into(), + expected_state: generation.initial_journal.state_token, + action: JournalAction::ConfirmUnchanged, + covered_ranges: vec![f.range.clone()], + } + } + fn assert_no_preaccept_attestation(f: &RuntimeFixture) { + let generation = f + .store + .active_journal_generation(&f.host, &f.id) + .unwrap() + .unwrap(); + let context = + JournalJobContext::validated(&f.store, &f.host, &generation.id, generation.job) + .unwrap(); + let error = subjektiv::journal::service::execute_snapshot( + &f.store, + &context, + &f.request.input_digest().unwrap(), + JournalJobRequest::ReadState, + ) + .unwrap_err(); + assert!( + error.to_string().contains("trusted Host delivery"), + "{error}" + ); + assert!( + f.store + .query_journal_operation(&f.host, &f.id, "first-tool-after-acceptance") + .unwrap() + .is_none() + ); + assert_eq!( + f.store.read_journal(&f.host, &f.id).unwrap().process_order, + 0 + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn dispatch_first_tool_waits_for_runtime_acceptance_then_progresses_without_ack_cycle() { + let f = RuntimeFixture::new().await; + let (entered, release) = acceptance_barrier(&f); + let dispatch = dispatch(&f); + let worker = entered.recv_timeout(DEADLINE).unwrap(); + let gate = subjektiv_journal::lifecycle_lock( + &f.api, + &f.request + .grants + .subjektiv_journal + .as_ref() + .unwrap() + .subject_id, + ) + .unwrap(); + assert!(matches!( + gate.try_lock(), + Err(std::sync::TryLockError::WouldBlock) + )); + let (attempted_tx, attempted_rx) = mpsc::channel(); + let api = f.api.clone(); + let callback_worker = worker.clone(); + let op = operation(&f); + let tool = spawn(move || { + attempted_tx.send(()).unwrap(); + subjektiv_journal::execute( + &api, + &callback_worker, + JournalJobRequest::Execute { operation: op }, + ) + }); + attempted_rx.recv_timeout(DEADLINE).unwrap(); + let (finish_tx, finish_rx) = mpsc::channel(); + let api = f.api.clone(); + let callback_worker = worker.clone(); + let early_finish = spawn(move || { + finish_tx.send(()).unwrap(); + subjektiv_journal::finish(&api, &callback_worker) + }); + finish_rx.recv_timeout(DEADLINE).unwrap(); + assert!( + matches!( + early_finish.1.recv_timeout(HELD_WINDOW), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "finish must wait for the same acceptance gate, not mutate before attestation" + ); + assert!( + matches!( + tool.1.recv_timeout(HELD_WINDOW), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "first Tool must not bypass dispatch's held Subject gate" + ); + assert_no_preaccept_attestation(&f); + release.send(()).unwrap(); + // Dispatch must independently finish after acceptance, even with the Tool waiting. + let dispatched = joined(dispatch).unwrap(); + let unfinished = joined(early_finish).unwrap_err().to_string(); + assert!( + !unfinished.contains("trusted Host delivery"), + "acceptance attestation now exists" + ); + let JournalJobResponse::Saved(record) = joined(tool).unwrap() else { + panic!("record receipt") + }; + assert_eq!(dispatched.attempt.worker.as_ref(), Some(&worker)); + assert_eq!( + f.store + .query_journal_operation(&f.host, &f.id, "first-tool-after-acceptance") + .unwrap(), + Some(record.clone()) + ); + assert_eq!( + f.store.read_journal(&f.host, &f.id).unwrap().process_order, + 0, + "acceptance and mutation alone are not a checkpoint" + ); + let position = save_fixture_position(&f.api, &worker, record.state_token); + subjektiv_journal::execute( + &f.api, + &worker, + JournalJobRequest::Execute { + operation: JournalOperation { + request_id: "checkpoint-after-barrier".into(), + expected_state: position, + action: JournalAction::Checkpoint { + record_operation_ids: vec!["first-tool-after-acceptance".into()], + }, + covered_ranges: vec![f.range.clone()], + }, + }, + ) + .unwrap(); + let api = f.api.clone(); + let callback_worker = worker.clone(); + let completion = joined(spawn(move || { + subjektiv_journal::finish(&api, &callback_worker) + })) + .unwrap(); + f.api + .accept_backend_job_result( + &worker, + &BackendJobResultSubmission { + job_id: f.request.job_id.clone(), + attempt_id: dispatched.attempt.attempt_id.clone(), + input_digest: f.request.input_digest().unwrap(), + result: serde_json::to_value(completion).unwrap(), + }, + ) + .unwrap(); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + f.assert_purged(&worker); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn erase_and_tool_race_dispatch_acceptance_without_inversion_and_require_cleanup_ack() { + let f = RuntimeFixture::new().await; + f.stop_source(); + let (entered, release) = acceptance_barrier(&f); + let dispatch = dispatch(&f); + let worker = entered.recv_timeout(DEADLINE).unwrap(); + f.api + .store + .update_worker_retention( + TEST_WORKSPACE_ID, + &worker, + "pinned", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + let gate = subjektiv_journal::lifecycle_lock( + &f.api, + &f.request + .grants + .subjektiv_journal + .as_ref() + .unwrap() + .subject_id, + ) + .unwrap(); + assert!(matches!( + gate.try_lock(), + Err(std::sync::TryLockError::WouldBlock) + )); + let (attempted_tx, attempted_rx) = mpsc::channel(); + let api = f.api.clone(); + let session = f.session.to_string(); + let key = "erase-racing-dispatch-acceptance"; + let erase_attempted = attempted_tx.clone(); + let erasure = spawn(move || { + erase_attempted.send(()).unwrap(); + api.erase_subjektiv_journal_session(&session, key) + }); + let api = f.api.clone(); + let callback_worker = worker.clone(); + let op = operation(&f); + let tool = spawn(move || { + attempted_tx.send(()).unwrap(); + subjektiv_journal::execute( + &api, + &callback_worker, + JournalJobRequest::Execute { operation: op }, + ) + }); + attempted_rx.recv_timeout(DEADLINE).unwrap(); + attempted_rx.recv_timeout(DEADLINE).unwrap(); + assert!(matches!( + erasure.1.recv_timeout(HELD_WINDOW), + Err(mpsc::RecvTimeoutError::Timeout) + )); + assert!(matches!( + tool.1.recv_timeout(HELD_WINDOW), + Err(mpsc::RecvTimeoutError::Timeout) + )); + assert!( + !f.store.journal_session_erased(&f.host).unwrap(), + "erase must wait for the same dispatch gate" + ); + assert_no_preaccept_attestation(&f); + release.send(()).unwrap(); + let dispatched = joined(dispatch).unwrap(); + let first = joined(erasure).unwrap_err(); + assert!(matches!(first, Error::RepositoryConflict(message) + if message == "Journal delivered-copy Purge remains pending; retry the same erasure after cleanup")); + // Mutex waiter order is deliberately unspecified. A Tool either saves before + // the tombstone or revalidates and is rejected; it cannot cross the fence. + if let Ok(response) = joined(tool) { + assert!(matches!(response, JournalJobResponse::Saved(_))); + } + assert!(f.store.journal_session_erased(&f.host).unwrap()); + let api = f.api.clone(); + let callback_worker = worker.clone(); + let op = JournalOperation { + request_id: "post-tombstone-must-not-save".into(), + expected_state: "irrelevant-after-erasure".into(), + action: JournalAction::ConfirmUnchanged, + covered_ranges: vec![f.range.clone()], + }; + assert!(joined(spawn(move || { + subjektiv_journal::execute(&api, &callback_worker, JournalJobRequest::ReadState) + .is_err() + && subjektiv_journal::execute( + &api, + &callback_worker, + JournalJobRequest::Execute { operation: op }, + ) + .is_err() + && subjektiv_journal::finish(&api, &callback_worker).is_err() + }))); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Failed, + ) + .await; + assert!(session_root(f.dir.path(), &worker).exists()); + assert!(f.erase_pending_or_acknowledged(key)); + f.api + .store + .update_worker_retention( + TEST_WORKSPACE_ID, + &worker, + "normal", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + f.api.recover_backend_jobs().unwrap(); + wait_for_backend_job_worker_cleanup( + &f.api, + &f.request.job_id, + &dispatched.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + f.assert_erasure_acknowledged(key); + f.assert_purged(&worker); + } +} + +mod journal_migration { + use super::lifecycle_barrier::{joined, spawn}; + use super::*; + use subjektiv::journal::recording::JournalInput; + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn reserved_legacy_descriptor_invalidates_exact_identity_and_migrates_only_after_saved_checkpoint() + { + let f = SubjectRemovalFixture::new().await; + let capture = subjektiv_journal::capture_input(&f.api, &f.attr, 0, 0, 48 * 1024).unwrap(); + let ranges = subjektiv::journal::recording::capture_ranges(&capture).unwrap(); + assert_eq!(ranges.len(), 2); + let id = f + .store + .accept_journal_sources(&f.host, "legacy admitted originals", &ranges) + .unwrap(); + let input = JournalInput::build(&f.store, &f.host, &id, |order, offset, budget| { + subjektiv_journal::capture_input(&f.api, &f.attr, order, offset, budget) + .map_err(|e| JournalError::Conflict(e.to_string())) + }) + .unwrap(); + let mut legacy = + subjektiv_journal::recording_request(&f.host, f.attr.clone(), &id, input).unwrap(); + legacy.job_id = "reserved-descriptor-only-legacy".into(); + legacy.input = json!({"journal_id":id, "selected_ranges":ranges}); + legacy + .grants + .subjektiv_journal + .as_mut() + .unwrap() + .binding_id + .clear(); + let reserved = f + .api + .store + .reserve_backend_job(TEST_WORKSPACE_ID, &legacy, &Utc::now().to_rfc3339()) + .unwrap(); + assert_eq!(reserved.attempt.state, BackendJobAttemptState::Reserved); + assert_eq!(reserved.job.request, legacy); + let digest = legacy.input_digest().unwrap(); + let fingerprint = legacy.fingerprint().unwrap(); + // A previous generation saved the first prefix before its acknowledgement + // was lost. Seed that real domain checkpoint, not a model success claim. + let binding = subjektiv::api::JobAttemptBinding { + job_id: "previous-checkpointed-generation".into(), + attempt_id: "previous-attempt".into(), + }; + let generation = f + .store + .prepare_journal_generation(&f.host, &id, binding.clone(), vec![ranges[0].clone()]) + .unwrap(); + let ctx = JournalJobContext::validated(&f.store, &f.host, &generation.id, binding.clone()) + .unwrap(); + let record = f + .store + .execute_journal( + &ctx, + JournalOperation { + request_id: "legacy-saved-record".into(), + expected_state: generation.initial_journal.state_token, + action: JournalAction::Create { + title: "Saved before migration".into(), + body: "First original reflected; second still pending".into(), + ongoing: "Continue second original only".into(), + state: JournalEntryState::Open, + source_end: None, + correction_refs: vec![], + }, + covered_ranges: vec![ranges[0].clone()], + }, + ) + .unwrap(); + let position = f + .store + .execute_journal( + &ctx, + JournalOperation { + request_id: "legacy-saved-position".into(), + expected_state: record.state_token.clone(), + action: JournalAction::UpdatePosition { + current_position: "First original saved; remaining work unconfirmed".into(), + }, + covered_ranges: vec![], + }, + ) + .unwrap(); + let checkpoint = f + .store + .checkpoint_journal( + &ctx, + JournalOperation { + request_id: "legacy-saved-checkpoint".into(), + expected_state: position.state_token, + action: JournalAction::Checkpoint { + record_operation_ids: vec!["legacy-saved-record".into()], + }, + covered_ranges: vec![ranges[0].clone()], + }, + ) + .unwrap(); + f.store + .close_journal_generation(&f.host, &generation.id, &binding) + .unwrap(); + assert_eq!(f.store.read_journal(&f.host, &id).unwrap().process_order, 1); + let api = f.api.clone(); + let migrated = joined(spawn(move || { + api.dispatch_backend_job_reservation(reserved) + })) + .unwrap(); + assert_eq!(migrated.job.request.job_id, legacy.job_id); + let old = f + .api + .store + .get_backend_job(TEST_WORKSPACE_ID, &legacy.job_id) + .unwrap() + .unwrap(); + assert_eq!(old.state, BackendJobState::Failed); + assert_eq!( + old.failure_category.as_deref(), + Some("journal_descriptor_invalidated") + ); + assert_eq!( + old.input_disposition, + crate::backend_job::JobInputDisposition::Invalidated + ); + assert_eq!(old.input_digest, digest); + assert_eq!(old.intent_fingerprint, fingerprint); + assert_eq!(old.request.job_id, legacy.job_id); + assert!(old.request.input.is_null()); + assert!(old.load_executable_request().is_err()); + assert_eq!(old.current_attempt, 1); + let old_attempt = f + .api + .store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &legacy.job_id, + &crate::backend_job::attempt_id(&legacy.job_id, 1), + ) + .unwrap() + .unwrap(); + assert_eq!(old_attempt.input_digest, digest); + assert!( + old_attempt.worker.is_none(), + "descriptor-only input must never reach a Worker" + ); + + let (new, attempt) = tokio::time::timeout(std::time::Duration::from_secs(20), async { + loop { + for job in f + .api + .store + .list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.attr.session_id, None, 100) + .unwrap() + { + if job.request.job_id == legacy.job_id { + continue; + } + let attempt = f + .api + .store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &job.request.job_id, + &crate::backend_job::attempt_id( + &job.request.job_id, + job.current_attempt, + ), + ) + .unwrap() + .unwrap(); + if attempt.state == BackendJobAttemptState::Dispatched { + return (job, attempt); + } + } + tokio::task::yield_now().await; + } + }) + .await + .expect("migration dispatch progresses without a lifecycle gate inversion"); + assert_ne!(new.request.job_id, legacy.job_id); + let frozen = JournalInput::decode(&new.request.input).unwrap(); + assert_eq!(frozen.selected_ranges, vec![ranges[1].clone()]); + assert_eq!(frozen.initial_state_token, checkpoint.state_token); + assert_eq!( + f.store + .query_journal_operation(&f.host, &id, "legacy-saved-record") + .unwrap(), + Some(record) + ); + assert_eq!( + f.store + .query_journal_operation(&f.host, &id, "legacy-saved-checkpoint") + .unwrap(), + Some(checkpoint) + ); + assert_eq!( + f.store + .list_journal_entries(&f.host, &id, None, 16) + .unwrap() + .items + .len(), + 1 + ); + let api = f.api.clone(); + let worker = attempt.worker.clone().unwrap(); + assert!( + joined(spawn(move || subjektiv_journal::execute( + &api, + &worker, + JournalJobRequest::ReadState + ))) + .is_ok() + ); + let api = f.api.clone(); + let stale = migrated.clone(); + assert!( + joined(spawn(move || api.dispatch_backend_job_reservation(stale))).is_err(), + "invalidated internal reservations cannot dispatch or silently replay" + ); + // Stop this intentionally unfinished new body Job without pretending its + // remaining original was reflected; normal cleanup still needs its receipt. + f.api + .finish_backend_job_attempt( + &new.request.job_id, + &attempt.attempt_id, + BackendJobAttemptState::Failed, + "test_finished", + "remaining original deliberately unprocessed", + ) + .unwrap(); + wait_for_backend_job_worker_cleanup( + &f.api, + &new.request.job_id, + &attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + assert_eq!(f.store.read_journal(&f.host, &id).unwrap().process_order, 1); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn tampered_or_missing_body_snapshot_is_not_legacy_and_never_falls_back_to_available_source() + { + for missing in [false, true] { + let f = RuntimeFixture::new().await; + f.api + .store + .finish_backend_job_attempt( + TEST_WORKSPACE_ID, + &f.request.job_id, + &f.reserved.attempt.attempt_id, + BackendJobAttemptState::Failed, + "predispatch_fixture", + "release resource before independently saved malformed request", + &Utc::now().to_rfc3339(), + ) + .unwrap(); + let mut malformed = f.request.clone(); + malformed.job_id = format!("malformed-body-snapshot-{missing}"); + if missing { + malformed.input.as_object_mut().unwrap().remove("capture"); + } else { + malformed.input["capture"]["entries"][0]["body"]["text"] = + json!("tampered body, same canonical descriptor"); + } + assert!(JournalInput::decode(&malformed.input).is_err()); + let bad = f + .api + .store + .reserve_backend_job(TEST_WORKSPACE_ID, &malformed, &Utc::now().to_rfc3339()) + .unwrap(); + assert_eq!(bad.job.request.input, malformed.input); + assert!( + bad.job.load_executable_request().is_ok(), + "generic immutable digest is valid; semantic body integrity is not" + ); + f.execution.take_inputs(); + let api = f.api.clone(); + let candidate = bad.clone(); + assert!( + !joined(spawn(move || subjektiv_journal::migrate_reserved_legacy( + &api, &candidate + ))) + .unwrap() + ); + let api = f.api.clone(); + assert!(joined(spawn(move || api.dispatch_backend_job_reservation(bad))).is_err()); + assert!(f.execution.take_inputs().is_empty()); + assert!( + session_root(f.dir.path(), &f.source).exists(), + "original source is available but must not repair saved input" + ); + let jobs = f + .api + .store + .list_backend_journal_jobs(TEST_WORKSPACE_ID, &f.session.to_string(), None, 100) + .unwrap(); + assert_eq!( + jobs.len(), + 2, + "malformed body must not create a replacement/migration Job" + ); + let saved = jobs + .iter() + .find(|job| job.request.job_id == malformed.job_id) + .unwrap(); + assert_eq!( + saved.input_disposition, + crate::backend_job::JobInputDisposition::Available + ); + assert_eq!(saved.input_digest, malformed.input_digest().unwrap()); + assert_eq!(saved.request.input, malformed.input); + assert!( + f.store + .active_journal_generation(&f.host, &f.id) + .unwrap() + .is_none() + ); + assert_eq!( + f.store.read_journal(&f.host, &f.id).unwrap().process_order, + 0 + ); + } + } +} diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index d742f0ec6..6da4ad1bd 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -119,7 +119,7 @@ pub use ticket::{ }; const OLDEST_SCHEMA_VERSION: i64 = 50; -const LATEST_SCHEMA_VERSION: i64 = 102; +const LATEST_SCHEMA_VERSION: i64 = 103; const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants"; const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs"; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; @@ -437,6 +437,11 @@ const MIGRATIONS: &[Migration] = &[ name: "Durable failed Subject creation recovery", apply: migrate_subject_create_recovery_v101_to_v102, }, + Migration { + version: 103, + name: "Job input retirement independent of immutable receipts", + apply: migrate_job_input_retention_v102_to_v103, + }, ]; #[derive(Clone, Copy)] @@ -1771,6 +1776,15 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { request: &BackendJobRequest, now: &str, ) -> Result; + /// Host-only retirement; fence domain mutations and delivered snapshots first for erasure. + /// Completed reclamation additionally requires canonical Worker cleanup for all attempts. + fn reclaim_backend_job_input( + &self, + workspace_id: &str, + job_id: &str, + expected_input_digest: &str, + reason: crate::backend_job::JobInputRetirementReason, + ) -> Result; fn reserve_backend_job_retry( &self, workspace_id: &str, @@ -7358,6 +7372,48 @@ impl ControlPlaneStore for SqliteWorkspaceStore { }) } + fn reclaim_backend_job_input( + &self, + workspace_id: &str, + job_id: &str, + expected_input_digest: &str, + reason: crate::backend_job::JobInputRetirementReason, + ) -> Result { + use crate::backend_job::{JobInputDisposition, JobInputRetirementReason}; + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let job = read_backend_job(&tx, workspace_id, job_id)? + .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; + if job.input_digest != expected_input_digest { + return Err(Error::InvalidInput("Backend Job input digest mismatch".into())); + } + let desired = reason.disposition(); + if job.input_disposition == desired { + tx.commit()?; + reclaim_backend_job_input_storage(conn)?; + return Ok(job); + } + if reason == JobInputRetirementReason::Completed { + job.load_executable_request()?; + let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, &attempt_id(job_id, job.current_attempt))? + .ok_or_else(|| Error::Store("Backend Job current attempt missing".into()))?; + if job.state != BackendJobState::Completed || attempt.state != BackendJobAttemptState::Completed + || job.result.is_none() || backend_job_has_unfinished_cleanup(&tx, workspace_id, job_id)? + { return Err(Error::InvalidInput("input reclaim requires known completed result and completed Worker cleanup".into())); } + } + if job.input_disposition != JobInputDisposition::Available + && reason != JobInputRetirementReason::ExplicitErasure + { return Err(Error::InvalidInput("retired Backend Job input cannot be restored or reclassified".into())); } + let mut metadata = job.request.clone(); + metadata.input = serde_json::Value::Null; + tx.execute("UPDATE backend_jobs SET request_json=?3, input_disposition=?4 WHERE workspace_id=?1 AND job_id=?2", params![workspace_id,job_id,serde_json::to_string(&metadata).map_err(|e| Error::Store(e.to_string()))?,desired.as_str()])?; + let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::Store("retired Backend Job disappeared".into()))?; + tx.commit()?; + reclaim_backend_job_input_storage(conn)?; + Ok(job) + }) + } + fn reserve_backend_job_retry( &self, workspace_id: &str, @@ -7368,6 +7424,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let mut job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; + job.load_executable_request()?; let previous_id = attempt_id(job_id, job.current_attempt); let previous = read_backend_job_attempt(&tx, workspace_id, job_id, &previous_id)? .ok_or_else(|| Error::Store("Backend Job current attempt is missing".to_string()))?; @@ -7407,7 +7464,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } tx.execute( "INSERT INTO backend_job_attempts (workspace_id, job_id, attempt_id, attempt, input_digest, state, deadline_at, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, 'reserved', ?6, ?7, ?7)", - params![workspace_id, job_id, next_id, next as i64, job.request.input_digest()?, deadline, now], + params![workspace_id, job_id, next_id, next as i64, job.input_digest, deadline, now], )?; job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::Store("retried Backend Job is missing".to_string()))?; @@ -7446,7 +7503,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { "SELECT job.job_id FROM backend_jobs job JOIN backend_job_attempts attempt ON attempt.workspace_id = job.workspace_id AND attempt.job_id = job.job_id AND attempt.attempt = job.current_attempt - WHERE job.workspace_id = ?1 AND job.state = 'pending' + WHERE job.workspace_id = ?1 AND job.state = 'pending' AND job.input_disposition = 'available' AND attempt.state = 'dispatched' AND attempt.runtime_id = ?2 AND attempt.worker_id = ?3 AND attempt.runtime_run_id IS NOT NULL", params![workspace_id, worker.runtime_id.as_str(), worker.worker_id.as_str()], @@ -7506,6 +7563,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; + job.load_executable_request()?; let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?; if attempt.state != BackendJobAttemptState::Reserved { @@ -7543,7 +7601,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result> { self.with_conn(|conn| { let mut statement = conn.prepare( - "SELECT job_id, attempt_id FROM backend_job_attempts WHERE workspace_id = ?1 AND state = 'reserved' ORDER BY created_at, attempt_id LIMIT ?2", + "SELECT a.job_id, a.attempt_id FROM backend_job_attempts a JOIN backend_jobs j ON j.workspace_id=a.workspace_id AND j.job_id=a.job_id WHERE a.workspace_id=?1 AND a.state='reserved' AND j.input_disposition='available' ORDER BY a.created_at, a.attempt_id LIMIT ?2", )?; let identities = statement.query_map(params![workspace_id, limit as i64], |row| { Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) @@ -7568,9 +7626,10 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let job = read_backend_job(&tx, workspace_id, job_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job `{job_id}`")))?; + job.load_executable_request()?; let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?; - if attempt.attempt != job.current_attempt || attempt.input_digest != job.request.input_digest()? { + if attempt.attempt != job.current_attempt || attempt.input_digest != job.input_digest { return Err(Error::InvalidInput( "Backend Job attempt is stale for the current input digest".to_string(), )); @@ -7629,6 +7688,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let job = read_backend_job(&tx, workspace_id, job_id)?.ok_or_else(|| Error::InvalidInput("Backend Job missing".into()))?; + job.load_executable_request()?; let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)? .ok_or_else(|| { Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")) @@ -7682,7 +7743,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let attempt = read_backend_job_attempt(&tx, workspace_id, &submission.job_id, &submission.attempt_id)? .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{}`", submission.attempt_id)))?; let (digest, result_json) = result_digest(&submission.result, job.request.limits.max_result_bytes)?; - if submission.input_digest != job.request.input_digest()? + if submission.input_digest != job.input_digest || attempt.input_digest != submission.input_digest || attempt.attempt != job.current_attempt { @@ -7709,6 +7770,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { tx.commit()?; return Ok(BackendJobResultAcceptance { job, attempt, replayed: true }); } + job.load_executable_request()?; if attempt.state != BackendJobAttemptState::Dispatched || job.state != BackendJobState::Pending { return Err(Error::InvalidInput( "Backend Job attempt no longer accepts results".to_string(), @@ -7776,7 +7838,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { // Full snapshot CAS includes Worker, tracked Run, input and cleanup evidence. if &attempt != expected || job.current_attempt != attempt.attempt || job.request.grants.subjektiv_journal.is_none() - || attempt.input_digest != job.request.input_digest()? + || attempt.input_digest != job.input_digest || !matches!(job.state, BackendJobState::Unknown | BackendJobState::Completed | BackendJobState::Failed) { return Err(Error::InvalidInput("Journal reconciliation binding changed".into())); } @@ -12109,6 +12171,20 @@ fn read_memory_staging_resolution_record( }) } +// The receipt/disposition commit precedes physical cleanup. An error here is +// safely retried with the same expected digest/reason, not a new execution. +fn reclaim_backend_job_input_storage(conn: &Connection) -> Result<()> { + conn.execute_batch("PRAGMA secure_delete=ON; VACUUM;")?; + let busy: i64 = conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| r.get(0))?; + if busy != 0 { + return Err(Error::Store( + "Backend Job input retired; SQLite body cleanup blocked by a reader, retry retirement" + .into(), + )); + } + Ok(()) +} + fn backend_job_deadline(now: &str, timeout_seconds: u32) -> Result { let now = chrono::DateTime::parse_from_rfc3339(now) .map_err(|error| Error::InvalidInput(format!("invalid Backend Job timestamp: {error}")))?; @@ -12172,7 +12248,7 @@ pub(crate) fn terminal_journal_job_worker( Ok(job.request.grants.subjektiv_journal.is_some() && attempt.state.terminal() && attempt.worker.as_ref() == Some(worker) - && attempt.input_digest == job.request.input_digest()?) + && attempt.input_digest == job.input_digest) } fn read_backend_job( @@ -12181,13 +12257,13 @@ fn read_backend_job( job_id: &str, ) -> Result> { let row = conn.query_row( - "SELECT request_json, intent_fingerprint, state, current_attempt, result_json, result_digest, failure_category, failure_detail, created_at, updated_at, completed_at FROM backend_jobs WHERE workspace_id = ?1 AND job_id = ?2", + "SELECT request_json, intent_fingerprint, state, current_attempt, result_json, result_digest, failure_category, failure_detail, created_at, updated_at, completed_at, input_digest, input_disposition FROM backend_jobs WHERE workspace_id = ?1 AND job_id = ?2", params![workspace_id, job_id], |row| Ok(( row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?, row.get::<_, i64>(3)?, row.get::<_, Option>(4)?, row.get::<_, Option>(5)?, row.get::<_, Option>(6)?, row.get::<_, Option>(7)?, row.get::<_, String>(8)?, - row.get::<_, String>(9)?, row.get::<_, Option>(10)?, + row.get::<_, String>(9)?, row.get::<_, Option>(10)?, row.get::<_, String>(11)?, row.get::<_, String>(12)?, )), ).optional()?; let Some(( @@ -12202,11 +12278,20 @@ fn read_backend_job( created_at, updated_at, completed_at, + input_digest, + input_disposition, )) = row else { return Ok(None); }; - let request: BackendJobRequest = serde_json::from_str(&request_json) + let request_value: serde_json::Value = serde_json::from_str(&request_json) + .map_err(|error| Error::Store(format!("invalid stored Backend Job request: {error}")))?; + if request_value.get("input").is_none() { + return Err(Error::Store( + "stored Backend Job input field is missing".into(), + )); + } + let request: BackendJobRequest = serde_json::from_value(request_value) .map_err(|error| Error::Store(format!("invalid stored Backend Job request: {error}")))?; let result = result_json .map(|value| serde_json::from_str(&value)) @@ -12218,6 +12303,9 @@ fn read_backend_job( workspace_id: workspace_id.to_string(), request, intent_fingerprint, + input_digest, + input_disposition: serde_json::from_value(serde_json::Value::String(input_disposition)) + .map_err(|e| Error::Store(format!("invalid Job input disposition: {e}")))?, state: BackendJobState::parse(&state)?, current_attempt, result, @@ -15130,6 +15218,24 @@ fn migrate_result_reports_v97_to_v98(conn: &Connection) -> Result<()> { Ok(()) } +fn migrate_job_input_retention_v102_to_v103(conn: &Connection) -> Result<()> { + if current_schema_version(conn)? != 102 { + return Err(Error::Store( + "Job input retention migration requires schema 102".into(), + )); + } + let tx = conn.unchecked_transaction()?; + tx.execute_batch("ALTER TABLE backend_jobs ADD COLUMN input_disposition TEXT NOT NULL DEFAULT 'available' CHECK (input_disposition IN ('available','reclaimed','erased','invalidated'));")?; + tx.execute_batch(crate::backend_job::INPUT_RETENTION_TRIGGERS)?; + tx.execute( + "INSERT INTO __yoi_schema_migrations(version,name) VALUES(103,?1)", + ["Job input retirement independent of immutable receipts"], + )?; + verify_workspace_resource_constraints(&tx)?; + tx.commit()?; + Ok(()) +} + fn migrate_subject_create_recovery_v101_to_v102(conn: &Connection) -> Result<()> { if current_schema_version(conn)? != 101 { return Err(Error::Store( @@ -18534,7 +18640,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx assert!(conn.execute("INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at) VALUES('invalid','missing','Invalid','active','1','1')", []).is_err()); } - fn prepare_retained_schema(path: &Path, version: i64) { + pub(super) fn prepare_retained_schema(path: &Path, version: i64) { // Cache only empty preconditions, built from frozen DDL and real retained // migrations. The test's seeded rows and migration under test are never shared. static TEMPLATES: std::sync::OnceLock>> = @@ -18914,6 +19020,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx WorkspaceSchemaMigrationStep {version:100, name:"Provisional users and scoped passkey activation".into()}, WorkspaceSchemaMigrationStep {version:101, name:"Durable explicit Subject body release".into()}, WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()}, + WorkspaceSchemaMigrationStep {version:103, name:"Job input retirement independent of immutable receipts".into()}, ] ); @@ -19026,6 +19133,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx (100, "Provisional users and scoped passkey activation".to_string()), (101, "Durable explicit Subject body release".to_string()), (102, "Durable failed Subject creation recovery".to_string()), +(103, "Job input retirement independent of immutable receipts".to_string()), ] ); assert!(!table_exists(conn, "trusted_runtime_records")?); @@ -21293,7 +21401,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx drop(conn); let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap(); reopened - .with_conn(|c| verify_schema_history(c, 102)) + .with_conn(|c| verify_schema_history(c, LATEST_SCHEMA_VERSION)) .unwrap(); } @@ -22232,7 +22340,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx (99, "Explicit hosting administrators and self-registration policy".into()), (100, "Provisional users and scoped passkey activation".into()), (101, "Durable explicit Subject body release".into()), - (102, "Durable failed Subject creation recovery".into())]); + (102, "Durable failed Subject creation recovery".into()), + (103, "Job input retirement independent of immutable receipts".into())]); for (table, rows) in unchanged.iter().zip(before.iter()) { assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}"); } @@ -26150,6 +26259,321 @@ INSERT INTO worker_registry ( .unwrap(); } + // Storage component tests: seed already-saved result/cleanup receipts, without + // constructing an execution Worker or claiming a model/dispatch integration. + fn seed_completed_job_receipts(store: &SqliteWorkspaceStore, id: &str, cleanup: &str) { + let result = serde_json::json!({"stored":true}); + let (digest, json) = result_digest(&result, job::ABSOLUTE_MAX_RESULT_BYTES).unwrap(); + store.with_conn(|conn| { + conn.execute("UPDATE backend_job_attempts SET state='completed',result_json=?2,result_digest=?3,worker_cleanup_state=?4,worker_cleanup_completed_at=?5 WHERE job_id=?1", params![id,json,digest,cleanup,RESOURCE_JOB_NOW])?; + conn.execute("UPDATE backend_jobs SET state='completed',result_json=?2,result_digest=?3,completed_at=?4 WHERE job_id=?1", params![id,json,digest,RESOURCE_JOB_NOW])?; + Ok(()) + }).unwrap(); + } + + #[tokio::test] + async fn backend_job_input_reclaim_preserves_receipts_reopen_and_exact_replay() { + use crate::backend_job::{JobInputDisposition, JobInputRetirementReason}; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("input.sqlite"); + let store = SqliteWorkspaceStore::open_api_fixture(&path).unwrap(); + seed_resource_job_workspace(&store).await; + let original = resource_job_request("reclaim", "subject-a", "candidate"); + let reservation = store + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap(); + assert!( + store + .reclaim_backend_job_input( + "resource-workspace", + "reclaim", + &reservation.job.input_digest, + JobInputRetirementReason::Completed + ) + .is_err() + ); + seed_completed_job_receipts(&store, "reclaim", "pending"); + assert!( + store + .reclaim_backend_job_input( + "resource-workspace", + "reclaim", + &reservation.job.input_digest, + JobInputRetirementReason::Completed + ) + .is_err() + ); + store.with_conn(|conn| { conn.execute("UPDATE backend_job_attempts SET worker_cleanup_state='completed' WHERE job_id='reclaim'",[])?; Ok(()) }).unwrap(); + let before = store + .get_backend_job("resource-workspace", "reclaim") + .unwrap() + .unwrap(); + let attempt = store + .get_backend_job_attempt( + "resource-workspace", + "reclaim", + &reservation.attempt.attempt_id, + ) + .unwrap() + .unwrap(); + assert!( + store + .reclaim_backend_job_input( + "resource-workspace", + "reclaim", + "wrong", + JobInputRetirementReason::Completed + ) + .is_err() + ); + let retired = store + .reclaim_backend_job_input( + "resource-workspace", + "reclaim", + &before.input_digest, + JobInputRetirementReason::Completed, + ) + .unwrap(); + assert_eq!(retired.input_disposition, JobInputDisposition::Reclaimed); + assert!(retired.request.input.is_null()); + assert!(retired.load_executable_request().is_err()); + assert_eq!(retired.input_digest, original.input_digest().unwrap()); + assert_eq!(retired.intent_fingerprint, before.intent_fingerprint); + assert_eq!(retired.request.grants, original.grants); + assert_eq!(retired.result, before.result); + assert_eq!( + store + .get_backend_job_attempt("resource-workspace", "reclaim", &attempt.attempt_id) + .unwrap() + .unwrap(), + attempt + ); + drop(store); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + let replay = reopened + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap(); + assert!(replay.replayed); + assert_eq!(replay.job, retired); + assert_eq!( + replay.outcome().input_digest, + original.input_digest().unwrap() + ); + let mut changed = original.clone(); + changed.input = serde_json::json!({"changed":true}); + assert!( + reopened + .reserve_backend_job("resource-workspace", &changed, RESOURCE_JOB_NOW) + .is_err() + ); + assert!( + reopened + .reserve_backend_job_retry("resource-workspace", "reclaim", RESOURCE_JOB_NOW) + .is_err() + ); + assert!( + reopened + .claim_backend_job_attempt_dispatch( + "resource-workspace", + "reclaim", + &attempt.attempt_id, + RESOURCE_JOB_NOW + ) + .is_err() + ); + assert!(reopened.with_conn(|conn| { conn.execute("UPDATE backend_jobs SET input_disposition='available',request_json=?1 WHERE job_id='reclaim'", [serde_json::to_string(&original).unwrap()])?; Ok(()) }).is_err()); + let erased = reopened + .reclaim_backend_job_input( + "resource-workspace", + "reclaim", + &retired.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert_eq!(erased.input_disposition, JobInputDisposition::Erased); + assert_eq!(erased.state, BackendJobState::Completed); + assert_eq!(erased.result, before.result); + } + + #[tokio::test] + async fn backend_job_input_unknown_retention_and_legacy_invalidation_do_not_reinterpret_outcome() + { + use crate::backend_job::{JobInputDisposition, JobInputRetirementReason}; + let store = SqliteWorkspaceStore::in_memory_fixture().unwrap(); + seed_resource_job_workspace(&store).await; + for (id, state) in [ + ("unfinished", None), + ("unknown-input", Some(BackendJobAttemptState::Unknown)), + ("failed-input", Some(BackendJobAttemptState::Failed)), + ] { + let original = resource_job_request(id, id, "candidate"); + let reservation = store + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap(); + if let Some(state) = state { + store + .finish_backend_job_attempt( + "resource-workspace", + id, + &reservation.attempt.attempt_id, + state, + "fixture", + "known stored outcome", + RESOURCE_JOB_NOW, + ) + .unwrap(); + } + let before = store + .get_backend_job("resource-workspace", id) + .unwrap() + .unwrap(); + assert!( + store + .reclaim_backend_job_input( + "resource-workspace", + id, + &before.input_digest, + JobInputRetirementReason::Completed + ) + .is_err() + ); + assert_eq!(before.load_executable_request().unwrap(), &original); + assert_eq!( + store + .get_backend_job("resource-workspace", id) + .unwrap() + .unwrap(), + before + ); + let retired = store + .reclaim_backend_job_input( + "resource-workspace", + id, + &before.input_digest, + JobInputRetirementReason::LegacyInvalidation, + ) + .unwrap(); + assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated); + assert_eq!(retired.state, before.state); + assert_eq!(retired.failure_category, before.failure_category); + assert_eq!(retired.completed_at, before.completed_at); + assert!(retired.load_executable_request().is_err()); + assert!( + store + .reserve_backend_job_retry("resource-workspace", id, RESOURCE_JOB_NOW) + .is_err() + ); + assert!( + store + .claim_backend_job_attempt_dispatch( + "resource-workspace", + id, + &reservation.attempt.attempt_id, + RESOURCE_JOB_NOW + ) + .is_err() + ); + assert_eq!( + store + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap() + .job, + retired + ); + } + } + + #[tokio::test] + async fn backend_job_input_available_loader_and_dispatch_refuse_missing_tampered_or_changed_intent() + { + for mutation in [ + "json_set(request_json,'$.input',json('{\"tampered\":true}'))", + "json_remove(request_json,'$.input')", + "json_set(request_json,'$.instruction','changed')", + ] { + let store = SqliteWorkspaceStore::in_memory_fixture().unwrap(); + seed_resource_job_workspace(&store).await; + let original = resource_job_request("tamper", "subject", "candidate"); + let reservation = store + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap(); + store + .with_conn(|conn| { + conn.execute_batch("DROP TRIGGER backend_job_input_retirement")?; + conn.execute_batch(&format!( + "UPDATE backend_jobs SET request_json={mutation}" + ))?; + Ok(()) + }) + .unwrap(); + if let Ok(Some(job)) = store.get_backend_job("resource-workspace", "tamper") { + assert!(job.load_executable_request().is_err()); + } + assert!( + store + .claim_backend_job_attempt_dispatch( + "resource-workspace", + "tamper", + &reservation.attempt.attempt_id, + RESOURCE_JOB_NOW + ) + .is_err() + ); + } + } + + #[tokio::test] + async fn backend_job_input_migration_preserves_original_identity_and_rolls_back_failure() { + use crate::backend_job::{JobInputDisposition, JobInputRetirementReason}; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("migration.sqlite"); + let store = SqliteWorkspaceStore::open_api_fixture(&path).unwrap(); + seed_resource_job_workspace(&store).await; + let original = resource_job_request("upgrade", "subject", "candidate"); + let before = store + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap(); + store.with_conn(|conn| { + conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable; + ALTER TABLE backend_jobs DROP COLUMN input_disposition; + UPDATE __yoi_schema_migrations SET version=102 WHERE version=103; + CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(()) + }).unwrap(); + drop(store); + assert!(SqliteWorkspaceStore::open_for_test(&path).is_err()); + let conn = Connection::open(&path).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 102); + assert!(!column_exists(&conn, "backend_jobs", "input_disposition").unwrap()); + conn.execute_batch("DROP TRIGGER backend_job_input_retirement") + .unwrap(); + drop(conn); + let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap(); + assert_eq!( + reopened + .get_backend_job("resource-workspace", "upgrade") + .unwrap() + .unwrap(), + before.job + ); + let erased = reopened + .reclaim_backend_job_input( + "resource-workspace", + "upgrade", + &before.job.input_digest, + JobInputRetirementReason::ExplicitErasure, + ) + .unwrap(); + assert_eq!(erased.input_disposition, JobInputDisposition::Erased); + assert_eq!(erased.state, BackendJobState::Pending); + assert_eq!( + reopened + .reserve_backend_job("resource-workspace", &original, RESOURCE_JOB_NOW) + .unwrap() + .job, + erased + ); + } + #[tokio::test] async fn backend_job_resource_reserve_converges_across_connections_without_changing_input() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/workspace-server/src/ticket_item_checker.rs b/crates/workspace-server/src/ticket_item_checker.rs index 254312e00..b1f683e77 100644 --- a/crates/workspace-server/src/ticket_item_checker.rs +++ b/crates/workspace-server/src/ticket_item_checker.rs @@ -231,12 +231,10 @@ pub fn result_facts( current_ticket: &ticket::Ticket, ) -> Result { let input = parse_input(&acceptance.job.request.input)?; - let result = validate_result( - &acceptance.job.request.input, - acceptance.job.result.as_ref().ok_or_else(|| { - Error::InvalidInput("completed Ticket item checker Job has no result".to_string()) - })?, - )?; + let result_value = acceptance.job.result.as_ref().ok_or_else(|| { + Error::InvalidInput("completed Ticket item checker Job has no result".to_string()) + })?; + let result = validate_result(&acceptance.job.request.input, result_value)?; if input.content_digest != content_digest(current_ticket) || input.ticket_id != current_ticket.meta.id { @@ -246,21 +244,17 @@ pub fn result_facts( return Ok(TicketItemCheckFacts::NoFindings); } - Ok(TicketItemCheckFacts::Findings(render_result(acceptance)?)) + Ok(TicketItemCheckFacts::Findings(render_result( + &acceptance.job.request.input, + result_value, + )?)) } /// Stable rendering from the original checked snapshot. Used to pin pre-upgrade /// unacknowledged requests without reinterpreting them against newer Ticket text. -pub(crate) fn render_result(acceptance: &BackendJobResultAcceptance) -> Result { - let input = parse_input(&acceptance.job.request.input)?; - let result = validate_result( - &acceptance.job.request.input, - acceptance - .job - .result - .as_ref() - .ok_or_else(|| Error::InvalidInput("completed checker has no result".into()))?, - )?; +pub(crate) fn render_result(input: &Value, result: &Value) -> Result { + let checked = parse_input(input)?; + let result = validate_result(input, result)?; if result.findings.is_empty() { return Err(Error::InvalidInput( "clean checker result has no advisory body".into(), @@ -268,7 +262,7 @@ pub(crate) fn render_result(acceptance: &BackendJobResultAcceptance) -> Result