diff --git a/Cargo.lock b/Cargo.lock index e067dc4a5..4bb517eb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6575,6 +6575,7 @@ dependencies = [ "flow", "fs4", "futures", + "libc", "manifest", "protocol", "reqwest", diff --git a/crates/worker-runtime/Cargo.toml b/crates/worker-runtime/Cargo.toml index 3f4a327c3..ef2f92af8 100644 --- a/crates/worker-runtime/Cargo.toml +++ b/crates/worker-runtime/Cargo.toml @@ -31,6 +31,7 @@ decodal.workspace = true flow = { path = "../flow" } fs4 = { workspace = true, optional = true } manifest.workspace = true +libc.workspace = true protocol.workspace = true runtime-api.workspace = true serde = { workspace = true, features = ["derive"] } diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 0326f899a..df39f7651 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -2486,6 +2486,8 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode { if matches!( diagnostic.code.as_str(), "repository_ref_provider_unavailable" + | "working_directory_cleanup_mount_check_unavailable" + | "working_directory_cleanup_storage_unavailable" | "repository_ref_provider_timeout" | "repository_access_provider_unavailable" ) => @@ -2504,6 +2506,23 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode { { StatusCode::FORBIDDEN } + RuntimeError::WorkingDirectory(diagnostic) + if matches!( + diagnostic.code.as_str(), + "working_directory_cleanup_mount_present" + | "working_directory_cleanup_resource_busy" + | "working_directory_cleanup_ownership_unknown" + | "working_directory_cleanup_changes_present" + | "working_directory_cleanup_changes_unknown" + ) => + { + StatusCode::CONFLICT + } + RuntimeError::WorkingDirectory(diagnostic) + if diagnostic.code == "working_directory_cleanup_permission_denied" => + { + StatusCode::FORBIDDEN + } RuntimeError::WorkingDirectory(diagnostic) if diagnostic.code == "repository_ref_not_found" => { @@ -4322,6 +4341,37 @@ mod tests { assert!(!error.to_string().contains('/')); } + #[tokio::test] + async fn cleanup_cause_codes_survive_http_error_serialization() { + let cases = [ + ("mount_present", StatusCode::CONFLICT), + ("mount_check_unavailable", StatusCode::SERVICE_UNAVAILABLE), + ("permission_denied", StatusCode::FORBIDDEN), + ("resource_busy", StatusCode::CONFLICT), + ("storage_unavailable", StatusCode::SERVICE_UNAVAILABLE), + ("ownership_unknown", StatusCode::CONFLICT), + ("changes_present", StatusCode::CONFLICT), + ("changes_unknown", StatusCode::CONFLICT), + ]; + for (cause, status) in cases { + let code = format!("working_directory_cleanup_{cause}"); + let diagnostic = crate::working_directory::WorkingDirectoryDiagnostic::rejected( + &code, + "Resolve the current blocker and retry removal.", + ); + let response = + RuntimeHttpRestError::runtime(RuntimeError::WorkingDirectory(diagnostic)) + .into_response(); + assert_eq!(response.status(), status); + let body = axum::body::to_bytes(response.into_body(), 2048) + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(json["error"]["code"], code, "{json}"); + assert!(json["error"]["message"].as_str().unwrap().contains("retry")); + } + } + #[test] fn workdir_runtime_errors_preserve_diagnostic_code() { let cases = [ diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 543d89244..3489140e9 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -819,23 +819,30 @@ impl Runtime { &self, working_directory_id: &str, ) -> Result { - let backend = { - let state = self.lock()?; - state.ensure_running()?; - if let Some(worker_id) = state.worker_id_for_workdir(working_directory_id) { - return Err(RuntimeError::InvalidRequest(format!( - "working directory {working_directory_id} is assigned to worker {worker_id}" - ))); + // Hold occupancy authority through the side effect, preventing an + // attachment from entering between the check and physical removal. + // This synchronous provider call must not reenter Runtime. The Git + // materializer does filesystem/Git work only. The global guard also + // delays unrelated state operations during hashing/unlink; releasing it + // would require equivalent per-workdir exclusion on ALL attachment paths, + // not just moving the occupied check before the provider call. + let state = self.lock()?; + state.ensure_running()?; + if let Some(worker_id) = state.worker_id_for_workdir(working_directory_id) { + return Err(RuntimeError::InvalidRequest(format!( + "working directory {working_directory_id} is assigned to worker {worker_id}" + ))); + } + let backend = state.execution_backend.clone().ok_or_else(|| { + RuntimeError::ExecutionBackendUnavailable { + message: "working directory cleanup requires an execution backend".to_string(), } - state.execution_backend.clone().ok_or_else(|| { - RuntimeError::ExecutionBackendUnavailable { - message: "working directory cleanup requires an execution backend".to_string(), - } - })? - }; - backend + })?; + let result = backend .cleanup_working_directory(working_directory_id) - .map_err(RuntimeError::from) + .map_err(RuntimeError::from); + drop(state); + result } fn annotate_working_directory_statuses( @@ -7665,6 +7672,7 @@ mod tests { repository_accesses: Mutex>, repository_access_available: AtomicBool, working_directory_requests: Mutex>, + cleanup_occupancy_probe: Mutex>>>, preserve_submission_acknowledgement_id: AtomicBool, #[cfg(feature = "ws-server")] snapshots: Mutex>, @@ -7705,6 +7713,22 @@ mod tests { } impl WorkerExecutionBackend for TestExecutionBackend { + fn cleanup_working_directory( + &self, + _id: &str, + ) -> Result { + if let Some(state) = self.cleanup_occupancy_probe.lock().unwrap().as_ref() { + assert!( + matches!(state.try_lock(), Err(std::sync::TryLockError::WouldBlock)), + "occupancy must remain exclusive during physical cleanup" + ); + } + Err(WorkingDirectoryDiagnostic::rejected( + "working_directory_cleanup_resource_busy", + "Retry removal after releasing the resource", + )) + } + fn backend_id(&self) -> &str { "test-execution-backend" } @@ -8043,6 +8067,22 @@ mod tests { .block_on(subscription.recv()) } + #[test] + fn workdir_removal_keeps_occupancy_exclusive_during_provider_side_effect() { + let (runtime, backend) = runtime_and_backend(); + *backend.cleanup_occupancy_probe.lock().unwrap() = Some(runtime.inner.clone()); + let error = runtime + .cleanup_working_directory("workdir-test") + .unwrap_err(); + assert!( + matches!(error, RuntimeError::WorkingDirectory(ref diagnostic) + if diagnostic.code == "working_directory_cleanup_resource_busy") + ); + // The lock is released on failure, permitting normal attachment/retry. + assert!(runtime.inner.try_lock().is_ok()); + *backend.cleanup_occupancy_probe.lock().unwrap() = None; + } + #[test] fn unknown_typed_intent_is_rejected_before_backend_dispatch() { let (runtime, backend) = runtime_and_backend(); diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index d338848a8..9ac79973b 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -19,7 +19,6 @@ use crate::catalog::{ CreateWorkerRequest, ProfileSourceArchiveSource, RepositoryRefObservation, RepositoryRefObservationRequest, WorkingDirectoryAttachmentStatus, WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest, WorkingDirectoryStatus, - WorkingDirectoryStatusKind, }; use crate::config_bundle::{ConfigBundle, workspace_config_etag}; #[cfg(feature = "ws-server")] @@ -3773,28 +3772,24 @@ where ) })?; for previous in &request.previous_workdir_attachments { - let current = materializer - .working_directory_status( - &previous.working_directory.summary.working_directory_id, - ) - .map_err(|message| { - WorkerExecutionResult::rejected( - WorkerExecutionOperation::Restore, - format!( - "Persisted Worker Workdir attachment `{}` is unavailable: {message}", - previous.alias - ), - ) - })?; - if current.summary.status != WorkingDirectoryStatusKind::Active { - return Err(WorkerExecutionResult::rejected( + let relative_cwd = request + .request + .workdir_attachments + .iter() + .find(|claim| claim.alias == previous.alias) + .and_then(|claim| claim.relative_cwd.as_deref()); + // Recovery attachments remain CleanupPending across restart. + // Validate current bindability/identity, not the old state name. + // Preflight is read-only; actual restore binds again below. + materializer.preflight_bind_working_directory( + &previous.working_directory.summary.working_directory_id, + relative_cwd, + ).map_err(|message| { + WorkerExecutionResult::rejected( WorkerExecutionOperation::Restore, - format!( - "Persisted Worker Workdir attachment `{}` is not active", - previous.alias - ), - )); - } + format!("Persisted Worker Workdir attachment `{}` is unavailable: {message}", previous.alias), + ) + })?; } } if request.previous_workdir_attachments.is_empty() @@ -4637,7 +4632,7 @@ mod tests { ConfigBundleRef, CreateWorkerRequest, LogicalWorkdirAttachment, MaterializerKind, ProfileSelector, RepositorySelector, WorkingDirectoryAttachmentClaim, WorkingDirectoryAttachmentRequest, WorkingDirectoryRepository, WorkingDirectoryRequest, - WorkspaceApiRef, + WorkingDirectoryStatusKind, WorkspaceApiRef, }; use crate::execution::WorkerExecutionContext; use crate::identity::WorkerId; @@ -8919,6 +8914,160 @@ mod tests { ); } + #[cfg(target_os = "linux")] + fn pending_owner_restore_fixture() -> ( + tempfile::TempDir, + WorkerRuntimeExecutionBackend, + WorkerExecutionRestoreRequest, + PathBuf, + Arc>>, + ) { + use crate::working_directory::{WorkingDirectoryMaterializer, fail_workdir_cleanup_once}; + let root = tempfile::tempdir().unwrap(); + let repo = create_clean_repo(); + let workdirs = root.path().join("workdirs"); + let materializer = RuntimeGitMaterializer::new(&workdirs); + let binding = materializer + .create(&working_directory_request(repo.path())) + .unwrap(); + let id = binding.working_directory.id.clone(); + fail_workdir_cleanup_once(&materializer, &id); + // The cleanup owner has legitimately reattached to preserve these changes. + fs::write(binding.root.join("README.md"), "owner recovery in progress").unwrap(); + let observed = Arc::new(Mutex::new(Vec::new())); + let factory = || MockFactory { + client: MockClient::new(simple_text_events()), + runtime_base: root.path().join("runtime"), + cwd: root.path().to_path_buf(), + store_dir: root.path().join("sessions"), + worker_metadata_dir: root.path().join("workers"), + observed_cwds: observed.clone(), + observed_workspace_clients: Arc::new(Mutex::new(Vec::new())), + }; + let backend = Arc::new( + WorkerRuntimeExecutionBackend::new(factory()) + .unwrap() + .with_working_directory_materializer(materializer), + ); + let runtime = + EmbeddedRuntime::with_execution_backend(RuntimeOptions::default(), backend.clone()) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let mut request = create_request("cleanup-owner"); + request.workdir_attachments = vec![WorkingDirectoryAttachmentClaim { + alias: WorkdirAttachmentAlias::new("checkout").unwrap(), + working_directory_id: id, + relative_cwd: None, + capabilities: WorkdirSessionCapabilities::ALL, + }]; + let detail = runtime.create_worker(request.clone()).unwrap(); + runtime.stop_worker(&detail.worker_ref, None).unwrap(); + let restore = WorkerExecutionRestoreRequest { + operation_id: crate::execution::WorkerLifecycleOperationId::new(), + worker_ref: detail.worker_ref.clone(), + request, + workspace_scope: None, + context: test_execution_context(detail.worker_ref), + previous_workdir_attachments: detail.workdir_attachments, + logical_workdir_attachments: Vec::new(), + workdir_attachments: BTreeMap::new(), + config_bundle: None, + }; + drop(runtime); + drop(backend); + // A fresh backend/materializer loads the durable pending identity evidence. + let restarted = WorkerRuntimeExecutionBackend::new(factory()) + .unwrap() + .with_working_directory_materializer(RuntimeGitMaterializer::new(&workdirs)); + (root, restarted, restore, binding.root, observed) + } + + #[cfg(target_os = "linux")] + #[test] + fn cleanup_pending_owner_restores_after_restart_with_validated_dirty_checkout() { + let (root, backend, request, checkout, observed) = pending_owner_restore_fixture(); + let before = persisted_files(&root.path().join("workdirs")); + backend.preflight_restore(&request).unwrap(); + assert_eq!( + persisted_files(&root.path().join("workdirs")), + before, + "preflight must not rewrite removal or Git authority" + ); + let restored = match backend.restore_worker(request.clone()) { + WorkerExecutionSpawnResult::Connected { + workdir_attachments, + .. + } => workdir_attachments, + other => panic!("cleanup-owner restore failed: {other:?}"), + }; + assert_eq!(restored.len(), 1); + assert_eq!( + restored[0].working_directory.summary.status, + WorkingDirectoryStatusKind::CleanupPending + ); + assert_eq!( + restored[0].working_directory.summary.cleanliness.as_deref(), + Some("dirty") + ); + assert_eq!( + observed.lock().unwrap().as_slice(), + &[checkout.clone(), checkout.clone()] + ); + assert_eq!( + fs::read_to_string(checkout.join("README.md")).unwrap(), + "owner recovery in progress" + ); + assert!(backend.stop_worker(&request.worker_ref).is_accepted()); + } + + #[cfg(target_os = "linux")] + #[test] + fn cleanup_pending_owner_restore_rejects_incomplete_replaced_and_raced_checkouts() { + for damage in ["missing_git", "replacement", "missing_after_preflight"] { + let (root, backend, request, checkout, observed) = pending_owner_restore_fixture(); + if damage == "missing_after_preflight" { + backend.preflight_restore(&request).unwrap(); + } + if damage == "replacement" { + fs::rename(&checkout, root.path().join("original-checkout")).unwrap(); + fs::create_dir(&checkout).unwrap(); + fs::create_dir(checkout.join(".git")).unwrap(); + } else { + fs::remove_dir_all(checkout.join(".git")).unwrap(); + } + let before = persisted_files(root.path()); + let error = backend.preflight_restore(&request).unwrap_err(); + assert_eq!( + error.outcome, + crate::execution::WorkerExecutionOutcome::Rejected, + "{damage}" + ); + assert!( + error + .message_or_default() + .contains("working_directory_cleanup_"), + "{error:?}" + ); + assert!( + matches!( + backend.restore_worker(request), + WorkerExecutionSpawnResult::Rejected(_) + ), + "{damage}" + ); + assert_eq!( + observed.lock().unwrap().len(), + 1, + "must not start a Controller on invalid checkout: {damage}" + ); + assert_eq!( + persisted_files(root.path()), + before, + "must preserve damaged/replaced checkout: {damage}" + ); + } + } + #[test] fn worker_spawn_receives_materialized_workspace_cwd_instead_of_source_repo() { let client = MockClient::new(simple_text_events()); diff --git a/crates/worker-runtime/src/working_directory.rs b/crates/worker-runtime/src/working_directory.rs index e44c574bc..178850a55 100644 --- a/crates/worker-runtime/src/working_directory.rs +++ b/crates/worker-runtime/src/working_directory.rs @@ -22,6 +22,11 @@ use std::thread::JoinHandle; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use workdir::WorkdirSessionResource; +mod cleanup; + +#[cfg(all(test, target_os = "linux"))] +pub(crate) use cleanup::tests::fail_once as fail_workdir_cleanup_once; + const CHECKOUT_DIR: &str = "checkout"; const MATERIALIZATION_RECORD: &str = "materialization.json"; const REPOSITORY_ACCESS_DIR: &str = ".repository-access"; @@ -127,7 +132,11 @@ impl WorkingDirectoryBinding { working_directory.status = WorkingDirectoryStatusKind::Corrupted; } let mut summary = working_directory.status_summary(); - summary.cleanliness = if summary.status == WorkingDirectoryStatusKind::Active { + summary.cleanliness = if matches!( + summary.status, + WorkingDirectoryStatusKind::Active | WorkingDirectoryStatusKind::CleanupPending + ) && binding_paths_are_available(self) + { let (current_selector, current_ref, current_tree) = binding_current_revision(self); summary.current_selector = current_selector; summary.current_ref = current_ref; @@ -201,6 +210,24 @@ pub trait WorkingDirectoryMaterializer: Send + Sync + 'static { relative_cwd: Option<&str>, ) -> Result; + /// Read-only bind validation: do not consume Repository credentials or open + /// session resources during restore preflight. Providers must opt in with + /// current usable-checkout validation to support non-active attachments. + fn preflight_bind_working_directory( + &self, + working_directory_id: &str, + _relative_cwd: Option<&str>, + ) -> Result<(), WorkingDirectoryDiagnostic> { + let current = self.working_directory_status(working_directory_id)?; + if current.summary.status != WorkingDirectoryStatusKind::Active { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_not_active", + "Persisted Workdir is not active and its provider cannot validate recovery binding", + )); + } + Ok(()) + } + fn list_working_directories( &self, ) -> Result, WorkingDirectoryDiagnostic>; @@ -627,6 +654,29 @@ impl RuntimeGitMaterializer { }) } + fn checked_binding( + &self, + working_directory_id: &str, + relative_cwd: Option<&str>, + ) -> Result { + validate_working_directory_id(working_directory_id)?; + let binding = self.read_binding(working_directory_id)?; + if !matches!( + binding.working_directory.status, + WorkingDirectoryStatusKind::Active | WorkingDirectoryStatusKind::CleanupPending + ) { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_not_active", + "working directory working_directory is not active", + )); + } + if binding.working_directory.status == WorkingDirectoryStatusKind::CleanupPending { + cleanup::validate_recovery_binding(self, working_directory_id, &binding)?; + } + let cwd = validate_relative_cwd(binding.root(), relative_cwd)?; + Ok(WorkingDirectoryBinding { cwd, ..binding }) + } + fn read_binding( &self, working_directory_id: &str, @@ -1137,17 +1187,17 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer { working_directory_id: &str, relative_cwd: Option<&str>, ) -> Result { - validate_working_directory_id(working_directory_id)?; - let binding = self.read_binding(working_directory_id)?; - if binding.working_directory.status != WorkingDirectoryStatusKind::Active { - return Err(WorkingDirectoryDiagnostic::new( - "working_directory_not_active", - "working directory working_directory is not active", - )); - } - let binding = self.bind_repository_access(working_directory_id, binding)?; - let cwd = validate_relative_cwd(binding.root(), relative_cwd)?; - Ok(WorkingDirectoryBinding { cwd, ..binding }) + let binding = self.checked_binding(working_directory_id, relative_cwd)?; + self.bind_repository_access(working_directory_id, binding) + } + + fn preflight_bind_working_directory( + &self, + working_directory_id: &str, + relative_cwd: Option<&str>, + ) -> Result<(), WorkingDirectoryDiagnostic> { + self.checked_binding(working_directory_id, relative_cwd) + .map(|_| ()) } fn list_working_directories( @@ -1178,6 +1228,10 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer { { continue; } + if let Some(status) = cleanup::remaining_status(self, &working_directory_id) { + statuses.push(status); + continue; + } match self.read_binding(&working_directory_id) { Ok(binding) => statuses.push(binding.status()), Err(_) => statuses.push(self.corrupted_status(&working_directory_id)), @@ -1197,11 +1251,24 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer { ) -> Result { validate_working_directory_id(working_directory_id)?; let working_directory_root = self.working_directory_root(working_directory_id); - if !working_directory_root.exists() { - return Err(WorkingDirectoryDiagnostic::new( - "working_directory_not_found", - "working directory working_directory was not found", - )); + match fs::symlink_metadata(&working_directory_root) { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_not_found", + "Runtime working directory was not found", + )); + } + Err(error) => { + return Err(cleanup::os_failure( + working_directory_id, + "lookup_cleanup_root", + error, + )); + } + } + if let Some(status) = cleanup::remaining_status(self, working_directory_id) { + return Ok(status); } match self.read_binding(working_directory_id) { Ok(binding) => Ok(binding.status()), @@ -1214,80 +1281,13 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer { working_directory_id: &str, ) -> Result { validate_working_directory_id(working_directory_id)?; - let status = self.working_directory_status(working_directory_id)?; - if status.summary.status == WorkingDirectoryStatusKind::Corrupted { - let working_directory_root = self.working_directory_root(working_directory_id); - if working_directory_root.exists() { - fs::remove_dir_all(&working_directory_root).map_err(|_| { - WorkingDirectoryDiagnostic::new( - "working_directory_corrupted_cleanup_failed", - "failed to remove corrupted working directory; backend-private path details were omitted", - ) - })?; - } - let mut summary = status.summary; - summary.status = WorkingDirectoryStatusKind::NotFound; - return Ok(WorkingDirectoryStatus { summary }); - } - let binding = self.read_binding(working_directory_id)?; - self.cleanup(&binding)?; - let mut summary = binding.working_directory.status_summary(); - summary.status = WorkingDirectoryStatusKind::NotFound; - summary.cleanliness = Some("unknown".to_string()); - if binding.working_directory_root.exists() { - fs::remove_dir_all(&binding.working_directory_root).map_err(|_| { - WorkingDirectoryDiagnostic::new( - "working_directory_record_cleanup_failed", - "failed to remove working directory record; backend-private path details were omitted", - ) - })?; - } - Ok(WorkingDirectoryStatus { summary }) + cleanup::remove(self, working_directory_id) } fn cleanup(&self, binding: &WorkingDirectoryBinding) -> Result<(), WorkingDirectoryDiagnostic> { - let mut working_directory = binding.working_directory.clone(); - let working_directory_root = binding.working_directory_root.canonicalize().map_err(|_| { - WorkingDirectoryDiagnostic::new( - "working_directory_cleanup_target_invalid", - "working directory working directory root is unavailable; backend-private path details were omitted", - ) - })?; - let root = binding.root.canonicalize().map_err(|_| { - WorkingDirectoryDiagnostic::new( - "working_directory_cleanup_target_invalid", - "working directory root is unavailable; backend-private path details were omitted", - ) - })?; - if !root.starts_with(&working_directory_root) { - return Err(WorkingDirectoryDiagnostic::new( - "working_directory_cleanup_escape_rejected", - "working directory cleanup target is outside the working directory root", - )); - } - let remove_result = fs::remove_dir_all(&working_directory_root).map_err(|_| { - WorkingDirectoryDiagnostic::new( - "working_directory_cleanup_failed", - "failed to remove working directory; backend-private path details were omitted", - ) - }); - if remove_result.is_err() { - working_directory.status = WorkingDirectoryStatusKind::CleanupPending; - let updated = WorkingDirectoryBinding { - working_directory, - root: binding.root.clone(), - cwd: binding.cwd.clone(), - working_directory_root: binding.working_directory_root.clone(), - command_environment: BTreeMap::new(), - session_resources: Vec::new(), - }; - let _ = self.write_record(&updated); - } else { - let _ = self - .repository_access - .remove_pending(&binding.working_directory.id); - } - remove_result + // Reload current authority rather than trusting a stale in-memory binding. + self.cleanup_working_directory(&binding.working_directory.id) + .map(|_| ()) } } @@ -2826,6 +2826,7 @@ where I: IntoIterator, { let output = Command::new("git") + .env("GIT_OPTIONAL_LOCKS", "0") .arg("-C") .arg(repository_path) .args(args) @@ -3094,7 +3095,7 @@ mod tests { assert!(scheduler_inner.upgrade().is_none()); } - fn git(path: &Path, args: &[&str]) { + pub(super) fn git(path: &Path, args: &[&str]) { let status = Command::new("git") .arg("-C") .arg(path) @@ -3104,7 +3105,7 @@ mod tests { assert!(status.success(), "git {:?} failed", args); } - fn create_clean_repo() -> tempfile::TempDir { + pub(super) fn create_clean_repo() -> tempfile::TempDir { let dir = tempfile::tempdir().unwrap(); git(dir.path(), &["init"]); git( @@ -3118,7 +3119,7 @@ mod tests { dir } - fn request(repo: &Path) -> WorkingDirectoryRequest { + pub(super) fn request(repo: &Path) -> WorkingDirectoryRequest { WorkingDirectoryRequest { repository: WorkingDirectoryRepository { id: "repo-main".to_string(), @@ -3678,6 +3679,8 @@ mod tests { .evidence .host_trust_revision = Some(1); materializer.write_record(&ssh_backed_binding).unwrap(); + #[cfg(target_os = "linux")] + fail_workdir_cleanup_once(&materializer, &id); let mut session_materialization = initial_materialization.clone(); let session_delivery_expiry = repository_access_now_epoch_seconds() + 1; session_materialization @@ -3691,6 +3694,17 @@ mod tests { materialization: session_materialization, }) .unwrap(); + // Restore preflight must validate without consuming this one-shot SSH + // delivery or creating agent/session resources needed by actual bind. + let record_before = + fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(); + materializer + .preflight_bind_working_directory(&id, None) + .unwrap(); + assert_eq!( + fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(), + record_before + ); assert_eq!(materializer.list_working_directories().unwrap().len(), 1); assert_eq!( fs::read_dir(runtime_root.path().join(".repository-agents")) @@ -3916,6 +3930,22 @@ mod tests { restored.bind_working_directory(&id, None).unwrap_err().code, "working_directory_remote_repository_access_required" ); + #[cfg(target_os = "linux")] + { + // Real SSH reauthorization updated access evidence repeatedly, but + // did not replace this clean materialization or require manual recovery. + assert_eq!( + restored + .working_directory_status(&id) + .unwrap() + .summary + .cleanliness + .as_deref(), + Some("clean") + ); + restored.cleanup_working_directory(&id).unwrap(); + assert!(!restored.working_directory_root(&id).exists()); + } } #[test] @@ -4393,7 +4423,7 @@ mod tests { } #[test] - fn corrupted_working_directory_record_can_be_removed() { + fn corrupted_working_directory_without_removal_authority_is_retained() { let runtime_root = tempfile::tempdir().unwrap(); let materializer = RuntimeGitMaterializer::new(runtime_root.path()); let working_directory_id = "workdir-corrupted"; @@ -4406,11 +4436,11 @@ mod tests { .unwrap(); assert_eq!(status.summary.status, WorkingDirectoryStatusKind::Corrupted); - let removed = materializer + let error = materializer .cleanup_working_directory(working_directory_id) - .unwrap(); - assert_eq!(removed.summary.status, WorkingDirectoryStatusKind::NotFound); - assert!(!root.exists()); + .unwrap_err(); + assert_eq!(error.code, "working_directory_cleanup_ownership_unknown"); + assert!(root.exists()); } #[test] diff --git a/crates/worker-runtime/src/working_directory/cleanup.rs b/crates/worker-runtime/src/working_directory/cleanup.rs new file mode 100644 index 000000000..037f01343 --- /dev/null +++ b/crates/worker-runtime/src/working_directory/cleanup.rs @@ -0,0 +1,1575 @@ +//! Ordinary removal authority, not a recovery lifecycle. The content witness is +//! outside the tree being removed so a crash/partial unlink cannot erase it. +//! Every retry checks current identities, mounts and remaining content. Missing +//! entries are expected after partial deletion. The same intact clean checkout +//! may renew its witness after owner recovery, never discarding new ignored content. +use super::*; +use std::io; + +fn cause(kind: &str) -> WorkingDirectoryDiagnostic { + let message = match kind { + "mount_present" => { + "A mounted resource remains. Its owner must safely release it before retrying removal; Runtime will not unmount external or unknown resources." + } + "mount_check_unavailable" => { + "Mount-safe removal is unavailable. Restore Runtime mount-check support and retry." + } + "permission_denied" => { + "Removal was denied by filesystem permissions. Ask the resource owner to resolve permissions and retry." + } + "resource_busy" => { + "A filesystem resource is busy or changed during removal. Release the resource and retry." + } + "storage_unavailable" => { + "Cleanup storage is unavailable. Restore storage access and retry." + } + "ownership_unknown" => { + "The remaining directory cannot be matched to Runtime removal authority. Verify its identity and ownership before removal." + } + "changes_present" => { + "New or changed content is present. Preserve or resolve those changes before retrying removal." + } + _ => { + "The checkout is incomplete and remaining content has no usable removal witness. Preserve residual content outside this Workdir and restore the same usable checkout, or ask its owner to remove verified residual content before retrying. Do not edit removal metadata." + } + }; + WorkingDirectoryDiagnostic::new(format!("working_directory_cleanup_{kind}"), message) +} + +pub(super) fn os_failure(id: &str, action: &str, error: io::Error) -> WorkingDirectoryDiagnostic { + let kind = match error.raw_os_error() { + Some(libc::EXDEV) => "mount_present", + Some(libc::ENOSYS | libc::EOPNOTSUPP | libc::EINVAL) => "mount_check_unavailable", + Some(libc::EACCES | libc::EPERM) => "permission_denied", + Some(libc::EBUSY | libc::ENOTEMPTY | libc::EAGAIN) => "resource_busy", + _ => "storage_unavailable", + }; + tracing::warn!(target: "yoi::workdir_cleanup", working_directory_id = id, + action, cause = kind, raw_os_error = ?error.raw_os_error(), error = %error, + "Runtime-private working directory cleanup failure"); + cause(kind) +} + +pub(super) fn remove( + materializer: &RuntimeGitMaterializer, + id: &str, +) -> Result { + let result = remove_checked(materializer, id); + if let Err(error) = &result { + tracing::warn!(target: "yoi::workdir_cleanup", working_directory_id = id, + action = "cleanup_working_directory", code = %error.code, + "working directory removal blocked"); + } + result +} + +#[cfg(target_os = "linux")] +pub(super) fn remaining_status( + materializer: &RuntimeGitMaterializer, + id: &str, +) -> Option { + let parent = fs::File::open(&materializer.runtime_root).ok()?; + let saved = load_authority(&parent, id).ok()??; + let tree = open_directory(&parent, std::ffi::OsStr::new(id)).ok()?; + if saved.working_directory.id != id || identity(&tree).ok()? != saved.root_identity { + return None; + } + let mut summary = saved.working_directory.status_summary(); + summary.status = WorkingDirectoryStatusKind::CleanupPending; + summary.cleanliness = Some( + match check_mounts( + &materializer.working_directory_root(id), + &fs::read("/proc/self/mountinfo").ok()?, + ) + .and_then(|()| retry_inventory(materializer, id, &tree, &saved)) + { + Ok(_) => "clean", + Err(error) if error.code == "working_directory_cleanup_changes_present" => "dirty", + Err(_) => "unknown", + } + .to_string(), + ); + Some(WorkingDirectoryStatus { summary }) +} + +#[cfg(not(target_os = "linux"))] +pub(super) fn remaining_status( + _materializer: &RuntimeGitMaterializer, + _id: &str, +) -> Option { + None +} + +#[cfg(not(target_os = "linux"))] +fn remove_checked( + _materializer: &RuntimeGitMaterializer, + _id: &str, +) -> Result { + // No fallback to remove_dir_all: mount safety is a precondition. + Err(cause("mount_check_unavailable")) +} + +#[cfg(target_os = "linux")] +use linux::*; + +#[cfg(target_os = "linux")] +fn remove_checked( + materializer: &RuntimeGitMaterializer, + id: &str, +) -> Result { + remove_with_eraser(materializer, id, erase) +} + +#[cfg(target_os = "linux")] +fn remove_with_eraser( + materializer: &RuntimeGitMaterializer, + id: &str, + erase_tree: impl FnOnce(&fs::File, &Path, &BTreeMap) -> Result<(), EraseError>, +) -> Result { + remove_with_checks( + materializer, + id, + || fs::read("/proc/self/mountinfo"), + erase_tree, + ) +} + +#[cfg(target_os = "linux")] +fn remove_with_checks( + materializer: &RuntimeGitMaterializer, + id: &str, + mount_info: impl FnOnce() -> io::Result>, + erase_tree: impl FnOnce(&fs::File, &Path, &BTreeMap) -> Result<(), EraseError>, +) -> Result { + let parent = match fs::File::open(&materializer.runtime_root) { + Ok(parent) => parent, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_not_found", + "Runtime working directory was not found", + )); + } + Err(error) => return Err(os_failure(id, "open_runtime_root", error)), + }; + let tree = match open_directory(&parent, std::ffi::OsStr::new(id)) { + Ok(tree) => tree, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return Err(WorkingDirectoryDiagnostic::new( + "working_directory_not_found", + "Runtime working directory was not found", + )); + } + Err(error) if matches!(error.raw_os_error(), Some(libc::ELOOP | libc::ENOTDIR)) => { + return Err(cause("ownership_unknown")); + } + Err(error) => return Err(os_failure(id, "open_cleanup_root", error)), + }; + let root_identity = identity(&tree).map_err(|e| os_failure(id, "identify_root", e))?; + let saved = load_authority(&parent, id).map_err(|error| { + if error.kind() == io::ErrorKind::InvalidData { + cause("ownership_unknown") + } else { + os_failure(id, "read_removal_authority", error) + } + })?; + // Mount table preflight catches mounted files and same-device bind mounts + // before Git/status/hash reads; descriptor NO_XDEV remains the race guard. + check_mounts( + &materializer.working_directory_root(id), + &mount_info().map_err(|e| { + let _ = os_failure(id, "read_mountinfo", e); + cause("mount_check_unavailable") + })?, + )?; + let authority = if let Some(saved) = saved { + if saved.working_directory.id != id || saved.root_identity != root_identity { + return Err(cause("ownership_unknown")); + } + let entries = retry_inventory(materializer, id, &tree, &saved)?; + if verify_remaining(&saved.entries, &entries).is_ok() { + saved + } else { + let authority = Authority { entries, ..saved }; + // Renew obsolete content evidence only from a current clean, intact + // checkout. Partial deletion without Git retains the old witness. + persist_authority(&parent, id, &authority) + .map_err(|e| os_failure(id, "refresh_removal_authority", e))?; + authority + } + } else { + let mut binding = materializer + .read_binding(id) + .map_err(|_| cause("ownership_unknown"))?; + validate_binding(materializer, id, &binding)?; + // Preflight the entire tree without following links or crossing mounts, + // before asking Git to inspect the checkout. + let before = inventory(&tree).map_err(|e| os_failure(id, "inspect_tree", e))?; + // Validated metadata-only residue contains no user content to discard. + if !metadata_only(&before) { + check_clean_checkout(id, &binding, &before, None)?; + } + if inventory(&tree).map_err(|e| os_failure(id, "recheck_inspected_tree", e))? != before { + return Err(cause("changes_present")); + } + binding.working_directory.status = WorkingDirectoryStatusKind::CleanupPending; + let record = WorkingDirectoryMaterializationRecord { + working_directory: binding.working_directory.clone(), + root: binding.root.clone(), + }; + let raw = serde_json::to_vec_pretty(&record).map_err(|_| cause("storage_unavailable"))?; + rewrite_record(&tree, &raw, before.get(Path::new(MATERIALIZATION_RECORD))) + .map_err(|e| os_failure(id, "write_cleanup_pending", e))?; + let entries = inventory(&tree).map_err(|e| os_failure(id, "capture_removal_content", e))?; + // Only our status-record write may differ between the clean observation + // and durable witness. Never authorize a newly arrived user change. + let mut without_record = entries.clone(); + without_record.remove(Path::new(MATERIALIZATION_RECORD)); + let mut before_without_record = before; + before_without_record.remove(Path::new(MATERIALIZATION_RECORD)); + if without_record != before_without_record { + return Err(cause("changes_present")); + } + let authority = Authority { + working_directory: binding.working_directory.clone(), + root_identity, + entries, + }; + persist_authority(&parent, id, &authority) + .map_err(|e| os_failure(id, "persist_removal_authority", e))?; + authority + }; + let current = inventory(&tree).map_err(|e| os_failure(id, "recheck_remaining_content", e))?; + verify_remaining(&authority.entries, ¤t)?; + // Repeat the content/identity comparison at each unlink. No unverified + // recursion into a replaced entry, even if a mount arrives after preflight. + erase_tree(&tree, Path::new(""), &authority.entries).map_err(|e| match e { + EraseError::Io(e) => os_failure(id, "unlink_remaining_tree", e), + EraseError::Changed => cause("changes_present"), + })?; + let current_root = open_directory(&parent, std::ffi::OsStr::new(id)) + .map_err(|e| os_failure(id, "recheck_root_identity", e))?; + if identity(¤t_root).map_err(|e| os_failure(id, "recheck_root_identity", e))? + != authority.root_identity + { + return Err(cause("ownership_unknown")); + } + unlink(&parent, std::ffi::OsStr::new(id), true) + .map_err(|e| os_failure(id, "unlink_cleanup_root", e))?; + parent + .sync_all() + .map_err(|e| os_failure(id, "sync_physical_removal", e))?; + // The authority is obsolete only after physical removal. Its removal cannot + // convert an already completed physical cleanup to an unknown outcome. + if let Err(error) = discard_authority(&parent, id) { + tracing::warn!(target: "yoi::workdir_cleanup", working_directory_id = id, + action = "remove_obsolete_authority", raw_os_error = ?error.raw_os_error(), error = %error); + } + let _ = materializer.repository_access.remove_pending(id); + let mut summary = authority.working_directory.status_summary(); + summary.status = WorkingDirectoryStatusKind::NotFound; + summary.cleanliness = Some("unknown".to_string()); + Ok(WorkingDirectoryStatus { summary }) +} + +#[cfg(target_os = "linux")] +fn metadata_only(entries: &BTreeMap) -> bool { + entries.iter().all(|(path, entry)| { + path == Path::new(MATERIALIZATION_RECORD) + || (path == Path::new(CHECKOUT_DIR) && entry.mode & libc::S_IFMT == libc::S_IFDIR) + }) +} + +#[cfg(target_os = "linux")] +fn same_materialization(saved: &WorkingDirectory, current: &WorkingDirectory) -> bool { + let immutable = |directory: &WorkingDirectory| { + let mut directory = directory.clone(); + // These fields are refreshed by authorize_repository_access, not a new + // materialization. All repository, creation and cleanup identity stays. + directory.evidence.operation_id = None; + directory.evidence.credential_revision = None; + directory.evidence.host_trust_revision = None; + directory + }; + immutable(saved) == immutable(current) +} + +#[cfg(target_os = "linux")] +fn retry_inventory( + materializer: &RuntimeGitMaterializer, + id: &str, + tree: &fs::File, + saved: &Authority, +) -> Result, WorkingDirectoryDiagnostic> { + let before = inventory(tree).map_err(|e| os_failure(id, "inspect_retry_tree", e))?; + if verify_remaining(&saved.entries, &before).is_ok() { + // With no Git left, only the old witness authorizes the surviving subset. + return Ok(before); + } + let binding = materializer + .read_binding(id) + .map_err(|_| cause("changes_present"))?; + validate_binding(materializer, id, &binding)?; + if !same_materialization(&saved.working_directory, &binding.working_directory) + || before + .get(Path::new(MATERIALIZATION_RECORD)) + .is_none_or(|entry| entry.mode & libc::S_IFMT != libc::S_IFREG) + { + return Err(cause("ownership_unknown")); + } + check_clean_checkout(id, &binding, &before, Some(saved))?; + let parent = fs::File::open(&materializer.runtime_root) + .map_err(|e| os_failure(id, "reopen_runtime_root", e))?; + let live = open_directory(&parent, std::ffi::OsStr::new(id)) + .map_err(|e| os_failure(id, "recheck_retry_identity", e))?; + if identity(&live).map_err(|e| os_failure(id, "recheck_retry_identity", e))? + != saved.root_identity + { + return Err(cause("ownership_unknown")); + } + if inventory(tree).map_err(|e| os_failure(id, "recheck_retry_content", e))? != before { + return Err(cause("changes_present")); + } + Ok(before) +} + +#[cfg(target_os = "linux")] +fn check_clean_checkout( + id: &str, + binding: &WorkingDirectoryBinding, + before: &BTreeMap, + saved: Option<&Authority>, +) -> Result<(), WorkingDirectoryDiagnostic> { + for required in [Path::new(CHECKOUT_DIR), Path::new("checkout/.git")] { + let Some(entry) = before + .get(required) + .filter(|entry| entry.mode & libc::S_IFMT == libc::S_IFDIR) + else { + return Err(cause("changes_unknown")); + }; + if let Some(saved) = saved { + if saved.entries.get(required) != Some(entry) { + return Err(cause("ownership_unknown")); + } + } + } + let run = |args: &[&str]| -> Result, WorkingDirectoryDiagnostic> { + let output = isolated_git_command() + .env("GIT_OPTIONAL_LOCKS", "0") + // Do not run repository-configured fsmonitor hooks while Runtime + // occupancy is held, or trust their cached cleanliness observation. + .args([ + "-c", + "core.fsmonitor=false", + "-c", + "core.untrackedCache=false", + ]) + .arg("-C") + .arg(binding.root()) + .args(args) + .output() + .map_err(|e| os_failure(id, "inspect_changes", e))?; + if !output.status.success() { + return Err(cause("changes_unknown")); + } + Ok(output.stdout) + }; + // An empty/unborn index is not a usable committed checkout. + run(&["rev-parse", "--verify", "HEAD^{tree}"])?; + if !run(&["status", "--porcelain", "--untracked-files=all"])?.is_empty() { + return Err(cause("changes_present")); + } + if let Some(saved) = saved { + use std::os::unix::ffi::OsStrExt; + let tracked = run(&["ls-files", "--cached", "-z"])?; + let mut git_paths = std::collections::BTreeSet::new(); + for raw in tracked.split(|b| *b == 0).filter(|raw| !raw.is_empty()) { + let path = Path::new(CHECKOUT_DIR).join(std::ffi::OsStr::from_bytes(raw)); + for path in path.ancestors().filter(|path| !path.as_os_str().is_empty()) { + git_paths.insert(path.to_path_buf()); + } + } + // Git clean is not ignored-content permission. Current tracked content + // and Git metadata may change; all other entries stay witness-protected. + for (path, entry) in before { + if path != Path::new(MATERIALIZATION_RECORD) + && !path.starts_with("checkout/.git") + && !git_paths.contains(path) + && saved.entries.get(path) != Some(entry) + { + return Err(cause("changes_present")); + } + } + } + Ok(()) +} + +pub(super) fn validate_recovery_binding( + materializer: &RuntimeGitMaterializer, + id: &str, + binding: &WorkingDirectoryBinding, +) -> Result<(), WorkingDirectoryDiagnostic> { + validate_binding(materializer, id, binding)?; + #[cfg(target_os = "linux")] + { + let parent = fs::File::open(&materializer.runtime_root) + .map_err(|e| os_failure(id, "open_bind_root", e))?; + let tree = open_directory(&parent, std::ffi::OsStr::new(id)) + .map_err(|_| cause("ownership_unknown"))?; + let checkout = open_directory(&tree, std::ffi::OsStr::new(CHECKOUT_DIR)) + .map_err(|_| cause("changes_unknown"))?; + let git = open_directory(&checkout, std::ffi::OsStr::new(".git")) + .map_err(|_| cause("changes_unknown"))?; + if inspect(&tree, std::ffi::OsStr::new(MATERIALIZATION_RECORD)) + .map_err(|_| cause("ownership_unknown"))? + .mode + & libc::S_IFMT + != libc::S_IFREG + { + return Err(cause("ownership_unknown")); + } + if let Some(saved) = load_authority(&parent, id).map_err(|_| cause("ownership_unknown"))? { + if saved.working_directory.id != id + || !same_materialization(&saved.working_directory, &binding.working_directory) + || identity(&tree).ok() != Some(saved.root_identity) + || saved + .entries + .get(Path::new(CHECKOUT_DIR)) + .map(|entry| entry.identity) + != identity(&checkout).ok() + || saved + .entries + .get(Path::new("checkout/.git")) + .map(|entry| entry.identity) + != identity(&git).ok() + { + return Err(cause("ownership_unknown")); + } + } + } + // Dirty is intentional here: ordinary attachment lets the owner preserve + // changes/release resources. Cleanup still separately rechecks occupancy. + if !binding_paths_are_available(binding) + || git_stdout(binding.root(), ["rev-parse", "--verify", "HEAD^{tree}"]).is_err() + { + return Err(cause("changes_unknown")); + } + Ok(()) +} + +pub(super) fn validate_binding( + materializer: &RuntimeGitMaterializer, + id: &str, + binding: &WorkingDirectoryBinding, +) -> Result<(), WorkingDirectoryDiagnostic> { + let directory = &binding.working_directory; + if directory.id != id + || directory.materializer_kind != MaterializerKind::RuntimeGitClone + || directory.evidence.materializer_kind != MaterializerKind::RuntimeGitClone + || directory.repository_id != directory.evidence.repository_id + || directory.cleanup_target.working_directory_id != id + || directory.cleanup_target.repository_id != directory.repository_id + || directory.cleanup_target.kind != "runtime_git_clone" + || binding.root != materializer.working_directory_root(id).join(CHECKOUT_DIR) + { + return Err(cause("ownership_unknown")); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +#[derive(Serialize, Deserialize)] +struct Authority { + working_directory: WorkingDirectory, + root_identity: (u64, u64), + entries: BTreeMap, +} + +#[cfg(target_os = "linux")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct Entry { + identity: (u64, u64), + mode: u32, + digest: Option, +} + +#[cfg(target_os = "linux")] +fn verify_remaining( + expected: &BTreeMap, + current: &BTreeMap, +) -> Result<(), WorkingDirectoryDiagnostic> { + if current + .iter() + .any(|(name, entry)| expected.get(name) != Some(entry)) + { + return Err(cause("changes_present")); + } + Ok(()) +} + +// mountinfo escapes spaces, tabs, newlines and backslashes as octal. Parse bytes +// instead of lossy Unicode; malformed input is not permission to recurse. +#[cfg(target_os = "linux")] +fn check_mounts(root: &Path, raw: &[u8]) -> Result<(), WorkingDirectoryDiagnostic> { + use std::os::unix::ffi::OsStringExt; + let root = root + .canonicalize() + .map_err(|_| cause("ownership_unknown"))?; + if raw.is_empty() { + return Err(cause("mount_check_unavailable")); + } + for line in raw.split(|b| *b == b'\n').filter(|line| !line.is_empty()) { + let fields: Vec<_> = line.split(|b| *b == b' ').collect(); + if fields.len() < 10 || !fields.contains(&b"-".as_slice()) { + return Err(cause("mount_check_unavailable")); + } + let mut path = Vec::new(); + let mut i = 0; + while i < fields[4].len() { + if fields[4][i] == b'\\' { + let digits = fields[4] + .get(i + 1..i + 4) + .ok_or_else(|| cause("mount_check_unavailable"))?; + if !digits.iter().all(|b| (b'0'..=b'7').contains(b)) { + return Err(cause("mount_check_unavailable")); + } + let value = (digits[0] - b'0') as u16 * 64 + + (digits[1] - b'0') as u16 * 8 + + (digits[2] - b'0') as u16; + path.push(u8::try_from(value).map_err(|_| cause("mount_check_unavailable"))?); + i += 4; + } else { + path.push(fields[4][i]); + i += 1; + } + } + let mount = PathBuf::from(std::ffi::OsString::from_vec(path)); + if !mount.is_absolute() { + return Err(cause("mount_check_unavailable")); + } + if mount.starts_with(&root) { + return Err(cause("mount_present")); + } + } + Ok(()) +} + +#[cfg(target_os = "linux")] +const MAX_AUTHORITY_BYTES: usize = 128 * 1024 * 1024; + +#[cfg(target_os = "linux")] +fn bounded_authority_bytes(authority: &Authority, limit: usize) -> io::Result> { + let raw = + serde_json::to_vec(authority).map_err(|_| io::Error::from(io::ErrorKind::InvalidData))?; + if raw.len() > limit { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "removal content authority exceeds its readable size limit", + )); + } + Ok(raw) +} + +#[cfg(target_os = "linux")] +mod linux { + use super::*; + use std::ffi::{CString, OsStr}; + use std::os::fd::{AsRawFd, FromRawFd}; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::fs::MetadataExt; + + #[repr(C)] + struct OpenHow { + flags: u64, + mode: u64, + resolve: u64, + } + + fn open(parent: &fs::File, name: &OsStr, flags: i32) -> io::Result { + let name = CString::new(name.as_bytes()) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + // NO_XDEV catches bind mounts too (st_dev comparisons do not). + // NO_SYMLINKS + O_PATH|O_NOFOLLOW allows reading a final symlink + // descriptor, but never following it to another directory. + let how = OpenHow { + // openat2 rejects O_NONBLOCK with O_PATH (unlike legacy open). + // Other opens remain nonblocking so raced-in FIFOs cannot hang. + flags: (flags + | libc::O_CLOEXEC + | libc::O_NOFOLLOW + | if flags & libc::O_PATH == 0 { + libc::O_NONBLOCK + } else { + 0 + }) as u64, + mode: if flags & libc::O_CREAT != 0 { 0o600 } else { 0 }, + resolve: 0x01 | 0x04 | 0x08, + }; // NO_XDEV | NO_SYMLINKS | BENEATH + // SAFETY: stable NUL-terminated name, valid descriptor and OpenHow ABI. + let fd = unsafe { + libc::syscall( + libc::SYS_openat2, + parent.as_raw_fd(), + name.as_ptr(), + &how, + std::mem::size_of::(), + ) + }; + if fd < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: syscall returned a newly owned descriptor. + Ok(unsafe { fs::File::from_raw_fd(fd as i32) }) + } + + pub(super) fn open_directory(parent: &fs::File, name: &OsStr) -> io::Result { + open(parent, name, libc::O_RDONLY | libc::O_DIRECTORY) + } + + fn authority_directory(parent: &fs::File, create: bool) -> io::Result { + if create { + // SAFETY: valid parent descriptor and fixed NUL-terminated name. + let result = + unsafe { libc::mkdirat(parent.as_raw_fd(), c".cleanup-authority".as_ptr(), 0o700) }; + if result < 0 { + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::AlreadyExists { + return Err(error); + } + } + } + let directory = open_directory(parent, OsStr::new(".cleanup-authority"))?; + let metadata = directory.metadata()?; + // SAFETY: geteuid has no arguments or memory access preconditions. + if metadata.uid() != unsafe { libc::geteuid() } || metadata.mode() & 0o022 != 0 { + return Err(io::Error::from_raw_os_error(libc::EPERM)); + } + Ok(directory) + } + + pub(super) fn load_authority(parent: &fs::File, id: &str) -> io::Result> { + let read = || -> io::Result { + let directory = authority_directory(parent, false)?; + let file = open( + &directory, + OsStr::new(&format!("{id}.json")), + libc::O_RDONLY, + )?; + if !file.metadata()?.is_file() || file.metadata()?.len() > MAX_AUTHORITY_BYTES as u64 { + return Err(io::Error::from(io::ErrorKind::InvalidData)); + } + let mut raw = Vec::new(); + file.take(MAX_AUTHORITY_BYTES as u64 + 1) + .read_to_end(&mut raw)?; + if raw.len() > MAX_AUTHORITY_BYTES { + return Err(io::Error::from(io::ErrorKind::InvalidData)); + } + serde_json::from_slice(&raw).map_err(|_| io::Error::from(io::ErrorKind::InvalidData)) + }; + match read() { + Ok(authority) => Ok(Some(authority)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } + } + + pub(super) fn persist_authority( + parent: &fs::File, + id: &str, + authority: &Authority, + ) -> io::Result<()> { + // Never publish a witness that the next normal retry cannot read. + let raw = bounded_authority_bytes(authority, MAX_AUTHORITY_BYTES)?; + let directory = authority_directory(parent, true)?; + let temporary = CString::new(format!(".{id}-{}.tmp", uuid::Uuid::now_v7())).unwrap(); + let target = CString::new(format!("{id}.json")).unwrap(); + let mut file = open( + &directory, + OsStr::from_bytes(temporary.as_bytes()), + libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL, + )?; + file.write_all(&raw)?; + file.sync_all()?; + // SAFETY: both names are valid C strings relative to the same owned fd. + let result = unsafe { + libc::renameat( + directory.as_raw_fd(), + temporary.as_ptr(), + directory.as_raw_fd(), + target.as_ptr(), + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + directory.sync_all()?; + // Persist the newly created authority directory before any deletion. + parent.sync_all() + } + + pub(super) fn discard_authority(parent: &fs::File, id: &str) -> io::Result<()> { + let directory = authority_directory(parent, false)?; + unlink(&directory, OsStr::new(&format!("{id}.json")), false)?; + directory.sync_all() + } + + pub(super) fn identity(file: &fs::File) -> io::Result<(u64, u64)> { + let metadata = file.metadata()?; + Ok((metadata.dev(), metadata.ino())) + } + + fn names(dir: &fs::File) -> io::Result> { + // /proc gives a descriptor-anchored directory listing; all access to + // listed entries still goes through mount-constrained openat2. + let mut names = fs::read_dir(format!("/proc/self/fd/{}", dir.as_raw_fd()))? + .map(|e| e.map(|e| e.file_name())) + .collect::>>()?; + names.sort(); + Ok(names) + } + + pub(super) fn inspect(dir: &fs::File, name: &OsStr) -> io::Result { + let file = open(dir, name, libc::O_PATH)?; + let metadata = file.metadata()?; + let digest = if metadata.is_file() { + let mut reader = open(dir, name, libc::O_RDONLY)?; + if identity(&reader)? != (metadata.dev(), metadata.ino()) { + return Err(io::Error::from_raw_os_error(libc::EAGAIN)); + } + let mut hash = Sha256::new(); + let mut buffer = [0u8; 65536]; + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } + hash.update(&buffer[..count]); + } + Some( + hash.finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(), + ) + } else if metadata.file_type().is_symlink() { + let mut buffer = vec![0u8; 65536]; + // SAFETY: valid O_PATH symlink descriptor; empty path reads that + // symlink, not its target. The mutable buffer has the stated length. + let count = unsafe { + libc::readlinkat( + file.as_raw_fd(), + c"".as_ptr(), + buffer.as_mut_ptr().cast(), + buffer.len(), + ) + }; + if count < 0 { + return Err(io::Error::last_os_error()); + } + buffer.truncate(count as usize); + Some( + Sha256::digest(&buffer) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(), + ) + } else if metadata.is_dir() { + None + } else { + return Err(io::Error::from_raw_os_error(libc::EBUSY)); + }; + Ok(Entry { + identity: (metadata.dev(), metadata.ino()), + // Permission repair is not a content change. Preserve the file type + // and executable bits, which Git treats as user-visible changes. + mode: metadata.mode() & libc::S_IFMT + | if metadata.is_file() { + metadata.mode() & 0o111 + } else { + 0 + }, + digest, + }) + } + + pub(super) fn inventory(dir: &fs::File) -> io::Result> { + fn walk( + dir: &fs::File, + path: &Path, + entries: &mut BTreeMap, + ) -> io::Result<()> { + if path.components().count() > 256 || entries.len() > 1_000_000 { + return Err(io::Error::from_raw_os_error(libc::EFBIG)); + } + for name in names(dir)? { + let entry = inspect(dir, &name)?; + let is_dir = entry.mode & libc::S_IFMT == libc::S_IFDIR; + let path = path.join(&name); + entries.insert(path.clone(), entry); + if is_dir { + walk(&open_directory(dir, &name)?, &path, entries)?; + } + } + Ok(()) + } + let mut entries = BTreeMap::new(); + walk(dir, Path::new(""), &mut entries)?; + Ok(entries) + } + + pub(super) fn rewrite_record( + dir: &fs::File, + bytes: &[u8], + expected: Option<&Entry>, + ) -> io::Result<()> { + let expected = expected.ok_or_else(|| io::Error::from_raw_os_error(libc::EAGAIN))?; + if expected.mode & libc::S_IFMT != libc::S_IFREG { + return Err(io::Error::from_raw_os_error(libc::EAGAIN)); + } + let mut file = open(dir, OsStr::new(MATERIALIZATION_RECORD), libc::O_WRONLY)?; + if identity(&file)? != expected.identity { + return Err(io::Error::from_raw_os_error(libc::EAGAIN)); + } + file.set_len(0)?; + file.write_all(bytes)?; + file.sync_all() + } + + pub(super) fn unlink(dir: &fs::File, name: &OsStr, directory: bool) -> io::Result<()> { + let name = CString::new(name.as_bytes()) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + // SAFETY: valid descriptor and NUL-terminated single entry name. + let result = unsafe { + libc::unlinkat( + dir.as_raw_fd(), + name.as_ptr(), + if directory { libc::AT_REMOVEDIR } else { 0 }, + ) + }; + if result < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } + } + + // Read-only kernel proof: use existing mounts, never mount/unmount anything. + // Linux /proc and openat2 are production prerequisites. Also exercise every + // accessible same-device mount when present (e.g. a read-only store bind). + #[test] + fn kernel_no_xdev_rejects_existing_cross_device_and_same_device_mounts() { + let root = fs::File::open("/").unwrap(); + let proc_error = open_directory(&root, OsStr::new("proc")).unwrap_err(); + assert_eq!(proc_error.raw_os_error(), Some(libc::EXDEV)); + let root_device = root.metadata().unwrap().dev(); + let mounts = fs::read_to_string("/proc/self/mountinfo").unwrap(); + let mut same_device = 0; + for line in mounts.lines() { + let mount = line.split(' ').nth(4).unwrap(); + if mount == "/" || mount.contains('\\') { + continue; + } + let Ok(metadata) = fs::metadata(mount) else { + continue; + }; + if metadata.is_dir() && metadata.dev() == root_device { + let error = + open_directory(&root, OsStr::new(mount.trim_start_matches('/'))).unwrap_err(); + assert_eq!(error.raw_os_error(), Some(libc::EXDEV), "{mount}: {error}"); + same_device += 1; + } + } + eprintln!( + "NO_XDEV kernel: /proc rejected; {same_device} existing same-device mounts rejected" + ); + } + + pub(super) enum EraseError { + Io(io::Error), + Changed, + } + impl From for EraseError { + fn from(error: io::Error) -> Self { + Self::Io(error) + } + } + + pub(super) fn erase( + dir: &fs::File, + path: &Path, + expected: &BTreeMap, + ) -> Result<(), EraseError> { + let mut children = names(dir)?; + // Keep identity evidence until the rest of the tree has been erased. + if path.as_os_str().is_empty() { + children.sort_by_key(|name| name == MATERIALIZATION_RECORD); + } + for name in children { + let child = path.join(&name); + let entry = inspect(dir, &name)?; + if expected.get(&child) != Some(&entry) { + return Err(EraseError::Changed); + } + let is_dir = entry.mode & libc::S_IFMT == libc::S_IFDIR; + if is_dir { + let directory = open_directory(dir, &name)?; + if identity(&directory)? != entry.identity { + return Err(EraseError::Changed); + } + erase(&directory, &child, expected)?; + if inspect(dir, &name)? != entry { + return Err(EraseError::Changed); + } + } + unlink(dir, &name, is_dir)?; + } + Ok(()) + } +} + +#[cfg(all(test, target_os = "linux"))] +pub(super) mod tests { + use super::super::tests::{create_clean_repo, request}; + use super::*; + + fn fixture() -> ( + tempfile::TempDir, + RuntimeGitMaterializer, + WorkingDirectoryBinding, + ) { + let repo = create_clean_repo(); + let runtime = tempfile::tempdir().unwrap(); + let materializer = RuntimeGitMaterializer::new(runtime.path()); + let binding = materializer.create(&request(repo.path())).unwrap(); + (runtime, materializer, binding) + } + + pub(crate) fn fail_once(materializer: &RuntimeGitMaterializer, id: &str) { + let error = remove_with_eraser(materializer, id, |_, _, _| { + Err(EraseError::Io(io::Error::from_raw_os_error(libc::EACCES))) + }) + .unwrap_err(); + assert_eq!(error.code, "working_directory_cleanup_permission_denied"); + } + + #[test] + fn persisted_removal_witness_must_fit_the_same_readable_limit() { + let (runtime, _, binding) = fixture(); + let authority = Authority { + working_directory: binding.working_directory.clone(), + root_identity: (0, 0), + entries: BTreeMap::new(), + }; + let bytes = bounded_authority_bytes(&authority, MAX_AUTHORITY_BYTES).unwrap(); + assert_eq!( + bounded_authority_bytes(&authority, bytes.len()).unwrap(), + bytes + ); + assert_eq!( + bounded_authority_bytes(&authority, bytes.len() - 1) + .unwrap_err() + .kind(), + io::ErrorKind::InvalidData + ); + assert!(!runtime.path().join(".cleanup-authority").exists()); + } + + #[test] + fn cleanup_pending_rechecks_current_cleanliness_and_retries_without_state_edit() { + let (_runtime, materializer, binding) = fixture(); + let id = &binding.working_directory.id; + fail_once(&materializer, id); + let status = materializer.working_directory_status(id).unwrap(); + assert_eq!( + status.summary.status, + WorkingDirectoryStatusKind::CleanupPending + ); + assert_eq!(status.summary.cleanliness.as_deref(), Some("clean")); + let removed = materializer.cleanup_working_directory(id).unwrap(); + assert_eq!(removed.summary.status, WorkingDirectoryStatusKind::NotFound); + assert!(!binding.working_directory_root.exists()); + } + + #[test] + fn partial_deletion_missing_checkout_and_restart_finish_with_durable_content_authority() { + for removed in [ + "checkout/README.md", + "checkout/.git", + CHECKOUT_DIR, + MATERIALIZATION_RECORD, + ] { + let (runtime, materializer, binding) = fixture(); + let id = &binding.working_directory.id; + let error = remove_with_eraser(&materializer, id, |_, _, _| { + let path = binding.working_directory_root.join(removed); + if path.is_dir() { + fs::remove_dir_all(path).unwrap(); + } else { + fs::remove_file(path).unwrap(); + } + Err(EraseError::Io(io::Error::from_raw_os_error(libc::EBUSY))) + }) + .unwrap_err(); + assert_eq!(error.code, "working_directory_cleanup_resource_busy"); + drop(materializer); + let restarted = RuntimeGitMaterializer::new(runtime.path()); + let retained = restarted.working_directory_status(id).unwrap(); + assert_eq!( + retained.summary.repository_id, + binding.working_directory.repository_id + ); + assert_eq!( + retained.summary.cleanup_target.as_ref(), + Some(&binding.working_directory.cleanup_target) + ); + assert_eq!(retained.summary.cleanliness.as_deref(), Some("clean")); + assert_eq!( + restarted.list_working_directories().unwrap()[0].summary, + retained.summary + ); + let removed_status = restarted.cleanup_working_directory(id).unwrap(); + assert_eq!( + removed_status.summary.status, + WorkingDirectoryStatusKind::NotFound, + "{removed}" + ); + assert!(!binding.working_directory_root.exists()); + } + } + + #[test] + fn missing_checkout_with_only_validated_metadata_can_be_removed_without_witness() { + let (_runtime, materializer, binding) = fixture(); + fs::remove_dir_all(binding.root()).unwrap(); + let removed = materializer + .cleanup_working_directory(&binding.working_directory.id) + .unwrap(); + assert_eq!(removed.summary.status, WorkingDirectoryStatusKind::NotFound); + assert!(!binding.working_directory_root.exists()); + } + + #[test] + fn incomplete_checkout_without_witness_preserves_residue_and_explains_recovery() { + let (_runtime, materializer, binding) = fixture(); + fs::remove_dir_all(binding.root.join(".git")).unwrap(); + let error = materializer + .cleanup_working_directory(&binding.working_directory.id) + .unwrap_err(); + assert_eq!(error.code, "working_directory_cleanup_changes_unknown"); + assert!(error.message.contains("Preserve residual content")); + assert!(binding.root.join("README.md").exists()); + } + + #[test] + fn failure_then_dirty_then_owner_resolution_renews_witness_on_ordinary_retry() { + use super::super::tests::git; + for resolution in ["commit", "restore"] { + let (runtime, materializer, binding) = fixture(); + let id = &binding.working_directory.id; + fail_once(&materializer, id); + fs::write(binding.root.join("README.md"), "preserved change").unwrap(); + assert_eq!( + materializer + .working_directory_status(id) + .unwrap() + .summary + .cleanliness + .as_deref(), + Some("dirty") + ); + assert_eq!( + materializer.cleanup_working_directory(id).unwrap_err().code, + "working_directory_cleanup_changes_present" + ); + let recovery = materializer.bind_working_directory(id, None).unwrap(); + assert_eq!( + recovery.working_directory.status, + WorkingDirectoryStatusKind::CleanupPending + ); + if resolution == "commit" { + git(binding.root(), &["add", "README.md"]); + git( + binding.root(), + &[ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "commit", + "-m", + "Preserve recovery changes", + ], + ); + } else { + fs::copy( + binding.root.join("README.md"), + runtime.path().join("preserved.txt"), + ) + .unwrap(); + // Replaces the tracked file inode and Git index legitimately. + fs::remove_file(binding.root.join("README.md")).unwrap(); + git(binding.root(), &["restore", "README.md"]); + } + let restarted = RuntimeGitMaterializer::new(runtime.path()); + assert_eq!( + restarted + .working_directory_status(id) + .unwrap() + .summary + .cleanliness + .as_deref(), + Some("clean"), + "{resolution}" + ); + // Renewal itself can fail at unlink. Its new witness must survive a + // later partial deletion/restart, even with no remaining Git. + fail_once(&restarted, id); + fs::remove_dir_all(binding.root.join(".git")).unwrap(); + let restarted = RuntimeGitMaterializer::new(runtime.path()); + restarted.cleanup_working_directory(id).unwrap(); + assert!(!binding.working_directory_root.exists()); + } + } + + #[test] + fn clean_git_retry_preserves_new_or_modified_ignored_and_untracked_content() { + use super::super::tests::git; + for change in ["ignored_new", "ignored_modified", "untracked", "sibling"] { + let (_runtime, materializer, binding) = fixture(); + let id = &binding.working_directory.id; + fs::write(binding.root.join(".git/info/exclude"), "ignored/\n").unwrap(); + fs::create_dir(binding.root.join("ignored")).unwrap(); + fs::write(binding.root.join("ignored/existing"), "initial").unwrap(); + fail_once(&materializer, id); + let file = match change { + "ignored_new" => binding.root.join("ignored/new"), + "ignored_modified" => binding.root.join("ignored/existing"), + "untracked" => binding.root.join("new"), + _ => binding.working_directory_root.join("new"), + }; + fs::write(&file, "preserve me").unwrap(); + git( + binding.root(), + &[ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "commit", + "--allow-empty", + "-m", + "Clean Git metadata changed", + ], + ); + assert_eq!( + materializer.cleanup_working_directory(id).unwrap_err().code, + "working_directory_cleanup_changes_present", + "{change}" + ); + assert_eq!(fs::read_to_string(&file).unwrap(), "preserve me"); + fs::remove_file(&file).unwrap(); + // Removing a pre-existing ignored file is safe; no new content remains. + materializer.cleanup_working_directory(id).unwrap(); + } + } + + #[test] + fn pending_witness_accepts_access_refresh_but_rejects_materialization_substitution() { + for update in ["access", "repository", "creation"] { + let (_runtime, materializer, binding) = fixture(); + let id = &binding.working_directory.id; + fail_once(&materializer, id); + let mut record = materializer.read_binding(id).unwrap(); + match update { + "access" => { + // The same serialized fields changed by authorize_repository_access. + record.working_directory.evidence.operation_id = + Some("reattach-operation".into()); + record.working_directory.evidence.credential_revision = Some(2); + record.working_directory.evidence.host_trust_revision = Some(2); + } + "repository" => { + record.working_directory.repository_id = "substitute".into(); + record.working_directory.evidence.repository_id = "substitute".into(); + record.working_directory.cleanup_target.repository_id = "substitute".into(); + } + _ => record.working_directory.evidence.resolved_commit = "substitute".into(), + } + materializer.write_record(&record).unwrap(); + if update == "access" { + materializer + .preflight_bind_working_directory(id, None) + .unwrap(); + // Read-only preflight does not bypass actual SSH delivery requirements. + assert_eq!( + materializer + .bind_working_directory(id, None) + .unwrap_err() + .code, + "working_directory_remote_repository_access_required" + ); + materializer.cleanup_working_directory(id).unwrap(); + assert!(!binding.working_directory_root.exists()); + } else { + assert_eq!( + materializer + .preflight_bind_working_directory(id, None) + .unwrap_err() + .code, + "working_directory_cleanup_ownership_unknown" + ); + assert_eq!( + materializer.cleanup_working_directory(id).unwrap_err().code, + "working_directory_cleanup_ownership_unknown" + ); + assert!(binding.root.join("README.md").exists()); + } + } + } + + #[test] + fn cleanup_pending_binding_rejects_missing_git_and_replaced_checkout() { + for replacement in [false, true] { + let (_runtime, materializer, binding) = fixture(); + let id = &binding.working_directory.id; + fail_once(&materializer, id); + if replacement { + fs::rename( + binding.root(), + binding.working_directory_root.join("old-checkout"), + ) + .unwrap(); + fs::create_dir(binding.root()).unwrap(); + fs::create_dir(binding.root.join(".git")).unwrap(); + } else { + fs::remove_dir_all(binding.root.join(".git")).unwrap(); + } + assert!(materializer.bind_working_directory(id, None).is_err()); + assert!(binding.working_directory_root.exists()); + } + } + + #[test] + fn retry_protects_new_files_modified_content_and_replaced_directories() { + for change in ["new", "modified", "replacement"] { + let (runtime, materializer, binding) = fixture(); + let id = &binding.working_directory.id; + fail_once(&materializer, id); + match change { + "new" => fs::write(binding.root.join("new.txt"), b"preserve me").unwrap(), + "modified" => fs::write(binding.root.join("README.md"), b"preserve me").unwrap(), + _ => { + fs::rename( + &binding.working_directory_root, + runtime.path().join("original"), + ) + .unwrap(); + fs::create_dir(&binding.working_directory_root).unwrap(); + } + } + let restarted = RuntimeGitMaterializer::new(runtime.path()); + let error = restarted.cleanup_working_directory(id).unwrap_err(); + assert_eq!( + error.code, + if change == "replacement" { + "working_directory_cleanup_ownership_unknown" + } else { + "working_directory_cleanup_changes_present" + }, + "{change}" + ); + assert!(binding.working_directory_root.exists()); + } + } + + #[test] + fn cleanup_does_not_execute_repository_fsmonitor_hooks_under_occupancy_guard() { + use super::super::tests::git; + use std::os::unix::fs::PermissionsExt; + let (runtime, materializer, binding) = fixture(); + let hook = runtime.path().join("fsmonitor"); + let marker = runtime.path().join("hook-called"); + fs::write( + &hook, + format!("#!/bin/sh\ntouch '{}'\nexit 1\n", marker.display()), + ) + .unwrap(); + fs::set_permissions(&hook, fs::Permissions::from_mode(0o700)).unwrap(); + git( + binding.root(), + &["config", "core.fsmonitor", hook.to_str().unwrap()], + ); + fail_once(&materializer, &binding.working_directory.id); + fs::write(binding.root.join("README.md"), "dirty").unwrap(); + assert_eq!( + materializer + .cleanup_working_directory(&binding.working_directory.id) + .unwrap_err() + .code, + "working_directory_cleanup_changes_present" + ); + assert!(!marker.exists()); + } + + #[test] + fn dirty_initial_checkout_is_never_authorized_for_removal() { + let (runtime, materializer, binding) = fixture(); + fs::write(binding.root.join("README.md"), "preserve me").unwrap(); + let error = materializer + .cleanup_working_directory(&binding.working_directory.id) + .unwrap_err(); + assert_eq!(error.code, "working_directory_cleanup_changes_present"); + assert_eq!( + fs::read_to_string(binding.root.join("README.md")).unwrap(), + "preserve me" + ); + assert!(!runtime.path().join(".cleanup-authority").exists()); + } + + #[test] + fn mounted_or_unknown_resources_are_preserved_then_normal_retry_succeeds() { + let (_runtime, materializer, mut binding) = fixture(); + // Legacy cleanup_pending is an input, not authority to discard content. + binding.working_directory.status = WorkingDirectoryStatusKind::CleanupPending; + materializer.write_record(&binding).unwrap(); + let mount = binding.root.join(".yoi/mount"); + fs::create_dir_all(&mount).unwrap(); + fs::write(mount.join("external.txt"), b"external resource").unwrap(); + // .yoi is ignored, exactly as in the reported mount residue. + fs::write(binding.root.join(".git/info/exclude"), ".yoi/\n").unwrap(); + let raw = format!( + "1 0 8:1 / / rw - ext4 /dev/root rw\n2 1 8:1 / {} rw - ext4 /dev/root rw\n", + mount.display() + ); + let error = remove_with_checks( + &materializer, + &binding.working_directory.id, + || Ok(raw.into_bytes()), + erase, + ) + .unwrap_err(); + assert_eq!(error.code, "working_directory_cleanup_mount_present"); + assert_eq!( + fs::read(mount.join("external.txt")).unwrap(), + b"external resource" + ); + assert!(!error.message.contains(&mount.to_string_lossy().to_string())); + // The mount has been safely released by its owner. There is no state edit. + let removed = materializer + .cleanup_working_directory(&binding.working_directory.id) + .unwrap(); + assert_eq!(removed.summary.status, WorkingDirectoryStatusKind::NotFound); + } + + #[test] + fn mountinfo_unavailable_or_malformed_never_reaches_deletion() { + let (_runtime, materializer, binding) = fixture(); + for raw in [ + Vec::new(), + b"invalid".to_vec(), + b"1 0 8:1 / /bad\\zzz rw - ext4 /dev/root rw\n".to_vec(), + ] { + let error = remove_with_checks( + &materializer, + &binding.working_directory.id, + || Ok(raw), + |_, _, _| panic!("must not unlink without a mount check"), + ) + .unwrap_err(); + assert_eq!( + error.code, + "working_directory_cleanup_mount_check_unavailable" + ); + } + let error = remove_with_checks( + &materializer, + &binding.working_directory.id, + || Err(io::Error::from_raw_os_error(libc::EACCES)), + erase, + ) + .unwrap_err(); + assert_eq!( + error.code, + "working_directory_cleanup_mount_check_unavailable" + ); + assert!(binding.root.exists()); + } + + #[test] + fn mountinfo_decodes_escaped_paths_and_uses_component_boundaries() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("work dir"); + fs::create_dir(&root).unwrap(); + let escaped = root.to_string_lossy().replace(' ', "\\040"); + let raw = format!("1 0 8:1 / {escaped}/mount rw - ext4 /dev/root rw\n"); + assert_eq!( + check_mounts(&root, raw.as_bytes()).unwrap_err().code, + "working_directory_cleanup_mount_present" + ); + let sibling = format!("1 0 8:1 / {escaped}-other/mount rw - ext4 /dev/root rw\n"); + check_mounts(&root, sibling.as_bytes()).unwrap(); + } + + #[test] + fn symlinks_are_unlinked_without_traversing_external_content() { + let (_runtime, materializer, binding) = fixture(); + let external = tempfile::tempdir().unwrap(); + fs::write(external.path().join("keep.txt"), b"keep").unwrap(); + fs::write(binding.root.join(".git/info/exclude"), "link\n").unwrap(); + std::os::unix::fs::symlink(external.path(), binding.root.join("link")).unwrap(); + materializer + .cleanup_working_directory(&binding.working_directory.id) + .unwrap(); + assert_eq!(fs::read(external.path().join("keep.txt")).unwrap(), b"keep"); + } + + #[test] + fn record_identity_mismatch_and_symlink_roots_never_grant_removal_authority() { + let (runtime, materializer, mut binding) = fixture(); + binding + .working_directory + .cleanup_target + .working_directory_id = "other".to_string(); + materializer.write_record(&binding).unwrap(); + let error = materializer + .cleanup_working_directory(&binding.working_directory.id) + .unwrap_err(); + assert_eq!(error.code, "working_directory_cleanup_ownership_unknown"); + fs::rename( + &binding.working_directory_root, + runtime.path().join("original"), + ) + .unwrap(); + std::os::unix::fs::symlink( + runtime.path().join("original"), + &binding.working_directory_root, + ) + .unwrap(); + assert!( + materializer + .cleanup_working_directory(&binding.working_directory.id) + .is_err() + ); + assert!(runtime.path().join("original/checkout/README.md").exists()); + } + + #[test] + fn permission_repair_without_content_changes_allows_normal_retry() { + use std::os::unix::fs::PermissionsExt; + let (_runtime, materializer, binding) = fixture(); + fail_once(&materializer, &binding.working_directory.id); + fs::set_permissions( + binding.root.join("README.md"), + fs::Permissions::from_mode(0o600), + ) + .unwrap(); + materializer + .cleanup_working_directory(&binding.working_directory.id) + .unwrap(); + assert!(!binding.working_directory_root.exists()); + } + + #[test] + fn malformed_or_external_removal_authority_is_never_used_or_overwritten() { + for external in [false, true] { + let (runtime, materializer, binding) = fixture(); + let private = runtime.path().join(".cleanup-authority"); + let outside = tempfile::tempdir().unwrap(); + let file = outside + .path() + .join(format!("{}.json", binding.working_directory.id)); + fs::write(&file, "do not overwrite").unwrap(); + if external { + std::os::unix::fs::symlink(outside.path(), &private).unwrap(); + } else { + fs::create_dir(&private).unwrap(); + fs::write( + private.join(format!("{}.json", binding.working_directory.id)), + "not json", + ) + .unwrap(); + } + let error = materializer + .cleanup_working_directory(&binding.working_directory.id) + .unwrap_err(); + assert_eq!( + error.code, + if external { + "working_directory_cleanup_storage_unavailable" + } else { + "working_directory_cleanup_ownership_unknown" + } + ); + assert!(binding.root.exists()); + assert_eq!(fs::read_to_string(file).unwrap(), "do not overwrite"); + } + } + + #[test] + fn cleanup_os_error_is_logged_privately_with_workdir_and_action_correlation() { + #[derive(Clone)] + struct Capture(Arc>>); + impl io::Write for Capture { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + let output = Arc::new(Mutex::new(Vec::new())); + let writer = Capture(output.clone()); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .without_time() + .with_writer(move || writer.clone()) + .finish(); + let public = tracing::subscriber::with_default(subscriber, || { + // Other parallel cleanup tests can register this callsite while no + // subscriber is installed on their thread. Refresh its interest for + // this thread-local capture; never install a process-global logger. + tracing::callsite::rebuild_interest_cache(); + os_failure( + "workdir-correlated", + "unlink_checkout", + io::Error::from_raw_os_error(libc::EACCES), + ) + }); + let log = String::from_utf8(output.lock().unwrap().clone()).unwrap(); + assert!( + log.contains("workdir-correlated") && log.contains("unlink_checkout"), + "{log}" + ); + assert!(log.contains("raw_os_error=Some(13)"), "{log}"); + assert!(!public.message.contains("os error")); + } + + #[test] + fn authoritative_absence_is_distinct_from_os_failure_and_unknown_changes() { + let runtime = tempfile::tempdir().unwrap(); + let materializer = RuntimeGitMaterializer::new(runtime.path()); + assert_eq!( + materializer + .cleanup_working_directory("absent") + .unwrap_err() + .code, + "working_directory_not_found" + ); + for (errno, code) in [ + (libc::EACCES, "permission_denied"), + (libc::EBUSY, "resource_busy"), + (libc::ENOSPC, "storage_unavailable"), + (libc::EXDEV, "mount_present"), + (libc::ENOSYS, "mount_check_unavailable"), + ] { + let error = os_failure( + "workdir-test", + "unlink", + io::Error::from_raw_os_error(errno), + ); + assert_eq!(error.code, format!("working_directory_cleanup_{code}")); + assert!(error.message.len() < 512); + assert!(!error.message.contains("/")); + } + } +} diff --git a/crates/worker/src/feature/builtin/manage_workdir.rs b/crates/worker/src/feature/builtin/manage_workdir.rs index 28867c0ca..498996cf3 100644 --- a/crates/worker/src/feature/builtin/manage_workdir.rs +++ b/crates/worker/src/feature/builtin/manage_workdir.rs @@ -734,8 +734,14 @@ impl WorkspaceHttpWorkdirBackend { ))?; workdir_output( format!( - "Workdir {workdir_id} removal disposition: {:?}", - response.disposition + "Workdir {workdir_id} removal disposition: {:?}. {}{}", + response.disposition, + removal_guidance(response.failure_category.as_deref()), + if response.retryable { + " After resolving the cause, repeat this WorkdirDelete with the same id and reason; current safety conditions will be rechecked." + } else { + "" + } ), &response, ) @@ -920,6 +926,48 @@ fn decode_error(error: serde_json::Error) -> ToolError { ToolError::ExecutionFailed(format!("decode Workspace Workdir response: {error}")) } +fn removal_guidance(category: Option<&str>) -> &'static str { + match category { + None => "", + Some("mount_present") => { + "A mount remains inside the Workdir. Confirm its owner and usage, and have that owner safely release it. Do not recursively delete mounted data or implicitly unmount external or unknown resources." + } + Some("mount_check_unavailable") => "Restore Runtime mount inspection before deletion.", + Some("changes_present" | "dirty_or_unknown") => { + "Current changes are protected; preserve or resolve them before deletion." + } + Some("changes_unknown") => { + "Restore checkout inspection or preserve remaining content so current changes can be verified." + } + Some( + "ownership_unknown" + | "provider_identity_changed" + | "authority_changed" + | "authority_invalid", + ) => { + "Confirm target identity and ownership with its owner; never remove a replacement or unknown resource." + } + Some("blocked_by_live_authority") => { + "Release current attachments, reservations and holds through their owning authority." + } + Some("permission_denied" | "resource_busy" | "storage_unavailable") => { + "Have the resource owner resolve access, usage or storage safely; do not force-clean or change shared resource permissions." + } + Some("runtime_unavailable" | "provider_unavailable") => { + "Restore the Runtime connection; the registry remains until deletion is confirmed." + } + Some("provider_observation_unknown" | "provider_cleanup_outcome_unknown") => { + "Deletion is not confirmed. Re-observe with normal removal after restoring provider access; the registry is retained." + } + Some("unsupported_target") => { + "Ask the owner for the provider's supported cleanup operation." + } + Some(_) => { + "Inspect correlated Runtime cleanup diagnostics with the owning operator and resolve the current cause." + } + } +} + fn workdir_output(summary: String, value: &T) -> Result { Ok(ToolOutput { summary, @@ -1218,6 +1266,45 @@ mod tests { }) } + #[test] + fn ordinary_workdir_delete_reports_safe_cause_and_exact_retry_guidance() { + let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({ + "working_directory_id": "wd-retry", + "disposition": "attention_required", + "retryable": true, + "failure_category": "mount_present" + }))])); + let backend = WorkspaceHttpWorkdirBackend::new(client.clone()); + let output = backend + .delete(WorkdirDeleteInput { + working_directory_id: "wd-retry".into(), + reason: "remove unused Workdir".into(), + }) + .unwrap(); + assert!(output.summary.contains("Confirm its owner")); + assert!( + output + .summary + .contains("Do not recursively delete mounted data") + ); + assert!(output.summary.contains("same id and reason")); + assert!(output.summary.len() <= 600); + let content: serde_json::Value = + serde_json::from_str(output.content.as_deref().unwrap()).unwrap(); + assert_eq!(content["failure_category"], "mount_present"); + assert_eq!(content["retryable"], true); + let requests = client.requests(); + assert_eq!(requests[0].method, WorkspaceRequestMethod::Delete); + assert_eq!( + serde_json::from_str::(requests[0].body.as_deref().unwrap()) + .unwrap()["reason"], + "remove unused Workdir" + ); + let unknown = removal_guidance(Some("secret /private/provider/path")); + assert!(!unknown.contains("/private")); + assert!(!unknown.contains("secret")); + } + #[test] fn workspace_request_failure_is_not_reported_as_session_unavailable() { let error = workspace_workdir_error(WorkspaceClientError::Request( diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 350326964..1135f5393 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -11927,10 +11927,6 @@ fn cleanup_workdir_cleanliness_from_runtime(value: Option<&str>) -> CleanupWorkd .unwrap_or(CleanupWorkdirCleanliness::Unknown) } -fn cleanup_workdir_cleanliness_is_clean(value: CleanupWorkdirCleanliness) -> bool { - matches!(value, CleanupWorkdirCleanliness::Clean) -} - #[derive(Debug, Deserialize)] struct LogQuery { limit: Option, @@ -17787,9 +17783,15 @@ async fn scoped_attach_current_worker_workdir( code: "working_directory_not_found".to_string(), message: format!("unknown Workdir `{workdir_id}`"), })?; - if workdir.materialization_status != "present" { + // A failed removal must not prevent its owner from attaching a still usable + // repository checkout to resolve current blockers. Runtime binding validates + // the actual checkout; durable pending-removal reservations still fence this + // mutation. Other sources and missing/corrupted materializations stay denied. + let repository_retry = workdir.materialization_status == "pending" + && matches!(workdir.source, WorkdirRegistrySource::Repository { .. }); + if workdir.materialization_status != "present" && !repository_retry { return Err(Error::WorkdirAttachmentConflict(format!( - "Workdir `{workdir_id}` is not active ({})", + "Workdir `{workdir_id}` is not available for attachment ({})", workdir.materialization_status )) .into()); @@ -24527,18 +24529,88 @@ fn runtime_reports_workdir_not_found(result: &crate::hosts::RuntimeWorkingDirect fn classify_workdir_provider_error(error: &RuntimeRegistryError) -> (&'static str, bool) { match error { RuntimeRegistryError::UnknownRuntime(_) => ("runtime_unavailable", true), - RuntimeRegistryError::RuntimeOperationFailed { code, .. } - if code == "working_directory_unsupported" => - { - ("unsupported_target", false) + RuntimeRegistryError::RuntimeOperationFailed { code, .. } => { + let category = + workdir_cleanup_diagnostic_category(code).unwrap_or("provider_unavailable"); + (category, category != "unsupported_target") } - RuntimeRegistryError::RuntimeOperationFailed { .. } => ("provider_unavailable", true), RuntimeRegistryError::InvalidIdentifier { .. } | RuntimeRegistryError::UnknownHost(_) | RuntimeRegistryError::UnknownWorker { .. } => ("authority_invalid", false), } } +// Only bounded classifications cross the public removal boundary. Provider +// messages may contain OS paths or credentials and are never persisted here. +fn workdir_cleanup_failure_category( + result: &crate::hosts::RuntimeWorkingDirectoryResult, +) -> &'static str { + result + .diagnostics + .iter() + .find_map(|diagnostic| workdir_cleanup_diagnostic_category(&diagnostic.code)) + .unwrap_or("provider_cleanup_failed") +} + +fn workdir_cleanup_diagnostic_category(code: &str) -> Option<&'static str> { + Some(match code { + "working_directory_unsupported" => "unsupported_target", + "working_directory_cleanup_mount_present" => "mount_present", + "working_directory_cleanup_mount_check_unavailable" => "mount_check_unavailable", + "working_directory_cleanup_permission_denied" => "permission_denied", + "working_directory_cleanup_resource_busy" => "resource_busy", + "working_directory_cleanup_storage_unavailable" => "storage_unavailable", + "working_directory_cleanup_ownership_unknown" => "ownership_unknown", + "working_directory_cleanup_changes_present" => "changes_present", + "working_directory_cleanup_changes_unknown" => "changes_unknown", + "working_directory_cleanup_target_invalid" + | "working_directory_cleanup_escape_rejected" => "ownership_unknown", + _ => return None, + }) +} + +fn workdir_removal_failure_guidance(category: &str) -> &'static str { + match category { + "mount_present" => { + "A mount remains inside the Workdir. Confirm its owner and usage, have that owner safely release it, then retry deletion. Yoi does not unmount external resources." + } + "mount_check_unavailable" => { + "Runtime cannot inspect mount boundaries. Restore mount inspection before retrying deletion." + } + "permission_denied" => { + "Runtime lacks permission to remove the Workdir. Have its owner resolve access safely, then retry; do not force-clean or change shared resource permissions." + } + "resource_busy" => { + "A resource is busy. Have its owner stop or release it safely, then retry deletion." + } + "storage_unavailable" => { + "Runtime storage is unavailable. Restore storage access, then retry deletion." + } + "ownership_unknown" | "provider_identity_changed" | "authority_changed" => { + "Workdir identity or ownership could not be confirmed. Verify the target with its owner before retrying." + } + "changes_present" | "dirty_or_unknown" => { + "Current Workdir changes are protected. Preserve or resolve them before retrying deletion." + } + "changes_unknown" => { + "Runtime cannot verify remaining Workdir changes. Restore repository inspection or preserve remaining files before retrying." + } + "blocked_by_live_authority" => { + "Workdir has a current attachment, reservation or retention hold. Release the owning authority before retrying deletion." + } + "runtime_unavailable" | "provider_unavailable" => { + "Runtime is unavailable. Reconnect it, then retry deletion; the Workdir registry is retained." + } + "provider_observation_unknown" | "provider_cleanup_outcome_unknown" => { + "Runtime has not confirmed deletion. Retry to re-observe the target; the Workdir registry is retained." + } + "unsupported_target" => "This provider does not support Workdir deletion.", + _ => { + "Runtime cleanup failed. Inspect correlated Runtime diagnostics, resolve the cause, then retry deletion." + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum WorkdirRemovalOwnerObservation { Running { process_start_marker: u64 }, @@ -24741,17 +24813,46 @@ fn execute_reserved_workdir_removal_with_provider( return api.config_store.commit_workdir_removal_removed(&operation); } let Some(status) = observed.working_directory.as_ref() else { + let category = workdir_cleanup_failure_category(&observed); + let category = if category == "provider_cleanup_failed" { + "provider_observation_unknown" + } else { + category + }; return api.config_store.fail_workdir_removal_operation( &operation, - "provider_observation_unknown", - true, + category, + category != "unsupported_target", ); }; - let corrupted = status.summary.status == WorkingDirectoryStatusKind::Corrupted; - if !corrupted - && (status.summary.cleanliness.as_deref() != Some("clean") - || status.summary.status != WorkingDirectoryStatusKind::Active) + // A provider summary is not permission to remove a different materialization. + // Unknown/corrupted metadata is delegated to Runtime's guarded cleanup, never + // repaired into a fabricated identity here. + let summary = &status.summary; + if summary.occupied_by.is_some() { + return api.config_store.complete_workdir_removal_retained( + &operation, + WorkdirRemovalDisposition::Retained, + "blocked_by_live_authority", + ); + } + if summary.working_directory_id != operation.working_directory_id + || (summary.repository_id == "unknown" + && summary.status != WorkingDirectoryStatusKind::Corrupted) + || (summary.repository_id != "unknown" + && crate::workdir_removal::workdir_materialization_fingerprint( + &workdir_record_from_summary(api, &operation.runtime_id, summary), + ) != operation.materialization_fingerprint) { + return api.config_store.fail_workdir_removal_operation( + &operation, + "provider_identity_changed", + true, + ); + } + // Past lifecycle failure is not a current dirtiness observation. Runtime + // must revalidate unknown/partial checkouts at the actual deletion boundary. + if summary.cleanliness.as_deref() == Some("dirty") { return api.config_store.complete_workdir_removal_retained( &operation, WorkdirRemovalDisposition::Retained, @@ -24769,18 +24870,25 @@ fn execute_reserved_workdir_removal_with_provider( .fail_workdir_removal_operation(&operation, category, retryable); } }; - if deleted.state != InternalWorkerOperationState::Accepted - && !runtime_reports_workdir_not_found(&deleted) - { - let category = if deleted - .diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "working_directory_unsupported") - { - "unsupported_target" + let deletion_confirmed = runtime_reports_workdir_not_found(&deleted) + || (deleted.state == InternalWorkerOperationState::Accepted + && deleted.working_directory.as_ref().is_some_and(|status| { + status.summary.working_directory_id == operation.working_directory_id + && status.summary.status == WorkingDirectoryStatusKind::NotFound + })); + if !deletion_confirmed { + let category = if deleted.state == InternalWorkerOperationState::Accepted { + "provider_cleanup_outcome_unknown" } else { - "provider_cleanup_failed" + workdir_cleanup_failure_category(&deleted) }; + tracing::warn!( + workdir_id = %operation.working_directory_id, + operation_id = %operation.operation_id, + runtime_id = %operation.runtime_id, + category, + "Runtime did not confirm Workdir removal; registry retained" + ); return api.config_store.fail_workdir_removal_operation( &operation, category, @@ -25066,9 +25174,7 @@ fn build_runtime_cleanup_plan( .unwrap_or_else(|| cleanup_workdir_cleanliness_from_registry(&record.cleanliness)); let action = if cleanup_workdir_file_status_is_record_only(file_status) { CleanupTargetKind::WorkdirRecordDelete - } else if cleanup_workdir_file_status_is_corrupted(file_status) - || cleanup_workdir_cleanliness_is_clean(cleanliness) - { + } else if !matches!(cleanliness, CleanupWorkdirCleanliness::Dirty) { CleanupTargetKind::WorkdirCleanCleanup } else { CleanupTargetKind::WorkdirDirtyDiscard @@ -25096,12 +25202,12 @@ fn build_runtime_cleanup_plan( } else if cleanup_workdir_file_status_is_record_only(file_status) { "Not-found Workdir record can be deleted from the Backend registry".to_string() } else if cleanup_workdir_file_status_is_corrupted(file_status) { - "Corrupted Workdir can be deleted from Runtime storage and Backend registry" + "Runtime will verify remaining files and removal authority for this incomplete Workdir" .to_string() } else if matches!(cleanliness, CleanupWorkdirCleanliness::Dirty) { - "Dirty Workdir requires explicit discard confirmation before cleanup".to_string() + "Dirty Workdir is protected; preserve or resolve current changes before retrying".to_string() } else if matches!(cleanliness, CleanupWorkdirCleanliness::Unknown) { - "Workdir clean state is unknown; explicit discard confirmation is required" + "Runtime will recheck current changes, identity and removal blockers before deleting" .to_string() } else { "Clean Workdir can be manually cleaned up".to_string() @@ -25303,7 +25409,12 @@ async fn execute_runtime_cleanup_with_context( target_id: candidate.target_id.clone(), action: candidate.action.clone(), status: status.to_string(), - message: message.to_string(), + message: removal.failure_category.as_deref().map_or_else( + || message.to_string(), + |category| { + format!("{category}: {}", workdir_removal_failure_guidance(category)) + }, + ), }); } CleanupTargetKind::WorkerDelete => { @@ -53019,7 +53130,7 @@ mod tests { }; let source_fingerprint = repository_source_fingerprint(&repository_source); for (workdir_id, _, _) in &fixtures { - runtime + let created = runtime .create_working_directory(WorkingDirectoryRequest { repository: WorkingDirectoryRepository { id: "repo-test".to_string(), @@ -53042,6 +53153,13 @@ mod tests { "present", "clean", ); + api.store + .upsert_workdir_registry(&workdir_record_from_summary( + &api, + provider_runtime_id, + &created.summary, + )) + .unwrap(); } let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let provider_base_url = format!("http://{}", listener.local_addr().unwrap()); @@ -53591,6 +53709,91 @@ mod tests { assert_eq!(response.status(), StatusCode::FORBIDDEN); } + #[tokio::test] + async fn current_worker_can_reattach_usable_pending_checkout_after_failed_removal_but_not_during_attempt() + { + let mut fixture = manual_worker_assignment_fixture().await; + let identity = + worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id) + .unwrap(); + configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id); + let api = &fixture.api; + api.store + .detach_worker_workdir( + TEST_WORKSPACE_ID, + &fixture.worker, + Some(&fixture.docs_workdir_id), + "release for removal", + ) + .unwrap(); + let mut record = api + .store + .get_workdir_registry(TEST_WORKSPACE_ID, &fixture.docs_workdir_id) + .unwrap() + .unwrap(); + record.materialization_status = "pending".into(); + api.store.upsert_workdir_registry(&record).unwrap(); + let intent = + workdir_removal_intent(&record, "account:owner", "remove docs checkout").unwrap(); + let pending = api + .config_store + .reserve_workdir_removal_operation(&intent) + .unwrap(); + let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-attachments"); + let request = || { + runtime_source_request( + &identity, + Some(&fixture.worker.worker_id), + "POST", + &path, + serde_json::to_vec( + &json!({ "alias": "docs", "working_directory_id": fixture.docs_workdir_id }), + ) + .unwrap(), + ) + }; + let blocked = build_router(api.clone()).oneshot(request()).await.unwrap(); + let blocked_status = blocked.status(); + let blocked_body = to_bytes(blocked.into_body(), usize::MAX).await.unwrap(); + assert_eq!( + blocked_status, + StatusCode::CONFLICT, + "{}", + String::from_utf8_lossy(&blocked_body) + ); + api.config_store + .fail_workdir_removal_operation(&pending, "provider_unavailable", true) + .unwrap(); + let attached = build_router(api.clone()).oneshot(request()).await.unwrap(); + let attached_status = attached.status(); + let attached_body = to_bytes(attached.into_body(), usize::MAX).await.unwrap(); + assert_eq!( + attached_status, + StatusCode::OK, + "{}", + String::from_utf8_lossy(&attached_body) + ); + let attached: server_api::CurrentWorkerWorkdirAttachmentResponse = + serde_json::from_slice(&attached_body).unwrap(); + assert!(attached.attached); + assert_eq!(attached.working_directory_id, fixture.docs_workdir_id); + assert_eq!( + api.store + .get_workdir_registry(TEST_WORKSPACE_ID, &fixture.docs_workdir_id) + .unwrap() + .unwrap() + .materialization_status, + "pending" + ); + assert!( + api.config_store + .workdir_removal_guards(&pending) + .unwrap() + .iter() + .any(|guard| guard.category == "active_attachment") + ); + } + #[tokio::test] async fn current_worker_attachment_list_and_stale_detach_use_signed_identity() { let mut fixture = manual_worker_assignment_fixture().await; @@ -54272,6 +54475,18 @@ mod tests { } } + fn confirmed_workdir_cleanup_result() -> crate::hosts::RuntimeWorkingDirectoryResult { + workdir_removal_result( + InternalWorkerOperationState::Rejected, + None, + vec![RuntimeDiagnostic { + code: "working_directory_not_found".into(), + severity: HostDiagnosticSeverity::Warning, + message: "authoritatively absent".into(), + }], + ) + } + fn reserve_removal_fixture( api: &WorkspaceApi, working_directory_id: &str, @@ -54312,7 +54527,7 @@ mod tests { Some(clean_summary), Vec::new(), ), - workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()), + confirmed_workdir_cleanup_result(), ); let removed = execute_reserved_workdir_removal_with_provider( &api, @@ -54378,7 +54593,7 @@ mod tests { message: "timeout".to_string(), }], ), - workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()), + confirmed_workdir_cleanup_result(), ); let unknown = execute_reserved_workdir_removal_with_provider( &api, @@ -54400,7 +54615,7 @@ mod tests { Some(dirty_summary), Vec::new(), ), - workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()), + confirmed_workdir_cleanup_result(), ); let dirty = execute_reserved_workdir_removal_with_provider( &api, @@ -54418,7 +54633,7 @@ mod tests { Some(clean_retry_summary), Vec::new(), ), - workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()), + confirmed_workdir_cleanup_result(), ); let removed_after_retry = execute_reserved_workdir_removal_with_provider( &api, @@ -54444,7 +54659,7 @@ mod tests { Some(corrupted_summary), Vec::new(), ), - workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()), + confirmed_workdir_cleanup_result(), ); let corrupted = execute_reserved_workdir_removal_with_provider( &api, @@ -54493,6 +54708,297 @@ mod tests { assert_eq!(unsupported_provider.cleanup_calls(), 1); } + #[test] + fn workdir_cleanup_failure_classification_is_bounded_and_guides_normal_retry() { + for (code, category) in [ + ("working_directory_cleanup_mount_present", "mount_present"), + ( + "working_directory_cleanup_mount_check_unavailable", + "mount_check_unavailable", + ), + ( + "working_directory_cleanup_permission_denied", + "permission_denied", + ), + ("working_directory_cleanup_resource_busy", "resource_busy"), + ( + "working_directory_cleanup_storage_unavailable", + "storage_unavailable", + ), + ( + "working_directory_cleanup_ownership_unknown", + "ownership_unknown", + ), + ( + "working_directory_cleanup_changes_present", + "changes_present", + ), + ( + "working_directory_cleanup_changes_unknown", + "changes_unknown", + ), + ( + "arbitrary provider text /private/path", + "provider_cleanup_failed", + ), + ] { + let result = workdir_removal_result( + InternalWorkerOperationState::Rejected, + None, + vec![RuntimeDiagnostic { + code: code.into(), + severity: HostDiagnosticSeverity::Error, + message: "secret /private/path".into(), + }], + ); + assert_eq!(workdir_cleanup_failure_category(&result), category); + let guidance = workdir_removal_failure_guidance(category); + assert!(!guidance.contains("/private/path")); + assert!(!guidance.contains("secret")); + assert!(guidance.len() <= 500); + assert!(guidance.contains("retry")); + } + } + + #[tokio::test] + async fn workdir_removal_retry_rechecks_pending_materialization_changes_and_attachments() { + let workspace = tempfile::tempdir().unwrap(); + init_clean_git_workspace(workspace.path()); + let api = test_api(workspace.path()).await; + let (operation, mut summary) = reserve_removal_fixture(&api, "cleanup-retry"); + let failed_provider = FakeWorkdirRemovalProvider::new( + workdir_removal_result( + InternalWorkerOperationState::Accepted, + Some(summary.clone()), + Vec::new(), + ), + workdir_removal_result( + InternalWorkerOperationState::Rejected, + None, + vec![RuntimeDiagnostic { + code: "working_directory_cleanup_mount_present".into(), + severity: HostDiagnosticSeverity::Error, + message: "/private/mount credentials must not escape".into(), + }], + ), + ); + let failed = execute_reserved_workdir_removal_with_provider( + &api, + operation, + false, + &failed_provider, + ) + .unwrap(); + assert!(failed.retryable); + assert_eq!(failed.failure_category.as_deref(), Some("mount_present")); + assert!( + api.store + .get_workdir_registry(TEST_WORKSPACE_ID, "cleanup-retry") + .unwrap() + .is_some() + ); + assert!(!serde_json::to_string(&failed).unwrap().contains("/private")); + + summary.status = WorkingDirectoryStatusKind::CleanupPending; + summary.cleanliness = Some("dirty".into()); + let dirty_provider = FakeWorkdirRemovalProvider::new( + workdir_removal_result( + InternalWorkerOperationState::Accepted, + Some(summary.clone()), + Vec::new(), + ), + confirmed_workdir_cleanup_result(), + ); + let dirty = + execute_reserved_workdir_removal_with_provider(&api, failed, false, &dirty_provider) + .unwrap(); + assert_eq!(dirty.disposition, Some(WorkdirRemovalDisposition::Retained)); + assert!(dirty.retryable); + assert_eq!(dirty_provider.cleanup_calls(), 0); + + let worker = seed_cleanup_worker(&api, 731, "normal"); + seed_cleanup_link(&api, &worker, "cleanup-retry"); + let occupied_provider = FakeWorkdirRemovalProvider::new( + workdir_removal_result( + InternalWorkerOperationState::Accepted, + Some(summary.clone()), + Vec::new(), + ), + confirmed_workdir_cleanup_result(), + ); + let occupied = + execute_reserved_workdir_removal_with_provider(&api, dirty, false, &occupied_provider) + .unwrap(); + assert_eq!( + occupied.failure_category.as_deref(), + Some("blocked_by_live_authority") + ); + assert!(occupied.retryable); + assert_eq!(occupied_provider.cleanup_calls(), 0); + api.store + .detach_worker_workdir( + TEST_WORKSPACE_ID, + &RuntimeWorkerRef::new("runtime-test", worker), + Some("cleanup-retry"), + "resolve retry attachment", + ) + .unwrap(); + + // Unknown because checkout was partially removed is delegated to Runtime + // for current change/identity protection, not denied by the old state. + summary.cleanliness = Some("unknown".into()); + let retry_provider = FakeWorkdirRemovalProvider::new( + workdir_removal_result( + InternalWorkerOperationState::Accepted, + Some(summary), + Vec::new(), + ), + confirmed_workdir_cleanup_result(), + ); + let removed = execute_reserved_workdir_removal_with_provider( + &api, + occupied.clone(), + false, + &retry_provider, + ) + .unwrap(); + assert_eq!(removed.operation_id, occupied.operation_id); + assert_eq!(removed.attempt_count, 4); + assert_eq!( + removed.disposition, + Some(WorkdirRemovalDisposition::Removed) + ); + assert_eq!(retry_provider.cleanup_calls(), 1); + assert!( + api.store + .get_workdir_registry(TEST_WORKSPACE_ID, "cleanup-retry") + .unwrap() + .is_none() + ); + assert_eq!( + execute_reserved_workdir_removal_with_provider( + &api, + removed.clone(), + false, + &retry_provider + ) + .unwrap(), + removed + ); + } + + #[tokio::test] + async fn workdir_observation_failure_preserves_bounded_os_cause_without_removing_registry() { + let workspace = tempfile::tempdir().unwrap(); + init_clean_git_workspace(workspace.path()); + let api = test_api(workspace.path()).await; + for (id, transport_error) in [ + ("observation-denied", false), + ("rpc-observation-denied", true), + ] { + let (operation, _) = reserve_removal_fixture(&api, id); + let provider = FakeWorkdirRemovalProvider::new( + workdir_removal_result( + InternalWorkerOperationState::Rejected, + None, + vec![RuntimeDiagnostic { + code: "working_directory_cleanup_permission_denied".into(), + severity: HostDiagnosticSeverity::Error, + message: "permission denied /private/path".into(), + }], + ), + confirmed_workdir_cleanup_result(), + ); + if transport_error { + *provider.observation.lock().unwrap() = + Some(Err(RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: "runtime-test".into(), + code: "working_directory_cleanup_permission_denied".into(), + message: "permission denied /private/path".into(), + })); + } + let result = + execute_reserved_workdir_removal_with_provider(&api, operation, false, &provider) + .unwrap(); + assert_eq!( + result.failure_category.as_deref(), + Some("permission_denied") + ); + assert!(result.retryable); + assert_eq!(provider.cleanup_calls(), 0); + assert!( + !serde_json::to_string(&result) + .unwrap() + .contains("/private/path") + ); + assert!( + api.store + .get_workdir_registry(TEST_WORKSPACE_ID, id) + .unwrap() + .is_some() + ); + } + } + + #[tokio::test] + async fn workdir_removal_keeps_registry_for_changed_identity_and_unknown_cleanup_outcome() { + let workspace = tempfile::tempdir().unwrap(); + init_clean_git_workspace(workspace.path()); + let api = test_api(workspace.path()).await; + for (id, changed_identity) in [("identity-mismatch", true), ("unknown-cleanup", false)] { + let (operation, mut summary) = reserve_removal_fixture(&api, id); + if changed_identity { + summary.repository_id = "another-repository".into(); + } + let provider = FakeWorkdirRemovalProvider::new( + workdir_removal_result( + InternalWorkerOperationState::Accepted, + Some(summary), + Vec::new(), + ), + workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()), + ); + let result = + execute_reserved_workdir_removal_with_provider(&api, operation, false, &provider) + .unwrap(); + assert!(result.retryable); + assert_eq!( + result.failure_category.as_deref(), + Some(if changed_identity { + "provider_identity_changed" + } else { + "provider_cleanup_outcome_unknown" + }) + ); + assert_eq!(provider.cleanup_calls(), usize::from(!changed_identity)); + assert!( + api.store + .get_workdir_registry(TEST_WORKSPACE_ID, id) + .unwrap() + .is_some() + ); + } + } + + #[tokio::test] + async fn cleanup_plan_offers_normal_revalidation_for_prior_failure_and_unknown() { + let workspace = tempfile::tempdir().unwrap(); + init_clean_git_workspace(workspace.path()); + let api = test_api(workspace.path()).await; + for (id, state) in [ + ("pending-retry", "pending"), + ("unknown-retry", "unknown"), + ("corrupt-retry", "corrupted"), + ] { + seed_cleanup_workdir(&api, id, state, "unknown"); + } + let plan = build_runtime_cleanup_plan(&api, "runtime-test").unwrap(); + for candidate in plan.workdirs { + assert_eq!(candidate.action, CleanupTargetKind::WorkdirCleanCleanup); + assert!(candidate.blocking_reason.is_none()); + } + } + #[tokio::test] async fn concurrent_orphan_recovery_runs_delayed_provider_cleanup_once() { let workspace = tempfile::tempdir().unwrap(); @@ -54518,7 +55024,7 @@ mod tests { Some(clean_summary), Vec::new(), ), - workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()), + confirmed_workdir_cleanup_result(), ) .with_observation_delay(std::time::Duration::from_millis(100)), ); diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 9d78665a7..2388611c3 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -10254,6 +10254,19 @@ impl ControlPlaneStore for SqliteWorkspaceStore { record.workdir_id, active.worker.runtime_id, active.worker.worker_id ))); } + // Direct attachments (including ordinary recovery/reconnection) + // must share the same atomic removal fence as reserved attachments. + let removal_pending: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM workdir_removal_operations WHERE workspace_id=?1 AND workdir_id=?2 AND state='pending')", + params![record.workspace_id, record.workdir_id], + |row| row.get(0), + )?; + if removal_pending { + return Err(Error::WorkdirAttachmentConflict(format!( + "Workdir {} has a pending durable removal operation", + record.workdir_id + ))); + } let write = tx.execute( r#"INSERT INTO worker_workdir_links ( workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id diff --git a/crates/workspace-server/src/workdir_removal.rs b/crates/workspace-server/src/workdir_removal.rs index 0208714fe..9356ebdf0 100644 --- a/crates/workspace-server/src/workdir_removal.rs +++ b/crates/workspace-server/src/workdir_removal.rs @@ -438,7 +438,9 @@ impl SqliteWorkspaceStore { "retained completion cannot use removed disposition".to_string(), )); } - self.finish_workdir_removal_operation(operation, disposition, false, Some(category), false) + // Retention describes this attempt's current blockers, not a permanent + // ban. A subsequent normal request rechecks all live authority. + self.finish_workdir_removal_operation(operation, disposition, true, Some(category), false) } pub fn fail_workdir_removal_operation( diff --git a/docs/development/workdir-removal.md b/docs/development/workdir-removal.md new file mode 100644 index 000000000..450d0ac5f --- /dev/null +++ b/docs/development/workdir-removal.md @@ -0,0 +1,73 @@ +# Workdir removal and ordinary retry + +Use the Runtime **Workdirs** page or the normal typed Workdir removal tool to +remove an unused repository Workdir. Deletion is authorized by the Backend's +existing durable removal operation; it is not a registry-only hide operation. + +A failed attempt does not require a new recovery state or a manual edit of +`materialization.json`. Retry the same removal request after resolving the +reported cause. Each attempt rechecks current attachments, attachment +reservations, retention holds, materialization identity and current file changes. +A previous clean observation is not permission to discard new changes. A dirty +Workdir must be preserved or its changes resolved by its owner before deletion. +When the same repository checkout is still usable, an authorized Worker may +reattach it through normal `WorkdirAttach` to preserve or resolve changes, then +detach and retry removal. An executing removal fences new attachments; a failed +attempt does not permanently prevent reconnection. Missing or replaced checkout +identity is not made usable by this permission. + +## Interpreting a retained Workdir + +- **Mount remains:** Git status being clean does not mean mounted resources are + released. Confirm the mount's owner, users and data destination. Have that + owner safely release the resource, then retry normal deletion. Yoi does not + recursively delete mounted data or implicitly unmount external/unknown + resources. Do not use force-clean, recursive chmod or raw recursive deletion + to override this boundary. +- **Permission denied / resource busy / storage unavailable:** have the owning + operator resolve access, release the resource or restore storage. Retry the + same normal action; registry retention is intentional until cleanup is proven. +- **Changes present / changes cannot be verified:** preserve current files and + restore repository inspection where possible. Do not treat a previous cleanup + failure or missing checkout metadata as proof that all remaining files are + expendable. +- **Identity or ownership unavailable:** verify the Workdir target with its + owner. Do not rewrite registry or materialization identity to bypass checks. +- **Runtime unavailable / outcome unknown:** reconnect the Runtime and retry. + Absence of a provider response is not authoritative not-found evidence. +- **Attachment, reservation or hold:** release the current owning authority + through its normal API before retrying. A stale failure does not freeze these + conditions; newly attached Workers remain protected. + +The Web page refreshes authoritative inventory after each attempt. Successful +removal commits registry deletion only after Runtime confirms physical deletion +or authoritative not-found. An unknown result leaves the registry in place so +retry can reconcile the provider's current state. + +## Runtime safety boundary + +The Runtime keeps removal content/identity evidence outside the tree being +removed. This is a factual witness for partial deletion, not a new recovery +state or cleanup stage. Surviving files are rechecked against it; an intact, +currently clean checkout can renew obsolete evidence without discarding newly +ignored/untracked content. Do not edit this witness manually. + +Mount-safe removal currently requires Linux `openat2` and `/proc` inspection. +Unavailable inspection fails closed with `mount_check_unavailable`; there is no +fallback to recursive deletion across an unchecked filesystem boundary. +Removal retains the Runtime occupancy mutex through its filesystem effect. +Unrelated Runtime state operations can therefore wait during hashing/unlink; +releasing it would require equivalent exclusion on all attachment paths. + +## Diagnostics + +Public responses expose a bounded cause category and safe guidance, not arbitrary +provider errors or host paths. For an OS failure, correlate Backend removal logs +(`operation_id`, `workdir_id`, Runtime) with Runtime cleanup logs for that Workdir +and cleanup action. Runtime logs retain the concrete OS error/errno for the +owning operator. These private diagnostics must not be copied unredacted into +public responses or Tickets. + +This procedure is not authorization to manipulate a live mount or another +Worker's storage. If ownership or required operator authority is unclear, stop +and request that specific owner/operator's decision. diff --git a/docs/report/2026-10-09-subworker-workdir-recovery-ergonomics.md b/docs/report/2026-10-09-subworker-workdir-recovery-ergonomics.md new file mode 100644 index 000000000..ae3a0d3ec --- /dev/null +++ b/docs/report/2026-10-09-subworker-workdir-recovery-ergonomics.md @@ -0,0 +1,34 @@ +# SubWorker / Workdir execution ergonomics during T-730 + +Observed during Workdir removal implementation in a dedicated clean checkout: + +- Children using `profile: builtin:default` were given explicit writable scope + and command permission, but their available tools/context did not support the + requested implementation. They performed read-only investigation and reported + the missing capabilities. Recreating coding children with `profile: inherit` + exposed the needed implementation/Ticket tools. No rejected capability was + bypassed. +- Concurrent Workdir reads sometimes returned `filesystem provider busy`. + This is transient provider contention, not an authorization rejection. A + child initially treated it as a stop condition; a subsequent parent turn + clarified the difference and sequential retries proceeded normally. +- After a user interruption, repository changes and parent tasks remained, but + `WorkerList` no longer listed the previously active children. Their work was + not treated as approved or complete: the parent inspected concrete diffs, + reran Web/browser validation and delegated a fresh bounded Runtime follow-up. + +Potential improvements: + +1. Show effective child tool capabilities together with filesystem/command + grants before spawn, so a writable scope is not mistaken for an available + implementation tool surface. +2. Distinguish retryable provider contention from capability rejection in typed + error metadata and recommended recovery text. +3. Make interruption disposition and retained child-session observation grants + explicit. Preserved files are useful evidence, but they do not replace a + committed final child report or verification of tests. + +These observations concern development-tool ergonomics, not product removal +requirements or permission to operate on live Workdirs/mounts. The T-730 work +uses temporary fixtures and ordinary Git publication; no live cleanup, unmount, +registry edits or runtime deployment were performed. diff --git a/tools/web-ux/browser-tests/workdir_fixture.ts b/tools/web-ux/browser-tests/workdir_fixture.ts new file mode 100644 index 000000000..4e3597870 --- /dev/null +++ b/tools/web-ux/browser-tests/workdir_fixture.ts @@ -0,0 +1,140 @@ +// Synthetic API boundary only; the production route and shell own all rendering. +export function workdirFixture() { + let mountReleased = false; + const deleted = new Set(); + const requests: { + workspace: string; + method: string; + path: string; + body?: unknown; + }[] = []; + const summary = ( + id: string, + status = "active", + cleanliness = "clean", + occupied = false, + ) => ({ + working_directory_id: id, + display_name: id === "retry-dir" ? "Cleanup retry — distributed development checkout" : id, + source: { kind: "repository", repository_key: "main" }, + materializer_kind: "runtime_git_clone", + status, + cleanliness, + current_ref: "a".repeat(40), + current_selector: "develop", + occupied_by: occupied + ? { + runtime_id: "fixture-runtime", + worker_id: "worker-1", + display_name: "Active Worker", + linked_at: "2026-01-01T00:00:00Z", + } + : null, + }); + const items = (workspace: string) => + workspace === "home-empty" ? [] : [ + ...(deleted.has(workspace) ? [] : [summary("retry-dir", "cleanup_pending", "unknown")]), + summary("dirty-dir", "active", "dirty"), + summary("occupied-dir", "active", "clean", true), + ]; + const plan = (workspace: string) => ({ + workspace_id: workspace, + runtime_id: "fixture-runtime", + generated_at: "2026-01-01T00:00:00Z", + revision: "plan-1", + digest: "digest-1", + workers: [], + diagnostics: [], + workdirs: items(workspace).map((item) => ({ + target_id: `workdir:${item.working_directory_id}`, + action: item.cleanliness === "dirty" ? "workdir_dirty_discard" : "workdir_clean_cleanup", + workdir_id: item.working_directory_id, + runtime_id: "fixture-runtime", + repository_key: "main", + reason: "Ordinary cleanup rechecks current conditions", + linked_worker_ids: item.occupied_by ? ["worker-1"] : [], + linked_running_worker_ids: item.occupied_by ? ["worker-1"] : [], + running_linked: Boolean(item.occupied_by), + pinned_linked: false, + file_status: item.status === "cleanup_pending" ? "pending" : "present", + cleanliness: item.cleanliness, + blocking_reason: item.occupied_by ? "Workdir is linked to a running Worker" : null, + })), + }); + return async (request: Request): Promise => { + const url = new URL(request.url); + if ( + url.pathname === "/fixture-workdirs-control" && request.method === "POST" + ) { + const control = await request.json(); + mountReleased = control.mountReleased === true; + if (control.reset) { + deleted.clear(); + requests.length = 0; + } + return Response.json({ ok: true }); + } + if (url.pathname === "/fixture-workdirs") { + return Response.json({ requests }); + } + const match = url.pathname.match(/^\/api\/w\/([^/]+)(\/runtimes.*)$/); + if (!match) return null; + const [, workspace, path] = match; + if (path === "/runtimes") { + return Response.json({ + workspace_id: workspace, + limit: 100, + source: "fixture", + diagnostics: [], + items: [], + }); + } + if (!path.startsWith("/runtimes/fixture-runtime/")) return null; + const body = request.method !== "GET" ? await request.json() : undefined; + requests.push({ workspace, method: request.method, path, body }); + if (path.endsWith("/working-directories")) { + return Response.json({ + workspace_id: workspace, + items: items(workspace), + diagnostics: [], + }); + } + if (path.endsWith("/cleanup-plan")) return Response.json(plan(workspace)); + // Baseline route uses cleanup-executions; same retained outcome for before captures. + if (path.endsWith("/cleanup-executions")) { + return Response.json({ + workspace_id: workspace, + runtime_id: "fixture-runtime", + executed_at: "2026-01-01T00:00:00Z", + diagnostics: [], + plan_after: plan(workspace), + results: [{ + target_id: "workdir:retry-dir", + action: "workdir_clean_cleanup", + status: "retained", + message: "Provider cleanup failed", + }], + }); + } + if ( + path.endsWith("/working-directories/retry-dir") && + request.method === "DELETE" + ) { + if (workspace === "home-error") { + return Response.json({ + message: "Synthetic private host path must not render", + }, { status: 503 }); + } + if (mountReleased) deleted.add(workspace); + return Response.json({ + working_directory_id: "retry-dir", + disposition: mountReleased ? "removed" : "attention_required", + retryable: !mountReleased, + failure_category: mountReleased ? null : "mount_present", + }); + } + return Response.json({ error: "Unexpected fixture Workdir request" }, { + status: 404, + }); + }; +} diff --git a/tools/web-ux/browser-tests/workdir_removal_shell_test.ts b/tools/web-ux/browser-tests/workdir_removal_shell_test.ts new file mode 100644 index 000000000..e48f0724e --- /dev/null +++ b/tools/web-ux/browser-tests/workdir_removal_shell_test.ts @@ -0,0 +1,456 @@ +/// +import { assert, assertEquals, assertStringIncludes } from "@std/assert"; +import { dirname, fromFileUrl, join } from "@std/path"; +import { chromium, type Page } from "playwright"; +import { startOwnedProcesses, stopOwnedProcesses } from "../src/processes.ts"; +const root = join(dirname(fromFileUrl(import.meta.url)), "../../.."); +const evidence = join(root, "target/web-ux/t730-browser"); + +async function noOverflow(page: Page) { + assertEquals( + await page.locator("main").evaluate((main) => ({ + page: document.documentElement.scrollWidth > innerWidth, + main: main.scrollWidth > main.clientWidth, + })), + { page: false, main: false }, + ); +} + +Deno.test("ordinary removal refreshes failed and successful inventory, supports exact retry, and preserves permission and live guards across viewports and themes", async () => { + await Deno.mkdir(evidence, { recursive: true }); + const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 }); + const port = (listener.addr as Deno.NetAddr).port; + listener.close(); + const url = `http://127.0.0.1:${port}`; + const server = await startOwnedProcesses( + [{ + id: "workdirs", + command: Deno.execPath(), + cwd: root, + args: [ + "run", + "--allow-net", + "--allow-read", + "tools/web-ux/browser-tests/workspace_home_fixture_server.ts", + String(port), + "web/workspace/build", + ], + readyUrl: `${url}/health`, + }], + join(root, "tools/web-ux/scenarios/workdir-removal-light.json"), + join(evidence, "process-logs"), + [], + ); + const observations: unknown[] = []; + const injectedConsoleErrors: string[] = []; + const injectedUnexpectedErrors: string[] = []; + const observeInjectedFailure = (page: Page) => { + page.on("console", (message) => { + if (message.type() === "error") { + injectedConsoleErrors.push(message.text()); + } + }); + page.on( + "pageerror", + (error) => injectedUnexpectedErrors.push(String(error)), + ); + page.on( + "requestfailed", + (request) => injectedUnexpectedErrors.push(request.url()), + ); + }; + const control = async (value: unknown) => { + const response = await fetch(`${url}/fixture-workdirs-control`, { + method: "POST", + body: JSON.stringify(value), + }); + assert(response.ok); + await response.body?.cancel(); + }; + const state = async () => await (await fetch(`${url}/fixture-workdirs`)).json(); + const browser = await chromium.launch({ headless: true }); + try { + for (const colorScheme of ["light", "dark"] as const) { + for (const width of [1440, 768, 390, 320]) { + await control({ reset: true }); + const context = await browser.newContext({ + viewport: { width, height: 900 }, + colorScheme, + reducedMotion: "reduce", + }); + try { + const page = await context.newPage(); + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(String(error))); + page.on("console", (message) => { + if (message.type() === "error") errors.push(message.text()); + }); + page.on("requestfailed", (request) => errors.push(request.url())); + const route = `${url}/w/home-owner/settings/runtimes/fixture-runtime/workdirs`; + await page.goto(route); + const deletion = page.getByRole("button", { + name: "Delete retry-dir", + exact: true, + }); + await deletion.waitFor(); + assert( + await deletion.isEnabled(), + "past pending/unknown state permanently disabled ordinary removal", + ); + assert( + await page.getByRole("button", { + name: "Delete dirty-dir", + exact: true, + }).isDisabled(), + ); + assert( + await page.getByRole("button", { + name: "Delete occupied-dir", + exact: true, + }).isDisabled(), + ); + await noOverflow(page); + assert( + await deletion.evaluate((button) => { + const r = button.getBoundingClientRect(); + return r.left >= 0 && r.right <= innerWidth; + }), + "Delete is outside initial viewport", + ); + await page.screenshot({ + path: join(evidence, `${colorScheme}-${width}-initial.png`), + }); + // Hold the actual request boundary to verify duplicate gestures cannot dispatch another DELETE. + let release!: () => void; + const gate = new Promise((resolve) => release = resolve); + let observed!: () => void; + const entered = new Promise((resolve) => observed = resolve); + let deletes = 0; + await page.route( + "**/working-directories/retry-dir", + async (intercept) => { + deletes++; + observed(); + await gate; + await intercept.continue(); + }, + ); + await deletion.focus(); + await page.keyboard.press("Enter"); + await page.getByRole("button", { + name: "Delete retry-dir", + exact: true, + }).filter({ hasText: "Deleting…" }).waitFor(); + assert(await deletion.isDisabled()); + await page.keyboard.press("Enter"); + await entered; + assertEquals(deletes, 1); + await page.screenshot({ + path: join(evidence, `${colorScheme}-${width}-deleting.png`), + }); + release(); + await page.getByRole("alert").filter({ + hasText: "mounted filesystem", + }).waitFor(); + const retry = page.getByRole("button", { + name: "Retry Delete retry-dir", + exact: true, + }); + await retry.waitFor(); + await page.waitForFunction(() => + !(document.querySelector( + "button[aria-label='Retry Delete retry-dir']", + ) as HTMLButtonElement)?.disabled + ); + const failed = await state(); + const beforeGets = failed.requests.filter(( + r: { workspace: string; method: string }, + ) => r.workspace === "home-owner" && r.method === "GET"); + assertEquals( + beforeGets.filter((r: { path: string }) => r.path.endsWith("/working-directories")) + .length, + 2, + ); + assertEquals( + beforeGets.filter((r: { path: string }) => r.path.endsWith("/cleanup-plan")).length, + 2, + ); + assertEquals(await page.locator("tbody tr").count(), 3); + await noOverflow(page); + await page.screenshot({ + path: join(evidence, `${colorScheme}-${width}-mount-failure.png`), + }); + await control({ mountReleased: true }); + await retry.focus(); + await page.keyboard.press("Enter"); + await page.getByRole("status").filter({ + hasText: "Deletion confirmed", + }).waitFor(); + await page.waitForFunction(() => document.querySelectorAll("tbody tr").length === 2); + assertEquals( + await page.getByRole("button", { name: /retry-dir/ }).count(), + 0, + ); + assert( + await page.getByRole("button", { + name: "Delete dirty-dir", + exact: true, + }).isDisabled(), + ); + const final = await state(); + const mutations = final.requests.filter((r: { method: string }) => r.method === "DELETE"); + assertEquals(mutations.length, 2); + assertEquals( + mutations[0], + mutations[1], + "ordinary retry changed its request", + ); + const gets = final.requests.filter(( + r: { workspace: string; method: string }, + ) => r.workspace === "home-owner" && r.method === "GET"); + assertEquals( + gets.filter((r: { path: string }) => r.path.endsWith("/working-directories")).length, + 3, + ); + assertEquals( + gets.filter((r: { path: string }) => r.path.endsWith("/cleanup-plan")).length, + 3, + ); + await page.screenshot({ + path: join(evidence, `${colorScheme}-${width}-removed.png`), + }); + await page.goto( + `${url}/w/home-member/settings/runtimes/fixture-runtime/workdirs`, + ); + await page.locator("tbody tr").first().waitFor(); + assertEquals( + await page.getByRole("button", { name: /Delete/ }).count(), + 0, + ); + await noOverflow(page); + await page.screenshot({ + path: join(evidence, `${colorScheme}-${width}-member.png`), + }); + await page.goto( + `${url}/w/home-empty/settings/runtimes/fixture-runtime/workdirs`, + ); + await page.getByText("No workdirs are visible for this Runtime.") + .waitFor(); + await noOverflow(page); + assertEquals(errors, []); + observations.push({ + colorScheme, + width, + requests: final.requests, + errors, + }); + } finally { + await context.close(); + } + } + } + + // Lost/HTTP failure responses still trigger fresh inventory and eligibility. Never echo provider bodies. + await control({ reset: true }); + const page = await browser.newPage(); + observeInjectedFailure(page); + await page.goto( + `${url}/w/home-error/settings/runtimes/fixture-runtime/workdirs`, + ); + await page.getByRole("button", { name: "Delete retry-dir", exact: true }) + .click(); + await page.getByRole("alert").filter({ hasText: "provider is unavailable" }) + .waitFor(); + await page.waitForFunction(() => + !(document.querySelector( + "button[aria-label='Delete retry-dir']", + ) as HTMLButtonElement)?.disabled + ); + assert( + !(await page.locator("main").innerText()).includes("Synthetic private"), + ); + const failed = await state(); + assertEquals( + failed.requests.filter((r: { method: string; path: string }) => + r.method === "GET" && r.path.endsWith("/working-directories") + ).length, + 2, + ); + assertEquals( + failed.requests.filter((r: { method: string; path: string }) => + r.method === "GET" && r.path.endsWith("/cleanup-plan") + ).length, + 2, + ); + await page.screenshot({ path: join(evidence, "http-failure.png") }); + await page.close(); + + // Plan refresh failure cannot leave stale delete authority enabled; manual Refresh recovers it. + await control({ reset: true }); + const stale = await browser.newPage(); + observeInjectedFailure(stale); + let planGets = 0; + let planFails = true; + await stale.route("**/cleanup-plan", async (route) => { + planGets++; + if (planGets > 1 && planFails) { + await route.fulfill({ + status: 503, + json: { message: "Synthetic private path" }, + }); + } else await route.continue(); + }); + await stale.goto( + `${url}/w/home-owner/settings/runtimes/fixture-runtime/workdirs`, + ); + await stale.getByRole("button", { name: "Delete retry-dir", exact: true }) + .click(); + await stale.getByRole("alert").filter({ hasText: "could not be refreshed" }) + .waitFor(); + assert( + await stale.getByRole("button", { + name: "Retry Delete retry-dir", + exact: true, + }).isDisabled(), + ); + assert( + !(await stale.locator("main").innerText()).includes("Synthetic private"), + ); + await stale.screenshot({ path: join(evidence, "refresh-failure.png") }); + planFails = false; + await stale.getByRole("button", { name: "Refresh", exact: true }).click(); + await stale.waitForFunction(() => + !(document.querySelector( + "button[aria-label='Retry Delete retry-dir']", + ) as HTMLButtonElement)?.disabled + ); + await stale.close(); + + // Current dirty changes after a failure override a retryable historical result. + await control({ reset: true }); + const changed = await browser.newPage(); + let changedGets = 0; + await changed.route("**/cleanup-plan", async (route) => { + const response = await route.fetch(); + const plan = await response.json(); + if (++changedGets > 1) plan.workdirs[0].action = "workdir_dirty_discard"; + await route.fulfill({ response, json: plan }); + }); + await changed.goto( + `${url}/w/home-owner/settings/runtimes/fixture-runtime/workdirs`, + ); + await changed.getByRole("button", { name: "Delete retry-dir", exact: true }) + .click(); + await changed.getByRole("button", { + name: "Retry Delete retry-dir", + exact: true, + }).waitFor(); + await changed.waitForFunction(() => + document.querySelector(".workdir-identity .removal-guard")?.textContent + ?.includes("Changes are protected") + ); + assert( + await changed.getByRole("button", { + name: "Retry Delete retry-dir", + exact: true, + }).isDisabled(), + ); + assertStringIncludes( + await changed.locator("main").innerText(), + "Changes are protected", + ); + await changed.screenshot({ + path: join(evidence, "new-changes-protected.png"), + }); + await changed.close(); + // A malformed response may follow a real side effect. Inventory still removes the now-absent row. + await control({ reset: true, mountReleased: true }); + const lost = await browser.newPage(); + observeInjectedFailure(lost); + await lost.route("**/working-directories/retry-dir", async (route) => { + await route.fetch(); + await route.fulfill({ + status: 200, + json: { + message: "Synthetic private path", + failure_category: "/private/path token=synthetic", + }, + }); + }); + await lost.goto( + `${url}/w/home-owner/settings/runtimes/fixture-runtime/workdirs`, + ); + await lost.getByRole("button", { name: "Delete retry-dir", exact: true }) + .click(); + await lost.getByRole("alert").filter({ + hasText: "Removal could not be confirmed", + }).waitFor(); + await lost.waitForFunction(() => document.querySelectorAll("tbody tr").length === 2); + assert( + !(await lost.locator("main").innerText()).includes("Synthetic private"), + ); + assert(!(await lost.locator("main").innerText()).includes("/private/path")); + await lost.screenshot({ + path: join(evidence, "malformed-response-refreshed.png"), + }); + await lost.close(); + + // Even a failed initial inventory can recover into a confirmed empty list with manual Refresh. + const emptyRecovery = await browser.newPage(); + observeInjectedFailure(emptyRecovery); + let inventoryFails = true; + await emptyRecovery.route( + "**/runtimes/fixture-runtime/working-directories", + async (route) => { + if (inventoryFails) { + await route.fulfill({ + status: 503, + json: { message: "Synthetic private path" }, + }); + } else await route.continue(); + }, + ); + await emptyRecovery.goto( + `${url}/w/home-empty/settings/runtimes/fixture-runtime/workdirs`, + ); + await emptyRecovery.getByText("Workdir inventory is unavailable.") + .waitFor(); + inventoryFails = false; + await emptyRecovery.getByRole("button", { name: "Refresh", exact: true }) + .click(); + await emptyRecovery.getByText("No workdirs are visible for this Runtime.") + .waitFor(); + await emptyRecovery.close(); + assertEquals(injectedUnexpectedErrors, []); + assert( + injectedConsoleErrors.every((message) => message.includes("503")), + "unexpected injected failure console diagnostic", + ); + await Deno.writeTextFile( + join(evidence, "evidence.json"), + JSON.stringify( + { + observations, + injectedConsoleErrors, + injectedUnexpectedErrors, + failureRefresh: failed.requests, + refreshFailureRecovered: true, + newChangesProtected: true, + malformedResponseReconciled: true, + emptyInventoryRecovered: true, + expectedFailureDiagnostics: [ + "Synthetic DELETE HTTP 503 on home-error", + "Synthetic cleanup-plan HTTP 503 after removal", + "Malformed DELETE result after fixture physical removal", + "Synthetic initial inventory HTTP 503", + ], + }, + null, + 2, + ), + ); + } finally { + await browser.close(); + assertEquals(await stopOwnedProcesses(server), []); + } +}); diff --git a/tools/web-ux/browser-tests/workspace_home_fixture_server.ts b/tools/web-ux/browser-tests/workspace_home_fixture_server.ts index c80c4f697..5d478044f 100644 --- a/tools/web-ux/browser-tests/workspace_home_fixture_server.ts +++ b/tools/web-ux/browser-tests/workspace_home_fixture_server.ts @@ -2,6 +2,8 @@ import { extname, join, normalize } from "jsr:@std/path@1.1.4"; import { dashboardFixture } from "../../../web/workspace/src/lib/workspace/home/dashboard.test-fixtures.ts"; import { settingsFixtureHandler } from "../../../web/workspace/src/lib/workspace/settings/identity.test-fixtures.ts"; +import { workdirFixture } from "./workdir_fixture.ts"; +const workdirsFixture = workdirFixture(); const settingsFixture = settingsFixtureHandler(); const port = Number(Deno.args[0]); @@ -30,6 +32,8 @@ const mime: Record = { Deno.serve({ hostname: "127.0.0.1", port }, async (request) => { const url = new URL(request.url); if (url.pathname === "/health") return new Response("ok"); + const workdirsResponse = await workdirsFixture(request); + if (workdirsResponse) return workdirsResponse; const settingsResponse = await settingsFixture(request); if (settingsResponse) return settingsResponse; if (url.pathname === "/api/workspaces") { diff --git a/tools/web-ux/deno.json b/tools/web-ux/deno.json index e3e278be5..0ffbee433 100644 --- a/tools/web-ux/deno.json +++ b/tools/web-ux/deno.json @@ -11,7 +11,7 @@ "web-ux": "deno run --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys cli.ts", "check": "deno check cli.ts src/*.ts tests/*.ts browser-tests/*.ts", "test": "deno test --allow-env --allow-read --allow-write --allow-run --allow-sys tests", - "test:browser": "deno test --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys browser-tests/ticket_shell_test.ts browser-tests/capture_smoke_test.ts browser-tests/memory_document_shell_test.ts browser-tests/console_history_shell_test.ts browser-tests/workspace_home_shell_test.ts browser-tests/workspace_settings_shell_test.ts browser-tests/console_pending_shell_test.ts browser-tests/console_live_shell_test.ts browser-tests/console_live_admission_shell_test.ts" + "test:browser": "deno test --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys browser-tests/ticket_shell_test.ts browser-tests/capture_smoke_test.ts browser-tests/memory_document_shell_test.ts browser-tests/console_history_shell_test.ts browser-tests/workspace_home_shell_test.ts browser-tests/workspace_settings_shell_test.ts browser-tests/workdir_removal_shell_test.ts browser-tests/console_pending_shell_test.ts browser-tests/console_live_shell_test.ts browser-tests/console_live_admission_shell_test.ts" }, "fmt": { "lineWidth": 100, diff --git a/tools/web-ux/scenarios/workdir-removal-dark.json b/tools/web-ux/scenarios/workdir-removal-dark.json new file mode 100644 index 000000000..b689c72d1 --- /dev/null +++ b/tools/web-ux/scenarios/workdir-removal-dark.json @@ -0,0 +1,117 @@ +{ + "schemaVersion": 1, + "id": "workdir-removal-dark", + "title": "Ordinary Workdir removal and guarded retry", + "baseUrl": "http://127.0.0.1:15290", + "colorScheme": "dark", + "reducedMotion": "reduce", + "personas": [ + { + "id": "fixture", + "label": "Synthetic permission projections", + "auth": { + "kind": "anonymous" + } + } + ], + "viewports": [ + { + "label": "desktop", + "width": 1440, + "height": 900 + }, + { + "label": "narrow", + "width": 768, + "height": 900 + }, + { + "label": "mobile", + "width": 390, + "height": 844 + } + ], + "routes": [ + { + "id": "owner", + "label": "Owner removal", + "path": "/w/home-owner/settings/runtimes/fixture-runtime/workdirs", + "goal": "Remove a failed-cleanup Workdir using ordinary retry without enabling dirty or occupied removal.", + "dataState": "Pending/unknown retry plus dirty and occupied guards. First attempt finds a mount.", + "ready": { + "kind": "selector", + "selector": "main table tbody tr" + }, + "capturePoints": [ + { + "id": "initial", + "label": "Current inventory" + }, + { + "id": "failed-attempt", + "label": "Removal blocked by mount", + "interaction": [ + { + "action": "click", + "selector": "button[aria-label='Delete retry-dir']" + } + ], + "ready": { + "kind": "selector", + "selector": "[role='alert']" + } + } + ] + }, + { + "id": "member", + "label": "No Runtime management permission", + "path": "/w/home-member/settings/runtimes/fixture-runtime/workdirs", + "goal": "Read inventory without owner-only removal controls.", + "dataState": "Management permission false.", + "ready": { + "kind": "selector", + "selector": "main table tbody tr" + }, + "capturePoints": [ + { + "id": "initial", + "label": "Read only inventory" + } + ] + }, + { + "id": "empty", + "label": "Empty inventory", + "path": "/w/home-empty/settings/runtimes/fixture-runtime/workdirs", + "goal": "Recognize no remaining Workdirs.", + "dataState": "Empty list.", + "ready": { + "kind": "selector", + "selector": "main:has-text('No workdirs')" + }, + "capturePoints": [ + { + "id": "initial", + "label": "Empty inventory" + } + ] + } + ], + "processes": [ + { + "id": "workdir-fixture", + "command": "deno", + "cwd": "..", + "args": [ + "run", + "--allow-net", + "--allow-read", + "browser-tests/workspace_home_fixture_server.ts", + "15290", + "../../web/workspace/build" + ], + "readyUrl": "http://127.0.0.1:15290/health" + } + ] +} diff --git a/tools/web-ux/scenarios/workdir-removal-light.json b/tools/web-ux/scenarios/workdir-removal-light.json new file mode 100644 index 000000000..2fd178d19 --- /dev/null +++ b/tools/web-ux/scenarios/workdir-removal-light.json @@ -0,0 +1,81 @@ +{ + "schemaVersion": 1, + "id": "workdir-removal-light", + "title": "Ordinary Workdir removal and guarded retry", + "baseUrl": "http://127.0.0.1:15290", + "colorScheme": "light", + "reducedMotion": "reduce", + "personas": [ + { + "id": "fixture", + "label": "Synthetic permission projections", + "auth": { "kind": "anonymous" } + } + ], + "viewports": [ + { "label": "desktop", "width": 1440, "height": 900 }, + { "label": "narrow", "width": 768, "height": 900 }, + { "label": "mobile", "width": 390, "height": 844 } + ], + "routes": [ + { + "id": "owner", + "label": "Owner removal", + "path": "/w/home-owner/settings/runtimes/fixture-runtime/workdirs", + "goal": "Remove a failed-cleanup Workdir using ordinary retry without enabling dirty or occupied removal.", + "dataState": "Pending/unknown retry plus dirty and occupied guards. First attempt finds a mount.", + "ready": { "kind": "selector", "selector": "main table tbody tr" }, + "capturePoints": [ + { "id": "initial", "label": "Current inventory" }, + { + "id": "failed-attempt", + "label": "Removal blocked by mount", + "interaction": [ + { + "action": "click", + "selector": "button[aria-label='Delete retry-dir']" + } + ], + "ready": { "kind": "selector", "selector": "[role='alert']" } + } + ] + }, + { + "id": "member", + "label": "No Runtime management permission", + "path": "/w/home-member/settings/runtimes/fixture-runtime/workdirs", + "goal": "Read inventory without owner-only removal controls.", + "dataState": "Management permission false.", + "ready": { "kind": "selector", "selector": "main table tbody tr" }, + "capturePoints": [{ "id": "initial", "label": "Read only inventory" }] + }, + { + "id": "empty", + "label": "Empty inventory", + "path": "/w/home-empty/settings/runtimes/fixture-runtime/workdirs", + "goal": "Recognize no remaining Workdirs.", + "dataState": "Empty list.", + "ready": { + "kind": "selector", + "selector": "main:has-text('No workdirs')" + }, + "capturePoints": [{ "id": "initial", "label": "Empty inventory" }] + } + ], + "processes": [ + { + "id": "workdir-fixture", + "command": "deno", + "cwd": "..", + "args": [ + "run", + "--allow-net", + "--allow-read", + "browser-tests/workspace_home_fixture_server.ts", + "15290", + "../../web/workspace/build" + ], + "readyUrl": "http://127.0.0.1:15290/health" + } + ] +} diff --git a/web/workspace/deno.json b/web/workspace/deno.json index 3159061ae..73ab2ea92 100644 --- a/web/workspace/deno.json +++ b/web/workspace/deno.json @@ -6,7 +6,7 @@ "dev": "deno run -A npm:vite@8.3.2 dev", "dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787", "check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json", - "test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts", + "test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts", "test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts", "test:browser:empty-workspace": "deno test --config ../../tools/web-ux/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/visual-review/empty-workspace.browser.test.ts", "test:browser:drive": "deno test --config test/drive-browser/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/drive-browser/drive.browser.test.ts", diff --git a/web/workspace/src/lib/workspace/api/workdirs.ts b/web/workspace/src/lib/workspace/api/workdirs.ts index dbfedcfb7..15619fe84 100644 --- a/web/workspace/src/lib/workspace/api/workdirs.ts +++ b/web/workspace/src/lib/workspace/api/workdirs.ts @@ -7,6 +7,7 @@ import type { WorkingDirectoryDetailResponse, WorkingDirectoryListResponse, WorkingDirectoryOccupancy, + WorkingDirectoryRemovalResponse, WorkingDirectorySource, WorkingDirectorySummary, } from "../../generated/workdir-api"; @@ -63,6 +64,40 @@ export function parseWorkingDirectoryListResponse( }; } +export function parseWorkingDirectoryRemovalResponse( + value: unknown, +): WorkingDirectoryRemovalResponse { + const record = exactRecord( + value, + new Set([ + "working_directory_id", + "disposition", + "retryable", + "failure_category", + ]), + "Workdir removal response", + ); + const working_directory_id = stringField(record, "working_directory_id"); + const category = record.failure_category; + if ( + working_directory_id.length > 128 || + (category != null && + (typeof category !== "string" || category.length > 128)) + ) { + throw new Error("Invalid Workdir removal response"); + } + return { + working_directory_id, + disposition: enumField(record, "disposition", [ + "removed", + "retained", + "attention_required", + ]), + retryable: booleanField(record, "retryable", "Workdir removal response"), + failure_category: category as string | null | undefined, + }; +} + export function parseWorkingDirectoryDetailResponse( value: unknown, ): WorkingDirectoryDetailResponse { @@ -178,13 +213,26 @@ export function parseWorkingDirectorySummary( function parseWorkingDirectorySource(value: unknown): WorkingDirectorySource { const source = exactRecord( value, - new Set(["kind", "repository_key", "grant_id", "grant_permissions", "access", "content_path", "purpose"]), + new Set([ + "kind", + "repository_key", + "grant_id", + "grant_permissions", + "access", + "content_path", + "purpose", + ]), "Workdir source", ); const kind = stringField(source, "kind"); if (kind === "workspace_config") { - if (source.repository_key !== undefined || source.grant_id !== undefined || source.grant_permissions !== undefined) { - throw new Error("Logical config source must not contain repository or External authority"); + if ( + source.repository_key !== undefined || source.grant_id !== undefined || + source.grant_permissions !== undefined + ) { + throw new Error( + "Logical config source must not contain repository or External authority", + ); } const access = stringField(source, "access"); if (access !== "read_only" && access !== "read_write") { @@ -194,10 +242,20 @@ function parseWorkingDirectorySource(value: unknown): WorkingDirectorySource { if (content_path !== "/workspace-config") { throw new Error("Logical config must reference its WIP content entrance"); } - return { kind, access, content_path, purpose: stringField(source, "purpose") }; + return { + kind, + access, + content_path, + purpose: stringField(source, "purpose"), + }; } - if (source.access !== undefined || source.content_path !== undefined || source.purpose !== undefined) { - throw new Error("Filesystem Workdir source must not contain logical config metadata"); + if ( + source.access !== undefined || source.content_path !== undefined || + source.purpose !== undefined + ) { + throw new Error( + "Filesystem Workdir source must not contain logical config metadata", + ); } if (kind === "repository") { if ( diff --git a/web/workspace/src/lib/workspace/settings/workdir-removal.ts b/web/workspace/src/lib/workspace/settings/workdir-removal.ts new file mode 100644 index 000000000..4c7194e86 --- /dev/null +++ b/web/workspace/src/lib/workspace/settings/workdir-removal.ts @@ -0,0 +1,167 @@ +import type { WorkingDirectoryRemovalResponse } from "#lib/generated/workdir-api.ts"; +import type { + CleanupWorkdirCandidate, + WorkingDirectorySummary, +} from "#lib/workspace/sidebar/types.ts"; +import { workspaceApiPath } from "#lib/workspace/api/http.ts"; +import { parseWorkingDirectoryRemovalResponse } from "#lib/workspace/api/workdirs.ts"; + +// Only reviewed category codes become public copy. Never render a provider message, +// category verbatim, OS error, or host path (including for unknown future codes). +const causes: Readonly> = { + mount_present: + "A mounted filesystem remains inside this Workdir. Ask its resource owner to verify ownership and safely release it before retrying. Unknown or shared mounts must not be forcibly unmounted.", + mount_check_unavailable: + "The Runtime could not check for mounted filesystems. Restore mount inspection on the Runtime before retrying.", + permission_denied: + "Removal permission was denied. Ask the Workspace or Runtime administrator to verify the required access; do not bypass protection with a forced deletion.", + resource_busy: + "A resource is still in use. Ask its owner to stop using it and release it safely before retrying.", + storage_unavailable: + "Runtime storage is unavailable. Restore storage access and capacity before retrying.", + ownership_unknown: + "Resource ownership could not be verified. Ask the Runtime administrator to verify ownership before any resource is released or removed.", + changes_present: + "This Workdir has changes. Preserve or clean those changes before requesting removal again.", + changes_unknown: + "Current changes could not be checked. Restore access so the Runtime can verify change protection before removal.", + provider_cleanup_failed: + "The provider could not finish removal. Ask the Runtime administrator to inspect the correlated removal diagnostics and resolve the cause before retrying.", + provider_observation_unknown: + "The provider could not confirm the remaining Workdir. Restore provider access so the next removal can recheck its current contents.", + provider_cleanup_outcome_unknown: + "The provider has not confirmed deletion. Restore provider access and retry the same removal request to recheck the remaining target. The Workdir stays listed until deletion is confirmed.", + authority_invalid: + "The removal authority could not be verified. Ask the Workspace administrator to verify the Workdir target and owning Runtime before requesting removal again.", + provider_identity_changed: + "The provider target identity changed. Ask the Runtime administrator to verify the target; do not delete a replacement resource.", + runtime_unavailable: + "The Runtime is unavailable. Restore its connection before retrying removal.", + provider_unavailable: + "The Workdir provider is unavailable. Restore its connection before retrying removal.", + unsupported_target: + "This target does not support ordinary removal. Ask its resource owner for the supported cleanup operation.", + blocked_by_live_authority: + "This Workdir has an active attachment, reservation, or hold. Release it through its owning operation before requesting removal again.", + dirty_or_unknown: + "Current change protection prevents removal. Preserve changes or restore change inspection before requesting removal again.", + authority_changed: + "Removal authority or current usage changed. Refresh the inventory and resolve current attachments, reservations, or holds before requesting removal again.", +}; +const unknownCause = + "Removal could not be confirmed. Check the Runtime's correlated removal diagnostics and resolve the cause before requesting removal again. The Workdir remains listed until deletion is confirmed."; + +export function removalCause(category: string | null | undefined): string { + return category != null && Object.hasOwn(causes, category) + ? causes[category] + : unknownCause; +} + +export function canRemoveWorkdir( + workdir: WorkingDirectorySummary, + candidate: CleanupWorkdirCandidate | undefined, +): boolean { + return Boolean( + candidate && !candidate.blocking_reason && !candidate.running_linked && + !candidate.pinned_linked && + (candidate.action === "workdir_clean_cleanup" || + candidate.action === "workdir_record_delete") && + workdir.cleanliness !== "dirty" && !workdir.occupied_by, + ); +} + +export function removalGuard( + workdir: WorkingDirectorySummary, + candidate: CleanupWorkdirCandidate | undefined, +): string { + if ( + workdir.occupied_by || candidate?.running_linked || + candidate?.pinned_linked || candidate?.blocking_reason + ) { + return "An attachment, reservation, or hold protects this Workdir. Release it through its owning operation, then refresh."; + } + if ( + workdir.cleanliness === "dirty" || + candidate?.action === "workdir_dirty_discard" + ) { + return "Changes are protected. Preserve or clean them, then refresh before deletion."; + } + return "Removal eligibility is unavailable. Refresh before deletion."; +} + +export class WorkdirRemovalError extends Error { + constructor(readonly category?: string) { + super(removalCause(category)); + } +} + +// Read only a small public DTO. Discard HTTP error bodies without interpreting or +// displaying them, and drain small bodies rather than aborting completed requests. +async function removalBody(response: Response): Promise { + const reader = response.body?.getReader(); + if (!reader) return ""; + const chunks: Uint8Array[] = []; + let bytes = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + bytes += value.byteLength; + if (bytes > 4096) { + await reader.cancel(); + throw new WorkdirRemovalError(); + } + chunks.push(value); + } + const body = new Uint8Array(bytes); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder().decode(body); + } finally { + reader.releaseLock(); + } +} + +export async function removeWorkdir( + fetchFn: typeof fetch, + workspaceId: string, + runtimeId: string, + workdirId: string, +): Promise { + const response = await fetchFn( + workspaceApiPath( + workspaceId, + `/runtimes/${encodeURIComponent(runtimeId)}/working-directories/${ + encodeURIComponent(workdirId) + }`, + ), + { + method: "DELETE", + headers: { "content-type": "application/json" }, + // An ordinary repeated request uses the existing durable Backend removal authority. + body: JSON.stringify({ reason: "User requested Workdir deletion" }), + }, + ); + if (!response.ok) { + await removalBody(response).catch(() => undefined); + throw new WorkdirRemovalError( + response.status === 403 + ? "permission_denied" + : response.status === 409 + ? "authority_changed" + : response.status === 503 + ? "provider_unavailable" + : undefined, + ); + } + const result = parseWorkingDirectoryRemovalResponse( + JSON.parse(await removalBody(response)), + ); + if (result.working_directory_id !== workdirId) { + throw new WorkdirRemovalError(); + } + return result; +} diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/workdirs/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/workdirs/+page.svelte index cb44d6047..6275f9d94 100644 --- a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/workdirs/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/workdirs/+page.svelte @@ -1,102 +1,89 @@