fix: integrate safe Workdir removal recovery

Merge approved T-730 source b1dc8f9f into develop. Resolve the test-registration conflict by preserving both Drive and Workdir-removal tests. Integration root check, Runtime/Server/Worker suites, real Web Drive adapter, generated Workdir contract, Web check/unit/components/build/browser, fmt and diff checks passed.
This commit is contained in:
2026-10-09 17:43:19 +09:00
24 changed files with 4120 additions and 355 deletions
Generated
+1
View File
@@ -6575,6 +6575,7 @@ dependencies = [
"flow",
"fs4",
"futures",
"libc",
"manifest",
"protocol",
"reqwest",
+1
View File
@@ -31,6 +31,7 @@ decodal.workspace = true
flow = { path = "../flow" }
fs4 = { workspace = true, optional = true }
manifest.workspace = true
libc.workspace = true
protocol.workspace = true
runtime-api.workspace = true
serde = { workspace = true, features = ["derive"] }
+50
View File
@@ -2486,6 +2486,8 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode {
if matches!(
diagnostic.code.as_str(),
"repository_ref_provider_unavailable"
| "working_directory_cleanup_mount_check_unavailable"
| "working_directory_cleanup_storage_unavailable"
| "repository_ref_provider_timeout"
| "repository_access_provider_unavailable"
) =>
@@ -2504,6 +2506,23 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode {
{
StatusCode::FORBIDDEN
}
RuntimeError::WorkingDirectory(diagnostic)
if matches!(
diagnostic.code.as_str(),
"working_directory_cleanup_mount_present"
| "working_directory_cleanup_resource_busy"
| "working_directory_cleanup_ownership_unknown"
| "working_directory_cleanup_changes_present"
| "working_directory_cleanup_changes_unknown"
) =>
{
StatusCode::CONFLICT
}
RuntimeError::WorkingDirectory(diagnostic)
if diagnostic.code == "working_directory_cleanup_permission_denied" =>
{
StatusCode::FORBIDDEN
}
RuntimeError::WorkingDirectory(diagnostic)
if diagnostic.code == "repository_ref_not_found" =>
{
@@ -4322,6 +4341,37 @@ mod tests {
assert!(!error.to_string().contains('/'));
}
#[tokio::test]
async fn cleanup_cause_codes_survive_http_error_serialization() {
let cases = [
("mount_present", StatusCode::CONFLICT),
("mount_check_unavailable", StatusCode::SERVICE_UNAVAILABLE),
("permission_denied", StatusCode::FORBIDDEN),
("resource_busy", StatusCode::CONFLICT),
("storage_unavailable", StatusCode::SERVICE_UNAVAILABLE),
("ownership_unknown", StatusCode::CONFLICT),
("changes_present", StatusCode::CONFLICT),
("changes_unknown", StatusCode::CONFLICT),
];
for (cause, status) in cases {
let code = format!("working_directory_cleanup_{cause}");
let diagnostic = crate::working_directory::WorkingDirectoryDiagnostic::rejected(
&code,
"Resolve the current blocker and retry removal.",
);
let response =
RuntimeHttpRestError::runtime(RuntimeError::WorkingDirectory(diagnostic))
.into_response();
assert_eq!(response.status(), status);
let body = axum::body::to_bytes(response.into_body(), 2048)
.await
.unwrap();
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(json["error"]["code"], code, "{json}");
assert!(json["error"]["message"].as_str().unwrap().contains("retry"));
}
}
#[test]
fn workdir_runtime_errors_preserve_diagnostic_code() {
let cases = [
+46 -6
View File
@@ -819,7 +819,13 @@ impl Runtime {
&self,
working_directory_id: &str,
) -> Result<CatalogWorkingDirectoryStatus, RuntimeError> {
let backend = {
// Hold occupancy authority through the side effect, preventing an
// attachment from entering between the check and physical removal.
// This synchronous provider call must not reenter Runtime. The Git
// materializer does filesystem/Git work only. The global guard also
// delays unrelated state operations during hashing/unlink; releasing it
// would require equivalent per-workdir exclusion on ALL attachment paths,
// not just moving the occupied check before the provider call.
let state = self.lock()?;
state.ensure_running()?;
if let Some(worker_id) = state.worker_id_for_workdir(working_directory_id) {
@@ -827,15 +833,16 @@ impl Runtime {
"working directory {working_directory_id} is assigned to worker {worker_id}"
)));
}
state.execution_backend.clone().ok_or_else(|| {
let backend = state.execution_backend.clone().ok_or_else(|| {
RuntimeError::ExecutionBackendUnavailable {
message: "working directory cleanup requires an execution backend".to_string(),
}
})?
};
backend
})?;
let result = backend
.cleanup_working_directory(working_directory_id)
.map_err(RuntimeError::from)
.map_err(RuntimeError::from);
drop(state);
result
}
fn annotate_working_directory_statuses(
@@ -7665,6 +7672,7 @@ mod tests {
repository_accesses: Mutex<Vec<WorkingDirectoryRepositoryAccessRequest>>,
repository_access_available: AtomicBool,
working_directory_requests: Mutex<Vec<WorkingDirectoryRequest>>,
cleanup_occupancy_probe: Mutex<Option<Arc<Mutex<RuntimeState>>>>,
preserve_submission_acknowledgement_id: AtomicBool,
#[cfg(feature = "ws-server")]
snapshots: Mutex<BTreeMap<WorkerId, protocol::Event>>,
@@ -7705,6 +7713,22 @@ mod tests {
}
impl WorkerExecutionBackend for TestExecutionBackend {
fn cleanup_working_directory(
&self,
_id: &str,
) -> Result<CatalogWorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
if let Some(state) = self.cleanup_occupancy_probe.lock().unwrap().as_ref() {
assert!(
matches!(state.try_lock(), Err(std::sync::TryLockError::WouldBlock)),
"occupancy must remain exclusive during physical cleanup"
);
}
Err(WorkingDirectoryDiagnostic::rejected(
"working_directory_cleanup_resource_busy",
"Retry removal after releasing the resource",
))
}
fn backend_id(&self) -> &str {
"test-execution-backend"
}
@@ -8043,6 +8067,22 @@ mod tests {
.block_on(subscription.recv())
}
#[test]
fn workdir_removal_keeps_occupancy_exclusive_during_provider_side_effect() {
let (runtime, backend) = runtime_and_backend();
*backend.cleanup_occupancy_probe.lock().unwrap() = Some(runtime.inner.clone());
let error = runtime
.cleanup_working_directory("workdir-test")
.unwrap_err();
assert!(
matches!(error, RuntimeError::WorkingDirectory(ref diagnostic)
if diagnostic.code == "working_directory_cleanup_resource_busy")
);
// The lock is released on failure, permitting normal attachment/retry.
assert!(runtime.inner.try_lock().is_ok());
*backend.cleanup_occupancy_probe.lock().unwrap() = None;
}
#[test]
fn unknown_typed_intent_is_rejected_before_backend_dispatch() {
let (runtime, backend) = runtime_and_backend();
+168 -19
View File
@@ -19,7 +19,6 @@ use crate::catalog::{
CreateWorkerRequest, ProfileSourceArchiveSource, RepositoryRefObservation,
RepositoryRefObservationRequest, WorkingDirectoryAttachmentStatus,
WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest, WorkingDirectoryStatus,
WorkingDirectoryStatusKind,
};
use crate::config_bundle::{ConfigBundle, workspace_config_etag};
#[cfg(feature = "ws-server")]
@@ -3773,28 +3772,24 @@ where
)
})?;
for previous in &request.previous_workdir_attachments {
let current = materializer
.working_directory_status(
let relative_cwd = request
.request
.workdir_attachments
.iter()
.find(|claim| claim.alias == previous.alias)
.and_then(|claim| claim.relative_cwd.as_deref());
// Recovery attachments remain CleanupPending across restart.
// Validate current bindability/identity, not the old state name.
// Preflight is read-only; actual restore binds again below.
materializer.preflight_bind_working_directory(
&previous.working_directory.summary.working_directory_id,
)
.map_err(|message| {
relative_cwd,
).map_err(|message| {
WorkerExecutionResult::rejected(
WorkerExecutionOperation::Restore,
format!(
"Persisted Worker Workdir attachment `{}` is unavailable: {message}",
previous.alias
),
format!("Persisted Worker Workdir attachment `{}` is unavailable: {message}", previous.alias),
)
})?;
if current.summary.status != WorkingDirectoryStatusKind::Active {
return Err(WorkerExecutionResult::rejected(
WorkerExecutionOperation::Restore,
format!(
"Persisted Worker Workdir attachment `{}` is not active",
previous.alias
),
));
}
}
}
if request.previous_workdir_attachments.is_empty()
@@ -4637,7 +4632,7 @@ mod tests {
ConfigBundleRef, CreateWorkerRequest, LogicalWorkdirAttachment, MaterializerKind,
ProfileSelector, RepositorySelector, WorkingDirectoryAttachmentClaim,
WorkingDirectoryAttachmentRequest, WorkingDirectoryRepository, WorkingDirectoryRequest,
WorkspaceApiRef,
WorkingDirectoryStatusKind, WorkspaceApiRef,
};
use crate::execution::WorkerExecutionContext;
use crate::identity::WorkerId;
@@ -8919,6 +8914,160 @@ mod tests {
);
}
#[cfg(target_os = "linux")]
fn pending_owner_restore_fixture() -> (
tempfile::TempDir,
WorkerRuntimeExecutionBackend<MockFactory>,
WorkerExecutionRestoreRequest,
PathBuf,
Arc<Mutex<Vec<PathBuf>>>,
) {
use crate::working_directory::{WorkingDirectoryMaterializer, fail_workdir_cleanup_once};
let root = tempfile::tempdir().unwrap();
let repo = create_clean_repo();
let workdirs = root.path().join("workdirs");
let materializer = RuntimeGitMaterializer::new(&workdirs);
let binding = materializer
.create(&working_directory_request(repo.path()))
.unwrap();
let id = binding.working_directory.id.clone();
fail_workdir_cleanup_once(&materializer, &id);
// The cleanup owner has legitimately reattached to preserve these changes.
fs::write(binding.root.join("README.md"), "owner recovery in progress").unwrap();
let observed = Arc::new(Mutex::new(Vec::new()));
let factory = || MockFactory {
client: MockClient::new(simple_text_events()),
runtime_base: root.path().join("runtime"),
cwd: root.path().to_path_buf(),
store_dir: root.path().join("sessions"),
worker_metadata_dir: root.path().join("workers"),
observed_cwds: observed.clone(),
observed_workspace_clients: Arc::new(Mutex::new(Vec::new())),
};
let backend = Arc::new(
WorkerRuntimeExecutionBackend::new(factory())
.unwrap()
.with_working_directory_materializer(materializer),
);
let runtime =
EmbeddedRuntime::with_execution_backend(RuntimeOptions::default(), backend.clone())
.unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let mut request = create_request("cleanup-owner");
request.workdir_attachments = vec![WorkingDirectoryAttachmentClaim {
alias: WorkdirAttachmentAlias::new("checkout").unwrap(),
working_directory_id: id,
relative_cwd: None,
capabilities: WorkdirSessionCapabilities::ALL,
}];
let detail = runtime.create_worker(request.clone()).unwrap();
runtime.stop_worker(&detail.worker_ref, None).unwrap();
let restore = WorkerExecutionRestoreRequest {
operation_id: crate::execution::WorkerLifecycleOperationId::new(),
worker_ref: detail.worker_ref.clone(),
request,
workspace_scope: None,
context: test_execution_context(detail.worker_ref),
previous_workdir_attachments: detail.workdir_attachments,
logical_workdir_attachments: Vec::new(),
workdir_attachments: BTreeMap::new(),
config_bundle: None,
};
drop(runtime);
drop(backend);
// A fresh backend/materializer loads the durable pending identity evidence.
let restarted = WorkerRuntimeExecutionBackend::new(factory())
.unwrap()
.with_working_directory_materializer(RuntimeGitMaterializer::new(&workdirs));
(root, restarted, restore, binding.root, observed)
}
#[cfg(target_os = "linux")]
#[test]
fn cleanup_pending_owner_restores_after_restart_with_validated_dirty_checkout() {
let (root, backend, request, checkout, observed) = pending_owner_restore_fixture();
let before = persisted_files(&root.path().join("workdirs"));
backend.preflight_restore(&request).unwrap();
assert_eq!(
persisted_files(&root.path().join("workdirs")),
before,
"preflight must not rewrite removal or Git authority"
);
let restored = match backend.restore_worker(request.clone()) {
WorkerExecutionSpawnResult::Connected {
workdir_attachments,
..
} => workdir_attachments,
other => panic!("cleanup-owner restore failed: {other:?}"),
};
assert_eq!(restored.len(), 1);
assert_eq!(
restored[0].working_directory.summary.status,
WorkingDirectoryStatusKind::CleanupPending
);
assert_eq!(
restored[0].working_directory.summary.cleanliness.as_deref(),
Some("dirty")
);
assert_eq!(
observed.lock().unwrap().as_slice(),
&[checkout.clone(), checkout.clone()]
);
assert_eq!(
fs::read_to_string(checkout.join("README.md")).unwrap(),
"owner recovery in progress"
);
assert!(backend.stop_worker(&request.worker_ref).is_accepted());
}
#[cfg(target_os = "linux")]
#[test]
fn cleanup_pending_owner_restore_rejects_incomplete_replaced_and_raced_checkouts() {
for damage in ["missing_git", "replacement", "missing_after_preflight"] {
let (root, backend, request, checkout, observed) = pending_owner_restore_fixture();
if damage == "missing_after_preflight" {
backend.preflight_restore(&request).unwrap();
}
if damage == "replacement" {
fs::rename(&checkout, root.path().join("original-checkout")).unwrap();
fs::create_dir(&checkout).unwrap();
fs::create_dir(checkout.join(".git")).unwrap();
} else {
fs::remove_dir_all(checkout.join(".git")).unwrap();
}
let before = persisted_files(root.path());
let error = backend.preflight_restore(&request).unwrap_err();
assert_eq!(
error.outcome,
crate::execution::WorkerExecutionOutcome::Rejected,
"{damage}"
);
assert!(
error
.message_or_default()
.contains("working_directory_cleanup_"),
"{error:?}"
);
assert!(
matches!(
backend.restore_worker(request),
WorkerExecutionSpawnResult::Rejected(_)
),
"{damage}"
);
assert_eq!(
observed.lock().unwrap().len(),
1,
"must not start a Controller on invalid checkout: {damage}"
);
assert_eq!(
persisted_files(root.path()),
before,
"must preserve damaged/replaced checkout: {damage}"
);
}
}
#[test]
fn worker_spawn_receives_materialized_workspace_cwd_instead_of_source_repo() {
let client = MockClient::new(simple_text_events());
+122 -92
View File
@@ -22,6 +22,11 @@ use std::thread::JoinHandle;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use workdir::WorkdirSessionResource;
mod cleanup;
#[cfg(all(test, target_os = "linux"))]
pub(crate) use cleanup::tests::fail_once as fail_workdir_cleanup_once;
const CHECKOUT_DIR: &str = "checkout";
const MATERIALIZATION_RECORD: &str = "materialization.json";
const REPOSITORY_ACCESS_DIR: &str = ".repository-access";
@@ -127,7 +132,11 @@ impl WorkingDirectoryBinding {
working_directory.status = WorkingDirectoryStatusKind::Corrupted;
}
let mut summary = working_directory.status_summary();
summary.cleanliness = if summary.status == WorkingDirectoryStatusKind::Active {
summary.cleanliness = if matches!(
summary.status,
WorkingDirectoryStatusKind::Active | WorkingDirectoryStatusKind::CleanupPending
) && binding_paths_are_available(self)
{
let (current_selector, current_ref, current_tree) = binding_current_revision(self);
summary.current_selector = current_selector;
summary.current_ref = current_ref;
@@ -201,6 +210,24 @@ pub trait WorkingDirectoryMaterializer: Send + Sync + 'static {
relative_cwd: Option<&str>,
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic>;
/// Read-only bind validation: do not consume Repository credentials or open
/// session resources during restore preflight. Providers must opt in with
/// current usable-checkout validation to support non-active attachments.
fn preflight_bind_working_directory(
&self,
working_directory_id: &str,
_relative_cwd: Option<&str>,
) -> Result<(), WorkingDirectoryDiagnostic> {
let current = self.working_directory_status(working_directory_id)?;
if current.summary.status != WorkingDirectoryStatusKind::Active {
return Err(WorkingDirectoryDiagnostic::new(
"working_directory_not_active",
"Persisted Workdir is not active and its provider cannot validate recovery binding",
));
}
Ok(())
}
fn list_working_directories(
&self,
) -> Result<Vec<WorkingDirectoryStatus>, WorkingDirectoryDiagnostic>;
@@ -627,6 +654,29 @@ impl RuntimeGitMaterializer {
})
}
fn checked_binding(
&self,
working_directory_id: &str,
relative_cwd: Option<&str>,
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
validate_working_directory_id(working_directory_id)?;
let binding = self.read_binding(working_directory_id)?;
if !matches!(
binding.working_directory.status,
WorkingDirectoryStatusKind::Active | WorkingDirectoryStatusKind::CleanupPending
) {
return Err(WorkingDirectoryDiagnostic::new(
"working_directory_not_active",
"working directory working_directory is not active",
));
}
if binding.working_directory.status == WorkingDirectoryStatusKind::CleanupPending {
cleanup::validate_recovery_binding(self, working_directory_id, &binding)?;
}
let cwd = validate_relative_cwd(binding.root(), relative_cwd)?;
Ok(WorkingDirectoryBinding { cwd, ..binding })
}
fn read_binding(
&self,
working_directory_id: &str,
@@ -1137,17 +1187,17 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
working_directory_id: &str,
relative_cwd: Option<&str>,
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
validate_working_directory_id(working_directory_id)?;
let binding = self.read_binding(working_directory_id)?;
if binding.working_directory.status != WorkingDirectoryStatusKind::Active {
return Err(WorkingDirectoryDiagnostic::new(
"working_directory_not_active",
"working directory working_directory is not active",
));
let binding = self.checked_binding(working_directory_id, relative_cwd)?;
self.bind_repository_access(working_directory_id, binding)
}
let binding = self.bind_repository_access(working_directory_id, binding)?;
let cwd = validate_relative_cwd(binding.root(), relative_cwd)?;
Ok(WorkingDirectoryBinding { cwd, ..binding })
fn preflight_bind_working_directory(
&self,
working_directory_id: &str,
relative_cwd: Option<&str>,
) -> Result<(), WorkingDirectoryDiagnostic> {
self.checked_binding(working_directory_id, relative_cwd)
.map(|_| ())
}
fn list_working_directories(
@@ -1178,6 +1228,10 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
{
continue;
}
if let Some(status) = cleanup::remaining_status(self, &working_directory_id) {
statuses.push(status);
continue;
}
match self.read_binding(&working_directory_id) {
Ok(binding) => statuses.push(binding.status()),
Err(_) => statuses.push(self.corrupted_status(&working_directory_id)),
@@ -1197,12 +1251,25 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
) -> Result<WorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
validate_working_directory_id(working_directory_id)?;
let working_directory_root = self.working_directory_root(working_directory_id);
if !working_directory_root.exists() {
match fs::symlink_metadata(&working_directory_root) {
Ok(_) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
return Err(WorkingDirectoryDiagnostic::new(
"working_directory_not_found",
"working directory working_directory was not found",
"Runtime working directory was not found",
));
}
Err(error) => {
return Err(cleanup::os_failure(
working_directory_id,
"lookup_cleanup_root",
error,
));
}
}
if let Some(status) = cleanup::remaining_status(self, working_directory_id) {
return Ok(status);
}
match self.read_binding(working_directory_id) {
Ok(binding) => Ok(binding.status()),
Err(_) => Ok(self.corrupted_status(working_directory_id)),
@@ -1214,80 +1281,13 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
working_directory_id: &str,
) -> Result<WorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
validate_working_directory_id(working_directory_id)?;
let status = self.working_directory_status(working_directory_id)?;
if status.summary.status == WorkingDirectoryStatusKind::Corrupted {
let working_directory_root = self.working_directory_root(working_directory_id);
if working_directory_root.exists() {
fs::remove_dir_all(&working_directory_root).map_err(|_| {
WorkingDirectoryDiagnostic::new(
"working_directory_corrupted_cleanup_failed",
"failed to remove corrupted working directory; backend-private path details were omitted",
)
})?;
}
let mut summary = status.summary;
summary.status = WorkingDirectoryStatusKind::NotFound;
return Ok(WorkingDirectoryStatus { summary });
}
let binding = self.read_binding(working_directory_id)?;
self.cleanup(&binding)?;
let mut summary = binding.working_directory.status_summary();
summary.status = WorkingDirectoryStatusKind::NotFound;
summary.cleanliness = Some("unknown".to_string());
if binding.working_directory_root.exists() {
fs::remove_dir_all(&binding.working_directory_root).map_err(|_| {
WorkingDirectoryDiagnostic::new(
"working_directory_record_cleanup_failed",
"failed to remove working directory record; backend-private path details were omitted",
)
})?;
}
Ok(WorkingDirectoryStatus { summary })
cleanup::remove(self, working_directory_id)
}
fn cleanup(&self, binding: &WorkingDirectoryBinding) -> Result<(), WorkingDirectoryDiagnostic> {
let mut working_directory = binding.working_directory.clone();
let working_directory_root = binding.working_directory_root.canonicalize().map_err(|_| {
WorkingDirectoryDiagnostic::new(
"working_directory_cleanup_target_invalid",
"working directory working directory root is unavailable; backend-private path details were omitted",
)
})?;
let root = binding.root.canonicalize().map_err(|_| {
WorkingDirectoryDiagnostic::new(
"working_directory_cleanup_target_invalid",
"working directory root is unavailable; backend-private path details were omitted",
)
})?;
if !root.starts_with(&working_directory_root) {
return Err(WorkingDirectoryDiagnostic::new(
"working_directory_cleanup_escape_rejected",
"working directory cleanup target is outside the working directory root",
));
}
let remove_result = fs::remove_dir_all(&working_directory_root).map_err(|_| {
WorkingDirectoryDiagnostic::new(
"working_directory_cleanup_failed",
"failed to remove working directory; backend-private path details were omitted",
)
});
if remove_result.is_err() {
working_directory.status = WorkingDirectoryStatusKind::CleanupPending;
let updated = WorkingDirectoryBinding {
working_directory,
root: binding.root.clone(),
cwd: binding.cwd.clone(),
working_directory_root: binding.working_directory_root.clone(),
command_environment: BTreeMap::new(),
session_resources: Vec::new(),
};
let _ = self.write_record(&updated);
} else {
let _ = self
.repository_access
.remove_pending(&binding.working_directory.id);
}
remove_result
// Reload current authority rather than trusting a stale in-memory binding.
self.cleanup_working_directory(&binding.working_directory.id)
.map(|_| ())
}
}
@@ -2826,6 +2826,7 @@ where
I: IntoIterator<Item = &'a str>,
{
let output = Command::new("git")
.env("GIT_OPTIONAL_LOCKS", "0")
.arg("-C")
.arg(repository_path)
.args(args)
@@ -3094,7 +3095,7 @@ mod tests {
assert!(scheduler_inner.upgrade().is_none());
}
fn git(path: &Path, args: &[&str]) {
pub(super) fn git(path: &Path, args: &[&str]) {
let status = Command::new("git")
.arg("-C")
.arg(path)
@@ -3104,7 +3105,7 @@ mod tests {
assert!(status.success(), "git {:?} failed", args);
}
fn create_clean_repo() -> tempfile::TempDir {
pub(super) fn create_clean_repo() -> tempfile::TempDir {
let dir = tempfile::tempdir().unwrap();
git(dir.path(), &["init"]);
git(
@@ -3118,7 +3119,7 @@ mod tests {
dir
}
fn request(repo: &Path) -> WorkingDirectoryRequest {
pub(super) fn request(repo: &Path) -> WorkingDirectoryRequest {
WorkingDirectoryRequest {
repository: WorkingDirectoryRepository {
id: "repo-main".to_string(),
@@ -3678,6 +3679,8 @@ mod tests {
.evidence
.host_trust_revision = Some(1);
materializer.write_record(&ssh_backed_binding).unwrap();
#[cfg(target_os = "linux")]
fail_workdir_cleanup_once(&materializer, &id);
let mut session_materialization = initial_materialization.clone();
let session_delivery_expiry = repository_access_now_epoch_seconds() + 1;
session_materialization
@@ -3691,6 +3694,17 @@ mod tests {
materialization: session_materialization,
})
.unwrap();
// Restore preflight must validate without consuming this one-shot SSH
// delivery or creating agent/session resources needed by actual bind.
let record_before =
fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap();
materializer
.preflight_bind_working_directory(&id, None)
.unwrap();
assert_eq!(
fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(),
record_before
);
assert_eq!(materializer.list_working_directories().unwrap().len(), 1);
assert_eq!(
fs::read_dir(runtime_root.path().join(".repository-agents"))
@@ -3916,6 +3930,22 @@ mod tests {
restored.bind_working_directory(&id, None).unwrap_err().code,
"working_directory_remote_repository_access_required"
);
#[cfg(target_os = "linux")]
{
// Real SSH reauthorization updated access evidence repeatedly, but
// did not replace this clean materialization or require manual recovery.
assert_eq!(
restored
.working_directory_status(&id)
.unwrap()
.summary
.cleanliness
.as_deref(),
Some("clean")
);
restored.cleanup_working_directory(&id).unwrap();
assert!(!restored.working_directory_root(&id).exists());
}
}
#[test]
@@ -4393,7 +4423,7 @@ mod tests {
}
#[test]
fn corrupted_working_directory_record_can_be_removed() {
fn corrupted_working_directory_without_removal_authority_is_retained() {
let runtime_root = tempfile::tempdir().unwrap();
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
let working_directory_id = "workdir-corrupted";
@@ -4406,11 +4436,11 @@ mod tests {
.unwrap();
assert_eq!(status.summary.status, WorkingDirectoryStatusKind::Corrupted);
let removed = materializer
let error = materializer
.cleanup_working_directory(working_directory_id)
.unwrap();
assert_eq!(removed.summary.status, WorkingDirectoryStatusKind::NotFound);
assert!(!root.exists());
.unwrap_err();
assert_eq!(error.code, "working_directory_cleanup_ownership_unknown");
assert!(root.exists());
}
#[test]
File diff suppressed because it is too large Load Diff
@@ -734,8 +734,14 @@ impl WorkspaceHttpWorkdirBackend {
))?;
workdir_output(
format!(
"Workdir {workdir_id} removal disposition: {:?}",
response.disposition
"Workdir {workdir_id} removal disposition: {:?}. {}{}",
response.disposition,
removal_guidance(response.failure_category.as_deref()),
if response.retryable {
" After resolving the cause, repeat this WorkdirDelete with the same id and reason; current safety conditions will be rechecked."
} else {
""
}
),
&response,
)
@@ -920,6 +926,48 @@ fn decode_error(error: serde_json::Error) -> ToolError {
ToolError::ExecutionFailed(format!("decode Workspace Workdir response: {error}"))
}
fn removal_guidance(category: Option<&str>) -> &'static str {
match category {
None => "",
Some("mount_present") => {
"A mount remains inside the Workdir. Confirm its owner and usage, and have that owner safely release it. Do not recursively delete mounted data or implicitly unmount external or unknown resources."
}
Some("mount_check_unavailable") => "Restore Runtime mount inspection before deletion.",
Some("changes_present" | "dirty_or_unknown") => {
"Current changes are protected; preserve or resolve them before deletion."
}
Some("changes_unknown") => {
"Restore checkout inspection or preserve remaining content so current changes can be verified."
}
Some(
"ownership_unknown"
| "provider_identity_changed"
| "authority_changed"
| "authority_invalid",
) => {
"Confirm target identity and ownership with its owner; never remove a replacement or unknown resource."
}
Some("blocked_by_live_authority") => {
"Release current attachments, reservations and holds through their owning authority."
}
Some("permission_denied" | "resource_busy" | "storage_unavailable") => {
"Have the resource owner resolve access, usage or storage safely; do not force-clean or change shared resource permissions."
}
Some("runtime_unavailable" | "provider_unavailable") => {
"Restore the Runtime connection; the registry remains until deletion is confirmed."
}
Some("provider_observation_unknown" | "provider_cleanup_outcome_unknown") => {
"Deletion is not confirmed. Re-observe with normal removal after restoring provider access; the registry is retained."
}
Some("unsupported_target") => {
"Ask the owner for the provider's supported cleanup operation."
}
Some(_) => {
"Inspect correlated Runtime cleanup diagnostics with the owning operator and resolve the current cause."
}
}
}
fn workdir_output<T: Serialize>(summary: String, value: &T) -> Result<ToolOutput, ToolError> {
Ok(ToolOutput {
summary,
@@ -1218,6 +1266,45 @@ mod tests {
})
}
#[test]
fn ordinary_workdir_delete_reports_safe_cause_and_exact_retry_guidance() {
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
"working_directory_id": "wd-retry",
"disposition": "attention_required",
"retryable": true,
"failure_category": "mount_present"
}))]));
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
let output = backend
.delete(WorkdirDeleteInput {
working_directory_id: "wd-retry".into(),
reason: "remove unused Workdir".into(),
})
.unwrap();
assert!(output.summary.contains("Confirm its owner"));
assert!(
output
.summary
.contains("Do not recursively delete mounted data")
);
assert!(output.summary.contains("same id and reason"));
assert!(output.summary.len() <= 600);
let content: serde_json::Value =
serde_json::from_str(output.content.as_deref().unwrap()).unwrap();
assert_eq!(content["failure_category"], "mount_present");
assert_eq!(content["retryable"], true);
let requests = client.requests();
assert_eq!(requests[0].method, WorkspaceRequestMethod::Delete);
assert_eq!(
serde_json::from_str::<serde_json::Value>(requests[0].body.as_deref().unwrap())
.unwrap()["reason"],
"remove unused Workdir"
);
let unknown = removal_guidance(Some("secret /private/provider/path"));
assert!(!unknown.contains("/private"));
assert!(!unknown.contains("secret"));
}
#[test]
fn workspace_request_failure_is_not_reported_as_session_unavailable() {
let error = workspace_workdir_error(WorkspaceClientError::Request(
+547 -41
View File
@@ -11927,10 +11927,6 @@ fn cleanup_workdir_cleanliness_from_runtime(value: Option<&str>) -> CleanupWorkd
.unwrap_or(CleanupWorkdirCleanliness::Unknown)
}
fn cleanup_workdir_cleanliness_is_clean(value: CleanupWorkdirCleanliness) -> bool {
matches!(value, CleanupWorkdirCleanliness::Clean)
}
#[derive(Debug, Deserialize)]
struct LogQuery {
limit: Option<usize>,
@@ -17787,9 +17783,15 @@ async fn scoped_attach_current_worker_workdir(
code: "working_directory_not_found".to_string(),
message: format!("unknown Workdir `{workdir_id}`"),
})?;
if workdir.materialization_status != "present" {
// A failed removal must not prevent its owner from attaching a still usable
// repository checkout to resolve current blockers. Runtime binding validates
// the actual checkout; durable pending-removal reservations still fence this
// mutation. Other sources and missing/corrupted materializations stay denied.
let repository_retry = workdir.materialization_status == "pending"
&& matches!(workdir.source, WorkdirRegistrySource::Repository { .. });
if workdir.materialization_status != "present" && !repository_retry {
return Err(Error::WorkdirAttachmentConflict(format!(
"Workdir `{workdir_id}` is not active ({})",
"Workdir `{workdir_id}` is not available for attachment ({})",
workdir.materialization_status
))
.into());
@@ -24527,18 +24529,88 @@ fn runtime_reports_workdir_not_found(result: &crate::hosts::RuntimeWorkingDirect
fn classify_workdir_provider_error(error: &RuntimeRegistryError) -> (&'static str, bool) {
match error {
RuntimeRegistryError::UnknownRuntime(_) => ("runtime_unavailable", true),
RuntimeRegistryError::RuntimeOperationFailed { code, .. }
if code == "working_directory_unsupported" =>
{
("unsupported_target", false)
RuntimeRegistryError::RuntimeOperationFailed { code, .. } => {
let category =
workdir_cleanup_diagnostic_category(code).unwrap_or("provider_unavailable");
(category, category != "unsupported_target")
}
RuntimeRegistryError::RuntimeOperationFailed { .. } => ("provider_unavailable", true),
RuntimeRegistryError::InvalidIdentifier { .. }
| RuntimeRegistryError::UnknownHost(_)
| RuntimeRegistryError::UnknownWorker { .. } => ("authority_invalid", false),
}
}
// Only bounded classifications cross the public removal boundary. Provider
// messages may contain OS paths or credentials and are never persisted here.
fn workdir_cleanup_failure_category(
result: &crate::hosts::RuntimeWorkingDirectoryResult,
) -> &'static str {
result
.diagnostics
.iter()
.find_map(|diagnostic| workdir_cleanup_diagnostic_category(&diagnostic.code))
.unwrap_or("provider_cleanup_failed")
}
fn workdir_cleanup_diagnostic_category(code: &str) -> Option<&'static str> {
Some(match code {
"working_directory_unsupported" => "unsupported_target",
"working_directory_cleanup_mount_present" => "mount_present",
"working_directory_cleanup_mount_check_unavailable" => "mount_check_unavailable",
"working_directory_cleanup_permission_denied" => "permission_denied",
"working_directory_cleanup_resource_busy" => "resource_busy",
"working_directory_cleanup_storage_unavailable" => "storage_unavailable",
"working_directory_cleanup_ownership_unknown" => "ownership_unknown",
"working_directory_cleanup_changes_present" => "changes_present",
"working_directory_cleanup_changes_unknown" => "changes_unknown",
"working_directory_cleanup_target_invalid"
| "working_directory_cleanup_escape_rejected" => "ownership_unknown",
_ => return None,
})
}
fn workdir_removal_failure_guidance(category: &str) -> &'static str {
match category {
"mount_present" => {
"A mount remains inside the Workdir. Confirm its owner and usage, have that owner safely release it, then retry deletion. Yoi does not unmount external resources."
}
"mount_check_unavailable" => {
"Runtime cannot inspect mount boundaries. Restore mount inspection before retrying deletion."
}
"permission_denied" => {
"Runtime lacks permission to remove the Workdir. Have its owner resolve access safely, then retry; do not force-clean or change shared resource permissions."
}
"resource_busy" => {
"A resource is busy. Have its owner stop or release it safely, then retry deletion."
}
"storage_unavailable" => {
"Runtime storage is unavailable. Restore storage access, then retry deletion."
}
"ownership_unknown" | "provider_identity_changed" | "authority_changed" => {
"Workdir identity or ownership could not be confirmed. Verify the target with its owner before retrying."
}
"changes_present" | "dirty_or_unknown" => {
"Current Workdir changes are protected. Preserve or resolve them before retrying deletion."
}
"changes_unknown" => {
"Runtime cannot verify remaining Workdir changes. Restore repository inspection or preserve remaining files before retrying."
}
"blocked_by_live_authority" => {
"Workdir has a current attachment, reservation or retention hold. Release the owning authority before retrying deletion."
}
"runtime_unavailable" | "provider_unavailable" => {
"Runtime is unavailable. Reconnect it, then retry deletion; the Workdir registry is retained."
}
"provider_observation_unknown" | "provider_cleanup_outcome_unknown" => {
"Runtime has not confirmed deletion. Retry to re-observe the target; the Workdir registry is retained."
}
"unsupported_target" => "This provider does not support Workdir deletion.",
_ => {
"Runtime cleanup failed. Inspect correlated Runtime diagnostics, resolve the cause, then retry deletion."
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum WorkdirRemovalOwnerObservation {
Running { process_start_marker: u64 },
@@ -24741,17 +24813,46 @@ fn execute_reserved_workdir_removal_with_provider(
return api.config_store.commit_workdir_removal_removed(&operation);
}
let Some(status) = observed.working_directory.as_ref() else {
let category = workdir_cleanup_failure_category(&observed);
let category = if category == "provider_cleanup_failed" {
"provider_observation_unknown"
} else {
category
};
return api.config_store.fail_workdir_removal_operation(
&operation,
"provider_observation_unknown",
true,
category,
category != "unsupported_target",
);
};
let corrupted = status.summary.status == WorkingDirectoryStatusKind::Corrupted;
if !corrupted
&& (status.summary.cleanliness.as_deref() != Some("clean")
|| status.summary.status != WorkingDirectoryStatusKind::Active)
// A provider summary is not permission to remove a different materialization.
// Unknown/corrupted metadata is delegated to Runtime's guarded cleanup, never
// repaired into a fabricated identity here.
let summary = &status.summary;
if summary.occupied_by.is_some() {
return api.config_store.complete_workdir_removal_retained(
&operation,
WorkdirRemovalDisposition::Retained,
"blocked_by_live_authority",
);
}
if summary.working_directory_id != operation.working_directory_id
|| (summary.repository_id == "unknown"
&& summary.status != WorkingDirectoryStatusKind::Corrupted)
|| (summary.repository_id != "unknown"
&& crate::workdir_removal::workdir_materialization_fingerprint(
&workdir_record_from_summary(api, &operation.runtime_id, summary),
) != operation.materialization_fingerprint)
{
return api.config_store.fail_workdir_removal_operation(
&operation,
"provider_identity_changed",
true,
);
}
// Past lifecycle failure is not a current dirtiness observation. Runtime
// must revalidate unknown/partial checkouts at the actual deletion boundary.
if summary.cleanliness.as_deref() == Some("dirty") {
return api.config_store.complete_workdir_removal_retained(
&operation,
WorkdirRemovalDisposition::Retained,
@@ -24769,18 +24870,25 @@ fn execute_reserved_workdir_removal_with_provider(
.fail_workdir_removal_operation(&operation, category, retryable);
}
};
if deleted.state != InternalWorkerOperationState::Accepted
&& !runtime_reports_workdir_not_found(&deleted)
{
let category = if deleted
.diagnostics
.iter()
.any(|diagnostic| diagnostic.code == "working_directory_unsupported")
{
"unsupported_target"
let deletion_confirmed = runtime_reports_workdir_not_found(&deleted)
|| (deleted.state == InternalWorkerOperationState::Accepted
&& deleted.working_directory.as_ref().is_some_and(|status| {
status.summary.working_directory_id == operation.working_directory_id
&& status.summary.status == WorkingDirectoryStatusKind::NotFound
}));
if !deletion_confirmed {
let category = if deleted.state == InternalWorkerOperationState::Accepted {
"provider_cleanup_outcome_unknown"
} else {
"provider_cleanup_failed"
workdir_cleanup_failure_category(&deleted)
};
tracing::warn!(
workdir_id = %operation.working_directory_id,
operation_id = %operation.operation_id,
runtime_id = %operation.runtime_id,
category,
"Runtime did not confirm Workdir removal; registry retained"
);
return api.config_store.fail_workdir_removal_operation(
&operation,
category,
@@ -25066,9 +25174,7 @@ fn build_runtime_cleanup_plan(
.unwrap_or_else(|| cleanup_workdir_cleanliness_from_registry(&record.cleanliness));
let action = if cleanup_workdir_file_status_is_record_only(file_status) {
CleanupTargetKind::WorkdirRecordDelete
} else if cleanup_workdir_file_status_is_corrupted(file_status)
|| cleanup_workdir_cleanliness_is_clean(cleanliness)
{
} else if !matches!(cleanliness, CleanupWorkdirCleanliness::Dirty) {
CleanupTargetKind::WorkdirCleanCleanup
} else {
CleanupTargetKind::WorkdirDirtyDiscard
@@ -25096,12 +25202,12 @@ fn build_runtime_cleanup_plan(
} else if cleanup_workdir_file_status_is_record_only(file_status) {
"Not-found Workdir record can be deleted from the Backend registry".to_string()
} else if cleanup_workdir_file_status_is_corrupted(file_status) {
"Corrupted Workdir can be deleted from Runtime storage and Backend registry"
"Runtime will verify remaining files and removal authority for this incomplete Workdir"
.to_string()
} else if matches!(cleanliness, CleanupWorkdirCleanliness::Dirty) {
"Dirty Workdir requires explicit discard confirmation before cleanup".to_string()
"Dirty Workdir is protected; preserve or resolve current changes before retrying".to_string()
} else if matches!(cleanliness, CleanupWorkdirCleanliness::Unknown) {
"Workdir clean state is unknown; explicit discard confirmation is required"
"Runtime will recheck current changes, identity and removal blockers before deleting"
.to_string()
} else {
"Clean Workdir can be manually cleaned up".to_string()
@@ -25303,7 +25409,12 @@ async fn execute_runtime_cleanup_with_context(
target_id: candidate.target_id.clone(),
action: candidate.action.clone(),
status: status.to_string(),
message: message.to_string(),
message: removal.failure_category.as_deref().map_or_else(
|| message.to_string(),
|category| {
format!("{category}: {}", workdir_removal_failure_guidance(category))
},
),
});
}
CleanupTargetKind::WorkerDelete => {
@@ -53019,7 +53130,7 @@ mod tests {
};
let source_fingerprint = repository_source_fingerprint(&repository_source);
for (workdir_id, _, _) in &fixtures {
runtime
let created = runtime
.create_working_directory(WorkingDirectoryRequest {
repository: WorkingDirectoryRepository {
id: "repo-test".to_string(),
@@ -53042,6 +53153,13 @@ mod tests {
"present",
"clean",
);
api.store
.upsert_workdir_registry(&workdir_record_from_summary(
&api,
provider_runtime_id,
&created.summary,
))
.unwrap();
}
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let provider_base_url = format!("http://{}", listener.local_addr().unwrap());
@@ -53591,6 +53709,91 @@ mod tests {
assert_eq!(response.status(), StatusCode::FORBIDDEN);
}
#[tokio::test]
async fn current_worker_can_reattach_usable_pending_checkout_after_failed_removal_but_not_during_attempt()
{
let mut fixture = manual_worker_assignment_fixture().await;
let identity =
worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id)
.unwrap();
configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id);
let api = &fixture.api;
api.store
.detach_worker_workdir(
TEST_WORKSPACE_ID,
&fixture.worker,
Some(&fixture.docs_workdir_id),
"release for removal",
)
.unwrap();
let mut record = api
.store
.get_workdir_registry(TEST_WORKSPACE_ID, &fixture.docs_workdir_id)
.unwrap()
.unwrap();
record.materialization_status = "pending".into();
api.store.upsert_workdir_registry(&record).unwrap();
let intent =
workdir_removal_intent(&record, "account:owner", "remove docs checkout").unwrap();
let pending = api
.config_store
.reserve_workdir_removal_operation(&intent)
.unwrap();
let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-attachments");
let request = || {
runtime_source_request(
&identity,
Some(&fixture.worker.worker_id),
"POST",
&path,
serde_json::to_vec(
&json!({ "alias": "docs", "working_directory_id": fixture.docs_workdir_id }),
)
.unwrap(),
)
};
let blocked = build_router(api.clone()).oneshot(request()).await.unwrap();
let blocked_status = blocked.status();
let blocked_body = to_bytes(blocked.into_body(), usize::MAX).await.unwrap();
assert_eq!(
blocked_status,
StatusCode::CONFLICT,
"{}",
String::from_utf8_lossy(&blocked_body)
);
api.config_store
.fail_workdir_removal_operation(&pending, "provider_unavailable", true)
.unwrap();
let attached = build_router(api.clone()).oneshot(request()).await.unwrap();
let attached_status = attached.status();
let attached_body = to_bytes(attached.into_body(), usize::MAX).await.unwrap();
assert_eq!(
attached_status,
StatusCode::OK,
"{}",
String::from_utf8_lossy(&attached_body)
);
let attached: server_api::CurrentWorkerWorkdirAttachmentResponse =
serde_json::from_slice(&attached_body).unwrap();
assert!(attached.attached);
assert_eq!(attached.working_directory_id, fixture.docs_workdir_id);
assert_eq!(
api.store
.get_workdir_registry(TEST_WORKSPACE_ID, &fixture.docs_workdir_id)
.unwrap()
.unwrap()
.materialization_status,
"pending"
);
assert!(
api.config_store
.workdir_removal_guards(&pending)
.unwrap()
.iter()
.any(|guard| guard.category == "active_attachment")
);
}
#[tokio::test]
async fn current_worker_attachment_list_and_stale_detach_use_signed_identity() {
let mut fixture = manual_worker_assignment_fixture().await;
@@ -54272,6 +54475,18 @@ mod tests {
}
}
fn confirmed_workdir_cleanup_result() -> crate::hosts::RuntimeWorkingDirectoryResult {
workdir_removal_result(
InternalWorkerOperationState::Rejected,
None,
vec![RuntimeDiagnostic {
code: "working_directory_not_found".into(),
severity: HostDiagnosticSeverity::Warning,
message: "authoritatively absent".into(),
}],
)
}
fn reserve_removal_fixture(
api: &WorkspaceApi,
working_directory_id: &str,
@@ -54312,7 +54527,7 @@ mod tests {
Some(clean_summary),
Vec::new(),
),
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
confirmed_workdir_cleanup_result(),
);
let removed = execute_reserved_workdir_removal_with_provider(
&api,
@@ -54378,7 +54593,7 @@ mod tests {
message: "timeout".to_string(),
}],
),
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
confirmed_workdir_cleanup_result(),
);
let unknown = execute_reserved_workdir_removal_with_provider(
&api,
@@ -54400,7 +54615,7 @@ mod tests {
Some(dirty_summary),
Vec::new(),
),
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
confirmed_workdir_cleanup_result(),
);
let dirty = execute_reserved_workdir_removal_with_provider(
&api,
@@ -54418,7 +54633,7 @@ mod tests {
Some(clean_retry_summary),
Vec::new(),
),
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
confirmed_workdir_cleanup_result(),
);
let removed_after_retry = execute_reserved_workdir_removal_with_provider(
&api,
@@ -54444,7 +54659,7 @@ mod tests {
Some(corrupted_summary),
Vec::new(),
),
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
confirmed_workdir_cleanup_result(),
);
let corrupted = execute_reserved_workdir_removal_with_provider(
&api,
@@ -54493,6 +54708,297 @@ mod tests {
assert_eq!(unsupported_provider.cleanup_calls(), 1);
}
#[test]
fn workdir_cleanup_failure_classification_is_bounded_and_guides_normal_retry() {
for (code, category) in [
("working_directory_cleanup_mount_present", "mount_present"),
(
"working_directory_cleanup_mount_check_unavailable",
"mount_check_unavailable",
),
(
"working_directory_cleanup_permission_denied",
"permission_denied",
),
("working_directory_cleanup_resource_busy", "resource_busy"),
(
"working_directory_cleanup_storage_unavailable",
"storage_unavailable",
),
(
"working_directory_cleanup_ownership_unknown",
"ownership_unknown",
),
(
"working_directory_cleanup_changes_present",
"changes_present",
),
(
"working_directory_cleanup_changes_unknown",
"changes_unknown",
),
(
"arbitrary provider text /private/path",
"provider_cleanup_failed",
),
] {
let result = workdir_removal_result(
InternalWorkerOperationState::Rejected,
None,
vec![RuntimeDiagnostic {
code: code.into(),
severity: HostDiagnosticSeverity::Error,
message: "secret /private/path".into(),
}],
);
assert_eq!(workdir_cleanup_failure_category(&result), category);
let guidance = workdir_removal_failure_guidance(category);
assert!(!guidance.contains("/private/path"));
assert!(!guidance.contains("secret"));
assert!(guidance.len() <= 500);
assert!(guidance.contains("retry"));
}
}
#[tokio::test]
async fn workdir_removal_retry_rechecks_pending_materialization_changes_and_attachments() {
let workspace = tempfile::tempdir().unwrap();
init_clean_git_workspace(workspace.path());
let api = test_api(workspace.path()).await;
let (operation, mut summary) = reserve_removal_fixture(&api, "cleanup-retry");
let failed_provider = FakeWorkdirRemovalProvider::new(
workdir_removal_result(
InternalWorkerOperationState::Accepted,
Some(summary.clone()),
Vec::new(),
),
workdir_removal_result(
InternalWorkerOperationState::Rejected,
None,
vec![RuntimeDiagnostic {
code: "working_directory_cleanup_mount_present".into(),
severity: HostDiagnosticSeverity::Error,
message: "/private/mount credentials must not escape".into(),
}],
),
);
let failed = execute_reserved_workdir_removal_with_provider(
&api,
operation,
false,
&failed_provider,
)
.unwrap();
assert!(failed.retryable);
assert_eq!(failed.failure_category.as_deref(), Some("mount_present"));
assert!(
api.store
.get_workdir_registry(TEST_WORKSPACE_ID, "cleanup-retry")
.unwrap()
.is_some()
);
assert!(!serde_json::to_string(&failed).unwrap().contains("/private"));
summary.status = WorkingDirectoryStatusKind::CleanupPending;
summary.cleanliness = Some("dirty".into());
let dirty_provider = FakeWorkdirRemovalProvider::new(
workdir_removal_result(
InternalWorkerOperationState::Accepted,
Some(summary.clone()),
Vec::new(),
),
confirmed_workdir_cleanup_result(),
);
let dirty =
execute_reserved_workdir_removal_with_provider(&api, failed, false, &dirty_provider)
.unwrap();
assert_eq!(dirty.disposition, Some(WorkdirRemovalDisposition::Retained));
assert!(dirty.retryable);
assert_eq!(dirty_provider.cleanup_calls(), 0);
let worker = seed_cleanup_worker(&api, 731, "normal");
seed_cleanup_link(&api, &worker, "cleanup-retry");
let occupied_provider = FakeWorkdirRemovalProvider::new(
workdir_removal_result(
InternalWorkerOperationState::Accepted,
Some(summary.clone()),
Vec::new(),
),
confirmed_workdir_cleanup_result(),
);
let occupied =
execute_reserved_workdir_removal_with_provider(&api, dirty, false, &occupied_provider)
.unwrap();
assert_eq!(
occupied.failure_category.as_deref(),
Some("blocked_by_live_authority")
);
assert!(occupied.retryable);
assert_eq!(occupied_provider.cleanup_calls(), 0);
api.store
.detach_worker_workdir(
TEST_WORKSPACE_ID,
&RuntimeWorkerRef::new("runtime-test", worker),
Some("cleanup-retry"),
"resolve retry attachment",
)
.unwrap();
// Unknown because checkout was partially removed is delegated to Runtime
// for current change/identity protection, not denied by the old state.
summary.cleanliness = Some("unknown".into());
let retry_provider = FakeWorkdirRemovalProvider::new(
workdir_removal_result(
InternalWorkerOperationState::Accepted,
Some(summary),
Vec::new(),
),
confirmed_workdir_cleanup_result(),
);
let removed = execute_reserved_workdir_removal_with_provider(
&api,
occupied.clone(),
false,
&retry_provider,
)
.unwrap();
assert_eq!(removed.operation_id, occupied.operation_id);
assert_eq!(removed.attempt_count, 4);
assert_eq!(
removed.disposition,
Some(WorkdirRemovalDisposition::Removed)
);
assert_eq!(retry_provider.cleanup_calls(), 1);
assert!(
api.store
.get_workdir_registry(TEST_WORKSPACE_ID, "cleanup-retry")
.unwrap()
.is_none()
);
assert_eq!(
execute_reserved_workdir_removal_with_provider(
&api,
removed.clone(),
false,
&retry_provider
)
.unwrap(),
removed
);
}
#[tokio::test]
async fn workdir_observation_failure_preserves_bounded_os_cause_without_removing_registry() {
let workspace = tempfile::tempdir().unwrap();
init_clean_git_workspace(workspace.path());
let api = test_api(workspace.path()).await;
for (id, transport_error) in [
("observation-denied", false),
("rpc-observation-denied", true),
] {
let (operation, _) = reserve_removal_fixture(&api, id);
let provider = FakeWorkdirRemovalProvider::new(
workdir_removal_result(
InternalWorkerOperationState::Rejected,
None,
vec![RuntimeDiagnostic {
code: "working_directory_cleanup_permission_denied".into(),
severity: HostDiagnosticSeverity::Error,
message: "permission denied /private/path".into(),
}],
),
confirmed_workdir_cleanup_result(),
);
if transport_error {
*provider.observation.lock().unwrap() =
Some(Err(RuntimeRegistryError::RuntimeOperationFailed {
runtime_id: "runtime-test".into(),
code: "working_directory_cleanup_permission_denied".into(),
message: "permission denied /private/path".into(),
}));
}
let result =
execute_reserved_workdir_removal_with_provider(&api, operation, false, &provider)
.unwrap();
assert_eq!(
result.failure_category.as_deref(),
Some("permission_denied")
);
assert!(result.retryable);
assert_eq!(provider.cleanup_calls(), 0);
assert!(
!serde_json::to_string(&result)
.unwrap()
.contains("/private/path")
);
assert!(
api.store
.get_workdir_registry(TEST_WORKSPACE_ID, id)
.unwrap()
.is_some()
);
}
}
#[tokio::test]
async fn workdir_removal_keeps_registry_for_changed_identity_and_unknown_cleanup_outcome() {
let workspace = tempfile::tempdir().unwrap();
init_clean_git_workspace(workspace.path());
let api = test_api(workspace.path()).await;
for (id, changed_identity) in [("identity-mismatch", true), ("unknown-cleanup", false)] {
let (operation, mut summary) = reserve_removal_fixture(&api, id);
if changed_identity {
summary.repository_id = "another-repository".into();
}
let provider = FakeWorkdirRemovalProvider::new(
workdir_removal_result(
InternalWorkerOperationState::Accepted,
Some(summary),
Vec::new(),
),
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
);
let result =
execute_reserved_workdir_removal_with_provider(&api, operation, false, &provider)
.unwrap();
assert!(result.retryable);
assert_eq!(
result.failure_category.as_deref(),
Some(if changed_identity {
"provider_identity_changed"
} else {
"provider_cleanup_outcome_unknown"
})
);
assert_eq!(provider.cleanup_calls(), usize::from(!changed_identity));
assert!(
api.store
.get_workdir_registry(TEST_WORKSPACE_ID, id)
.unwrap()
.is_some()
);
}
}
#[tokio::test]
async fn cleanup_plan_offers_normal_revalidation_for_prior_failure_and_unknown() {
let workspace = tempfile::tempdir().unwrap();
init_clean_git_workspace(workspace.path());
let api = test_api(workspace.path()).await;
for (id, state) in [
("pending-retry", "pending"),
("unknown-retry", "unknown"),
("corrupt-retry", "corrupted"),
] {
seed_cleanup_workdir(&api, id, state, "unknown");
}
let plan = build_runtime_cleanup_plan(&api, "runtime-test").unwrap();
for candidate in plan.workdirs {
assert_eq!(candidate.action, CleanupTargetKind::WorkdirCleanCleanup);
assert!(candidate.blocking_reason.is_none());
}
}
#[tokio::test]
async fn concurrent_orphan_recovery_runs_delayed_provider_cleanup_once() {
let workspace = tempfile::tempdir().unwrap();
@@ -54518,7 +55024,7 @@ mod tests {
Some(clean_summary),
Vec::new(),
),
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
confirmed_workdir_cleanup_result(),
)
.with_observation_delay(std::time::Duration::from_millis(100)),
);
+13
View File
@@ -10254,6 +10254,19 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
record.workdir_id, active.worker.runtime_id, active.worker.worker_id
)));
}
// Direct attachments (including ordinary recovery/reconnection)
// must share the same atomic removal fence as reserved attachments.
let removal_pending: bool = tx.query_row(
"SELECT EXISTS(SELECT 1 FROM workdir_removal_operations WHERE workspace_id=?1 AND workdir_id=?2 AND state='pending')",
params![record.workspace_id, record.workdir_id],
|row| row.get(0),
)?;
if removal_pending {
return Err(Error::WorkdirAttachmentConflict(format!(
"Workdir {} has a pending durable removal operation",
record.workdir_id
)));
}
let write = tx.execute(
r#"INSERT INTO worker_workdir_links (
workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
@@ -438,7 +438,9 @@ impl SqliteWorkspaceStore {
"retained completion cannot use removed disposition".to_string(),
));
}
self.finish_workdir_removal_operation(operation, disposition, false, Some(category), false)
// Retention describes this attempt's current blockers, not a permanent
// ban. A subsequent normal request rechecks all live authority.
self.finish_workdir_removal_operation(operation, disposition, true, Some(category), false)
}
pub fn fail_workdir_removal_operation(
+73
View File
@@ -0,0 +1,73 @@
# Workdir removal and ordinary retry
Use the Runtime **Workdirs** page or the normal typed Workdir removal tool to
remove an unused repository Workdir. Deletion is authorized by the Backend's
existing durable removal operation; it is not a registry-only hide operation.
A failed attempt does not require a new recovery state or a manual edit of
`materialization.json`. Retry the same removal request after resolving the
reported cause. Each attempt rechecks current attachments, attachment
reservations, retention holds, materialization identity and current file changes.
A previous clean observation is not permission to discard new changes. A dirty
Workdir must be preserved or its changes resolved by its owner before deletion.
When the same repository checkout is still usable, an authorized Worker may
reattach it through normal `WorkdirAttach` to preserve or resolve changes, then
detach and retry removal. An executing removal fences new attachments; a failed
attempt does not permanently prevent reconnection. Missing or replaced checkout
identity is not made usable by this permission.
## Interpreting a retained Workdir
- **Mount remains:** Git status being clean does not mean mounted resources are
released. Confirm the mount's owner, users and data destination. Have that
owner safely release the resource, then retry normal deletion. Yoi does not
recursively delete mounted data or implicitly unmount external/unknown
resources. Do not use force-clean, recursive chmod or raw recursive deletion
to override this boundary.
- **Permission denied / resource busy / storage unavailable:** have the owning
operator resolve access, release the resource or restore storage. Retry the
same normal action; registry retention is intentional until cleanup is proven.
- **Changes present / changes cannot be verified:** preserve current files and
restore repository inspection where possible. Do not treat a previous cleanup
failure or missing checkout metadata as proof that all remaining files are
expendable.
- **Identity or ownership unavailable:** verify the Workdir target with its
owner. Do not rewrite registry or materialization identity to bypass checks.
- **Runtime unavailable / outcome unknown:** reconnect the Runtime and retry.
Absence of a provider response is not authoritative not-found evidence.
- **Attachment, reservation or hold:** release the current owning authority
through its normal API before retrying. A stale failure does not freeze these
conditions; newly attached Workers remain protected.
The Web page refreshes authoritative inventory after each attempt. Successful
removal commits registry deletion only after Runtime confirms physical deletion
or authoritative not-found. An unknown result leaves the registry in place so
retry can reconcile the provider's current state.
## Runtime safety boundary
The Runtime keeps removal content/identity evidence outside the tree being
removed. This is a factual witness for partial deletion, not a new recovery
state or cleanup stage. Surviving files are rechecked against it; an intact,
currently clean checkout can renew obsolete evidence without discarding newly
ignored/untracked content. Do not edit this witness manually.
Mount-safe removal currently requires Linux `openat2` and `/proc` inspection.
Unavailable inspection fails closed with `mount_check_unavailable`; there is no
fallback to recursive deletion across an unchecked filesystem boundary.
Removal retains the Runtime occupancy mutex through its filesystem effect.
Unrelated Runtime state operations can therefore wait during hashing/unlink;
releasing it would require equivalent exclusion on all attachment paths.
## Diagnostics
Public responses expose a bounded cause category and safe guidance, not arbitrary
provider errors or host paths. For an OS failure, correlate Backend removal logs
(`operation_id`, `workdir_id`, Runtime) with Runtime cleanup logs for that Workdir
and cleanup action. Runtime logs retain the concrete OS error/errno for the
owning operator. These private diagnostics must not be copied unredacted into
public responses or Tickets.
This procedure is not authorization to manipulate a live mount or another
Worker's storage. If ownership or required operator authority is unclear, stop
and request that specific owner/operator's decision.
@@ -0,0 +1,34 @@
# SubWorker / Workdir execution ergonomics during T-730
Observed during Workdir removal implementation in a dedicated clean checkout:
- Children using `profile: builtin:default` were given explicit writable scope
and command permission, but their available tools/context did not support the
requested implementation. They performed read-only investigation and reported
the missing capabilities. Recreating coding children with `profile: inherit`
exposed the needed implementation/Ticket tools. No rejected capability was
bypassed.
- Concurrent Workdir reads sometimes returned `filesystem provider busy`.
This is transient provider contention, not an authorization rejection. A
child initially treated it as a stop condition; a subsequent parent turn
clarified the difference and sequential retries proceeded normally.
- After a user interruption, repository changes and parent tasks remained, but
`WorkerList` no longer listed the previously active children. Their work was
not treated as approved or complete: the parent inspected concrete diffs,
reran Web/browser validation and delegated a fresh bounded Runtime follow-up.
Potential improvements:
1. Show effective child tool capabilities together with filesystem/command
grants before spawn, so a writable scope is not mistaken for an available
implementation tool surface.
2. Distinguish retryable provider contention from capability rejection in typed
error metadata and recommended recovery text.
3. Make interruption disposition and retained child-session observation grants
explicit. Preserved files are useful evidence, but they do not replace a
committed final child report or verification of tests.
These observations concern development-tool ergonomics, not product removal
requirements or permission to operate on live Workdirs/mounts. The T-730 work
uses temporary fixtures and ordinary Git publication; no live cleanup, unmount,
registry edits or runtime deployment were performed.
@@ -0,0 +1,140 @@
// Synthetic API boundary only; the production route and shell own all rendering.
export function workdirFixture() {
let mountReleased = false;
const deleted = new Set<string>();
const requests: {
workspace: string;
method: string;
path: string;
body?: unknown;
}[] = [];
const summary = (
id: string,
status = "active",
cleanliness = "clean",
occupied = false,
) => ({
working_directory_id: id,
display_name: id === "retry-dir" ? "Cleanup retry — distributed development checkout" : id,
source: { kind: "repository", repository_key: "main" },
materializer_kind: "runtime_git_clone",
status,
cleanliness,
current_ref: "a".repeat(40),
current_selector: "develop",
occupied_by: occupied
? {
runtime_id: "fixture-runtime",
worker_id: "worker-1",
display_name: "Active Worker",
linked_at: "2026-01-01T00:00:00Z",
}
: null,
});
const items = (workspace: string) =>
workspace === "home-empty" ? [] : [
...(deleted.has(workspace) ? [] : [summary("retry-dir", "cleanup_pending", "unknown")]),
summary("dirty-dir", "active", "dirty"),
summary("occupied-dir", "active", "clean", true),
];
const plan = (workspace: string) => ({
workspace_id: workspace,
runtime_id: "fixture-runtime",
generated_at: "2026-01-01T00:00:00Z",
revision: "plan-1",
digest: "digest-1",
workers: [],
diagnostics: [],
workdirs: items(workspace).map((item) => ({
target_id: `workdir:${item.working_directory_id}`,
action: item.cleanliness === "dirty" ? "workdir_dirty_discard" : "workdir_clean_cleanup",
workdir_id: item.working_directory_id,
runtime_id: "fixture-runtime",
repository_key: "main",
reason: "Ordinary cleanup rechecks current conditions",
linked_worker_ids: item.occupied_by ? ["worker-1"] : [],
linked_running_worker_ids: item.occupied_by ? ["worker-1"] : [],
running_linked: Boolean(item.occupied_by),
pinned_linked: false,
file_status: item.status === "cleanup_pending" ? "pending" : "present",
cleanliness: item.cleanliness,
blocking_reason: item.occupied_by ? "Workdir is linked to a running Worker" : null,
})),
});
return async (request: Request): Promise<Response | null> => {
const url = new URL(request.url);
if (
url.pathname === "/fixture-workdirs-control" && request.method === "POST"
) {
const control = await request.json();
mountReleased = control.mountReleased === true;
if (control.reset) {
deleted.clear();
requests.length = 0;
}
return Response.json({ ok: true });
}
if (url.pathname === "/fixture-workdirs") {
return Response.json({ requests });
}
const match = url.pathname.match(/^\/api\/w\/([^/]+)(\/runtimes.*)$/);
if (!match) return null;
const [, workspace, path] = match;
if (path === "/runtimes") {
return Response.json({
workspace_id: workspace,
limit: 100,
source: "fixture",
diagnostics: [],
items: [],
});
}
if (!path.startsWith("/runtimes/fixture-runtime/")) return null;
const body = request.method !== "GET" ? await request.json() : undefined;
requests.push({ workspace, method: request.method, path, body });
if (path.endsWith("/working-directories")) {
return Response.json({
workspace_id: workspace,
items: items(workspace),
diagnostics: [],
});
}
if (path.endsWith("/cleanup-plan")) return Response.json(plan(workspace));
// Baseline route uses cleanup-executions; same retained outcome for before captures.
if (path.endsWith("/cleanup-executions")) {
return Response.json({
workspace_id: workspace,
runtime_id: "fixture-runtime",
executed_at: "2026-01-01T00:00:00Z",
diagnostics: [],
plan_after: plan(workspace),
results: [{
target_id: "workdir:retry-dir",
action: "workdir_clean_cleanup",
status: "retained",
message: "Provider cleanup failed",
}],
});
}
if (
path.endsWith("/working-directories/retry-dir") &&
request.method === "DELETE"
) {
if (workspace === "home-error") {
return Response.json({
message: "Synthetic private host path must not render",
}, { status: 503 });
}
if (mountReleased) deleted.add(workspace);
return Response.json({
working_directory_id: "retry-dir",
disposition: mountReleased ? "removed" : "attention_required",
retryable: !mountReleased,
failure_category: mountReleased ? null : "mount_present",
});
}
return Response.json({ error: "Unexpected fixture Workdir request" }, {
status: 404,
});
};
}
@@ -0,0 +1,456 @@
/// <reference lib="dom" />
import { assert, assertEquals, assertStringIncludes } from "@std/assert";
import { dirname, fromFileUrl, join } from "@std/path";
import { chromium, type Page } from "playwright";
import { startOwnedProcesses, stopOwnedProcesses } from "../src/processes.ts";
const root = join(dirname(fromFileUrl(import.meta.url)), "../../..");
const evidence = join(root, "target/web-ux/t730-browser");
async function noOverflow(page: Page) {
assertEquals(
await page.locator("main").evaluate((main) => ({
page: document.documentElement.scrollWidth > innerWidth,
main: main.scrollWidth > main.clientWidth,
})),
{ page: false, main: false },
);
}
Deno.test("ordinary removal refreshes failed and successful inventory, supports exact retry, and preserves permission and live guards across viewports and themes", async () => {
await Deno.mkdir(evidence, { recursive: true });
const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 });
const port = (listener.addr as Deno.NetAddr).port;
listener.close();
const url = `http://127.0.0.1:${port}`;
const server = await startOwnedProcesses(
[{
id: "workdirs",
command: Deno.execPath(),
cwd: root,
args: [
"run",
"--allow-net",
"--allow-read",
"tools/web-ux/browser-tests/workspace_home_fixture_server.ts",
String(port),
"web/workspace/build",
],
readyUrl: `${url}/health`,
}],
join(root, "tools/web-ux/scenarios/workdir-removal-light.json"),
join(evidence, "process-logs"),
[],
);
const observations: unknown[] = [];
const injectedConsoleErrors: string[] = [];
const injectedUnexpectedErrors: string[] = [];
const observeInjectedFailure = (page: Page) => {
page.on("console", (message) => {
if (message.type() === "error") {
injectedConsoleErrors.push(message.text());
}
});
page.on(
"pageerror",
(error) => injectedUnexpectedErrors.push(String(error)),
);
page.on(
"requestfailed",
(request) => injectedUnexpectedErrors.push(request.url()),
);
};
const control = async (value: unknown) => {
const response = await fetch(`${url}/fixture-workdirs-control`, {
method: "POST",
body: JSON.stringify(value),
});
assert(response.ok);
await response.body?.cancel();
};
const state = async () => await (await fetch(`${url}/fixture-workdirs`)).json();
const browser = await chromium.launch({ headless: true });
try {
for (const colorScheme of ["light", "dark"] as const) {
for (const width of [1440, 768, 390, 320]) {
await control({ reset: true });
const context = await browser.newContext({
viewport: { width, height: 900 },
colorScheme,
reducedMotion: "reduce",
});
try {
const page = await context.newPage();
const errors: string[] = [];
page.on("pageerror", (error) => errors.push(String(error)));
page.on("console", (message) => {
if (message.type() === "error") errors.push(message.text());
});
page.on("requestfailed", (request) => errors.push(request.url()));
const route = `${url}/w/home-owner/settings/runtimes/fixture-runtime/workdirs`;
await page.goto(route);
const deletion = page.getByRole("button", {
name: "Delete retry-dir",
exact: true,
});
await deletion.waitFor();
assert(
await deletion.isEnabled(),
"past pending/unknown state permanently disabled ordinary removal",
);
assert(
await page.getByRole("button", {
name: "Delete dirty-dir",
exact: true,
}).isDisabled(),
);
assert(
await page.getByRole("button", {
name: "Delete occupied-dir",
exact: true,
}).isDisabled(),
);
await noOverflow(page);
assert(
await deletion.evaluate((button) => {
const r = button.getBoundingClientRect();
return r.left >= 0 && r.right <= innerWidth;
}),
"Delete is outside initial viewport",
);
await page.screenshot({
path: join(evidence, `${colorScheme}-${width}-initial.png`),
});
// Hold the actual request boundary to verify duplicate gestures cannot dispatch another DELETE.
let release!: () => void;
const gate = new Promise<void>((resolve) => release = resolve);
let observed!: () => void;
const entered = new Promise<void>((resolve) => observed = resolve);
let deletes = 0;
await page.route(
"**/working-directories/retry-dir",
async (intercept) => {
deletes++;
observed();
await gate;
await intercept.continue();
},
);
await deletion.focus();
await page.keyboard.press("Enter");
await page.getByRole("button", {
name: "Delete retry-dir",
exact: true,
}).filter({ hasText: "Deleting…" }).waitFor();
assert(await deletion.isDisabled());
await page.keyboard.press("Enter");
await entered;
assertEquals(deletes, 1);
await page.screenshot({
path: join(evidence, `${colorScheme}-${width}-deleting.png`),
});
release();
await page.getByRole("alert").filter({
hasText: "mounted filesystem",
}).waitFor();
const retry = page.getByRole("button", {
name: "Retry Delete retry-dir",
exact: true,
});
await retry.waitFor();
await page.waitForFunction(() =>
!(document.querySelector(
"button[aria-label='Retry Delete retry-dir']",
) as HTMLButtonElement)?.disabled
);
const failed = await state();
const beforeGets = failed.requests.filter((
r: { workspace: string; method: string },
) => r.workspace === "home-owner" && r.method === "GET");
assertEquals(
beforeGets.filter((r: { path: string }) => r.path.endsWith("/working-directories"))
.length,
2,
);
assertEquals(
beforeGets.filter((r: { path: string }) => r.path.endsWith("/cleanup-plan")).length,
2,
);
assertEquals(await page.locator("tbody tr").count(), 3);
await noOverflow(page);
await page.screenshot({
path: join(evidence, `${colorScheme}-${width}-mount-failure.png`),
});
await control({ mountReleased: true });
await retry.focus();
await page.keyboard.press("Enter");
await page.getByRole("status").filter({
hasText: "Deletion confirmed",
}).waitFor();
await page.waitForFunction(() => document.querySelectorAll("tbody tr").length === 2);
assertEquals(
await page.getByRole("button", { name: /retry-dir/ }).count(),
0,
);
assert(
await page.getByRole("button", {
name: "Delete dirty-dir",
exact: true,
}).isDisabled(),
);
const final = await state();
const mutations = final.requests.filter((r: { method: string }) => r.method === "DELETE");
assertEquals(mutations.length, 2);
assertEquals(
mutations[0],
mutations[1],
"ordinary retry changed its request",
);
const gets = final.requests.filter((
r: { workspace: string; method: string },
) => r.workspace === "home-owner" && r.method === "GET");
assertEquals(
gets.filter((r: { path: string }) => r.path.endsWith("/working-directories")).length,
3,
);
assertEquals(
gets.filter((r: { path: string }) => r.path.endsWith("/cleanup-plan")).length,
3,
);
await page.screenshot({
path: join(evidence, `${colorScheme}-${width}-removed.png`),
});
await page.goto(
`${url}/w/home-member/settings/runtimes/fixture-runtime/workdirs`,
);
await page.locator("tbody tr").first().waitFor();
assertEquals(
await page.getByRole("button", { name: /Delete/ }).count(),
0,
);
await noOverflow(page);
await page.screenshot({
path: join(evidence, `${colorScheme}-${width}-member.png`),
});
await page.goto(
`${url}/w/home-empty/settings/runtimes/fixture-runtime/workdirs`,
);
await page.getByText("No workdirs are visible for this Runtime.")
.waitFor();
await noOverflow(page);
assertEquals(errors, []);
observations.push({
colorScheme,
width,
requests: final.requests,
errors,
});
} finally {
await context.close();
}
}
}
// Lost/HTTP failure responses still trigger fresh inventory and eligibility. Never echo provider bodies.
await control({ reset: true });
const page = await browser.newPage();
observeInjectedFailure(page);
await page.goto(
`${url}/w/home-error/settings/runtimes/fixture-runtime/workdirs`,
);
await page.getByRole("button", { name: "Delete retry-dir", exact: true })
.click();
await page.getByRole("alert").filter({ hasText: "provider is unavailable" })
.waitFor();
await page.waitForFunction(() =>
!(document.querySelector(
"button[aria-label='Delete retry-dir']",
) as HTMLButtonElement)?.disabled
);
assert(
!(await page.locator("main").innerText()).includes("Synthetic private"),
);
const failed = await state();
assertEquals(
failed.requests.filter((r: { method: string; path: string }) =>
r.method === "GET" && r.path.endsWith("/working-directories")
).length,
2,
);
assertEquals(
failed.requests.filter((r: { method: string; path: string }) =>
r.method === "GET" && r.path.endsWith("/cleanup-plan")
).length,
2,
);
await page.screenshot({ path: join(evidence, "http-failure.png") });
await page.close();
// Plan refresh failure cannot leave stale delete authority enabled; manual Refresh recovers it.
await control({ reset: true });
const stale = await browser.newPage();
observeInjectedFailure(stale);
let planGets = 0;
let planFails = true;
await stale.route("**/cleanup-plan", async (route) => {
planGets++;
if (planGets > 1 && planFails) {
await route.fulfill({
status: 503,
json: { message: "Synthetic private path" },
});
} else await route.continue();
});
await stale.goto(
`${url}/w/home-owner/settings/runtimes/fixture-runtime/workdirs`,
);
await stale.getByRole("button", { name: "Delete retry-dir", exact: true })
.click();
await stale.getByRole("alert").filter({ hasText: "could not be refreshed" })
.waitFor();
assert(
await stale.getByRole("button", {
name: "Retry Delete retry-dir",
exact: true,
}).isDisabled(),
);
assert(
!(await stale.locator("main").innerText()).includes("Synthetic private"),
);
await stale.screenshot({ path: join(evidence, "refresh-failure.png") });
planFails = false;
await stale.getByRole("button", { name: "Refresh", exact: true }).click();
await stale.waitForFunction(() =>
!(document.querySelector(
"button[aria-label='Retry Delete retry-dir']",
) as HTMLButtonElement)?.disabled
);
await stale.close();
// Current dirty changes after a failure override a retryable historical result.
await control({ reset: true });
const changed = await browser.newPage();
let changedGets = 0;
await changed.route("**/cleanup-plan", async (route) => {
const response = await route.fetch();
const plan = await response.json();
if (++changedGets > 1) plan.workdirs[0].action = "workdir_dirty_discard";
await route.fulfill({ response, json: plan });
});
await changed.goto(
`${url}/w/home-owner/settings/runtimes/fixture-runtime/workdirs`,
);
await changed.getByRole("button", { name: "Delete retry-dir", exact: true })
.click();
await changed.getByRole("button", {
name: "Retry Delete retry-dir",
exact: true,
}).waitFor();
await changed.waitForFunction(() =>
document.querySelector(".workdir-identity .removal-guard")?.textContent
?.includes("Changes are protected")
);
assert(
await changed.getByRole("button", {
name: "Retry Delete retry-dir",
exact: true,
}).isDisabled(),
);
assertStringIncludes(
await changed.locator("main").innerText(),
"Changes are protected",
);
await changed.screenshot({
path: join(evidence, "new-changes-protected.png"),
});
await changed.close();
// A malformed response may follow a real side effect. Inventory still removes the now-absent row.
await control({ reset: true, mountReleased: true });
const lost = await browser.newPage();
observeInjectedFailure(lost);
await lost.route("**/working-directories/retry-dir", async (route) => {
await route.fetch();
await route.fulfill({
status: 200,
json: {
message: "Synthetic private path",
failure_category: "/private/path token=synthetic",
},
});
});
await lost.goto(
`${url}/w/home-owner/settings/runtimes/fixture-runtime/workdirs`,
);
await lost.getByRole("button", { name: "Delete retry-dir", exact: true })
.click();
await lost.getByRole("alert").filter({
hasText: "Removal could not be confirmed",
}).waitFor();
await lost.waitForFunction(() => document.querySelectorAll("tbody tr").length === 2);
assert(
!(await lost.locator("main").innerText()).includes("Synthetic private"),
);
assert(!(await lost.locator("main").innerText()).includes("/private/path"));
await lost.screenshot({
path: join(evidence, "malformed-response-refreshed.png"),
});
await lost.close();
// Even a failed initial inventory can recover into a confirmed empty list with manual Refresh.
const emptyRecovery = await browser.newPage();
observeInjectedFailure(emptyRecovery);
let inventoryFails = true;
await emptyRecovery.route(
"**/runtimes/fixture-runtime/working-directories",
async (route) => {
if (inventoryFails) {
await route.fulfill({
status: 503,
json: { message: "Synthetic private path" },
});
} else await route.continue();
},
);
await emptyRecovery.goto(
`${url}/w/home-empty/settings/runtimes/fixture-runtime/workdirs`,
);
await emptyRecovery.getByText("Workdir inventory is unavailable.")
.waitFor();
inventoryFails = false;
await emptyRecovery.getByRole("button", { name: "Refresh", exact: true })
.click();
await emptyRecovery.getByText("No workdirs are visible for this Runtime.")
.waitFor();
await emptyRecovery.close();
assertEquals(injectedUnexpectedErrors, []);
assert(
injectedConsoleErrors.every((message) => message.includes("503")),
"unexpected injected failure console diagnostic",
);
await Deno.writeTextFile(
join(evidence, "evidence.json"),
JSON.stringify(
{
observations,
injectedConsoleErrors,
injectedUnexpectedErrors,
failureRefresh: failed.requests,
refreshFailureRecovered: true,
newChangesProtected: true,
malformedResponseReconciled: true,
emptyInventoryRecovered: true,
expectedFailureDiagnostics: [
"Synthetic DELETE HTTP 503 on home-error",
"Synthetic cleanup-plan HTTP 503 after removal",
"Malformed DELETE result after fixture physical removal",
"Synthetic initial inventory HTTP 503",
],
},
null,
2,
),
);
} finally {
await browser.close();
assertEquals(await stopOwnedProcesses(server), []);
}
});
@@ -2,6 +2,8 @@ import { extname, join, normalize } from "jsr:@std/path@1.1.4";
import { dashboardFixture } from "../../../web/workspace/src/lib/workspace/home/dashboard.test-fixtures.ts";
import { settingsFixtureHandler } from "../../../web/workspace/src/lib/workspace/settings/identity.test-fixtures.ts";
import { workdirFixture } from "./workdir_fixture.ts";
const workdirsFixture = workdirFixture();
const settingsFixture = settingsFixtureHandler();
const port = Number(Deno.args[0]);
@@ -30,6 +32,8 @@ const mime: Record<string, string> = {
Deno.serve({ hostname: "127.0.0.1", port }, async (request) => {
const url = new URL(request.url);
if (url.pathname === "/health") return new Response("ok");
const workdirsResponse = await workdirsFixture(request);
if (workdirsResponse) return workdirsResponse;
const settingsResponse = await settingsFixture(request);
if (settingsResponse) return settingsResponse;
if (url.pathname === "/api/workspaces") {
+1 -1
View File
@@ -11,7 +11,7 @@
"web-ux": "deno run --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys cli.ts",
"check": "deno check cli.ts src/*.ts tests/*.ts browser-tests/*.ts",
"test": "deno test --allow-env --allow-read --allow-write --allow-run --allow-sys tests",
"test:browser": "deno test --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys browser-tests/ticket_shell_test.ts browser-tests/capture_smoke_test.ts browser-tests/memory_document_shell_test.ts browser-tests/console_history_shell_test.ts browser-tests/workspace_home_shell_test.ts browser-tests/workspace_settings_shell_test.ts browser-tests/console_pending_shell_test.ts browser-tests/console_live_shell_test.ts browser-tests/console_live_admission_shell_test.ts"
"test:browser": "deno test --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys browser-tests/ticket_shell_test.ts browser-tests/capture_smoke_test.ts browser-tests/memory_document_shell_test.ts browser-tests/console_history_shell_test.ts browser-tests/workspace_home_shell_test.ts browser-tests/workspace_settings_shell_test.ts browser-tests/workdir_removal_shell_test.ts browser-tests/console_pending_shell_test.ts browser-tests/console_live_shell_test.ts browser-tests/console_live_admission_shell_test.ts"
},
"fmt": {
"lineWidth": 100,
@@ -0,0 +1,117 @@
{
"schemaVersion": 1,
"id": "workdir-removal-dark",
"title": "Ordinary Workdir removal and guarded retry",
"baseUrl": "http://127.0.0.1:15290",
"colorScheme": "dark",
"reducedMotion": "reduce",
"personas": [
{
"id": "fixture",
"label": "Synthetic permission projections",
"auth": {
"kind": "anonymous"
}
}
],
"viewports": [
{
"label": "desktop",
"width": 1440,
"height": 900
},
{
"label": "narrow",
"width": 768,
"height": 900
},
{
"label": "mobile",
"width": 390,
"height": 844
}
],
"routes": [
{
"id": "owner",
"label": "Owner removal",
"path": "/w/home-owner/settings/runtimes/fixture-runtime/workdirs",
"goal": "Remove a failed-cleanup Workdir using ordinary retry without enabling dirty or occupied removal.",
"dataState": "Pending/unknown retry plus dirty and occupied guards. First attempt finds a mount.",
"ready": {
"kind": "selector",
"selector": "main table tbody tr"
},
"capturePoints": [
{
"id": "initial",
"label": "Current inventory"
},
{
"id": "failed-attempt",
"label": "Removal blocked by mount",
"interaction": [
{
"action": "click",
"selector": "button[aria-label='Delete retry-dir']"
}
],
"ready": {
"kind": "selector",
"selector": "[role='alert']"
}
}
]
},
{
"id": "member",
"label": "No Runtime management permission",
"path": "/w/home-member/settings/runtimes/fixture-runtime/workdirs",
"goal": "Read inventory without owner-only removal controls.",
"dataState": "Management permission false.",
"ready": {
"kind": "selector",
"selector": "main table tbody tr"
},
"capturePoints": [
{
"id": "initial",
"label": "Read only inventory"
}
]
},
{
"id": "empty",
"label": "Empty inventory",
"path": "/w/home-empty/settings/runtimes/fixture-runtime/workdirs",
"goal": "Recognize no remaining Workdirs.",
"dataState": "Empty list.",
"ready": {
"kind": "selector",
"selector": "main:has-text('No workdirs')"
},
"capturePoints": [
{
"id": "initial",
"label": "Empty inventory"
}
]
}
],
"processes": [
{
"id": "workdir-fixture",
"command": "deno",
"cwd": "..",
"args": [
"run",
"--allow-net",
"--allow-read",
"browser-tests/workspace_home_fixture_server.ts",
"15290",
"../../web/workspace/build"
],
"readyUrl": "http://127.0.0.1:15290/health"
}
]
}
@@ -0,0 +1,81 @@
{
"schemaVersion": 1,
"id": "workdir-removal-light",
"title": "Ordinary Workdir removal and guarded retry",
"baseUrl": "http://127.0.0.1:15290",
"colorScheme": "light",
"reducedMotion": "reduce",
"personas": [
{
"id": "fixture",
"label": "Synthetic permission projections",
"auth": { "kind": "anonymous" }
}
],
"viewports": [
{ "label": "desktop", "width": 1440, "height": 900 },
{ "label": "narrow", "width": 768, "height": 900 },
{ "label": "mobile", "width": 390, "height": 844 }
],
"routes": [
{
"id": "owner",
"label": "Owner removal",
"path": "/w/home-owner/settings/runtimes/fixture-runtime/workdirs",
"goal": "Remove a failed-cleanup Workdir using ordinary retry without enabling dirty or occupied removal.",
"dataState": "Pending/unknown retry plus dirty and occupied guards. First attempt finds a mount.",
"ready": { "kind": "selector", "selector": "main table tbody tr" },
"capturePoints": [
{ "id": "initial", "label": "Current inventory" },
{
"id": "failed-attempt",
"label": "Removal blocked by mount",
"interaction": [
{
"action": "click",
"selector": "button[aria-label='Delete retry-dir']"
}
],
"ready": { "kind": "selector", "selector": "[role='alert']" }
}
]
},
{
"id": "member",
"label": "No Runtime management permission",
"path": "/w/home-member/settings/runtimes/fixture-runtime/workdirs",
"goal": "Read inventory without owner-only removal controls.",
"dataState": "Management permission false.",
"ready": { "kind": "selector", "selector": "main table tbody tr" },
"capturePoints": [{ "id": "initial", "label": "Read only inventory" }]
},
{
"id": "empty",
"label": "Empty inventory",
"path": "/w/home-empty/settings/runtimes/fixture-runtime/workdirs",
"goal": "Recognize no remaining Workdirs.",
"dataState": "Empty list.",
"ready": {
"kind": "selector",
"selector": "main:has-text('No workdirs')"
},
"capturePoints": [{ "id": "initial", "label": "Empty inventory" }]
}
],
"processes": [
{
"id": "workdir-fixture",
"command": "deno",
"cwd": "..",
"args": [
"run",
"--allow-net",
"--allow-read",
"browser-tests/workspace_home_fixture_server.ts",
"15290",
"../../web/workspace/build"
],
"readyUrl": "http://127.0.0.1:15290/health"
}
]
}
+1 -1
View File
@@ -6,7 +6,7 @@
"dev": "deno run -A npm:vite@8.3.2 dev",
"dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787",
"check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json",
"test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts",
"test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts",
"test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts",
"test:browser:empty-workspace": "deno test --config ../../tools/web-ux/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/visual-review/empty-workspace.browser.test.ts",
"test:browser:drive": "deno test --config test/drive-browser/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/drive-browser/drive.browser.test.ts",
@@ -7,6 +7,7 @@ import type {
WorkingDirectoryDetailResponse,
WorkingDirectoryListResponse,
WorkingDirectoryOccupancy,
WorkingDirectoryRemovalResponse,
WorkingDirectorySource,
WorkingDirectorySummary,
} from "../../generated/workdir-api";
@@ -63,6 +64,40 @@ export function parseWorkingDirectoryListResponse(
};
}
export function parseWorkingDirectoryRemovalResponse(
value: unknown,
): WorkingDirectoryRemovalResponse {
const record = exactRecord(
value,
new Set([
"working_directory_id",
"disposition",
"retryable",
"failure_category",
]),
"Workdir removal response",
);
const working_directory_id = stringField(record, "working_directory_id");
const category = record.failure_category;
if (
working_directory_id.length > 128 ||
(category != null &&
(typeof category !== "string" || category.length > 128))
) {
throw new Error("Invalid Workdir removal response");
}
return {
working_directory_id,
disposition: enumField(record, "disposition", [
"removed",
"retained",
"attention_required",
]),
retryable: booleanField(record, "retryable", "Workdir removal response"),
failure_category: category as string | null | undefined,
};
}
export function parseWorkingDirectoryDetailResponse(
value: unknown,
): WorkingDirectoryDetailResponse {
@@ -178,13 +213,26 @@ export function parseWorkingDirectorySummary(
function parseWorkingDirectorySource(value: unknown): WorkingDirectorySource {
const source = exactRecord(
value,
new Set(["kind", "repository_key", "grant_id", "grant_permissions", "access", "content_path", "purpose"]),
new Set([
"kind",
"repository_key",
"grant_id",
"grant_permissions",
"access",
"content_path",
"purpose",
]),
"Workdir source",
);
const kind = stringField(source, "kind");
if (kind === "workspace_config") {
if (source.repository_key !== undefined || source.grant_id !== undefined || source.grant_permissions !== undefined) {
throw new Error("Logical config source must not contain repository or External authority");
if (
source.repository_key !== undefined || source.grant_id !== undefined ||
source.grant_permissions !== undefined
) {
throw new Error(
"Logical config source must not contain repository or External authority",
);
}
const access = stringField(source, "access");
if (access !== "read_only" && access !== "read_write") {
@@ -194,10 +242,20 @@ function parseWorkingDirectorySource(value: unknown): WorkingDirectorySource {
if (content_path !== "/workspace-config") {
throw new Error("Logical config must reference its WIP content entrance");
}
return { kind, access, content_path, purpose: stringField(source, "purpose") };
return {
kind,
access,
content_path,
purpose: stringField(source, "purpose"),
};
}
if (source.access !== undefined || source.content_path !== undefined || source.purpose !== undefined) {
throw new Error("Filesystem Workdir source must not contain logical config metadata");
if (
source.access !== undefined || source.content_path !== undefined ||
source.purpose !== undefined
) {
throw new Error(
"Filesystem Workdir source must not contain logical config metadata",
);
}
if (kind === "repository") {
if (
@@ -0,0 +1,167 @@
import type { WorkingDirectoryRemovalResponse } from "#lib/generated/workdir-api.ts";
import type {
CleanupWorkdirCandidate,
WorkingDirectorySummary,
} from "#lib/workspace/sidebar/types.ts";
import { workspaceApiPath } from "#lib/workspace/api/http.ts";
import { parseWorkingDirectoryRemovalResponse } from "#lib/workspace/api/workdirs.ts";
// Only reviewed category codes become public copy. Never render a provider message,
// category verbatim, OS error, or host path (including for unknown future codes).
const causes: Readonly<Record<string, string>> = {
mount_present:
"A mounted filesystem remains inside this Workdir. Ask its resource owner to verify ownership and safely release it before retrying. Unknown or shared mounts must not be forcibly unmounted.",
mount_check_unavailable:
"The Runtime could not check for mounted filesystems. Restore mount inspection on the Runtime before retrying.",
permission_denied:
"Removal permission was denied. Ask the Workspace or Runtime administrator to verify the required access; do not bypass protection with a forced deletion.",
resource_busy:
"A resource is still in use. Ask its owner to stop using it and release it safely before retrying.",
storage_unavailable:
"Runtime storage is unavailable. Restore storage access and capacity before retrying.",
ownership_unknown:
"Resource ownership could not be verified. Ask the Runtime administrator to verify ownership before any resource is released or removed.",
changes_present:
"This Workdir has changes. Preserve or clean those changes before requesting removal again.",
changes_unknown:
"Current changes could not be checked. Restore access so the Runtime can verify change protection before removal.",
provider_cleanup_failed:
"The provider could not finish removal. Ask the Runtime administrator to inspect the correlated removal diagnostics and resolve the cause before retrying.",
provider_observation_unknown:
"The provider could not confirm the remaining Workdir. Restore provider access so the next removal can recheck its current contents.",
provider_cleanup_outcome_unknown:
"The provider has not confirmed deletion. Restore provider access and retry the same removal request to recheck the remaining target. The Workdir stays listed until deletion is confirmed.",
authority_invalid:
"The removal authority could not be verified. Ask the Workspace administrator to verify the Workdir target and owning Runtime before requesting removal again.",
provider_identity_changed:
"The provider target identity changed. Ask the Runtime administrator to verify the target; do not delete a replacement resource.",
runtime_unavailable:
"The Runtime is unavailable. Restore its connection before retrying removal.",
provider_unavailable:
"The Workdir provider is unavailable. Restore its connection before retrying removal.",
unsupported_target:
"This target does not support ordinary removal. Ask its resource owner for the supported cleanup operation.",
blocked_by_live_authority:
"This Workdir has an active attachment, reservation, or hold. Release it through its owning operation before requesting removal again.",
dirty_or_unknown:
"Current change protection prevents removal. Preserve changes or restore change inspection before requesting removal again.",
authority_changed:
"Removal authority or current usage changed. Refresh the inventory and resolve current attachments, reservations, or holds before requesting removal again.",
};
const unknownCause =
"Removal could not be confirmed. Check the Runtime's correlated removal diagnostics and resolve the cause before requesting removal again. The Workdir remains listed until deletion is confirmed.";
export function removalCause(category: string | null | undefined): string {
return category != null && Object.hasOwn(causes, category)
? causes[category]
: unknownCause;
}
export function canRemoveWorkdir(
workdir: WorkingDirectorySummary,
candidate: CleanupWorkdirCandidate | undefined,
): boolean {
return Boolean(
candidate && !candidate.blocking_reason && !candidate.running_linked &&
!candidate.pinned_linked &&
(candidate.action === "workdir_clean_cleanup" ||
candidate.action === "workdir_record_delete") &&
workdir.cleanliness !== "dirty" && !workdir.occupied_by,
);
}
export function removalGuard(
workdir: WorkingDirectorySummary,
candidate: CleanupWorkdirCandidate | undefined,
): string {
if (
workdir.occupied_by || candidate?.running_linked ||
candidate?.pinned_linked || candidate?.blocking_reason
) {
return "An attachment, reservation, or hold protects this Workdir. Release it through its owning operation, then refresh.";
}
if (
workdir.cleanliness === "dirty" ||
candidate?.action === "workdir_dirty_discard"
) {
return "Changes are protected. Preserve or clean them, then refresh before deletion.";
}
return "Removal eligibility is unavailable. Refresh before deletion.";
}
export class WorkdirRemovalError extends Error {
constructor(readonly category?: string) {
super(removalCause(category));
}
}
// Read only a small public DTO. Discard HTTP error bodies without interpreting or
// displaying them, and drain small bodies rather than aborting completed requests.
async function removalBody(response: Response): Promise<string> {
const reader = response.body?.getReader();
if (!reader) return "";
const chunks: Uint8Array[] = [];
let bytes = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
bytes += value.byteLength;
if (bytes > 4096) {
await reader.cancel();
throw new WorkdirRemovalError();
}
chunks.push(value);
}
const body = new Uint8Array(bytes);
let offset = 0;
for (const chunk of chunks) {
body.set(chunk, offset);
offset += chunk.byteLength;
}
return new TextDecoder().decode(body);
} finally {
reader.releaseLock();
}
}
export async function removeWorkdir(
fetchFn: typeof fetch,
workspaceId: string,
runtimeId: string,
workdirId: string,
): Promise<WorkingDirectoryRemovalResponse> {
const response = await fetchFn(
workspaceApiPath(
workspaceId,
`/runtimes/${encodeURIComponent(runtimeId)}/working-directories/${
encodeURIComponent(workdirId)
}`,
),
{
method: "DELETE",
headers: { "content-type": "application/json" },
// An ordinary repeated request uses the existing durable Backend removal authority.
body: JSON.stringify({ reason: "User requested Workdir deletion" }),
},
);
if (!response.ok) {
await removalBody(response).catch(() => undefined);
throw new WorkdirRemovalError(
response.status === 403
? "permission_denied"
: response.status === 409
? "authority_changed"
: response.status === 503
? "provider_unavailable"
: undefined,
);
}
const result = parseWorkingDirectoryRemovalResponse(
JSON.parse(await removalBody(response)),
);
if (result.working_directory_id !== workdirId) {
throw new WorkdirRemovalError();
}
return result;
}
@@ -1,102 +1,89 @@
<script lang="ts">
import { pushWorkspaceAlert } from '#lib/workspace/alerts/store.ts';
import { workspaceApiPath } from '#lib/workspace/api/http.ts';
import { parseRuntimeCleanupExecution } from '#lib/workspace/api/runtime-workers.ts';
import { loadJson, workspaceApiPath } from '#lib/workspace/api/http.ts';
import { parseRuntimeCleanupPlan } from '#lib/workspace/api/runtime-workers.ts';
import { parseWorkingDirectoryListResponse } from '#lib/workspace/api/workdirs.ts';
import { canRemoveWorkdir, removalCause, removalGuard, removeWorkdir, WorkdirRemovalError } from '#lib/workspace/settings/workdir-removal.ts';
import { formatCurrentWorkdirRevision } from '#lib/workspace/settings/workdir-revision.ts';
import type {
CleanupWorkdirCandidate,
RuntimeCleanupPlanResponse,
WorkingDirectorySummary,
} from '#lib/workspace/sidebar/types.ts';
import type { CleanupWorkdirCandidate, RuntimeCleanupPlanResponse, WorkingDirectorySummary } from '#lib/workspace/sidebar/types.ts';
import type { PageProps } from './$types';
let { data }: PageProps = $props();
let cleanupBusyTarget = $state<string | null>(null);
let refreshing = $state(false);
let refreshError = $state<string | null>(null);
let cleanupPlan = $state<RuntimeCleanupPlanResponse | null>(null);
let workdirs = $state<WorkingDirectorySummary[]>([]);
let runtimeLabel = $derived(
data.runtimes?.items.find((runtime) => runtime.runtime_id === data.runtimeId)?.label ?? data.runtimeId,
);
let inventoryLoaded = $state(false);
let feedback = $state<{ id: string; message: string; retryable: boolean; removed: boolean } | null>(null);
let runtimeLabel = $derived(data.runtimes?.items.find((runtime) => runtime.runtime_id === data.runtimeId)?.label ?? data.runtimeId);
let canManage = $derived(data.workspace?.permissions.manage_runtimes === true);
let cleanupCandidates = $derived(cleanupPlan?.workdirs ?? []);
$effect(() => {
cleanupPlan = data.cleanupPlan ?? null;
workdirs = data.workdirs?.items ?? [];
inventoryLoaded = Boolean(data.workdirs);
refreshError = data.workdirsError || data.cleanupPlanError ? 'Inventory or removal eligibility is unavailable. Refresh before deletion.' : null;
feedback = null;
});
function repositoryKey(workdir: WorkingDirectorySummary): string | null {
return workdir.source.kind === 'repository' ? workdir.source.repository_key : null;
}
function repositoryProvider(workdir: WorkingDirectorySummary): string | null {
const key = repositoryKey(workdir);
return key
? data.repositories?.items.find((repository) => repository.repository_key === key)?.provider ?? null
: null;
}
function currentRevision(workdir: WorkingDirectorySummary): string {
return formatCurrentWorkdirRevision(workdir, repositoryProvider(workdir));
const key = repositoryKey(workdir);
const provider = key ? data.repositories?.items.find((repository) => repository.repository_key === key)?.provider ?? null : null;
return formatCurrentWorkdirRevision(workdir, provider);
}
function cleanupCandidate(workdir: WorkingDirectorySummary): CleanupWorkdirCandidate | undefined {
return cleanupCandidates.find((candidate) => candidate.workdir_id === workdir.working_directory_id);
}
function isDeleteDisabled(candidate: CleanupWorkdirCandidate): boolean {
return Boolean(candidate.blocking_reason) || candidate.action === 'workdir_dirty_discard' || cleanupBusyTarget !== null;
function isDeleteDisabled(workdir: WorkingDirectorySummary): boolean {
return !canManage || Boolean(refreshError) || refreshing || cleanupBusyTarget !== null || !canRemoveWorkdir(workdir, cleanupCandidate(workdir));
}
function errorMessage(payload: unknown, fallback: string): string {
if (payload && typeof payload === 'object') {
if ('message' in payload && typeof payload.message === 'string') return payload.message;
if ('error' in payload) {
const error = payload.error;
if (typeof error === 'string') return error;
if (error && typeof error === 'object' && 'message' in error && typeof error.message === 'string') return error.message;
}
if ('diagnostics' in payload && Array.isArray(payload.diagnostics)) {
const diagnostic = payload.diagnostics.find(
(entry): entry is { message: string } => Boolean(entry) && typeof entry === 'object' && 'message' in entry && typeof entry.message === 'string',
);
if (diagnostic) return diagnostic.message;
}
}
return fallback;
}
async function deleteWorkdir(workdir: WorkingDirectorySummary, candidate: CleanupWorkdirCandidate): Promise<void> {
if (!cleanupPlan || isDeleteDisabled(candidate)) return;
cleanupBusyTarget = candidate.target_id;
async function refreshInventory(): Promise<void> {
refreshing = true;
// Capture route authority before awaiting; a departed route must not receive this result.
const workspaceId = data.workspaceId;
const runtimeId = data.runtimeId;
try {
const response = await fetch(
workspaceApiPath(data.workspaceId, `/runtimes/${encodeURIComponent(data.runtimeId)}/cleanup-executions`),
{
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
expected_plan_revision: cleanupPlan.revision,
expected_plan_digest: cleanupPlan.digest,
worker_target_ids: [],
workdir_target_ids: [candidate.target_id],
confirm_dirty_discard_target_ids: [],
}),
},
);
const payload = await response.json().catch(() => null);
if (!response.ok) throw new Error(errorMessage(payload, response.statusText));
const execution = parseRuntimeCleanupExecution(payload);
cleanupPlan = execution.plan_after;
const result = execution.results.find((entry) => entry.target_id === candidate.target_id);
if (!result || result.status !== 'deleted') {
throw new Error(result?.message ?? 'Runtime did not delete the selected Workdir');
}
workdirs = workdirs.filter((item) => item.working_directory_id !== workdir.working_directory_id);
} catch (error) {
pushWorkspaceAlert('error', error instanceof Error ? error.message : 'Workdir deletion failed', {
title: 'Workdir deletion failed',
});
const base = `/runtimes/${encodeURIComponent(runtimeId)}`;
const [inventory, plan] = await Promise.all([
loadJson(fetch, workspaceApiPath(workspaceId, `${base}/working-directories`), undefined, parseWorkingDirectoryListResponse),
loadJson(fetch, workspaceApiPath(workspaceId, `${base}/cleanup-plan`), undefined, parseRuntimeCleanupPlan),
]);
if (workspaceId !== data.workspaceId || runtimeId !== data.runtimeId) return;
// Keep the last observed list on failure, but never authorize another deletion with stale eligibility.
if (inventory.data) { workdirs = inventory.data.items; inventoryLoaded = true; }
cleanupPlan = plan.data;
refreshError = inventory.error || plan.error ? 'Inventory or removal eligibility could not be refreshed. Refresh before another deletion.' : null;
} finally {
refreshing = false;
}
}
async function deleteWorkdir(workdir: WorkingDirectorySummary): Promise<void> {
if (isDeleteDisabled(workdir)) return;
const workspaceId = data.workspaceId;
const runtimeId = data.runtimeId;
cleanupBusyTarget = workdir.working_directory_id;
feedback = null;
try {
const result = await removeWorkdir(fetch, workspaceId, runtimeId, workdir.working_directory_id);
if (workspaceId !== data.workspaceId || runtimeId !== data.runtimeId) return;
feedback = {
id: workdir.working_directory_id,
message: result.disposition === 'removed' ? 'Deletion confirmed.' : removalCause(result.failure_category),
retryable: result.retryable && result.disposition !== 'removed',
removed: result.disposition === 'removed',
};
} catch (error) {
if (workspaceId !== data.workspaceId || runtimeId !== data.runtimeId) return;
feedback = { id: workdir.working_directory_id, message: error instanceof WorkdirRemovalError ? error.message : removalCause(null), retryable: false, removed: false };
} finally {
// Even a lost/error response can follow a real provider side effect. Re-read inventory AND guards.
if (workspaceId === data.workspaceId && runtimeId === data.runtimeId) await refreshInventory();
cleanupBusyTarget = null;
}
}
@@ -107,76 +94,55 @@
<meta name="description" content="Runtime workdirs" />
</svelte:head>
<section class="workdirs-page" aria-labelledby="workdirs-heading">
<section class="workdirs-page" aria-labelledby="workdirs-heading" aria-busy={refreshing}>
<header class="page-header-row">
<div>
<p class="breadcrumb"><a href={`/w/${data.workspaceId}/settings/runtimes`}>Runtimes</a> / {runtimeLabel}</p>
<h1 id="workdirs-heading">Workdirs</h1>
<p>Workdirs owned by <code>{data.runtimeId}</code>.</p>
{#if data.cleanupPlanError}<p class="section-state error">{data.cleanupPlanError}</p>{/if}
</div>
<button class="workdir-action" type="button" disabled={refreshing || cleanupBusyTarget !== null} onclick={refreshInventory}>{refreshing ? 'Refreshing…' : 'Refresh'}</button>
</header>
{#if data.workdirsError}
<p class="section-state error">{data.workdirsError}</p>
{:else if !data.workdirs}
<p class="section-state">Loading workdirs…</p>
{#if feedback}
<div class="removal-feedback" class:error={!feedback.removed} role={feedback.removed ? 'status' : 'alert'}>
<p><strong>{feedback.removed ? 'Workdir deleted' : 'Workdir not deleted'}</strong> · <code>{feedback.id}</code></p>
<p>{feedback.message}</p>
{#if feedback.retryable}<p>After resolving this cause, retry Delete. Current safety conditions will be rechecked.</p>{/if}
</div>
{/if}
{#if refreshError}<p class="section-state error" role="alert">{refreshError}</p>{/if}
{#if !inventoryLoaded}
<p class="section-state">{data.workdirsError ? 'Workdir inventory is unavailable.' : 'Loading workdirs…'}</p>
{:else if workdirs.length === 0}
<p class="section-state">No workdirs are visible for this Runtime.</p>
{:else}
<div class="table-wrap">
<!-- svelte-ignore a11y_no_noninteractive_tabindex (Horizontal inventory scroll must be keyboard-focusable.) -->
<div class="table-wrap" role="region" aria-label="Workdir inventory" tabindex="0">
<table class="workdirs-table">
<thead>
<tr>
<th>Workdir</th>
<th>Repository</th>
<th>Revision</th>
<th>Status</th>
<th>Cleanliness</th>
<th>Occupied by</th>
<th>Action</th>
</tr>
</thead>
<thead><tr><th>Workdir</th><th>Repository</th><th>Revision</th><th>Status</th><th>Cleanliness</th><th>Occupied by</th></tr></thead>
<tbody>
{#each workdirs as workdir}
{#each workdirs as workdir (workdir.working_directory_id)}
{@const cleanup = cleanupCandidate(workdir)}
<tr>
<td>
<td class="workdir-identity">
<span>{workdir.display_name ?? '—'}</span>
<small><code>{workdir.working_directory_id}</code></small>
{#if canManage}
<button class="workdir-action danger" type="button" disabled={isDeleteDisabled(workdir)}
aria-label={`${feedback?.id === workdir.working_directory_id && feedback.retryable ? 'Retry Delete' : 'Delete'} ${workdir.working_directory_id}`}
onclick={() => deleteWorkdir(workdir)}>
{cleanupBusyTarget === workdir.working_directory_id ? 'Deleting…' : feedback?.id === workdir.working_directory_id && feedback.retryable ? 'Retry Delete' : 'Delete'}
</button>
{#if !canRemoveWorkdir(workdir, cleanup)}<p class="removal-guard">{removalGuard(workdir, cleanup)}</p>{/if}
{/if}
</td>
<td>{repositoryKey(workdir) ?? 'External'}</td>
<td><code>{currentRevision(workdir)}</code></td>
<td>{workdir.status}</td>
<td>{workdir.cleanliness ?? 'unknown'}</td>
<td>
{#if workdir.occupied_by}
<span>{workdir.occupied_by.display_name}</span>
<small>{workdir.occupied_by.runtime_id}:{workdir.occupied_by.worker_id}</small>
{:else}
<span class="muted">—</span>
{/if}
</td>
<td>
{#if cleanup}
<button
class="icon-action danger"
type="button"
disabled={isDeleteDisabled(cleanup)}
aria-label={`Delete ${workdir.working_directory_id}`}
title={cleanup.action === 'workdir_dirty_discard' ? 'Dirty Workdirs must be cleaned before deletion' : (cleanup.blocking_reason ?? cleanup.reason)}
onclick={() => deleteWorkdir(workdir, cleanup)}
>
{#if cleanupBusyTarget === cleanup.target_id}
<span class="spinner" aria-hidden="true"></span>
{:else}
<svg class="action-icon" aria-hidden="true" viewBox="0 0 24 24"><path d="M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6" /><path d="M3 6h18" /><path d="M8 6V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2" /></svg>
{/if}
</button>
{:else}
<span class="muted">—</span>
{/if}
</td>
<td>{#if workdir.occupied_by}<span>{workdir.occupied_by.display_name}</span><small>{workdir.occupied_by.runtime_id}:{workdir.occupied_by.worker_id}</small>{:else}<span class="muted">—</span>{/if}</td>
</tr>
{/each}
</tbody>
@@ -186,53 +152,20 @@
</section>
<style>
.icon-action {
display: inline-flex;
align-items: center;
justify-content: center;
width: 2rem;
height: 2rem;
padding: 0;
border: 1px solid var(--line);
border-radius: 0.5rem;
background: var(--bg-raised);
color: var(--text);
cursor: pointer;
}
.icon-action.danger:hover:not(:disabled),
.icon-action.danger:focus-visible:not(:disabled) {
border-color: var(--danger, oklch(60% 0.18 30));
color: var(--danger, oklch(60% 0.18 30));
}
.icon-action:disabled {
cursor: not-allowed;
opacity: 0.45;
}
.action-icon {
width: 1rem;
height: 1rem;
fill: none;
stroke: currentColor;
stroke-width: 2;
stroke-linecap: round;
stroke-linejoin: round;
}
.spinner {
width: 1rem;
height: 1rem;
border: 2px solid currentColor;
border-right-color: transparent;
border-radius: 999px;
animation: workdir-action-spin 0.8s linear infinite;
}
@keyframes workdir-action-spin {
to {
transform: rotate(360deg);
}
}
.workdirs-page { min-width: 0; }
.page-header-row { display: flex; flex-wrap: wrap; justify-content: space-between; align-items: start; gap: var(--space-3); }
.table-wrap { max-width: 100%; overflow-x: auto; }
.workdirs-table { width: 100%; min-width: 54rem; border-collapse: collapse; }
th, td { padding: var(--space-3); text-align: left; vertical-align: top; border-bottom: 1px solid var(--line); }
th { color: var(--text-muted); font-weight: 500; }
.workdir-identity { width: 18rem; min-width: 18rem; overflow-wrap: anywhere; }
td small { display: block; color: var(--text-muted); margin-block: var(--space-1); }
.workdir-action { display: inline-flex; align-items: center; justify-content: center; padding: var(--space-2) var(--space-3); border: 1px solid var(--line); border-radius: var(--radius-soft); background: var(--bg-raised); color: var(--text); cursor: pointer; white-space: nowrap; }
.workdir-action:hover:not(:disabled) { background: var(--interactive-hover); }
.workdir-action.danger { color: var(--danger); }
.workdir-action:disabled { cursor: not-allowed; opacity: 0.45; }
.removal-feedback { margin-block: var(--space-4); overflow-wrap: anywhere; }
.removal-feedback p { margin-block: var(--space-2); }
.removal-feedback.error strong { color: var(--danger); }
.removal-guard { margin-block: var(--space-2) 0; color: var(--text-muted); }
</style>
+248
View File
@@ -0,0 +1,248 @@
declare const Deno: {
test(name: string, fn: () => void | Promise<void>): void;
};
import { parseWorkingDirectoryRemovalResponse } from "../src/lib/workspace/api/workdirs.ts";
import {
canRemoveWorkdir,
removalCause,
removeWorkdir,
} from "../src/lib/workspace/settings/workdir-removal.ts";
import type {
CleanupWorkdirCandidate,
WorkingDirectorySummary,
} from "../src/lib/workspace/sidebar/types.ts";
function assert(value: unknown, message = "assertion failed"): asserts value {
if (!value) throw new Error(message);
}
Deno.test("removal wire boundary rejects malformed results and mismatched identities", async () => {
for (
const extra of [
{ retryable: "true" },
{ disposition: "forgotten" },
{ failure_category: {} },
{ failure_category: "x".repeat(129) },
{ working_directory_id: "x".repeat(129) },
{ unexpected: true },
]
) {
let rejected = false;
try {
parseWorkingDirectoryRemovalResponse({
working_directory_id: "dir",
disposition: "retained",
retryable: true,
...extra,
});
} catch {
rejected = true;
}
assert(
rejected,
`accepted malformed removal result: ${JSON.stringify(extra)}`,
);
}
let rejected = false;
try {
await removeWorkdir(
(() =>
Promise.resolve(
Response.json({
working_directory_id: "other",
disposition: "removed",
retryable: false,
}),
)) as typeof fetch,
"workspace",
"runtime",
"dir",
);
} catch {
rejected = true;
}
assert(rejected, "accepted removal evidence for a different Workdir");
});
Deno.test("removal response size is bounded before decoding provider content", async () => {
let rejected = false;
try {
await removeWorkdir(
(() => Promise.resolve(new Response("x".repeat(4097)))) as typeof fetch,
"workspace",
"runtime",
"dir",
);
} catch {
rejected = true;
}
assert(rejected, "accepted an oversized removal response");
});
Deno.test("removal causes give bounded actions without echoing untrusted provider details", () => {
const categories = [
"mount_present",
"mount_check_unavailable",
"permission_denied",
"resource_busy",
"storage_unavailable",
"ownership_unknown",
"changes_present",
"changes_unknown",
"provider_cleanup_failed",
"provider_observation_unknown",
"provider_cleanup_outcome_unknown",
"authority_invalid",
"provider_identity_changed",
"runtime_unavailable",
"provider_unavailable",
"unsupported_target",
"blocked_by_live_authority",
"dirty_or_unknown",
"authority_changed",
];
for (const category of categories) {
const text = removalCause(category);
assert(text !== removalCause(null), `missing category ${category}`);
assert(
text.length < 400 && !text.includes("/"),
`unbounded or pathful category ${category}`,
);
}
for (
const untrusted of [
"/private/host/path",
"token=synthetic-secret",
"toString",
"__proto__",
"future_category",
"x".repeat(10000),
]
) {
const text = removalCause(untrusted);
assert(
text === removalCause(null) && !text.includes(untrusted),
"echoed unknown category",
);
}
});
Deno.test("pending unknown cleanup permits ordinary removal while current dirty and live guards remain protected", () => {
const workdir = {
working_directory_id: "dir",
status: "cleanup_pending",
cleanliness: "unknown",
source: { kind: "repository", repository_key: "main" },
materializer_kind: "runtime_git_clone",
} as WorkingDirectorySummary;
const candidate = {
action: "workdir_clean_cleanup",
blocking_reason: null,
running_linked: false,
pinned_linked: false,
} as CleanupWorkdirCandidate;
assert(canRemoveWorkdir(workdir, candidate));
for (
const guard of [
{ action: "workdir_dirty_discard" },
{ blocking_reason: "protected" },
{ running_linked: true },
{ pinned_linked: true },
]
) {
assert(
!canRemoveWorkdir(
workdir,
{ ...candidate, ...guard } as CleanupWorkdirCandidate,
),
);
}
assert(!canRemoveWorkdir({ ...workdir, cleanliness: "dirty" }, candidate));
assert(
canRemoveWorkdir(workdir, {
...candidate,
action: "workdir_record_delete",
}),
);
assert(!canRemoveWorkdir(workdir, { ...candidate, action: "worker_delete" }));
assert(!canRemoveWorkdir(workdir, undefined));
assert(
!canRemoveWorkdir({
...workdir,
occupied_by: {
worker_id: "worker",
runtime_id: "runtime",
display_name: "Worker",
linked_at: "now",
},
}, candidate),
);
});
Deno.test("ordinary retries send the same scoped DELETE body and expose only allowlisted HTTP causes", async () => {
const requests: { path: string; body: string; method: string }[] = [];
const fetchFn = (async (path: RequestInfo | URL, init?: RequestInit) => {
requests.push({
path: String(path),
body: String(init?.body),
method: String(init?.method),
});
return Response.json({
working_directory_id: "dir/x",
disposition: "attention_required",
retryable: true,
failure_category: "mount_present",
});
}) as typeof fetch;
for (let i = 0; i < 2; i++) {
const result = await removeWorkdir(
fetchFn,
"workspace/x",
"runtime/x",
"dir/x",
);
assert(result.retryable && result.failure_category === "mount_present");
}
assert(
JSON.stringify(requests[0]) === JSON.stringify(requests[1]),
"retry changed ordinary deletion request",
);
assert(
requests[0].path ===
"/api/w/workspace%2Fx/runtimes/runtime%2Fx/working-directories/dir%2Fx",
);
assert(
requests[0].method === "DELETE" &&
requests[0].body ===
JSON.stringify({ reason: "User requested Workdir deletion" }),
);
for (
const [status, category] of [
[403, "permission_denied"],
[409, "authority_changed"],
[503, "provider_unavailable"],
[500, null],
] as const
) {
let message = "";
try {
await removeWorkdir(
(() =>
Promise.resolve(
Response.json({ message: "/private/path token=synthetic-secret" }, {
status,
}),
)) as typeof fetch,
"workspace",
"runtime",
"dir",
);
} catch (error) {
message = error instanceof Error ? error.message : String(error);
}
assert(
message === removalCause(category),
`HTTP ${status} leaked provider text`,
);
}
});