fix: integrate safe Workdir removal recovery
Merge approved T-730 source b1dc8f9f into develop. Resolve the test-registration conflict by preserving both Drive and Workdir-removal tests. Integration root check, Runtime/Server/Worker suites, real Web Drive adapter, generated Workdir contract, Web check/unit/components/build/browser, fmt and diff checks passed.
This commit is contained in:
Generated
+1
@@ -6575,6 +6575,7 @@ dependencies = [
|
||||
"flow",
|
||||
"fs4",
|
||||
"futures",
|
||||
"libc",
|
||||
"manifest",
|
||||
"protocol",
|
||||
"reqwest",
|
||||
|
||||
@@ -31,6 +31,7 @@ decodal.workspace = true
|
||||
flow = { path = "../flow" }
|
||||
fs4 = { workspace = true, optional = true }
|
||||
manifest.workspace = true
|
||||
libc.workspace = true
|
||||
protocol.workspace = true
|
||||
runtime-api.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -2486,6 +2486,8 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode {
|
||||
if matches!(
|
||||
diagnostic.code.as_str(),
|
||||
"repository_ref_provider_unavailable"
|
||||
| "working_directory_cleanup_mount_check_unavailable"
|
||||
| "working_directory_cleanup_storage_unavailable"
|
||||
| "repository_ref_provider_timeout"
|
||||
| "repository_access_provider_unavailable"
|
||||
) =>
|
||||
@@ -2504,6 +2506,23 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode {
|
||||
{
|
||||
StatusCode::FORBIDDEN
|
||||
}
|
||||
RuntimeError::WorkingDirectory(diagnostic)
|
||||
if matches!(
|
||||
diagnostic.code.as_str(),
|
||||
"working_directory_cleanup_mount_present"
|
||||
| "working_directory_cleanup_resource_busy"
|
||||
| "working_directory_cleanup_ownership_unknown"
|
||||
| "working_directory_cleanup_changes_present"
|
||||
| "working_directory_cleanup_changes_unknown"
|
||||
) =>
|
||||
{
|
||||
StatusCode::CONFLICT
|
||||
}
|
||||
RuntimeError::WorkingDirectory(diagnostic)
|
||||
if diagnostic.code == "working_directory_cleanup_permission_denied" =>
|
||||
{
|
||||
StatusCode::FORBIDDEN
|
||||
}
|
||||
RuntimeError::WorkingDirectory(diagnostic)
|
||||
if diagnostic.code == "repository_ref_not_found" =>
|
||||
{
|
||||
@@ -4322,6 +4341,37 @@ mod tests {
|
||||
assert!(!error.to_string().contains('/'));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_cause_codes_survive_http_error_serialization() {
|
||||
let cases = [
|
||||
("mount_present", StatusCode::CONFLICT),
|
||||
("mount_check_unavailable", StatusCode::SERVICE_UNAVAILABLE),
|
||||
("permission_denied", StatusCode::FORBIDDEN),
|
||||
("resource_busy", StatusCode::CONFLICT),
|
||||
("storage_unavailable", StatusCode::SERVICE_UNAVAILABLE),
|
||||
("ownership_unknown", StatusCode::CONFLICT),
|
||||
("changes_present", StatusCode::CONFLICT),
|
||||
("changes_unknown", StatusCode::CONFLICT),
|
||||
];
|
||||
for (cause, status) in cases {
|
||||
let code = format!("working_directory_cleanup_{cause}");
|
||||
let diagnostic = crate::working_directory::WorkingDirectoryDiagnostic::rejected(
|
||||
&code,
|
||||
"Resolve the current blocker and retry removal.",
|
||||
);
|
||||
let response =
|
||||
RuntimeHttpRestError::runtime(RuntimeError::WorkingDirectory(diagnostic))
|
||||
.into_response();
|
||||
assert_eq!(response.status(), status);
|
||||
let body = axum::body::to_bytes(response.into_body(), 2048)
|
||||
.await
|
||||
.unwrap();
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(json["error"]["code"], code, "{json}");
|
||||
assert!(json["error"]["message"].as_str().unwrap().contains("retry"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workdir_runtime_errors_preserve_diagnostic_code() {
|
||||
let cases = [
|
||||
|
||||
@@ -819,23 +819,30 @@ impl Runtime {
|
||||
&self,
|
||||
working_directory_id: &str,
|
||||
) -> Result<CatalogWorkingDirectoryStatus, RuntimeError> {
|
||||
let backend = {
|
||||
let state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
if let Some(worker_id) = state.worker_id_for_workdir(working_directory_id) {
|
||||
return Err(RuntimeError::InvalidRequest(format!(
|
||||
"working directory {working_directory_id} is assigned to worker {worker_id}"
|
||||
)));
|
||||
// Hold occupancy authority through the side effect, preventing an
|
||||
// attachment from entering between the check and physical removal.
|
||||
// This synchronous provider call must not reenter Runtime. The Git
|
||||
// materializer does filesystem/Git work only. The global guard also
|
||||
// delays unrelated state operations during hashing/unlink; releasing it
|
||||
// would require equivalent per-workdir exclusion on ALL attachment paths,
|
||||
// not just moving the occupied check before the provider call.
|
||||
let state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
if let Some(worker_id) = state.worker_id_for_workdir(working_directory_id) {
|
||||
return Err(RuntimeError::InvalidRequest(format!(
|
||||
"working directory {working_directory_id} is assigned to worker {worker_id}"
|
||||
)));
|
||||
}
|
||||
let backend = state.execution_backend.clone().ok_or_else(|| {
|
||||
RuntimeError::ExecutionBackendUnavailable {
|
||||
message: "working directory cleanup requires an execution backend".to_string(),
|
||||
}
|
||||
state.execution_backend.clone().ok_or_else(|| {
|
||||
RuntimeError::ExecutionBackendUnavailable {
|
||||
message: "working directory cleanup requires an execution backend".to_string(),
|
||||
}
|
||||
})?
|
||||
};
|
||||
backend
|
||||
})?;
|
||||
let result = backend
|
||||
.cleanup_working_directory(working_directory_id)
|
||||
.map_err(RuntimeError::from)
|
||||
.map_err(RuntimeError::from);
|
||||
drop(state);
|
||||
result
|
||||
}
|
||||
|
||||
fn annotate_working_directory_statuses(
|
||||
@@ -7665,6 +7672,7 @@ mod tests {
|
||||
repository_accesses: Mutex<Vec<WorkingDirectoryRepositoryAccessRequest>>,
|
||||
repository_access_available: AtomicBool,
|
||||
working_directory_requests: Mutex<Vec<WorkingDirectoryRequest>>,
|
||||
cleanup_occupancy_probe: Mutex<Option<Arc<Mutex<RuntimeState>>>>,
|
||||
preserve_submission_acknowledgement_id: AtomicBool,
|
||||
#[cfg(feature = "ws-server")]
|
||||
snapshots: Mutex<BTreeMap<WorkerId, protocol::Event>>,
|
||||
@@ -7705,6 +7713,22 @@ mod tests {
|
||||
}
|
||||
|
||||
impl WorkerExecutionBackend for TestExecutionBackend {
|
||||
fn cleanup_working_directory(
|
||||
&self,
|
||||
_id: &str,
|
||||
) -> Result<CatalogWorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
|
||||
if let Some(state) = self.cleanup_occupancy_probe.lock().unwrap().as_ref() {
|
||||
assert!(
|
||||
matches!(state.try_lock(), Err(std::sync::TryLockError::WouldBlock)),
|
||||
"occupancy must remain exclusive during physical cleanup"
|
||||
);
|
||||
}
|
||||
Err(WorkingDirectoryDiagnostic::rejected(
|
||||
"working_directory_cleanup_resource_busy",
|
||||
"Retry removal after releasing the resource",
|
||||
))
|
||||
}
|
||||
|
||||
fn backend_id(&self) -> &str {
|
||||
"test-execution-backend"
|
||||
}
|
||||
@@ -8043,6 +8067,22 @@ mod tests {
|
||||
.block_on(subscription.recv())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workdir_removal_keeps_occupancy_exclusive_during_provider_side_effect() {
|
||||
let (runtime, backend) = runtime_and_backend();
|
||||
*backend.cleanup_occupancy_probe.lock().unwrap() = Some(runtime.inner.clone());
|
||||
let error = runtime
|
||||
.cleanup_working_directory("workdir-test")
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(error, RuntimeError::WorkingDirectory(ref diagnostic)
|
||||
if diagnostic.code == "working_directory_cleanup_resource_busy")
|
||||
);
|
||||
// The lock is released on failure, permitting normal attachment/retry.
|
||||
assert!(runtime.inner.try_lock().is_ok());
|
||||
*backend.cleanup_occupancy_probe.lock().unwrap() = None;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_typed_intent_is_rejected_before_backend_dispatch() {
|
||||
let (runtime, backend) = runtime_and_backend();
|
||||
|
||||
@@ -19,7 +19,6 @@ use crate::catalog::{
|
||||
CreateWorkerRequest, ProfileSourceArchiveSource, RepositoryRefObservation,
|
||||
RepositoryRefObservationRequest, WorkingDirectoryAttachmentStatus,
|
||||
WorkingDirectoryRepositoryAccessRequest, WorkingDirectoryRequest, WorkingDirectoryStatus,
|
||||
WorkingDirectoryStatusKind,
|
||||
};
|
||||
use crate::config_bundle::{ConfigBundle, workspace_config_etag};
|
||||
#[cfg(feature = "ws-server")]
|
||||
@@ -3773,28 +3772,24 @@ where
|
||||
)
|
||||
})?;
|
||||
for previous in &request.previous_workdir_attachments {
|
||||
let current = materializer
|
||||
.working_directory_status(
|
||||
&previous.working_directory.summary.working_directory_id,
|
||||
)
|
||||
.map_err(|message| {
|
||||
WorkerExecutionResult::rejected(
|
||||
WorkerExecutionOperation::Restore,
|
||||
format!(
|
||||
"Persisted Worker Workdir attachment `{}` is unavailable: {message}",
|
||||
previous.alias
|
||||
),
|
||||
)
|
||||
})?;
|
||||
if current.summary.status != WorkingDirectoryStatusKind::Active {
|
||||
return Err(WorkerExecutionResult::rejected(
|
||||
let relative_cwd = request
|
||||
.request
|
||||
.workdir_attachments
|
||||
.iter()
|
||||
.find(|claim| claim.alias == previous.alias)
|
||||
.and_then(|claim| claim.relative_cwd.as_deref());
|
||||
// Recovery attachments remain CleanupPending across restart.
|
||||
// Validate current bindability/identity, not the old state name.
|
||||
// Preflight is read-only; actual restore binds again below.
|
||||
materializer.preflight_bind_working_directory(
|
||||
&previous.working_directory.summary.working_directory_id,
|
||||
relative_cwd,
|
||||
).map_err(|message| {
|
||||
WorkerExecutionResult::rejected(
|
||||
WorkerExecutionOperation::Restore,
|
||||
format!(
|
||||
"Persisted Worker Workdir attachment `{}` is not active",
|
||||
previous.alias
|
||||
),
|
||||
));
|
||||
}
|
||||
format!("Persisted Worker Workdir attachment `{}` is unavailable: {message}", previous.alias),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
}
|
||||
if request.previous_workdir_attachments.is_empty()
|
||||
@@ -4637,7 +4632,7 @@ mod tests {
|
||||
ConfigBundleRef, CreateWorkerRequest, LogicalWorkdirAttachment, MaterializerKind,
|
||||
ProfileSelector, RepositorySelector, WorkingDirectoryAttachmentClaim,
|
||||
WorkingDirectoryAttachmentRequest, WorkingDirectoryRepository, WorkingDirectoryRequest,
|
||||
WorkspaceApiRef,
|
||||
WorkingDirectoryStatusKind, WorkspaceApiRef,
|
||||
};
|
||||
use crate::execution::WorkerExecutionContext;
|
||||
use crate::identity::WorkerId;
|
||||
@@ -8919,6 +8914,160 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn pending_owner_restore_fixture() -> (
|
||||
tempfile::TempDir,
|
||||
WorkerRuntimeExecutionBackend<MockFactory>,
|
||||
WorkerExecutionRestoreRequest,
|
||||
PathBuf,
|
||||
Arc<Mutex<Vec<PathBuf>>>,
|
||||
) {
|
||||
use crate::working_directory::{WorkingDirectoryMaterializer, fail_workdir_cleanup_once};
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let repo = create_clean_repo();
|
||||
let workdirs = root.path().join("workdirs");
|
||||
let materializer = RuntimeGitMaterializer::new(&workdirs);
|
||||
let binding = materializer
|
||||
.create(&working_directory_request(repo.path()))
|
||||
.unwrap();
|
||||
let id = binding.working_directory.id.clone();
|
||||
fail_workdir_cleanup_once(&materializer, &id);
|
||||
// The cleanup owner has legitimately reattached to preserve these changes.
|
||||
fs::write(binding.root.join("README.md"), "owner recovery in progress").unwrap();
|
||||
let observed = Arc::new(Mutex::new(Vec::new()));
|
||||
let factory = || MockFactory {
|
||||
client: MockClient::new(simple_text_events()),
|
||||
runtime_base: root.path().join("runtime"),
|
||||
cwd: root.path().to_path_buf(),
|
||||
store_dir: root.path().join("sessions"),
|
||||
worker_metadata_dir: root.path().join("workers"),
|
||||
observed_cwds: observed.clone(),
|
||||
observed_workspace_clients: Arc::new(Mutex::new(Vec::new())),
|
||||
};
|
||||
let backend = Arc::new(
|
||||
WorkerRuntimeExecutionBackend::new(factory())
|
||||
.unwrap()
|
||||
.with_working_directory_materializer(materializer),
|
||||
);
|
||||
let runtime =
|
||||
EmbeddedRuntime::with_execution_backend(RuntimeOptions::default(), backend.clone())
|
||||
.unwrap();
|
||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||
let mut request = create_request("cleanup-owner");
|
||||
request.workdir_attachments = vec![WorkingDirectoryAttachmentClaim {
|
||||
alias: WorkdirAttachmentAlias::new("checkout").unwrap(),
|
||||
working_directory_id: id,
|
||||
relative_cwd: None,
|
||||
capabilities: WorkdirSessionCapabilities::ALL,
|
||||
}];
|
||||
let detail = runtime.create_worker(request.clone()).unwrap();
|
||||
runtime.stop_worker(&detail.worker_ref, None).unwrap();
|
||||
let restore = WorkerExecutionRestoreRequest {
|
||||
operation_id: crate::execution::WorkerLifecycleOperationId::new(),
|
||||
worker_ref: detail.worker_ref.clone(),
|
||||
request,
|
||||
workspace_scope: None,
|
||||
context: test_execution_context(detail.worker_ref),
|
||||
previous_workdir_attachments: detail.workdir_attachments,
|
||||
logical_workdir_attachments: Vec::new(),
|
||||
workdir_attachments: BTreeMap::new(),
|
||||
config_bundle: None,
|
||||
};
|
||||
drop(runtime);
|
||||
drop(backend);
|
||||
// A fresh backend/materializer loads the durable pending identity evidence.
|
||||
let restarted = WorkerRuntimeExecutionBackend::new(factory())
|
||||
.unwrap()
|
||||
.with_working_directory_materializer(RuntimeGitMaterializer::new(&workdirs));
|
||||
(root, restarted, restore, binding.root, observed)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn cleanup_pending_owner_restores_after_restart_with_validated_dirty_checkout() {
|
||||
let (root, backend, request, checkout, observed) = pending_owner_restore_fixture();
|
||||
let before = persisted_files(&root.path().join("workdirs"));
|
||||
backend.preflight_restore(&request).unwrap();
|
||||
assert_eq!(
|
||||
persisted_files(&root.path().join("workdirs")),
|
||||
before,
|
||||
"preflight must not rewrite removal or Git authority"
|
||||
);
|
||||
let restored = match backend.restore_worker(request.clone()) {
|
||||
WorkerExecutionSpawnResult::Connected {
|
||||
workdir_attachments,
|
||||
..
|
||||
} => workdir_attachments,
|
||||
other => panic!("cleanup-owner restore failed: {other:?}"),
|
||||
};
|
||||
assert_eq!(restored.len(), 1);
|
||||
assert_eq!(
|
||||
restored[0].working_directory.summary.status,
|
||||
WorkingDirectoryStatusKind::CleanupPending
|
||||
);
|
||||
assert_eq!(
|
||||
restored[0].working_directory.summary.cleanliness.as_deref(),
|
||||
Some("dirty")
|
||||
);
|
||||
assert_eq!(
|
||||
observed.lock().unwrap().as_slice(),
|
||||
&[checkout.clone(), checkout.clone()]
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read_to_string(checkout.join("README.md")).unwrap(),
|
||||
"owner recovery in progress"
|
||||
);
|
||||
assert!(backend.stop_worker(&request.worker_ref).is_accepted());
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn cleanup_pending_owner_restore_rejects_incomplete_replaced_and_raced_checkouts() {
|
||||
for damage in ["missing_git", "replacement", "missing_after_preflight"] {
|
||||
let (root, backend, request, checkout, observed) = pending_owner_restore_fixture();
|
||||
if damage == "missing_after_preflight" {
|
||||
backend.preflight_restore(&request).unwrap();
|
||||
}
|
||||
if damage == "replacement" {
|
||||
fs::rename(&checkout, root.path().join("original-checkout")).unwrap();
|
||||
fs::create_dir(&checkout).unwrap();
|
||||
fs::create_dir(checkout.join(".git")).unwrap();
|
||||
} else {
|
||||
fs::remove_dir_all(checkout.join(".git")).unwrap();
|
||||
}
|
||||
let before = persisted_files(root.path());
|
||||
let error = backend.preflight_restore(&request).unwrap_err();
|
||||
assert_eq!(
|
||||
error.outcome,
|
||||
crate::execution::WorkerExecutionOutcome::Rejected,
|
||||
"{damage}"
|
||||
);
|
||||
assert!(
|
||||
error
|
||||
.message_or_default()
|
||||
.contains("working_directory_cleanup_"),
|
||||
"{error:?}"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
backend.restore_worker(request),
|
||||
WorkerExecutionSpawnResult::Rejected(_)
|
||||
),
|
||||
"{damage}"
|
||||
);
|
||||
assert_eq!(
|
||||
observed.lock().unwrap().len(),
|
||||
1,
|
||||
"must not start a Controller on invalid checkout: {damage}"
|
||||
);
|
||||
assert_eq!(
|
||||
persisted_files(root.path()),
|
||||
before,
|
||||
"must preserve damaged/replaced checkout: {damage}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_spawn_receives_materialized_workspace_cwd_instead_of_source_repo() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
|
||||
@@ -22,6 +22,11 @@ use std::thread::JoinHandle;
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use workdir::WorkdirSessionResource;
|
||||
|
||||
mod cleanup;
|
||||
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
pub(crate) use cleanup::tests::fail_once as fail_workdir_cleanup_once;
|
||||
|
||||
const CHECKOUT_DIR: &str = "checkout";
|
||||
const MATERIALIZATION_RECORD: &str = "materialization.json";
|
||||
const REPOSITORY_ACCESS_DIR: &str = ".repository-access";
|
||||
@@ -127,7 +132,11 @@ impl WorkingDirectoryBinding {
|
||||
working_directory.status = WorkingDirectoryStatusKind::Corrupted;
|
||||
}
|
||||
let mut summary = working_directory.status_summary();
|
||||
summary.cleanliness = if summary.status == WorkingDirectoryStatusKind::Active {
|
||||
summary.cleanliness = if matches!(
|
||||
summary.status,
|
||||
WorkingDirectoryStatusKind::Active | WorkingDirectoryStatusKind::CleanupPending
|
||||
) && binding_paths_are_available(self)
|
||||
{
|
||||
let (current_selector, current_ref, current_tree) = binding_current_revision(self);
|
||||
summary.current_selector = current_selector;
|
||||
summary.current_ref = current_ref;
|
||||
@@ -201,6 +210,24 @@ pub trait WorkingDirectoryMaterializer: Send + Sync + 'static {
|
||||
relative_cwd: Option<&str>,
|
||||
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic>;
|
||||
|
||||
/// Read-only bind validation: do not consume Repository credentials or open
|
||||
/// session resources during restore preflight. Providers must opt in with
|
||||
/// current usable-checkout validation to support non-active attachments.
|
||||
fn preflight_bind_working_directory(
|
||||
&self,
|
||||
working_directory_id: &str,
|
||||
_relative_cwd: Option<&str>,
|
||||
) -> Result<(), WorkingDirectoryDiagnostic> {
|
||||
let current = self.working_directory_status(working_directory_id)?;
|
||||
if current.summary.status != WorkingDirectoryStatusKind::Active {
|
||||
return Err(WorkingDirectoryDiagnostic::new(
|
||||
"working_directory_not_active",
|
||||
"Persisted Workdir is not active and its provider cannot validate recovery binding",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn list_working_directories(
|
||||
&self,
|
||||
) -> Result<Vec<WorkingDirectoryStatus>, WorkingDirectoryDiagnostic>;
|
||||
@@ -627,6 +654,29 @@ impl RuntimeGitMaterializer {
|
||||
})
|
||||
}
|
||||
|
||||
fn checked_binding(
|
||||
&self,
|
||||
working_directory_id: &str,
|
||||
relative_cwd: Option<&str>,
|
||||
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
|
||||
validate_working_directory_id(working_directory_id)?;
|
||||
let binding = self.read_binding(working_directory_id)?;
|
||||
if !matches!(
|
||||
binding.working_directory.status,
|
||||
WorkingDirectoryStatusKind::Active | WorkingDirectoryStatusKind::CleanupPending
|
||||
) {
|
||||
return Err(WorkingDirectoryDiagnostic::new(
|
||||
"working_directory_not_active",
|
||||
"working directory working_directory is not active",
|
||||
));
|
||||
}
|
||||
if binding.working_directory.status == WorkingDirectoryStatusKind::CleanupPending {
|
||||
cleanup::validate_recovery_binding(self, working_directory_id, &binding)?;
|
||||
}
|
||||
let cwd = validate_relative_cwd(binding.root(), relative_cwd)?;
|
||||
Ok(WorkingDirectoryBinding { cwd, ..binding })
|
||||
}
|
||||
|
||||
fn read_binding(
|
||||
&self,
|
||||
working_directory_id: &str,
|
||||
@@ -1137,17 +1187,17 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
|
||||
working_directory_id: &str,
|
||||
relative_cwd: Option<&str>,
|
||||
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
|
||||
validate_working_directory_id(working_directory_id)?;
|
||||
let binding = self.read_binding(working_directory_id)?;
|
||||
if binding.working_directory.status != WorkingDirectoryStatusKind::Active {
|
||||
return Err(WorkingDirectoryDiagnostic::new(
|
||||
"working_directory_not_active",
|
||||
"working directory working_directory is not active",
|
||||
));
|
||||
}
|
||||
let binding = self.bind_repository_access(working_directory_id, binding)?;
|
||||
let cwd = validate_relative_cwd(binding.root(), relative_cwd)?;
|
||||
Ok(WorkingDirectoryBinding { cwd, ..binding })
|
||||
let binding = self.checked_binding(working_directory_id, relative_cwd)?;
|
||||
self.bind_repository_access(working_directory_id, binding)
|
||||
}
|
||||
|
||||
fn preflight_bind_working_directory(
|
||||
&self,
|
||||
working_directory_id: &str,
|
||||
relative_cwd: Option<&str>,
|
||||
) -> Result<(), WorkingDirectoryDiagnostic> {
|
||||
self.checked_binding(working_directory_id, relative_cwd)
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
fn list_working_directories(
|
||||
@@ -1178,6 +1228,10 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if let Some(status) = cleanup::remaining_status(self, &working_directory_id) {
|
||||
statuses.push(status);
|
||||
continue;
|
||||
}
|
||||
match self.read_binding(&working_directory_id) {
|
||||
Ok(binding) => statuses.push(binding.status()),
|
||||
Err(_) => statuses.push(self.corrupted_status(&working_directory_id)),
|
||||
@@ -1197,11 +1251,24 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
|
||||
) -> Result<WorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
|
||||
validate_working_directory_id(working_directory_id)?;
|
||||
let working_directory_root = self.working_directory_root(working_directory_id);
|
||||
if !working_directory_root.exists() {
|
||||
return Err(WorkingDirectoryDiagnostic::new(
|
||||
"working_directory_not_found",
|
||||
"working directory working_directory was not found",
|
||||
));
|
||||
match fs::symlink_metadata(&working_directory_root) {
|
||||
Ok(_) => {}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
return Err(WorkingDirectoryDiagnostic::new(
|
||||
"working_directory_not_found",
|
||||
"Runtime working directory was not found",
|
||||
));
|
||||
}
|
||||
Err(error) => {
|
||||
return Err(cleanup::os_failure(
|
||||
working_directory_id,
|
||||
"lookup_cleanup_root",
|
||||
error,
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(status) = cleanup::remaining_status(self, working_directory_id) {
|
||||
return Ok(status);
|
||||
}
|
||||
match self.read_binding(working_directory_id) {
|
||||
Ok(binding) => Ok(binding.status()),
|
||||
@@ -1214,80 +1281,13 @@ impl WorkingDirectoryMaterializer for RuntimeGitMaterializer {
|
||||
working_directory_id: &str,
|
||||
) -> Result<WorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
|
||||
validate_working_directory_id(working_directory_id)?;
|
||||
let status = self.working_directory_status(working_directory_id)?;
|
||||
if status.summary.status == WorkingDirectoryStatusKind::Corrupted {
|
||||
let working_directory_root = self.working_directory_root(working_directory_id);
|
||||
if working_directory_root.exists() {
|
||||
fs::remove_dir_all(&working_directory_root).map_err(|_| {
|
||||
WorkingDirectoryDiagnostic::new(
|
||||
"working_directory_corrupted_cleanup_failed",
|
||||
"failed to remove corrupted working directory; backend-private path details were omitted",
|
||||
)
|
||||
})?;
|
||||
}
|
||||
let mut summary = status.summary;
|
||||
summary.status = WorkingDirectoryStatusKind::NotFound;
|
||||
return Ok(WorkingDirectoryStatus { summary });
|
||||
}
|
||||
let binding = self.read_binding(working_directory_id)?;
|
||||
self.cleanup(&binding)?;
|
||||
let mut summary = binding.working_directory.status_summary();
|
||||
summary.status = WorkingDirectoryStatusKind::NotFound;
|
||||
summary.cleanliness = Some("unknown".to_string());
|
||||
if binding.working_directory_root.exists() {
|
||||
fs::remove_dir_all(&binding.working_directory_root).map_err(|_| {
|
||||
WorkingDirectoryDiagnostic::new(
|
||||
"working_directory_record_cleanup_failed",
|
||||
"failed to remove working directory record; backend-private path details were omitted",
|
||||
)
|
||||
})?;
|
||||
}
|
||||
Ok(WorkingDirectoryStatus { summary })
|
||||
cleanup::remove(self, working_directory_id)
|
||||
}
|
||||
|
||||
fn cleanup(&self, binding: &WorkingDirectoryBinding) -> Result<(), WorkingDirectoryDiagnostic> {
|
||||
let mut working_directory = binding.working_directory.clone();
|
||||
let working_directory_root = binding.working_directory_root.canonicalize().map_err(|_| {
|
||||
WorkingDirectoryDiagnostic::new(
|
||||
"working_directory_cleanup_target_invalid",
|
||||
"working directory working directory root is unavailable; backend-private path details were omitted",
|
||||
)
|
||||
})?;
|
||||
let root = binding.root.canonicalize().map_err(|_| {
|
||||
WorkingDirectoryDiagnostic::new(
|
||||
"working_directory_cleanup_target_invalid",
|
||||
"working directory root is unavailable; backend-private path details were omitted",
|
||||
)
|
||||
})?;
|
||||
if !root.starts_with(&working_directory_root) {
|
||||
return Err(WorkingDirectoryDiagnostic::new(
|
||||
"working_directory_cleanup_escape_rejected",
|
||||
"working directory cleanup target is outside the working directory root",
|
||||
));
|
||||
}
|
||||
let remove_result = fs::remove_dir_all(&working_directory_root).map_err(|_| {
|
||||
WorkingDirectoryDiagnostic::new(
|
||||
"working_directory_cleanup_failed",
|
||||
"failed to remove working directory; backend-private path details were omitted",
|
||||
)
|
||||
});
|
||||
if remove_result.is_err() {
|
||||
working_directory.status = WorkingDirectoryStatusKind::CleanupPending;
|
||||
let updated = WorkingDirectoryBinding {
|
||||
working_directory,
|
||||
root: binding.root.clone(),
|
||||
cwd: binding.cwd.clone(),
|
||||
working_directory_root: binding.working_directory_root.clone(),
|
||||
command_environment: BTreeMap::new(),
|
||||
session_resources: Vec::new(),
|
||||
};
|
||||
let _ = self.write_record(&updated);
|
||||
} else {
|
||||
let _ = self
|
||||
.repository_access
|
||||
.remove_pending(&binding.working_directory.id);
|
||||
}
|
||||
remove_result
|
||||
// Reload current authority rather than trusting a stale in-memory binding.
|
||||
self.cleanup_working_directory(&binding.working_directory.id)
|
||||
.map(|_| ())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2826,6 +2826,7 @@ where
|
||||
I: IntoIterator<Item = &'a str>,
|
||||
{
|
||||
let output = Command::new("git")
|
||||
.env("GIT_OPTIONAL_LOCKS", "0")
|
||||
.arg("-C")
|
||||
.arg(repository_path)
|
||||
.args(args)
|
||||
@@ -3094,7 +3095,7 @@ mod tests {
|
||||
assert!(scheduler_inner.upgrade().is_none());
|
||||
}
|
||||
|
||||
fn git(path: &Path, args: &[&str]) {
|
||||
pub(super) fn git(path: &Path, args: &[&str]) {
|
||||
let status = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(path)
|
||||
@@ -3104,7 +3105,7 @@ mod tests {
|
||||
assert!(status.success(), "git {:?} failed", args);
|
||||
}
|
||||
|
||||
fn create_clean_repo() -> tempfile::TempDir {
|
||||
pub(super) fn create_clean_repo() -> tempfile::TempDir {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
git(dir.path(), &["init"]);
|
||||
git(
|
||||
@@ -3118,7 +3119,7 @@ mod tests {
|
||||
dir
|
||||
}
|
||||
|
||||
fn request(repo: &Path) -> WorkingDirectoryRequest {
|
||||
pub(super) fn request(repo: &Path) -> WorkingDirectoryRequest {
|
||||
WorkingDirectoryRequest {
|
||||
repository: WorkingDirectoryRepository {
|
||||
id: "repo-main".to_string(),
|
||||
@@ -3678,6 +3679,8 @@ mod tests {
|
||||
.evidence
|
||||
.host_trust_revision = Some(1);
|
||||
materializer.write_record(&ssh_backed_binding).unwrap();
|
||||
#[cfg(target_os = "linux")]
|
||||
fail_workdir_cleanup_once(&materializer, &id);
|
||||
let mut session_materialization = initial_materialization.clone();
|
||||
let session_delivery_expiry = repository_access_now_epoch_seconds() + 1;
|
||||
session_materialization
|
||||
@@ -3691,6 +3694,17 @@ mod tests {
|
||||
materialization: session_materialization,
|
||||
})
|
||||
.unwrap();
|
||||
// Restore preflight must validate without consuming this one-shot SSH
|
||||
// delivery or creating agent/session resources needed by actual bind.
|
||||
let record_before =
|
||||
fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap();
|
||||
materializer
|
||||
.preflight_bind_working_directory(&id, None)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(),
|
||||
record_before
|
||||
);
|
||||
assert_eq!(materializer.list_working_directories().unwrap().len(), 1);
|
||||
assert_eq!(
|
||||
fs::read_dir(runtime_root.path().join(".repository-agents"))
|
||||
@@ -3916,6 +3930,22 @@ mod tests {
|
||||
restored.bind_working_directory(&id, None).unwrap_err().code,
|
||||
"working_directory_remote_repository_access_required"
|
||||
);
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
// Real SSH reauthorization updated access evidence repeatedly, but
|
||||
// did not replace this clean materialization or require manual recovery.
|
||||
assert_eq!(
|
||||
restored
|
||||
.working_directory_status(&id)
|
||||
.unwrap()
|
||||
.summary
|
||||
.cleanliness
|
||||
.as_deref(),
|
||||
Some("clean")
|
||||
);
|
||||
restored.cleanup_working_directory(&id).unwrap();
|
||||
assert!(!restored.working_directory_root(&id).exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4393,7 +4423,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupted_working_directory_record_can_be_removed() {
|
||||
fn corrupted_working_directory_without_removal_authority_is_retained() {
|
||||
let runtime_root = tempfile::tempdir().unwrap();
|
||||
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
|
||||
let working_directory_id = "workdir-corrupted";
|
||||
@@ -4406,11 +4436,11 @@ mod tests {
|
||||
.unwrap();
|
||||
assert_eq!(status.summary.status, WorkingDirectoryStatusKind::Corrupted);
|
||||
|
||||
let removed = materializer
|
||||
let error = materializer
|
||||
.cleanup_working_directory(working_directory_id)
|
||||
.unwrap();
|
||||
assert_eq!(removed.summary.status, WorkingDirectoryStatusKind::NotFound);
|
||||
assert!(!root.exists());
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code, "working_directory_cleanup_ownership_unknown");
|
||||
assert!(root.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -734,8 +734,14 @@ impl WorkspaceHttpWorkdirBackend {
|
||||
))?;
|
||||
workdir_output(
|
||||
format!(
|
||||
"Workdir {workdir_id} removal disposition: {:?}",
|
||||
response.disposition
|
||||
"Workdir {workdir_id} removal disposition: {:?}. {}{}",
|
||||
response.disposition,
|
||||
removal_guidance(response.failure_category.as_deref()),
|
||||
if response.retryable {
|
||||
" After resolving the cause, repeat this WorkdirDelete with the same id and reason; current safety conditions will be rechecked."
|
||||
} else {
|
||||
""
|
||||
}
|
||||
),
|
||||
&response,
|
||||
)
|
||||
@@ -920,6 +926,48 @@ fn decode_error(error: serde_json::Error) -> ToolError {
|
||||
ToolError::ExecutionFailed(format!("decode Workspace Workdir response: {error}"))
|
||||
}
|
||||
|
||||
fn removal_guidance(category: Option<&str>) -> &'static str {
|
||||
match category {
|
||||
None => "",
|
||||
Some("mount_present") => {
|
||||
"A mount remains inside the Workdir. Confirm its owner and usage, and have that owner safely release it. Do not recursively delete mounted data or implicitly unmount external or unknown resources."
|
||||
}
|
||||
Some("mount_check_unavailable") => "Restore Runtime mount inspection before deletion.",
|
||||
Some("changes_present" | "dirty_or_unknown") => {
|
||||
"Current changes are protected; preserve or resolve them before deletion."
|
||||
}
|
||||
Some("changes_unknown") => {
|
||||
"Restore checkout inspection or preserve remaining content so current changes can be verified."
|
||||
}
|
||||
Some(
|
||||
"ownership_unknown"
|
||||
| "provider_identity_changed"
|
||||
| "authority_changed"
|
||||
| "authority_invalid",
|
||||
) => {
|
||||
"Confirm target identity and ownership with its owner; never remove a replacement or unknown resource."
|
||||
}
|
||||
Some("blocked_by_live_authority") => {
|
||||
"Release current attachments, reservations and holds through their owning authority."
|
||||
}
|
||||
Some("permission_denied" | "resource_busy" | "storage_unavailable") => {
|
||||
"Have the resource owner resolve access, usage or storage safely; do not force-clean or change shared resource permissions."
|
||||
}
|
||||
Some("runtime_unavailable" | "provider_unavailable") => {
|
||||
"Restore the Runtime connection; the registry remains until deletion is confirmed."
|
||||
}
|
||||
Some("provider_observation_unknown" | "provider_cleanup_outcome_unknown") => {
|
||||
"Deletion is not confirmed. Re-observe with normal removal after restoring provider access; the registry is retained."
|
||||
}
|
||||
Some("unsupported_target") => {
|
||||
"Ask the owner for the provider's supported cleanup operation."
|
||||
}
|
||||
Some(_) => {
|
||||
"Inspect correlated Runtime cleanup diagnostics with the owning operator and resolve the current cause."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn workdir_output<T: Serialize>(summary: String, value: &T) -> Result<ToolOutput, ToolError> {
|
||||
Ok(ToolOutput {
|
||||
summary,
|
||||
@@ -1218,6 +1266,45 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_workdir_delete_reports_safe_cause_and_exact_retry_guidance() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
||||
"working_directory_id": "wd-retry",
|
||||
"disposition": "attention_required",
|
||||
"retryable": true,
|
||||
"failure_category": "mount_present"
|
||||
}))]));
|
||||
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
||||
let output = backend
|
||||
.delete(WorkdirDeleteInput {
|
||||
working_directory_id: "wd-retry".into(),
|
||||
reason: "remove unused Workdir".into(),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(output.summary.contains("Confirm its owner"));
|
||||
assert!(
|
||||
output
|
||||
.summary
|
||||
.contains("Do not recursively delete mounted data")
|
||||
);
|
||||
assert!(output.summary.contains("same id and reason"));
|
||||
assert!(output.summary.len() <= 600);
|
||||
let content: serde_json::Value =
|
||||
serde_json::from_str(output.content.as_deref().unwrap()).unwrap();
|
||||
assert_eq!(content["failure_category"], "mount_present");
|
||||
assert_eq!(content["retryable"], true);
|
||||
let requests = client.requests();
|
||||
assert_eq!(requests[0].method, WorkspaceRequestMethod::Delete);
|
||||
assert_eq!(
|
||||
serde_json::from_str::<serde_json::Value>(requests[0].body.as_deref().unwrap())
|
||||
.unwrap()["reason"],
|
||||
"remove unused Workdir"
|
||||
);
|
||||
let unknown = removal_guidance(Some("secret /private/provider/path"));
|
||||
assert!(!unknown.contains("/private"));
|
||||
assert!(!unknown.contains("secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workspace_request_failure_is_not_reported_as_session_unavailable() {
|
||||
let error = workspace_workdir_error(WorkspaceClientError::Request(
|
||||
|
||||
@@ -11927,10 +11927,6 @@ fn cleanup_workdir_cleanliness_from_runtime(value: Option<&str>) -> CleanupWorkd
|
||||
.unwrap_or(CleanupWorkdirCleanliness::Unknown)
|
||||
}
|
||||
|
||||
fn cleanup_workdir_cleanliness_is_clean(value: CleanupWorkdirCleanliness) -> bool {
|
||||
matches!(value, CleanupWorkdirCleanliness::Clean)
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LogQuery {
|
||||
limit: Option<usize>,
|
||||
@@ -17787,9 +17783,15 @@ async fn scoped_attach_current_worker_workdir(
|
||||
code: "working_directory_not_found".to_string(),
|
||||
message: format!("unknown Workdir `{workdir_id}`"),
|
||||
})?;
|
||||
if workdir.materialization_status != "present" {
|
||||
// A failed removal must not prevent its owner from attaching a still usable
|
||||
// repository checkout to resolve current blockers. Runtime binding validates
|
||||
// the actual checkout; durable pending-removal reservations still fence this
|
||||
// mutation. Other sources and missing/corrupted materializations stay denied.
|
||||
let repository_retry = workdir.materialization_status == "pending"
|
||||
&& matches!(workdir.source, WorkdirRegistrySource::Repository { .. });
|
||||
if workdir.materialization_status != "present" && !repository_retry {
|
||||
return Err(Error::WorkdirAttachmentConflict(format!(
|
||||
"Workdir `{workdir_id}` is not active ({})",
|
||||
"Workdir `{workdir_id}` is not available for attachment ({})",
|
||||
workdir.materialization_status
|
||||
))
|
||||
.into());
|
||||
@@ -24527,18 +24529,88 @@ fn runtime_reports_workdir_not_found(result: &crate::hosts::RuntimeWorkingDirect
|
||||
fn classify_workdir_provider_error(error: &RuntimeRegistryError) -> (&'static str, bool) {
|
||||
match error {
|
||||
RuntimeRegistryError::UnknownRuntime(_) => ("runtime_unavailable", true),
|
||||
RuntimeRegistryError::RuntimeOperationFailed { code, .. }
|
||||
if code == "working_directory_unsupported" =>
|
||||
{
|
||||
("unsupported_target", false)
|
||||
RuntimeRegistryError::RuntimeOperationFailed { code, .. } => {
|
||||
let category =
|
||||
workdir_cleanup_diagnostic_category(code).unwrap_or("provider_unavailable");
|
||||
(category, category != "unsupported_target")
|
||||
}
|
||||
RuntimeRegistryError::RuntimeOperationFailed { .. } => ("provider_unavailable", true),
|
||||
RuntimeRegistryError::InvalidIdentifier { .. }
|
||||
| RuntimeRegistryError::UnknownHost(_)
|
||||
| RuntimeRegistryError::UnknownWorker { .. } => ("authority_invalid", false),
|
||||
}
|
||||
}
|
||||
|
||||
// Only bounded classifications cross the public removal boundary. Provider
|
||||
// messages may contain OS paths or credentials and are never persisted here.
|
||||
fn workdir_cleanup_failure_category(
|
||||
result: &crate::hosts::RuntimeWorkingDirectoryResult,
|
||||
) -> &'static str {
|
||||
result
|
||||
.diagnostics
|
||||
.iter()
|
||||
.find_map(|diagnostic| workdir_cleanup_diagnostic_category(&diagnostic.code))
|
||||
.unwrap_or("provider_cleanup_failed")
|
||||
}
|
||||
|
||||
fn workdir_cleanup_diagnostic_category(code: &str) -> Option<&'static str> {
|
||||
Some(match code {
|
||||
"working_directory_unsupported" => "unsupported_target",
|
||||
"working_directory_cleanup_mount_present" => "mount_present",
|
||||
"working_directory_cleanup_mount_check_unavailable" => "mount_check_unavailable",
|
||||
"working_directory_cleanup_permission_denied" => "permission_denied",
|
||||
"working_directory_cleanup_resource_busy" => "resource_busy",
|
||||
"working_directory_cleanup_storage_unavailable" => "storage_unavailable",
|
||||
"working_directory_cleanup_ownership_unknown" => "ownership_unknown",
|
||||
"working_directory_cleanup_changes_present" => "changes_present",
|
||||
"working_directory_cleanup_changes_unknown" => "changes_unknown",
|
||||
"working_directory_cleanup_target_invalid"
|
||||
| "working_directory_cleanup_escape_rejected" => "ownership_unknown",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
fn workdir_removal_failure_guidance(category: &str) -> &'static str {
|
||||
match category {
|
||||
"mount_present" => {
|
||||
"A mount remains inside the Workdir. Confirm its owner and usage, have that owner safely release it, then retry deletion. Yoi does not unmount external resources."
|
||||
}
|
||||
"mount_check_unavailable" => {
|
||||
"Runtime cannot inspect mount boundaries. Restore mount inspection before retrying deletion."
|
||||
}
|
||||
"permission_denied" => {
|
||||
"Runtime lacks permission to remove the Workdir. Have its owner resolve access safely, then retry; do not force-clean or change shared resource permissions."
|
||||
}
|
||||
"resource_busy" => {
|
||||
"A resource is busy. Have its owner stop or release it safely, then retry deletion."
|
||||
}
|
||||
"storage_unavailable" => {
|
||||
"Runtime storage is unavailable. Restore storage access, then retry deletion."
|
||||
}
|
||||
"ownership_unknown" | "provider_identity_changed" | "authority_changed" => {
|
||||
"Workdir identity or ownership could not be confirmed. Verify the target with its owner before retrying."
|
||||
}
|
||||
"changes_present" | "dirty_or_unknown" => {
|
||||
"Current Workdir changes are protected. Preserve or resolve them before retrying deletion."
|
||||
}
|
||||
"changes_unknown" => {
|
||||
"Runtime cannot verify remaining Workdir changes. Restore repository inspection or preserve remaining files before retrying."
|
||||
}
|
||||
"blocked_by_live_authority" => {
|
||||
"Workdir has a current attachment, reservation or retention hold. Release the owning authority before retrying deletion."
|
||||
}
|
||||
"runtime_unavailable" | "provider_unavailable" => {
|
||||
"Runtime is unavailable. Reconnect it, then retry deletion; the Workdir registry is retained."
|
||||
}
|
||||
"provider_observation_unknown" | "provider_cleanup_outcome_unknown" => {
|
||||
"Runtime has not confirmed deletion. Retry to re-observe the target; the Workdir registry is retained."
|
||||
}
|
||||
"unsupported_target" => "This provider does not support Workdir deletion.",
|
||||
_ => {
|
||||
"Runtime cleanup failed. Inspect correlated Runtime diagnostics, resolve the cause, then retry deletion."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum WorkdirRemovalOwnerObservation {
|
||||
Running { process_start_marker: u64 },
|
||||
@@ -24741,17 +24813,46 @@ fn execute_reserved_workdir_removal_with_provider(
|
||||
return api.config_store.commit_workdir_removal_removed(&operation);
|
||||
}
|
||||
let Some(status) = observed.working_directory.as_ref() else {
|
||||
let category = workdir_cleanup_failure_category(&observed);
|
||||
let category = if category == "provider_cleanup_failed" {
|
||||
"provider_observation_unknown"
|
||||
} else {
|
||||
category
|
||||
};
|
||||
return api.config_store.fail_workdir_removal_operation(
|
||||
&operation,
|
||||
"provider_observation_unknown",
|
||||
true,
|
||||
category,
|
||||
category != "unsupported_target",
|
||||
);
|
||||
};
|
||||
let corrupted = status.summary.status == WorkingDirectoryStatusKind::Corrupted;
|
||||
if !corrupted
|
||||
&& (status.summary.cleanliness.as_deref() != Some("clean")
|
||||
|| status.summary.status != WorkingDirectoryStatusKind::Active)
|
||||
// A provider summary is not permission to remove a different materialization.
|
||||
// Unknown/corrupted metadata is delegated to Runtime's guarded cleanup, never
|
||||
// repaired into a fabricated identity here.
|
||||
let summary = &status.summary;
|
||||
if summary.occupied_by.is_some() {
|
||||
return api.config_store.complete_workdir_removal_retained(
|
||||
&operation,
|
||||
WorkdirRemovalDisposition::Retained,
|
||||
"blocked_by_live_authority",
|
||||
);
|
||||
}
|
||||
if summary.working_directory_id != operation.working_directory_id
|
||||
|| (summary.repository_id == "unknown"
|
||||
&& summary.status != WorkingDirectoryStatusKind::Corrupted)
|
||||
|| (summary.repository_id != "unknown"
|
||||
&& crate::workdir_removal::workdir_materialization_fingerprint(
|
||||
&workdir_record_from_summary(api, &operation.runtime_id, summary),
|
||||
) != operation.materialization_fingerprint)
|
||||
{
|
||||
return api.config_store.fail_workdir_removal_operation(
|
||||
&operation,
|
||||
"provider_identity_changed",
|
||||
true,
|
||||
);
|
||||
}
|
||||
// Past lifecycle failure is not a current dirtiness observation. Runtime
|
||||
// must revalidate unknown/partial checkouts at the actual deletion boundary.
|
||||
if summary.cleanliness.as_deref() == Some("dirty") {
|
||||
return api.config_store.complete_workdir_removal_retained(
|
||||
&operation,
|
||||
WorkdirRemovalDisposition::Retained,
|
||||
@@ -24769,18 +24870,25 @@ fn execute_reserved_workdir_removal_with_provider(
|
||||
.fail_workdir_removal_operation(&operation, category, retryable);
|
||||
}
|
||||
};
|
||||
if deleted.state != InternalWorkerOperationState::Accepted
|
||||
&& !runtime_reports_workdir_not_found(&deleted)
|
||||
{
|
||||
let category = if deleted
|
||||
.diagnostics
|
||||
.iter()
|
||||
.any(|diagnostic| diagnostic.code == "working_directory_unsupported")
|
||||
{
|
||||
"unsupported_target"
|
||||
let deletion_confirmed = runtime_reports_workdir_not_found(&deleted)
|
||||
|| (deleted.state == InternalWorkerOperationState::Accepted
|
||||
&& deleted.working_directory.as_ref().is_some_and(|status| {
|
||||
status.summary.working_directory_id == operation.working_directory_id
|
||||
&& status.summary.status == WorkingDirectoryStatusKind::NotFound
|
||||
}));
|
||||
if !deletion_confirmed {
|
||||
let category = if deleted.state == InternalWorkerOperationState::Accepted {
|
||||
"provider_cleanup_outcome_unknown"
|
||||
} else {
|
||||
"provider_cleanup_failed"
|
||||
workdir_cleanup_failure_category(&deleted)
|
||||
};
|
||||
tracing::warn!(
|
||||
workdir_id = %operation.working_directory_id,
|
||||
operation_id = %operation.operation_id,
|
||||
runtime_id = %operation.runtime_id,
|
||||
category,
|
||||
"Runtime did not confirm Workdir removal; registry retained"
|
||||
);
|
||||
return api.config_store.fail_workdir_removal_operation(
|
||||
&operation,
|
||||
category,
|
||||
@@ -25066,9 +25174,7 @@ fn build_runtime_cleanup_plan(
|
||||
.unwrap_or_else(|| cleanup_workdir_cleanliness_from_registry(&record.cleanliness));
|
||||
let action = if cleanup_workdir_file_status_is_record_only(file_status) {
|
||||
CleanupTargetKind::WorkdirRecordDelete
|
||||
} else if cleanup_workdir_file_status_is_corrupted(file_status)
|
||||
|| cleanup_workdir_cleanliness_is_clean(cleanliness)
|
||||
{
|
||||
} else if !matches!(cleanliness, CleanupWorkdirCleanliness::Dirty) {
|
||||
CleanupTargetKind::WorkdirCleanCleanup
|
||||
} else {
|
||||
CleanupTargetKind::WorkdirDirtyDiscard
|
||||
@@ -25096,12 +25202,12 @@ fn build_runtime_cleanup_plan(
|
||||
} else if cleanup_workdir_file_status_is_record_only(file_status) {
|
||||
"Not-found Workdir record can be deleted from the Backend registry".to_string()
|
||||
} else if cleanup_workdir_file_status_is_corrupted(file_status) {
|
||||
"Corrupted Workdir can be deleted from Runtime storage and Backend registry"
|
||||
"Runtime will verify remaining files and removal authority for this incomplete Workdir"
|
||||
.to_string()
|
||||
} else if matches!(cleanliness, CleanupWorkdirCleanliness::Dirty) {
|
||||
"Dirty Workdir requires explicit discard confirmation before cleanup".to_string()
|
||||
"Dirty Workdir is protected; preserve or resolve current changes before retrying".to_string()
|
||||
} else if matches!(cleanliness, CleanupWorkdirCleanliness::Unknown) {
|
||||
"Workdir clean state is unknown; explicit discard confirmation is required"
|
||||
"Runtime will recheck current changes, identity and removal blockers before deleting"
|
||||
.to_string()
|
||||
} else {
|
||||
"Clean Workdir can be manually cleaned up".to_string()
|
||||
@@ -25303,7 +25409,12 @@ async fn execute_runtime_cleanup_with_context(
|
||||
target_id: candidate.target_id.clone(),
|
||||
action: candidate.action.clone(),
|
||||
status: status.to_string(),
|
||||
message: message.to_string(),
|
||||
message: removal.failure_category.as_deref().map_or_else(
|
||||
|| message.to_string(),
|
||||
|category| {
|
||||
format!("{category}: {}", workdir_removal_failure_guidance(category))
|
||||
},
|
||||
),
|
||||
});
|
||||
}
|
||||
CleanupTargetKind::WorkerDelete => {
|
||||
@@ -53019,7 +53130,7 @@ mod tests {
|
||||
};
|
||||
let source_fingerprint = repository_source_fingerprint(&repository_source);
|
||||
for (workdir_id, _, _) in &fixtures {
|
||||
runtime
|
||||
let created = runtime
|
||||
.create_working_directory(WorkingDirectoryRequest {
|
||||
repository: WorkingDirectoryRepository {
|
||||
id: "repo-test".to_string(),
|
||||
@@ -53042,6 +53153,13 @@ mod tests {
|
||||
"present",
|
||||
"clean",
|
||||
);
|
||||
api.store
|
||||
.upsert_workdir_registry(&workdir_record_from_summary(
|
||||
&api,
|
||||
provider_runtime_id,
|
||||
&created.summary,
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let provider_base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||
@@ -53591,6 +53709,91 @@ mod tests {
|
||||
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_worker_can_reattach_usable_pending_checkout_after_failed_removal_but_not_during_attempt()
|
||||
{
|
||||
let mut fixture = manual_worker_assignment_fixture().await;
|
||||
let identity =
|
||||
worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id)
|
||||
.unwrap();
|
||||
configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id);
|
||||
let api = &fixture.api;
|
||||
api.store
|
||||
.detach_worker_workdir(
|
||||
TEST_WORKSPACE_ID,
|
||||
&fixture.worker,
|
||||
Some(&fixture.docs_workdir_id),
|
||||
"release for removal",
|
||||
)
|
||||
.unwrap();
|
||||
let mut record = api
|
||||
.store
|
||||
.get_workdir_registry(TEST_WORKSPACE_ID, &fixture.docs_workdir_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
record.materialization_status = "pending".into();
|
||||
api.store.upsert_workdir_registry(&record).unwrap();
|
||||
let intent =
|
||||
workdir_removal_intent(&record, "account:owner", "remove docs checkout").unwrap();
|
||||
let pending = api
|
||||
.config_store
|
||||
.reserve_workdir_removal_operation(&intent)
|
||||
.unwrap();
|
||||
let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-attachments");
|
||||
let request = || {
|
||||
runtime_source_request(
|
||||
&identity,
|
||||
Some(&fixture.worker.worker_id),
|
||||
"POST",
|
||||
&path,
|
||||
serde_json::to_vec(
|
||||
&json!({ "alias": "docs", "working_directory_id": fixture.docs_workdir_id }),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
};
|
||||
let blocked = build_router(api.clone()).oneshot(request()).await.unwrap();
|
||||
let blocked_status = blocked.status();
|
||||
let blocked_body = to_bytes(blocked.into_body(), usize::MAX).await.unwrap();
|
||||
assert_eq!(
|
||||
blocked_status,
|
||||
StatusCode::CONFLICT,
|
||||
"{}",
|
||||
String::from_utf8_lossy(&blocked_body)
|
||||
);
|
||||
api.config_store
|
||||
.fail_workdir_removal_operation(&pending, "provider_unavailable", true)
|
||||
.unwrap();
|
||||
let attached = build_router(api.clone()).oneshot(request()).await.unwrap();
|
||||
let attached_status = attached.status();
|
||||
let attached_body = to_bytes(attached.into_body(), usize::MAX).await.unwrap();
|
||||
assert_eq!(
|
||||
attached_status,
|
||||
StatusCode::OK,
|
||||
"{}",
|
||||
String::from_utf8_lossy(&attached_body)
|
||||
);
|
||||
let attached: server_api::CurrentWorkerWorkdirAttachmentResponse =
|
||||
serde_json::from_slice(&attached_body).unwrap();
|
||||
assert!(attached.attached);
|
||||
assert_eq!(attached.working_directory_id, fixture.docs_workdir_id);
|
||||
assert_eq!(
|
||||
api.store
|
||||
.get_workdir_registry(TEST_WORKSPACE_ID, &fixture.docs_workdir_id)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.materialization_status,
|
||||
"pending"
|
||||
);
|
||||
assert!(
|
||||
api.config_store
|
||||
.workdir_removal_guards(&pending)
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|guard| guard.category == "active_attachment")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_worker_attachment_list_and_stale_detach_use_signed_identity() {
|
||||
let mut fixture = manual_worker_assignment_fixture().await;
|
||||
@@ -54272,6 +54475,18 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn confirmed_workdir_cleanup_result() -> crate::hosts::RuntimeWorkingDirectoryResult {
|
||||
workdir_removal_result(
|
||||
InternalWorkerOperationState::Rejected,
|
||||
None,
|
||||
vec![RuntimeDiagnostic {
|
||||
code: "working_directory_not_found".into(),
|
||||
severity: HostDiagnosticSeverity::Warning,
|
||||
message: "authoritatively absent".into(),
|
||||
}],
|
||||
)
|
||||
}
|
||||
|
||||
fn reserve_removal_fixture(
|
||||
api: &WorkspaceApi,
|
||||
working_directory_id: &str,
|
||||
@@ -54312,7 +54527,7 @@ mod tests {
|
||||
Some(clean_summary),
|
||||
Vec::new(),
|
||||
),
|
||||
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
|
||||
confirmed_workdir_cleanup_result(),
|
||||
);
|
||||
let removed = execute_reserved_workdir_removal_with_provider(
|
||||
&api,
|
||||
@@ -54378,7 +54593,7 @@ mod tests {
|
||||
message: "timeout".to_string(),
|
||||
}],
|
||||
),
|
||||
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
|
||||
confirmed_workdir_cleanup_result(),
|
||||
);
|
||||
let unknown = execute_reserved_workdir_removal_with_provider(
|
||||
&api,
|
||||
@@ -54400,7 +54615,7 @@ mod tests {
|
||||
Some(dirty_summary),
|
||||
Vec::new(),
|
||||
),
|
||||
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
|
||||
confirmed_workdir_cleanup_result(),
|
||||
);
|
||||
let dirty = execute_reserved_workdir_removal_with_provider(
|
||||
&api,
|
||||
@@ -54418,7 +54633,7 @@ mod tests {
|
||||
Some(clean_retry_summary),
|
||||
Vec::new(),
|
||||
),
|
||||
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
|
||||
confirmed_workdir_cleanup_result(),
|
||||
);
|
||||
let removed_after_retry = execute_reserved_workdir_removal_with_provider(
|
||||
&api,
|
||||
@@ -54444,7 +54659,7 @@ mod tests {
|
||||
Some(corrupted_summary),
|
||||
Vec::new(),
|
||||
),
|
||||
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
|
||||
confirmed_workdir_cleanup_result(),
|
||||
);
|
||||
let corrupted = execute_reserved_workdir_removal_with_provider(
|
||||
&api,
|
||||
@@ -54493,6 +54708,297 @@ mod tests {
|
||||
assert_eq!(unsupported_provider.cleanup_calls(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workdir_cleanup_failure_classification_is_bounded_and_guides_normal_retry() {
|
||||
for (code, category) in [
|
||||
("working_directory_cleanup_mount_present", "mount_present"),
|
||||
(
|
||||
"working_directory_cleanup_mount_check_unavailable",
|
||||
"mount_check_unavailable",
|
||||
),
|
||||
(
|
||||
"working_directory_cleanup_permission_denied",
|
||||
"permission_denied",
|
||||
),
|
||||
("working_directory_cleanup_resource_busy", "resource_busy"),
|
||||
(
|
||||
"working_directory_cleanup_storage_unavailable",
|
||||
"storage_unavailable",
|
||||
),
|
||||
(
|
||||
"working_directory_cleanup_ownership_unknown",
|
||||
"ownership_unknown",
|
||||
),
|
||||
(
|
||||
"working_directory_cleanup_changes_present",
|
||||
"changes_present",
|
||||
),
|
||||
(
|
||||
"working_directory_cleanup_changes_unknown",
|
||||
"changes_unknown",
|
||||
),
|
||||
(
|
||||
"arbitrary provider text /private/path",
|
||||
"provider_cleanup_failed",
|
||||
),
|
||||
] {
|
||||
let result = workdir_removal_result(
|
||||
InternalWorkerOperationState::Rejected,
|
||||
None,
|
||||
vec![RuntimeDiagnostic {
|
||||
code: code.into(),
|
||||
severity: HostDiagnosticSeverity::Error,
|
||||
message: "secret /private/path".into(),
|
||||
}],
|
||||
);
|
||||
assert_eq!(workdir_cleanup_failure_category(&result), category);
|
||||
let guidance = workdir_removal_failure_guidance(category);
|
||||
assert!(!guidance.contains("/private/path"));
|
||||
assert!(!guidance.contains("secret"));
|
||||
assert!(guidance.len() <= 500);
|
||||
assert!(guidance.contains("retry"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn workdir_removal_retry_rechecks_pending_materialization_changes_and_attachments() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
init_clean_git_workspace(workspace.path());
|
||||
let api = test_api(workspace.path()).await;
|
||||
let (operation, mut summary) = reserve_removal_fixture(&api, "cleanup-retry");
|
||||
let failed_provider = FakeWorkdirRemovalProvider::new(
|
||||
workdir_removal_result(
|
||||
InternalWorkerOperationState::Accepted,
|
||||
Some(summary.clone()),
|
||||
Vec::new(),
|
||||
),
|
||||
workdir_removal_result(
|
||||
InternalWorkerOperationState::Rejected,
|
||||
None,
|
||||
vec![RuntimeDiagnostic {
|
||||
code: "working_directory_cleanup_mount_present".into(),
|
||||
severity: HostDiagnosticSeverity::Error,
|
||||
message: "/private/mount credentials must not escape".into(),
|
||||
}],
|
||||
),
|
||||
);
|
||||
let failed = execute_reserved_workdir_removal_with_provider(
|
||||
&api,
|
||||
operation,
|
||||
false,
|
||||
&failed_provider,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(failed.retryable);
|
||||
assert_eq!(failed.failure_category.as_deref(), Some("mount_present"));
|
||||
assert!(
|
||||
api.store
|
||||
.get_workdir_registry(TEST_WORKSPACE_ID, "cleanup-retry")
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
assert!(!serde_json::to_string(&failed).unwrap().contains("/private"));
|
||||
|
||||
summary.status = WorkingDirectoryStatusKind::CleanupPending;
|
||||
summary.cleanliness = Some("dirty".into());
|
||||
let dirty_provider = FakeWorkdirRemovalProvider::new(
|
||||
workdir_removal_result(
|
||||
InternalWorkerOperationState::Accepted,
|
||||
Some(summary.clone()),
|
||||
Vec::new(),
|
||||
),
|
||||
confirmed_workdir_cleanup_result(),
|
||||
);
|
||||
let dirty =
|
||||
execute_reserved_workdir_removal_with_provider(&api, failed, false, &dirty_provider)
|
||||
.unwrap();
|
||||
assert_eq!(dirty.disposition, Some(WorkdirRemovalDisposition::Retained));
|
||||
assert!(dirty.retryable);
|
||||
assert_eq!(dirty_provider.cleanup_calls(), 0);
|
||||
|
||||
let worker = seed_cleanup_worker(&api, 731, "normal");
|
||||
seed_cleanup_link(&api, &worker, "cleanup-retry");
|
||||
let occupied_provider = FakeWorkdirRemovalProvider::new(
|
||||
workdir_removal_result(
|
||||
InternalWorkerOperationState::Accepted,
|
||||
Some(summary.clone()),
|
||||
Vec::new(),
|
||||
),
|
||||
confirmed_workdir_cleanup_result(),
|
||||
);
|
||||
let occupied =
|
||||
execute_reserved_workdir_removal_with_provider(&api, dirty, false, &occupied_provider)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
occupied.failure_category.as_deref(),
|
||||
Some("blocked_by_live_authority")
|
||||
);
|
||||
assert!(occupied.retryable);
|
||||
assert_eq!(occupied_provider.cleanup_calls(), 0);
|
||||
api.store
|
||||
.detach_worker_workdir(
|
||||
TEST_WORKSPACE_ID,
|
||||
&RuntimeWorkerRef::new("runtime-test", worker),
|
||||
Some("cleanup-retry"),
|
||||
"resolve retry attachment",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Unknown because checkout was partially removed is delegated to Runtime
|
||||
// for current change/identity protection, not denied by the old state.
|
||||
summary.cleanliness = Some("unknown".into());
|
||||
let retry_provider = FakeWorkdirRemovalProvider::new(
|
||||
workdir_removal_result(
|
||||
InternalWorkerOperationState::Accepted,
|
||||
Some(summary),
|
||||
Vec::new(),
|
||||
),
|
||||
confirmed_workdir_cleanup_result(),
|
||||
);
|
||||
let removed = execute_reserved_workdir_removal_with_provider(
|
||||
&api,
|
||||
occupied.clone(),
|
||||
false,
|
||||
&retry_provider,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(removed.operation_id, occupied.operation_id);
|
||||
assert_eq!(removed.attempt_count, 4);
|
||||
assert_eq!(
|
||||
removed.disposition,
|
||||
Some(WorkdirRemovalDisposition::Removed)
|
||||
);
|
||||
assert_eq!(retry_provider.cleanup_calls(), 1);
|
||||
assert!(
|
||||
api.store
|
||||
.get_workdir_registry(TEST_WORKSPACE_ID, "cleanup-retry")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
execute_reserved_workdir_removal_with_provider(
|
||||
&api,
|
||||
removed.clone(),
|
||||
false,
|
||||
&retry_provider
|
||||
)
|
||||
.unwrap(),
|
||||
removed
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn workdir_observation_failure_preserves_bounded_os_cause_without_removing_registry() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
init_clean_git_workspace(workspace.path());
|
||||
let api = test_api(workspace.path()).await;
|
||||
for (id, transport_error) in [
|
||||
("observation-denied", false),
|
||||
("rpc-observation-denied", true),
|
||||
] {
|
||||
let (operation, _) = reserve_removal_fixture(&api, id);
|
||||
let provider = FakeWorkdirRemovalProvider::new(
|
||||
workdir_removal_result(
|
||||
InternalWorkerOperationState::Rejected,
|
||||
None,
|
||||
vec![RuntimeDiagnostic {
|
||||
code: "working_directory_cleanup_permission_denied".into(),
|
||||
severity: HostDiagnosticSeverity::Error,
|
||||
message: "permission denied /private/path".into(),
|
||||
}],
|
||||
),
|
||||
confirmed_workdir_cleanup_result(),
|
||||
);
|
||||
if transport_error {
|
||||
*provider.observation.lock().unwrap() =
|
||||
Some(Err(RuntimeRegistryError::RuntimeOperationFailed {
|
||||
runtime_id: "runtime-test".into(),
|
||||
code: "working_directory_cleanup_permission_denied".into(),
|
||||
message: "permission denied /private/path".into(),
|
||||
}));
|
||||
}
|
||||
let result =
|
||||
execute_reserved_workdir_removal_with_provider(&api, operation, false, &provider)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
result.failure_category.as_deref(),
|
||||
Some("permission_denied")
|
||||
);
|
||||
assert!(result.retryable);
|
||||
assert_eq!(provider.cleanup_calls(), 0);
|
||||
assert!(
|
||||
!serde_json::to_string(&result)
|
||||
.unwrap()
|
||||
.contains("/private/path")
|
||||
);
|
||||
assert!(
|
||||
api.store
|
||||
.get_workdir_registry(TEST_WORKSPACE_ID, id)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn workdir_removal_keeps_registry_for_changed_identity_and_unknown_cleanup_outcome() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
init_clean_git_workspace(workspace.path());
|
||||
let api = test_api(workspace.path()).await;
|
||||
for (id, changed_identity) in [("identity-mismatch", true), ("unknown-cleanup", false)] {
|
||||
let (operation, mut summary) = reserve_removal_fixture(&api, id);
|
||||
if changed_identity {
|
||||
summary.repository_id = "another-repository".into();
|
||||
}
|
||||
let provider = FakeWorkdirRemovalProvider::new(
|
||||
workdir_removal_result(
|
||||
InternalWorkerOperationState::Accepted,
|
||||
Some(summary),
|
||||
Vec::new(),
|
||||
),
|
||||
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
|
||||
);
|
||||
let result =
|
||||
execute_reserved_workdir_removal_with_provider(&api, operation, false, &provider)
|
||||
.unwrap();
|
||||
assert!(result.retryable);
|
||||
assert_eq!(
|
||||
result.failure_category.as_deref(),
|
||||
Some(if changed_identity {
|
||||
"provider_identity_changed"
|
||||
} else {
|
||||
"provider_cleanup_outcome_unknown"
|
||||
})
|
||||
);
|
||||
assert_eq!(provider.cleanup_calls(), usize::from(!changed_identity));
|
||||
assert!(
|
||||
api.store
|
||||
.get_workdir_registry(TEST_WORKSPACE_ID, id)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_plan_offers_normal_revalidation_for_prior_failure_and_unknown() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
init_clean_git_workspace(workspace.path());
|
||||
let api = test_api(workspace.path()).await;
|
||||
for (id, state) in [
|
||||
("pending-retry", "pending"),
|
||||
("unknown-retry", "unknown"),
|
||||
("corrupt-retry", "corrupted"),
|
||||
] {
|
||||
seed_cleanup_workdir(&api, id, state, "unknown");
|
||||
}
|
||||
let plan = build_runtime_cleanup_plan(&api, "runtime-test").unwrap();
|
||||
for candidate in plan.workdirs {
|
||||
assert_eq!(candidate.action, CleanupTargetKind::WorkdirCleanCleanup);
|
||||
assert!(candidate.blocking_reason.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_orphan_recovery_runs_delayed_provider_cleanup_once() {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
@@ -54518,7 +55024,7 @@ mod tests {
|
||||
Some(clean_summary),
|
||||
Vec::new(),
|
||||
),
|
||||
workdir_removal_result(InternalWorkerOperationState::Accepted, None, Vec::new()),
|
||||
confirmed_workdir_cleanup_result(),
|
||||
)
|
||||
.with_observation_delay(std::time::Duration::from_millis(100)),
|
||||
);
|
||||
|
||||
@@ -10254,6 +10254,19 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
||||
record.workdir_id, active.worker.runtime_id, active.worker.worker_id
|
||||
)));
|
||||
}
|
||||
// Direct attachments (including ordinary recovery/reconnection)
|
||||
// must share the same atomic removal fence as reserved attachments.
|
||||
let removal_pending: bool = tx.query_row(
|
||||
"SELECT EXISTS(SELECT 1 FROM workdir_removal_operations WHERE workspace_id=?1 AND workdir_id=?2 AND state='pending')",
|
||||
params![record.workspace_id, record.workdir_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if removal_pending {
|
||||
return Err(Error::WorkdirAttachmentConflict(format!(
|
||||
"Workdir {} has a pending durable removal operation",
|
||||
record.workdir_id
|
||||
)));
|
||||
}
|
||||
let write = tx.execute(
|
||||
r#"INSERT INTO worker_workdir_links (
|
||||
workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
|
||||
|
||||
@@ -438,7 +438,9 @@ impl SqliteWorkspaceStore {
|
||||
"retained completion cannot use removed disposition".to_string(),
|
||||
));
|
||||
}
|
||||
self.finish_workdir_removal_operation(operation, disposition, false, Some(category), false)
|
||||
// Retention describes this attempt's current blockers, not a permanent
|
||||
// ban. A subsequent normal request rechecks all live authority.
|
||||
self.finish_workdir_removal_operation(operation, disposition, true, Some(category), false)
|
||||
}
|
||||
|
||||
pub fn fail_workdir_removal_operation(
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
# Workdir removal and ordinary retry
|
||||
|
||||
Use the Runtime **Workdirs** page or the normal typed Workdir removal tool to
|
||||
remove an unused repository Workdir. Deletion is authorized by the Backend's
|
||||
existing durable removal operation; it is not a registry-only hide operation.
|
||||
|
||||
A failed attempt does not require a new recovery state or a manual edit of
|
||||
`materialization.json`. Retry the same removal request after resolving the
|
||||
reported cause. Each attempt rechecks current attachments, attachment
|
||||
reservations, retention holds, materialization identity and current file changes.
|
||||
A previous clean observation is not permission to discard new changes. A dirty
|
||||
Workdir must be preserved or its changes resolved by its owner before deletion.
|
||||
When the same repository checkout is still usable, an authorized Worker may
|
||||
reattach it through normal `WorkdirAttach` to preserve or resolve changes, then
|
||||
detach and retry removal. An executing removal fences new attachments; a failed
|
||||
attempt does not permanently prevent reconnection. Missing or replaced checkout
|
||||
identity is not made usable by this permission.
|
||||
|
||||
## Interpreting a retained Workdir
|
||||
|
||||
- **Mount remains:** Git status being clean does not mean mounted resources are
|
||||
released. Confirm the mount's owner, users and data destination. Have that
|
||||
owner safely release the resource, then retry normal deletion. Yoi does not
|
||||
recursively delete mounted data or implicitly unmount external/unknown
|
||||
resources. Do not use force-clean, recursive chmod or raw recursive deletion
|
||||
to override this boundary.
|
||||
- **Permission denied / resource busy / storage unavailable:** have the owning
|
||||
operator resolve access, release the resource or restore storage. Retry the
|
||||
same normal action; registry retention is intentional until cleanup is proven.
|
||||
- **Changes present / changes cannot be verified:** preserve current files and
|
||||
restore repository inspection where possible. Do not treat a previous cleanup
|
||||
failure or missing checkout metadata as proof that all remaining files are
|
||||
expendable.
|
||||
- **Identity or ownership unavailable:** verify the Workdir target with its
|
||||
owner. Do not rewrite registry or materialization identity to bypass checks.
|
||||
- **Runtime unavailable / outcome unknown:** reconnect the Runtime and retry.
|
||||
Absence of a provider response is not authoritative not-found evidence.
|
||||
- **Attachment, reservation or hold:** release the current owning authority
|
||||
through its normal API before retrying. A stale failure does not freeze these
|
||||
conditions; newly attached Workers remain protected.
|
||||
|
||||
The Web page refreshes authoritative inventory after each attempt. Successful
|
||||
removal commits registry deletion only after Runtime confirms physical deletion
|
||||
or authoritative not-found. An unknown result leaves the registry in place so
|
||||
retry can reconcile the provider's current state.
|
||||
|
||||
## Runtime safety boundary
|
||||
|
||||
The Runtime keeps removal content/identity evidence outside the tree being
|
||||
removed. This is a factual witness for partial deletion, not a new recovery
|
||||
state or cleanup stage. Surviving files are rechecked against it; an intact,
|
||||
currently clean checkout can renew obsolete evidence without discarding newly
|
||||
ignored/untracked content. Do not edit this witness manually.
|
||||
|
||||
Mount-safe removal currently requires Linux `openat2` and `/proc` inspection.
|
||||
Unavailable inspection fails closed with `mount_check_unavailable`; there is no
|
||||
fallback to recursive deletion across an unchecked filesystem boundary.
|
||||
Removal retains the Runtime occupancy mutex through its filesystem effect.
|
||||
Unrelated Runtime state operations can therefore wait during hashing/unlink;
|
||||
releasing it would require equivalent exclusion on all attachment paths.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Public responses expose a bounded cause category and safe guidance, not arbitrary
|
||||
provider errors or host paths. For an OS failure, correlate Backend removal logs
|
||||
(`operation_id`, `workdir_id`, Runtime) with Runtime cleanup logs for that Workdir
|
||||
and cleanup action. Runtime logs retain the concrete OS error/errno for the
|
||||
owning operator. These private diagnostics must not be copied unredacted into
|
||||
public responses or Tickets.
|
||||
|
||||
This procedure is not authorization to manipulate a live mount or another
|
||||
Worker's storage. If ownership or required operator authority is unclear, stop
|
||||
and request that specific owner/operator's decision.
|
||||
@@ -0,0 +1,34 @@
|
||||
# SubWorker / Workdir execution ergonomics during T-730
|
||||
|
||||
Observed during Workdir removal implementation in a dedicated clean checkout:
|
||||
|
||||
- Children using `profile: builtin:default` were given explicit writable scope
|
||||
and command permission, but their available tools/context did not support the
|
||||
requested implementation. They performed read-only investigation and reported
|
||||
the missing capabilities. Recreating coding children with `profile: inherit`
|
||||
exposed the needed implementation/Ticket tools. No rejected capability was
|
||||
bypassed.
|
||||
- Concurrent Workdir reads sometimes returned `filesystem provider busy`.
|
||||
This is transient provider contention, not an authorization rejection. A
|
||||
child initially treated it as a stop condition; a subsequent parent turn
|
||||
clarified the difference and sequential retries proceeded normally.
|
||||
- After a user interruption, repository changes and parent tasks remained, but
|
||||
`WorkerList` no longer listed the previously active children. Their work was
|
||||
not treated as approved or complete: the parent inspected concrete diffs,
|
||||
reran Web/browser validation and delegated a fresh bounded Runtime follow-up.
|
||||
|
||||
Potential improvements:
|
||||
|
||||
1. Show effective child tool capabilities together with filesystem/command
|
||||
grants before spawn, so a writable scope is not mistaken for an available
|
||||
implementation tool surface.
|
||||
2. Distinguish retryable provider contention from capability rejection in typed
|
||||
error metadata and recommended recovery text.
|
||||
3. Make interruption disposition and retained child-session observation grants
|
||||
explicit. Preserved files are useful evidence, but they do not replace a
|
||||
committed final child report or verification of tests.
|
||||
|
||||
These observations concern development-tool ergonomics, not product removal
|
||||
requirements or permission to operate on live Workdirs/mounts. The T-730 work
|
||||
uses temporary fixtures and ordinary Git publication; no live cleanup, unmount,
|
||||
registry edits or runtime deployment were performed.
|
||||
@@ -0,0 +1,140 @@
|
||||
// Synthetic API boundary only; the production route and shell own all rendering.
|
||||
export function workdirFixture() {
|
||||
let mountReleased = false;
|
||||
const deleted = new Set<string>();
|
||||
const requests: {
|
||||
workspace: string;
|
||||
method: string;
|
||||
path: string;
|
||||
body?: unknown;
|
||||
}[] = [];
|
||||
const summary = (
|
||||
id: string,
|
||||
status = "active",
|
||||
cleanliness = "clean",
|
||||
occupied = false,
|
||||
) => ({
|
||||
working_directory_id: id,
|
||||
display_name: id === "retry-dir" ? "Cleanup retry — distributed development checkout" : id,
|
||||
source: { kind: "repository", repository_key: "main" },
|
||||
materializer_kind: "runtime_git_clone",
|
||||
status,
|
||||
cleanliness,
|
||||
current_ref: "a".repeat(40),
|
||||
current_selector: "develop",
|
||||
occupied_by: occupied
|
||||
? {
|
||||
runtime_id: "fixture-runtime",
|
||||
worker_id: "worker-1",
|
||||
display_name: "Active Worker",
|
||||
linked_at: "2026-01-01T00:00:00Z",
|
||||
}
|
||||
: null,
|
||||
});
|
||||
const items = (workspace: string) =>
|
||||
workspace === "home-empty" ? [] : [
|
||||
...(deleted.has(workspace) ? [] : [summary("retry-dir", "cleanup_pending", "unknown")]),
|
||||
summary("dirty-dir", "active", "dirty"),
|
||||
summary("occupied-dir", "active", "clean", true),
|
||||
];
|
||||
const plan = (workspace: string) => ({
|
||||
workspace_id: workspace,
|
||||
runtime_id: "fixture-runtime",
|
||||
generated_at: "2026-01-01T00:00:00Z",
|
||||
revision: "plan-1",
|
||||
digest: "digest-1",
|
||||
workers: [],
|
||||
diagnostics: [],
|
||||
workdirs: items(workspace).map((item) => ({
|
||||
target_id: `workdir:${item.working_directory_id}`,
|
||||
action: item.cleanliness === "dirty" ? "workdir_dirty_discard" : "workdir_clean_cleanup",
|
||||
workdir_id: item.working_directory_id,
|
||||
runtime_id: "fixture-runtime",
|
||||
repository_key: "main",
|
||||
reason: "Ordinary cleanup rechecks current conditions",
|
||||
linked_worker_ids: item.occupied_by ? ["worker-1"] : [],
|
||||
linked_running_worker_ids: item.occupied_by ? ["worker-1"] : [],
|
||||
running_linked: Boolean(item.occupied_by),
|
||||
pinned_linked: false,
|
||||
file_status: item.status === "cleanup_pending" ? "pending" : "present",
|
||||
cleanliness: item.cleanliness,
|
||||
blocking_reason: item.occupied_by ? "Workdir is linked to a running Worker" : null,
|
||||
})),
|
||||
});
|
||||
return async (request: Request): Promise<Response | null> => {
|
||||
const url = new URL(request.url);
|
||||
if (
|
||||
url.pathname === "/fixture-workdirs-control" && request.method === "POST"
|
||||
) {
|
||||
const control = await request.json();
|
||||
mountReleased = control.mountReleased === true;
|
||||
if (control.reset) {
|
||||
deleted.clear();
|
||||
requests.length = 0;
|
||||
}
|
||||
return Response.json({ ok: true });
|
||||
}
|
||||
if (url.pathname === "/fixture-workdirs") {
|
||||
return Response.json({ requests });
|
||||
}
|
||||
const match = url.pathname.match(/^\/api\/w\/([^/]+)(\/runtimes.*)$/);
|
||||
if (!match) return null;
|
||||
const [, workspace, path] = match;
|
||||
if (path === "/runtimes") {
|
||||
return Response.json({
|
||||
workspace_id: workspace,
|
||||
limit: 100,
|
||||
source: "fixture",
|
||||
diagnostics: [],
|
||||
items: [],
|
||||
});
|
||||
}
|
||||
if (!path.startsWith("/runtimes/fixture-runtime/")) return null;
|
||||
const body = request.method !== "GET" ? await request.json() : undefined;
|
||||
requests.push({ workspace, method: request.method, path, body });
|
||||
if (path.endsWith("/working-directories")) {
|
||||
return Response.json({
|
||||
workspace_id: workspace,
|
||||
items: items(workspace),
|
||||
diagnostics: [],
|
||||
});
|
||||
}
|
||||
if (path.endsWith("/cleanup-plan")) return Response.json(plan(workspace));
|
||||
// Baseline route uses cleanup-executions; same retained outcome for before captures.
|
||||
if (path.endsWith("/cleanup-executions")) {
|
||||
return Response.json({
|
||||
workspace_id: workspace,
|
||||
runtime_id: "fixture-runtime",
|
||||
executed_at: "2026-01-01T00:00:00Z",
|
||||
diagnostics: [],
|
||||
plan_after: plan(workspace),
|
||||
results: [{
|
||||
target_id: "workdir:retry-dir",
|
||||
action: "workdir_clean_cleanup",
|
||||
status: "retained",
|
||||
message: "Provider cleanup failed",
|
||||
}],
|
||||
});
|
||||
}
|
||||
if (
|
||||
path.endsWith("/working-directories/retry-dir") &&
|
||||
request.method === "DELETE"
|
||||
) {
|
||||
if (workspace === "home-error") {
|
||||
return Response.json({
|
||||
message: "Synthetic private host path must not render",
|
||||
}, { status: 503 });
|
||||
}
|
||||
if (mountReleased) deleted.add(workspace);
|
||||
return Response.json({
|
||||
working_directory_id: "retry-dir",
|
||||
disposition: mountReleased ? "removed" : "attention_required",
|
||||
retryable: !mountReleased,
|
||||
failure_category: mountReleased ? null : "mount_present",
|
||||
});
|
||||
}
|
||||
return Response.json({ error: "Unexpected fixture Workdir request" }, {
|
||||
status: 404,
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,456 @@
|
||||
/// <reference lib="dom" />
|
||||
import { assert, assertEquals, assertStringIncludes } from "@std/assert";
|
||||
import { dirname, fromFileUrl, join } from "@std/path";
|
||||
import { chromium, type Page } from "playwright";
|
||||
import { startOwnedProcesses, stopOwnedProcesses } from "../src/processes.ts";
|
||||
const root = join(dirname(fromFileUrl(import.meta.url)), "../../..");
|
||||
const evidence = join(root, "target/web-ux/t730-browser");
|
||||
|
||||
async function noOverflow(page: Page) {
|
||||
assertEquals(
|
||||
await page.locator("main").evaluate((main) => ({
|
||||
page: document.documentElement.scrollWidth > innerWidth,
|
||||
main: main.scrollWidth > main.clientWidth,
|
||||
})),
|
||||
{ page: false, main: false },
|
||||
);
|
||||
}
|
||||
|
||||
Deno.test("ordinary removal refreshes failed and successful inventory, supports exact retry, and preserves permission and live guards across viewports and themes", async () => {
|
||||
await Deno.mkdir(evidence, { recursive: true });
|
||||
const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 });
|
||||
const port = (listener.addr as Deno.NetAddr).port;
|
||||
listener.close();
|
||||
const url = `http://127.0.0.1:${port}`;
|
||||
const server = await startOwnedProcesses(
|
||||
[{
|
||||
id: "workdirs",
|
||||
command: Deno.execPath(),
|
||||
cwd: root,
|
||||
args: [
|
||||
"run",
|
||||
"--allow-net",
|
||||
"--allow-read",
|
||||
"tools/web-ux/browser-tests/workspace_home_fixture_server.ts",
|
||||
String(port),
|
||||
"web/workspace/build",
|
||||
],
|
||||
readyUrl: `${url}/health`,
|
||||
}],
|
||||
join(root, "tools/web-ux/scenarios/workdir-removal-light.json"),
|
||||
join(evidence, "process-logs"),
|
||||
[],
|
||||
);
|
||||
const observations: unknown[] = [];
|
||||
const injectedConsoleErrors: string[] = [];
|
||||
const injectedUnexpectedErrors: string[] = [];
|
||||
const observeInjectedFailure = (page: Page) => {
|
||||
page.on("console", (message) => {
|
||||
if (message.type() === "error") {
|
||||
injectedConsoleErrors.push(message.text());
|
||||
}
|
||||
});
|
||||
page.on(
|
||||
"pageerror",
|
||||
(error) => injectedUnexpectedErrors.push(String(error)),
|
||||
);
|
||||
page.on(
|
||||
"requestfailed",
|
||||
(request) => injectedUnexpectedErrors.push(request.url()),
|
||||
);
|
||||
};
|
||||
const control = async (value: unknown) => {
|
||||
const response = await fetch(`${url}/fixture-workdirs-control`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify(value),
|
||||
});
|
||||
assert(response.ok);
|
||||
await response.body?.cancel();
|
||||
};
|
||||
const state = async () => await (await fetch(`${url}/fixture-workdirs`)).json();
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
try {
|
||||
for (const colorScheme of ["light", "dark"] as const) {
|
||||
for (const width of [1440, 768, 390, 320]) {
|
||||
await control({ reset: true });
|
||||
const context = await browser.newContext({
|
||||
viewport: { width, height: 900 },
|
||||
colorScheme,
|
||||
reducedMotion: "reduce",
|
||||
});
|
||||
try {
|
||||
const page = await context.newPage();
|
||||
const errors: string[] = [];
|
||||
page.on("pageerror", (error) => errors.push(String(error)));
|
||||
page.on("console", (message) => {
|
||||
if (message.type() === "error") errors.push(message.text());
|
||||
});
|
||||
page.on("requestfailed", (request) => errors.push(request.url()));
|
||||
const route = `${url}/w/home-owner/settings/runtimes/fixture-runtime/workdirs`;
|
||||
await page.goto(route);
|
||||
const deletion = page.getByRole("button", {
|
||||
name: "Delete retry-dir",
|
||||
exact: true,
|
||||
});
|
||||
await deletion.waitFor();
|
||||
assert(
|
||||
await deletion.isEnabled(),
|
||||
"past pending/unknown state permanently disabled ordinary removal",
|
||||
);
|
||||
assert(
|
||||
await page.getByRole("button", {
|
||||
name: "Delete dirty-dir",
|
||||
exact: true,
|
||||
}).isDisabled(),
|
||||
);
|
||||
assert(
|
||||
await page.getByRole("button", {
|
||||
name: "Delete occupied-dir",
|
||||
exact: true,
|
||||
}).isDisabled(),
|
||||
);
|
||||
await noOverflow(page);
|
||||
assert(
|
||||
await deletion.evaluate((button) => {
|
||||
const r = button.getBoundingClientRect();
|
||||
return r.left >= 0 && r.right <= innerWidth;
|
||||
}),
|
||||
"Delete is outside initial viewport",
|
||||
);
|
||||
await page.screenshot({
|
||||
path: join(evidence, `${colorScheme}-${width}-initial.png`),
|
||||
});
|
||||
// Hold the actual request boundary to verify duplicate gestures cannot dispatch another DELETE.
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((resolve) => release = resolve);
|
||||
let observed!: () => void;
|
||||
const entered = new Promise<void>((resolve) => observed = resolve);
|
||||
let deletes = 0;
|
||||
await page.route(
|
||||
"**/working-directories/retry-dir",
|
||||
async (intercept) => {
|
||||
deletes++;
|
||||
observed();
|
||||
await gate;
|
||||
await intercept.continue();
|
||||
},
|
||||
);
|
||||
await deletion.focus();
|
||||
await page.keyboard.press("Enter");
|
||||
await page.getByRole("button", {
|
||||
name: "Delete retry-dir",
|
||||
exact: true,
|
||||
}).filter({ hasText: "Deleting…" }).waitFor();
|
||||
assert(await deletion.isDisabled());
|
||||
await page.keyboard.press("Enter");
|
||||
await entered;
|
||||
assertEquals(deletes, 1);
|
||||
await page.screenshot({
|
||||
path: join(evidence, `${colorScheme}-${width}-deleting.png`),
|
||||
});
|
||||
release();
|
||||
await page.getByRole("alert").filter({
|
||||
hasText: "mounted filesystem",
|
||||
}).waitFor();
|
||||
const retry = page.getByRole("button", {
|
||||
name: "Retry Delete retry-dir",
|
||||
exact: true,
|
||||
});
|
||||
await retry.waitFor();
|
||||
await page.waitForFunction(() =>
|
||||
!(document.querySelector(
|
||||
"button[aria-label='Retry Delete retry-dir']",
|
||||
) as HTMLButtonElement)?.disabled
|
||||
);
|
||||
const failed = await state();
|
||||
const beforeGets = failed.requests.filter((
|
||||
r: { workspace: string; method: string },
|
||||
) => r.workspace === "home-owner" && r.method === "GET");
|
||||
assertEquals(
|
||||
beforeGets.filter((r: { path: string }) => r.path.endsWith("/working-directories"))
|
||||
.length,
|
||||
2,
|
||||
);
|
||||
assertEquals(
|
||||
beforeGets.filter((r: { path: string }) => r.path.endsWith("/cleanup-plan")).length,
|
||||
2,
|
||||
);
|
||||
assertEquals(await page.locator("tbody tr").count(), 3);
|
||||
await noOverflow(page);
|
||||
await page.screenshot({
|
||||
path: join(evidence, `${colorScheme}-${width}-mount-failure.png`),
|
||||
});
|
||||
await control({ mountReleased: true });
|
||||
await retry.focus();
|
||||
await page.keyboard.press("Enter");
|
||||
await page.getByRole("status").filter({
|
||||
hasText: "Deletion confirmed",
|
||||
}).waitFor();
|
||||
await page.waitForFunction(() => document.querySelectorAll("tbody tr").length === 2);
|
||||
assertEquals(
|
||||
await page.getByRole("button", { name: /retry-dir/ }).count(),
|
||||
0,
|
||||
);
|
||||
assert(
|
||||
await page.getByRole("button", {
|
||||
name: "Delete dirty-dir",
|
||||
exact: true,
|
||||
}).isDisabled(),
|
||||
);
|
||||
const final = await state();
|
||||
const mutations = final.requests.filter((r: { method: string }) => r.method === "DELETE");
|
||||
assertEquals(mutations.length, 2);
|
||||
assertEquals(
|
||||
mutations[0],
|
||||
mutations[1],
|
||||
"ordinary retry changed its request",
|
||||
);
|
||||
const gets = final.requests.filter((
|
||||
r: { workspace: string; method: string },
|
||||
) => r.workspace === "home-owner" && r.method === "GET");
|
||||
assertEquals(
|
||||
gets.filter((r: { path: string }) => r.path.endsWith("/working-directories")).length,
|
||||
3,
|
||||
);
|
||||
assertEquals(
|
||||
gets.filter((r: { path: string }) => r.path.endsWith("/cleanup-plan")).length,
|
||||
3,
|
||||
);
|
||||
await page.screenshot({
|
||||
path: join(evidence, `${colorScheme}-${width}-removed.png`),
|
||||
});
|
||||
await page.goto(
|
||||
`${url}/w/home-member/settings/runtimes/fixture-runtime/workdirs`,
|
||||
);
|
||||
await page.locator("tbody tr").first().waitFor();
|
||||
assertEquals(
|
||||
await page.getByRole("button", { name: /Delete/ }).count(),
|
||||
0,
|
||||
);
|
||||
await noOverflow(page);
|
||||
await page.screenshot({
|
||||
path: join(evidence, `${colorScheme}-${width}-member.png`),
|
||||
});
|
||||
await page.goto(
|
||||
`${url}/w/home-empty/settings/runtimes/fixture-runtime/workdirs`,
|
||||
);
|
||||
await page.getByText("No workdirs are visible for this Runtime.")
|
||||
.waitFor();
|
||||
await noOverflow(page);
|
||||
assertEquals(errors, []);
|
||||
observations.push({
|
||||
colorScheme,
|
||||
width,
|
||||
requests: final.requests,
|
||||
errors,
|
||||
});
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Lost/HTTP failure responses still trigger fresh inventory and eligibility. Never echo provider bodies.
|
||||
await control({ reset: true });
|
||||
const page = await browser.newPage();
|
||||
observeInjectedFailure(page);
|
||||
await page.goto(
|
||||
`${url}/w/home-error/settings/runtimes/fixture-runtime/workdirs`,
|
||||
);
|
||||
await page.getByRole("button", { name: "Delete retry-dir", exact: true })
|
||||
.click();
|
||||
await page.getByRole("alert").filter({ hasText: "provider is unavailable" })
|
||||
.waitFor();
|
||||
await page.waitForFunction(() =>
|
||||
!(document.querySelector(
|
||||
"button[aria-label='Delete retry-dir']",
|
||||
) as HTMLButtonElement)?.disabled
|
||||
);
|
||||
assert(
|
||||
!(await page.locator("main").innerText()).includes("Synthetic private"),
|
||||
);
|
||||
const failed = await state();
|
||||
assertEquals(
|
||||
failed.requests.filter((r: { method: string; path: string }) =>
|
||||
r.method === "GET" && r.path.endsWith("/working-directories")
|
||||
).length,
|
||||
2,
|
||||
);
|
||||
assertEquals(
|
||||
failed.requests.filter((r: { method: string; path: string }) =>
|
||||
r.method === "GET" && r.path.endsWith("/cleanup-plan")
|
||||
).length,
|
||||
2,
|
||||
);
|
||||
await page.screenshot({ path: join(evidence, "http-failure.png") });
|
||||
await page.close();
|
||||
|
||||
// Plan refresh failure cannot leave stale delete authority enabled; manual Refresh recovers it.
|
||||
await control({ reset: true });
|
||||
const stale = await browser.newPage();
|
||||
observeInjectedFailure(stale);
|
||||
let planGets = 0;
|
||||
let planFails = true;
|
||||
await stale.route("**/cleanup-plan", async (route) => {
|
||||
planGets++;
|
||||
if (planGets > 1 && planFails) {
|
||||
await route.fulfill({
|
||||
status: 503,
|
||||
json: { message: "Synthetic private path" },
|
||||
});
|
||||
} else await route.continue();
|
||||
});
|
||||
await stale.goto(
|
||||
`${url}/w/home-owner/settings/runtimes/fixture-runtime/workdirs`,
|
||||
);
|
||||
await stale.getByRole("button", { name: "Delete retry-dir", exact: true })
|
||||
.click();
|
||||
await stale.getByRole("alert").filter({ hasText: "could not be refreshed" })
|
||||
.waitFor();
|
||||
assert(
|
||||
await stale.getByRole("button", {
|
||||
name: "Retry Delete retry-dir",
|
||||
exact: true,
|
||||
}).isDisabled(),
|
||||
);
|
||||
assert(
|
||||
!(await stale.locator("main").innerText()).includes("Synthetic private"),
|
||||
);
|
||||
await stale.screenshot({ path: join(evidence, "refresh-failure.png") });
|
||||
planFails = false;
|
||||
await stale.getByRole("button", { name: "Refresh", exact: true }).click();
|
||||
await stale.waitForFunction(() =>
|
||||
!(document.querySelector(
|
||||
"button[aria-label='Retry Delete retry-dir']",
|
||||
) as HTMLButtonElement)?.disabled
|
||||
);
|
||||
await stale.close();
|
||||
|
||||
// Current dirty changes after a failure override a retryable historical result.
|
||||
await control({ reset: true });
|
||||
const changed = await browser.newPage();
|
||||
let changedGets = 0;
|
||||
await changed.route("**/cleanup-plan", async (route) => {
|
||||
const response = await route.fetch();
|
||||
const plan = await response.json();
|
||||
if (++changedGets > 1) plan.workdirs[0].action = "workdir_dirty_discard";
|
||||
await route.fulfill({ response, json: plan });
|
||||
});
|
||||
await changed.goto(
|
||||
`${url}/w/home-owner/settings/runtimes/fixture-runtime/workdirs`,
|
||||
);
|
||||
await changed.getByRole("button", { name: "Delete retry-dir", exact: true })
|
||||
.click();
|
||||
await changed.getByRole("button", {
|
||||
name: "Retry Delete retry-dir",
|
||||
exact: true,
|
||||
}).waitFor();
|
||||
await changed.waitForFunction(() =>
|
||||
document.querySelector(".workdir-identity .removal-guard")?.textContent
|
||||
?.includes("Changes are protected")
|
||||
);
|
||||
assert(
|
||||
await changed.getByRole("button", {
|
||||
name: "Retry Delete retry-dir",
|
||||
exact: true,
|
||||
}).isDisabled(),
|
||||
);
|
||||
assertStringIncludes(
|
||||
await changed.locator("main").innerText(),
|
||||
"Changes are protected",
|
||||
);
|
||||
await changed.screenshot({
|
||||
path: join(evidence, "new-changes-protected.png"),
|
||||
});
|
||||
await changed.close();
|
||||
// A malformed response may follow a real side effect. Inventory still removes the now-absent row.
|
||||
await control({ reset: true, mountReleased: true });
|
||||
const lost = await browser.newPage();
|
||||
observeInjectedFailure(lost);
|
||||
await lost.route("**/working-directories/retry-dir", async (route) => {
|
||||
await route.fetch();
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
json: {
|
||||
message: "Synthetic private path",
|
||||
failure_category: "/private/path token=synthetic",
|
||||
},
|
||||
});
|
||||
});
|
||||
await lost.goto(
|
||||
`${url}/w/home-owner/settings/runtimes/fixture-runtime/workdirs`,
|
||||
);
|
||||
await lost.getByRole("button", { name: "Delete retry-dir", exact: true })
|
||||
.click();
|
||||
await lost.getByRole("alert").filter({
|
||||
hasText: "Removal could not be confirmed",
|
||||
}).waitFor();
|
||||
await lost.waitForFunction(() => document.querySelectorAll("tbody tr").length === 2);
|
||||
assert(
|
||||
!(await lost.locator("main").innerText()).includes("Synthetic private"),
|
||||
);
|
||||
assert(!(await lost.locator("main").innerText()).includes("/private/path"));
|
||||
await lost.screenshot({
|
||||
path: join(evidence, "malformed-response-refreshed.png"),
|
||||
});
|
||||
await lost.close();
|
||||
|
||||
// Even a failed initial inventory can recover into a confirmed empty list with manual Refresh.
|
||||
const emptyRecovery = await browser.newPage();
|
||||
observeInjectedFailure(emptyRecovery);
|
||||
let inventoryFails = true;
|
||||
await emptyRecovery.route(
|
||||
"**/runtimes/fixture-runtime/working-directories",
|
||||
async (route) => {
|
||||
if (inventoryFails) {
|
||||
await route.fulfill({
|
||||
status: 503,
|
||||
json: { message: "Synthetic private path" },
|
||||
});
|
||||
} else await route.continue();
|
||||
},
|
||||
);
|
||||
await emptyRecovery.goto(
|
||||
`${url}/w/home-empty/settings/runtimes/fixture-runtime/workdirs`,
|
||||
);
|
||||
await emptyRecovery.getByText("Workdir inventory is unavailable.")
|
||||
.waitFor();
|
||||
inventoryFails = false;
|
||||
await emptyRecovery.getByRole("button", { name: "Refresh", exact: true })
|
||||
.click();
|
||||
await emptyRecovery.getByText("No workdirs are visible for this Runtime.")
|
||||
.waitFor();
|
||||
await emptyRecovery.close();
|
||||
assertEquals(injectedUnexpectedErrors, []);
|
||||
assert(
|
||||
injectedConsoleErrors.every((message) => message.includes("503")),
|
||||
"unexpected injected failure console diagnostic",
|
||||
);
|
||||
await Deno.writeTextFile(
|
||||
join(evidence, "evidence.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
observations,
|
||||
injectedConsoleErrors,
|
||||
injectedUnexpectedErrors,
|
||||
failureRefresh: failed.requests,
|
||||
refreshFailureRecovered: true,
|
||||
newChangesProtected: true,
|
||||
malformedResponseReconciled: true,
|
||||
emptyInventoryRecovered: true,
|
||||
expectedFailureDiagnostics: [
|
||||
"Synthetic DELETE HTTP 503 on home-error",
|
||||
"Synthetic cleanup-plan HTTP 503 after removal",
|
||||
"Malformed DELETE result after fixture physical removal",
|
||||
"Synthetic initial inventory HTTP 503",
|
||||
],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
} finally {
|
||||
await browser.close();
|
||||
assertEquals(await stopOwnedProcesses(server), []);
|
||||
}
|
||||
});
|
||||
@@ -2,6 +2,8 @@ import { extname, join, normalize } from "jsr:@std/path@1.1.4";
|
||||
import { dashboardFixture } from "../../../web/workspace/src/lib/workspace/home/dashboard.test-fixtures.ts";
|
||||
|
||||
import { settingsFixtureHandler } from "../../../web/workspace/src/lib/workspace/settings/identity.test-fixtures.ts";
|
||||
import { workdirFixture } from "./workdir_fixture.ts";
|
||||
const workdirsFixture = workdirFixture();
|
||||
const settingsFixture = settingsFixtureHandler();
|
||||
|
||||
const port = Number(Deno.args[0]);
|
||||
@@ -30,6 +32,8 @@ const mime: Record<string, string> = {
|
||||
Deno.serve({ hostname: "127.0.0.1", port }, async (request) => {
|
||||
const url = new URL(request.url);
|
||||
if (url.pathname === "/health") return new Response("ok");
|
||||
const workdirsResponse = await workdirsFixture(request);
|
||||
if (workdirsResponse) return workdirsResponse;
|
||||
const settingsResponse = await settingsFixture(request);
|
||||
if (settingsResponse) return settingsResponse;
|
||||
if (url.pathname === "/api/workspaces") {
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
"web-ux": "deno run --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys cli.ts",
|
||||
"check": "deno check cli.ts src/*.ts tests/*.ts browser-tests/*.ts",
|
||||
"test": "deno test --allow-env --allow-read --allow-write --allow-run --allow-sys tests",
|
||||
"test:browser": "deno test --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys browser-tests/ticket_shell_test.ts browser-tests/capture_smoke_test.ts browser-tests/memory_document_shell_test.ts browser-tests/console_history_shell_test.ts browser-tests/workspace_home_shell_test.ts browser-tests/workspace_settings_shell_test.ts browser-tests/console_pending_shell_test.ts browser-tests/console_live_shell_test.ts browser-tests/console_live_admission_shell_test.ts"
|
||||
"test:browser": "deno test --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys browser-tests/ticket_shell_test.ts browser-tests/capture_smoke_test.ts browser-tests/memory_document_shell_test.ts browser-tests/console_history_shell_test.ts browser-tests/workspace_home_shell_test.ts browser-tests/workspace_settings_shell_test.ts browser-tests/workdir_removal_shell_test.ts browser-tests/console_pending_shell_test.ts browser-tests/console_live_shell_test.ts browser-tests/console_live_admission_shell_test.ts"
|
||||
},
|
||||
"fmt": {
|
||||
"lineWidth": 100,
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"id": "workdir-removal-dark",
|
||||
"title": "Ordinary Workdir removal and guarded retry",
|
||||
"baseUrl": "http://127.0.0.1:15290",
|
||||
"colorScheme": "dark",
|
||||
"reducedMotion": "reduce",
|
||||
"personas": [
|
||||
{
|
||||
"id": "fixture",
|
||||
"label": "Synthetic permission projections",
|
||||
"auth": {
|
||||
"kind": "anonymous"
|
||||
}
|
||||
}
|
||||
],
|
||||
"viewports": [
|
||||
{
|
||||
"label": "desktop",
|
||||
"width": 1440,
|
||||
"height": 900
|
||||
},
|
||||
{
|
||||
"label": "narrow",
|
||||
"width": 768,
|
||||
"height": 900
|
||||
},
|
||||
{
|
||||
"label": "mobile",
|
||||
"width": 390,
|
||||
"height": 844
|
||||
}
|
||||
],
|
||||
"routes": [
|
||||
{
|
||||
"id": "owner",
|
||||
"label": "Owner removal",
|
||||
"path": "/w/home-owner/settings/runtimes/fixture-runtime/workdirs",
|
||||
"goal": "Remove a failed-cleanup Workdir using ordinary retry without enabling dirty or occupied removal.",
|
||||
"dataState": "Pending/unknown retry plus dirty and occupied guards. First attempt finds a mount.",
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": "main table tbody tr"
|
||||
},
|
||||
"capturePoints": [
|
||||
{
|
||||
"id": "initial",
|
||||
"label": "Current inventory"
|
||||
},
|
||||
{
|
||||
"id": "failed-attempt",
|
||||
"label": "Removal blocked by mount",
|
||||
"interaction": [
|
||||
{
|
||||
"action": "click",
|
||||
"selector": "button[aria-label='Delete retry-dir']"
|
||||
}
|
||||
],
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": "[role='alert']"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "member",
|
||||
"label": "No Runtime management permission",
|
||||
"path": "/w/home-member/settings/runtimes/fixture-runtime/workdirs",
|
||||
"goal": "Read inventory without owner-only removal controls.",
|
||||
"dataState": "Management permission false.",
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": "main table tbody tr"
|
||||
},
|
||||
"capturePoints": [
|
||||
{
|
||||
"id": "initial",
|
||||
"label": "Read only inventory"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "empty",
|
||||
"label": "Empty inventory",
|
||||
"path": "/w/home-empty/settings/runtimes/fixture-runtime/workdirs",
|
||||
"goal": "Recognize no remaining Workdirs.",
|
||||
"dataState": "Empty list.",
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": "main:has-text('No workdirs')"
|
||||
},
|
||||
"capturePoints": [
|
||||
{
|
||||
"id": "initial",
|
||||
"label": "Empty inventory"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"processes": [
|
||||
{
|
||||
"id": "workdir-fixture",
|
||||
"command": "deno",
|
||||
"cwd": "..",
|
||||
"args": [
|
||||
"run",
|
||||
"--allow-net",
|
||||
"--allow-read",
|
||||
"browser-tests/workspace_home_fixture_server.ts",
|
||||
"15290",
|
||||
"../../web/workspace/build"
|
||||
],
|
||||
"readyUrl": "http://127.0.0.1:15290/health"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"id": "workdir-removal-light",
|
||||
"title": "Ordinary Workdir removal and guarded retry",
|
||||
"baseUrl": "http://127.0.0.1:15290",
|
||||
"colorScheme": "light",
|
||||
"reducedMotion": "reduce",
|
||||
"personas": [
|
||||
{
|
||||
"id": "fixture",
|
||||
"label": "Synthetic permission projections",
|
||||
"auth": { "kind": "anonymous" }
|
||||
}
|
||||
],
|
||||
"viewports": [
|
||||
{ "label": "desktop", "width": 1440, "height": 900 },
|
||||
{ "label": "narrow", "width": 768, "height": 900 },
|
||||
{ "label": "mobile", "width": 390, "height": 844 }
|
||||
],
|
||||
"routes": [
|
||||
{
|
||||
"id": "owner",
|
||||
"label": "Owner removal",
|
||||
"path": "/w/home-owner/settings/runtimes/fixture-runtime/workdirs",
|
||||
"goal": "Remove a failed-cleanup Workdir using ordinary retry without enabling dirty or occupied removal.",
|
||||
"dataState": "Pending/unknown retry plus dirty and occupied guards. First attempt finds a mount.",
|
||||
"ready": { "kind": "selector", "selector": "main table tbody tr" },
|
||||
"capturePoints": [
|
||||
{ "id": "initial", "label": "Current inventory" },
|
||||
{
|
||||
"id": "failed-attempt",
|
||||
"label": "Removal blocked by mount",
|
||||
"interaction": [
|
||||
{
|
||||
"action": "click",
|
||||
"selector": "button[aria-label='Delete retry-dir']"
|
||||
}
|
||||
],
|
||||
"ready": { "kind": "selector", "selector": "[role='alert']" }
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "member",
|
||||
"label": "No Runtime management permission",
|
||||
"path": "/w/home-member/settings/runtimes/fixture-runtime/workdirs",
|
||||
"goal": "Read inventory without owner-only removal controls.",
|
||||
"dataState": "Management permission false.",
|
||||
"ready": { "kind": "selector", "selector": "main table tbody tr" },
|
||||
"capturePoints": [{ "id": "initial", "label": "Read only inventory" }]
|
||||
},
|
||||
{
|
||||
"id": "empty",
|
||||
"label": "Empty inventory",
|
||||
"path": "/w/home-empty/settings/runtimes/fixture-runtime/workdirs",
|
||||
"goal": "Recognize no remaining Workdirs.",
|
||||
"dataState": "Empty list.",
|
||||
"ready": {
|
||||
"kind": "selector",
|
||||
"selector": "main:has-text('No workdirs')"
|
||||
},
|
||||
"capturePoints": [{ "id": "initial", "label": "Empty inventory" }]
|
||||
}
|
||||
],
|
||||
"processes": [
|
||||
{
|
||||
"id": "workdir-fixture",
|
||||
"command": "deno",
|
||||
"cwd": "..",
|
||||
"args": [
|
||||
"run",
|
||||
"--allow-net",
|
||||
"--allow-read",
|
||||
"browser-tests/workspace_home_fixture_server.ts",
|
||||
"15290",
|
||||
"../../web/workspace/build"
|
||||
],
|
||||
"readyUrl": "http://127.0.0.1:15290/health"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -6,7 +6,7 @@
|
||||
"dev": "deno run -A npm:vite@8.3.2 dev",
|
||||
"dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787",
|
||||
"check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json",
|
||||
"test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts",
|
||||
"test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts",
|
||||
"test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts",
|
||||
"test:browser:empty-workspace": "deno test --config ../../tools/web-ux/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/visual-review/empty-workspace.browser.test.ts",
|
||||
"test:browser:drive": "deno test --config test/drive-browser/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/drive-browser/drive.browser.test.ts",
|
||||
|
||||
@@ -7,6 +7,7 @@ import type {
|
||||
WorkingDirectoryDetailResponse,
|
||||
WorkingDirectoryListResponse,
|
||||
WorkingDirectoryOccupancy,
|
||||
WorkingDirectoryRemovalResponse,
|
||||
WorkingDirectorySource,
|
||||
WorkingDirectorySummary,
|
||||
} from "../../generated/workdir-api";
|
||||
@@ -63,6 +64,40 @@ export function parseWorkingDirectoryListResponse(
|
||||
};
|
||||
}
|
||||
|
||||
export function parseWorkingDirectoryRemovalResponse(
|
||||
value: unknown,
|
||||
): WorkingDirectoryRemovalResponse {
|
||||
const record = exactRecord(
|
||||
value,
|
||||
new Set([
|
||||
"working_directory_id",
|
||||
"disposition",
|
||||
"retryable",
|
||||
"failure_category",
|
||||
]),
|
||||
"Workdir removal response",
|
||||
);
|
||||
const working_directory_id = stringField(record, "working_directory_id");
|
||||
const category = record.failure_category;
|
||||
if (
|
||||
working_directory_id.length > 128 ||
|
||||
(category != null &&
|
||||
(typeof category !== "string" || category.length > 128))
|
||||
) {
|
||||
throw new Error("Invalid Workdir removal response");
|
||||
}
|
||||
return {
|
||||
working_directory_id,
|
||||
disposition: enumField(record, "disposition", [
|
||||
"removed",
|
||||
"retained",
|
||||
"attention_required",
|
||||
]),
|
||||
retryable: booleanField(record, "retryable", "Workdir removal response"),
|
||||
failure_category: category as string | null | undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function parseWorkingDirectoryDetailResponse(
|
||||
value: unknown,
|
||||
): WorkingDirectoryDetailResponse {
|
||||
@@ -178,13 +213,26 @@ export function parseWorkingDirectorySummary(
|
||||
function parseWorkingDirectorySource(value: unknown): WorkingDirectorySource {
|
||||
const source = exactRecord(
|
||||
value,
|
||||
new Set(["kind", "repository_key", "grant_id", "grant_permissions", "access", "content_path", "purpose"]),
|
||||
new Set([
|
||||
"kind",
|
||||
"repository_key",
|
||||
"grant_id",
|
||||
"grant_permissions",
|
||||
"access",
|
||||
"content_path",
|
||||
"purpose",
|
||||
]),
|
||||
"Workdir source",
|
||||
);
|
||||
const kind = stringField(source, "kind");
|
||||
if (kind === "workspace_config") {
|
||||
if (source.repository_key !== undefined || source.grant_id !== undefined || source.grant_permissions !== undefined) {
|
||||
throw new Error("Logical config source must not contain repository or External authority");
|
||||
if (
|
||||
source.repository_key !== undefined || source.grant_id !== undefined ||
|
||||
source.grant_permissions !== undefined
|
||||
) {
|
||||
throw new Error(
|
||||
"Logical config source must not contain repository or External authority",
|
||||
);
|
||||
}
|
||||
const access = stringField(source, "access");
|
||||
if (access !== "read_only" && access !== "read_write") {
|
||||
@@ -194,10 +242,20 @@ function parseWorkingDirectorySource(value: unknown): WorkingDirectorySource {
|
||||
if (content_path !== "/workspace-config") {
|
||||
throw new Error("Logical config must reference its WIP content entrance");
|
||||
}
|
||||
return { kind, access, content_path, purpose: stringField(source, "purpose") };
|
||||
return {
|
||||
kind,
|
||||
access,
|
||||
content_path,
|
||||
purpose: stringField(source, "purpose"),
|
||||
};
|
||||
}
|
||||
if (source.access !== undefined || source.content_path !== undefined || source.purpose !== undefined) {
|
||||
throw new Error("Filesystem Workdir source must not contain logical config metadata");
|
||||
if (
|
||||
source.access !== undefined || source.content_path !== undefined ||
|
||||
source.purpose !== undefined
|
||||
) {
|
||||
throw new Error(
|
||||
"Filesystem Workdir source must not contain logical config metadata",
|
||||
);
|
||||
}
|
||||
if (kind === "repository") {
|
||||
if (
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
import type { WorkingDirectoryRemovalResponse } from "#lib/generated/workdir-api.ts";
|
||||
import type {
|
||||
CleanupWorkdirCandidate,
|
||||
WorkingDirectorySummary,
|
||||
} from "#lib/workspace/sidebar/types.ts";
|
||||
import { workspaceApiPath } from "#lib/workspace/api/http.ts";
|
||||
import { parseWorkingDirectoryRemovalResponse } from "#lib/workspace/api/workdirs.ts";
|
||||
|
||||
// Only reviewed category codes become public copy. Never render a provider message,
|
||||
// category verbatim, OS error, or host path (including for unknown future codes).
|
||||
const causes: Readonly<Record<string, string>> = {
|
||||
mount_present:
|
||||
"A mounted filesystem remains inside this Workdir. Ask its resource owner to verify ownership and safely release it before retrying. Unknown or shared mounts must not be forcibly unmounted.",
|
||||
mount_check_unavailable:
|
||||
"The Runtime could not check for mounted filesystems. Restore mount inspection on the Runtime before retrying.",
|
||||
permission_denied:
|
||||
"Removal permission was denied. Ask the Workspace or Runtime administrator to verify the required access; do not bypass protection with a forced deletion.",
|
||||
resource_busy:
|
||||
"A resource is still in use. Ask its owner to stop using it and release it safely before retrying.",
|
||||
storage_unavailable:
|
||||
"Runtime storage is unavailable. Restore storage access and capacity before retrying.",
|
||||
ownership_unknown:
|
||||
"Resource ownership could not be verified. Ask the Runtime administrator to verify ownership before any resource is released or removed.",
|
||||
changes_present:
|
||||
"This Workdir has changes. Preserve or clean those changes before requesting removal again.",
|
||||
changes_unknown:
|
||||
"Current changes could not be checked. Restore access so the Runtime can verify change protection before removal.",
|
||||
provider_cleanup_failed:
|
||||
"The provider could not finish removal. Ask the Runtime administrator to inspect the correlated removal diagnostics and resolve the cause before retrying.",
|
||||
provider_observation_unknown:
|
||||
"The provider could not confirm the remaining Workdir. Restore provider access so the next removal can recheck its current contents.",
|
||||
provider_cleanup_outcome_unknown:
|
||||
"The provider has not confirmed deletion. Restore provider access and retry the same removal request to recheck the remaining target. The Workdir stays listed until deletion is confirmed.",
|
||||
authority_invalid:
|
||||
"The removal authority could not be verified. Ask the Workspace administrator to verify the Workdir target and owning Runtime before requesting removal again.",
|
||||
provider_identity_changed:
|
||||
"The provider target identity changed. Ask the Runtime administrator to verify the target; do not delete a replacement resource.",
|
||||
runtime_unavailable:
|
||||
"The Runtime is unavailable. Restore its connection before retrying removal.",
|
||||
provider_unavailable:
|
||||
"The Workdir provider is unavailable. Restore its connection before retrying removal.",
|
||||
unsupported_target:
|
||||
"This target does not support ordinary removal. Ask its resource owner for the supported cleanup operation.",
|
||||
blocked_by_live_authority:
|
||||
"This Workdir has an active attachment, reservation, or hold. Release it through its owning operation before requesting removal again.",
|
||||
dirty_or_unknown:
|
||||
"Current change protection prevents removal. Preserve changes or restore change inspection before requesting removal again.",
|
||||
authority_changed:
|
||||
"Removal authority or current usage changed. Refresh the inventory and resolve current attachments, reservations, or holds before requesting removal again.",
|
||||
};
|
||||
const unknownCause =
|
||||
"Removal could not be confirmed. Check the Runtime's correlated removal diagnostics and resolve the cause before requesting removal again. The Workdir remains listed until deletion is confirmed.";
|
||||
|
||||
export function removalCause(category: string | null | undefined): string {
|
||||
return category != null && Object.hasOwn(causes, category)
|
||||
? causes[category]
|
||||
: unknownCause;
|
||||
}
|
||||
|
||||
export function canRemoveWorkdir(
|
||||
workdir: WorkingDirectorySummary,
|
||||
candidate: CleanupWorkdirCandidate | undefined,
|
||||
): boolean {
|
||||
return Boolean(
|
||||
candidate && !candidate.blocking_reason && !candidate.running_linked &&
|
||||
!candidate.pinned_linked &&
|
||||
(candidate.action === "workdir_clean_cleanup" ||
|
||||
candidate.action === "workdir_record_delete") &&
|
||||
workdir.cleanliness !== "dirty" && !workdir.occupied_by,
|
||||
);
|
||||
}
|
||||
|
||||
export function removalGuard(
|
||||
workdir: WorkingDirectorySummary,
|
||||
candidate: CleanupWorkdirCandidate | undefined,
|
||||
): string {
|
||||
if (
|
||||
workdir.occupied_by || candidate?.running_linked ||
|
||||
candidate?.pinned_linked || candidate?.blocking_reason
|
||||
) {
|
||||
return "An attachment, reservation, or hold protects this Workdir. Release it through its owning operation, then refresh.";
|
||||
}
|
||||
if (
|
||||
workdir.cleanliness === "dirty" ||
|
||||
candidate?.action === "workdir_dirty_discard"
|
||||
) {
|
||||
return "Changes are protected. Preserve or clean them, then refresh before deletion.";
|
||||
}
|
||||
return "Removal eligibility is unavailable. Refresh before deletion.";
|
||||
}
|
||||
|
||||
export class WorkdirRemovalError extends Error {
|
||||
constructor(readonly category?: string) {
|
||||
super(removalCause(category));
|
||||
}
|
||||
}
|
||||
|
||||
// Read only a small public DTO. Discard HTTP error bodies without interpreting or
|
||||
// displaying them, and drain small bodies rather than aborting completed requests.
|
||||
async function removalBody(response: Response): Promise<string> {
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) return "";
|
||||
const chunks: Uint8Array[] = [];
|
||||
let bytes = 0;
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
bytes += value.byteLength;
|
||||
if (bytes > 4096) {
|
||||
await reader.cancel();
|
||||
throw new WorkdirRemovalError();
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
const body = new Uint8Array(bytes);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
body.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return new TextDecoder().decode(body);
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
export async function removeWorkdir(
|
||||
fetchFn: typeof fetch,
|
||||
workspaceId: string,
|
||||
runtimeId: string,
|
||||
workdirId: string,
|
||||
): Promise<WorkingDirectoryRemovalResponse> {
|
||||
const response = await fetchFn(
|
||||
workspaceApiPath(
|
||||
workspaceId,
|
||||
`/runtimes/${encodeURIComponent(runtimeId)}/working-directories/${
|
||||
encodeURIComponent(workdirId)
|
||||
}`,
|
||||
),
|
||||
{
|
||||
method: "DELETE",
|
||||
headers: { "content-type": "application/json" },
|
||||
// An ordinary repeated request uses the existing durable Backend removal authority.
|
||||
body: JSON.stringify({ reason: "User requested Workdir deletion" }),
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
await removalBody(response).catch(() => undefined);
|
||||
throw new WorkdirRemovalError(
|
||||
response.status === 403
|
||||
? "permission_denied"
|
||||
: response.status === 409
|
||||
? "authority_changed"
|
||||
: response.status === 503
|
||||
? "provider_unavailable"
|
||||
: undefined,
|
||||
);
|
||||
}
|
||||
const result = parseWorkingDirectoryRemovalResponse(
|
||||
JSON.parse(await removalBody(response)),
|
||||
);
|
||||
if (result.working_directory_id !== workdirId) {
|
||||
throw new WorkdirRemovalError();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
+102
-169
@@ -1,102 +1,89 @@
|
||||
<script lang="ts">
|
||||
import { pushWorkspaceAlert } from '#lib/workspace/alerts/store.ts';
|
||||
import { workspaceApiPath } from '#lib/workspace/api/http.ts';
|
||||
import { parseRuntimeCleanupExecution } from '#lib/workspace/api/runtime-workers.ts';
|
||||
import { loadJson, workspaceApiPath } from '#lib/workspace/api/http.ts';
|
||||
import { parseRuntimeCleanupPlan } from '#lib/workspace/api/runtime-workers.ts';
|
||||
import { parseWorkingDirectoryListResponse } from '#lib/workspace/api/workdirs.ts';
|
||||
import { canRemoveWorkdir, removalCause, removalGuard, removeWorkdir, WorkdirRemovalError } from '#lib/workspace/settings/workdir-removal.ts';
|
||||
import { formatCurrentWorkdirRevision } from '#lib/workspace/settings/workdir-revision.ts';
|
||||
import type {
|
||||
CleanupWorkdirCandidate,
|
||||
RuntimeCleanupPlanResponse,
|
||||
WorkingDirectorySummary,
|
||||
} from '#lib/workspace/sidebar/types.ts';
|
||||
import type { CleanupWorkdirCandidate, RuntimeCleanupPlanResponse, WorkingDirectorySummary } from '#lib/workspace/sidebar/types.ts';
|
||||
import type { PageProps } from './$types';
|
||||
|
||||
let { data }: PageProps = $props();
|
||||
let cleanupBusyTarget = $state<string | null>(null);
|
||||
let refreshing = $state(false);
|
||||
let refreshError = $state<string | null>(null);
|
||||
let cleanupPlan = $state<RuntimeCleanupPlanResponse | null>(null);
|
||||
let workdirs = $state<WorkingDirectorySummary[]>([]);
|
||||
let runtimeLabel = $derived(
|
||||
data.runtimes?.items.find((runtime) => runtime.runtime_id === data.runtimeId)?.label ?? data.runtimeId,
|
||||
);
|
||||
let inventoryLoaded = $state(false);
|
||||
let feedback = $state<{ id: string; message: string; retryable: boolean; removed: boolean } | null>(null);
|
||||
let runtimeLabel = $derived(data.runtimes?.items.find((runtime) => runtime.runtime_id === data.runtimeId)?.label ?? data.runtimeId);
|
||||
let canManage = $derived(data.workspace?.permissions.manage_runtimes === true);
|
||||
let cleanupCandidates = $derived(cleanupPlan?.workdirs ?? []);
|
||||
|
||||
$effect(() => {
|
||||
cleanupPlan = data.cleanupPlan ?? null;
|
||||
workdirs = data.workdirs?.items ?? [];
|
||||
inventoryLoaded = Boolean(data.workdirs);
|
||||
refreshError = data.workdirsError || data.cleanupPlanError ? 'Inventory or removal eligibility is unavailable. Refresh before deletion.' : null;
|
||||
feedback = null;
|
||||
});
|
||||
|
||||
function repositoryKey(workdir: WorkingDirectorySummary): string | null {
|
||||
return workdir.source.kind === 'repository' ? workdir.source.repository_key : null;
|
||||
}
|
||||
|
||||
function repositoryProvider(workdir: WorkingDirectorySummary): string | null {
|
||||
const key = repositoryKey(workdir);
|
||||
return key
|
||||
? data.repositories?.items.find((repository) => repository.repository_key === key)?.provider ?? null
|
||||
: null;
|
||||
}
|
||||
|
||||
function currentRevision(workdir: WorkingDirectorySummary): string {
|
||||
return formatCurrentWorkdirRevision(workdir, repositoryProvider(workdir));
|
||||
const key = repositoryKey(workdir);
|
||||
const provider = key ? data.repositories?.items.find((repository) => repository.repository_key === key)?.provider ?? null : null;
|
||||
return formatCurrentWorkdirRevision(workdir, provider);
|
||||
}
|
||||
|
||||
function cleanupCandidate(workdir: WorkingDirectorySummary): CleanupWorkdirCandidate | undefined {
|
||||
return cleanupCandidates.find((candidate) => candidate.workdir_id === workdir.working_directory_id);
|
||||
}
|
||||
|
||||
function isDeleteDisabled(candidate: CleanupWorkdirCandidate): boolean {
|
||||
return Boolean(candidate.blocking_reason) || candidate.action === 'workdir_dirty_discard' || cleanupBusyTarget !== null;
|
||||
function isDeleteDisabled(workdir: WorkingDirectorySummary): boolean {
|
||||
return !canManage || Boolean(refreshError) || refreshing || cleanupBusyTarget !== null || !canRemoveWorkdir(workdir, cleanupCandidate(workdir));
|
||||
}
|
||||
|
||||
function errorMessage(payload: unknown, fallback: string): string {
|
||||
if (payload && typeof payload === 'object') {
|
||||
if ('message' in payload && typeof payload.message === 'string') return payload.message;
|
||||
if ('error' in payload) {
|
||||
const error = payload.error;
|
||||
if (typeof error === 'string') return error;
|
||||
if (error && typeof error === 'object' && 'message' in error && typeof error.message === 'string') return error.message;
|
||||
}
|
||||
if ('diagnostics' in payload && Array.isArray(payload.diagnostics)) {
|
||||
const diagnostic = payload.diagnostics.find(
|
||||
(entry): entry is { message: string } => Boolean(entry) && typeof entry === 'object' && 'message' in entry && typeof entry.message === 'string',
|
||||
);
|
||||
if (diagnostic) return diagnostic.message;
|
||||
}
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
async function deleteWorkdir(workdir: WorkingDirectorySummary, candidate: CleanupWorkdirCandidate): Promise<void> {
|
||||
if (!cleanupPlan || isDeleteDisabled(candidate)) return;
|
||||
cleanupBusyTarget = candidate.target_id;
|
||||
async function refreshInventory(): Promise<void> {
|
||||
refreshing = true;
|
||||
// Capture route authority before awaiting; a departed route must not receive this result.
|
||||
const workspaceId = data.workspaceId;
|
||||
const runtimeId = data.runtimeId;
|
||||
try {
|
||||
const response = await fetch(
|
||||
workspaceApiPath(data.workspaceId, `/runtimes/${encodeURIComponent(data.runtimeId)}/cleanup-executions`),
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
expected_plan_revision: cleanupPlan.revision,
|
||||
expected_plan_digest: cleanupPlan.digest,
|
||||
worker_target_ids: [],
|
||||
workdir_target_ids: [candidate.target_id],
|
||||
confirm_dirty_discard_target_ids: [],
|
||||
}),
|
||||
},
|
||||
);
|
||||
const payload = await response.json().catch(() => null);
|
||||
if (!response.ok) throw new Error(errorMessage(payload, response.statusText));
|
||||
const execution = parseRuntimeCleanupExecution(payload);
|
||||
cleanupPlan = execution.plan_after;
|
||||
const result = execution.results.find((entry) => entry.target_id === candidate.target_id);
|
||||
if (!result || result.status !== 'deleted') {
|
||||
throw new Error(result?.message ?? 'Runtime did not delete the selected Workdir');
|
||||
}
|
||||
workdirs = workdirs.filter((item) => item.working_directory_id !== workdir.working_directory_id);
|
||||
} catch (error) {
|
||||
pushWorkspaceAlert('error', error instanceof Error ? error.message : 'Workdir deletion failed', {
|
||||
title: 'Workdir deletion failed',
|
||||
});
|
||||
const base = `/runtimes/${encodeURIComponent(runtimeId)}`;
|
||||
const [inventory, plan] = await Promise.all([
|
||||
loadJson(fetch, workspaceApiPath(workspaceId, `${base}/working-directories`), undefined, parseWorkingDirectoryListResponse),
|
||||
loadJson(fetch, workspaceApiPath(workspaceId, `${base}/cleanup-plan`), undefined, parseRuntimeCleanupPlan),
|
||||
]);
|
||||
if (workspaceId !== data.workspaceId || runtimeId !== data.runtimeId) return;
|
||||
// Keep the last observed list on failure, but never authorize another deletion with stale eligibility.
|
||||
if (inventory.data) { workdirs = inventory.data.items; inventoryLoaded = true; }
|
||||
cleanupPlan = plan.data;
|
||||
refreshError = inventory.error || plan.error ? 'Inventory or removal eligibility could not be refreshed. Refresh before another deletion.' : null;
|
||||
} finally {
|
||||
refreshing = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteWorkdir(workdir: WorkingDirectorySummary): Promise<void> {
|
||||
if (isDeleteDisabled(workdir)) return;
|
||||
const workspaceId = data.workspaceId;
|
||||
const runtimeId = data.runtimeId;
|
||||
cleanupBusyTarget = workdir.working_directory_id;
|
||||
feedback = null;
|
||||
try {
|
||||
const result = await removeWorkdir(fetch, workspaceId, runtimeId, workdir.working_directory_id);
|
||||
if (workspaceId !== data.workspaceId || runtimeId !== data.runtimeId) return;
|
||||
feedback = {
|
||||
id: workdir.working_directory_id,
|
||||
message: result.disposition === 'removed' ? 'Deletion confirmed.' : removalCause(result.failure_category),
|
||||
retryable: result.retryable && result.disposition !== 'removed',
|
||||
removed: result.disposition === 'removed',
|
||||
};
|
||||
} catch (error) {
|
||||
if (workspaceId !== data.workspaceId || runtimeId !== data.runtimeId) return;
|
||||
feedback = { id: workdir.working_directory_id, message: error instanceof WorkdirRemovalError ? error.message : removalCause(null), retryable: false, removed: false };
|
||||
} finally {
|
||||
// Even a lost/error response can follow a real provider side effect. Re-read inventory AND guards.
|
||||
if (workspaceId === data.workspaceId && runtimeId === data.runtimeId) await refreshInventory();
|
||||
cleanupBusyTarget = null;
|
||||
}
|
||||
}
|
||||
@@ -107,76 +94,55 @@
|
||||
<meta name="description" content="Runtime workdirs" />
|
||||
</svelte:head>
|
||||
|
||||
<section class="workdirs-page" aria-labelledby="workdirs-heading">
|
||||
<section class="workdirs-page" aria-labelledby="workdirs-heading" aria-busy={refreshing}>
|
||||
<header class="page-header-row">
|
||||
<div>
|
||||
<p class="breadcrumb"><a href={`/w/${data.workspaceId}/settings/runtimes`}>Runtimes</a> / {runtimeLabel}</p>
|
||||
<h1 id="workdirs-heading">Workdirs</h1>
|
||||
<p>Workdirs owned by <code>{data.runtimeId}</code>.</p>
|
||||
{#if data.cleanupPlanError}<p class="section-state error">{data.cleanupPlanError}</p>{/if}
|
||||
</div>
|
||||
<button class="workdir-action" type="button" disabled={refreshing || cleanupBusyTarget !== null} onclick={refreshInventory}>{refreshing ? 'Refreshing…' : 'Refresh'}</button>
|
||||
</header>
|
||||
|
||||
{#if data.workdirsError}
|
||||
<p class="section-state error">{data.workdirsError}</p>
|
||||
{:else if !data.workdirs}
|
||||
<p class="section-state">Loading workdirs…</p>
|
||||
{#if feedback}
|
||||
<div class="removal-feedback" class:error={!feedback.removed} role={feedback.removed ? 'status' : 'alert'}>
|
||||
<p><strong>{feedback.removed ? 'Workdir deleted' : 'Workdir not deleted'}</strong> · <code>{feedback.id}</code></p>
|
||||
<p>{feedback.message}</p>
|
||||
{#if feedback.retryable}<p>After resolving this cause, retry Delete. Current safety conditions will be rechecked.</p>{/if}
|
||||
</div>
|
||||
{/if}
|
||||
{#if refreshError}<p class="section-state error" role="alert">{refreshError}</p>{/if}
|
||||
|
||||
{#if !inventoryLoaded}
|
||||
<p class="section-state">{data.workdirsError ? 'Workdir inventory is unavailable.' : 'Loading workdirs…'}</p>
|
||||
{:else if workdirs.length === 0}
|
||||
<p class="section-state">No workdirs are visible for this Runtime.</p>
|
||||
{:else}
|
||||
<div class="table-wrap">
|
||||
<!-- svelte-ignore a11y_no_noninteractive_tabindex (Horizontal inventory scroll must be keyboard-focusable.) -->
|
||||
<div class="table-wrap" role="region" aria-label="Workdir inventory" tabindex="0">
|
||||
<table class="workdirs-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Workdir</th>
|
||||
<th>Repository</th>
|
||||
<th>Revision</th>
|
||||
<th>Status</th>
|
||||
<th>Cleanliness</th>
|
||||
<th>Occupied by</th>
|
||||
<th>Action</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<thead><tr><th>Workdir</th><th>Repository</th><th>Revision</th><th>Status</th><th>Cleanliness</th><th>Occupied by</th></tr></thead>
|
||||
<tbody>
|
||||
{#each workdirs as workdir}
|
||||
{#each workdirs as workdir (workdir.working_directory_id)}
|
||||
{@const cleanup = cleanupCandidate(workdir)}
|
||||
<tr>
|
||||
<td>
|
||||
<td class="workdir-identity">
|
||||
<span>{workdir.display_name ?? '—'}</span>
|
||||
<small><code>{workdir.working_directory_id}</code></small>
|
||||
{#if canManage}
|
||||
<button class="workdir-action danger" type="button" disabled={isDeleteDisabled(workdir)}
|
||||
aria-label={`${feedback?.id === workdir.working_directory_id && feedback.retryable ? 'Retry Delete' : 'Delete'} ${workdir.working_directory_id}`}
|
||||
onclick={() => deleteWorkdir(workdir)}>
|
||||
{cleanupBusyTarget === workdir.working_directory_id ? 'Deleting…' : feedback?.id === workdir.working_directory_id && feedback.retryable ? 'Retry Delete' : 'Delete'}
|
||||
</button>
|
||||
{#if !canRemoveWorkdir(workdir, cleanup)}<p class="removal-guard">{removalGuard(workdir, cleanup)}</p>{/if}
|
||||
{/if}
|
||||
</td>
|
||||
<td>{repositoryKey(workdir) ?? 'External'}</td>
|
||||
<td><code>{currentRevision(workdir)}</code></td>
|
||||
<td>{workdir.status}</td>
|
||||
<td>{workdir.cleanliness ?? 'unknown'}</td>
|
||||
<td>
|
||||
{#if workdir.occupied_by}
|
||||
<span>{workdir.occupied_by.display_name}</span>
|
||||
<small>{workdir.occupied_by.runtime_id}:{workdir.occupied_by.worker_id}</small>
|
||||
{:else}
|
||||
<span class="muted">—</span>
|
||||
{/if}
|
||||
</td>
|
||||
<td>
|
||||
{#if cleanup}
|
||||
<button
|
||||
class="icon-action danger"
|
||||
type="button"
|
||||
disabled={isDeleteDisabled(cleanup)}
|
||||
aria-label={`Delete ${workdir.working_directory_id}`}
|
||||
title={cleanup.action === 'workdir_dirty_discard' ? 'Dirty Workdirs must be cleaned before deletion' : (cleanup.blocking_reason ?? cleanup.reason)}
|
||||
onclick={() => deleteWorkdir(workdir, cleanup)}
|
||||
>
|
||||
{#if cleanupBusyTarget === cleanup.target_id}
|
||||
<span class="spinner" aria-hidden="true"></span>
|
||||
{:else}
|
||||
<svg class="action-icon" aria-hidden="true" viewBox="0 0 24 24"><path d="M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6" /><path d="M3 6h18" /><path d="M8 6V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2" /></svg>
|
||||
{/if}
|
||||
</button>
|
||||
{:else}
|
||||
<span class="muted">—</span>
|
||||
{/if}
|
||||
</td>
|
||||
<td>{#if workdir.occupied_by}<span>{workdir.occupied_by.display_name}</span><small>{workdir.occupied_by.runtime_id}:{workdir.occupied_by.worker_id}</small>{:else}<span class="muted">—</span>{/if}</td>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
@@ -186,53 +152,20 @@
|
||||
</section>
|
||||
|
||||
<style>
|
||||
.icon-action {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 2rem;
|
||||
height: 2rem;
|
||||
padding: 0;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 0.5rem;
|
||||
background: var(--bg-raised);
|
||||
color: var(--text);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.icon-action.danger:hover:not(:disabled),
|
||||
.icon-action.danger:focus-visible:not(:disabled) {
|
||||
border-color: var(--danger, oklch(60% 0.18 30));
|
||||
color: var(--danger, oklch(60% 0.18 30));
|
||||
}
|
||||
|
||||
.icon-action:disabled {
|
||||
cursor: not-allowed;
|
||||
opacity: 0.45;
|
||||
}
|
||||
|
||||
.action-icon {
|
||||
width: 1rem;
|
||||
height: 1rem;
|
||||
fill: none;
|
||||
stroke: currentColor;
|
||||
stroke-width: 2;
|
||||
stroke-linecap: round;
|
||||
stroke-linejoin: round;
|
||||
}
|
||||
|
||||
.spinner {
|
||||
width: 1rem;
|
||||
height: 1rem;
|
||||
border: 2px solid currentColor;
|
||||
border-right-color: transparent;
|
||||
border-radius: 999px;
|
||||
animation: workdir-action-spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes workdir-action-spin {
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
.workdirs-page { min-width: 0; }
|
||||
.page-header-row { display: flex; flex-wrap: wrap; justify-content: space-between; align-items: start; gap: var(--space-3); }
|
||||
.table-wrap { max-width: 100%; overflow-x: auto; }
|
||||
.workdirs-table { width: 100%; min-width: 54rem; border-collapse: collapse; }
|
||||
th, td { padding: var(--space-3); text-align: left; vertical-align: top; border-bottom: 1px solid var(--line); }
|
||||
th { color: var(--text-muted); font-weight: 500; }
|
||||
.workdir-identity { width: 18rem; min-width: 18rem; overflow-wrap: anywhere; }
|
||||
td small { display: block; color: var(--text-muted); margin-block: var(--space-1); }
|
||||
.workdir-action { display: inline-flex; align-items: center; justify-content: center; padding: var(--space-2) var(--space-3); border: 1px solid var(--line); border-radius: var(--radius-soft); background: var(--bg-raised); color: var(--text); cursor: pointer; white-space: nowrap; }
|
||||
.workdir-action:hover:not(:disabled) { background: var(--interactive-hover); }
|
||||
.workdir-action.danger { color: var(--danger); }
|
||||
.workdir-action:disabled { cursor: not-allowed; opacity: 0.45; }
|
||||
.removal-feedback { margin-block: var(--space-4); overflow-wrap: anywhere; }
|
||||
.removal-feedback p { margin-block: var(--space-2); }
|
||||
.removal-feedback.error strong { color: var(--danger); }
|
||||
.removal-guard { margin-block: var(--space-2) 0; color: var(--text-muted); }
|
||||
</style>
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
declare const Deno: {
|
||||
test(name: string, fn: () => void | Promise<void>): void;
|
||||
};
|
||||
import { parseWorkingDirectoryRemovalResponse } from "../src/lib/workspace/api/workdirs.ts";
|
||||
import {
|
||||
canRemoveWorkdir,
|
||||
removalCause,
|
||||
removeWorkdir,
|
||||
} from "../src/lib/workspace/settings/workdir-removal.ts";
|
||||
import type {
|
||||
CleanupWorkdirCandidate,
|
||||
WorkingDirectorySummary,
|
||||
} from "../src/lib/workspace/sidebar/types.ts";
|
||||
|
||||
function assert(value: unknown, message = "assertion failed"): asserts value {
|
||||
if (!value) throw new Error(message);
|
||||
}
|
||||
|
||||
Deno.test("removal wire boundary rejects malformed results and mismatched identities", async () => {
|
||||
for (
|
||||
const extra of [
|
||||
{ retryable: "true" },
|
||||
{ disposition: "forgotten" },
|
||||
{ failure_category: {} },
|
||||
{ failure_category: "x".repeat(129) },
|
||||
{ working_directory_id: "x".repeat(129) },
|
||||
{ unexpected: true },
|
||||
]
|
||||
) {
|
||||
let rejected = false;
|
||||
try {
|
||||
parseWorkingDirectoryRemovalResponse({
|
||||
working_directory_id: "dir",
|
||||
disposition: "retained",
|
||||
retryable: true,
|
||||
...extra,
|
||||
});
|
||||
} catch {
|
||||
rejected = true;
|
||||
}
|
||||
assert(
|
||||
rejected,
|
||||
`accepted malformed removal result: ${JSON.stringify(extra)}`,
|
||||
);
|
||||
}
|
||||
let rejected = false;
|
||||
try {
|
||||
await removeWorkdir(
|
||||
(() =>
|
||||
Promise.resolve(
|
||||
Response.json({
|
||||
working_directory_id: "other",
|
||||
disposition: "removed",
|
||||
retryable: false,
|
||||
}),
|
||||
)) as typeof fetch,
|
||||
"workspace",
|
||||
"runtime",
|
||||
"dir",
|
||||
);
|
||||
} catch {
|
||||
rejected = true;
|
||||
}
|
||||
assert(rejected, "accepted removal evidence for a different Workdir");
|
||||
});
|
||||
|
||||
Deno.test("removal response size is bounded before decoding provider content", async () => {
|
||||
let rejected = false;
|
||||
try {
|
||||
await removeWorkdir(
|
||||
(() => Promise.resolve(new Response("x".repeat(4097)))) as typeof fetch,
|
||||
"workspace",
|
||||
"runtime",
|
||||
"dir",
|
||||
);
|
||||
} catch {
|
||||
rejected = true;
|
||||
}
|
||||
assert(rejected, "accepted an oversized removal response");
|
||||
});
|
||||
|
||||
Deno.test("removal causes give bounded actions without echoing untrusted provider details", () => {
|
||||
const categories = [
|
||||
"mount_present",
|
||||
"mount_check_unavailable",
|
||||
"permission_denied",
|
||||
"resource_busy",
|
||||
"storage_unavailable",
|
||||
"ownership_unknown",
|
||||
"changes_present",
|
||||
"changes_unknown",
|
||||
"provider_cleanup_failed",
|
||||
"provider_observation_unknown",
|
||||
"provider_cleanup_outcome_unknown",
|
||||
"authority_invalid",
|
||||
"provider_identity_changed",
|
||||
"runtime_unavailable",
|
||||
"provider_unavailable",
|
||||
"unsupported_target",
|
||||
"blocked_by_live_authority",
|
||||
"dirty_or_unknown",
|
||||
"authority_changed",
|
||||
];
|
||||
for (const category of categories) {
|
||||
const text = removalCause(category);
|
||||
assert(text !== removalCause(null), `missing category ${category}`);
|
||||
assert(
|
||||
text.length < 400 && !text.includes("/"),
|
||||
`unbounded or pathful category ${category}`,
|
||||
);
|
||||
}
|
||||
for (
|
||||
const untrusted of [
|
||||
"/private/host/path",
|
||||
"token=synthetic-secret",
|
||||
"toString",
|
||||
"__proto__",
|
||||
"future_category",
|
||||
"x".repeat(10000),
|
||||
]
|
||||
) {
|
||||
const text = removalCause(untrusted);
|
||||
assert(
|
||||
text === removalCause(null) && !text.includes(untrusted),
|
||||
"echoed unknown category",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("pending unknown cleanup permits ordinary removal while current dirty and live guards remain protected", () => {
|
||||
const workdir = {
|
||||
working_directory_id: "dir",
|
||||
status: "cleanup_pending",
|
||||
cleanliness: "unknown",
|
||||
source: { kind: "repository", repository_key: "main" },
|
||||
materializer_kind: "runtime_git_clone",
|
||||
} as WorkingDirectorySummary;
|
||||
const candidate = {
|
||||
action: "workdir_clean_cleanup",
|
||||
blocking_reason: null,
|
||||
running_linked: false,
|
||||
pinned_linked: false,
|
||||
} as CleanupWorkdirCandidate;
|
||||
assert(canRemoveWorkdir(workdir, candidate));
|
||||
for (
|
||||
const guard of [
|
||||
{ action: "workdir_dirty_discard" },
|
||||
{ blocking_reason: "protected" },
|
||||
{ running_linked: true },
|
||||
{ pinned_linked: true },
|
||||
]
|
||||
) {
|
||||
assert(
|
||||
!canRemoveWorkdir(
|
||||
workdir,
|
||||
{ ...candidate, ...guard } as CleanupWorkdirCandidate,
|
||||
),
|
||||
);
|
||||
}
|
||||
assert(!canRemoveWorkdir({ ...workdir, cleanliness: "dirty" }, candidate));
|
||||
assert(
|
||||
canRemoveWorkdir(workdir, {
|
||||
...candidate,
|
||||
action: "workdir_record_delete",
|
||||
}),
|
||||
);
|
||||
assert(!canRemoveWorkdir(workdir, { ...candidate, action: "worker_delete" }));
|
||||
assert(!canRemoveWorkdir(workdir, undefined));
|
||||
assert(
|
||||
!canRemoveWorkdir({
|
||||
...workdir,
|
||||
occupied_by: {
|
||||
worker_id: "worker",
|
||||
runtime_id: "runtime",
|
||||
display_name: "Worker",
|
||||
linked_at: "now",
|
||||
},
|
||||
}, candidate),
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("ordinary retries send the same scoped DELETE body and expose only allowlisted HTTP causes", async () => {
|
||||
const requests: { path: string; body: string; method: string }[] = [];
|
||||
const fetchFn = (async (path: RequestInfo | URL, init?: RequestInit) => {
|
||||
requests.push({
|
||||
path: String(path),
|
||||
body: String(init?.body),
|
||||
method: String(init?.method),
|
||||
});
|
||||
return Response.json({
|
||||
working_directory_id: "dir/x",
|
||||
disposition: "attention_required",
|
||||
retryable: true,
|
||||
failure_category: "mount_present",
|
||||
});
|
||||
}) as typeof fetch;
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const result = await removeWorkdir(
|
||||
fetchFn,
|
||||
"workspace/x",
|
||||
"runtime/x",
|
||||
"dir/x",
|
||||
);
|
||||
assert(result.retryable && result.failure_category === "mount_present");
|
||||
}
|
||||
assert(
|
||||
JSON.stringify(requests[0]) === JSON.stringify(requests[1]),
|
||||
"retry changed ordinary deletion request",
|
||||
);
|
||||
assert(
|
||||
requests[0].path ===
|
||||
"/api/w/workspace%2Fx/runtimes/runtime%2Fx/working-directories/dir%2Fx",
|
||||
);
|
||||
assert(
|
||||
requests[0].method === "DELETE" &&
|
||||
requests[0].body ===
|
||||
JSON.stringify({ reason: "User requested Workdir deletion" }),
|
||||
);
|
||||
for (
|
||||
const [status, category] of [
|
||||
[403, "permission_denied"],
|
||||
[409, "authority_changed"],
|
||||
[503, "provider_unavailable"],
|
||||
[500, null],
|
||||
] as const
|
||||
) {
|
||||
let message = "";
|
||||
try {
|
||||
await removeWorkdir(
|
||||
(() =>
|
||||
Promise.resolve(
|
||||
Response.json({ message: "/private/path token=synthetic-secret" }, {
|
||||
status,
|
||||
}),
|
||||
)) as typeof fetch,
|
||||
"workspace",
|
||||
"runtime",
|
||||
"dir",
|
||||
);
|
||||
} catch (error) {
|
||||
message = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
assert(
|
||||
message === removalCause(category),
|
||||
`HTTP ${status} leaked provider text`,
|
||||
);
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user