fix: enforce workspace deletion blockers
This commit is contained in:
@@ -673,7 +673,6 @@ pub struct WorkspaceDeletionPreflightResponse {
|
|||||||
/// Opaque persisted Workspace metadata revision used as a CAS fence.
|
/// Opaque persisted Workspace metadata revision used as a CAS fence.
|
||||||
pub expected_revision: String,
|
pub expected_revision: String,
|
||||||
pub can_delete: bool,
|
pub can_delete: bool,
|
||||||
pub force_delete_dirty_workdirs_available: bool,
|
|
||||||
pub resources: WorkspaceDeletionResourceCounts,
|
pub resources: WorkspaceDeletionResourceCounts,
|
||||||
pub blockers: Vec<WorkspaceDeletionBlocker>,
|
pub blockers: Vec<WorkspaceDeletionBlocker>,
|
||||||
}
|
}
|
||||||
@@ -686,8 +685,6 @@ pub struct WorkspaceDeletionRequest {
|
|||||||
pub operation_id: String,
|
pub operation_id: String,
|
||||||
pub expected_revision: String,
|
pub expected_revision: String,
|
||||||
pub confirmation: String,
|
pub confirmation: String,
|
||||||
#[serde(default)]
|
|
||||||
pub force_delete_dirty_workdirs: bool,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Durable deletion operation projection used by request responses and polling.
|
/// Durable deletion operation projection used by request responses and polling.
|
||||||
@@ -699,7 +696,6 @@ pub struct WorkspaceDeletionOperationResponse {
|
|||||||
pub workspace_id: String,
|
pub workspace_id: String,
|
||||||
pub display_name: String,
|
pub display_name: String,
|
||||||
pub state: WorkspaceDeletionState,
|
pub state: WorkspaceDeletionState,
|
||||||
pub force_delete_dirty_workdirs: bool,
|
|
||||||
pub resources: WorkspaceDeletionResourceCounts,
|
pub resources: WorkspaceDeletionResourceCounts,
|
||||||
pub child_operation_ids: Vec<String>,
|
pub child_operation_ids: Vec<String>,
|
||||||
pub blockers: Vec<WorkspaceDeletionBlocker>,
|
pub blockers: Vec<WorkspaceDeletionBlocker>,
|
||||||
@@ -3262,7 +3258,6 @@ mod tests {
|
|||||||
display_name: "Test".to_string(),
|
display_name: "Test".to_string(),
|
||||||
expected_revision: "revision-7".to_string(),
|
expected_revision: "revision-7".to_string(),
|
||||||
can_delete: true,
|
can_delete: true,
|
||||||
force_delete_dirty_workdirs_available: true,
|
|
||||||
resources: WorkspaceDeletionResourceCounts {
|
resources: WorkspaceDeletionResourceCounts {
|
||||||
workers: 2,
|
workers: 2,
|
||||||
workdirs: 1,
|
workdirs: 1,
|
||||||
@@ -3288,8 +3283,7 @@ mod tests {
|
|||||||
serde_json::from_value::<WorkspaceDeletionRequest>(serde_json::json!({
|
serde_json::from_value::<WorkspaceDeletionRequest>(serde_json::json!({
|
||||||
"operation_id": "delete-test",
|
"operation_id": "delete-test",
|
||||||
"expected_revision": "revision-7",
|
"expected_revision": "revision-7",
|
||||||
"confirmation": "delete Test",
|
"confirmation": "Test",
|
||||||
"force_delete_dirty_workdirs": false,
|
|
||||||
"workspace_id": "caller-controlled"
|
"workspace_id": "caller-controlled"
|
||||||
}))
|
}))
|
||||||
.is_err()
|
.is_err()
|
||||||
|
|||||||
@@ -948,7 +948,6 @@ CREATE TABLE workspace_deletion_operations (
|
|||||||
workspace_revision TEXT NOT NULL,
|
workspace_revision TEXT NOT NULL,
|
||||||
owner_account_id TEXT NOT NULL,
|
owner_account_id TEXT NOT NULL,
|
||||||
actor_account_id TEXT NOT NULL,
|
actor_account_id TEXT NOT NULL,
|
||||||
force_delete_dirty_workdirs INTEGER NOT NULL CHECK(force_delete_dirty_workdirs IN (0, 1)),
|
|
||||||
state TEXT NOT NULL CHECK(state IN ('queued', 'running', 'blocked', 'failed', 'succeeded')),
|
state TEXT NOT NULL CHECK(state IN ('queued', 'running', 'blocked', 'failed', 'succeeded')),
|
||||||
resource_counts_json TEXT NOT NULL,
|
resource_counts_json TEXT NOT NULL,
|
||||||
child_operation_ids_json TEXT NOT NULL,
|
child_operation_ids_json TEXT NOT NULL,
|
||||||
|
|||||||
@@ -90,11 +90,11 @@ use workspace_api::{
|
|||||||
WorkingDirectoryRemovalDisposition, WorkingDirectoryRemovalRequest,
|
WorkingDirectoryRemovalDisposition, WorkingDirectoryRemovalRequest,
|
||||||
WorkingDirectoryRemovalResponse, WorkingDirectoryRepositoryOption,
|
WorkingDirectoryRemovalResponse, WorkingDirectoryRepositoryOption,
|
||||||
WorkspaceCatalogListResponse, WorkspaceCreateResponse, WorkspaceDeletionBlocker,
|
WorkspaceCatalogListResponse, WorkspaceCreateResponse, WorkspaceDeletionBlocker,
|
||||||
WorkspaceDeletionBlockerKind, WorkspaceDeletionOperationResponse, WorkspaceDeletionRequest,
|
WorkspaceDeletionBlockerKind, WorkspaceDeletionOperationResponse,
|
||||||
WorkspaceDeletionState, WorkspaceExtensionPointState, WorkspaceExtensionPoints,
|
WorkspaceDeletionPreflightResponse, WorkspaceDeletionRequest, WorkspaceDeletionState,
|
||||||
WorkspaceMetadataMutationResponse, WorkspaceMetadataSettingsResponse,
|
WorkspaceExtensionPointState, WorkspaceExtensionPoints, WorkspaceMetadataMutationResponse,
|
||||||
WorkspacePermissionSummary, WorkspaceRepositoryRecord, WorkspaceResponse,
|
WorkspaceMetadataSettingsResponse, WorkspacePermissionSummary, WorkspaceRepositoryRecord,
|
||||||
WorkspaceRuntimeDetail, WorkspaceRuntimeResource, WorkspaceSummary,
|
WorkspaceResponse, WorkspaceRuntimeDetail, WorkspaceRuntimeResource, WorkspaceSummary,
|
||||||
WorkspaceWorkerDiscoveryItem, WorkspaceWorkerDiscoveryPage, WorkspaceWorkerSubject,
|
WorkspaceWorkerDiscoveryItem, WorkspaceWorkerDiscoveryPage, WorkspaceWorkerSubject,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -746,8 +746,7 @@ impl WorkspaceWorkerRemoveExecutor {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
self.execute_target_removal(&runtime, &target, reason, false)
|
self.execute_target_removal(&runtime, &target, reason).await
|
||||||
.await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn execute_target_removal(
|
async fn execute_target_removal(
|
||||||
@@ -755,7 +754,6 @@ impl WorkspaceWorkerRemoveExecutor {
|
|||||||
runtime: &RuntimeRegistry,
|
runtime: &RuntimeRegistry,
|
||||||
target: &RuntimeWorkerRef,
|
target: &RuntimeWorkerRef,
|
||||||
reason: &str,
|
reason: &str,
|
||||||
allow_internal: bool,
|
|
||||||
) -> std::result::Result<worker::WorkspaceResponse, String> {
|
) -> std::result::Result<worker::WorkspaceResponse, String> {
|
||||||
let remove_lock = {
|
let remove_lock = {
|
||||||
let mut locks = self
|
let mut locks = self
|
||||||
@@ -857,7 +855,7 @@ impl WorkspaceWorkerRemoveExecutor {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if worker.singleton_key.is_some() && !allow_internal {
|
if worker.singleton_key.is_some() {
|
||||||
return Ok(worker_remove_error_response(
|
return Ok(worker_remove_error_response(
|
||||||
StatusCode::CONFLICT,
|
StatusCode::CONFLICT,
|
||||||
"internal_worker_forbidden",
|
"internal_worker_forbidden",
|
||||||
@@ -1009,6 +1007,7 @@ pub struct WorkspaceServerApi {
|
|||||||
store: Arc<dyn ControlPlaneStore>,
|
store: Arc<dyn ControlPlaneStore>,
|
||||||
catalog: WorkspaceCatalogService,
|
catalog: WorkspaceCatalogService,
|
||||||
routers: Arc<AsyncMutex<HashMap<String, Router>>>,
|
routers: Arc<AsyncMutex<HashMap<String, Router>>>,
|
||||||
|
apis: Arc<AsyncMutex<HashMap<String, WorkspaceApi>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WorkspaceServerApi {
|
impl WorkspaceServerApi {
|
||||||
@@ -1018,9 +1017,71 @@ impl WorkspaceServerApi {
|
|||||||
catalog: WorkspaceCatalogService::new(store.clone()),
|
catalog: WorkspaceCatalogService::new(store.clone()),
|
||||||
store,
|
store,
|
||||||
routers: Arc::new(AsyncMutex::new(HashMap::new())),
|
routers: Arc::new(AsyncMutex::new(HashMap::new())),
|
||||||
|
apis: Arc::new(AsyncMutex::new(HashMap::new())),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn api_for_workspace(&self, workspace_id: &str) -> Result<Option<WorkspaceApi>> {
|
||||||
|
let mut apis = self.apis.lock().await;
|
||||||
|
if let Some(api) = apis.get(workspace_id).cloned() {
|
||||||
|
return Ok(Some(api));
|
||||||
|
}
|
||||||
|
let Some(workspace) = self.store.get_workspace(workspace_id).await? else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let repositories = self.store.list_repositories(workspace_id)?;
|
||||||
|
let config = self
|
||||||
|
.template
|
||||||
|
.for_catalog_workspace(&workspace, repositories)?;
|
||||||
|
let api = WorkspaceApi::new(config, self.store.clone()).await?;
|
||||||
|
apis.insert(workspace_id.to_string(), api.clone());
|
||||||
|
Ok(Some(api))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn workspace_deletion_preflight(
|
||||||
|
&self,
|
||||||
|
actor_account_id: &str,
|
||||||
|
workspace_id: &str,
|
||||||
|
) -> Result<WorkspaceDeletionPreflightResponse> {
|
||||||
|
let mut preflight = self
|
||||||
|
.store
|
||||||
|
.workspace_deletion_preflight(actor_account_id, workspace_id)?;
|
||||||
|
let Some(api) = self.api_for_workspace(workspace_id).await? else {
|
||||||
|
return Err(Error::InvalidInput("Workspace does not exist".to_string()));
|
||||||
|
};
|
||||||
|
for registry_worker in self.store.list_worker_registry(workspace_id, 10_000)? {
|
||||||
|
let worker_key = registry_worker.display_name;
|
||||||
|
match api.runtime.worker(®istry_worker.worker) {
|
||||||
|
Ok(worker) if worker.state == "stopped" && worker.singleton_key.is_none() => {}
|
||||||
|
Ok(worker) => preflight.blockers.push(WorkspaceDeletionBlocker {
|
||||||
|
kind: if worker.singleton_key.is_some() {
|
||||||
|
WorkspaceDeletionBlockerKind::RetentionHold
|
||||||
|
} else {
|
||||||
|
WorkspaceDeletionBlockerKind::WorkerRemovalBlocked
|
||||||
|
},
|
||||||
|
resource_kind: Some("worker".to_string()),
|
||||||
|
resource_key: Some(worker_key),
|
||||||
|
message: if worker.singleton_key.is_some() {
|
||||||
|
"Internal or singleton Workers must be released by their owning service first."
|
||||||
|
.to_string()
|
||||||
|
} else {
|
||||||
|
"Stop running, restoring, or otherwise active Workers before deleting the Workspace."
|
||||||
|
.to_string()
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
Err(_) => preflight.blockers.push(WorkspaceDeletionBlocker {
|
||||||
|
kind: WorkspaceDeletionBlockerKind::CleanupUnavailable,
|
||||||
|
resource_kind: Some("worker".to_string()),
|
||||||
|
resource_key: Some(worker_key),
|
||||||
|
message: "Worker state is unavailable; retry after Runtime state is healthy."
|
||||||
|
.to_string(),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
preflight.can_delete = preflight.blockers.is_empty();
|
||||||
|
Ok(preflight)
|
||||||
|
}
|
||||||
|
|
||||||
async fn execute_workspace_deletion(
|
async fn execute_workspace_deletion(
|
||||||
&self,
|
&self,
|
||||||
operation_id: &str,
|
operation_id: &str,
|
||||||
@@ -1032,18 +1093,10 @@ impl WorkspaceServerApi {
|
|||||||
&[],
|
&[],
|
||||||
None,
|
None,
|
||||||
)?;
|
)?;
|
||||||
let workspace = self
|
let api = self
|
||||||
.store
|
.api_for_workspace(&operation.workspace_id)
|
||||||
.get_workspace(&operation.workspace_id)
|
|
||||||
.await?
|
.await?
|
||||||
.ok_or_else(|| Error::InvalidInput("Workspace no longer exists".to_string()))?;
|
.ok_or_else(|| Error::InvalidInput("Workspace no longer exists".to_string()))?;
|
||||||
let repositories = self.store.list_repositories(&operation.workspace_id)?;
|
|
||||||
let config = self
|
|
||||||
.template
|
|
||||||
.for_catalog_workspace(&workspace, repositories)?;
|
|
||||||
let api = WorkspaceApi::new(config, self.store.clone()).await?;
|
|
||||||
self.store
|
|
||||||
.release_workspace_assignments_for_deletion(&operation.workspace_id)?;
|
|
||||||
|
|
||||||
let mut child_operation_ids = Vec::new();
|
let mut child_operation_ids = Vec::new();
|
||||||
let mut blockers = Vec::new();
|
let mut blockers = Vec::new();
|
||||||
@@ -1053,22 +1106,8 @@ impl WorkspaceServerApi {
|
|||||||
{
|
{
|
||||||
let worker_key = worker.display_name.clone();
|
let worker_key = worker.display_name.clone();
|
||||||
let target = worker.worker;
|
let target = worker.worker;
|
||||||
let lifecycle = WorkerLifecycleRequest {
|
|
||||||
reason: Some("Workspace deletion".to_string()),
|
|
||||||
ticket_assignment: None,
|
|
||||||
};
|
|
||||||
let _ = api.runtime.cancel_worker(&target, lifecycle.clone());
|
|
||||||
if api.runtime.stop_worker(&target, lifecycle).is_err() {
|
|
||||||
blockers.push(WorkspaceDeletionBlocker {
|
|
||||||
kind: WorkspaceDeletionBlockerKind::WorkerRemovalBlocked,
|
|
||||||
resource_kind: Some("worker".to_string()),
|
|
||||||
resource_key: Some(worker_key.clone()),
|
|
||||||
message: "Worker stop did not reach a retryable terminal state.".to_string(),
|
|
||||||
});
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let response = WorkspaceWorkerRemoveExecutor::new(&api)
|
let response = WorkspaceWorkerRemoveExecutor::new(&api)
|
||||||
.execute_target_removal(api.runtime.as_ref(), &target, "Workspace deletion", true)
|
.execute_target_removal(api.runtime.as_ref(), &target, "Workspace deletion")
|
||||||
.await
|
.await
|
||||||
.map_err(Error::Store)?;
|
.map_err(Error::Store)?;
|
||||||
if let Some(child_operation_id) = self.store.latest_worker_removal_operation_id(
|
if let Some(child_operation_id) = self.store.latest_worker_removal_operation_id(
|
||||||
@@ -1097,7 +1136,6 @@ impl WorkspaceServerApi {
|
|||||||
&api,
|
&api,
|
||||||
&workdir.workdir_id,
|
&workdir.workdir_id,
|
||||||
operation_id,
|
operation_id,
|
||||||
operation.force_delete_dirty_workdirs,
|
|
||||||
) {
|
) {
|
||||||
Ok(child) => {
|
Ok(child) => {
|
||||||
child_operation_ids.push(child.operation_id.clone());
|
child_operation_ids.push(child.operation_id.clone());
|
||||||
@@ -1115,7 +1153,7 @@ impl WorkspaceServerApi {
|
|||||||
resource_kind: Some("workdir".to_string()),
|
resource_kind: Some("workdir".to_string()),
|
||||||
resource_key: Some(workdir.workdir_id),
|
resource_key: Some(workdir.workdir_id),
|
||||||
message: if dirty {
|
message: if dirty {
|
||||||
"Workdir is dirty or its cleanliness is unknown. Enable force deletion only after reviewing the impact."
|
"Workdir is dirty or its cleanliness is unknown. Clean it and refresh status before retrying deletion."
|
||||||
.to_string()
|
.to_string()
|
||||||
} else {
|
} else {
|
||||||
"Workdir removal did not complete; retry the Workspace deletion operation."
|
"Workdir removal did not complete; retry the Workspace deletion operation."
|
||||||
@@ -1145,25 +1183,23 @@ impl WorkspaceServerApi {
|
|||||||
}
|
}
|
||||||
let completed = self.store.finalize_workspace_deletion(operation_id)?;
|
let completed = self.store.finalize_workspace_deletion(operation_id)?;
|
||||||
self.routers.lock().await.remove(&completed.workspace_id);
|
self.routers.lock().await.remove(&completed.workspace_id);
|
||||||
|
self.apis.lock().await.remove(&completed.workspace_id);
|
||||||
Ok(completed)
|
Ok(completed)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn router_for_workspace(&self, workspace_id: &str) -> Result<Option<Router>> {
|
async fn router_for_workspace(&self, workspace_id: &str) -> Result<Option<Router>> {
|
||||||
let mut routers = self.routers.lock().await;
|
if let Some(router) = self.routers.lock().await.get(workspace_id).cloned() {
|
||||||
if let Some(router) = routers.get(workspace_id) {
|
return Ok(Some(router));
|
||||||
return Ok(Some(router.clone()));
|
|
||||||
}
|
}
|
||||||
let Some(workspace) = self.store.get_workspace(workspace_id).await? else {
|
let Some(api) = self.api_for_workspace(workspace_id).await? else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let repositories = self.store.list_repositories(workspace_id)?;
|
|
||||||
let config = self
|
|
||||||
.template
|
|
||||||
.for_catalog_workspace(&workspace, repositories)?;
|
|
||||||
let api = WorkspaceApi::new(config, self.store.clone()).await?;
|
|
||||||
tokio::spawn(run_orchestrator_turn_end_hook(api.clone()));
|
tokio::spawn(run_orchestrator_turn_end_hook(api.clone()));
|
||||||
let router = build_inner_router(api);
|
let router = build_inner_router(api);
|
||||||
routers.insert(workspace_id.to_string(), router.clone());
|
self.routers
|
||||||
|
.lock()
|
||||||
|
.await
|
||||||
|
.insert(workspace_id.to_string(), router.clone());
|
||||||
Ok(Some(router))
|
Ok(Some(router))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1269,8 +1305,8 @@ async fn preflight_server_workspace_deletion(
|
|||||||
Err(error) => return server_error_response(error),
|
Err(error) => return server_error_response(error),
|
||||||
};
|
};
|
||||||
match api
|
match api
|
||||||
.store
|
|
||||||
.workspace_deletion_preflight(&actor_account_id, &workspace_id)
|
.workspace_deletion_preflight(&actor_account_id, &workspace_id)
|
||||||
|
.await
|
||||||
{
|
{
|
||||||
Ok(preflight) => Json(preflight).into_response(),
|
Ok(preflight) => Json(preflight).into_response(),
|
||||||
Err(error) => server_error_response(error),
|
Err(error) => server_error_response(error),
|
||||||
@@ -1288,6 +1324,25 @@ async fn start_server_workspace_deletion(
|
|||||||
Ok(None) => return forbidden_server_response("Workspace deletion requires its owner"),
|
Ok(None) => return forbidden_server_response("Workspace deletion requires its owner"),
|
||||||
Err(error) => return server_error_response(error),
|
Err(error) => return server_error_response(error),
|
||||||
};
|
};
|
||||||
|
let existing = match api
|
||||||
|
.store
|
||||||
|
.workspace_deletion_operation(&actor_account_id, &request.operation_id)
|
||||||
|
{
|
||||||
|
Ok(existing) => existing,
|
||||||
|
Err(error) => return server_error_response(error),
|
||||||
|
};
|
||||||
|
if existing.is_none() {
|
||||||
|
let preflight = match api
|
||||||
|
.workspace_deletion_preflight(&actor_account_id, &workspace_id)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(preflight) => preflight,
|
||||||
|
Err(error) => return server_error_response(error),
|
||||||
|
};
|
||||||
|
if !preflight.can_delete {
|
||||||
|
return (StatusCode::CONFLICT, Json(preflight)).into_response();
|
||||||
|
}
|
||||||
|
}
|
||||||
let reservation =
|
let reservation =
|
||||||
match api
|
match api
|
||||||
.store
|
.store
|
||||||
@@ -10520,27 +10575,7 @@ fn execute_reserved_workdir_removal(
|
|||||||
operation: WorkdirRemovalOperation,
|
operation: WorkdirRemovalOperation,
|
||||||
recovery: bool,
|
recovery: bool,
|
||||||
) -> Result<WorkdirRemovalOperation> {
|
) -> Result<WorkdirRemovalOperation> {
|
||||||
execute_reserved_workdir_removal_with_provider(
|
execute_reserved_workdir_removal_with_provider(api, operation, recovery, api.runtime.as_ref())
|
||||||
api,
|
|
||||||
operation,
|
|
||||||
recovery,
|
|
||||||
api.runtime.as_ref(),
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn execute_reserved_workdir_removal_for_workspace_deletion(
|
|
||||||
api: &WorkspaceApi,
|
|
||||||
operation: WorkdirRemovalOperation,
|
|
||||||
force_dirty: bool,
|
|
||||||
) -> Result<WorkdirRemovalOperation> {
|
|
||||||
execute_reserved_workdir_removal_with_provider(
|
|
||||||
api,
|
|
||||||
operation,
|
|
||||||
false,
|
|
||||||
api.runtime.as_ref(),
|
|
||||||
force_dirty,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn execute_reserved_workdir_removal_with_provider(
|
fn execute_reserved_workdir_removal_with_provider(
|
||||||
@@ -10548,7 +10583,6 @@ fn execute_reserved_workdir_removal_with_provider(
|
|||||||
operation: WorkdirRemovalOperation,
|
operation: WorkdirRemovalOperation,
|
||||||
recovery: bool,
|
recovery: bool,
|
||||||
provider: &dyn WorkdirRemovalRuntimeProvider,
|
provider: &dyn WorkdirRemovalRuntimeProvider,
|
||||||
force_dirty: bool,
|
|
||||||
) -> Result<WorkdirRemovalOperation> {
|
) -> Result<WorkdirRemovalOperation> {
|
||||||
if operation.state == WorkdirRemovalOperationState::Completed {
|
if operation.state == WorkdirRemovalOperationState::Completed {
|
||||||
return Ok(operation);
|
return Ok(operation);
|
||||||
@@ -10623,9 +10657,8 @@ fn execute_reserved_workdir_removal_with_provider(
|
|||||||
true,
|
true,
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
if !force_dirty
|
if status.summary.cleanliness.as_deref() != Some("clean")
|
||||||
&& (status.summary.cleanliness.as_deref() != Some("clean")
|
|| status.summary.status != WorkingDirectoryStatusKind::Active
|
||||||
|| status.summary.status != WorkingDirectoryStatusKind::Active)
|
|
||||||
{
|
{
|
||||||
return api.config_store.complete_workdir_removal_retained(
|
return api.config_store.complete_workdir_removal_retained(
|
||||||
&operation,
|
&operation,
|
||||||
@@ -10723,7 +10756,6 @@ fn execute_workdir_removal_for_workspace_deletion(
|
|||||||
api: &WorkspaceApi,
|
api: &WorkspaceApi,
|
||||||
working_directory_id: &str,
|
working_directory_id: &str,
|
||||||
parent_operation_id: &str,
|
parent_operation_id: &str,
|
||||||
force_dirty: bool,
|
|
||||||
) -> Result<WorkdirRemovalOperation> {
|
) -> Result<WorkdirRemovalOperation> {
|
||||||
let source_actor = format!("workspace-deletion:{parent_operation_id}");
|
let source_actor = format!("workspace-deletion:{parent_operation_id}");
|
||||||
let reason = "Workspace deletion";
|
let reason = "Workspace deletion";
|
||||||
@@ -10750,7 +10782,7 @@ fn execute_workdir_removal_for_workspace_deletion(
|
|||||||
api.config_store
|
api.config_store
|
||||||
.reserve_workdir_removal_operation(&intent)?
|
.reserve_workdir_removal_operation(&intent)?
|
||||||
};
|
};
|
||||||
execute_reserved_workdir_removal_for_workspace_deletion(api, operation, force_dirty)
|
execute_reserved_workdir_removal(api, operation, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn recover_workdir_removals(api: &WorkspaceApi) -> Result<()> {
|
fn recover_workdir_removals(api: &WorkspaceApi) -> Result<()> {
|
||||||
@@ -24371,7 +24403,6 @@ mod tests {
|
|||||||
clean_operation.clone(),
|
clean_operation.clone(),
|
||||||
false,
|
false,
|
||||||
&clean_provider,
|
&clean_provider,
|
||||||
false,
|
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -24384,7 +24415,6 @@ mod tests {
|
|||||||
removed.clone(),
|
removed.clone(),
|
||||||
false,
|
false,
|
||||||
&clean_provider,
|
&clean_provider,
|
||||||
false,
|
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(replay, removed);
|
assert_eq!(replay, removed);
|
||||||
@@ -24409,7 +24439,6 @@ mod tests {
|
|||||||
missing_operation,
|
missing_operation,
|
||||||
false,
|
false,
|
||||||
&missing_provider,
|
&missing_provider,
|
||||||
false,
|
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -24436,7 +24465,6 @@ mod tests {
|
|||||||
unknown_operation,
|
unknown_operation,
|
||||||
false,
|
false,
|
||||||
&unknown_provider,
|
&unknown_provider,
|
||||||
false,
|
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(unknown.state, WorkdirRemovalOperationState::Failed);
|
assert_eq!(unknown.state, WorkdirRemovalOperationState::Failed);
|
||||||
@@ -24458,34 +24486,11 @@ mod tests {
|
|||||||
dirty_operation,
|
dirty_operation,
|
||||||
false,
|
false,
|
||||||
&dirty_provider,
|
&dirty_provider,
|
||||||
false,
|
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(dirty.disposition, Some(WorkdirRemovalDisposition::Retained));
|
assert_eq!(dirty.disposition, Some(WorkdirRemovalDisposition::Retained));
|
||||||
assert_eq!(dirty_provider.cleanup_calls(), 0);
|
assert_eq!(dirty_provider.cleanup_calls(), 0);
|
||||||
|
|
||||||
let (forced_operation, mut forced_summary) =
|
|
||||||
reserve_removal_fixture(&api, "provider-dirty-forced");
|
|
||||||
forced_summary.cleanliness = Some("dirty".to_string());
|
|
||||||
let forced_provider = FakeWorkdirRemovalProvider::new(
|
|
||||||
workdir_removal_result(
|
|
||||||
WorkerOperationState::Accepted,
|
|
||||||
Some(forced_summary),
|
|
||||||
Vec::new(),
|
|
||||||
),
|
|
||||||
workdir_removal_result(WorkerOperationState::Accepted, None, Vec::new()),
|
|
||||||
);
|
|
||||||
let forced = execute_reserved_workdir_removal_with_provider(
|
|
||||||
&api,
|
|
||||||
forced_operation,
|
|
||||||
false,
|
|
||||||
&forced_provider,
|
|
||||||
true,
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(forced.disposition, Some(WorkdirRemovalDisposition::Removed));
|
|
||||||
assert_eq!(forced_provider.cleanup_calls(), 1);
|
|
||||||
|
|
||||||
let (unsupported_operation, unsupported_summary) =
|
let (unsupported_operation, unsupported_summary) =
|
||||||
reserve_removal_fixture(&api, "provider-unsupported");
|
reserve_removal_fixture(&api, "provider-unsupported");
|
||||||
let unsupported_provider = FakeWorkdirRemovalProvider::new(
|
let unsupported_provider = FakeWorkdirRemovalProvider::new(
|
||||||
@@ -24509,7 +24514,6 @@ mod tests {
|
|||||||
unsupported_operation,
|
unsupported_operation,
|
||||||
false,
|
false,
|
||||||
&unsupported_provider,
|
&unsupported_provider,
|
||||||
false,
|
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(unsupported.state, WorkdirRemovalOperationState::Failed);
|
assert_eq!(unsupported.state, WorkdirRemovalOperationState::Failed);
|
||||||
@@ -24564,7 +24568,6 @@ mod tests {
|
|||||||
operation,
|
operation,
|
||||||
true,
|
true,
|
||||||
provider.as_ref(),
|
provider.as_ref(),
|
||||||
false,
|
|
||||||
)
|
)
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6438,7 +6438,6 @@ fn migrate_workspace_deletion_v52_to_v53(conn: &Connection) -> Result<()> {
|
|||||||
workspace_revision TEXT NOT NULL,
|
workspace_revision TEXT NOT NULL,
|
||||||
owner_account_id TEXT NOT NULL,
|
owner_account_id TEXT NOT NULL,
|
||||||
actor_account_id TEXT NOT NULL,
|
actor_account_id TEXT NOT NULL,
|
||||||
force_delete_dirty_workdirs INTEGER NOT NULL CHECK(force_delete_dirty_workdirs IN (0, 1)),
|
|
||||||
state TEXT NOT NULL CHECK(state IN ('queued', 'running', 'blocked', 'failed', 'succeeded')),
|
state TEXT NOT NULL CHECK(state IN ('queued', 'running', 'blocked', 'failed', 'succeeded')),
|
||||||
resource_counts_json TEXT NOT NULL,
|
resource_counts_json TEXT NOT NULL,
|
||||||
child_operation_ids_json TEXT NOT NULL,
|
child_operation_ids_json TEXT NOT NULL,
|
||||||
@@ -6475,7 +6474,6 @@ fn verify_workspace_deletion_schema(conn: &Connection) -> Result<()> {
|
|||||||
"workspace_revision",
|
"workspace_revision",
|
||||||
"owner_account_id",
|
"owner_account_id",
|
||||||
"actor_account_id",
|
"actor_account_id",
|
||||||
"force_delete_dirty_workdirs",
|
|
||||||
"state",
|
"state",
|
||||||
"resource_counts_json",
|
"resource_counts_json",
|
||||||
"child_operation_ids_json",
|
"child_operation_ids_json",
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ use crate::store::{SqliteWorkspaceStore, WorkspaceRecord};
|
|||||||
use crate::{Error, Result};
|
use crate::{Error, Result};
|
||||||
|
|
||||||
const MAX_OPERATION_ID_BYTES: usize = 128;
|
const MAX_OPERATION_ID_BYTES: usize = 128;
|
||||||
const CONFIRMATION_PREFIX: &str = "delete ";
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct WorkspaceDeletionReservation {
|
pub struct WorkspaceDeletionReservation {
|
||||||
@@ -40,8 +39,6 @@ pub trait WorkspaceDeletionStore: Send + Sync {
|
|||||||
operation_id: &str,
|
operation_id: &str,
|
||||||
) -> Result<Option<WorkspaceDeletionOperationResponse>>;
|
) -> Result<Option<WorkspaceDeletionOperationResponse>>;
|
||||||
|
|
||||||
fn release_workspace_assignments_for_deletion(&self, workspace_id: &str) -> Result<u64>;
|
|
||||||
|
|
||||||
fn latest_worker_removal_operation_id(
|
fn latest_worker_removal_operation_id(
|
||||||
&self,
|
&self,
|
||||||
workspace_id: &str,
|
workspace_id: &str,
|
||||||
@@ -74,11 +71,11 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
|||||||
let workspace = owner_workspace(conn, actor_account_id, workspace_id)?;
|
let workspace = owner_workspace(conn, actor_account_id, workspace_id)?;
|
||||||
let resources = resource_counts(conn, workspace_id)?;
|
let resources = resource_counts(conn, workspace_id)?;
|
||||||
let accessible: u64 = conn.query_row(
|
let accessible: u64 = conn.query_row(
|
||||||
"SELECT COUNT(*) FROM workspaces WHERE owner_account_id = ?1 AND state = 'active'",
|
"SELECT COUNT(*) FROM workspaces WHERE owner_account_id = ?1",
|
||||||
params![actor_account_id],
|
params![actor_account_id],
|
||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)?;
|
)?;
|
||||||
let mut blockers = Vec::new();
|
let mut blockers = workspace_database_blockers(conn, workspace_id)?;
|
||||||
if accessible <= 1 {
|
if accessible <= 1 {
|
||||||
blockers.push(WorkspaceDeletionBlocker {
|
blockers.push(WorkspaceDeletionBlocker {
|
||||||
kind: WorkspaceDeletionBlockerKind::LastAccessibleWorkspace,
|
kind: WorkspaceDeletionBlockerKind::LastAccessibleWorkspace,
|
||||||
@@ -92,7 +89,6 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
|||||||
display_name: workspace.display_name,
|
display_name: workspace.display_name,
|
||||||
expected_revision: workspace.updated_at,
|
expected_revision: workspace.updated_at,
|
||||||
can_delete: blockers.is_empty(),
|
can_delete: blockers.is_empty(),
|
||||||
force_delete_dirty_workdirs_available: true,
|
|
||||||
resources,
|
resources,
|
||||||
blockers,
|
blockers,
|
||||||
})
|
})
|
||||||
@@ -126,8 +122,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let workspace = owner_workspace(tx, actor_account_id, workspace_id)?;
|
let workspace = owner_workspace(tx, actor_account_id, workspace_id)?;
|
||||||
let expected_confirmation = format!("{CONFIRMATION_PREFIX}{}", workspace.display_name);
|
if request.confirmation != workspace.display_name {
|
||||||
if request.confirmation != expected_confirmation {
|
|
||||||
return Err(Error::InvalidInput(
|
return Err(Error::InvalidInput(
|
||||||
"confirmation must exactly match the displayed Workspace name".to_string(),
|
"confirmation must exactly match the displayed Workspace name".to_string(),
|
||||||
));
|
));
|
||||||
@@ -138,7 +133,7 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
let accessible: u64 = tx.query_row(
|
let accessible: u64 = tx.query_row(
|
||||||
"SELECT COUNT(*) FROM workspaces WHERE owner_account_id = ?1 AND state = 'active'",
|
"SELECT COUNT(*) FROM workspaces WHERE owner_account_id = ?1",
|
||||||
params![actor_account_id],
|
params![actor_account_id],
|
||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)?;
|
)?;
|
||||||
@@ -148,6 +143,11 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
|||||||
.to_string(),
|
.to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
if !workspace_database_blockers(tx, workspace_id)?.is_empty() {
|
||||||
|
return Err(Error::WorkspaceConfigConflict(
|
||||||
|
"Workspace deletion preflight changed; reload current blockers".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
let resources = resource_counts(tx, workspace_id)?;
|
let resources = resource_counts(tx, workspace_id)?;
|
||||||
let now = Utc::now().to_rfc3339();
|
let now = Utc::now().to_rfc3339();
|
||||||
@@ -156,10 +156,10 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
|||||||
"INSERT INTO workspace_deletion_operations (
|
"INSERT INTO workspace_deletion_operations (
|
||||||
operation_id, request_fingerprint, workspace_id, workspace_display_name,
|
operation_id, request_fingerprint, workspace_id, workspace_display_name,
|
||||||
workspace_revision, owner_account_id, actor_account_id,
|
workspace_revision, owner_account_id, actor_account_id,
|
||||||
force_delete_dirty_workdirs, state, resource_counts_json,
|
state, resource_counts_json,
|
||||||
child_operation_ids_json, blockers_json, failure_category,
|
child_operation_ids_json, blockers_json, failure_category,
|
||||||
created_at, updated_at, completed_at
|
created_at, updated_at, completed_at
|
||||||
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 'queued', ?9, '[]', '[]', NULL, ?10, ?10, NULL)",
|
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'queued', ?8, '[]', '[]', NULL, ?9, ?9, NULL)",
|
||||||
params![
|
params![
|
||||||
request.operation_id,
|
request.operation_id,
|
||||||
fingerprint,
|
fingerprint,
|
||||||
@@ -168,7 +168,6 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
|||||||
request.expected_revision,
|
request.expected_revision,
|
||||||
workspace.owner_account_id,
|
workspace.owner_account_id,
|
||||||
actor_account_id,
|
actor_account_id,
|
||||||
request.force_delete_dirty_workdirs as i64,
|
|
||||||
serde_json::to_string(&resources).map_err(|error| Error::Store(error.to_string()))?,
|
serde_json::to_string(&resources).map_err(|error| Error::Store(error.to_string()))?,
|
||||||
now,
|
now,
|
||||||
],
|
],
|
||||||
@@ -210,17 +209,6 @@ impl WorkspaceDeletionStore for SqliteWorkspaceStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn release_workspace_assignments_for_deletion(&self, workspace_id: &str) -> Result<u64> {
|
|
||||||
self.with_conn(|conn| {
|
|
||||||
let changed = conn.execute(
|
|
||||||
"DELETE FROM ticket_current_worker_assignments WHERE workspace_id = ?1",
|
|
||||||
params![workspace_id],
|
|
||||||
)?;
|
|
||||||
u64::try_from(changed)
|
|
||||||
.map_err(|_| Error::Store("assignment deletion count overflow".to_string()))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn latest_worker_removal_operation_id(
|
fn latest_worker_removal_operation_id(
|
||||||
&self,
|
&self,
|
||||||
workspace_id: &str,
|
workspace_id: &str,
|
||||||
@@ -373,30 +361,28 @@ fn read_operation(
|
|||||||
) -> Result<Option<StoredOperation>> {
|
) -> Result<Option<StoredOperation>> {
|
||||||
conn.query_row(
|
conn.query_row(
|
||||||
"SELECT request_fingerprint, actor_account_id, workspace_id, workspace_display_name,
|
"SELECT request_fingerprint, actor_account_id, workspace_id, workspace_display_name,
|
||||||
state, force_delete_dirty_workdirs, resource_counts_json,
|
state, resource_counts_json, child_operation_ids_json,
|
||||||
child_operation_ids_json, blockers_json, failure_category,
|
blockers_json, failure_category, created_at, updated_at, completed_at
|
||||||
created_at, updated_at, completed_at
|
|
||||||
FROM workspace_deletion_operations WHERE operation_id = ?1",
|
FROM workspace_deletion_operations WHERE operation_id = ?1",
|
||||||
params![operation_id],
|
params![operation_id],
|
||||||
|row| {
|
|row| {
|
||||||
let state: String = row.get(4)?;
|
let state: String = row.get(4)?;
|
||||||
let resource_counts_json: String = row.get(6)?;
|
let resource_counts_json: String = row.get(5)?;
|
||||||
let child_operation_ids_json: String = row.get(7)?;
|
let child_operation_ids_json: String = row.get(6)?;
|
||||||
let blockers_json: String = row.get(8)?;
|
let blockers_json: String = row.get(7)?;
|
||||||
Ok((
|
Ok((
|
||||||
row.get::<_, String>(0)?,
|
row.get::<_, String>(0)?,
|
||||||
row.get::<_, String>(1)?,
|
row.get::<_, String>(1)?,
|
||||||
row.get::<_, String>(2)?,
|
row.get::<_, String>(2)?,
|
||||||
row.get::<_, String>(3)?,
|
row.get::<_, String>(3)?,
|
||||||
state,
|
state,
|
||||||
row.get::<_, bool>(5)?,
|
|
||||||
resource_counts_json,
|
resource_counts_json,
|
||||||
child_operation_ids_json,
|
child_operation_ids_json,
|
||||||
blockers_json,
|
blockers_json,
|
||||||
row.get::<_, Option<String>>(9)?,
|
row.get::<_, Option<String>>(8)?,
|
||||||
|
row.get::<_, String>(9)?,
|
||||||
row.get::<_, String>(10)?,
|
row.get::<_, String>(10)?,
|
||||||
row.get::<_, String>(11)?,
|
row.get::<_, Option<String>>(11)?,
|
||||||
row.get::<_, Option<String>>(12)?,
|
|
||||||
))
|
))
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -408,7 +394,6 @@ fn read_operation(
|
|||||||
workspace_id,
|
workspace_id,
|
||||||
display_name,
|
display_name,
|
||||||
state,
|
state,
|
||||||
force,
|
|
||||||
resources,
|
resources,
|
||||||
children,
|
children,
|
||||||
blockers,
|
blockers,
|
||||||
@@ -425,7 +410,6 @@ fn read_operation(
|
|||||||
workspace_id,
|
workspace_id,
|
||||||
display_name,
|
display_name,
|
||||||
state: parse_deletion_state(&state)?,
|
state: parse_deletion_state(&state)?,
|
||||||
force_delete_dirty_workdirs: force,
|
|
||||||
resources: serde_json::from_str(&resources)
|
resources: serde_json::from_str(&resources)
|
||||||
.map_err(|error| Error::Store(error.to_string()))?,
|
.map_err(|error| Error::Store(error.to_string()))?,
|
||||||
child_operation_ids: serde_json::from_str(&children)
|
child_operation_ids: serde_json::from_str(&children)
|
||||||
@@ -474,6 +458,98 @@ fn owner_workspace(
|
|||||||
Ok(workspace)
|
Ok(workspace)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn workspace_database_blockers(
|
||||||
|
conn: &rusqlite::Connection,
|
||||||
|
workspace_id: &str,
|
||||||
|
) -> Result<Vec<WorkspaceDeletionBlocker>> {
|
||||||
|
let mut blockers = Vec::new();
|
||||||
|
for (sql, kind, resource_kind, message) in [
|
||||||
|
(
|
||||||
|
"SELECT workdir_id FROM worker_workdir_links WHERE workspace_id = ?1 AND unlinked_at IS NULL",
|
||||||
|
WorkspaceDeletionBlockerKind::WorkdirRemovalBlocked,
|
||||||
|
"workdir",
|
||||||
|
"Release this active Worker–Workdir attachment before deleting the Workspace.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"SELECT workdir_id FROM worker_workdir_attachment_reservations WHERE workspace_id = ?1",
|
||||||
|
WorkspaceDeletionBlockerKind::WorkdirRemovalBlocked,
|
||||||
|
"workdir",
|
||||||
|
"Wait for or cancel this pending Workdir attachment reservation.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"SELECT display_name FROM worker_registry WHERE workspace_id = ?1 AND retention_state = 'pinned'",
|
||||||
|
WorkspaceDeletionBlockerKind::RetentionHold,
|
||||||
|
"worker",
|
||||||
|
"Remove this Worker retention pin before deleting the Workspace.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"SELECT workdir_id FROM workdir_registry WHERE workspace_id = ?1 AND COALESCE(cleanliness, '') != 'clean'",
|
||||||
|
WorkspaceDeletionBlockerKind::DirtyWorkdir,
|
||||||
|
"workdir",
|
||||||
|
"Clean this Workdir and refresh unknown cleanliness before deleting the Workspace.",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
for resource_key in query_resource_keys(conn, sql, workspace_id)? {
|
||||||
|
blockers.push(WorkspaceDeletionBlocker {
|
||||||
|
kind,
|
||||||
|
resource_kind: Some(resource_kind.to_string()),
|
||||||
|
resource_key: Some(resource_key),
|
||||||
|
message: message.to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (sql, kind, resource_kind, message) in [
|
||||||
|
(
|
||||||
|
"SELECT COUNT(*) FROM ticket_current_worker_assignments WHERE workspace_id = ?1",
|
||||||
|
WorkspaceDeletionBlockerKind::WorkerRemovalBlocked,
|
||||||
|
"ticket",
|
||||||
|
"Remove current Ticket assignments before deleting the Workspace.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"SELECT COUNT(*) FROM worker_removal_operations WHERE workspace_id = ?1 AND state IN ('planned', 'blocked', 'executing', 'failed', 'stale')",
|
||||||
|
WorkspaceDeletionBlockerKind::CleanupUnavailable,
|
||||||
|
"worker",
|
||||||
|
"Resolve pending or failed Worker removal operations first.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"SELECT COUNT(*) FROM workdir_removal_operations WHERE workspace_id = ?1 AND state IN ('pending', 'failed')",
|
||||||
|
WorkspaceDeletionBlockerKind::CleanupUnavailable,
|
||||||
|
"workdir",
|
||||||
|
"Resolve pending or failed Workdir removal operations first.",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"SELECT COUNT(*) FROM workdir_create_operations WHERE workspace_id = ?1 AND state = 'pending'",
|
||||||
|
WorkspaceDeletionBlockerKind::CleanupUnavailable,
|
||||||
|
"workdir",
|
||||||
|
"Wait for pending Workdir creation operations to finish.",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let count: u64 = conn.query_row(sql, params![workspace_id], |row| row.get(0))?;
|
||||||
|
if count != 0 {
|
||||||
|
blockers.push(WorkspaceDeletionBlocker {
|
||||||
|
kind,
|
||||||
|
resource_kind: Some(resource_kind.to_string()),
|
||||||
|
resource_key: None,
|
||||||
|
message: format!("{message} ({count})"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(blockers)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn query_resource_keys(
|
||||||
|
conn: &rusqlite::Connection,
|
||||||
|
sql: &str,
|
||||||
|
workspace_id: &str,
|
||||||
|
) -> Result<Vec<String>> {
|
||||||
|
let mut statement = conn.prepare(sql)?;
|
||||||
|
statement
|
||||||
|
.query_map(params![workspace_id], |row| row.get(0))?
|
||||||
|
.collect::<std::result::Result<Vec<_>, _>>()
|
||||||
|
.map_err(Into::into)
|
||||||
|
}
|
||||||
|
|
||||||
fn resource_counts(
|
fn resource_counts(
|
||||||
conn: &rusqlite::Connection,
|
conn: &rusqlite::Connection,
|
||||||
workspace_id: &str,
|
workspace_id: &str,
|
||||||
@@ -515,8 +591,8 @@ fn request_fingerprint(
|
|||||||
request: &WorkspaceDeletionRequest,
|
request: &WorkspaceDeletionRequest,
|
||||||
) -> String {
|
) -> String {
|
||||||
let canonical = format!(
|
let canonical = format!(
|
||||||
"workspace-delete-v1\0{actor_account_id}\0{workspace_id}\0{}\0{}\0{}",
|
"workspace-delete-v1\0{actor_account_id}\0{workspace_id}\0{}\0{}",
|
||||||
request.expected_revision, request.confirmation, request.force_delete_dirty_workdirs
|
request.expected_revision, request.confirmation
|
||||||
);
|
);
|
||||||
encode_hex(&Sha256::digest(canonical.as_bytes()))
|
encode_hex(&Sha256::digest(canonical.as_bytes()))
|
||||||
}
|
}
|
||||||
@@ -590,8 +666,7 @@ mod tests {
|
|||||||
let request = WorkspaceDeletionRequest {
|
let request = WorkspaceDeletionRequest {
|
||||||
operation_id: "delete-workspace-a".to_string(),
|
operation_id: "delete-workspace-a".to_string(),
|
||||||
expected_revision: preflight.expected_revision,
|
expected_revision: preflight.expected_revision,
|
||||||
confirmation: "delete Alpha".to_string(),
|
confirmation: "Alpha".to_string(),
|
||||||
force_delete_dirty_workdirs: false,
|
|
||||||
};
|
};
|
||||||
let first = store
|
let first = store
|
||||||
.reserve_workspace_deletion(&owner, &workspace_id, &request)
|
.reserve_workspace_deletion(&owner, &workspace_id, &request)
|
||||||
@@ -644,21 +719,54 @@ mod tests {
|
|||||||
let mut request = WorkspaceDeletionRequest {
|
let mut request = WorkspaceDeletionRequest {
|
||||||
operation_id: "delete-alpha-guarded".to_string(),
|
operation_id: "delete-alpha-guarded".to_string(),
|
||||||
expected_revision: "stale".to_string(),
|
expected_revision: "stale".to_string(),
|
||||||
confirmation: "delete Alpha".to_string(),
|
confirmation: "Alpha".to_string(),
|
||||||
force_delete_dirty_workdirs: false,
|
|
||||||
};
|
};
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
store.reserve_workspace_deletion(&owner, &workspace_id, &request),
|
store.reserve_workspace_deletion(&owner, &workspace_id, &request),
|
||||||
Err(Error::WorkspaceConfigConflict(_))
|
Err(Error::WorkspaceConfigConflict(_))
|
||||||
));
|
));
|
||||||
request.expected_revision = preflight.expected_revision;
|
request.expected_revision = preflight.expected_revision;
|
||||||
request.confirmation = "Alpha".to_string();
|
request.confirmation = "delete Alpha".to_string();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
store.reserve_workspace_deletion(&owner, &workspace_id, &request),
|
store.reserve_workspace_deletion(&owner, &workspace_id, &request),
|
||||||
Err(Error::InvalidInput(_))
|
Err(Error::InvalidInput(_))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pinned_worker_blocks_preflight_before_operation_reservation() {
|
||||||
|
let (store, owner, workspace_id) = setup();
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO worker_registry (
|
||||||
|
workspace_id, runtime_id, worker_id, display_name,
|
||||||
|
created_at, updated_at, retention_state
|
||||||
|
) VALUES (?1, 'runtime-a', 'worker-a', 'Pinned worker', '1', '1', 'pinned')",
|
||||||
|
params![workspace_id],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.expect("worker");
|
||||||
|
let preflight = store
|
||||||
|
.workspace_deletion_preflight(&owner, &workspace_id)
|
||||||
|
.expect("preflight");
|
||||||
|
assert!(!preflight.can_delete);
|
||||||
|
assert!(preflight.blockers.iter().any(|blocker| {
|
||||||
|
blocker.kind == WorkspaceDeletionBlockerKind::RetentionHold
|
||||||
|
&& blocker.resource_key.as_deref() == Some("Pinned worker")
|
||||||
|
}));
|
||||||
|
let request = WorkspaceDeletionRequest {
|
||||||
|
operation_id: "delete-pinned".to_string(),
|
||||||
|
expected_revision: preflight.expected_revision,
|
||||||
|
confirmation: "Alpha".to_string(),
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
store.reserve_workspace_deletion(&owner, &workspace_id, &request),
|
||||||
|
Err(Error::WorkspaceConfigConflict(_))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn last_accessible_workspace_and_revision_conflicts_fail_closed() {
|
fn last_accessible_workspace_and_revision_conflicts_fail_closed() {
|
||||||
let (store, owner, workspace_id) = setup();
|
let (store, owner, workspace_id) = setup();
|
||||||
@@ -675,8 +783,7 @@ mod tests {
|
|||||||
&WorkspaceDeletionRequest {
|
&WorkspaceDeletionRequest {
|
||||||
operation_id: "delete-beta".to_string(),
|
operation_id: "delete-beta".to_string(),
|
||||||
expected_revision: preflight.expected_revision,
|
expected_revision: preflight.expected_revision,
|
||||||
confirmation: "delete Beta".to_string(),
|
confirmation: "Beta".to_string(),
|
||||||
force_delete_dirty_workdirs: false,
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.expect("reserve")
|
.expect("reserve")
|
||||||
|
|||||||
@@ -91,7 +91,6 @@ export type WorkspaceDeletionPreflightResponse = {
|
|||||||
*/
|
*/
|
||||||
expected_revision: string;
|
expected_revision: string;
|
||||||
can_delete: boolean;
|
can_delete: boolean;
|
||||||
force_delete_dirty_workdirs_available: boolean;
|
|
||||||
resources: WorkspaceDeletionResourceCounts;
|
resources: WorkspaceDeletionResourceCounts;
|
||||||
blockers: Array<WorkspaceDeletionBlocker>;
|
blockers: Array<WorkspaceDeletionBlocker>;
|
||||||
};
|
};
|
||||||
@@ -100,7 +99,6 @@ export type WorkspaceDeletionRequest = {
|
|||||||
operation_id: string;
|
operation_id: string;
|
||||||
expected_revision: string;
|
expected_revision: string;
|
||||||
confirmation: string;
|
confirmation: string;
|
||||||
force_delete_dirty_workdirs: boolean;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export type WorkspaceDeletionOperationResponse = {
|
export type WorkspaceDeletionOperationResponse = {
|
||||||
@@ -108,7 +106,6 @@ export type WorkspaceDeletionOperationResponse = {
|
|||||||
workspace_id: string;
|
workspace_id: string;
|
||||||
display_name: string;
|
display_name: string;
|
||||||
state: WorkspaceDeletionState;
|
state: WorkspaceDeletionState;
|
||||||
force_delete_dirty_workdirs: boolean;
|
|
||||||
resources: WorkspaceDeletionResourceCounts;
|
resources: WorkspaceDeletionResourceCounts;
|
||||||
child_operation_ids: Array<string>;
|
child_operation_ids: Array<string>;
|
||||||
blockers: Array<WorkspaceDeletionBlocker>;
|
blockers: Array<WorkspaceDeletionBlocker>;
|
||||||
|
|||||||
@@ -665,7 +665,6 @@ export function parseWorkspaceDeletionPreflightResponse(
|
|||||||
"display_name",
|
"display_name",
|
||||||
"expected_revision",
|
"expected_revision",
|
||||||
"can_delete",
|
"can_delete",
|
||||||
"force_delete_dirty_workdirs_available",
|
|
||||||
"resources",
|
"resources",
|
||||||
"blockers",
|
"blockers",
|
||||||
], "Workspace deletion preflight");
|
], "Workspace deletion preflight");
|
||||||
@@ -686,10 +685,6 @@ export function parseWorkspaceDeletionPreflightResponse(
|
|||||||
item.can_delete,
|
item.can_delete,
|
||||||
"Workspace deletion preflight.can_delete",
|
"Workspace deletion preflight.can_delete",
|
||||||
),
|
),
|
||||||
force_delete_dirty_workdirs_available: boolean(
|
|
||||||
item.force_delete_dirty_workdirs_available,
|
|
||||||
"Workspace deletion preflight.force_delete_dirty_workdirs_available",
|
|
||||||
),
|
|
||||||
resources: deletionResourceCounts(
|
resources: deletionResourceCounts(
|
||||||
item.resources,
|
item.resources,
|
||||||
"Workspace deletion preflight.resources",
|
"Workspace deletion preflight.resources",
|
||||||
@@ -713,7 +708,6 @@ export function parseWorkspaceDeletionOperationResponse(
|
|||||||
"workspace_id",
|
"workspace_id",
|
||||||
"display_name",
|
"display_name",
|
||||||
"state",
|
"state",
|
||||||
"force_delete_dirty_workdirs",
|
|
||||||
"resources",
|
"resources",
|
||||||
"child_operation_ids",
|
"child_operation_ids",
|
||||||
"blockers",
|
"blockers",
|
||||||
@@ -736,10 +730,6 @@ export function parseWorkspaceDeletionOperationResponse(
|
|||||||
"Workspace deletion operation.display_name",
|
"Workspace deletion operation.display_name",
|
||||||
),
|
),
|
||||||
state: deletionState(item.state, "Workspace deletion operation.state"),
|
state: deletionState(item.state, "Workspace deletion operation.state"),
|
||||||
force_delete_dirty_workdirs: boolean(
|
|
||||||
item.force_delete_dirty_workdirs,
|
|
||||||
"Workspace deletion operation.force_delete_dirty_workdirs",
|
|
||||||
),
|
|
||||||
resources: deletionResourceCounts(
|
resources: deletionResourceCounts(
|
||||||
item.resources,
|
item.resources,
|
||||||
"Workspace deletion operation.resources",
|
"Workspace deletion operation.resources",
|
||||||
|
|||||||
@@ -32,7 +32,6 @@
|
|||||||
let deletionLoading = $state(false);
|
let deletionLoading = $state(false);
|
||||||
let deletionSubmitting = $state(false);
|
let deletionSubmitting = $state(false);
|
||||||
let deletionConfirmation = $state('');
|
let deletionConfirmation = $state('');
|
||||||
let forceDeleteDirtyWorkdirs = $state(false);
|
|
||||||
let deletionPreflight = $state<WorkspaceDeletionPreflightResponse | null>(null);
|
let deletionPreflight = $state<WorkspaceDeletionPreflightResponse | null>(null);
|
||||||
let deletionOperation = $state<WorkspaceDeletionOperationResponse | null>(null);
|
let deletionOperation = $state<WorkspaceDeletionOperationResponse | null>(null);
|
||||||
let deletionOperationId = $state('');
|
let deletionOperationId = $state('');
|
||||||
@@ -95,7 +94,6 @@
|
|||||||
deletionOperation = null;
|
deletionOperation = null;
|
||||||
deletionOperationId = crypto.randomUUID();
|
deletionOperationId = crypto.randomUUID();
|
||||||
deletionConfirmation = '';
|
deletionConfirmation = '';
|
||||||
forceDeleteDirtyWorkdirs = false;
|
|
||||||
try {
|
try {
|
||||||
deletionPreflight = await preflightWorkspaceDeletion(workspaceId);
|
deletionPreflight = await preflightWorkspaceDeletion(workspaceId);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -114,7 +112,6 @@
|
|||||||
operation_id: deletionOperationId,
|
operation_id: deletionOperationId,
|
||||||
expected_revision: deletionPreflight.expected_revision,
|
expected_revision: deletionPreflight.expected_revision,
|
||||||
confirmation: deletionConfirmation,
|
confirmation: deletionConfirmation,
|
||||||
force_delete_dirty_workdirs: forceDeleteDirtyWorkdirs,
|
|
||||||
});
|
});
|
||||||
deletionOperation = operation;
|
deletionOperation = operation;
|
||||||
while (operation.state === 'queued' || operation.state === 'running') {
|
while (operation.state === 'queued' || operation.state === 'running') {
|
||||||
@@ -207,15 +204,9 @@
|
|||||||
<p class="status-message error">{blocker.message}</p>
|
<p class="status-message error">{blocker.message}</p>
|
||||||
{/each}
|
{/each}
|
||||||
<label>
|
<label>
|
||||||
<span>Type <strong>delete {deletionPreflight.display_name}</strong> to confirm</span>
|
<span>Type <strong>{deletionPreflight.display_name}</strong> to confirm</span>
|
||||||
<input bind:value={deletionConfirmation} autocomplete="off" />
|
<input bind:value={deletionConfirmation} autocomplete="off" />
|
||||||
</label>
|
</label>
|
||||||
{#if deletionPreflight.force_delete_dirty_workdirs_available}
|
|
||||||
<label class="force-delete-option">
|
|
||||||
<input type="checkbox" bind:checked={forceDeleteDirtyWorkdirs} disabled={deletionOperation !== null} />
|
|
||||||
<span>Force-delete dirty Workdirs after reviewing the impact</span>
|
|
||||||
</label>
|
|
||||||
{/if}
|
|
||||||
{/if}
|
{/if}
|
||||||
{#if deletionOperation}
|
{#if deletionOperation}
|
||||||
<p class="status-message">Deletion state: {deletionOperation.state}</p>
|
<p class="status-message">Deletion state: {deletionOperation.state}</p>
|
||||||
@@ -230,7 +221,7 @@
|
|||||||
class="danger-button"
|
class="danger-button"
|
||||||
type="button"
|
type="button"
|
||||||
onclick={() => void deleteWorkspace()}
|
onclick={() => void deleteWorkspace()}
|
||||||
disabled={deletionSubmitting || !deletionPreflight?.can_delete || deletionConfirmation !== `delete ${deletionPreflight?.display_name ?? ''}`}
|
disabled={deletionSubmitting || !deletionPreflight?.can_delete || deletionConfirmation !== (deletionPreflight?.display_name ?? '')}
|
||||||
>{deletionSubmitting ? 'Deleting…' : 'Delete Workspace'}</button>
|
>{deletionSubmitting ? 'Deleting…' : 'Delete Workspace'}</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -244,6 +235,5 @@
|
|||||||
.modal-backdrop { position: fixed; inset: 0; z-index: 100; display: grid; place-items: center; padding: var(--space-4); background: rgb(0 0 0 / 0.55); }
|
.modal-backdrop { position: fixed; inset: 0; z-index: 100; display: grid; place-items: center; padding: var(--space-4); background: rgb(0 0 0 / 0.55); }
|
||||||
.deletion-dialog { width: min(34rem, 100%); max-height: calc(100vh - 2rem); overflow: auto; padding: var(--space-5); background: var(--color-surface, white); border: 1px solid var(--color-border); }
|
.deletion-dialog { width: min(34rem, 100%); max-height: calc(100vh - 2rem); overflow: auto; padding: var(--space-5); background: var(--color-surface, white); border: 1px solid var(--color-border); }
|
||||||
.deletion-dialog label { display: grid; gap: var(--space-2); margin-block: var(--space-4); }
|
.deletion-dialog label { display: grid; gap: var(--space-2); margin-block: var(--space-4); }
|
||||||
.force-delete-option { grid-template-columns: auto 1fr !important; align-items: start; }
|
|
||||||
.dialog-actions { display: flex; justify-content: flex-end; gap: var(--space-2); margin-top: var(--space-5); }
|
.dialog-actions { display: flex; justify-content: flex-end; gap: var(--space-2); margin-top: var(--space-5); }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -117,7 +117,6 @@ Deno.test("Workspace deletion DTOs fail closed and preserve durable operation st
|
|||||||
display_name: "Alpha",
|
display_name: "Alpha",
|
||||||
expected_revision: "2026-01-01T00:00:00Z",
|
expected_revision: "2026-01-01T00:00:00Z",
|
||||||
can_delete: true,
|
can_delete: true,
|
||||||
force_delete_dirty_workdirs_available: true,
|
|
||||||
resources: {
|
resources: {
|
||||||
workers: 2,
|
workers: 2,
|
||||||
workdirs: 1,
|
workdirs: 1,
|
||||||
@@ -137,7 +136,6 @@ Deno.test("Workspace deletion DTOs fail closed and preserve durable operation st
|
|||||||
workspace_id: "workspace-a",
|
workspace_id: "workspace-a",
|
||||||
display_name: "Alpha",
|
display_name: "Alpha",
|
||||||
state: "blocked",
|
state: "blocked",
|
||||||
force_delete_dirty_workdirs: false,
|
|
||||||
resources: preflight.resources,
|
resources: preflight.resources,
|
||||||
child_operation_ids: ["worker-remove:arcadia/7"],
|
child_operation_ids: ["worker-remove:arcadia/7"],
|
||||||
blockers: [{
|
blockers: [{
|
||||||
@@ -185,8 +183,7 @@ Deno.test("Workspace settings exposes owner-gated typed destructive confirmation
|
|||||||
"permissions.delete_workspace",
|
"permissions.delete_workspace",
|
||||||
"preflightWorkspaceDeletion",
|
"preflightWorkspaceDeletion",
|
||||||
"startWorkspaceDeletion",
|
"startWorkspaceDeletion",
|
||||||
"delete ${",
|
"deletionConfirmation",
|
||||||
"force_delete_dirty_workdirs",
|
|
||||||
"disposeWorkspaceMultiplexer(workspaceId)",
|
"disposeWorkspaceMultiplexer(workspaceId)",
|
||||||
]
|
]
|
||||||
) {
|
) {
|
||||||
|
|||||||
Reference in New Issue
Block a user