fix: fence removal while permitting safe Workdir reconnection

This commit is contained in:
2026-10-09 16:51:59 +09:00
parent ef642b34be
commit ed50847afe
3 changed files with 111 additions and 2 deletions
+93 -2
View File
@@ -17775,9 +17775,15 @@ async fn scoped_attach_current_worker_workdir(
code: "working_directory_not_found".to_string(), code: "working_directory_not_found".to_string(),
message: format!("unknown Workdir `{workdir_id}`"), message: format!("unknown Workdir `{workdir_id}`"),
})?; })?;
if workdir.materialization_status != "present" { // A failed removal must not prevent its owner from attaching a still usable
// repository checkout to resolve current blockers. Runtime binding validates
// the actual checkout; durable pending-removal reservations still fence this
// mutation. Other sources and missing/corrupted materializations stay denied.
let repository_retry = workdir.materialization_status == "pending"
&& matches!(workdir.source, WorkdirRegistrySource::Repository { .. });
if workdir.materialization_status != "present" && !repository_retry {
return Err(Error::WorkdirAttachmentConflict(format!( return Err(Error::WorkdirAttachmentConflict(format!(
"Workdir `{workdir_id}` is not active ({})", "Workdir `{workdir_id}` is not available for attachment ({})",
workdir.materialization_status workdir.materialization_status
)) ))
.into()); .into());
@@ -53649,6 +53655,91 @@ mod tests {
assert_eq!(response.status(), StatusCode::FORBIDDEN); assert_eq!(response.status(), StatusCode::FORBIDDEN);
} }
#[tokio::test]
async fn current_worker_can_reattach_usable_pending_checkout_after_failed_removal_but_not_during_attempt()
{
let mut fixture = manual_worker_assignment_fixture().await;
let identity =
worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id)
.unwrap();
configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id);
let api = &fixture.api;
api.store
.detach_worker_workdir(
TEST_WORKSPACE_ID,
&fixture.worker,
Some(&fixture.docs_workdir_id),
"release for removal",
)
.unwrap();
let mut record = api
.store
.get_workdir_registry(TEST_WORKSPACE_ID, &fixture.docs_workdir_id)
.unwrap()
.unwrap();
record.materialization_status = "pending".into();
api.store.upsert_workdir_registry(&record).unwrap();
let intent =
workdir_removal_intent(&record, "account:owner", "remove docs checkout").unwrap();
let pending = api
.config_store
.reserve_workdir_removal_operation(&intent)
.unwrap();
let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-attachments");
let request = || {
runtime_source_request(
&identity,
Some(&fixture.worker.worker_id),
"POST",
&path,
serde_json::to_vec(
&json!({ "alias": "docs", "working_directory_id": fixture.docs_workdir_id }),
)
.unwrap(),
)
};
let blocked = build_router(api.clone()).oneshot(request()).await.unwrap();
let blocked_status = blocked.status();
let blocked_body = to_bytes(blocked.into_body(), usize::MAX).await.unwrap();
assert_eq!(
blocked_status,
StatusCode::CONFLICT,
"{}",
String::from_utf8_lossy(&blocked_body)
);
api.config_store
.fail_workdir_removal_operation(&pending, "provider_unavailable", true)
.unwrap();
let attached = build_router(api.clone()).oneshot(request()).await.unwrap();
let attached_status = attached.status();
let attached_body = to_bytes(attached.into_body(), usize::MAX).await.unwrap();
assert_eq!(
attached_status,
StatusCode::OK,
"{}",
String::from_utf8_lossy(&attached_body)
);
let attached: server_api::CurrentWorkerWorkdirAttachmentResponse =
serde_json::from_slice(&attached_body).unwrap();
assert!(attached.attached);
assert_eq!(attached.working_directory_id, fixture.docs_workdir_id);
assert_eq!(
api.store
.get_workdir_registry(TEST_WORKSPACE_ID, &fixture.docs_workdir_id)
.unwrap()
.unwrap()
.materialization_status,
"pending"
);
assert!(
api.config_store
.workdir_removal_guards(&pending)
.unwrap()
.iter()
.any(|guard| guard.category == "active_attachment")
);
}
#[tokio::test] #[tokio::test]
async fn current_worker_attachment_list_and_stale_detach_use_signed_identity() { async fn current_worker_attachment_list_and_stale_detach_use_signed_identity() {
let mut fixture = manual_worker_assignment_fixture().await; let mut fixture = manual_worker_assignment_fixture().await;
+13
View File
@@ -10254,6 +10254,19 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
record.workdir_id, active.worker.runtime_id, active.worker.worker_id record.workdir_id, active.worker.runtime_id, active.worker.worker_id
))); )));
} }
// Direct attachments (including ordinary recovery/reconnection)
// must share the same atomic removal fence as reserved attachments.
let removal_pending: bool = tx.query_row(
"SELECT EXISTS(SELECT 1 FROM workdir_removal_operations WHERE workspace_id=?1 AND workdir_id=?2 AND state='pending')",
params![record.workspace_id, record.workdir_id],
|row| row.get(0),
)?;
if removal_pending {
return Err(Error::WorkdirAttachmentConflict(format!(
"Workdir {} has a pending durable removal operation",
record.workdir_id
)));
}
let write = tx.execute( let write = tx.execute(
r#"INSERT INTO worker_workdir_links ( r#"INSERT INTO worker_workdir_links (
workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
+5
View File
@@ -10,6 +10,11 @@ reported cause. Each attempt rechecks current attachments, attachment
reservations, retention holds, materialization identity and current file changes. reservations, retention holds, materialization identity and current file changes.
A previous clean observation is not permission to discard new changes. A dirty A previous clean observation is not permission to discard new changes. A dirty
Workdir must be preserved or its changes resolved by its owner before deletion. Workdir must be preserved or its changes resolved by its owner before deletion.
When the same repository checkout is still usable, an authorized Worker may
reattach it through normal `WorkdirAttach` to preserve or resolve changes, then
detach and retry removal. An executing removal fences new attachments; a failed
attempt does not permanently prevent reconnection. Missing or replaced checkout
identity is not made usable by this permission.
## Interpreting a retained Workdir ## Interpreting a retained Workdir