From d2cb50d081336f0a8b5e6aaa912eb323033b7d8a Mon Sep 17 00:00:00 2001 From: Hare Date: Sun, 13 Sep 2026 02:54:01 +0900 Subject: [PATCH] fix: fence runtime removal races and retries --- crates/workspace-server/src/latest_schema.sql | 144 ++++++++++++ crates/workspace-server/src/store.rs | 212 ++++++++++++++++-- .../lib/workspace/api/runtime-management.ts | 17 ++ .../runtimes/[runtimeId]/+page.svelte | 7 +- .../tests/runtime-management-source.test.ts | 4 +- .../tests/runtime-management.test.ts | 53 +++++ 6 files changed, 416 insertions(+), 21 deletions(-) diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index 8e00b1ac..b2d8c083 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -1107,6 +1107,150 @@ BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; +CREATE TRIGGER worker_registry_insert_blocked_by_runtime_removal +BEFORE INSERT ON worker_registry FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER worker_registry_update_blocked_by_runtime_removal +BEFORE UPDATE ON worker_registry FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER workdir_registry_insert_blocked_by_runtime_removal +BEFORE INSERT ON workdir_registry FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER workdir_registry_update_blocked_by_runtime_removal +BEFORE UPDATE ON workdir_registry FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER worker_assignment_insert_blocked_by_runtime_removal +BEFORE INSERT ON ticket_current_worker_assignments FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER worker_assignment_update_blocked_by_runtime_removal +BEFORE UPDATE ON ticket_current_worker_assignments FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER workdir_attachment_insert_blocked_by_runtime_removal +BEFORE INSERT ON worker_workdir_links FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER workdir_attachment_update_blocked_by_runtime_removal +BEFORE UPDATE ON worker_workdir_links FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER worker_create_insert_blocked_by_runtime_removal +BEFORE INSERT ON worker_create_reservations FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER worker_create_update_blocked_by_runtime_removal +BEFORE UPDATE ON worker_create_reservations FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER workdir_create_insert_blocked_by_runtime_removal +BEFORE INSERT ON workdir_create_operations FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.resolved_runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER workdir_create_update_blocked_by_runtime_removal +BEFORE UPDATE ON workdir_create_operations FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.resolved_runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER worker_removal_insert_blocked_by_runtime_removal +BEFORE INSERT ON worker_removal_operations FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER worker_removal_update_blocked_by_runtime_removal +BEFORE UPDATE ON worker_removal_operations FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER workdir_removal_insert_blocked_by_runtime_removal +BEFORE INSERT ON workdir_removal_operations FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + +CREATE TRIGGER workdir_removal_update_blocked_by_runtime_removal +BEFORE UPDATE ON workdir_removal_operations FOR EACH ROW +WHEN EXISTS ( + SELECT 1 FROM runtime_removal_operations operation + WHERE operation.runtime_id = NEW.runtime_id + AND operation.state IN ('pending', 'cleanup_pending') +) +BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TABLE workspace_deletion_operations ( operation_id TEXT PRIMARY KEY, request_fingerprint TEXT NOT NULL, diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 800e7d50..5833be3c 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -9587,7 +9587,8 @@ fn verify_canonical_workspace_runtime_binding(binding: WorkspaceRuntimeBinding) } fn migrate_runtime_removal_operations_v59_to_v60(conn: &Connection) -> Result<()> { - conn.execute_batch( + let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; + tx.execute_batch( r#"CREATE TABLE runtime_removal_operations ( operation_id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL, @@ -9632,13 +9633,82 @@ fn migrate_runtime_removal_operations_v59_to_v60(conn: &Connection) -> Result<() SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;"#, )?; - conn.execute( + tx.execute_batch( + r#" + CREATE TRIGGER worker_registry_insert_blocked_by_runtime_removal + BEFORE INSERT ON worker_registry FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER worker_registry_update_blocked_by_runtime_removal + BEFORE UPDATE ON worker_registry FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER workdir_registry_insert_blocked_by_runtime_removal + BEFORE INSERT ON workdir_registry FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER workdir_registry_update_blocked_by_runtime_removal + BEFORE UPDATE ON workdir_registry FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER worker_assignment_insert_blocked_by_runtime_removal + BEFORE INSERT ON ticket_current_worker_assignments FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER worker_assignment_update_blocked_by_runtime_removal + BEFORE UPDATE ON ticket_current_worker_assignments FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER workdir_attachment_insert_blocked_by_runtime_removal + BEFORE INSERT ON worker_workdir_links FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER workdir_attachment_update_blocked_by_runtime_removal + BEFORE UPDATE ON worker_workdir_links FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER worker_create_insert_blocked_by_runtime_removal + BEFORE INSERT ON worker_create_reservations FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER worker_create_update_blocked_by_runtime_removal + BEFORE UPDATE ON worker_create_reservations FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER workdir_create_insert_blocked_by_runtime_removal + BEFORE INSERT ON workdir_create_operations FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.resolved_runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER workdir_create_update_blocked_by_runtime_removal + BEFORE UPDATE ON workdir_create_operations FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.resolved_runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER worker_removal_insert_blocked_by_runtime_removal + BEFORE INSERT ON worker_removal_operations FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER worker_removal_update_blocked_by_runtime_removal + BEFORE UPDATE ON worker_removal_operations FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER workdir_removal_insert_blocked_by_runtime_removal + BEFORE INSERT ON workdir_removal_operations FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + CREATE TRIGGER workdir_removal_update_blocked_by_runtime_removal + BEFORE UPDATE ON workdir_removal_operations FOR EACH ROW + WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending')) + BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END; + "#, + )?; + tx.execute( "INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)", params![ LATEST_SCHEMA_VERSION, RUNTIME_REMOVAL_OPERATION_MIGRATION_NAME ], )?; + tx.commit()?; Ok(()) } @@ -10438,7 +10508,7 @@ mod tests { #[test] fn runtime_removal_rejects_shared_binding_and_fences_new_binding_until_cleanup() { - let (_temp, store) = runtime_removal_test_store(); + let (temp, store) = runtime_removal_test_store(); store .with_conn(|conn| { conn.execute_batch( @@ -10500,26 +10570,50 @@ mod tests { 1, ) .unwrap(); - let blocked_insert = store.with_conn(|conn| { - conn.execute( - "INSERT INTO workspace_runtime_bindings(\ - workspace_id, runtime_id, display_name, base_url, public_key, \ - public_key_fingerprint, binding_revision, state, authentication_mode, \ - created_at, updated_at\ - ) VALUES (\ - 'workspace-b', 'runtime-a', 'Runtime A', 'https://runtime.invalid', \ - 'key-a', 'fingerprint-b', 1, 'verified', 'legacy_server_issuer', '1', '1'\ - )", - [], - )?; - Ok(()) - }); + let competing_conn = Connection::open(temp.path().join("server.db")).unwrap(); + competing_conn + .execute_batch("PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;") + .unwrap(); + let blocked_insert = competing_conn.execute( + "INSERT INTO workspace_runtime_bindings(\ + workspace_id, runtime_id, display_name, base_url, public_key, \ + public_key_fingerprint, binding_revision, state, authentication_mode, \ + created_at, updated_at\ + ) VALUES (\ + 'workspace-b', 'runtime-a', 'Runtime A', 'https://runtime.invalid', \ + 'key-a', 'fingerprint-b', 1, 'verified', 'legacy_server_issuer', '1', '1'\ + )", + [], + ); assert!( blocked_insert .unwrap_err() .to_string() .contains("runtime_removal_in_progress") ); + + for competing_mutation in [ + "INSERT INTO worker_registry(\ + workspace_id, worker_id, runtime_id, display_name, retention_state, created_at, updated_at\ + ) VALUES ('workspace-a', 'racing-worker', 'runtime-a', 'Racing Worker', 'normal', '2', '2')", + "INSERT INTO worker_create_reservations(\ + workspace_id, allocation_key, worker_id, runtime_id, create_fingerprint, state, created_at, updated_at\ + ) VALUES ('workspace-a', 'racing-allocation', 'racing-worker', 'runtime-a', 'racing-create', 'reserved', '2', '2')", + "INSERT INTO workdir_create_operations(\ + workspace_id, operation_id, request_fingerprint, repository_id, resolved_runtime_id, \ + config_revision, config_projection_digest, working_directory_id, state, created_at, updated_at\ + ) VALUES (\ + 'workspace-a', 'racing-workdir-create', 'racing-request', 'repository-a', 'runtime-a', \ + 1, 'racing-projection', 'racing-workdir', 'pending', '2', '2'\ + )", + ] { + let error = competing_conn.execute(competing_mutation, []).unwrap_err(); + assert!( + error.to_string().contains("runtime_removal_in_progress"), + "competing Runtime resource mutation was not fenced: {error}" + ); + } + assert_runtime_removal_binding_unchanged(&store); } #[test] @@ -10602,6 +10696,22 @@ mod tests { conn.execute_batch( "DROP TRIGGER runtime_binding_insert_blocked_by_removal; \ DROP TRIGGER runtime_binding_update_blocked_by_removal; \ + DROP TRIGGER worker_registry_insert_blocked_by_runtime_removal; \ + DROP TRIGGER worker_registry_update_blocked_by_runtime_removal; \ + DROP TRIGGER workdir_registry_insert_blocked_by_runtime_removal; \ + DROP TRIGGER workdir_registry_update_blocked_by_runtime_removal; \ + DROP TRIGGER worker_assignment_insert_blocked_by_runtime_removal; \ + DROP TRIGGER worker_assignment_update_blocked_by_runtime_removal; \ + DROP TRIGGER workdir_attachment_insert_blocked_by_runtime_removal; \ + DROP TRIGGER workdir_attachment_update_blocked_by_runtime_removal; \ + DROP TRIGGER worker_create_insert_blocked_by_runtime_removal; \ + DROP TRIGGER worker_create_update_blocked_by_runtime_removal; \ + DROP TRIGGER workdir_create_insert_blocked_by_runtime_removal; \ + DROP TRIGGER workdir_create_update_blocked_by_runtime_removal; \ + DROP TRIGGER worker_removal_insert_blocked_by_runtime_removal; \ + DROP TRIGGER worker_removal_update_blocked_by_runtime_removal; \ + DROP TRIGGER workdir_removal_insert_blocked_by_runtime_removal; \ + DROP TRIGGER workdir_removal_update_blocked_by_runtime_removal; \ DROP TABLE runtime_removal_operations; \ DELETE FROM __yoi_schema_migrations; \ INSERT INTO __yoi_schema_migrations(version, name) \ @@ -10623,7 +10733,10 @@ mod tests { )?; let trigger_count: i64 = conn.query_row( "SELECT COUNT(*) FROM sqlite_master \ - WHERE type = 'trigger' AND name LIKE 'runtime_binding_%_blocked_by_removal'", + WHERE type = 'trigger' AND (\ + name LIKE '%_blocked_by_runtime_removal' OR \ + name LIKE 'runtime_binding_%_blocked_by_removal'\ + )", [], |row| row.get(0), )?; @@ -10632,13 +10745,58 @@ mod tests { row.get(0) })?; assert_eq!(table_count, 1); - assert_eq!(trigger_count, 2); + assert_eq!(trigger_count, 18); assert_eq!(foreign_key_failures, 0); Ok(()) }) .unwrap(); } + #[test] + fn schema_v59_runtime_removal_migration_rolls_back_partial_ddl_and_marker() { + let temp = tempfile::tempdir().unwrap(); + let store = SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap(); + store + .with_conn(|conn| { + let trigger_names = { + let mut stmt = conn.prepare( + "SELECT name FROM sqlite_master \ + WHERE type = 'trigger' AND sql LIKE '%runtime_removal_operations%'", + )?; + stmt.query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()? + }; + for trigger_name in trigger_names { + conn.execute_batch(&format!("DROP TRIGGER \"{trigger_name}\";"))?; + } + conn.execute_batch( + "DROP TABLE runtime_removal_operations; \ + DELETE FROM __yoi_schema_migrations WHERE version = 60; \ + CREATE INDEX runtime_removal_operations_one_active_runtime \ + ON workspace_runtime_bindings(runtime_id);", + )?; + + let migration_error = migrate_runtime_removal_operations_v59_to_v60(conn) + .expect_err("injected index-name collision must fail migration"); + assert!(migration_error.to_string().contains("already exists")); + let table_count: i64 = conn.query_row( + "SELECT COUNT(*) FROM sqlite_master \ + WHERE type = 'table' AND name = 'runtime_removal_operations'", + [], + |row| row.get(0), + )?; + let marker_count: i64 = conn.query_row( + "SELECT COUNT(*) FROM __yoi_schema_migrations WHERE version = 60", + [], + |row| row.get(0), + )?; + assert_eq!(table_count, 0, "partial Runtime removal DDL leaked"); + assert_eq!(marker_count, 0, "failed migration advanced schema marker"); + Ok(()) + }) + .unwrap(); + } + #[test] fn current_schema_accepts_every_retained_canonical_provenance() { for baseline_version in OLDEST_SCHEMA_VERSION..=LATEST_SCHEMA_VERSION { @@ -10707,6 +10865,22 @@ mod tests { DROP TABLE workdir_create_credential_candidates; DROP TRIGGER runtime_binding_insert_blocked_by_removal; DROP TRIGGER runtime_binding_update_blocked_by_removal; + DROP TRIGGER worker_registry_insert_blocked_by_runtime_removal; + DROP TRIGGER worker_registry_update_blocked_by_runtime_removal; + DROP TRIGGER workdir_registry_insert_blocked_by_runtime_removal; + DROP TRIGGER workdir_registry_update_blocked_by_runtime_removal; + DROP TRIGGER worker_assignment_insert_blocked_by_runtime_removal; + DROP TRIGGER worker_assignment_update_blocked_by_runtime_removal; + DROP TRIGGER workdir_attachment_insert_blocked_by_runtime_removal; + DROP TRIGGER workdir_attachment_update_blocked_by_runtime_removal; + DROP TRIGGER worker_create_insert_blocked_by_runtime_removal; + DROP TRIGGER worker_create_update_blocked_by_runtime_removal; + DROP TRIGGER workdir_create_insert_blocked_by_runtime_removal; + DROP TRIGGER workdir_create_update_blocked_by_runtime_removal; + DROP TRIGGER worker_removal_insert_blocked_by_runtime_removal; + DROP TRIGGER worker_removal_update_blocked_by_runtime_removal; + DROP TRIGGER workdir_removal_insert_blocked_by_runtime_removal; + DROP TRIGGER workdir_removal_update_blocked_by_runtime_removal; DROP TABLE runtime_removal_operations; DROP INDEX workspace_signing_identity_audit_workspace_idx; DROP TABLE workspace_signing_identity_audit; diff --git a/web/workspace/src/lib/workspace/api/runtime-management.ts b/web/workspace/src/lib/workspace/api/runtime-management.ts index 00fe6470..d7e363cf 100644 --- a/web/workspace/src/lib/workspace/api/runtime-management.ts +++ b/web/workspace/src/lib/workspace/api/runtime-management.ts @@ -1026,6 +1026,23 @@ export async function updateRemoteRuntime( return finishMutation(response, workspaceId, runtimeId); } +export class RuntimeRemovalAttempt { + #operationId: string | null = null; + + operationId(create: () => string = () => crypto.randomUUID()): string { + if (this.#operationId === null) this.#operationId = create(); + return this.#operationId; + } + + complete(operationId: string): void { + if (this.#operationId === operationId) this.#operationId = null; + } + + reset(): void { + this.#operationId = null; + } +} + export async function removeRemoteRuntime( workspaceId: string, runtimeId: string, diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte index 7ce80ef9..49e5167a 100644 --- a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte @@ -13,6 +13,7 @@ updateRemoteRuntime, RuntimeTrustConflictError, RuntimeTrustRouteFence, + RuntimeRemovalAttempt, RuntimeTrustRequestError, type RuntimeTrustRouteOperation, } from '$lib/workspace/api/runtime-management'; @@ -37,6 +38,7 @@ let replacementFingerprintError = $state(null); let fingerprintGeneration = 0; const routeFence = new RuntimeTrustRouteFence(); + const runtimeRemovalAttempt = new RuntimeRemovalAttempt(); let routeGeneration = 0; $effect(() => { @@ -51,6 +53,7 @@ endpoint = data.runtimeDetail?.endpoint ?? ''; editingMetadata = false; deleteRuntimeConfirmation = ''; + runtimeRemovalAttempt.reset(); busyAction = null; fieldError = null; deleteRuntimeError = null; @@ -290,6 +293,7 @@ deleteRuntimeError = 'The authoritative Runtime binding revision is unavailable. Reload before removal.'; return; } + const operationId = runtimeRemovalAttempt.operationId(); busyAction = 'delete'; deleteRuntimeError = null; try { @@ -297,11 +301,12 @@ data.workspaceId, routeOperation.runtimeId, { - operation_id: crypto.randomUUID(), + operation_id: operationId, expected_binding_revision: trust.revision, }, ); if (!isCurrentRoute(routeOperation)) return; + runtimeRemovalAttempt.complete(operationId); await goto(`/w/${encodeURIComponent(data.workspaceId)}/settings/runtimes`, { replaceState: true, }); diff --git a/web/workspace/tests/runtime-management-source.test.ts b/web/workspace/tests/runtime-management-source.test.ts index 37c3229a..080bdc28 100644 --- a/web/workspace/tests/runtime-management-source.test.ts +++ b/web/workspace/tests/runtime-management-source.test.ts @@ -178,7 +178,9 @@ Deno.test("Runtime detail keeps trust controls owner-only and conflict-safe", as "Workspace trust only; this does not delete the Runtime process, Workers, or Workdirs.", "await revokeRuntimeTrustKey(", "await removeRemoteRuntime(", - "operation_id: crypto.randomUUID()", + "new RuntimeRemovalAttempt()", + "runtimeRemovalAttempt.operationId()", + "operation_id: operationId", "expected_binding_revision: trust.revision", "The Backend removes Workspace trust and this Runtime registration as one guarded operation.", "deleteRuntimeConfirmation.trim() !== data.runtimeId", diff --git a/web/workspace/tests/runtime-management.test.ts b/web/workspace/tests/runtime-management.test.ts index 67c9c3bf..05f42dfa 100644 --- a/web/workspace/tests/runtime-management.test.ts +++ b/web/workspace/tests/runtime-management.test.ts @@ -12,6 +12,7 @@ import { previewRuntimePublicKeyFingerprint, removeRemoteRuntime, revokeRuntimeTrustKey, + RuntimeRemovalAttempt, RuntimeTrustConflictError, RuntimeTrustRouteFence, updateRemoteRuntime, @@ -314,6 +315,58 @@ Deno.test("Runtime metadata update never sends public key authority", async () = assert(!("public_key" in body), "metadata update sent public_key"); }); +Deno.test("Runtime removal attempt retains its id across response-loss retry", async () => { + const attempt = new RuntimeRemovalAttempt(); + const operationIds: string[] = []; + let calls = 0; + const submit = async () => { + const operationId = attempt.operationId(() => "stable-removal-operation"); + operationIds.push(operationId); + const operation = await removeRemoteRuntime( + "workspace-a", + "runtime-a", + { operation_id: operationId, expected_binding_revision: 4 }, + () => { + calls += 1; + if (calls === 1) return Promise.reject(new Error("response lost")); + return Promise.resolve(Response.json({ + operation_id: operationId, + workspace_id: "workspace-a", + runtime_id: "runtime-a", + state: "succeeded", + binding_removed: true, + runtime_registration_removed: true, + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-01T00:00:01Z", + completed_at: "2026-01-01T00:00:01Z", + })); + }, + ); + attempt.complete(operation.operation_id); + }; + + try { + await submit(); + throw new Error("expected response-loss retry to fail"); + } catch (error) { + assert( + error instanceof Error && error.message.includes("response lost"), + `unexpected response-loss error: ${String(error)}`, + ); + } + await submit(); + assert( + operationIds.length === 2 && + operationIds.every((id) => id === "stable-removal-operation"), + `response-loss retry changed operation id: ${operationIds.join(",")}`, + ); + assert( + attempt.operationId(() => "next-removal-operation") === + "next-removal-operation", + "authoritative success did not clear the completed operation id", + ); +}); + Deno.test("Runtime removal uses the Workspace-scoped operation route", async () => { let requestedUrl = ""; let requestedMethod = "";