fix: fence runtime removal races and retries

crates/workspace-server/src/latest_schema.sql

@@ -1107,6 +1107,150 @@ BEGIN
     SELECT RAISE(ABORT, 'runtime_removal_in_progress');
 END;
 
+CREATE TRIGGER worker_registry_insert_blocked_by_runtime_removal
+BEFORE INSERT ON worker_registry FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER worker_registry_update_blocked_by_runtime_removal
+BEFORE UPDATE ON worker_registry FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER workdir_registry_insert_blocked_by_runtime_removal
+BEFORE INSERT ON workdir_registry FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER workdir_registry_update_blocked_by_runtime_removal
+BEFORE UPDATE ON workdir_registry FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER worker_assignment_insert_blocked_by_runtime_removal
+BEFORE INSERT ON ticket_current_worker_assignments FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER worker_assignment_update_blocked_by_runtime_removal
+BEFORE UPDATE ON ticket_current_worker_assignments FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER workdir_attachment_insert_blocked_by_runtime_removal
+BEFORE INSERT ON worker_workdir_links FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER workdir_attachment_update_blocked_by_runtime_removal
+BEFORE UPDATE ON worker_workdir_links FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER worker_create_insert_blocked_by_runtime_removal
+BEFORE INSERT ON worker_create_reservations FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER worker_create_update_blocked_by_runtime_removal
+BEFORE UPDATE ON worker_create_reservations FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER workdir_create_insert_blocked_by_runtime_removal
+BEFORE INSERT ON workdir_create_operations FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.resolved_runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER workdir_create_update_blocked_by_runtime_removal
+BEFORE UPDATE ON workdir_create_operations FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.resolved_runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER worker_removal_insert_blocked_by_runtime_removal
+BEFORE INSERT ON worker_removal_operations FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER worker_removal_update_blocked_by_runtime_removal
+BEFORE UPDATE ON worker_removal_operations FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER workdir_removal_insert_blocked_by_runtime_removal
+BEFORE INSERT ON workdir_removal_operations FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
+CREATE TRIGGER workdir_removal_update_blocked_by_runtime_removal
+BEFORE UPDATE ON workdir_removal_operations FOR EACH ROW
+WHEN EXISTS (
+    SELECT 1 FROM runtime_removal_operations operation
+    WHERE operation.runtime_id = NEW.runtime_id
+      AND operation.state IN ('pending', 'cleanup_pending')
+)
+BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+
 CREATE TABLE workspace_deletion_operations (
     operation_id TEXT PRIMARY KEY,
     request_fingerprint TEXT NOT NULL,

crates/workspace-server/src/store.rs

@@ -9587,7 +9587,8 @@ fn verify_canonical_workspace_runtime_binding(binding: WorkspaceRuntimeBinding)
 }
 
 fn migrate_runtime_removal_operations_v59_to_v60(conn: &Connection) -> Result<()> {
-    conn.execute_batch(
+    let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?;
+    tx.execute_batch(
         r#"CREATE TABLE runtime_removal_operations (
              operation_id TEXT PRIMARY KEY,
              workspace_id TEXT NOT NULL,
@@ -9632,13 +9633,82 @@ fn migrate_runtime_removal_operations_v59_to_v60(conn: &Connection) -> Result<()
              SELECT RAISE(ABORT, 'runtime_removal_in_progress');
          END;"#,
     )?;
-    conn.execute(
+    tx.execute_batch(
+        r#"
+        CREATE TRIGGER worker_registry_insert_blocked_by_runtime_removal
+        BEFORE INSERT ON worker_registry FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER worker_registry_update_blocked_by_runtime_removal
+        BEFORE UPDATE ON worker_registry FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER workdir_registry_insert_blocked_by_runtime_removal
+        BEFORE INSERT ON workdir_registry FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER workdir_registry_update_blocked_by_runtime_removal
+        BEFORE UPDATE ON workdir_registry FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER worker_assignment_insert_blocked_by_runtime_removal
+        BEFORE INSERT ON ticket_current_worker_assignments FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER worker_assignment_update_blocked_by_runtime_removal
+        BEFORE UPDATE ON ticket_current_worker_assignments FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER workdir_attachment_insert_blocked_by_runtime_removal
+        BEFORE INSERT ON worker_workdir_links FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER workdir_attachment_update_blocked_by_runtime_removal
+        BEFORE UPDATE ON worker_workdir_links FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER worker_create_insert_blocked_by_runtime_removal
+        BEFORE INSERT ON worker_create_reservations FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER worker_create_update_blocked_by_runtime_removal
+        BEFORE UPDATE ON worker_create_reservations FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER workdir_create_insert_blocked_by_runtime_removal
+        BEFORE INSERT ON workdir_create_operations FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.resolved_runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER workdir_create_update_blocked_by_runtime_removal
+        BEFORE UPDATE ON workdir_create_operations FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.resolved_runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER worker_removal_insert_blocked_by_runtime_removal
+        BEFORE INSERT ON worker_removal_operations FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER worker_removal_update_blocked_by_runtime_removal
+        BEFORE UPDATE ON worker_removal_operations FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER workdir_removal_insert_blocked_by_runtime_removal
+        BEFORE INSERT ON workdir_removal_operations FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        CREATE TRIGGER workdir_removal_update_blocked_by_runtime_removal
+        BEFORE UPDATE ON workdir_removal_operations FOR EACH ROW
+        WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
+        BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
+        "#,
+    )?;
+    tx.execute(
         "INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)",
         params![
             LATEST_SCHEMA_VERSION,
             RUNTIME_REMOVAL_OPERATION_MIGRATION_NAME
         ],
     )?;
+    tx.commit()?;
     Ok(())
 }
 
@@ -10438,7 +10508,7 @@ mod tests {
 
     #[test]
     fn runtime_removal_rejects_shared_binding_and_fences_new_binding_until_cleanup() {
-        let (_temp, store) = runtime_removal_test_store();
+        let (temp, store) = runtime_removal_test_store();
         store
             .with_conn(|conn| {
                 conn.execute_batch(
@@ -10500,8 +10570,11 @@ mod tests {
                 1,
             )
             .unwrap();
-        let blocked_insert = store.with_conn(|conn| {
+        let competing_conn = Connection::open(temp.path().join("server.db")).unwrap();
-            conn.execute(
+        competing_conn
+            .execute_batch("PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;")
+            .unwrap();
+        let blocked_insert = competing_conn.execute(
             "INSERT INTO workspace_runtime_bindings(\
                  workspace_id, runtime_id, display_name, base_url, public_key, \
                  public_key_fingerprint, binding_revision, state, authentication_mode, \
@@ -10511,15 +10584,36 @@ mod tests {
                  'key-a', 'fingerprint-b', 1, 'verified', 'legacy_server_issuer', '1', '1'\
              )",
             [],
-            )?;
+        );
-            Ok(())
-        });
         assert!(
             blocked_insert
                 .unwrap_err()
                 .to_string()
                 .contains("runtime_removal_in_progress")
         );
+
+        for competing_mutation in [
+            "INSERT INTO worker_registry(\
+                 workspace_id, worker_id, runtime_id, display_name, retention_state, created_at, updated_at\
+             ) VALUES ('workspace-a', 'racing-worker', 'runtime-a', 'Racing Worker', 'normal', '2', '2')",
+            "INSERT INTO worker_create_reservations(\
+                 workspace_id, allocation_key, worker_id, runtime_id, create_fingerprint, state, created_at, updated_at\
+             ) VALUES ('workspace-a', 'racing-allocation', 'racing-worker', 'runtime-a', 'racing-create', 'reserved', '2', '2')",
+            "INSERT INTO workdir_create_operations(\
+                 workspace_id, operation_id, request_fingerprint, repository_id, resolved_runtime_id, \
+                 config_revision, config_projection_digest, working_directory_id, state, created_at, updated_at\
+             ) VALUES (\
+                 'workspace-a', 'racing-workdir-create', 'racing-request', 'repository-a', 'runtime-a', \
+                 1, 'racing-projection', 'racing-workdir', 'pending', '2', '2'\
+             )",
+        ] {
+            let error = competing_conn.execute(competing_mutation, []).unwrap_err();
+            assert!(
+                error.to_string().contains("runtime_removal_in_progress"),
+                "competing Runtime resource mutation was not fenced: {error}"
+            );
+        }
+        assert_runtime_removal_binding_unchanged(&store);
     }
 
     #[test]
@@ -10602,6 +10696,22 @@ mod tests {
                 conn.execute_batch(
                     "DROP TRIGGER runtime_binding_insert_blocked_by_removal; \
                      DROP TRIGGER runtime_binding_update_blocked_by_removal; \
+                     DROP TRIGGER worker_registry_insert_blocked_by_runtime_removal; \
+                     DROP TRIGGER worker_registry_update_blocked_by_runtime_removal; \
+                     DROP TRIGGER workdir_registry_insert_blocked_by_runtime_removal; \
+                     DROP TRIGGER workdir_registry_update_blocked_by_runtime_removal; \
+                     DROP TRIGGER worker_assignment_insert_blocked_by_runtime_removal; \
+                     DROP TRIGGER worker_assignment_update_blocked_by_runtime_removal; \
+                     DROP TRIGGER workdir_attachment_insert_blocked_by_runtime_removal; \
+                     DROP TRIGGER workdir_attachment_update_blocked_by_runtime_removal; \
+                     DROP TRIGGER worker_create_insert_blocked_by_runtime_removal; \
+                     DROP TRIGGER worker_create_update_blocked_by_runtime_removal; \
+                     DROP TRIGGER workdir_create_insert_blocked_by_runtime_removal; \
+                     DROP TRIGGER workdir_create_update_blocked_by_runtime_removal; \
+                     DROP TRIGGER worker_removal_insert_blocked_by_runtime_removal; \
+                     DROP TRIGGER worker_removal_update_blocked_by_runtime_removal; \
+                     DROP TRIGGER workdir_removal_insert_blocked_by_runtime_removal; \
+                     DROP TRIGGER workdir_removal_update_blocked_by_runtime_removal; \
                      DROP TABLE runtime_removal_operations; \
                      DELETE FROM __yoi_schema_migrations; \
                      INSERT INTO __yoi_schema_migrations(version, name) \
@@ -10623,7 +10733,10 @@ mod tests {
                 )?;
                 let trigger_count: i64 = conn.query_row(
                     "SELECT COUNT(*) FROM sqlite_master \
-                     WHERE type = 'trigger' AND name LIKE 'runtime_binding_%_blocked_by_removal'",
+                     WHERE type = 'trigger' AND (\
+                         name LIKE '%_blocked_by_runtime_removal' OR \
+                         name LIKE 'runtime_binding_%_blocked_by_removal'\
+                     )",
                     [],
                     |row| row.get(0),
                 )?;
@@ -10632,13 +10745,58 @@ mod tests {
                         row.get(0)
                     })?;
                 assert_eq!(table_count, 1);
-                assert_eq!(trigger_count, 2);
+                assert_eq!(trigger_count, 18);
                 assert_eq!(foreign_key_failures, 0);
                 Ok(())
             })
             .unwrap();
     }
 
+    #[test]
+    fn schema_v59_runtime_removal_migration_rolls_back_partial_ddl_and_marker() {
+        let temp = tempfile::tempdir().unwrap();
+        let store = SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap();
+        store
+            .with_conn(|conn| {
+                let trigger_names = {
+                    let mut stmt = conn.prepare(
+                        "SELECT name FROM sqlite_master \
+                         WHERE type = 'trigger' AND sql LIKE '%runtime_removal_operations%'",
+                    )?;
+                    stmt.query_map([], |row| row.get::<_, String>(0))?
+                        .collect::<std::result::Result<Vec<_>, _>>()?
+                };
+                for trigger_name in trigger_names {
+                    conn.execute_batch(&format!("DROP TRIGGER \"{trigger_name}\";"))?;
+                }
+                conn.execute_batch(
+                    "DROP TABLE runtime_removal_operations; \
+                     DELETE FROM __yoi_schema_migrations WHERE version = 60; \
+                     CREATE INDEX runtime_removal_operations_one_active_runtime \
+                     ON workspace_runtime_bindings(runtime_id);",
+                )?;
+
+                let migration_error = migrate_runtime_removal_operations_v59_to_v60(conn)
+                    .expect_err("injected index-name collision must fail migration");
+                assert!(migration_error.to_string().contains("already exists"));
+                let table_count: i64 = conn.query_row(
+                    "SELECT COUNT(*) FROM sqlite_master \
+                     WHERE type = 'table' AND name = 'runtime_removal_operations'",
+                    [],
+                    |row| row.get(0),
+                )?;
+                let marker_count: i64 = conn.query_row(
+                    "SELECT COUNT(*) FROM __yoi_schema_migrations WHERE version = 60",
+                    [],
+                    |row| row.get(0),
+                )?;
+                assert_eq!(table_count, 0, "partial Runtime removal DDL leaked");
+                assert_eq!(marker_count, 0, "failed migration advanced schema marker");
+                Ok(())
+            })
+            .unwrap();
+    }
+
     #[test]
     fn current_schema_accepts_every_retained_canonical_provenance() {
         for baseline_version in OLDEST_SCHEMA_VERSION..=LATEST_SCHEMA_VERSION {
@@ -10707,6 +10865,22 @@ mod tests {
             DROP TABLE workdir_create_credential_candidates;
             DROP TRIGGER runtime_binding_insert_blocked_by_removal;
             DROP TRIGGER runtime_binding_update_blocked_by_removal;
+            DROP TRIGGER worker_registry_insert_blocked_by_runtime_removal;
+            DROP TRIGGER worker_registry_update_blocked_by_runtime_removal;
+            DROP TRIGGER workdir_registry_insert_blocked_by_runtime_removal;
+            DROP TRIGGER workdir_registry_update_blocked_by_runtime_removal;
+            DROP TRIGGER worker_assignment_insert_blocked_by_runtime_removal;
+            DROP TRIGGER worker_assignment_update_blocked_by_runtime_removal;
+            DROP TRIGGER workdir_attachment_insert_blocked_by_runtime_removal;
+            DROP TRIGGER workdir_attachment_update_blocked_by_runtime_removal;
+            DROP TRIGGER worker_create_insert_blocked_by_runtime_removal;
+            DROP TRIGGER worker_create_update_blocked_by_runtime_removal;
+            DROP TRIGGER workdir_create_insert_blocked_by_runtime_removal;
+            DROP TRIGGER workdir_create_update_blocked_by_runtime_removal;
+            DROP TRIGGER worker_removal_insert_blocked_by_runtime_removal;
+            DROP TRIGGER worker_removal_update_blocked_by_runtime_removal;
+            DROP TRIGGER workdir_removal_insert_blocked_by_runtime_removal;
+            DROP TRIGGER workdir_removal_update_blocked_by_runtime_removal;
             DROP TABLE runtime_removal_operations;
             DROP INDEX workspace_signing_identity_audit_workspace_idx;
             DROP TABLE workspace_signing_identity_audit;

web/workspace/src/lib/workspace/api/runtime-management.ts

@@ -1026,6 +1026,23 @@ export async function updateRemoteRuntime(
   return finishMutation(response, workspaceId, runtimeId);
 }
 
+export class RuntimeRemovalAttempt {
+  #operationId: string | null = null;
+
+  operationId(create: () => string = () => crypto.randomUUID()): string {
+    if (this.#operationId === null) this.#operationId = create();
+    return this.#operationId;
+  }
+
+  complete(operationId: string): void {
+    if (this.#operationId === operationId) this.#operationId = null;
+  }
+
+  reset(): void {
+    this.#operationId = null;
+  }
+}
+
 export async function removeRemoteRuntime(
   workspaceId: string,
   runtimeId: string,

web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte

@@ -13,6 +13,7 @@
     updateRemoteRuntime,
     RuntimeTrustConflictError,
     RuntimeTrustRouteFence,
+    RuntimeRemovalAttempt,
     RuntimeTrustRequestError,
     type RuntimeTrustRouteOperation,
   } from '$lib/workspace/api/runtime-management';
@@ -37,6 +38,7 @@
   let replacementFingerprintError = $state<string | null>(null);
   let fingerprintGeneration = 0;
   const routeFence = new RuntimeTrustRouteFence();
+  const runtimeRemovalAttempt = new RuntimeRemovalAttempt();
   let routeGeneration = 0;
 
   $effect(() => {
@@ -51,6 +53,7 @@
     endpoint = data.runtimeDetail?.endpoint ?? '';
     editingMetadata = false;
     deleteRuntimeConfirmation = '';
+    runtimeRemovalAttempt.reset();
     busyAction = null;
     fieldError = null;
     deleteRuntimeError = null;
@@ -290,6 +293,7 @@
       deleteRuntimeError = 'The authoritative Runtime binding revision is unavailable. Reload before removal.';
       return;
     }
+    const operationId = runtimeRemovalAttempt.operationId();
     busyAction = 'delete';
     deleteRuntimeError = null;
     try {
@@ -297,11 +301,12 @@
         data.workspaceId,
         routeOperation.runtimeId,
         {
-          operation_id: crypto.randomUUID(),
+          operation_id: operationId,
           expected_binding_revision: trust.revision,
         },
       );
       if (!isCurrentRoute(routeOperation)) return;
+      runtimeRemovalAttempt.complete(operationId);
       await goto(`/w/${encodeURIComponent(data.workspaceId)}/settings/runtimes`, {
         replaceState: true,
       });

web/workspace/tests/runtime-management-source.test.ts

@@ -178,7 +178,9 @@ Deno.test("Runtime detail keeps trust controls owner-only and conflict-safe", as
       "Workspace trust only; this does not delete the Runtime process, Workers, or Workdirs.",
       "await revokeRuntimeTrustKey(",
       "await removeRemoteRuntime(",
-      "operation_id: crypto.randomUUID()",
+      "new RuntimeRemovalAttempt()",
+      "runtimeRemovalAttempt.operationId()",
+      "operation_id: operationId",
       "expected_binding_revision: trust.revision",
       "The Backend removes Workspace trust and this Runtime registration as one guarded operation.",
       "deleteRuntimeConfirmation.trim() !== data.runtimeId",

web/workspace/tests/runtime-management.test.ts

@@ -12,6 +12,7 @@ import {
   previewRuntimePublicKeyFingerprint,
   removeRemoteRuntime,
   revokeRuntimeTrustKey,
+  RuntimeRemovalAttempt,
   RuntimeTrustConflictError,
   RuntimeTrustRouteFence,
   updateRemoteRuntime,
@@ -314,6 +315,58 @@ Deno.test("Runtime metadata update never sends public key authority", async () =
   assert(!("public_key" in body), "metadata update sent public_key");
 });
 
+Deno.test("Runtime removal attempt retains its id across response-loss retry", async () => {
+  const attempt = new RuntimeRemovalAttempt();
+  const operationIds: string[] = [];
+  let calls = 0;
+  const submit = async () => {
+    const operationId = attempt.operationId(() => "stable-removal-operation");
+    operationIds.push(operationId);
+    const operation = await removeRemoteRuntime(
+      "workspace-a",
+      "runtime-a",
+      { operation_id: operationId, expected_binding_revision: 4 },
+      () => {
+        calls += 1;
+        if (calls === 1) return Promise.reject(new Error("response lost"));
+        return Promise.resolve(Response.json({
+          operation_id: operationId,
+          workspace_id: "workspace-a",
+          runtime_id: "runtime-a",
+          state: "succeeded",
+          binding_removed: true,
+          runtime_registration_removed: true,
+          created_at: "2026-01-01T00:00:00Z",
+          updated_at: "2026-01-01T00:00:01Z",
+          completed_at: "2026-01-01T00:00:01Z",
+        }));
+      },
+    );
+    attempt.complete(operation.operation_id);
+  };
+
+  try {
+    await submit();
+    throw new Error("expected response-loss retry to fail");
+  } catch (error) {
+    assert(
+      error instanceof Error && error.message.includes("response lost"),
+      `unexpected response-loss error: ${String(error)}`,
+    );
+  }
+  await submit();
+  assert(
+    operationIds.length === 2 &&
+      operationIds.every((id) => id === "stable-removal-operation"),
+    `response-loss retry changed operation id: ${operationIds.join(",")}`,
+  );
+  assert(
+    attempt.operationId(() => "next-removal-operation") ===
+      "next-removal-operation",
+    "authoritative success did not clear the completed operation id",
+  );
+});
+
 Deno.test("Runtime removal uses the Workspace-scoped operation route", async () => {
   let requestedUrl = "";
   let requestedMethod = "";