Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

fix: fence runtime removal races and retries

Browse Source
This commit is contained in:
Hare
2026-09-13 02:54:01 +09:00
parent 6c609808c9
commit d2cb50d081
6 changed files with 416 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/workspace-server/src/latest_schema.sql
+144
View File
@@ -1107,6 +1107,150 @@ BEGIN
SELECT RAISE(ABORT, 'runtime_removal_in_progress');
END;
CREATE TRIGGER worker_registry_insert_blocked_by_runtime_removal
BEFORE INSERT ON worker_registry FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_registry_update_blocked_by_runtime_removal
BEFORE UPDATE ON worker_registry FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_registry_insert_blocked_by_runtime_removal
BEFORE INSERT ON workdir_registry FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_registry_update_blocked_by_runtime_removal
BEFORE UPDATE ON workdir_registry FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_assignment_insert_blocked_by_runtime_removal
BEFORE INSERT ON ticket_current_worker_assignments FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_assignment_update_blocked_by_runtime_removal
BEFORE UPDATE ON ticket_current_worker_assignments FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_attachment_insert_blocked_by_runtime_removal
BEFORE INSERT ON worker_workdir_links FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_attachment_update_blocked_by_runtime_removal
BEFORE UPDATE ON worker_workdir_links FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_create_insert_blocked_by_runtime_removal
BEFORE INSERT ON worker_create_reservations FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_create_update_blocked_by_runtime_removal
BEFORE UPDATE ON worker_create_reservations FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_create_insert_blocked_by_runtime_removal
BEFORE INSERT ON workdir_create_operations FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.resolved_runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_create_update_blocked_by_runtime_removal
BEFORE UPDATE ON workdir_create_operations FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.resolved_runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_removal_insert_blocked_by_runtime_removal
BEFORE INSERT ON worker_removal_operations FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_removal_update_blocked_by_runtime_removal
BEFORE UPDATE ON worker_removal_operations FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_removal_insert_blocked_by_runtime_removal
BEFORE INSERT ON workdir_removal_operations FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_removal_update_blocked_by_runtime_removal
BEFORE UPDATE ON workdir_removal_operations FOR EACH ROW
WHEN EXISTS (
SELECT 1 FROM runtime_removal_operations operation
WHERE operation.runtime_id = NEW.runtime_id
AND operation.state IN ('pending', 'cleanup_pending')
)
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TABLE workspace_deletion_operations (
operation_id TEXT PRIMARY KEY,
request_fingerprint TEXT NOT NULL,
crates/workspace-server/src/store.rs
+184 -10
View File
@@ -9587,7 +9587,8 @@ fn verify_canonical_workspace_runtime_binding(binding: WorkspaceRuntimeBinding)
}
fn migrate_runtime_removal_operations_v59_to_v60(conn: &Connection) -> Result<()> {
conn.execute_batch(
let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?;
tx.execute_batch(
r#"CREATE TABLE runtime_removal_operations (
operation_id TEXT PRIMARY KEY,
workspace_id TEXT NOT NULL,
@@ -9632,13 +9633,82 @@ fn migrate_runtime_removal_operations_v59_to_v60(conn: &Connection) -> Result<()
SELECT RAISE(ABORT, 'runtime_removal_in_progress');
END;"#,
)?;
conn.execute(
tx.execute_batch(
r#"
CREATE TRIGGER worker_registry_insert_blocked_by_runtime_removal
BEFORE INSERT ON worker_registry FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_registry_update_blocked_by_runtime_removal
BEFORE UPDATE ON worker_registry FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_registry_insert_blocked_by_runtime_removal
BEFORE INSERT ON workdir_registry FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_registry_update_blocked_by_runtime_removal
BEFORE UPDATE ON workdir_registry FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_assignment_insert_blocked_by_runtime_removal
BEFORE INSERT ON ticket_current_worker_assignments FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_assignment_update_blocked_by_runtime_removal
BEFORE UPDATE ON ticket_current_worker_assignments FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_attachment_insert_blocked_by_runtime_removal
BEFORE INSERT ON worker_workdir_links FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_attachment_update_blocked_by_runtime_removal
BEFORE UPDATE ON worker_workdir_links FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_create_insert_blocked_by_runtime_removal
BEFORE INSERT ON worker_create_reservations FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_create_update_blocked_by_runtime_removal
BEFORE UPDATE ON worker_create_reservations FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_create_insert_blocked_by_runtime_removal
BEFORE INSERT ON workdir_create_operations FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.resolved_runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_create_update_blocked_by_runtime_removal
BEFORE UPDATE ON workdir_create_operations FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.resolved_runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_removal_insert_blocked_by_runtime_removal
BEFORE INSERT ON worker_removal_operations FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER worker_removal_update_blocked_by_runtime_removal
BEFORE UPDATE ON worker_removal_operations FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_removal_insert_blocked_by_runtime_removal
BEFORE INSERT ON workdir_removal_operations FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
CREATE TRIGGER workdir_removal_update_blocked_by_runtime_removal
BEFORE UPDATE ON workdir_removal_operations FOR EACH ROW
WHEN EXISTS (SELECT 1 FROM runtime_removal_operations operation WHERE operation.runtime_id = NEW.runtime_id AND operation.state IN ('pending', 'cleanup_pending'))
BEGIN SELECT RAISE(ABORT, 'runtime_removal_in_progress'); END;
"#,
)?;
tx.execute(
"INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)",
params![
LATEST_SCHEMA_VERSION,
RUNTIME_REMOVAL_OPERATION_MIGRATION_NAME
],
)?;
tx.commit()?;
Ok(())
}
@@ -10438,7 +10508,7 @@ mod tests {
#[test]
fn runtime_removal_rejects_shared_binding_and_fences_new_binding_until_cleanup() {
let (_temp, store) = runtime_removal_test_store();
let (temp, store) = runtime_removal_test_store();
store
.with_conn(|conn| {
conn.execute_batch(
@@ -10500,8 +10570,11 @@ mod tests {
1,
)
.unwrap();
let blocked_insert = store.with_conn(|conn| {
conn.execute(
let competing_conn = Connection::open(temp.path().join("server.db")).unwrap();
competing_conn
.execute_batch("PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;")
.unwrap();
let blocked_insert = competing_conn.execute(
"INSERT INTO workspace_runtime_bindings(\
workspace_id, runtime_id, display_name, base_url, public_key, \
public_key_fingerprint, binding_revision, state, authentication_mode, \
@@ -10511,15 +10584,36 @@ mod tests {
'key-a', 'fingerprint-b', 1, 'verified', 'legacy_server_issuer', '1', '1'\
)",
[],
)?;
Ok(())
});
);
assert!(
blocked_insert
.unwrap_err()
.to_string()
.contains("runtime_removal_in_progress")
);
for competing_mutation in [
"INSERT INTO worker_registry(\
workspace_id, worker_id, runtime_id, display_name, retention_state, created_at, updated_at\
) VALUES ('workspace-a', 'racing-worker', 'runtime-a', 'Racing Worker', 'normal', '2', '2')",
"INSERT INTO worker_create_reservations(\
workspace_id, allocation_key, worker_id, runtime_id, create_fingerprint, state, created_at, updated_at\
) VALUES ('workspace-a', 'racing-allocation', 'racing-worker', 'runtime-a', 'racing-create', 'reserved', '2', '2')",
"INSERT INTO workdir_create_operations(\
workspace_id, operation_id, request_fingerprint, repository_id, resolved_runtime_id, \
config_revision, config_projection_digest, working_directory_id, state, created_at, updated_at\
) VALUES (\
'workspace-a', 'racing-workdir-create', 'racing-request', 'repository-a', 'runtime-a', \
1, 'racing-projection', 'racing-workdir', 'pending', '2', '2'\
)",
] {
let error = competing_conn.execute(competing_mutation, []).unwrap_err();
assert!(
error.to_string().contains("runtime_removal_in_progress"),
"competing Runtime resource mutation was not fenced: {error}"
);
}
assert_runtime_removal_binding_unchanged(&store);
}
#[test]
@@ -10602,6 +10696,22 @@ mod tests {
conn.execute_batch(
"DROP TRIGGER runtime_binding_insert_blocked_by_removal; \
DROP TRIGGER runtime_binding_update_blocked_by_removal; \
DROP TRIGGER worker_registry_insert_blocked_by_runtime_removal; \
DROP TRIGGER worker_registry_update_blocked_by_runtime_removal; \
DROP TRIGGER workdir_registry_insert_blocked_by_runtime_removal; \
DROP TRIGGER workdir_registry_update_blocked_by_runtime_removal; \
DROP TRIGGER worker_assignment_insert_blocked_by_runtime_removal; \
DROP TRIGGER worker_assignment_update_blocked_by_runtime_removal; \
DROP TRIGGER workdir_attachment_insert_blocked_by_runtime_removal; \
DROP TRIGGER workdir_attachment_update_blocked_by_runtime_removal; \
DROP TRIGGER worker_create_insert_blocked_by_runtime_removal; \
DROP TRIGGER worker_create_update_blocked_by_runtime_removal; \
DROP TRIGGER workdir_create_insert_blocked_by_runtime_removal; \
DROP TRIGGER workdir_create_update_blocked_by_runtime_removal; \
DROP TRIGGER worker_removal_insert_blocked_by_runtime_removal; \
DROP TRIGGER worker_removal_update_blocked_by_runtime_removal; \
DROP TRIGGER workdir_removal_insert_blocked_by_runtime_removal; \
DROP TRIGGER workdir_removal_update_blocked_by_runtime_removal; \
DROP TABLE runtime_removal_operations; \
DELETE FROM __yoi_schema_migrations; \
INSERT INTO __yoi_schema_migrations(version, name) \
@@ -10623,7 +10733,10 @@ mod tests {
)?;
let trigger_count: i64 = conn.query_row(
"SELECT COUNT(*) FROM sqlite_master \
WHERE type = 'trigger' AND name LIKE 'runtime_binding_%_blocked_by_removal'",
WHERE type = 'trigger' AND (\
name LIKE '%_blocked_by_runtime_removal' OR \
name LIKE 'runtime_binding_%_blocked_by_removal'\
)",
[],
|row| row.get(0),
)?;
@@ -10632,13 +10745,58 @@ mod tests {
row.get(0)
})?;
assert_eq!(table_count, 1);
assert_eq!(trigger_count, 2);
assert_eq!(trigger_count, 18);
assert_eq!(foreign_key_failures, 0);
Ok(())
})
.unwrap();
}
#[test]
fn schema_v59_runtime_removal_migration_rolls_back_partial_ddl_and_marker() {
let temp = tempfile::tempdir().unwrap();
let store = SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap();
store
.with_conn(|conn| {
let trigger_names = {
let mut stmt = conn.prepare(
"SELECT name FROM sqlite_master \
WHERE type = 'trigger' AND sql LIKE '%runtime_removal_operations%'",
)?;
stmt.query_map([], |row| row.get::<_, String>(0))?
.collect::<std::result::Result<Vec<_>, _>>()?
};
for trigger_name in trigger_names {
conn.execute_batch(&format!("DROP TRIGGER \"{trigger_name}\";"))?;
}
conn.execute_batch(
"DROP TABLE runtime_removal_operations; \
DELETE FROM __yoi_schema_migrations WHERE version = 60; \
CREATE INDEX runtime_removal_operations_one_active_runtime \
ON workspace_runtime_bindings(runtime_id);",
)?;
let migration_error = migrate_runtime_removal_operations_v59_to_v60(conn)
.expect_err("injected index-name collision must fail migration");
assert!(migration_error.to_string().contains("already exists"));
let table_count: i64 = conn.query_row(
"SELECT COUNT(*) FROM sqlite_master \
WHERE type = 'table' AND name = 'runtime_removal_operations'",
[],
|row| row.get(0),
)?;
let marker_count: i64 = conn.query_row(
"SELECT COUNT(*) FROM __yoi_schema_migrations WHERE version = 60",
[],
|row| row.get(0),
)?;
assert_eq!(table_count, 0, "partial Runtime removal DDL leaked");
assert_eq!(marker_count, 0, "failed migration advanced schema marker");
Ok(())
})
.unwrap();
}
#[test]
fn current_schema_accepts_every_retained_canonical_provenance() {
for baseline_version in OLDEST_SCHEMA_VERSION..=LATEST_SCHEMA_VERSION {
@@ -10707,6 +10865,22 @@ mod tests {
DROP TABLE workdir_create_credential_candidates;
DROP TRIGGER runtime_binding_insert_blocked_by_removal;
DROP TRIGGER runtime_binding_update_blocked_by_removal;
DROP TRIGGER worker_registry_insert_blocked_by_runtime_removal;
DROP TRIGGER worker_registry_update_blocked_by_runtime_removal;
DROP TRIGGER workdir_registry_insert_blocked_by_runtime_removal;
DROP TRIGGER workdir_registry_update_blocked_by_runtime_removal;
DROP TRIGGER worker_assignment_insert_blocked_by_runtime_removal;
DROP TRIGGER worker_assignment_update_blocked_by_runtime_removal;
DROP TRIGGER workdir_attachment_insert_blocked_by_runtime_removal;
DROP TRIGGER workdir_attachment_update_blocked_by_runtime_removal;
DROP TRIGGER worker_create_insert_blocked_by_runtime_removal;
DROP TRIGGER worker_create_update_blocked_by_runtime_removal;
DROP TRIGGER workdir_create_insert_blocked_by_runtime_removal;
DROP TRIGGER workdir_create_update_blocked_by_runtime_removal;
DROP TRIGGER worker_removal_insert_blocked_by_runtime_removal;
DROP TRIGGER worker_removal_update_blocked_by_runtime_removal;
DROP TRIGGER workdir_removal_insert_blocked_by_runtime_removal;
DROP TRIGGER workdir_removal_update_blocked_by_runtime_removal;
DROP TABLE runtime_removal_operations;
DROP INDEX workspace_signing_identity_audit_workspace_idx;
DROP TABLE workspace_signing_identity_audit;
web/workspace/src/lib/workspace/api/runtime-management.ts
+17
View File
@@ -1026,6 +1026,23 @@ export async function updateRemoteRuntime(
return finishMutation(response, workspaceId, runtimeId);
}
export class RuntimeRemovalAttempt {
#operationId: string | null = null;
operationId(create: () => string = () => crypto.randomUUID()): string {
if (this.#operationId === null) this.#operationId = create();
return this.#operationId;
}
complete(operationId: string): void {
if (this.#operationId === operationId) this.#operationId = null;
}
reset(): void {
this.#operationId = null;
}
}
export async function removeRemoteRuntime(
workspaceId: string,
runtimeId: string,
web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte
+6 -1
View File
@@ -13,6 +13,7 @@
updateRemoteRuntime,
RuntimeTrustConflictError,
RuntimeTrustRouteFence,
RuntimeRemovalAttempt,
RuntimeTrustRequestError,
type RuntimeTrustRouteOperation,
} from '$lib/workspace/api/runtime-management';
@@ -37,6 +38,7 @@
let replacementFingerprintError = $state<string | null>(null);
let fingerprintGeneration = 0;
const routeFence = new RuntimeTrustRouteFence();
const runtimeRemovalAttempt = new RuntimeRemovalAttempt();
let routeGeneration = 0;
$effect(() => {
@@ -51,6 +53,7 @@
endpoint = data.runtimeDetail?.endpoint ?? '';
editingMetadata = false;
deleteRuntimeConfirmation = '';
runtimeRemovalAttempt.reset();
busyAction = null;
fieldError = null;
deleteRuntimeError = null;
@@ -290,6 +293,7 @@
deleteRuntimeError = 'The authoritative Runtime binding revision is unavailable. Reload before removal.';
return;
}
const operationId = runtimeRemovalAttempt.operationId();
busyAction = 'delete';
deleteRuntimeError = null;
try {
@@ -297,11 +301,12 @@
data.workspaceId,
routeOperation.runtimeId,
{
operation_id: crypto.randomUUID(),
operation_id: operationId,
expected_binding_revision: trust.revision,
},
);
if (!isCurrentRoute(routeOperation)) return;
runtimeRemovalAttempt.complete(operationId);
await goto(`/w/${encodeURIComponent(data.workspaceId)}/settings/runtimes`, {
replaceState: true,
});
web/workspace/tests/runtime-management-source.test.ts
+3 -1
View File
@@ -178,7 +178,9 @@ Deno.test("Runtime detail keeps trust controls owner-only and conflict-safe", as
"Workspace trust only; this does not delete the Runtime process, Workers, or Workdirs.",
"await revokeRuntimeTrustKey(",
"await removeRemoteRuntime(",
"operation_id: crypto.randomUUID()",
"new RuntimeRemovalAttempt()",
"runtimeRemovalAttempt.operationId()",
"operation_id: operationId",
"expected_binding_revision: trust.revision",
"The Backend removes Workspace trust and this Runtime registration as one guarded operation.",
"deleteRuntimeConfirmation.trim() !== data.runtimeId",
web/workspace/tests/runtime-management.test.ts
+53
View File
@@ -12,6 +12,7 @@ import {
previewRuntimePublicKeyFingerprint,
removeRemoteRuntime,
revokeRuntimeTrustKey,
RuntimeRemovalAttempt,
RuntimeTrustConflictError,
RuntimeTrustRouteFence,
updateRemoteRuntime,
@@ -314,6 +315,58 @@ Deno.test("Runtime metadata update never sends public key authority", async () =
assert(!("public_key" in body), "metadata update sent public_key");
});
Deno.test("Runtime removal attempt retains its id across response-loss retry", async () => {
const attempt = new RuntimeRemovalAttempt();
const operationIds: string[] = [];
let calls = 0;
const submit = async () => {
const operationId = attempt.operationId(() => "stable-removal-operation");
operationIds.push(operationId);
const operation = await removeRemoteRuntime(
"workspace-a",
"runtime-a",
{ operation_id: operationId, expected_binding_revision: 4 },
() => {
calls += 1;
if (calls === 1) return Promise.reject(new Error("response lost"));
return Promise.resolve(Response.json({
operation_id: operationId,
workspace_id: "workspace-a",
runtime_id: "runtime-a",
state: "succeeded",
binding_removed: true,
runtime_registration_removed: true,
created_at: "2026-01-01T00:00:00Z",
updated_at: "2026-01-01T00:00:01Z",
completed_at: "2026-01-01T00:00:01Z",
}));
},
);
attempt.complete(operation.operation_id);
};
try {
await submit();
throw new Error("expected response-loss retry to fail");
} catch (error) {
assert(
error instanceof Error && error.message.includes("response lost"),
`unexpected response-loss error: ${String(error)}`,
);
}
await submit();
assert(
operationIds.length === 2 &&
operationIds.every((id) => id === "stable-removal-operation"),
`response-loss retry changed operation id: ${operationIds.join(",")}`,
);
assert(
attempt.operationId(() => "next-removal-operation") ===
"next-removal-operation",
"authoritative success did not clear the completed operation id",
);
});
Deno.test("Runtime removal uses the Workspace-scoped operation route", async () => {
let requestedUrl = "";
let requestedMethod = "";