refactor: make worker state snapshots authoritative

This commit is contained in:
2026-09-17 02:02:51 +09:00
parent d6bef5d1c7
commit 10ebac142e
31 changed files with 473 additions and 1132 deletions
+4 -9
View File
@@ -10,16 +10,11 @@ export type CompletionKind = "file";
export type WorkerStatus = "idle" | "running" | "paused" | "stopped";
export type WorkerCommandEnvelope = {
/**
* Caller-owned sequence. A controller accepts command ids in strictly
* increasing order for one execution generation.
*/
command_id: number, expected_execution_generation: number, expected_worker_state_revision: number, };
export type WorkerCommandEnvelope = { command_id: number, };
export type WorkerCommandKind = "resume" | "cancel" | "pause" | "compact" | "shutdown";
export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "conflict" | "invalid_state";
export type WorkerCommandDisposition = "accepted" | "stale_command_id" | "conflict" | "invalid_state";
export type WorkerCommandAcknowledgement = { command_id: number, command: WorkerCommandKind, disposition: WorkerCommandDisposition,
/**
@@ -35,9 +30,9 @@ export type WorkerBusyState = { "kind": "run", "state": WorkerRunState } | { "ki
export type WorkerState = { "kind": "idle" } | { "kind": "busy", "state": WorkerBusyState };
export type WorkerStateSnapshot = { execution_generation: number, revision: number,
export type WorkerStateSnapshot = {
/**
* Highest lifecycle command id observed by this controller generation.
* Highest lifecycle command id observed by this controller instance.
*/
last_command_id: number, state: WorkerState, };
@@ -21,8 +21,6 @@ declare const Deno: {
function workerState(status: WorkerStatus): WorkerStateSnapshot {
return {
execution_generation: 1,
revision: status === "idle" ? 0 : 1,
last_command_id: 0,
state: status === "idle"
? { kind: "idle" }
@@ -218,17 +216,17 @@ Deno.test("console routing projects live errors but not completion replies", ()
);
});
Deno.test("Worker state events and acknowledgements apply monotonically", () => {
Deno.test("Worker state events and acknowledgements replace the full state", () => {
const projector = createConsoleProjector();
const running: WorkerStateSnapshot = {
execution_generation: 4,
revision: 3,
last_command_id: 2,
state: { kind: "busy", state: { kind: "run", state: "running" } },
};
const freshIdle: WorkerStateSnapshot = {
last_command_id: 0,
state: { kind: "idle" },
};
const paused: WorkerStateSnapshot = {
...running,
revision: 4,
last_command_id: 3,
state: { kind: "busy", state: { kind: "run", state: "paused" } },
};
@@ -238,11 +236,8 @@ Deno.test("Worker state events and acknowledgements apply monotonically", () =>
event: { event: "worker_state", data: { snapshot: running } },
},
{
eventId: "stale",
event: {
event: "worker_state",
data: { snapshot: { ...running, revision: 2, state: { kind: "idle" } } },
},
eventId: "fresh-idle",
event: { event: "worker_state", data: { snapshot: freshIdle } },
},
{
eventId: "pause-ack",
@@ -263,18 +258,17 @@ Deno.test("Worker state events and acknowledgements apply monotonically", () =>
assertEquals(projection.status, "paused");
projection = projector.append([{
eventId: "conflict",
eventId: "replacement",
event: {
event: "worker_state",
data: { snapshot: { ...paused, state: { kind: "idle" } } },
data: { snapshot: freshIdle },
},
}]);
assertEquals(projection.workerState, paused);
assertEquals(projection.workerState, freshIdle);
assertEquals(projection.status, "idle");
assert(
projection.lines.some((line) =>
line.eventId === "conflict:worker-state-conflict" && line.error
),
"conflicting equal-version snapshots must fail closed",
!projection.lines.some((line) => line.eventId?.includes("worker-state-conflict")),
"full snapshots must not be rejected by a client-side version comparison",
);
});
@@ -784,58 +784,12 @@ function refreshCompactionActivity(
return changed ? { ...projection, lines } : projection;
}
function workerStateEqual(left: WorkerState, right: WorkerState): boolean {
if (left.kind !== right.kind) return false;
if (left.kind === "idle" || right.kind === "idle") return true;
return left.state.kind === right.state.kind &&
left.state.state === right.state.state;
}
function workerStateSnapshotEqual(
left: WorkerStateSnapshot,
right: WorkerStateSnapshot,
): boolean {
return left.execution_generation === right.execution_generation &&
left.revision === right.revision &&
left.last_command_id === right.last_command_id &&
workerStateEqual(left.state, right.state);
}
function applyWorkerStateSnapshot(
projection: ConsoleProjection,
incoming: WorkerStateSnapshot,
eventId: string,
): void {
const current = projection.workerState;
if (!current) {
projection.workerState = incoming;
projection.status = workerStatusFromState(incoming);
return;
}
const generationOrder = incoming.execution_generation -
current.execution_generation;
const revisionOrder = incoming.revision - current.revision;
if (generationOrder > 0 || (generationOrder === 0 && revisionOrder > 0)) {
projection.workerState = incoming;
projection.status = workerStatusFromState(incoming);
return;
}
if (generationOrder < 0 || (generationOrder === 0 && revisionOrder < 0)) {
return;
}
if (!workerStateSnapshotEqual(current, incoming)) {
projection.lines.push(
line(
`${eventId}:worker-state-conflict`,
"error",
"error · internal",
`worker state stream rejected: conflicting snapshots at generation ${incoming.execution_generation} revision ${incoming.revision}`,
undefined,
false,
true,
),
);
}
projection.workerState = incoming;
projection.status = workerStatusFromState(incoming);
}
export function applyProtocolEvent(
@@ -1004,7 +958,7 @@ export function applyProtocolEvent(
};
}
}
applyWorkerStateSnapshot(next, event.data.state, envelope.eventId);
applyWorkerStateSnapshot(next, event.data.state);
break;
}
case "internal_worker": {
@@ -1053,14 +1007,10 @@ export function applyProtocolEvent(
break;
}
case "worker_state":
applyWorkerStateSnapshot(next, event.data.snapshot, envelope.eventId);
applyWorkerStateSnapshot(next, event.data.snapshot);
break;
case "command_acknowledged":
applyWorkerStateSnapshot(
next,
event.data.acknowledgement.state,
envelope.eventId,
);
applyWorkerStateSnapshot(next, event.data.acknowledgement.state);
break;
case "command":
applyCommandEvent(next, envelope.eventId, event.data.event);
@@ -76,7 +76,6 @@ Deno.test("new invoke and running snapshot reset run activity", () => {
entries: [],
greeting: { text: "", profile: "" },
state: {
execution_generation: 1,
revision: 0,
last_command_id: 0,
state: { kind: "idle" },
@@ -794,8 +794,9 @@ Deno.test("Worker Console route resolves logical Worker authority before Runtime
) &&
consolePage.includes('sendWorkerControl("cancel")') &&
consolePage.includes("lifecycleMethod(command)") &&
consolePage.includes("expected_worker_state_revision") &&
consolePage.includes("expected_execution_generation") &&
consolePage.includes("command_id: commandId") &&
!consolePage.includes("expected_worker_state_revision") &&
!consolePage.includes("expected_execution_generation") &&
consolePage.includes("onsubmit={handleComposerSubmit}") &&
consolePage.includes("disabled={!composerEditable}") &&
consolePage.includes("class:stop={workerRunning}") &&
@@ -37,8 +37,6 @@ function worker(
Deno.test('Worker list state uses the authoritative live snapshot separately from lifecycle', () => {
const active = worker('runtime-a', 'worker-1', 1);
active.worker_state = {
execution_generation: 4,
revision: 2,
last_command_id: 1,
state: { kind: 'busy', state: { kind: 'run', state: 'paused' } },
};
@@ -562,8 +562,6 @@
nextWorkerCommandId = commandId + 1;
const envelope = {
command_id: commandId,
expected_execution_generation: state.execution_generation,
expected_worker_state_revision: state.revision,
};
switch (command) {
case "pause":