fix: enforce monotonic worker state projection

This commit is contained in:
2026-09-06 07:56:25 +09:00
parent 4269ebec04
commit 282a8d31b5
7 changed files with 485 additions and 241 deletions
@@ -218,6 +218,66 @@ Deno.test("console routing projects live errors but not completion replies", ()
);
});
Deno.test("Worker state events and acknowledgements apply monotonically", () => {
const projector = createConsoleProjector();
const running: WorkerStateSnapshot = {
execution_generation: 4,
revision: 3,
last_command_id: 2,
state: { kind: "busy", state: { kind: "run", state: "running" } },
};
const paused: WorkerStateSnapshot = {
...running,
revision: 4,
last_command_id: 3,
state: { kind: "busy", state: { kind: "run", state: "paused" } },
};
let projection = projector.append([
{
eventId: "running",
event: { event: "worker_state", data: { snapshot: running } },
},
{
eventId: "stale",
event: {
event: "worker_state",
data: { snapshot: { ...running, revision: 2, state: { kind: "idle" } } },
},
},
{
eventId: "pause-ack",
event: {
event: "command_acknowledged",
data: {
acknowledgement: {
command_id: 3,
command: "pause",
disposition: "accepted",
state: paused,
},
},
},
},
]);
assertEquals(projection.workerState, paused);
assertEquals(projection.status, "paused");
projection = projector.append([{
eventId: "conflict",
event: {
event: "worker_state",
data: { snapshot: { ...paused, state: { kind: "idle" } } },
},
}]);
assertEquals(projection.workerState, paused);
assert(
projection.lines.some((line) =>
line.eventId === "conflict:worker-state-conflict" && line.error
),
"conflicting equal-version snapshots must fail closed",
);
});
Deno.test("snapshot replaces a live error with one durable run_errored row", () => {
const projector = createConsoleProjector();
let projection = projector.append([
@@ -10,6 +10,7 @@ import type {
InternalWorkerRef,
InternalWorkerSnapshot,
Segment,
WorkerState,
WorkerStateSnapshot,
WorkerStatus,
} from "$lib/generated/protocol";
@@ -796,6 +797,60 @@ function refreshCompactionActivity(
return changed ? { ...projection, lines } : projection;
}
function workerStateEqual(left: WorkerState, right: WorkerState): boolean {
if (left.kind !== right.kind) return false;
if (left.kind === "idle" || right.kind === "idle") return true;
return left.state.kind === right.state.kind &&
left.state.state === right.state.state;
}
function workerStateSnapshotEqual(
left: WorkerStateSnapshot,
right: WorkerStateSnapshot,
): boolean {
return left.execution_generation === right.execution_generation &&
left.revision === right.revision &&
left.last_command_id === right.last_command_id &&
workerStateEqual(left.state, right.state);
}
function applyWorkerStateSnapshot(
projection: ConsoleProjection,
incoming: WorkerStateSnapshot,
eventId: string,
): void {
const current = projection.workerState;
if (!current) {
projection.workerState = incoming;
projection.status = workerStatusFromState(incoming);
return;
}
const generationOrder = incoming.execution_generation -
current.execution_generation;
const revisionOrder = incoming.revision - current.revision;
if (generationOrder > 0 || (generationOrder === 0 && revisionOrder > 0)) {
projection.workerState = incoming;
projection.status = workerStatusFromState(incoming);
return;
}
if (generationOrder < 0 || (generationOrder === 0 && revisionOrder < 0)) {
return;
}
if (!workerStateSnapshotEqual(current, incoming)) {
projection.lines.push(
line(
`${eventId}:worker-state-conflict`,
"error",
"error · internal",
`worker state stream rejected: conflicting snapshots at generation ${incoming.execution_generation} revision ${incoming.revision}`,
undefined,
false,
true,
),
);
}
}
export function applyProtocolEvent(
projection: ConsoleProjection,
envelope: ConsoleEventInput,
@@ -917,8 +972,6 @@ export function applyProtocolEvent(
);
break;
case "snapshot": {
next.workerState = event.data.state;
next.status = workerStatusFromState(event.data.state);
next.cwd = event.data.greeting.cwd;
const snapshot = snapshotProjectionFromSession(
envelope.eventId,
@@ -968,6 +1021,7 @@ export function applyProtocolEvent(
};
}
}
applyWorkerStateSnapshot(next, event.data.state, envelope.eventId);
break;
}
case "internal_worker": {
@@ -1016,12 +1070,14 @@ export function applyProtocolEvent(
break;
}
case "worker_state":
next.workerState = event.data.snapshot;
next.status = workerStatusFromState(event.data.snapshot);
applyWorkerStateSnapshot(next, event.data.snapshot, envelope.eventId);
break;
case "command_acknowledged":
next.workerState = event.data.acknowledgement.state;
next.status = workerStatusFromState(event.data.acknowledgement.state);
applyWorkerStateSnapshot(
next,
event.data.acknowledgement.state,
envelope.eventId,
);
break;
case "command":
applyCommandEvent(next, envelope.eventId, event.data.event);