fix: enforce monotonic worker state projection
This commit is contained in:
@@ -218,6 +218,66 @@ Deno.test("console routing projects live errors but not completion replies", ()
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("Worker state events and acknowledgements apply monotonically", () => {
|
||||
const projector = createConsoleProjector();
|
||||
const running: WorkerStateSnapshot = {
|
||||
execution_generation: 4,
|
||||
revision: 3,
|
||||
last_command_id: 2,
|
||||
state: { kind: "busy", state: { kind: "run", state: "running" } },
|
||||
};
|
||||
const paused: WorkerStateSnapshot = {
|
||||
...running,
|
||||
revision: 4,
|
||||
last_command_id: 3,
|
||||
state: { kind: "busy", state: { kind: "run", state: "paused" } },
|
||||
};
|
||||
let projection = projector.append([
|
||||
{
|
||||
eventId: "running",
|
||||
event: { event: "worker_state", data: { snapshot: running } },
|
||||
},
|
||||
{
|
||||
eventId: "stale",
|
||||
event: {
|
||||
event: "worker_state",
|
||||
data: { snapshot: { ...running, revision: 2, state: { kind: "idle" } } },
|
||||
},
|
||||
},
|
||||
{
|
||||
eventId: "pause-ack",
|
||||
event: {
|
||||
event: "command_acknowledged",
|
||||
data: {
|
||||
acknowledgement: {
|
||||
command_id: 3,
|
||||
command: "pause",
|
||||
disposition: "accepted",
|
||||
state: paused,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
assertEquals(projection.workerState, paused);
|
||||
assertEquals(projection.status, "paused");
|
||||
|
||||
projection = projector.append([{
|
||||
eventId: "conflict",
|
||||
event: {
|
||||
event: "worker_state",
|
||||
data: { snapshot: { ...paused, state: { kind: "idle" } } },
|
||||
},
|
||||
}]);
|
||||
assertEquals(projection.workerState, paused);
|
||||
assert(
|
||||
projection.lines.some((line) =>
|
||||
line.eventId === "conflict:worker-state-conflict" && line.error
|
||||
),
|
||||
"conflicting equal-version snapshots must fail closed",
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("snapshot replaces a live error with one durable run_errored row", () => {
|
||||
const projector = createConsoleProjector();
|
||||
let projection = projector.append([
|
||||
|
||||
@@ -10,6 +10,7 @@ import type {
|
||||
InternalWorkerRef,
|
||||
InternalWorkerSnapshot,
|
||||
Segment,
|
||||
WorkerState,
|
||||
WorkerStateSnapshot,
|
||||
WorkerStatus,
|
||||
} from "$lib/generated/protocol";
|
||||
@@ -796,6 +797,60 @@ function refreshCompactionActivity(
|
||||
return changed ? { ...projection, lines } : projection;
|
||||
}
|
||||
|
||||
function workerStateEqual(left: WorkerState, right: WorkerState): boolean {
|
||||
if (left.kind !== right.kind) return false;
|
||||
if (left.kind === "idle" || right.kind === "idle") return true;
|
||||
return left.state.kind === right.state.kind &&
|
||||
left.state.state === right.state.state;
|
||||
}
|
||||
|
||||
function workerStateSnapshotEqual(
|
||||
left: WorkerStateSnapshot,
|
||||
right: WorkerStateSnapshot,
|
||||
): boolean {
|
||||
return left.execution_generation === right.execution_generation &&
|
||||
left.revision === right.revision &&
|
||||
left.last_command_id === right.last_command_id &&
|
||||
workerStateEqual(left.state, right.state);
|
||||
}
|
||||
|
||||
function applyWorkerStateSnapshot(
|
||||
projection: ConsoleProjection,
|
||||
incoming: WorkerStateSnapshot,
|
||||
eventId: string,
|
||||
): void {
|
||||
const current = projection.workerState;
|
||||
if (!current) {
|
||||
projection.workerState = incoming;
|
||||
projection.status = workerStatusFromState(incoming);
|
||||
return;
|
||||
}
|
||||
const generationOrder = incoming.execution_generation -
|
||||
current.execution_generation;
|
||||
const revisionOrder = incoming.revision - current.revision;
|
||||
if (generationOrder > 0 || (generationOrder === 0 && revisionOrder > 0)) {
|
||||
projection.workerState = incoming;
|
||||
projection.status = workerStatusFromState(incoming);
|
||||
return;
|
||||
}
|
||||
if (generationOrder < 0 || (generationOrder === 0 && revisionOrder < 0)) {
|
||||
return;
|
||||
}
|
||||
if (!workerStateSnapshotEqual(current, incoming)) {
|
||||
projection.lines.push(
|
||||
line(
|
||||
`${eventId}:worker-state-conflict`,
|
||||
"error",
|
||||
"error · internal",
|
||||
`worker state stream rejected: conflicting snapshots at generation ${incoming.execution_generation} revision ${incoming.revision}`,
|
||||
undefined,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function applyProtocolEvent(
|
||||
projection: ConsoleProjection,
|
||||
envelope: ConsoleEventInput,
|
||||
@@ -917,8 +972,6 @@ export function applyProtocolEvent(
|
||||
);
|
||||
break;
|
||||
case "snapshot": {
|
||||
next.workerState = event.data.state;
|
||||
next.status = workerStatusFromState(event.data.state);
|
||||
next.cwd = event.data.greeting.cwd;
|
||||
const snapshot = snapshotProjectionFromSession(
|
||||
envelope.eventId,
|
||||
@@ -968,6 +1021,7 @@ export function applyProtocolEvent(
|
||||
};
|
||||
}
|
||||
}
|
||||
applyWorkerStateSnapshot(next, event.data.state, envelope.eventId);
|
||||
break;
|
||||
}
|
||||
case "internal_worker": {
|
||||
@@ -1016,12 +1070,14 @@ export function applyProtocolEvent(
|
||||
break;
|
||||
}
|
||||
case "worker_state":
|
||||
next.workerState = event.data.snapshot;
|
||||
next.status = workerStatusFromState(event.data.snapshot);
|
||||
applyWorkerStateSnapshot(next, event.data.snapshot, envelope.eventId);
|
||||
break;
|
||||
case "command_acknowledged":
|
||||
next.workerState = event.data.acknowledgement.state;
|
||||
next.status = workerStatusFromState(event.data.acknowledgement.state);
|
||||
applyWorkerStateSnapshot(
|
||||
next,
|
||||
event.data.acknowledgement.state,
|
||||
envelope.eventId,
|
||||
);
|
||||
break;
|
||||
case "command":
|
||||
applyCommandEvent(next, envelope.eventId, event.data.event);
|
||||
|
||||
Reference in New Issue
Block a user