refactor: make worker state snapshots authoritative
This commit is contained in:
@@ -10,16 +10,11 @@ export type CompletionKind = "file";
|
||||
|
||||
export type WorkerStatus = "idle" | "running" | "paused" | "stopped";
|
||||
|
||||
export type WorkerCommandEnvelope = {
|
||||
/**
|
||||
* Caller-owned sequence. A controller accepts command ids in strictly
|
||||
* increasing order for one execution generation.
|
||||
*/
|
||||
command_id: number, expected_execution_generation: number, expected_worker_state_revision: number, };
|
||||
export type WorkerCommandEnvelope = { command_id: number, };
|
||||
|
||||
export type WorkerCommandKind = "resume" | "cancel" | "pause" | "compact" | "shutdown";
|
||||
|
||||
export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "conflict" | "invalid_state";
|
||||
export type WorkerCommandDisposition = "accepted" | "stale_command_id" | "conflict" | "invalid_state";
|
||||
|
||||
export type WorkerCommandAcknowledgement = { command_id: number, command: WorkerCommandKind, disposition: WorkerCommandDisposition,
|
||||
/**
|
||||
@@ -35,9 +30,9 @@ export type WorkerBusyState = { "kind": "run", "state": WorkerRunState } | { "ki
|
||||
|
||||
export type WorkerState = { "kind": "idle" } | { "kind": "busy", "state": WorkerBusyState };
|
||||
|
||||
export type WorkerStateSnapshot = { execution_generation: number, revision: number,
|
||||
export type WorkerStateSnapshot = {
|
||||
/**
|
||||
* Highest lifecycle command id observed by this controller generation.
|
||||
* Highest lifecycle command id observed by this controller instance.
|
||||
*/
|
||||
last_command_id: number, state: WorkerState, };
|
||||
|
||||
|
||||
@@ -21,8 +21,6 @@ declare const Deno: {
|
||||
|
||||
function workerState(status: WorkerStatus): WorkerStateSnapshot {
|
||||
return {
|
||||
execution_generation: 1,
|
||||
revision: status === "idle" ? 0 : 1,
|
||||
last_command_id: 0,
|
||||
state: status === "idle"
|
||||
? { kind: "idle" }
|
||||
@@ -218,17 +216,17 @@ Deno.test("console routing projects live errors but not completion replies", ()
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("Worker state events and acknowledgements apply monotonically", () => {
|
||||
Deno.test("Worker state events and acknowledgements replace the full state", () => {
|
||||
const projector = createConsoleProjector();
|
||||
const running: WorkerStateSnapshot = {
|
||||
execution_generation: 4,
|
||||
revision: 3,
|
||||
last_command_id: 2,
|
||||
state: { kind: "busy", state: { kind: "run", state: "running" } },
|
||||
};
|
||||
const freshIdle: WorkerStateSnapshot = {
|
||||
last_command_id: 0,
|
||||
state: { kind: "idle" },
|
||||
};
|
||||
const paused: WorkerStateSnapshot = {
|
||||
...running,
|
||||
revision: 4,
|
||||
last_command_id: 3,
|
||||
state: { kind: "busy", state: { kind: "run", state: "paused" } },
|
||||
};
|
||||
@@ -238,11 +236,8 @@ Deno.test("Worker state events and acknowledgements apply monotonically", () =>
|
||||
event: { event: "worker_state", data: { snapshot: running } },
|
||||
},
|
||||
{
|
||||
eventId: "stale",
|
||||
event: {
|
||||
event: "worker_state",
|
||||
data: { snapshot: { ...running, revision: 2, state: { kind: "idle" } } },
|
||||
},
|
||||
eventId: "fresh-idle",
|
||||
event: { event: "worker_state", data: { snapshot: freshIdle } },
|
||||
},
|
||||
{
|
||||
eventId: "pause-ack",
|
||||
@@ -263,18 +258,17 @@ Deno.test("Worker state events and acknowledgements apply monotonically", () =>
|
||||
assertEquals(projection.status, "paused");
|
||||
|
||||
projection = projector.append([{
|
||||
eventId: "conflict",
|
||||
eventId: "replacement",
|
||||
event: {
|
||||
event: "worker_state",
|
||||
data: { snapshot: { ...paused, state: { kind: "idle" } } },
|
||||
data: { snapshot: freshIdle },
|
||||
},
|
||||
}]);
|
||||
assertEquals(projection.workerState, paused);
|
||||
assertEquals(projection.workerState, freshIdle);
|
||||
assertEquals(projection.status, "idle");
|
||||
assert(
|
||||
projection.lines.some((line) =>
|
||||
line.eventId === "conflict:worker-state-conflict" && line.error
|
||||
),
|
||||
"conflicting equal-version snapshots must fail closed",
|
||||
!projection.lines.some((line) => line.eventId?.includes("worker-state-conflict")),
|
||||
"full snapshots must not be rejected by a client-side version comparison",
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -784,58 +784,12 @@ function refreshCompactionActivity(
|
||||
return changed ? { ...projection, lines } : projection;
|
||||
}
|
||||
|
||||
function workerStateEqual(left: WorkerState, right: WorkerState): boolean {
|
||||
if (left.kind !== right.kind) return false;
|
||||
if (left.kind === "idle" || right.kind === "idle") return true;
|
||||
return left.state.kind === right.state.kind &&
|
||||
left.state.state === right.state.state;
|
||||
}
|
||||
|
||||
function workerStateSnapshotEqual(
|
||||
left: WorkerStateSnapshot,
|
||||
right: WorkerStateSnapshot,
|
||||
): boolean {
|
||||
return left.execution_generation === right.execution_generation &&
|
||||
left.revision === right.revision &&
|
||||
left.last_command_id === right.last_command_id &&
|
||||
workerStateEqual(left.state, right.state);
|
||||
}
|
||||
|
||||
function applyWorkerStateSnapshot(
|
||||
projection: ConsoleProjection,
|
||||
incoming: WorkerStateSnapshot,
|
||||
eventId: string,
|
||||
): void {
|
||||
const current = projection.workerState;
|
||||
if (!current) {
|
||||
projection.workerState = incoming;
|
||||
projection.status = workerStatusFromState(incoming);
|
||||
return;
|
||||
}
|
||||
const generationOrder = incoming.execution_generation -
|
||||
current.execution_generation;
|
||||
const revisionOrder = incoming.revision - current.revision;
|
||||
if (generationOrder > 0 || (generationOrder === 0 && revisionOrder > 0)) {
|
||||
projection.workerState = incoming;
|
||||
projection.status = workerStatusFromState(incoming);
|
||||
return;
|
||||
}
|
||||
if (generationOrder < 0 || (generationOrder === 0 && revisionOrder < 0)) {
|
||||
return;
|
||||
}
|
||||
if (!workerStateSnapshotEqual(current, incoming)) {
|
||||
projection.lines.push(
|
||||
line(
|
||||
`${eventId}:worker-state-conflict`,
|
||||
"error",
|
||||
"error · internal",
|
||||
`worker state stream rejected: conflicting snapshots at generation ${incoming.execution_generation} revision ${incoming.revision}`,
|
||||
undefined,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
);
|
||||
}
|
||||
projection.workerState = incoming;
|
||||
projection.status = workerStatusFromState(incoming);
|
||||
}
|
||||
|
||||
export function applyProtocolEvent(
|
||||
@@ -1004,7 +958,7 @@ export function applyProtocolEvent(
|
||||
};
|
||||
}
|
||||
}
|
||||
applyWorkerStateSnapshot(next, event.data.state, envelope.eventId);
|
||||
applyWorkerStateSnapshot(next, event.data.state);
|
||||
break;
|
||||
}
|
||||
case "internal_worker": {
|
||||
@@ -1053,14 +1007,10 @@ export function applyProtocolEvent(
|
||||
break;
|
||||
}
|
||||
case "worker_state":
|
||||
applyWorkerStateSnapshot(next, event.data.snapshot, envelope.eventId);
|
||||
applyWorkerStateSnapshot(next, event.data.snapshot);
|
||||
break;
|
||||
case "command_acknowledged":
|
||||
applyWorkerStateSnapshot(
|
||||
next,
|
||||
event.data.acknowledgement.state,
|
||||
envelope.eventId,
|
||||
);
|
||||
applyWorkerStateSnapshot(next, event.data.acknowledgement.state);
|
||||
break;
|
||||
case "command":
|
||||
applyCommandEvent(next, envelope.eventId, event.data.event);
|
||||
|
||||
@@ -76,7 +76,6 @@ Deno.test("new invoke and running snapshot reset run activity", () => {
|
||||
entries: [],
|
||||
greeting: { text: "", profile: "" },
|
||||
state: {
|
||||
execution_generation: 1,
|
||||
revision: 0,
|
||||
last_command_id: 0,
|
||||
state: { kind: "idle" },
|
||||
|
||||
@@ -794,8 +794,9 @@ Deno.test("Worker Console route resolves logical Worker authority before Runtime
|
||||
) &&
|
||||
consolePage.includes('sendWorkerControl("cancel")') &&
|
||||
consolePage.includes("lifecycleMethod(command)") &&
|
||||
consolePage.includes("expected_worker_state_revision") &&
|
||||
consolePage.includes("expected_execution_generation") &&
|
||||
consolePage.includes("command_id: commandId") &&
|
||||
!consolePage.includes("expected_worker_state_revision") &&
|
||||
!consolePage.includes("expected_execution_generation") &&
|
||||
consolePage.includes("onsubmit={handleComposerSubmit}") &&
|
||||
consolePage.includes("disabled={!composerEditable}") &&
|
||||
consolePage.includes("class:stop={workerRunning}") &&
|
||||
|
||||
@@ -37,8 +37,6 @@ function worker(
|
||||
Deno.test('Worker list state uses the authoritative live snapshot separately from lifecycle', () => {
|
||||
const active = worker('runtime-a', 'worker-1', 1);
|
||||
active.worker_state = {
|
||||
execution_generation: 4,
|
||||
revision: 2,
|
||||
last_command_id: 1,
|
||||
state: { kind: 'busy', state: { kind: 'run', state: 'paused' } },
|
||||
};
|
||||
|
||||
@@ -562,8 +562,6 @@
|
||||
nextWorkerCommandId = commandId + 1;
|
||||
const envelope = {
|
||||
command_id: commandId,
|
||||
expected_execution_generation: state.execution_generation,
|
||||
expected_worker_state_revision: state.revision,
|
||||
};
|
||||
switch (command) {
|
||||
case "pause":
|
||||
|
||||
Reference in New Issue
Block a user