feat: integrate Worker state authority
This commit is contained in:
@@ -10,6 +10,37 @@ export type CompletionKind = "file";
|
||||
|
||||
export type WorkerStatus = "idle" | "running" | "paused" | "stopped";
|
||||
|
||||
export type WorkerCommandEnvelope = {
|
||||
/**
|
||||
* Caller-owned sequence. A controller accepts command ids in strictly
|
||||
* increasing order for one execution generation.
|
||||
*/
|
||||
command_id: number, expected_execution_generation: number, expected_worker_state_revision: number, };
|
||||
|
||||
export type WorkerCommandKind = "resume" | "cancel" | "pause" | "compact" | "shutdown";
|
||||
|
||||
export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "conflict" | "invalid_state";
|
||||
|
||||
export type WorkerCommandAcknowledgement = { command_id: number, command: WorkerCommandKind, disposition: WorkerCommandDisposition,
|
||||
/**
|
||||
* The complete authoritative state observed after command admission.
|
||||
*/
|
||||
state: WorkerStateSnapshot, };
|
||||
|
||||
export type WorkerRunState = "running" | "pausing" | "paused" | "cancelling";
|
||||
|
||||
export type WorkerMaintenanceState = "compacting";
|
||||
|
||||
export type WorkerBusyState = { "kind": "run", "state": WorkerRunState } | { "kind": "maintenance", "state": WorkerMaintenanceState };
|
||||
|
||||
export type WorkerState = { "kind": "idle" } | { "kind": "busy", "state": WorkerBusyState };
|
||||
|
||||
export type WorkerStateSnapshot = { execution_generation: number, revision: number,
|
||||
/**
|
||||
* Highest lifecycle command id observed by this controller generation.
|
||||
*/
|
||||
last_command_id: number, state: WorkerState, };
|
||||
|
||||
export type TurnResult = "finished" | "paused";
|
||||
|
||||
export type InvokeKind = "user_send" | "notify" | "worker_event" | "system_reminder" | "wakeup";
|
||||
@@ -202,7 +233,16 @@ resource_key?: string | null,
|
||||
/**
|
||||
* Producer-owned monotonic revision for this Worker subject.
|
||||
*/
|
||||
subject_revision: number, state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null,
|
||||
subject_revision: number,
|
||||
/**
|
||||
* Latest revisioned foreground state observed from the Worker. This remains
|
||||
* absent until an authoritative Worker snapshot/event has been applied.
|
||||
*/
|
||||
worker_state?: WorkerStateSnapshot | null,
|
||||
/**
|
||||
* Runtime catalog lifecycle compatibility projection; not foreground-state authority.
|
||||
*/
|
||||
state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null,
|
||||
/**
|
||||
* Workspace-facing Repository key. Runtime producers leave this unset and
|
||||
* Workspace Server projections replace `repository_id` with this field.
|
||||
@@ -231,7 +271,7 @@ export type SubscriptionFramePayload = { "frame": "request", "message": Subscrip
|
||||
|
||||
export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod });
|
||||
|
||||
export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array<Segment>, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } };
|
||||
export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array<Segment>, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume", "params": { command: WorkerCommandEnvelope, } } | { "method": "cancel", "params": { command: WorkerCommandEnvelope, } } | { "method": "pause", "params": { command: WorkerCommandEnvelope, } } | { "method": "compact", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } };
|
||||
|
||||
export type Event = { "event": "submission_accepted", "data": { submission_request_id: string, submission_id: string, disposition: SubmissionDisposition, } } | { "event": "submission_rejected", "data": { submission_request_id: string, message: string, } } | { "event": "pending_submissions_changed", "data": { pending: PendingSubmissionsSnapshot, } } | { "event": "user_message", "data": { segments: Array<Segment>, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number,
|
||||
/**
|
||||
@@ -247,7 +287,12 @@ summary: string,
|
||||
* Full tool output. Absent when the tool chose to return
|
||||
* summary-only, or when the result was pruned.
|
||||
*/
|
||||
output?: string | null, disposition?: ToolResultDisposition | null, is_error: boolean, } } | { "event": "usage", "data": { input_tokens: number | null, output_tokens: number | null, cache_read_input_tokens?: number | null, } } | { "event": "run_end", "data": { result: RunResult, } } | { "event": "error", "data": { code: ErrorCode, message: string, } } | { "event": "snapshot", "data": { session: SessionSnapshot, greeting: Greeting, status: WorkerStatus,
|
||||
output?: string | null, disposition?: ToolResultDisposition | null, is_error: boolean, } } | { "event": "usage", "data": { input_tokens: number | null, output_tokens: number | null, cache_read_input_tokens?: number | null, } } | { "event": "run_end", "data": { result: RunResult, } } | { "event": "error", "data": { code: ErrorCode, message: string, } } | { "event": "snapshot", "data": { session: SessionSnapshot, greeting: Greeting,
|
||||
/**
|
||||
* Full revisioned live execution state. `Stopped` remains Runtime
|
||||
* catalog authority and is deliberately not represented here.
|
||||
*/
|
||||
state: WorkerStateSnapshot,
|
||||
/**
|
||||
* Unfinished model output that has already streamed in the current
|
||||
* run but is not yet represented by committed snapshot entries.
|
||||
@@ -257,4 +302,4 @@ in_flight?: InFlightSnapshot,
|
||||
* Parent-owned Internal Worker sessions visible to this client.
|
||||
* Service-private Internal Workers are deliberately excluded.
|
||||
*/
|
||||
internal_workers?: Array<InternalWorkerSnapshot>, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "status", "data": { status: WorkerStatus, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array<CompletionEntry>, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array<RewindTarget>, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array<Segment>, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" };
|
||||
internal_workers?: Array<InternalWorkerSnapshot>, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "worker_state", "data": { snapshot: WorkerStateSnapshot, } } | { "event": "command_acknowledged", "data": { acknowledgement: WorkerCommandAcknowledgement, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array<CompletionEntry>, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array<RewindTarget>, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array<Segment>, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" };
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { Event } from "$lib/generated/protocol";
|
||||
import type { Event, WorkerStateSnapshot, WorkerStatus } from "$lib/generated/protocol";
|
||||
import {
|
||||
type ConsoleEventInput,
|
||||
type ConsoleLine,
|
||||
@@ -19,6 +19,23 @@ declare const Deno: {
|
||||
test(name: string, fn: () => void): void;
|
||||
};
|
||||
|
||||
function workerState(status: WorkerStatus): WorkerStateSnapshot {
|
||||
return {
|
||||
execution_generation: 1,
|
||||
revision: status === "idle" ? 0 : 1,
|
||||
last_command_id: 0,
|
||||
state: status === "idle"
|
||||
? { kind: "idle" }
|
||||
: {
|
||||
kind: "busy",
|
||||
state: {
|
||||
kind: "run",
|
||||
state: status === "paused" ? "paused" : "running",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function assert(condition: unknown, message: string): asserts condition {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
@@ -131,7 +148,7 @@ function snapshotEvent(cwd: string, entries: unknown[] = []): Event {
|
||||
context_window: 100,
|
||||
context_tokens: 20,
|
||||
},
|
||||
status: "idle",
|
||||
state: workerState("idle"),
|
||||
in_flight: { blocks: [] },
|
||||
},
|
||||
};
|
||||
@@ -201,6 +218,66 @@ Deno.test("console routing projects live errors but not completion replies", ()
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("Worker state events and acknowledgements apply monotonically", () => {
|
||||
const projector = createConsoleProjector();
|
||||
const running: WorkerStateSnapshot = {
|
||||
execution_generation: 4,
|
||||
revision: 3,
|
||||
last_command_id: 2,
|
||||
state: { kind: "busy", state: { kind: "run", state: "running" } },
|
||||
};
|
||||
const paused: WorkerStateSnapshot = {
|
||||
...running,
|
||||
revision: 4,
|
||||
last_command_id: 3,
|
||||
state: { kind: "busy", state: { kind: "run", state: "paused" } },
|
||||
};
|
||||
let projection = projector.append([
|
||||
{
|
||||
eventId: "running",
|
||||
event: { event: "worker_state", data: { snapshot: running } },
|
||||
},
|
||||
{
|
||||
eventId: "stale",
|
||||
event: {
|
||||
event: "worker_state",
|
||||
data: { snapshot: { ...running, revision: 2, state: { kind: "idle" } } },
|
||||
},
|
||||
},
|
||||
{
|
||||
eventId: "pause-ack",
|
||||
event: {
|
||||
event: "command_acknowledged",
|
||||
data: {
|
||||
acknowledgement: {
|
||||
command_id: 3,
|
||||
command: "pause",
|
||||
disposition: "accepted",
|
||||
state: paused,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
assertEquals(projection.workerState, paused);
|
||||
assertEquals(projection.status, "paused");
|
||||
|
||||
projection = projector.append([{
|
||||
eventId: "conflict",
|
||||
event: {
|
||||
event: "worker_state",
|
||||
data: { snapshot: { ...paused, state: { kind: "idle" } } },
|
||||
},
|
||||
}]);
|
||||
assertEquals(projection.workerState, paused);
|
||||
assert(
|
||||
projection.lines.some((line) =>
|
||||
line.eventId === "conflict:worker-state-conflict" && line.error
|
||||
),
|
||||
"conflicting equal-version snapshots must fail closed",
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("snapshot replaces a live error with one durable run_errored row", () => {
|
||||
const projector = createConsoleProjector();
|
||||
let projection = projector.append([
|
||||
@@ -213,7 +290,7 @@ Deno.test("snapshot replaces a live error with one durable run_errored row", ()
|
||||
},
|
||||
{
|
||||
eventId: "idle-after-error",
|
||||
event: { event: "status", data: { status: "idle" } } satisfies Event,
|
||||
event: { event: "worker_state", data: { snapshot: workerState("idle") } } satisfies Event,
|
||||
},
|
||||
]);
|
||||
|
||||
@@ -653,7 +730,7 @@ Deno.test("projectConsole streams distinct Bash stdout and stderr through termin
|
||||
Deno.test("snapshot restores bounded in-flight Bash command output", () => {
|
||||
const snapshot = snapshotEvent("/repo");
|
||||
if (snapshot.event !== "snapshot") throw new Error("snapshot fixture expected");
|
||||
snapshot.data.status = "running";
|
||||
snapshot.data.state = workerState("running");
|
||||
snapshot.data.in_flight = {
|
||||
blocks: [{
|
||||
kind: "tool_call",
|
||||
@@ -1403,7 +1480,7 @@ Deno.test("projectConsole hides lifecycle events and renders system items", () =
|
||||
const projection = projectConsole([
|
||||
{
|
||||
eventId: "30",
|
||||
event: { event: "status", data: { status: "running" } } satisfies Event,
|
||||
event: { event: "worker_state", data: { snapshot: workerState("running") } } satisfies Event,
|
||||
},
|
||||
{
|
||||
eventId: "31",
|
||||
@@ -1527,7 +1604,7 @@ Deno.test("projectConsole renders snapshot entries and in-flight output", () =>
|
||||
context_window: 100,
|
||||
context_tokens: 20,
|
||||
},
|
||||
status: "running",
|
||||
state: workerState("running"),
|
||||
in_flight: {
|
||||
blocks: [
|
||||
{ kind: "text", text: "partial" },
|
||||
@@ -1578,7 +1655,7 @@ Deno.test("projectConsole restores system items from snapshot entries", () => {
|
||||
context_window: 100,
|
||||
context_tokens: 20,
|
||||
},
|
||||
status: "idle",
|
||||
state: workerState("idle"),
|
||||
},
|
||||
} satisfies Event,
|
||||
}]);
|
||||
@@ -1922,7 +1999,7 @@ Deno.test("console Worker views expose only direct Internal Workers", () => {
|
||||
kind: "sub_worker",
|
||||
},
|
||||
revision: 1,
|
||||
event: { event: "status", data: { status: "running" } },
|
||||
event: { event: "worker_state", data: { snapshot: workerState("running") } },
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1941,7 +2018,7 @@ Deno.test("console Worker views expose only direct Internal Workers", () => {
|
||||
kind: "sub_worker",
|
||||
},
|
||||
revision: 1,
|
||||
event: { event: "status", data: { status: "idle" } },
|
||||
event: { event: "worker_state", data: { snapshot: workerState("idle") } },
|
||||
},
|
||||
},
|
||||
}]);
|
||||
@@ -2033,7 +2110,7 @@ Deno.test("parent snapshot authoritatively replaces Internal Worker projections"
|
||||
kind: "sub_worker",
|
||||
},
|
||||
revision: 1,
|
||||
event: { event: "status", data: { status: "running" } },
|
||||
event: { event: "worker_state", data: { snapshot: workerState("running") } },
|
||||
},
|
||||
},
|
||||
}]);
|
||||
|
||||
@@ -10,6 +10,9 @@ import type {
|
||||
InternalWorkerRef,
|
||||
InternalWorkerSnapshot,
|
||||
Segment,
|
||||
WorkerState,
|
||||
WorkerStateSnapshot,
|
||||
WorkerStatus,
|
||||
} from "$lib/generated/protocol";
|
||||
import { stringify as stringifyYaml } from "yaml";
|
||||
import { workspaceRoute } from "$lib/workspace/api/http";
|
||||
@@ -169,6 +172,7 @@ export type ConsoleProjection = {
|
||||
tasks: ConsoleTask[];
|
||||
taskNextId: number;
|
||||
status: string | null;
|
||||
workerState: WorkerStateSnapshot | null;
|
||||
usage: string | null;
|
||||
runActivity: RunActivityStats;
|
||||
cwd: string | null;
|
||||
@@ -251,12 +255,22 @@ export function isConsoleProjectionEvent(event: ProtocolEvent): boolean {
|
||||
return event.event !== "completions";
|
||||
}
|
||||
|
||||
function workerStatusFromState(snapshot: WorkerStateSnapshot): WorkerStatus {
|
||||
if (snapshot.state.kind === "idle") return "idle";
|
||||
if (
|
||||
snapshot.state.state.kind === "run" &&
|
||||
snapshot.state.state.state === "paused"
|
||||
) return "paused";
|
||||
return "running";
|
||||
}
|
||||
|
||||
export function emptyConsoleProjection(): ConsoleProjection {
|
||||
return {
|
||||
lines: [],
|
||||
tasks: [],
|
||||
taskNextId: 1,
|
||||
status: null,
|
||||
workerState: null,
|
||||
usage: null,
|
||||
runActivity: emptyRunActivityStats(),
|
||||
cwd: null,
|
||||
@@ -783,6 +797,60 @@ function refreshCompactionActivity(
|
||||
return changed ? { ...projection, lines } : projection;
|
||||
}
|
||||
|
||||
function workerStateEqual(left: WorkerState, right: WorkerState): boolean {
|
||||
if (left.kind !== right.kind) return false;
|
||||
if (left.kind === "idle" || right.kind === "idle") return true;
|
||||
return left.state.kind === right.state.kind &&
|
||||
left.state.state === right.state.state;
|
||||
}
|
||||
|
||||
function workerStateSnapshotEqual(
|
||||
left: WorkerStateSnapshot,
|
||||
right: WorkerStateSnapshot,
|
||||
): boolean {
|
||||
return left.execution_generation === right.execution_generation &&
|
||||
left.revision === right.revision &&
|
||||
left.last_command_id === right.last_command_id &&
|
||||
workerStateEqual(left.state, right.state);
|
||||
}
|
||||
|
||||
function applyWorkerStateSnapshot(
|
||||
projection: ConsoleProjection,
|
||||
incoming: WorkerStateSnapshot,
|
||||
eventId: string,
|
||||
): void {
|
||||
const current = projection.workerState;
|
||||
if (!current) {
|
||||
projection.workerState = incoming;
|
||||
projection.status = workerStatusFromState(incoming);
|
||||
return;
|
||||
}
|
||||
const generationOrder = incoming.execution_generation -
|
||||
current.execution_generation;
|
||||
const revisionOrder = incoming.revision - current.revision;
|
||||
if (generationOrder > 0 || (generationOrder === 0 && revisionOrder > 0)) {
|
||||
projection.workerState = incoming;
|
||||
projection.status = workerStatusFromState(incoming);
|
||||
return;
|
||||
}
|
||||
if (generationOrder < 0 || (generationOrder === 0 && revisionOrder < 0)) {
|
||||
return;
|
||||
}
|
||||
if (!workerStateSnapshotEqual(current, incoming)) {
|
||||
projection.lines.push(
|
||||
line(
|
||||
`${eventId}:worker-state-conflict`,
|
||||
"error",
|
||||
"error · internal",
|
||||
`worker state stream rejected: conflicting snapshots at generation ${incoming.execution_generation} revision ${incoming.revision}`,
|
||||
undefined,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function applyProtocolEvent(
|
||||
projection: ConsoleProjection,
|
||||
envelope: ConsoleEventInput,
|
||||
@@ -793,6 +861,7 @@ export function applyProtocolEvent(
|
||||
tasks: [...projection.tasks],
|
||||
taskNextId: projection.taskNextId,
|
||||
status: projection.status,
|
||||
workerState: projection.workerState,
|
||||
usage: projection.usage,
|
||||
runActivity: applyRunActivityEvent(
|
||||
projection.runActivity,
|
||||
@@ -903,7 +972,6 @@ export function applyProtocolEvent(
|
||||
);
|
||||
break;
|
||||
case "snapshot": {
|
||||
next.status = event.data.status;
|
||||
next.cwd = event.data.greeting.cwd;
|
||||
const snapshot = snapshotProjectionFromSession(
|
||||
envelope.eventId,
|
||||
@@ -953,6 +1021,7 @@ export function applyProtocolEvent(
|
||||
};
|
||||
}
|
||||
}
|
||||
applyWorkerStateSnapshot(next, event.data.state, envelope.eventId);
|
||||
break;
|
||||
}
|
||||
case "internal_worker": {
|
||||
@@ -1000,8 +1069,15 @@ export function applyProtocolEvent(
|
||||
if (existingIndex >= 0) next.internalWorkers.splice(existingIndex, 1);
|
||||
break;
|
||||
}
|
||||
case "status":
|
||||
next.status = event.data.status;
|
||||
case "worker_state":
|
||||
applyWorkerStateSnapshot(next, event.data.snapshot, envelope.eventId);
|
||||
break;
|
||||
case "command_acknowledged":
|
||||
applyWorkerStateSnapshot(
|
||||
next,
|
||||
event.data.acknowledgement.state,
|
||||
envelope.eventId,
|
||||
);
|
||||
break;
|
||||
case "command":
|
||||
applyCommandEvent(next, envelope.eventId, event.data.event);
|
||||
@@ -1939,6 +2015,7 @@ function snapshotProjectionFromSession(
|
||||
tasks: [],
|
||||
taskNextId: 1,
|
||||
status: null,
|
||||
workerState: null,
|
||||
usage: null,
|
||||
runActivity: emptyRunActivityStats(),
|
||||
cwd,
|
||||
|
||||
@@ -75,7 +75,12 @@ Deno.test("new invoke and running snapshot reset run activity", () => {
|
||||
data: {
|
||||
entries: [],
|
||||
greeting: { text: "", profile: "" },
|
||||
status: "idle",
|
||||
state: {
|
||||
execution_generation: 1,
|
||||
revision: 0,
|
||||
last_command_id: 0,
|
||||
state: { kind: "idle" },
|
||||
},
|
||||
in_flight: {},
|
||||
internal_workers: [],
|
||||
},
|
||||
|
||||
@@ -25,7 +25,9 @@ export function applyRunActivityEvent(
|
||||
case "invoke_start":
|
||||
return { ...emptyRunActivityStats(), startedAtMs: observedAtMs };
|
||||
case "snapshot":
|
||||
return event.data.status === "running"
|
||||
return event.data.state.state.kind === "busy" &&
|
||||
!(event.data.state.state.state.kind === "run" &&
|
||||
event.data.state.state.state.state === "paused")
|
||||
? { ...emptyRunActivityStats(), startedAtMs: observedAtMs }
|
||||
: emptyRunActivityStats();
|
||||
case "turn_start":
|
||||
|
||||
@@ -620,7 +620,7 @@ Deno.test("Worker Console paste chips preserve typed draft and target authority"
|
||||
consolePage.includes("preserveExactText: value.textPastes.length > 0") &&
|
||||
consolePage.includes("composerDrafts.set(activeComposerTargetKey") &&
|
||||
consolePage.includes("switchComposerTarget(target)") &&
|
||||
consolePage.includes('sendControl({ method: "cancel" }, "Stop")'),
|
||||
consolePage.includes('sendWorkerControl("cancel")'),
|
||||
"Paste chips should use shared threshold classification, atomic keyboard behavior, accessible labels, typed restore, and per-Worker draft authority",
|
||||
);
|
||||
});
|
||||
@@ -787,7 +787,10 @@ Deno.test("Worker Console page is routed by runtime_id and worker_id through bac
|
||||
consolePage.includes(
|
||||
'const composerEditable = $derived(protocolState === "open" && !sending);',
|
||||
) &&
|
||||
consolePage.includes('sendControl({ method: "cancel" }, "Stop")') &&
|
||||
consolePage.includes('sendWorkerControl("cancel")') &&
|
||||
consolePage.includes("lifecycleMethod(command)") &&
|
||||
consolePage.includes("expected_worker_state_revision") &&
|
||||
consolePage.includes("expected_execution_generation") &&
|
||||
consolePage.includes("onsubmit={handleComposerSubmit}") &&
|
||||
consolePage.includes("disabled={!composerEditable}") &&
|
||||
consolePage.includes("class:stop={workerRunning}") &&
|
||||
|
||||
@@ -19,6 +19,7 @@ import type {
|
||||
Event as PodProtocolEvent,
|
||||
Method as PodProtocolMethod,
|
||||
Segment as PodProtocolSegment,
|
||||
WorkerStateSnapshot,
|
||||
} from "$lib/generated/protocol";
|
||||
import type {
|
||||
GitCommitSummary as SharedGitCommitSummary,
|
||||
@@ -99,6 +100,7 @@ export type Worker = {
|
||||
tags: string[];
|
||||
workspace: { visibility: string; identity: string };
|
||||
state: string;
|
||||
worker_state?: WorkerStateSnapshot | null;
|
||||
pinned?: boolean;
|
||||
retention_state?: string;
|
||||
last_seen_at?: string | null;
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import type { WorkerStateSnapshot } from "$lib/generated/protocol";
|
||||
|
||||
export function liveWorkerState(worker: {
|
||||
state: string;
|
||||
worker_state?: WorkerStateSnapshot | null;
|
||||
}): string {
|
||||
const state = worker.worker_state?.state;
|
||||
if (!state) return worker.state === "stopped" ? "stopped" : "unknown";
|
||||
if (state.kind === "idle") return "idle";
|
||||
if (state.state.kind === "maintenance") return "running";
|
||||
return state.state.state === "paused" ? "paused" : "running";
|
||||
}
|
||||
@@ -5,6 +5,7 @@ function assertEquals(actual: unknown, expected: unknown): void {
|
||||
throw new Error(`expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
|
||||
}
|
||||
}
|
||||
import { liveWorkerState } from './worker-state';
|
||||
import {
|
||||
applyWorkspaceWorkersFrame,
|
||||
createWorkspaceWorkersProjection,
|
||||
@@ -33,6 +34,22 @@ function worker(
|
||||
};
|
||||
}
|
||||
|
||||
Deno.test('Worker list state uses the authoritative live snapshot separately from lifecycle', () => {
|
||||
const active = worker('runtime-a', 'worker-1', 1);
|
||||
active.worker_state = {
|
||||
execution_generation: 4,
|
||||
revision: 2,
|
||||
last_command_id: 1,
|
||||
state: { kind: 'busy', state: { kind: 'run', state: 'paused' } },
|
||||
};
|
||||
assertEquals(liveWorkerState(active), 'paused');
|
||||
|
||||
const unavailable = worker('runtime-a', 'worker-2', 1);
|
||||
assertEquals(liveWorkerState(unavailable), 'unknown');
|
||||
unavailable.state = 'stopped';
|
||||
assertEquals(liveWorkerState(unavailable), 'stopped');
|
||||
});
|
||||
|
||||
Deno.test('workspace Worker snapshot keeps equal local ids from different Runtimes', () => {
|
||||
const projection = createWorkspaceWorkersProjection();
|
||||
const frame: SubscriptionFrame = {
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
applyWorkspaceWorkersFrame,
|
||||
createWorkspaceWorkersProjection,
|
||||
} from './worker-subscription-model';
|
||||
import { liveWorkerState } from './worker-state';
|
||||
import { compareWorkersForSidebar } from './workers';
|
||||
import type { Worker } from './types';
|
||||
|
||||
@@ -90,7 +91,8 @@ function projectWorker(worker: SubscriptionWorker): SidebarWorker {
|
||||
profile: worker.profile ?? null,
|
||||
tags: [],
|
||||
workspace: { visibility: 'workspace', identity: 'runtime_subscription_worker' },
|
||||
state: worker.state,
|
||||
state: liveWorkerState(worker),
|
||||
worker_state: worker.worker_state,
|
||||
pinned: false,
|
||||
retention_state: 'transient',
|
||||
implementation: {
|
||||
|
||||
+60
-31
@@ -52,11 +52,7 @@
|
||||
import { pushWorkspaceAlert } from "$lib/workspace/alerts/store";
|
||||
import { workspaceApiPath } from "$lib/workspace/api/http";
|
||||
import { workspaceMultiplexer, type WorkspaceMultiplexerSubscription } from "$lib/workspace/multiplexer";
|
||||
import type {
|
||||
Diagnostic,
|
||||
Worker,
|
||||
PodProtocolEvent,
|
||||
} from "$lib/workspace/sidebar/types";
|
||||
import type { Diagnostic, Worker } from "$lib/workspace/sidebar/types";
|
||||
|
||||
type Props = {
|
||||
data: {
|
||||
@@ -207,7 +203,6 @@
|
||||
);
|
||||
let pendingObservationEvents: ConsoleEventInput[] = [];
|
||||
let protocolEventSequence = 0;
|
||||
let pendingObservedStates: Array<string | null> = [];
|
||||
let pendingStreamDiagnostics: Diagnostic[] = [];
|
||||
let observationFlushHandle: number | null = null;
|
||||
let nextReloadToken = 0;
|
||||
@@ -249,7 +244,9 @@
|
||||
const diagnostics = $derived(
|
||||
mergeDiagnostics(worker?.diagnostics ?? [], streamDiagnostics),
|
||||
);
|
||||
const workerState = $derived(liveWorkerState ?? worker?.state ?? "loading");
|
||||
const workerState = $derived(
|
||||
liveWorkerState ?? (worker?.state === "stopped" ? "stopped" : "loading"),
|
||||
);
|
||||
const workerRunning = $derived(workerState === "running");
|
||||
const workerPaused = $derived(workerState === "paused");
|
||||
const composerEditable = $derived(protocolState === "open" && !sending);
|
||||
@@ -343,7 +340,6 @@
|
||||
observationFlushHandle = null;
|
||||
}
|
||||
pendingObservationEvents = [];
|
||||
pendingObservedStates = [];
|
||||
pendingStreamDiagnostics = [];
|
||||
}
|
||||
|
||||
@@ -359,18 +355,15 @@
|
||||
function flushObservationBatch() {
|
||||
observationFlushHandle = null;
|
||||
const eventBatch = pendingObservationEvents;
|
||||
const stateBatch = pendingObservedStates;
|
||||
const diagnosticBatch = pendingStreamDiagnostics;
|
||||
pendingObservationEvents = [];
|
||||
pendingObservedStates = [];
|
||||
pendingStreamDiagnostics = [];
|
||||
|
||||
if (eventBatch.length > 0) {
|
||||
const latestState = stateBatch.findLast((state) => state !== null);
|
||||
if (latestState) {
|
||||
liveWorkerState = latestState;
|
||||
}
|
||||
consoleProjection = consoleProjector.append(eventBatch);
|
||||
liveWorkerState = consoleProjection.status === "shutdown"
|
||||
? "shutdown"
|
||||
: workerStateFromSnapshot(consoleProjection.workerState);
|
||||
advanceEventObservedAtVersion();
|
||||
}
|
||||
|
||||
@@ -407,7 +400,6 @@
|
||||
event: payload,
|
||||
observedAtMs,
|
||||
});
|
||||
pendingObservedStates.push(workerStateFromProtocolEvent(payload));
|
||||
scheduleObservationFlush();
|
||||
}
|
||||
|
||||
@@ -541,9 +533,42 @@
|
||||
}
|
||||
}
|
||||
|
||||
let nextWorkerCommandId = 1;
|
||||
|
||||
function lifecycleMethod(
|
||||
command: "pause" | "cancel" | "resume" | "compact",
|
||||
): ProtocolMethod | null {
|
||||
const state = consoleProjection.workerState;
|
||||
if (!state) {
|
||||
sendError = "Worker state snapshot is not available; reconnect before sending control.";
|
||||
return null;
|
||||
}
|
||||
const commandId = Math.max(
|
||||
nextWorkerCommandId,
|
||||
state.last_command_id + 1,
|
||||
);
|
||||
nextWorkerCommandId = commandId + 1;
|
||||
const envelope = {
|
||||
command_id: commandId,
|
||||
expected_execution_generation: state.execution_generation,
|
||||
expected_worker_state_revision: state.revision,
|
||||
};
|
||||
switch (command) {
|
||||
case "pause":
|
||||
return { method: "pause", params: { command: envelope } };
|
||||
case "cancel":
|
||||
return { method: "cancel", params: { command: envelope } };
|
||||
case "resume":
|
||||
return { method: "resume", params: { command: envelope } };
|
||||
case "compact":
|
||||
return { method: "compact", params: { command: envelope } };
|
||||
}
|
||||
}
|
||||
|
||||
function sendWorkerControl(command: "pause" | "cancel" | "resume") {
|
||||
const label = command[0].toUpperCase() + command.slice(1);
|
||||
sendControl({ method: command }, label);
|
||||
const method = lifecycleMethod(command);
|
||||
if (method) sendControl(method, label);
|
||||
}
|
||||
|
||||
function isEditableTarget(target: EventTarget | null): boolean {
|
||||
@@ -627,8 +652,11 @@
|
||||
auto_run: true,
|
||||
},
|
||||
};
|
||||
case "compact":
|
||||
return { method: "compact" };
|
||||
case "compact": {
|
||||
const method = lifecycleMethod("compact");
|
||||
if (!method) throw new Error("Worker state snapshot is not available");
|
||||
return method;
|
||||
}
|
||||
case "list_rewind_targets":
|
||||
return { method: "list_rewind_targets" };
|
||||
case "register_peer":
|
||||
@@ -691,7 +719,7 @@
|
||||
|
||||
function handleComposerSubmit() {
|
||||
if (workerRunning) {
|
||||
sendControl({ method: "cancel" }, "Stop");
|
||||
sendWorkerControl("cancel");
|
||||
return;
|
||||
}
|
||||
void submitDraft(composerInputElement?.snapshot() ?? draft);
|
||||
@@ -889,18 +917,16 @@
|
||||
handleComposerSubmit();
|
||||
}
|
||||
|
||||
function workerStateFromProtocolEvent(
|
||||
event: PodProtocolEvent,
|
||||
function workerStateFromSnapshot(
|
||||
snapshot: ConsoleProjection["workerState"],
|
||||
): string | null {
|
||||
switch (event.event) {
|
||||
case "snapshot":
|
||||
case "status":
|
||||
return event.data.status;
|
||||
case "shutdown":
|
||||
return "shutdown";
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
if (!snapshot) return null;
|
||||
return snapshot.state.kind === "idle"
|
||||
? "idle"
|
||||
: snapshot.state.state.kind === "run" &&
|
||||
snapshot.state.state.state === "paused"
|
||||
? "paused"
|
||||
: "running";
|
||||
}
|
||||
|
||||
function connectProtocolTransport(
|
||||
@@ -1620,7 +1646,10 @@
|
||||
type="button"
|
||||
class="secondary-button"
|
||||
disabled={protocolState !== "open"}
|
||||
onclick={() => sendControl({ method: "compact" }, "Compact")}
|
||||
onclick={() => {
|
||||
const method = lifecycleMethod("compact");
|
||||
if (method) sendControl(method, "Compact");
|
||||
}}
|
||||
>
|
||||
Compact
|
||||
</button>
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import { workerHref } from '$lib/workspace/resource-links';
|
||||
import { formatCurrentWorkdirRevision } from '$lib/workspace/settings/workdir-revision';
|
||||
import { canOpenWorkerConsole } from '$lib/workspace/sidebar/workers';
|
||||
import { liveWorkerState } from '$lib/workspace/sidebar/worker-state';
|
||||
import type { CleanupWorkerCandidate, RuntimeCleanupExecutionResponse, RuntimeCleanupPlanResponse, Worker } from '$lib/workspace/sidebar/types';
|
||||
import type { PageProps } from './$types';
|
||||
|
||||
@@ -136,7 +137,7 @@
|
||||
}
|
||||
|
||||
function workerStatus(worker: Worker): string {
|
||||
return worker.state;
|
||||
return liveWorkerState(worker);
|
||||
}
|
||||
|
||||
function workerProfile(worker: Worker): string {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<script lang="ts">
|
||||
import { workspaceRoute } from '$lib/workspace/api/http';
|
||||
import { liveWorkerState } from '$lib/workspace/sidebar/worker-state';
|
||||
import type { PageData } from './$types';
|
||||
let { data }: { data: PageData } = $props();
|
||||
</script>
|
||||
@@ -24,7 +25,7 @@
|
||||
>Open console</a>
|
||||
</header>
|
||||
<dl class="resource-meta">
|
||||
<dt>Status</dt><dd>{data.worker.state}</dd>
|
||||
<dt>Status</dt><dd>{liveWorkerState(data.worker)}</dd>
|
||||
<dt>Profile</dt><dd>{data.worker.profile}</dd>
|
||||
<dt>Internal ID</dt><dd><code>{data.worker.worker_id}</code></dd>
|
||||
</dl>
|
||||
|
||||
Reference in New Issue
Block a user