fix: clear stale internal worker in-flight state

This commit is contained in:
2026-08-20 06:18:21 +09:00
parent 80ffff642f
commit de72afd9a1
3 changed files with 72 additions and 14 deletions
+12
View File
@@ -555,6 +555,7 @@ pub(crate) async fn prepare_internal_worker_session(
spawn_internal_log_event_bridge(sink.clone(), event_tx.clone()); spawn_internal_log_event_bridge(sink.clone(), event_tx.clone());
let alerter = Alerter::new(event_tx.clone()); let alerter = Alerter::new(event_tx.clone());
let in_flight = InFlightEvents::new(event_tx.clone()); let in_flight = InFlightEvents::new(event_tx.clone());
let actor_in_flight = in_flight.clone();
worker.attach_alerter(alerter.clone()); worker.attach_alerter(alerter.clone());
worker.attach_event_tx(event_tx.clone()); worker.attach_event_tx(event_tx.clone());
worker.attach_in_flight_events(in_flight.clone()); worker.attach_in_flight_events(in_flight.clone());
@@ -587,6 +588,7 @@ pub(crate) async fn prepare_internal_worker_session(
while let Some(command) = command_rx.recv().await { while let Some(command) = command_rx.recv().await {
match command { match command {
InternalWorkerSessionCommand::Run(input) => { InternalWorkerSessionCommand::Run(input) => {
actor_in_flight.clear();
let cancel_sender = worker.engine_mut().cancel_sender(); let cancel_sender = worker.engine_mut().cancel_sender();
let mut run = std::pin::pin!(worker.run_text(&input)); let mut run = std::pin::pin!(worker.run_text(&input));
loop { loop {
@@ -599,6 +601,7 @@ pub(crate) async fn prepare_internal_worker_session(
Some(error.to_string()), Some(error.to_string()),
), ),
}; };
actor_in_flight.clear();
status.store(turn_status.encode(), std::sync::atomic::Ordering::Release); status.store(turn_status.encode(), std::sync::atomic::Ordering::Release);
if let Some(message) = error { if let Some(message) = error {
*last_error.lock().unwrap() = Some(message.clone()); *last_error.lock().unwrap() = Some(message.clone());
@@ -622,6 +625,7 @@ pub(crate) async fn prepare_internal_worker_session(
Some(InternalWorkerSessionCommand::Stop(done)) => { Some(InternalWorkerSessionCommand::Stop(done)) => {
let _ = cancel_sender.send(()).await; let _ = cancel_sender.send(()).await;
let _ = (&mut run).await; let _ = (&mut run).await;
actor_in_flight.clear();
status.store(InternalWorkerSessionStatus::Stopped.encode(), std::sync::atomic::Ordering::Release); status.store(InternalWorkerSessionStatus::Stopped.encode(), std::sync::atomic::Ordering::Release);
let _ = event_tx.send(Event::Status { status: WorkerStatus::Paused }); let _ = event_tx.send(Event::Status { status: WorkerStatus::Paused });
let _ = event_tx.send(Event::Shutdown); let _ = event_tx.send(Event::Shutdown);
@@ -634,6 +638,7 @@ pub(crate) async fn prepare_internal_worker_session(
} }
None => { None => {
let _ = cancel_sender.send(()).await; let _ = cancel_sender.send(()).await;
actor_in_flight.clear();
return; return;
} }
} }
@@ -642,6 +647,7 @@ pub(crate) async fn prepare_internal_worker_session(
} }
} }
InternalWorkerSessionCommand::Stop(done) => { InternalWorkerSessionCommand::Stop(done) => {
actor_in_flight.clear();
status.store( status.store(
InternalWorkerSessionStatus::Stopped.encode(), InternalWorkerSessionStatus::Stopped.encode(),
std::sync::atomic::Ordering::Release, std::sync::atomic::Ordering::Release,
@@ -656,6 +662,7 @@ pub(crate) async fn prepare_internal_worker_session(
} }
} }
} }
actor_in_flight.clear();
}); });
Ok(handle) Ok(handle)
@@ -1068,6 +1075,10 @@ permission = "write"
handle.wait_until_idle().await, handle.wait_until_idle().await,
InternalWorkerSessionStatus::Idle InternalWorkerSessionStatus::Idle
); );
handle
.in_flight
.tool_call_start("stale-call".to_string(), "Read".to_string());
assert_eq!(handle.protocol_snapshot().in_flight.blocks.len(), 1);
let entries_after_first = handle.entries().len(); let entries_after_first = handle.entries().len();
assert!(entries_after_first >= 4); assert!(entries_after_first >= 4);
handle.send("follow-up").await.expect("send follow-up turn"); handle.send("follow-up").await.expect("send follow-up turn");
@@ -1075,6 +1086,7 @@ permission = "write"
handle.wait_until_idle().await, handle.wait_until_idle().await,
InternalWorkerSessionStatus::Idle InternalWorkerSessionStatus::Idle
); );
assert!(handle.protocol_snapshot().in_flight.blocks.is_empty());
assert_eq!(calls.load(Ordering::SeqCst), 2); assert_eq!(calls.load(Ordering::SeqCst), 2);
assert!(handle.entries().len() > entries_after_first); assert!(handle.entries().len() > entries_after_first);
@@ -1314,9 +1314,36 @@ Deno.test("parent snapshot authoritatively replaces Internal Worker projections"
kind: "sub_worker", kind: "sub_worker",
}, },
revision: 4, revision: 4,
entries: [], entries: [{
kind: "assistant_item",
ts: 1,
item: {
kind: "tool_call",
call_id: "committed-call",
name: "Read",
arguments: JSON.stringify({ file_path: "/repo/a.md" }),
},
}, {
kind: "tool_result",
ts: 2,
item: {
kind: "tool_result",
call_id: "committed-call",
summary: "read file",
content: "content",
is_error: false,
},
}],
status: "idle", status: "idle",
in_flight: { blocks: [] }, in_flight: {
blocks: [{
kind: "tool_call",
id: "committed-call",
name: "Read",
args: JSON.stringify({ file_path: "/repo/a.md" }),
state: "done",
}],
},
internal_workers: [], internal_workers: [],
}]; }];
const projector = createConsoleProjector(); const projector = createConsoleProjector();
@@ -1340,6 +1367,10 @@ Deno.test("parent snapshot authoritatively replaces Internal Worker projections"
assertEquals(projection.internalWorkers.map((worker) => worker.worker.session_id), [ assertEquals(projection.internalWorkers.map((worker) => worker.worker.session_id), [
"replacement", "replacement",
]); ]);
const childLines = projection.internalWorkers[0].console.lines;
assertEquals(childLines.length, 1);
assertEquals(new Set(childLines.map((line) => line.id)).size, 1);
assertEquals(childLines[0].kind, "tool");
}); });
Deno.test("snapshot restores TaskStore state from system history", () => { Deno.test("snapshot restores TaskStore state from system history", () => {
@@ -224,6 +224,21 @@ function projectVisibleConsole(
}; };
} }
function appendSnapshotInFlightLines(
projection: ConsoleProjection,
blocks: InFlightBlock[],
eventId: string,
cwd: string | null,
): void {
const lineIds = new Set(projection.lines.map((line) => line.id));
blocks.forEach((block, index) => {
const pending = inFlightLine(`${eventId}:${index}`, block, cwd);
if (lineIds.has(pending.id)) return;
projection.lines.push(pending);
lineIds.add(pending.id);
});
}
function projectInternalWorkerSnapshot( function projectInternalWorkerSnapshot(
snapshot: InternalWorkerSnapshot, snapshot: InternalWorkerSnapshot,
eventId: string, eventId: string,
@@ -235,15 +250,12 @@ function projectInternalWorkerSnapshot(
cwd, cwd,
); );
console.status = snapshot.status; console.status = snapshot.status;
for (const block of snapshot.in_flight?.blocks ?? []) { appendSnapshotInFlightLines(
console.lines.push( console,
inFlightLine( snapshot.in_flight?.blocks ?? [],
`${eventId}:internal:${snapshot.worker.session_id}:in-flight`, `${eventId}:internal:${snapshot.worker.session_id}:in-flight`,
block, cwd,
cwd, );
),
);
}
if (snapshot.error) { if (snapshot.error) {
console.lines.push({ console.lines.push({
id: `${eventId}:internal:${snapshot.worker.session_id}:error`, id: `${eventId}:internal:${snapshot.worker.session_id}:error`,
@@ -385,9 +397,12 @@ export function applyProtocolEvent(
next.lines = snapshot.lines; next.lines = snapshot.lines;
next.tasks = snapshot.tasks; next.tasks = snapshot.tasks;
next.taskNextId = snapshot.taskNextId; next.taskNextId = snapshot.taskNextId;
for (const block of event.data.in_flight?.blocks ?? []) { appendSnapshotInFlightLines(
next.lines.push(inFlightLine(envelope.eventId, block, next.cwd)); next,
} event.data.in_flight?.blocks ?? [],
`${envelope.eventId}:snapshot-in-flight`,
next.cwd,
);
next.internalWorkers = (event.data.internal_workers ?? []).map((worker) => next.internalWorkers = (event.data.internal_workers ?? []).map((worker) =>
projectInternalWorkerSnapshot(worker, envelope.eventId, next.cwd) projectInternalWorkerSnapshot(worker, envelope.eventId, next.cwd)
); );