Hare/yoi
1
1
Fork 0
Code Actions Packages Releases Activity

fix: map internal worker terminal lifecycles

Browse Source
This commit is contained in:
Hare
2026-08-27 12:27:55 +09:00
parent 975b4fa700
commit 3a7a3307ef
9 changed files with 225 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/session-store/tests/session_test.rs
+28 -10
View File
@@ -102,7 +102,7 @@ async fn run_and_persist(
session_id: session_store::SessionId,
segment_id: session_store::SegmentId,
input: &str,
) -> (Engine<MockLlmClient>, agen::EngineResult) {
) -> (Engine<MockLlmClient>, agen::EngineRunExit) {
// Mirror Worker's run-entry contract: log the user input as segments
// before the worker pushes its flattened user_message; save_delta
// skips the resulting user_message item to avoid double-write.
@@ -125,31 +125,49 @@ async fn run_and_persist(
session_store::save_turn_end(store, session_id, segment_id, worker.turn_count()).unwrap();
match &result {
Ok(r) => {
agen::EngineRunExit::Finished
| agen::EngineRunExit::Paused
| agen::EngineRunExit::Yielded => {
let (legacy_result, interrupted) = match &result {
agen::EngineRunExit::Finished => (agen::EngineResult::Finished, false),
agen::EngineRunExit::Paused => (agen::EngineResult::Paused, true),
agen::EngineRunExit::Yielded => (agen::EngineResult::Yielded, true),
agen::EngineRunExit::Interrupted(_) => unreachable!(),
};
session_store::save_run_completed(
store,
session_id,
segment_id,
r.clone(),
worker.last_run_interrupted(),
legacy_result,
interrupted,
worker.active_run_turn_count(),
)
.unwrap();
}
Err(e) => {
agen::EngineRunExit::Interrupted(agen::StopReason::LimitReached) => {
session_store::save_run_completed(
store,
session_id,
segment_id,
agen::EngineResult::LimitReached,
false,
worker.active_run_turn_count(),
)
.unwrap();
}
agen::EngineRunExit::Interrupted(reason) => {
session_store::save_run_errored(
store,
session_id,
segment_id,
e.to_string(),
worker.last_run_interrupted(),
format!("{reason:?}"),
true,
)
.unwrap();
}
}
let r = result.unwrap();
(worker, r)
(worker, result)
}
// =============================================================================
@@ -292,7 +310,7 @@ async fn session_resume_after_pause() {
.unwrap();
let (_worker, result) = run_and_persist(worker, &store, sid, segid, "Weather?").await;
assert!(matches!(result, agen::EngineResult::Paused));
assert!(matches!(result, agen::EngineRunExit::Paused));
// Check RunCompleted is Paused
let entries = store.read_all(sid, segid).unwrap();
crates/worker/examples/worker_cli.rs
+1
View File
@@ -66,6 +66,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
WorkerRunResult::Finished => println!("(finished)"),
WorkerRunResult::Paused => println!("(paused)"),
WorkerRunResult::LimitReached => println!("(turn limit reached)"),
WorkerRunResult::Interrupted { message, .. } => println!("(interrupted: {message})"),
WorkerRunResult::RolledBack => println!("(empty turn rolled back)"),
}
crates/worker/src/controller.rs
+3 -3
View File
@@ -1650,13 +1650,13 @@ where
WorkerRunResult::Paused => (WorkerStatus::Paused, RunResult::Paused),
WorkerRunResult::LimitReached => (WorkerStatus::Idle, RunResult::LimitReached),
WorkerRunResult::RolledBack => (WorkerStatus::Idle, RunResult::RolledBack),
WorkerRunResult::Interrupted(_message) if pause_requested => {
WorkerRunResult::Interrupted { .. } if pause_requested => {
let _ = event_tx.send(Event::RunEnd { result: RunResult::Paused });
return (WorkerStatus::Paused, shutdown_requested);
}
WorkerRunResult::Interrupted(message) => {
WorkerRunResult::Interrupted { code, message } => {
let _ = event_tx.send(Event::Error {
code: ErrorCode::Internal,
code,
message: message.clone(),
});
if parent_originated {
crates/worker/src/internal_worker.rs
+93 -28
View File
@@ -199,10 +199,12 @@ where
on_cancel_sender(worker.engine_mut().cancel_sender());
match worker.run_text(&input).await {
Ok(WorkerRunResult::Interrupted(message)) => Err(InternalWorkerError {
source: WorkerError::Engine(EngineError::Aborted(message)),
Ok(lifecycle @ WorkerRunResult::Finished)
| Ok(lifecycle @ WorkerRunResult::Paused)
| Ok(lifecycle @ WorkerRunResult::RolledBack) => Ok(InternalWorkerResult {
usage: last_usage.lock().ok().and_then(|slot| slot.clone()),
identity,
lifecycle,
history_entries: store.entries_count(session_id, segment_id),
}),
Ok(WorkerRunResult::LimitReached) => Err(InternalWorkerError {
@@ -213,10 +215,10 @@ where
identity,
history_entries: store.entries_count(session_id, segment_id),
}),
Ok(lifecycle) => Ok(InternalWorkerResult {
Ok(WorkerRunResult::Interrupted { message, .. }) => Err(InternalWorkerError {
source: WorkerError::Engine(EngineError::Aborted(message)),
usage: last_usage.lock().ok().and_then(|slot| slot.clone()),
identity,
lifecycle,
history_entries: store.entries_count(session_id, segment_id),
}),
Err(source) => Err(InternalWorkerError {
@@ -246,6 +248,7 @@ impl Default for InternalWorkerVisibility {
pub(crate) enum InternalWorkerSessionStatus {
Idle,
Running,
Paused,
Stopping,
Stopped,
Failed,
@@ -256,9 +259,10 @@ impl InternalWorkerSessionStatus {
match self {
Self::Idle => 0,
Self::Running => 1,
Self::Stopping => 2,
Self::Stopped => 3,
Self::Failed => 4,
Self::Paused => 2,
Self::Stopping => 3,
Self::Stopped => 4,
Self::Failed => 5,
}
}
@@ -266,13 +270,35 @@ impl InternalWorkerSessionStatus {
match value {
0 => Self::Idle,
1 => Self::Running,
2 => Self::Stopping,
3 => Self::Stopped,
2 => Self::Paused,
3 => Self::Stopping,
4 => Self::Stopped,
_ => Self::Failed,
}
}
}
fn classify_internal_turn_result(
result: Result<WorkerRunResult, WorkerError>,
) -> (InternalWorkerSessionStatus, Option<String>) {
match result {
Ok(WorkerRunResult::Finished) => (InternalWorkerSessionStatus::Idle, None),
Ok(WorkerRunResult::Paused) => (InternalWorkerSessionStatus::Paused, None),
Ok(WorkerRunResult::LimitReached) => (
InternalWorkerSessionStatus::Stopped,
Some("internal Worker reached its turn limit".to_string()),
),
Ok(WorkerRunResult::Interrupted { message, .. }) => {
(InternalWorkerSessionStatus::Stopped, Some(message))
}
Ok(WorkerRunResult::RolledBack) => (
InternalWorkerSessionStatus::Stopped,
Some("internal Worker run was cancelled before AI output".to_string()),
),
Err(error) => (InternalWorkerSessionStatus::Failed, Some(error.to_string())),
}
}
#[derive(Debug, thiserror::Error)]
pub(crate) enum InternalWorkerSessionError {
#[error("failed to build internal Worker session: {message}")]
@@ -367,6 +393,7 @@ impl InternalWorkerSessionHandle {
entries,
status: match self.status() {
InternalWorkerSessionStatus::Running => WorkerStatus::Running,
InternalWorkerSessionStatus::Paused => WorkerStatus::Paused,
InternalWorkerSessionStatus::Idle => WorkerStatus::Idle,
InternalWorkerSessionStatus::Stopping
| InternalWorkerSessionStatus::Stopped
@@ -402,6 +429,7 @@ impl InternalWorkerSessionHandle {
.map_err(
|current| match InternalWorkerSessionStatus::decode(current) {
InternalWorkerSessionStatus::Running
| InternalWorkerSessionStatus::Paused
| InternalWorkerSessionStatus::Stopping => InternalWorkerSessionError::Busy,
InternalWorkerSessionStatus::Stopped | InternalWorkerSessionStatus::Failed => {
InternalWorkerSessionError::Stopped
@@ -747,21 +775,7 @@ pub(crate) async fn prepare_internal_worker_session(
loop {
tokio::select! {
result = &mut run => {
let (turn_status, error) = match result {
Ok(WorkerRunResult::Interrupted(message)) => (
InternalWorkerSessionStatus::Stopped,
Some(message),
),
Ok(WorkerRunResult::LimitReached) => (
InternalWorkerSessionStatus::Stopped,
Some("internal Worker reached its turn limit".to_string()),
),
Ok(_) => (InternalWorkerSessionStatus::Idle, None),
Err(error) => (
InternalWorkerSessionStatus::Failed,
Some(error.to_string()),
),
};
let (turn_status, error) = classify_internal_turn_result(result);
actor_in_flight.clear();
status.store(turn_status.encode(), std::sync::atomic::Ordering::Release);
if let Some(message) = error {
@@ -771,10 +785,15 @@ pub(crate) async fn prepare_internal_worker_session(
message,
});
}
let protocol_status = if turn_status == InternalWorkerSessionStatus::Idle {
WorkerStatus::Idle
} else {
WorkerStatus::Stopped
let protocol_status = match turn_status {
InternalWorkerSessionStatus::Idle => WorkerStatus::Idle,
InternalWorkerSessionStatus::Paused => WorkerStatus::Paused,
InternalWorkerSessionStatus::Stopped
| InternalWorkerSessionStatus::Failed => WorkerStatus::Stopped,
InternalWorkerSessionStatus::Running
| InternalWorkerSessionStatus::Stopping => {
unreachable!("run completion cannot remain active")
}
};
let _ = event_tx.send(Event::Status {
status: protocol_status,
@@ -1261,6 +1280,52 @@ permission = "write"
assert_eq!(result.identity.kind, "test");
}
#[test]
fn internal_turn_result_mapping_is_exhaustive() {
let cases = [
(
WorkerRunResult::Finished,
InternalWorkerSessionStatus::Idle,
false,
),
(
WorkerRunResult::Paused,
InternalWorkerSessionStatus::Paused,
false,
),
(
WorkerRunResult::LimitReached,
InternalWorkerSessionStatus::Stopped,
true,
),
(
WorkerRunResult::Interrupted {
code: protocol::ErrorCode::Internal,
message: "cancelled".to_string(),
},
InternalWorkerSessionStatus::Stopped,
true,
),
(
WorkerRunResult::RolledBack,
InternalWorkerSessionStatus::Stopped,
true,
),
];
for (result, expected_status, expects_error) in cases {
let (status, error) = classify_internal_turn_result(Ok(result));
assert_eq!(status, expected_status);
assert_eq!(error.is_some(), expects_error);
}
let (status, error) = classify_internal_turn_result(Err(WorkerError::Engine(
EngineError::Aborted("fatal".to_string()),
)));
assert_eq!(status, InternalWorkerSessionStatus::Failed);
assert!(error.is_some_and(|message| message.contains("fatal")));
}
#[tokio::test]
async fn fatal_internal_run_transitions_to_stopped_protocol_status() {
let calls = Arc::new(AtomicUsize::new(0));
crates/worker/src/worker.rs
+25 -5
View File
@@ -77,8 +77,8 @@ use crate::skill::{SkillActivationResponse, SkillClientError};
#[cfg(test)]
use async_trait::async_trait;
use protocol::{
AlertLevel, AlertSource, CompactionLifecycle, CompactionLifecycleState, Event, RewindSummary,
RewindTarget, RewindTargetId, Segment,
AlertLevel, AlertSource, CompactionLifecycle, CompactionLifecycleState, ErrorCode, Event,
RewindSummary, RewindTarget, RewindTargetId, Segment,
};
use tokio::net::UnixStream;
use tokio::sync::broadcast;
@@ -2964,7 +2964,10 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
}
EngineRunExit::Interrupted(reason) => {
self.last_run_interrupted = true;
Ok(WorkerRunResult::Interrupted(stop_reason_message(&reason)))
Ok(WorkerRunResult::Interrupted {
code: stop_reason_error_code(&reason),
message: stop_reason_message(&reason),
})
}
EngineRunExit::Yielded => unreachable!("yielded handled above"),
}
@@ -4512,7 +4515,7 @@ impl<C: LlmClient, St: Store> Worker<C, St> {
fn extract_internal_worker_lifecycle_error(lifecycle: &WorkerRunResult) -> Option<WorkerError> {
match lifecycle {
WorkerRunResult::RolledBack => Some(WorkerError::Engine(EngineError::Cancelled)),
WorkerRunResult::Interrupted(message) => {
WorkerRunResult::Interrupted { message, .. } => {
Some(WorkerError::Engine(EngineError::Aborted(message.clone())))
}
WorkerRunResult::Finished | WorkerRunResult::Paused | WorkerRunResult::LimitReached => None,
@@ -5521,6 +5524,23 @@ fn restore_manifest_from_worker_metadata_snapshot(
}
}
fn stop_reason_error_code(reason: &StopReason) -> ErrorCode {
match reason {
StopReason::ContextWindowExceeded | StopReason::Unexpected(EngineError::Client(_)) => {
ErrorCode::ProviderError
}
StopReason::Unexpected(EngineError::Tool(_)) => ErrorCode::ToolError,
StopReason::LimitReached
| StopReason::Cancelled
| StopReason::Unexpected(
EngineError::Aborted(_)
| EngineError::Cancelled
| EngineError::ConfigWarnings(_)
| EngineError::HistoryAppend(_),
) => ErrorCode::Internal,
}
}
fn stop_reason_message(reason: &StopReason) -> String {
match reason {
StopReason::LimitReached => "engine turn limit reached".to_string(),
@@ -5540,7 +5560,7 @@ pub enum WorkerRunResult {
/// The worker reached its configured max_turns limit.
LimitReached,
/// The run was interrupted by a known or unexpected terminal cause.
Interrupted(String),
Interrupted { code: ErrorCode, message: String },
/// The submit-time user turn was rolled back because no AI output was materialized.
RolledBack,
}
web/workspace/src/lib/workspace/sidebar/WorkersNavSection.svelte
+5 -4
View File
@@ -5,7 +5,7 @@
workspaceWorkersStore,
type SidebarWorker,
} from './worker-subscription';
import { canShowWorkerInSidebar } from './workers';
import { canShowWorkerInSidebar, sidebarWorkerActivity } from './workers';
const COLLAPSED_WORKER_COUNT = 6;
@@ -69,6 +69,7 @@
<ul class="nav-list" aria-label="Workers">
{#each visibleWorkers as worker (`${worker.runtime_id}:${worker.worker_id}`)}
{@const href = workerConsoleHref(worker, workspaceId)}
{@const activity = sidebarWorkerActivity(worker)}
<li>
<a
href={href}
@@ -77,11 +78,11 @@
aria-current={currentPath === href ? 'page' : undefined}
>
<span class="worker-status-indicator">
{#if worker.state === 'running'}
{#if activity === 'worker-running'}
<span class="worker-status-spinner"><Spinner label="Running" /></span>
{:else if worker.has_running_internal_workers}
{:else if activity === 'subworker-running'}
<span class="worker-status-spinner is-subworker"><Spinner label="SubWorker running" /></span>
{:else if worker.state === 'idle'}
{:else if activity === 'idle'}
<span class="worker-status-dot" aria-label="Idle"></span>
{/if}
</span>
web/workspace/src/lib/workspace/sidebar/worker-subscription.test.ts
+32 -2
View File
@@ -14,13 +14,18 @@ declare const Deno: {
test(name: string, fn: () => void | Promise<void>): void;
};
function worker(runtimeId: string, workerId: string, revision: number): SubscriptionWorker {
function worker(
runtimeId: string,
workerId: string,
revision: number,
hasRunningInternalWorkers = false,
): SubscriptionWorker {
return {
worker_id: workerId,
runtime_id: runtimeId,
subject_revision: revision,
state: 'idle',
has_running_internal_workers: false,
has_running_internal_workers: hasRunningInternalWorkers,
workspace_id: 'workspace-test',
display_name: null,
profile: null,
@@ -88,3 +93,28 @@ Deno.test('workspace Worker reducer ignores stale events and removes composite s
assertEquals(projection.workers.size, 0);
assertEquals(projection.revisions.get('runtime-a:1'), 4);
});
Deno.test('fatal child stop replaces the running-child sidebar projection', () => {
const projection = createWorkspaceWorkersProjection();
projection.workers.set('runtime-a:1', worker('runtime-a', '1', 1, true));
projection.revisions.set('runtime-a:1', 1);
applyWorkspaceWorkersFrame(projection, {
protocol_version: 1,
frame: 'event',
message: {
event: 'event',
data: {
subscription_id: 'subscription-1',
subject_revision: 2,
payload: {
event: 'worker_upserted',
data: { worker: worker('runtime-a', '1', 2, false) },
},
},
},
});
assertEquals(projection.workers.get('runtime-a:1')?.has_running_internal_workers, false);
assertEquals(projection.revisions.get('runtime-a:1'), 2);
});
web/workspace/src/lib/workspace/sidebar/workers.test.ts
+19
View File
@@ -2,6 +2,7 @@ import {
canOpenWorkerConsole,
canShowWorkerInSidebar,
compareWorkersForSidebar,
sidebarWorkerActivity,
} from "./workers.ts";
import type { Worker } from "./types.ts";
@@ -77,3 +78,21 @@ Deno.test("sidebar workers sort running then idle then stopped", () => {
workers.sort(compareWorkersForSidebar);
assertEquals(workers.map((candidate) => candidate.worker_id).join(","), "2,1,4,3");
});
Deno.test("fatal child stop clears the sidebar SubWorker spinner activity", () => {
const parent = { state: "idle", has_running_internal_workers: true };
assertEquals(sidebarWorkerActivity(parent), "subworker-running");
parent.has_running_internal_workers = false;
assertEquals(sidebarWorkerActivity(parent), "idle");
});
Deno.test("stopped parents do not fall back to the idle indicator", () => {
assertEquals(
sidebarWorkerActivity({
state: "stopped",
has_running_internal_workers: false,
}),
"none",
);
});
web/workspace/src/lib/workspace/sidebar/workers.ts
+19
View File
@@ -1,5 +1,24 @@
import type { Worker } from './types';
export type SidebarWorkerActivity =
| 'worker-running'
| 'subworker-running'
| 'idle'
| 'none';
type WorkerActivitySource = Pick<Worker, 'state'> & {
has_running_internal_workers: boolean;
};
export function sidebarWorkerActivity(
worker: WorkerActivitySource,
): SidebarWorkerActivity {
if (worker.state === 'running') return 'worker-running';
if (worker.has_running_internal_workers) return 'subworker-running';
if (worker.state === 'idle') return 'idle';
return 'none';
}
export function canShowWorkerInSidebar(worker: Worker): boolean {
return worker.implementation.kind !== 'backend_worker_registry';
}