feat: integrate command output streaming

This commit is contained in:
2026-08-21 03:54:47 +09:00
16 changed files with 1614 additions and 77 deletions
+133 -3
View File
@@ -556,6 +556,12 @@ pub enum Event {
Status {
status: WorkerStatus,
},
/// Bounded, provider-owned command telemetry for the live Console. This is
/// intentionally not a history entry and is reconstructed from
/// `Snapshot.in_flight.commands` after reconnect.
Command {
event: CommandEvent,
},
/// Reply to `Method::ListCompletions`. Delivered only to the
/// requesting socket (not broadcast). `entries` is empty when no
/// candidates match or when the requested kind has no resolver
@@ -723,8 +729,79 @@ pub struct RewindSummary {
pub tool_side_effect_warning: bool,
}
/// Unfinished model output included in `Event::Snapshot` for clients that
/// attach while an LLM response is still streaming.
/// Live provider-owned command status. These values are operational Console
/// state only and are never appended to Worker history.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")]
pub enum CommandStatus {
Running,
Completed,
Failed,
TimedOut,
Cancelled,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")]
pub enum CommandStream {
Stdout,
Stderr,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
pub struct CommandStreamSlice {
pub start_offset: u64,
pub end_offset: u64,
pub content: String,
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
pub struct CommandSnapshot {
pub command_id: String,
pub tool_call_id: Option<String>,
pub status: CommandStatus,
pub started_at_ms: u64,
pub observed_at_ms: u64,
pub last_output_at_ms: Option<u64>,
pub stdout: CommandStreamSlice,
pub stderr: CommandStreamSlice,
pub exit_code: Option<i32>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum CommandEvent {
Started {
command_id: String,
tool_call_id: Option<String>,
observed_at_ms: u64,
},
Output {
command_id: String,
stream: CommandStream,
start_offset: u64,
end_offset: u64,
content: String,
observed_at_ms: u64,
},
Terminal {
command_id: String,
status: CommandStatus,
exit_code: Option<i32>,
stdout_end_offset: u64,
stderr_end_offset: u64,
observed_at_ms: u64,
},
}
/// Unfinished model output and active command state included in
/// `Event::Snapshot` for clients that attach while work is still streaming.
///
/// These blocks are presentation state only: they are reconstructed from the
/// active Worker controller and must not be treated as committed assistant
@@ -735,11 +812,13 @@ pub struct RewindSummary {
pub struct InFlightSnapshot {
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub blocks: Vec<InFlightBlock>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub commands: Vec<CommandSnapshot>,
}
impl InFlightSnapshot {
pub fn is_empty(&self) -> bool {
self.blocks.is_empty()
self.blocks.is_empty() && self.commands.is_empty()
}
}
@@ -1384,6 +1463,22 @@ mod tests {
state: InFlightToolCallState::StreamingArgs,
},
],
commands: vec![CommandSnapshot {
command_id: "command-1".into(),
tool_call_id: Some("call_1".into()),
status: CommandStatus::Running,
started_at_ms: 100,
observed_at_ms: 120,
last_output_at_ms: Some(120),
stdout: CommandStreamSlice {
start_offset: 4,
end_offset: 8,
content: "tail".into(),
truncated: true,
},
stderr: CommandStreamSlice::default(),
exit_code: None,
}],
},
internal_workers: Vec::new(),
};
@@ -1453,6 +1548,41 @@ mod tests {
));
}
#[test]
fn event_command_output_roundtrip_preserves_stream_and_offsets() {
let event = Event::Command {
event: CommandEvent::Output {
command_id: "command-1".into(),
stream: CommandStream::Stderr,
start_offset: 8,
end_offset: 12,
content: "warn".into(),
observed_at_ms: 42,
},
};
let json = serde_json::to_string(&event).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed["event"], "command");
assert_eq!(parsed["data"]["event"]["kind"], "output");
assert_eq!(parsed["data"]["event"]["stream"], "stderr");
assert_eq!(parsed["data"]["event"]["start_offset"], 8);
assert_eq!(parsed["data"]["event"]["end_offset"], 12);
assert_eq!(parsed["data"]["event"]["observed_at_ms"], 42);
assert!(matches!(
serde_json::from_str::<Event>(&json).unwrap(),
Event::Command {
event: CommandEvent::Output {
command_id,
stream: CommandStream::Stderr,
start_offset: 8,
end_offset: 12,
content,
observed_at_ms: 42,
}
} if command_id == "command-1" && content == "warn"
));
}
#[test]
fn event_snapshot_legacy_without_status_defaults_to_idle() {
let json = r#"{"event":"snapshot","data":{"entries":[],"greeting":{"worker_name":"test","cwd":"/tmp","provider":"anthropic","model":"claude","scope_summary":"","tools":[]}}}"#;
+8 -2
View File
@@ -3,8 +3,9 @@ use std::path::PathBuf;
use ts_rs::{Config, TS};
use crate::{
Alert, AlertLevel, AlertSource, CompletionEntry, CompletionKind, ErrorCode, Event, Greeting,
InFlightBlock, InFlightSnapshot, InFlightToolCallState, InternalWorkerKind, InternalWorkerRef,
Alert, AlertLevel, AlertSource, CommandEvent, CommandSnapshot, CommandStatus, CommandStream,
CommandStreamSlice, CompletionEntry, CompletionKind, ErrorCode, Event, Greeting, InFlightBlock,
InFlightSnapshot, InFlightToolCallState, InternalWorkerKind, InternalWorkerRef,
InternalWorkerSnapshot, InvokeKind, MemoryWorkerEvent, Method, Permission, RewindSummary,
RewindTarget, RewindTargetId, RunResult, ScopeRule, Segment, TurnResult, WorkerEvent,
WorkerStatus,
@@ -47,6 +48,11 @@ pub fn generated_protocol_types() -> String {
push_decl::<ErrorCode>(&cfg, &mut output);
push_decl::<Permission>(&cfg, &mut output);
push_decl::<InFlightToolCallState>(&cfg, &mut output);
push_decl::<CommandStatus>(&cfg, &mut output);
push_decl::<CommandStream>(&cfg, &mut output);
push_decl::<CommandStreamSlice>(&cfg, &mut output);
push_decl::<CommandSnapshot>(&cfg, &mut output);
push_decl::<CommandEvent>(&cfg, &mut output);
push_decl::<ScopeRule>(&cfg, &mut output);
push_decl::<CompletionEntry>(&cfg, &mut output);
push_decl::<RewindTargetId>(&cfg, &mut output);
+2 -1
View File
@@ -43,7 +43,7 @@ impl Tool for BashTool {
async fn execute(
&self,
input_json: &str,
_ctx: llm_engine::tool::ToolExecutionContext,
ctx: llm_engine::tool::ToolExecutionContext,
) -> Result<ToolOutput, ToolError> {
let params: BashParams = serde_json::from_str(input_json)
.map_err(|error| ToolError::InvalidArgument(format!("invalid Bash input: {error}")))?;
@@ -58,6 +58,7 @@ impl Tool for BashTool {
command: params.command,
timeout_secs,
output_limit: INLINE_BYTE_BUDGET,
tool_call_id: Some(ctx.call_id),
})
.await
.map_err(crate::ToolsError::from)?;
+3
View File
@@ -1328,6 +1328,9 @@ impl App {
self.rewind_refresh_fence = false;
self.set_worker_status(status);
}
// Command telemetry is an operational Web Console surface. The
// TUI continues to render the final Bash ToolResult from history.
Event::Command { .. } => {}
Event::Completions { kind, entries } => {
// Apply only if the popup is still on the same
// (kind, prefix) the request was issued for; an
+14
View File
@@ -16,6 +16,7 @@ use std::sync::Arc;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use tokio::sync::broadcast;
pub use delegation::{
AppliedWorkdirDelegation, ReadOnlyWorkdirSession, WorkdirDelegation,
@@ -192,6 +193,19 @@ pub trait WorkdirSession: std::fmt::Debug + Send + Sync {
request: CommandOutputRequest,
) -> Result<CommandOutput, WorkdirError>;
async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError>;
/// Subscribe to bounded provider-owned command telemetry. Implementations
/// that do not expose live command observation may keep the default.
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<CommandEvent>> {
None
}
/// Return the bounded current command state used to recover from a lagged
/// provider subscription without replaying command output into history.
fn command_snapshot(&self) -> Vec<CommandSnapshot> {
Vec::new()
}
/// Terminal, idempotent release of this Worker-bound operation session.
async fn close(&self) -> Result<(), WorkdirError>;
}
+633 -55
View File
@@ -14,21 +14,22 @@ use std::io::Write as _;
use std::io::{Read as _, Seek as _, SeekFrom};
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::Duration;
use std::sync::{Arc, Mutex as StdMutex};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use async_trait::async_trait;
use manifest::{Permission, Scope, ScopeConfig, ScopeRule, SharedScope};
use sha2::{Digest, Sha256};
use tokio::process::Command;
use tokio::sync::{Mutex, Notify};
use tokio::sync::{Mutex, Notify, broadcast, watch};
use tokio::task::JoinHandle;
use crate::{
CommandHandle, CommandOutput, CommandOutputRequest, CommandRequest, CommandStatus, EditRequest,
EditResult, GlobRequest, GlobResult, GrepRequest, GrepResult, ListRequest, ListResult,
ReadRequest, ReadResult, StatRequest, StatResult, Workdir, WorkdirDelegationPermission,
CommandEvent, CommandHandle, CommandOutput, CommandOutputRequest, CommandRequest,
CommandSnapshot, CommandStatus, CommandStream, CommandStreamSlice, EditRequest, EditResult,
GlobRequest, GlobResult, GrepRequest, GrepResult, ListRequest, ListResult, ReadRequest,
ReadResult, StatRequest, StatResult, Workdir, WorkdirDelegationPermission,
WorkdirDelegationRequest, WorkdirError, WorkdirPath, WorkdirSession,
WorkdirSessionCapabilities, WorkdirSessionCapability, WorkdirSessionHandle, WriteRequest,
WriteResult,
@@ -36,15 +37,172 @@ use crate::{
#[cfg(test)]
use crate::{EntryKind, WriteOutcome};
const COMMAND_EVENT_CHANNEL_CAPACITY: usize = 256;
const COMMAND_EVENT_CHUNK_BYTES: usize = 8 * 1024;
const COMMAND_SNAPSHOT_STREAM_BYTES: usize = 32 * 1024;
fn command_observed_at_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis()
.try_into()
.unwrap_or(u64::MAX)
}
#[derive(Debug)]
enum LocalCommand {
Running {
task: JoinHandle<Result<CommandOutput, WorkdirError>>,
completion: Arc<Notify>,
cancel: watch::Sender<bool>,
},
Completed(CommandOutput),
}
#[derive(Debug, Clone)]
struct CommandTelemetry {
inner: Arc<CommandTelemetryInner>,
}
#[derive(Debug)]
struct CommandTelemetryInner {
snapshots: StdMutex<HashMap<String, CommandSnapshot>>,
events: broadcast::Sender<CommandEvent>,
}
impl CommandTelemetry {
fn new() -> Self {
let (events, _) = broadcast::channel(COMMAND_EVENT_CHANNEL_CAPACITY);
Self {
inner: Arc::new(CommandTelemetryInner {
snapshots: StdMutex::new(HashMap::new()),
events,
}),
}
}
fn subscribe(&self) -> broadcast::Receiver<CommandEvent> {
self.inner.events.subscribe()
}
fn snapshot(&self) -> Vec<CommandSnapshot> {
let mut snapshots = self
.inner
.snapshots
.lock()
.expect("command telemetry mutex poisoned")
.values()
.cloned()
.collect::<Vec<_>>();
snapshots.sort_by(|left, right| left.command_id.cmp(&right.command_id));
snapshots
}
fn started(&self, command_id: &str, tool_call_id: Option<String>) {
let observed_at_ms = command_observed_at_ms();
self.inner
.snapshots
.lock()
.expect("command telemetry mutex poisoned")
.insert(
command_id.to_string(),
CommandSnapshot {
command_id: command_id.to_string(),
tool_call_id: tool_call_id.clone(),
status: CommandStatus::Running,
started_at_ms: observed_at_ms,
observed_at_ms,
last_output_at_ms: None,
stdout: CommandStreamSlice::default(),
stderr: CommandStreamSlice::default(),
exit_code: None,
},
);
let _ = self.inner.events.send(CommandEvent::Started {
command_id: command_id.to_string(),
tool_call_id,
observed_at_ms,
});
}
fn output(&self, command_id: &str, stream: CommandStream, start_offset: u64, bytes: &[u8]) {
if bytes.is_empty() {
return;
}
let end_offset = start_offset.saturating_add(bytes.len() as u64);
let content = String::from_utf8_lossy(bytes).into_owned();
let observed_at_ms = command_observed_at_ms();
if let Some(snapshot) = self
.inner
.snapshots
.lock()
.expect("command telemetry mutex poisoned")
.get_mut(command_id)
{
snapshot.observed_at_ms = observed_at_ms;
snapshot.last_output_at_ms = Some(observed_at_ms);
let target = match stream {
CommandStream::Stdout => &mut snapshot.stdout,
CommandStream::Stderr => &mut snapshot.stderr,
};
target.end_offset = end_offset;
target.content.push_str(&content);
if target.content.len() > COMMAND_SNAPSHOT_STREAM_BYTES {
let mut cut = target.content.len() - COMMAND_SNAPSHOT_STREAM_BYTES;
while cut < target.content.len() && !target.content.is_char_boundary(cut) {
cut += 1;
}
target.content.drain(..cut);
target.start_offset = end_offset.saturating_sub(target.content.len() as u64);
target.truncated = true;
}
}
let _ = self.inner.events.send(CommandEvent::Output {
command_id: command_id.to_string(),
stream,
start_offset,
end_offset,
content,
observed_at_ms,
});
}
fn terminal(&self, command_id: &str, status: CommandStatus, exit_code: Option<i32>) {
let observed_at_ms = command_observed_at_ms();
let (stdout_end_offset, stderr_end_offset) = if let Some(snapshot) = self
.inner
.snapshots
.lock()
.expect("command telemetry mutex poisoned")
.get_mut(command_id)
{
snapshot.status = status;
snapshot.exit_code = exit_code;
snapshot.observed_at_ms = observed_at_ms;
(snapshot.stdout.end_offset, snapshot.stderr.end_offset)
} else {
(0, 0)
};
let _ = self.inner.events.send(CommandEvent::Terminal {
command_id: command_id.to_string(),
status,
exit_code,
stdout_end_offset,
stderr_end_offset,
observed_at_ms,
});
}
fn remove(&self, command_id: &str) {
self.inner
.snapshots
.lock()
.expect("command telemetry mutex poisoned")
.remove(command_id);
}
}
#[derive(Debug)]
struct ScopeAccess(Arc<Scope>);
@@ -69,6 +227,7 @@ struct LocalWorkdirSessionInner {
close_lock: Mutex<()>,
next_command_id: AtomicU64,
commands: Mutex<HashMap<String, LocalCommand>>,
command_telemetry: CommandTelemetry,
}
impl Drop for LocalWorkdirSessionInner {
@@ -171,6 +330,7 @@ impl LocalWorkdirSession {
close_lock: Mutex::new(()),
next_command_id: AtomicU64::new(1),
commands: Mutex::new(HashMap::new()),
command_telemetry: CommandTelemetry::new(),
}),
}
}
@@ -502,23 +662,35 @@ impl WorkdirSession for LocalWorkdirSession {
async fn start_command(&self, request: CommandRequest) -> Result<CommandHandle, WorkdirError> {
self.ensure_capability(WorkdirSessionCapability::Command)?;
self.ensure_open()?;
let id = self.inner.next_command_id.fetch_add(1, Ordering::Relaxed);
let handle = CommandHandle(format!("command-{id}"));
let cwd = self.inner.cwd.clone();
let completion = Arc::new(Notify::new());
let task_completion = Arc::clone(&completion);
let command_id = handle.0.clone();
let telemetry = self.inner.command_telemetry.clone();
let (cancel, cancel_rx) = watch::channel(false);
let task = tokio::spawn(async move {
let output = run_command(cwd, request).await;
let output = run_command(cwd, request, command_id, telemetry, cancel_rx).await;
task_completion.notify_one();
output
});
let mut commands = self.inner.commands.lock().await;
if let Err(error) = self.ensure_open() {
let _ = cancel.send(true);
task.abort();
completion.notify_one();
return Err(error);
}
commands.insert(handle.0.clone(), LocalCommand::Running { task, completion });
commands.insert(
handle.0.clone(),
LocalCommand::Running {
task,
completion,
cancel,
},
);
Ok(handle)
}
@@ -530,7 +702,13 @@ impl WorkdirSession for LocalWorkdirSession {
.ok_or_else(|| WorkdirError::UnknownCommand(handle.0.clone()))?;
Ok(match command {
LocalCommand::Running { task, .. } if !task.is_finished() => CommandStatus::Running,
LocalCommand::Running { .. } => CommandStatus::Completed,
LocalCommand::Running { .. } => self
.inner
.command_telemetry
.snapshot()
.into_iter()
.find(|snapshot| snapshot.command_id == handle.0)
.map_or(CommandStatus::Completed, |snapshot| snapshot.status),
LocalCommand::Completed(output) => output.status,
})
}
@@ -584,36 +762,75 @@ impl WorkdirSession for LocalWorkdirSession {
if !self.inner.closed.load(Ordering::Acquire) {
commands.insert(request.handle.0, LocalCommand::Completed(output));
}
} else {
self.inner.command_telemetry.remove(&request.handle.0);
}
Ok(page)
}
async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError> {
self.ensure_capability(WorkdirSessionCapability::Command)?;
let command = self
.inner
.commands
.lock()
.await
.remove(&handle.0)
.ok_or_else(|| WorkdirError::UnknownCommand(handle.0))?;
if let LocalCommand::Running { task, completion } = command {
task.abort();
completion.notify_one();
let cancel = {
let commands = self.inner.commands.lock().await;
let command = commands
.get(&handle.0)
.ok_or_else(|| WorkdirError::UnknownCommand(handle.0.clone()))?;
match command {
LocalCommand::Running { task, cancel, .. } if !task.is_finished() => {
Some(cancel.clone())
}
_ => None,
}
};
if let Some(cancel) = cancel {
let _ = cancel.send(true);
}
Ok(())
}
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<CommandEvent>> {
self.inner
.capabilities
.supports(WorkdirSessionCapability::Command)
.then(|| self.inner.command_telemetry.subscribe())
}
fn command_snapshot(&self) -> Vec<CommandSnapshot> {
if self
.inner
.capabilities
.supports(WorkdirSessionCapability::Command)
{
self.inner.command_telemetry.snapshot()
} else {
Vec::new()
}
}
async fn close(&self) -> Result<(), WorkdirError> {
let _close_guard = self.inner.close_lock.lock().await;
if self.inner.closed.swap(true, Ordering::AcqRel) {
return Ok(());
}
let mut commands = self.inner.commands.lock().await;
for (_, command) in commands.drain() {
if let LocalCommand::Running { task, completion } = command {
task.abort();
completion.notify_one();
let commands = {
let mut commands = self.inner.commands.lock().await;
commands
.drain()
.map(|(_, command)| command)
.collect::<Vec<_>>()
};
for command in commands {
match command {
LocalCommand::Running {
task,
completion,
cancel,
} => {
let _ = cancel.send(true);
let _ = task.await;
completion.notify_one();
}
LocalCommand::Completed(_) => {}
}
}
Ok(())
@@ -680,7 +897,13 @@ fn sanitize_error(error: WorkdirError, logical: &WorkdirPath) -> WorkdirError {
}
}
async fn run_command(cwd: PathBuf, request: CommandRequest) -> Result<CommandOutput, WorkdirError> {
async fn run_command(
cwd: PathBuf,
request: CommandRequest,
command_id: String,
telemetry: CommandTelemetry,
mut cancel: watch::Receiver<bool>,
) -> Result<CommandOutput, WorkdirError> {
let stdout = tempfile::NamedTempFile::new().map_err(|error| WorkdirError::io(&cwd, error))?;
let stderr = tempfile::NamedTempFile::new().map_err(|error| WorkdirError::io(&cwd, error))?;
let stdout_path = stdout.into_temp_path();
@@ -690,7 +913,8 @@ async fn run_command(cwd: PathBuf, request: CommandRequest) -> Result<CommandOut
let stderr_file = std::fs::File::create(&stderr_path)
.map_err(|error| WorkdirError::io(&stderr_path, error))?;
let mut child = Command::new("bash")
telemetry.started(&command_id, request.tool_call_id.clone());
let mut child = match Command::new("bash")
.arg("-c")
.arg(&request.command)
.current_dir(&cwd)
@@ -699,45 +923,188 @@ async fn run_command(cwd: PathBuf, request: CommandRequest) -> Result<CommandOut
.stderr(Stdio::from(stderr_file))
.kill_on_drop(true)
.spawn()
.map_err(|error| WorkdirError::io(&cwd, error))?;
let timed_out = match tokio::time::timeout(
Duration::from_secs(request.timeout_secs.max(1)),
child.wait(),
)
.await
{
Ok(result) => {
let status = result.map_err(|error| WorkdirError::io(&cwd, error))?;
let (content, truncated) =
read_command_output_files(&stdout_path, &stderr_path, request.output_limit.max(1))?;
return Ok(CommandOutput {
status: CommandStatus::Completed,
exit_code: status.code(),
timed_out: false,
content,
next_cursor: None,
truncated,
});
}
Err(_) => {
let _ = child.kill().await;
true
Ok(child) => child,
Err(error) => {
telemetry.terminal(&command_id, CommandStatus::Failed, None);
return Err(WorkdirError::io(&cwd, error));
}
};
let mut stdout_reader =
std::fs::File::open(&stdout_path).map_err(|error| WorkdirError::io(&stdout_path, error))?;
let mut stderr_reader =
std::fs::File::open(&stderr_path).map_err(|error| WorkdirError::io(&stderr_path, error))?;
let mut stdout_decoder = CommandOutputDecoder::default();
let mut stderr_decoder = CommandOutputDecoder::default();
let mut timeout = Box::pin(tokio::time::sleep(Duration::from_secs(
request.timeout_secs.max(1),
)));
let mut interval = tokio::time::interval(Duration::from_millis(50));
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
interval.tick().await;
let (status, exit_code) = loop {
tokio::select! {
exit = child.wait() => {
let exit = exit.map_err(|error| WorkdirError::io(&cwd, error))?;
break (
if exit.success() { CommandStatus::Completed } else { CommandStatus::Failed },
exit.code(),
);
}
_ = &mut timeout => {
let _ = child.start_kill();
let exit_code = child.wait().await.ok().and_then(|status| status.code());
break (CommandStatus::TimedOut, exit_code);
}
changed = cancel.changed() => {
if changed.is_err() || *cancel.borrow() {
let _ = child.start_kill();
let exit_code = child.wait().await.ok().and_then(|status| status.code());
break (CommandStatus::Cancelled, exit_code);
}
}
_ = interval.tick() => {
publish_available_output(
&mut stdout_reader,
&mut stdout_decoder,
&telemetry,
&command_id,
CommandStream::Stdout,
&stdout_path,
false,
)?;
publish_available_output(
&mut stderr_reader,
&mut stderr_decoder,
&telemetry,
&command_id,
CommandStream::Stderr,
&stderr_path,
false,
)?;
}
}
};
publish_available_output(
&mut stdout_reader,
&mut stdout_decoder,
&telemetry,
&command_id,
CommandStream::Stdout,
&stdout_path,
true,
)?;
publish_available_output(
&mut stderr_reader,
&mut stderr_decoder,
&telemetry,
&command_id,
CommandStream::Stderr,
&stderr_path,
true,
)?;
telemetry.terminal(&command_id, status, exit_code);
let (content, truncated) =
read_command_output_files(&stdout_path, &stderr_path, request.output_limit.max(1))?;
Ok(CommandOutput {
status: CommandStatus::Failed,
exit_code: None,
timed_out,
status,
exit_code,
timed_out: status == CommandStatus::TimedOut,
content,
next_cursor: None,
truncated,
})
}
#[derive(Debug, Default)]
struct CommandOutputDecoder {
read_offset: u64,
emitted_offset: u64,
pending: Vec<u8>,
}
fn publish_available_output(
file: &mut std::fs::File,
decoder: &mut CommandOutputDecoder,
telemetry: &CommandTelemetry,
command_id: &str,
stream: CommandStream,
path: &Path,
flush: bool,
) -> Result<(), WorkdirError> {
file.seek(SeekFrom::Start(decoder.read_offset))
.map_err(|error| WorkdirError::io(path, error))?;
loop {
let mut buffer = vec![0; COMMAND_EVENT_CHUNK_BYTES];
let read = file
.read(&mut buffer)
.map_err(|error| WorkdirError::io(path, error))?;
if read == 0 {
publish_decoded_output(decoder, telemetry, command_id, stream, flush);
return Ok(());
}
decoder.pending.extend_from_slice(&buffer[..read]);
decoder.read_offset = decoder.read_offset.saturating_add(read as u64);
publish_decoded_output(decoder, telemetry, command_id, stream, false);
if read < COMMAND_EVENT_CHUNK_BYTES {
if flush {
publish_decoded_output(decoder, telemetry, command_id, stream, true);
}
return Ok(());
}
}
}
fn publish_decoded_output(
decoder: &mut CommandOutputDecoder,
telemetry: &CommandTelemetry,
command_id: &str,
stream: CommandStream,
flush: bool,
) {
let prefix_len = if flush {
decoder.pending.len()
} else {
stable_utf8_prefix_len(&decoder.pending)
};
if prefix_len == 0 {
return;
}
telemetry.output(
command_id,
stream,
decoder.emitted_offset,
&decoder.pending[..prefix_len],
);
decoder.emitted_offset = decoder.emitted_offset.saturating_add(prefix_len as u64);
decoder.pending.drain(..prefix_len);
}
/// Return the byte prefix that can be decoded now without replacing a valid
/// UTF-8 scalar whose remaining bytes may arrive in a later file read. Definite
/// invalid sequences remain in the prefix and are rendered lossily, preserving
/// the existing arbitrary-byte output behavior.
fn stable_utf8_prefix_len(bytes: &[u8]) -> usize {
let mut inspected = 0;
while inspected < bytes.len() {
match std::str::from_utf8(&bytes[inspected..]) {
Ok(_) => return bytes.len(),
Err(error) => {
inspected += error.valid_up_to();
match error.error_len() {
Some(invalid_len) => inspected += invalid_len,
None => return inspected,
}
}
}
}
inspected
}
fn read_command_output_files(
stdout_path: &Path,
stderr_path: &Path,
@@ -1024,6 +1391,7 @@ mod tests {
command: "sleep 30".to_owned(),
timeout_secs: 60,
output_limit: 1024,
tool_call_id: None,
},
)
.await
@@ -1549,6 +1917,7 @@ mod tests {
command: "pwd && printf provider-command".into(),
timeout_secs: 5,
output_limit: 4096,
tool_call_id: None,
},
)
.await
@@ -1583,6 +1952,7 @@ mod tests {
command: "printf 'aéz'".into(),
timeout_secs: 5,
output_limit: 1024,
tool_call_id: None,
},
)
.await
@@ -1620,6 +1990,213 @@ mod tests {
));
}
#[test]
fn command_output_decoder_preserves_utf8_split_across_file_reads() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("command.out");
let mut first_write = vec![b'a'; COMMAND_EVENT_CHUNK_BYTES - 1];
first_write.push(0xe2);
std::fs::write(&path, first_write).unwrap();
let telemetry = CommandTelemetry::new();
let mut events = telemetry.subscribe();
telemetry.started("command-utf8", None);
let mut decoder = CommandOutputDecoder::default();
let mut reader = std::fs::File::open(&path).unwrap();
publish_available_output(
&mut reader,
&mut decoder,
&telemetry,
"command-utf8",
CommandStream::Stdout,
&path,
false,
)
.unwrap();
assert_eq!(decoder.pending, vec![0xe2]);
let mut writer = std::fs::OpenOptions::new()
.append(true)
.open(&path)
.unwrap();
writer.write_all(&[0x82, 0xac]).unwrap();
writer.flush().unwrap();
publish_available_output(
&mut reader,
&mut decoder,
&telemetry,
"command-utf8",
CommandStream::Stdout,
&path,
false,
)
.unwrap();
let output = std::iter::from_fn(|| events.try_recv().ok())
.filter_map(|event| match event {
CommandEvent::Output {
stream: CommandStream::Stdout,
content,
..
} => Some(content),
_ => None,
})
.collect::<String>();
assert_eq!(output.len(), COMMAND_EVENT_CHUNK_BYTES - 1 + "".len());
assert!(output.ends_with('€'));
assert!(!output.contains('\u{fffd}'));
assert!(decoder.pending.is_empty());
}
#[tokio::test]
async fn provider_streams_bounded_command_lifecycle_and_distinct_output() {
let dir = TempDir::new().unwrap();
let workdir = make_fs(&dir);
let mut events = WorkdirSession::subscribe_command_events(&workdir)
.expect("local command observation must be available");
let handle = WorkdirSession::start_command(
&workdir,
CommandRequest {
command: "printf ready; printf warning >&2; sleep 0.2; printf done".into(),
timeout_secs: 5,
output_limit: 1024,
tool_call_id: Some("tool-7".into()),
},
)
.await
.unwrap();
let mut stdout = String::new();
let mut stdout_chunks = 0;
let mut stderr = String::new();
let mut terminal = None;
while terminal.is_none() {
let event = tokio::time::timeout(Duration::from_secs(2), events.recv())
.await
.expect("command telemetry should not stall")
.unwrap();
match event {
CommandEvent::Started {
command_id,
tool_call_id,
..
} => {
assert_eq!(command_id, handle.0);
assert_eq!(tool_call_id.as_deref(), Some("tool-7"));
}
CommandEvent::Output {
command_id,
stream,
content,
..
} => {
assert_eq!(command_id, handle.0);
match stream {
CommandStream::Stdout => {
stdout_chunks += 1;
stdout.push_str(&content);
}
CommandStream::Stderr => stderr.push_str(&content),
}
}
CommandEvent::Terminal {
command_id,
status,
exit_code,
stdout_end_offset,
stderr_end_offset,
observed_at_ms,
} => {
assert_eq!(command_id, handle.0);
terminal = Some((
status,
exit_code,
stdout_end_offset,
stderr_end_offset,
observed_at_ms,
));
}
}
}
let (status, exit_code, stdout_end_offset, stderr_end_offset, observed_at_ms) =
terminal.unwrap();
assert_eq!(status, CommandStatus::Completed);
assert_eq!(exit_code, Some(0));
assert_eq!(stdout_end_offset, "readydone".len() as u64);
assert_eq!(stderr_end_offset, "warning".len() as u64);
assert!(observed_at_ms > 0);
assert!(
stdout_chunks >= 2,
"long-running output should stream incrementally"
);
assert_eq!(stdout, "readydone");
assert_eq!(stderr, "warning");
let snapshot = WorkdirSession::command_snapshot(&workdir);
assert_eq!(snapshot.len(), 1);
assert_eq!(snapshot[0].status, CommandStatus::Completed);
assert_eq!(snapshot[0].stdout.content, "readydone");
assert_eq!(snapshot[0].stderr.content, "warning");
let output = WorkdirSession::command_output(
&workdir,
CommandOutputRequest {
handle,
cursor: 0,
limit: 1024,
wait: true,
},
)
.await
.unwrap();
assert_eq!(output.status, CommandStatus::Completed);
assert!(WorkdirSession::command_snapshot(&workdir).is_empty());
}
#[tokio::test]
async fn provider_distinguishes_timed_out_terminal_state() {
let dir = TempDir::new().unwrap();
let workdir = make_fs(&dir);
let mut events = WorkdirSession::subscribe_command_events(&workdir).unwrap();
let handle = WorkdirSession::start_command(
&workdir,
CommandRequest {
command: "sleep 30".into(),
timeout_secs: 1,
output_limit: 1024,
tool_call_id: None,
},
)
.await
.unwrap();
let output = WorkdirSession::command_output(
&workdir,
CommandOutputRequest {
handle: handle.clone(),
cursor: 0,
limit: 1024,
wait: true,
},
)
.await
.unwrap();
assert_eq!(output.status, CommandStatus::TimedOut);
assert!(output.timed_out);
let mut terminal = None;
while let Ok(event) = events.try_recv() {
if let CommandEvent::Terminal {
command_id,
status,
exit_code,
..
} = event
{
terminal = Some((command_id, status, exit_code));
}
}
assert_eq!(terminal, Some((handle.0, CommandStatus::TimedOut, None)));
}
#[tokio::test]
async fn provider_cancels_active_command() {
let dir = TempDir::new().unwrap();
@@ -1630,6 +2207,7 @@ mod tests {
command: "sleep 30".into(),
timeout_secs: 60,
output_limit: 1024,
tool_call_id: None,
},
)
.await
@@ -1658,12 +2236,12 @@ mod tests {
WorkdirSession::cancel_command(&workdir, handle.clone())
.await
.unwrap();
let waiter_error = tokio::time::timeout(Duration::from_secs(1), waiter)
let output = tokio::time::timeout(Duration::from_secs(1), waiter)
.await
.expect("cancel should wake command output waiters")
.unwrap()
.unwrap_err();
assert!(matches!(waiter_error, WorkdirError::UnknownCommand(_)));
.unwrap();
assert_eq!(output.status, CommandStatus::Cancelled);
assert!(matches!(
WorkdirSession::command_status(&workdir, handle).await,
Err(WorkdirError::UnknownCommand(_))
+61 -1
View File
@@ -9,6 +9,11 @@ pub struct CommandRequest {
pub command: String,
pub timeout_secs: u64,
pub output_limit: usize,
/// Optional caller-owned correlation id. Bash supplies its tool-call id so
/// user-facing command telemetry can update the corresponding Console row
/// without exposing provider/session handles.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tool_call_id: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -24,8 +29,63 @@ pub struct CommandOutputRequest {
pub enum CommandStatus {
Running,
Completed,
Cancelled,
Failed,
TimedOut,
Cancelled,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CommandStream {
Stdout,
Stderr,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct CommandStreamSlice {
pub start_offset: u64,
pub end_offset: u64,
pub content: String,
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CommandSnapshot {
pub command_id: String,
pub tool_call_id: Option<String>,
pub status: CommandStatus,
pub started_at_ms: u64,
pub observed_at_ms: u64,
pub last_output_at_ms: Option<u64>,
pub stdout: CommandStreamSlice,
pub stderr: CommandStreamSlice,
pub exit_code: Option<i32>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum CommandEvent {
Started {
command_id: String,
tool_call_id: Option<String>,
observed_at_ms: u64,
},
Output {
command_id: String,
stream: CommandStream,
start_offset: u64,
end_offset: u64,
content: String,
observed_at_ms: u64,
},
Terminal {
command_id: String,
status: CommandStatus,
exit_code: Option<i32>,
stdout_end_offset: u64,
stderr_end_offset: u64,
observed_at_ms: u64,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+8 -2
View File
@@ -1430,7 +1430,10 @@ impl Runtime {
context_tokens: 0,
},
status: protocol::WorkerStatus::Idle,
in_flight: protocol::InFlightSnapshot { blocks: Vec::new() },
in_flight: protocol::InFlightSnapshot {
blocks: Vec::new(),
commands: Vec::new(),
},
internal_workers: Vec::new(),
})
}
@@ -3765,7 +3768,10 @@ mod tests {
context_tokens: 64,
},
status: protocol::WorkerStatus::Running,
in_flight: protocol::InFlightSnapshot { blocks: Vec::new() },
in_flight: protocol::InFlightSnapshot {
blocks: Vec::new(),
commands: Vec::new(),
},
internal_workers: Vec::new(),
},
);
+123 -2
View File
@@ -28,8 +28,14 @@ use crate::worker::{
WorkerRunResult,
};
use protocol::{
AlertLevel, AlertSource, ErrorCode, Event, Method, RewindTargetId, RunResult, Segment,
TurnResult, WorkerStatus,
AlertLevel, AlertSource, CommandEvent as ProtocolCommandEvent,
CommandSnapshot as ProtocolCommandSnapshot, CommandStatus as ProtocolCommandStatus,
CommandStream as ProtocolCommandStream, CommandStreamSlice as ProtocolCommandStreamSlice,
ErrorCode, Event, Method, RewindTargetId, RunResult, Segment, TurnResult, WorkerStatus,
};
use workdir::{
CommandEvent as WorkdirCommandEvent, CommandSnapshot as WorkdirCommandSnapshot,
CommandStatus as WorkdirCommandStatus, CommandStream as WorkdirCommandStream, WorkdirSession,
};
// ---------------------------------------------------------------------------
@@ -424,6 +430,9 @@ impl WorkerController {
Some(method_tx.downgrade()),
)
.await?;
if let Some(session) = fs_for_view.as_ref() {
wire_workdir_command_events(session, &in_flight);
}
// Intake role Workers self-terminate only after a successful
// TicketIntakeReady turn has fully settled back to Idle. The request
@@ -498,6 +507,118 @@ impl WorkerController {
}
}
pub(crate) fn wire_workdir_command_events(
session: &Arc<dyn WorkdirSession>,
in_flight: &InFlightEvents,
) {
in_flight.replace_command_snapshot(
session
.command_snapshot()
.into_iter()
.map(protocol_command_snapshot)
.collect(),
);
let Some(mut events) = session.subscribe_command_events() else {
return;
};
let in_flight = in_flight.clone();
tokio::spawn(async move {
loop {
match events.recv().await {
Ok(event) => in_flight.publish_command_event(protocol_command_event(event)),
Err(broadcast::error::RecvError::Lagged(_)) => {
// Never retain stale command output after a provider-local
// observer lag. The next chunk reconstructs a bounded tail
// with its absolute offset and marks the gap truncated.
in_flight.replace_command_snapshot(Vec::new());
}
Err(broadcast::error::RecvError::Closed) => break,
}
}
});
}
fn protocol_command_snapshot(snapshot: WorkdirCommandSnapshot) -> ProtocolCommandSnapshot {
ProtocolCommandSnapshot {
command_id: snapshot.command_id,
tool_call_id: snapshot.tool_call_id,
status: protocol_command_status(snapshot.status),
started_at_ms: snapshot.started_at_ms,
observed_at_ms: snapshot.observed_at_ms,
last_output_at_ms: snapshot.last_output_at_ms,
stdout: ProtocolCommandStreamSlice {
start_offset: snapshot.stdout.start_offset,
end_offset: snapshot.stdout.end_offset,
content: snapshot.stdout.content,
truncated: snapshot.stdout.truncated,
},
stderr: ProtocolCommandStreamSlice {
start_offset: snapshot.stderr.start_offset,
end_offset: snapshot.stderr.end_offset,
content: snapshot.stderr.content,
truncated: snapshot.stderr.truncated,
},
exit_code: snapshot.exit_code,
}
}
fn protocol_command_event(event: WorkdirCommandEvent) -> ProtocolCommandEvent {
match event {
WorkdirCommandEvent::Started {
command_id,
tool_call_id,
observed_at_ms,
} => ProtocolCommandEvent::Started {
command_id,
tool_call_id,
observed_at_ms,
},
WorkdirCommandEvent::Output {
command_id,
stream,
start_offset,
end_offset,
content,
observed_at_ms,
} => ProtocolCommandEvent::Output {
command_id,
stream: match stream {
WorkdirCommandStream::Stdout => ProtocolCommandStream::Stdout,
WorkdirCommandStream::Stderr => ProtocolCommandStream::Stderr,
},
start_offset,
end_offset,
content,
observed_at_ms,
},
WorkdirCommandEvent::Terminal {
command_id,
status,
exit_code,
stdout_end_offset,
stderr_end_offset,
observed_at_ms,
} => ProtocolCommandEvent::Terminal {
command_id,
status: protocol_command_status(status),
exit_code,
stdout_end_offset,
stderr_end_offset,
observed_at_ms,
},
}
}
fn protocol_command_status(status: WorkdirCommandStatus) -> ProtocolCommandStatus {
match status {
WorkdirCommandStatus::Running => ProtocolCommandStatus::Running,
WorkdirCommandStatus::Completed => ProtocolCommandStatus::Completed,
WorkdirCommandStatus::Failed => ProtocolCommandStatus::Failed,
WorkdirCommandStatus::TimedOut => ProtocolCommandStatus::TimedOut,
WorkdirCommandStatus::Cancelled => ProtocolCommandStatus::Cancelled,
}
}
/// Wire the per-event broadcast bridges on the Worker's Engine. Each callback
/// re-publishes a worker-level signal as a `protocol::Event` on `event_tx`
/// so subscribers (TUI, socket clients) get a single typed stream.
+157 -2
View File
@@ -1,9 +1,14 @@
use std::sync::{Arc, Mutex, MutexGuard};
use protocol::{Event, InFlightBlock, InFlightSnapshot, InFlightToolCallState};
use protocol::{
CommandEvent, CommandSnapshot, CommandStatus, CommandStream, CommandStreamSlice, Event,
InFlightBlock, InFlightSnapshot, InFlightToolCallState,
};
use session_store::{LoggedContentPart, LoggedItem};
use tokio::sync::broadcast;
const COMMAND_SNAPSHOT_STREAM_BYTES: usize = 32 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct InFlightBlockId(u64);
@@ -17,6 +22,7 @@ pub struct InFlightEvents {
pub(crate) struct InFlightInner {
next_block_id: u64,
blocks: Vec<TrackedBlock>,
commands: Vec<CommandSnapshot>,
}
#[derive(Debug, Clone)]
@@ -46,6 +52,7 @@ impl InFlightEvents {
inner: Arc::new(Mutex::new(InFlightInner {
next_block_id: 1,
blocks: Vec::new(),
commands: Vec::new(),
})),
event_tx,
}
@@ -201,6 +208,15 @@ impl InFlightEvents {
f()
}
pub(crate) fn publish_command_event(&self, event: CommandEvent) {
self.lock().apply_command_event(&event);
let _ = self.event_tx.send(Event::Command { event });
}
pub(crate) fn replace_command_snapshot(&self, commands: Vec<CommandSnapshot>) {
self.lock().commands = commands;
}
pub(crate) fn clear(&self) {
let mut inner = self.lock();
inner.clear();
@@ -224,6 +240,92 @@ impl InFlightInner {
.find(|block| block.block_id() == block_id)
}
fn apply_command_event(&mut self, event: &CommandEvent) {
match event {
CommandEvent::Started {
command_id,
tool_call_id,
observed_at_ms,
} => {
self.commands
.retain(|command| command.command_id != *command_id);
self.commands.push(CommandSnapshot {
command_id: command_id.clone(),
tool_call_id: tool_call_id.clone(),
status: CommandStatus::Running,
started_at_ms: *observed_at_ms,
observed_at_ms: *observed_at_ms,
last_output_at_ms: None,
stdout: CommandStreamSlice::default(),
stderr: CommandStreamSlice::default(),
exit_code: None,
});
}
CommandEvent::Output {
command_id,
stream,
start_offset,
end_offset,
content,
observed_at_ms,
} => {
let command = match self
.commands
.iter_mut()
.find(|command| command.command_id == *command_id)
{
Some(command) => command,
None => {
self.commands.push(CommandSnapshot {
command_id: command_id.clone(),
tool_call_id: None,
status: CommandStatus::Running,
started_at_ms: *observed_at_ms,
observed_at_ms: *observed_at_ms,
last_output_at_ms: Some(*observed_at_ms),
stdout: CommandStreamSlice::default(),
stderr: CommandStreamSlice::default(),
exit_code: None,
});
self.commands.last_mut().expect("command was inserted")
}
};
command.observed_at_ms = *observed_at_ms;
command.last_output_at_ms = Some(*observed_at_ms);
let target = match stream {
CommandStream::Stdout => &mut command.stdout,
CommandStream::Stderr => &mut command.stderr,
};
if target.end_offset != *start_offset {
target.content.clear();
target.start_offset = *start_offset;
target.truncated = *start_offset > 0;
}
target.content.push_str(content);
target.end_offset = *end_offset;
if target.content.len() > COMMAND_SNAPSHOT_STREAM_BYTES {
let mut cut = target.content.len() - COMMAND_SNAPSHOT_STREAM_BYTES;
while cut < target.content.len() && !target.content.is_char_boundary(cut) {
cut += 1;
}
target.content.drain(..cut);
target.start_offset = target
.end_offset
.saturating_sub(target.content.len() as u64);
target.truncated = true;
}
}
CommandEvent::Terminal { command_id, .. } => {
// Terminal state is delivered as a live protocol event. It is
// no longer in-flight snapshot state, and removing it here
// also prevents queued output from an aborted turn from
// surviving the subsequent terminal event after `clear()`.
self.commands
.retain(|command| command.command_id != *command_id);
}
}
}
fn clear_for_committed_item(&mut self, item: &LoggedItem) {
match item {
LoggedItem::Message { role, content }
@@ -273,14 +375,16 @@ impl InFlightInner {
.iter()
.filter_map(TrackedBlock::to_snapshot_block)
.collect(),
commands: self.commands.clone(),
}
}
fn clear(&mut self) -> bool {
if self.blocks.is_empty() {
if self.blocks.is_empty() && self.commands.is_empty() {
false
} else {
self.blocks.clear();
self.commands.clear();
true
}
}
@@ -583,6 +687,57 @@ mod tests {
);
}
#[test]
fn command_events_are_bounded_and_recoverable_from_snapshot() {
let (event_tx, _) = broadcast::channel(16);
let mut rx = event_tx.subscribe();
let in_flight = InFlightEvents::new(event_tx);
in_flight.publish_command_event(CommandEvent::Started {
command_id: "command-1".into(),
tool_call_id: Some("tool-1".into()),
observed_at_ms: 100,
});
in_flight.publish_command_event(CommandEvent::Output {
command_id: "command-1".into(),
stream: CommandStream::Stdout,
start_offset: 0,
end_offset: 5,
content: "ready".into(),
observed_at_ms: 110,
});
let guard = in_flight.snapshot_guard();
let snapshot = snapshot_from_guard(&guard);
assert_eq!(snapshot.commands.len(), 1);
assert_eq!(snapshot.commands[0].tool_call_id.as_deref(), Some("tool-1"));
assert_eq!(snapshot.commands[0].stdout.content, "ready");
assert_eq!(snapshot.commands[0].status, CommandStatus::Running);
drop(guard);
assert!(matches!(
rx.try_recv().unwrap(),
Event::Command {
event: CommandEvent::Started { .. }
}
));
assert!(matches!(
rx.try_recv().unwrap(),
Event::Command {
event: CommandEvent::Output { .. }
}
));
in_flight.publish_command_event(CommandEvent::Terminal {
command_id: "command-1".into(),
status: CommandStatus::TimedOut,
exit_code: None,
stdout_end_offset: 5,
stderr_end_offset: 0,
observed_at_ms: 200,
});
let guard = in_flight.snapshot_guard();
assert!(snapshot_from_guard(&guard).commands.is_empty());
}
#[test]
fn clear_discards_uncommitted_blocks_without_protocol_event() {
let (event_tx, _) = broadcast::channel(16);
+4 -1
View File
@@ -17,7 +17,7 @@ use session_store::{LogEntry, SegmentId, SessionId, Store, StoreError, TraceEntr
use tokio::sync::broadcast;
use uuid::Uuid;
use crate::controller::wire_event_bridges_on_engine;
use crate::controller::{wire_event_bridges_on_engine, wire_workdir_command_events};
use crate::feature::FeatureRegistryBuilder;
use crate::in_flight::{InFlightEvents, snapshot_from_guard};
use crate::ipc::alerter::Alerter;
@@ -576,6 +576,9 @@ pub(crate) async fn prepare_internal_worker_session(
spawn_internal_log_event_bridge(sink.clone(), event_tx.clone());
let alerter = Alerter::new(event_tx.clone());
let in_flight = InFlightEvents::new(event_tx.clone());
if let Some(session) = worker.workdir_session() {
wire_workdir_command_events(session, &in_flight);
}
let actor_in_flight = in_flight.clone();
worker.attach_alerter(alerter.clone());
worker.attach_event_tx(event_tx.clone());
+108 -2
View File
@@ -12,8 +12,8 @@ use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput};
use session_store::{CombinedStore, FsWorkerStore};
use session_store::{FsStore, LogEntry};
use workdir::{
CommandRequest, LocalWorkdirSession, Workdir, WorkdirError, WorkdirSessionCapabilities,
WorkdirSessionHandle,
CommandOutputRequest, CommandRequest, LocalWorkdirSession, Workdir, WorkdirError,
WorkdirSessionCapabilities, WorkdirSessionHandle,
};
use worker::{
@@ -232,6 +232,7 @@ async fn shutdown_closes_bound_workdir_session() {
command: "sleep 30".to_owned(),
timeout_secs: 60,
output_limit: 1024,
tool_call_id: None,
})
.await
.unwrap();
@@ -253,6 +254,110 @@ async fn shutdown_closes_bound_workdir_session() {
));
}
#[tokio::test]
async fn controller_projects_workdir_command_events_and_snapshot_state() {
let (mut worker, pwd) = make_worker_with_pwd(MockClient::new(simple_text_events())).await;
let session: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::materialized_bound(
Workdir::new("controller-command-observation-workdir"),
pwd.clone(),
pwd,
worker.scope().clone(),
WorkdirSessionCapabilities::ALL,
));
worker.bind_workdir_session(Some(Arc::clone(&session)));
let handle = spawn_controller(worker).await;
let mut events = handle.subscribe();
let command = session
.start_command(CommandRequest {
command: "printf ready; sleep 0.3; printf done".to_owned(),
timeout_secs: 5,
output_limit: 1024,
tool_call_id: Some("tool-command-1".into()),
})
.await
.unwrap();
let mut saw_started = false;
let mut saw_output = false;
while !saw_output {
let event = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
.await
.expect("command event should arrive")
.unwrap();
match event {
Event::Command {
event:
protocol::CommandEvent::Started {
command_id,
tool_call_id,
..
},
} => {
assert_eq!(command_id, command.0);
assert_eq!(tool_call_id.as_deref(), Some("tool-command-1"));
saw_started = true;
}
Event::Command {
event:
protocol::CommandEvent::Output {
command_id,
stream: protocol::CommandStream::Stdout,
content,
..
},
} if command_id == command.0 && content.contains("ready") => saw_output = true,
_ => {}
}
}
assert!(saw_started);
let Event::Snapshot { in_flight, .. } = handle.snapshot_event() else {
panic!("worker snapshot expected");
};
assert_eq!(in_flight.commands.len(), 1);
assert_eq!(in_flight.commands[0].command_id, command.0);
assert_eq!(in_flight.commands[0].stdout.content, "ready");
assert_eq!(
in_flight.commands[0].status,
protocol::CommandStatus::Running
);
let saw_terminal = drain_until(&mut events, std::time::Duration::from_secs(2), |event| {
matches!(
event,
Event::Command {
event: protocol::CommandEvent::Terminal {
command_id,
status: protocol::CommandStatus::Completed,
exit_code: Some(0),
..
}
} if command_id == &command.0
)
})
.await;
assert!(saw_terminal, "completed command event should arrive");
let output = session
.command_output(CommandOutputRequest {
handle: command,
cursor: 0,
limit: 1024,
wait: true,
})
.await
.unwrap();
assert_eq!(output.status, workdir::CommandStatus::Completed);
let (entries, _) = handle.sink.subscribe_with_snapshot();
let durable_history = serde_json::to_string(&entries).unwrap();
assert!(
!durable_history.contains("ready") && !durable_history.contains("done"),
"operational command chunks must not be appended to Worker history: {durable_history}"
);
handle.send(Method::Shutdown).await.unwrap();
}
#[tokio::test]
async fn controller_startup_failure_closes_bound_workdir_session() {
let (mut worker, pwd) = make_worker_with_pwd(MockClient::new(simple_text_events())).await;
@@ -279,6 +384,7 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
command: "printf unreachable".to_owned(),
timeout_secs: 5,
output_limit: 1024,
tool_call_id: None,
})
.await,
Err(WorkdirError::Unavailable(_))