feat: stream workdir command output to consoles
This commit is contained in:
+119
-3
@@ -547,6 +547,12 @@ pub enum Event {
|
|||||||
Status {
|
Status {
|
||||||
status: WorkerStatus,
|
status: WorkerStatus,
|
||||||
},
|
},
|
||||||
|
/// Bounded, provider-owned command telemetry for the live Console. This is
|
||||||
|
/// intentionally not a history entry and is reconstructed from
|
||||||
|
/// `Snapshot.in_flight.commands` after reconnect.
|
||||||
|
Command {
|
||||||
|
event: CommandEvent,
|
||||||
|
},
|
||||||
/// Reply to `Method::ListCompletions`. Delivered only to the
|
/// Reply to `Method::ListCompletions`. Delivered only to the
|
||||||
/// requesting socket (not broadcast). `entries` is empty when no
|
/// requesting socket (not broadcast). `entries` is empty when no
|
||||||
/// candidates match or when the requested kind has no resolver
|
/// candidates match or when the requested kind has no resolver
|
||||||
@@ -714,8 +720,71 @@ pub struct RewindSummary {
|
|||||||
pub tool_side_effect_warning: bool,
|
pub tool_side_effect_warning: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Unfinished model output included in `Event::Snapshot` for clients that
|
/// Live provider-owned command status. These values are operational Console
|
||||||
/// attach while an LLM response is still streaming.
|
/// state only and are never appended to Worker history.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum CommandStatus {
|
||||||
|
Running,
|
||||||
|
Completed,
|
||||||
|
Failed,
|
||||||
|
TimedOut,
|
||||||
|
Cancelled,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum CommandStream {
|
||||||
|
Stdout,
|
||||||
|
Stderr,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
pub struct CommandStreamSlice {
|
||||||
|
pub start_offset: u64,
|
||||||
|
pub end_offset: u64,
|
||||||
|
pub content: String,
|
||||||
|
pub truncated: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
pub struct CommandSnapshot {
|
||||||
|
pub command_id: String,
|
||||||
|
pub tool_call_id: Option<String>,
|
||||||
|
pub status: CommandStatus,
|
||||||
|
pub stdout: CommandStreamSlice,
|
||||||
|
pub stderr: CommandStreamSlice,
|
||||||
|
pub exit_code: Option<i32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||||
|
pub enum CommandEvent {
|
||||||
|
Started {
|
||||||
|
command_id: String,
|
||||||
|
tool_call_id: Option<String>,
|
||||||
|
},
|
||||||
|
Output {
|
||||||
|
command_id: String,
|
||||||
|
stream: CommandStream,
|
||||||
|
start_offset: u64,
|
||||||
|
end_offset: u64,
|
||||||
|
content: String,
|
||||||
|
},
|
||||||
|
Terminal {
|
||||||
|
command_id: String,
|
||||||
|
status: CommandStatus,
|
||||||
|
exit_code: Option<i32>,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unfinished model output and active command state included in
|
||||||
|
/// `Event::Snapshot` for clients that attach while work is still streaming.
|
||||||
///
|
///
|
||||||
/// These blocks are presentation state only: they are reconstructed from the
|
/// These blocks are presentation state only: they are reconstructed from the
|
||||||
/// active Worker controller and must not be treated as committed assistant
|
/// active Worker controller and must not be treated as committed assistant
|
||||||
@@ -726,11 +795,13 @@ pub struct RewindSummary {
|
|||||||
pub struct InFlightSnapshot {
|
pub struct InFlightSnapshot {
|
||||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
pub blocks: Vec<InFlightBlock>,
|
pub blocks: Vec<InFlightBlock>,
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub commands: Vec<CommandSnapshot>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl InFlightSnapshot {
|
impl InFlightSnapshot {
|
||||||
pub fn is_empty(&self) -> bool {
|
pub fn is_empty(&self) -> bool {
|
||||||
self.blocks.is_empty()
|
self.blocks.is_empty() && self.commands.is_empty()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1375,6 +1446,19 @@ mod tests {
|
|||||||
state: InFlightToolCallState::StreamingArgs,
|
state: InFlightToolCallState::StreamingArgs,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
commands: vec![CommandSnapshot {
|
||||||
|
command_id: "command-1".into(),
|
||||||
|
tool_call_id: Some("call_1".into()),
|
||||||
|
status: CommandStatus::Running,
|
||||||
|
stdout: CommandStreamSlice {
|
||||||
|
start_offset: 4,
|
||||||
|
end_offset: 8,
|
||||||
|
content: "tail".into(),
|
||||||
|
truncated: true,
|
||||||
|
},
|
||||||
|
stderr: CommandStreamSlice::default(),
|
||||||
|
exit_code: None,
|
||||||
|
}],
|
||||||
},
|
},
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
};
|
};
|
||||||
@@ -1444,6 +1528,38 @@ mod tests {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn event_command_output_roundtrip_preserves_stream_and_offsets() {
|
||||||
|
let event = Event::Command {
|
||||||
|
event: CommandEvent::Output {
|
||||||
|
command_id: "command-1".into(),
|
||||||
|
stream: CommandStream::Stderr,
|
||||||
|
start_offset: 8,
|
||||||
|
end_offset: 12,
|
||||||
|
content: "warn".into(),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let json = serde_json::to_string(&event).unwrap();
|
||||||
|
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
|
||||||
|
assert_eq!(parsed["event"], "command");
|
||||||
|
assert_eq!(parsed["data"]["event"]["kind"], "output");
|
||||||
|
assert_eq!(parsed["data"]["event"]["stream"], "stderr");
|
||||||
|
assert_eq!(parsed["data"]["event"]["start_offset"], 8);
|
||||||
|
assert_eq!(parsed["data"]["event"]["end_offset"], 12);
|
||||||
|
assert!(matches!(
|
||||||
|
serde_json::from_str::<Event>(&json).unwrap(),
|
||||||
|
Event::Command {
|
||||||
|
event: CommandEvent::Output {
|
||||||
|
command_id,
|
||||||
|
stream: CommandStream::Stderr,
|
||||||
|
start_offset: 8,
|
||||||
|
end_offset: 12,
|
||||||
|
content,
|
||||||
|
}
|
||||||
|
} if command_id == "command-1" && content == "warn"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn event_snapshot_legacy_without_status_defaults_to_idle() {
|
fn event_snapshot_legacy_without_status_defaults_to_idle() {
|
||||||
let json = r#"{"event":"snapshot","data":{"entries":[],"greeting":{"worker_name":"test","cwd":"/tmp","provider":"anthropic","model":"claude","scope_summary":"","tools":[]}}}"#;
|
let json = r#"{"event":"snapshot","data":{"entries":[],"greeting":{"worker_name":"test","cwd":"/tmp","provider":"anthropic","model":"claude","scope_summary":"","tools":[]}}}"#;
|
||||||
|
|||||||
@@ -3,8 +3,9 @@ use std::path::PathBuf;
|
|||||||
use ts_rs::{Config, TS};
|
use ts_rs::{Config, TS};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
Alert, AlertLevel, AlertSource, CompletionEntry, CompletionKind, ErrorCode, Event, Greeting,
|
Alert, AlertLevel, AlertSource, CommandEvent, CommandSnapshot, CommandStatus, CommandStream,
|
||||||
InFlightBlock, InFlightSnapshot, InFlightToolCallState, InternalWorkerKind, InternalWorkerRef,
|
CommandStreamSlice, CompletionEntry, CompletionKind, ErrorCode, Event, Greeting, InFlightBlock,
|
||||||
|
InFlightSnapshot, InFlightToolCallState, InternalWorkerKind, InternalWorkerRef,
|
||||||
InternalWorkerSnapshot, InvokeKind, MemoryWorkerEvent, Method, Permission, RewindSummary,
|
InternalWorkerSnapshot, InvokeKind, MemoryWorkerEvent, Method, Permission, RewindSummary,
|
||||||
RewindTarget, RewindTargetId, RunResult, ScopeRule, Segment, TurnResult, WorkerEvent,
|
RewindTarget, RewindTargetId, RunResult, ScopeRule, Segment, TurnResult, WorkerEvent,
|
||||||
WorkerStatus,
|
WorkerStatus,
|
||||||
@@ -47,6 +48,11 @@ pub fn generated_protocol_types() -> String {
|
|||||||
push_decl::<ErrorCode>(&cfg, &mut output);
|
push_decl::<ErrorCode>(&cfg, &mut output);
|
||||||
push_decl::<Permission>(&cfg, &mut output);
|
push_decl::<Permission>(&cfg, &mut output);
|
||||||
push_decl::<InFlightToolCallState>(&cfg, &mut output);
|
push_decl::<InFlightToolCallState>(&cfg, &mut output);
|
||||||
|
push_decl::<CommandStatus>(&cfg, &mut output);
|
||||||
|
push_decl::<CommandStream>(&cfg, &mut output);
|
||||||
|
push_decl::<CommandStreamSlice>(&cfg, &mut output);
|
||||||
|
push_decl::<CommandSnapshot>(&cfg, &mut output);
|
||||||
|
push_decl::<CommandEvent>(&cfg, &mut output);
|
||||||
push_decl::<ScopeRule>(&cfg, &mut output);
|
push_decl::<ScopeRule>(&cfg, &mut output);
|
||||||
push_decl::<CompletionEntry>(&cfg, &mut output);
|
push_decl::<CompletionEntry>(&cfg, &mut output);
|
||||||
push_decl::<RewindTargetId>(&cfg, &mut output);
|
push_decl::<RewindTargetId>(&cfg, &mut output);
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ impl Tool for BashTool {
|
|||||||
async fn execute(
|
async fn execute(
|
||||||
&self,
|
&self,
|
||||||
input_json: &str,
|
input_json: &str,
|
||||||
_ctx: llm_engine::tool::ToolExecutionContext,
|
ctx: llm_engine::tool::ToolExecutionContext,
|
||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let params: BashParams = serde_json::from_str(input_json)
|
let params: BashParams = serde_json::from_str(input_json)
|
||||||
.map_err(|error| ToolError::InvalidArgument(format!("invalid Bash input: {error}")))?;
|
.map_err(|error| ToolError::InvalidArgument(format!("invalid Bash input: {error}")))?;
|
||||||
@@ -58,6 +58,7 @@ impl Tool for BashTool {
|
|||||||
command: params.command,
|
command: params.command,
|
||||||
timeout_secs,
|
timeout_secs,
|
||||||
output_limit: INLINE_BYTE_BUDGET,
|
output_limit: INLINE_BYTE_BUDGET,
|
||||||
|
tool_call_id: Some(ctx.call_id),
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(crate::ToolsError::from)?;
|
.map_err(crate::ToolsError::from)?;
|
||||||
|
|||||||
@@ -1322,6 +1322,9 @@ impl App {
|
|||||||
self.rewind_refresh_fence = false;
|
self.rewind_refresh_fence = false;
|
||||||
self.set_worker_status(status);
|
self.set_worker_status(status);
|
||||||
}
|
}
|
||||||
|
// Command telemetry is an operational Web Console surface. The
|
||||||
|
// TUI continues to render the final Bash ToolResult from history.
|
||||||
|
Event::Command { .. } => {}
|
||||||
Event::Completions { kind, entries } => {
|
Event::Completions { kind, entries } => {
|
||||||
// Apply only if the popup is still on the same
|
// Apply only if the popup is still on the same
|
||||||
// (kind, prefix) the request was issued for; an
|
// (kind, prefix) the request was issued for; an
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
pub use delegation::{
|
pub use delegation::{
|
||||||
AppliedWorkdirDelegation, ReadOnlyWorkdirSession, WorkdirDelegation,
|
AppliedWorkdirDelegation, ReadOnlyWorkdirSession, WorkdirDelegation,
|
||||||
@@ -192,6 +193,19 @@ pub trait WorkdirSession: std::fmt::Debug + Send + Sync {
|
|||||||
request: CommandOutputRequest,
|
request: CommandOutputRequest,
|
||||||
) -> Result<CommandOutput, WorkdirError>;
|
) -> Result<CommandOutput, WorkdirError>;
|
||||||
async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError>;
|
async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError>;
|
||||||
|
|
||||||
|
/// Subscribe to bounded provider-owned command telemetry. Implementations
|
||||||
|
/// that do not expose live command observation may keep the default.
|
||||||
|
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<CommandEvent>> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return the bounded current command state used to recover from a lagged
|
||||||
|
/// provider subscription without replaying command output into history.
|
||||||
|
fn command_snapshot(&self) -> Vec<CommandSnapshot> {
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
|
|
||||||
/// Terminal, idempotent release of this Worker-bound operation session.
|
/// Terminal, idempotent release of this Worker-bound operation session.
|
||||||
async fn close(&self) -> Result<(), WorkdirError>;
|
async fn close(&self) -> Result<(), WorkdirError>;
|
||||||
}
|
}
|
||||||
|
|||||||
+461
-54
@@ -14,21 +14,22 @@ use std::io::Write as _;
|
|||||||
use std::io::{Read as _, Seek as _, SeekFrom};
|
use std::io::{Read as _, Seek as _, SeekFrom};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::Stdio;
|
use std::process::Stdio;
|
||||||
use std::sync::Arc;
|
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||||
|
use std::sync::{Arc, Mutex as StdMutex};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use manifest::{Permission, Scope, ScopeConfig, ScopeRule, SharedScope};
|
use manifest::{Permission, Scope, ScopeConfig, ScopeRule, SharedScope};
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
use tokio::sync::{Mutex, Notify};
|
use tokio::sync::{Mutex, Notify, broadcast, watch};
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
CommandHandle, CommandOutput, CommandOutputRequest, CommandRequest, CommandStatus, EditRequest,
|
CommandEvent, CommandHandle, CommandOutput, CommandOutputRequest, CommandRequest,
|
||||||
EditResult, GlobRequest, GlobResult, GrepRequest, GrepResult, ListRequest, ListResult,
|
CommandSnapshot, CommandStatus, CommandStream, CommandStreamSlice, EditRequest, EditResult,
|
||||||
ReadRequest, ReadResult, StatRequest, StatResult, Workdir, WorkdirDelegationPermission,
|
GlobRequest, GlobResult, GrepRequest, GrepResult, ListRequest, ListResult, ReadRequest,
|
||||||
|
ReadResult, StatRequest, StatResult, Workdir, WorkdirDelegationPermission,
|
||||||
WorkdirDelegationRequest, WorkdirError, WorkdirPath, WorkdirSession,
|
WorkdirDelegationRequest, WorkdirError, WorkdirPath, WorkdirSession,
|
||||||
WorkdirSessionCapabilities, WorkdirSessionCapability, WorkdirSessionHandle, WriteRequest,
|
WorkdirSessionCapabilities, WorkdirSessionCapability, WorkdirSessionHandle, WriteRequest,
|
||||||
WriteResult,
|
WriteResult,
|
||||||
@@ -36,15 +37,146 @@ use crate::{
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use crate::{EntryKind, WriteOutcome};
|
use crate::{EntryKind, WriteOutcome};
|
||||||
|
|
||||||
|
const COMMAND_EVENT_CHANNEL_CAPACITY: usize = 256;
|
||||||
|
const COMMAND_EVENT_CHUNK_BYTES: usize = 8 * 1024;
|
||||||
|
const COMMAND_SNAPSHOT_STREAM_BYTES: usize = 32 * 1024;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
enum LocalCommand {
|
enum LocalCommand {
|
||||||
Running {
|
Running {
|
||||||
task: JoinHandle<Result<CommandOutput, WorkdirError>>,
|
task: JoinHandle<Result<CommandOutput, WorkdirError>>,
|
||||||
completion: Arc<Notify>,
|
completion: Arc<Notify>,
|
||||||
|
cancel: watch::Sender<bool>,
|
||||||
},
|
},
|
||||||
Completed(CommandOutput),
|
Completed(CommandOutput),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct CommandTelemetry {
|
||||||
|
inner: Arc<CommandTelemetryInner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct CommandTelemetryInner {
|
||||||
|
snapshots: StdMutex<HashMap<String, CommandSnapshot>>,
|
||||||
|
events: broadcast::Sender<CommandEvent>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CommandTelemetry {
|
||||||
|
fn new() -> Self {
|
||||||
|
let (events, _) = broadcast::channel(COMMAND_EVENT_CHANNEL_CAPACITY);
|
||||||
|
Self {
|
||||||
|
inner: Arc::new(CommandTelemetryInner {
|
||||||
|
snapshots: StdMutex::new(HashMap::new()),
|
||||||
|
events,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn subscribe(&self) -> broadcast::Receiver<CommandEvent> {
|
||||||
|
self.inner.events.subscribe()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn snapshot(&self) -> Vec<CommandSnapshot> {
|
||||||
|
let mut snapshots = self
|
||||||
|
.inner
|
||||||
|
.snapshots
|
||||||
|
.lock()
|
||||||
|
.expect("command telemetry mutex poisoned")
|
||||||
|
.values()
|
||||||
|
.cloned()
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
snapshots.sort_by(|left, right| left.command_id.cmp(&right.command_id));
|
||||||
|
snapshots
|
||||||
|
}
|
||||||
|
|
||||||
|
fn started(&self, command_id: &str, tool_call_id: Option<String>) {
|
||||||
|
self.inner
|
||||||
|
.snapshots
|
||||||
|
.lock()
|
||||||
|
.expect("command telemetry mutex poisoned")
|
||||||
|
.insert(
|
||||||
|
command_id.to_string(),
|
||||||
|
CommandSnapshot {
|
||||||
|
command_id: command_id.to_string(),
|
||||||
|
tool_call_id: tool_call_id.clone(),
|
||||||
|
status: CommandStatus::Running,
|
||||||
|
stdout: CommandStreamSlice::default(),
|
||||||
|
stderr: CommandStreamSlice::default(),
|
||||||
|
exit_code: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let _ = self.inner.events.send(CommandEvent::Started {
|
||||||
|
command_id: command_id.to_string(),
|
||||||
|
tool_call_id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn output(&self, command_id: &str, stream: CommandStream, start_offset: u64, bytes: &[u8]) {
|
||||||
|
if bytes.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let end_offset = start_offset.saturating_add(bytes.len() as u64);
|
||||||
|
let content = String::from_utf8_lossy(bytes).into_owned();
|
||||||
|
if let Some(snapshot) = self
|
||||||
|
.inner
|
||||||
|
.snapshots
|
||||||
|
.lock()
|
||||||
|
.expect("command telemetry mutex poisoned")
|
||||||
|
.get_mut(command_id)
|
||||||
|
{
|
||||||
|
let target = match stream {
|
||||||
|
CommandStream::Stdout => &mut snapshot.stdout,
|
||||||
|
CommandStream::Stderr => &mut snapshot.stderr,
|
||||||
|
};
|
||||||
|
target.end_offset = end_offset;
|
||||||
|
target.content.push_str(&content);
|
||||||
|
if target.content.len() > COMMAND_SNAPSHOT_STREAM_BYTES {
|
||||||
|
let mut cut = target.content.len() - COMMAND_SNAPSHOT_STREAM_BYTES;
|
||||||
|
while cut < target.content.len() && !target.content.is_char_boundary(cut) {
|
||||||
|
cut += 1;
|
||||||
|
}
|
||||||
|
target.content.drain(..cut);
|
||||||
|
target.start_offset = end_offset.saturating_sub(target.content.len() as u64);
|
||||||
|
target.truncated = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _ = self.inner.events.send(CommandEvent::Output {
|
||||||
|
command_id: command_id.to_string(),
|
||||||
|
stream,
|
||||||
|
start_offset,
|
||||||
|
end_offset,
|
||||||
|
content,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn terminal(&self, command_id: &str, status: CommandStatus, exit_code: Option<i32>) {
|
||||||
|
if let Some(snapshot) = self
|
||||||
|
.inner
|
||||||
|
.snapshots
|
||||||
|
.lock()
|
||||||
|
.expect("command telemetry mutex poisoned")
|
||||||
|
.get_mut(command_id)
|
||||||
|
{
|
||||||
|
snapshot.status = status;
|
||||||
|
snapshot.exit_code = exit_code;
|
||||||
|
}
|
||||||
|
let _ = self.inner.events.send(CommandEvent::Terminal {
|
||||||
|
command_id: command_id.to_string(),
|
||||||
|
status,
|
||||||
|
exit_code,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove(&self, command_id: &str) {
|
||||||
|
self.inner
|
||||||
|
.snapshots
|
||||||
|
.lock()
|
||||||
|
.expect("command telemetry mutex poisoned")
|
||||||
|
.remove(command_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct ScopeAccess(Arc<Scope>);
|
struct ScopeAccess(Arc<Scope>);
|
||||||
|
|
||||||
@@ -69,6 +201,7 @@ struct LocalWorkdirSessionInner {
|
|||||||
close_lock: Mutex<()>,
|
close_lock: Mutex<()>,
|
||||||
next_command_id: AtomicU64,
|
next_command_id: AtomicU64,
|
||||||
commands: Mutex<HashMap<String, LocalCommand>>,
|
commands: Mutex<HashMap<String, LocalCommand>>,
|
||||||
|
command_telemetry: CommandTelemetry,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for LocalWorkdirSessionInner {
|
impl Drop for LocalWorkdirSessionInner {
|
||||||
@@ -171,6 +304,7 @@ impl LocalWorkdirSession {
|
|||||||
close_lock: Mutex::new(()),
|
close_lock: Mutex::new(()),
|
||||||
next_command_id: AtomicU64::new(1),
|
next_command_id: AtomicU64::new(1),
|
||||||
commands: Mutex::new(HashMap::new()),
|
commands: Mutex::new(HashMap::new()),
|
||||||
|
command_telemetry: CommandTelemetry::new(),
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -502,23 +636,35 @@ impl WorkdirSession for LocalWorkdirSession {
|
|||||||
|
|
||||||
async fn start_command(&self, request: CommandRequest) -> Result<CommandHandle, WorkdirError> {
|
async fn start_command(&self, request: CommandRequest) -> Result<CommandHandle, WorkdirError> {
|
||||||
self.ensure_capability(WorkdirSessionCapability::Command)?;
|
self.ensure_capability(WorkdirSessionCapability::Command)?;
|
||||||
|
self.ensure_open()?;
|
||||||
let id = self.inner.next_command_id.fetch_add(1, Ordering::Relaxed);
|
let id = self.inner.next_command_id.fetch_add(1, Ordering::Relaxed);
|
||||||
let handle = CommandHandle(format!("command-{id}"));
|
let handle = CommandHandle(format!("command-{id}"));
|
||||||
let cwd = self.inner.cwd.clone();
|
let cwd = self.inner.cwd.clone();
|
||||||
let completion = Arc::new(Notify::new());
|
let completion = Arc::new(Notify::new());
|
||||||
let task_completion = Arc::clone(&completion);
|
let task_completion = Arc::clone(&completion);
|
||||||
|
let command_id = handle.0.clone();
|
||||||
|
let telemetry = self.inner.command_telemetry.clone();
|
||||||
|
let (cancel, cancel_rx) = watch::channel(false);
|
||||||
let task = tokio::spawn(async move {
|
let task = tokio::spawn(async move {
|
||||||
let output = run_command(cwd, request).await;
|
let output = run_command(cwd, request, command_id, telemetry, cancel_rx).await;
|
||||||
task_completion.notify_one();
|
task_completion.notify_one();
|
||||||
output
|
output
|
||||||
});
|
});
|
||||||
let mut commands = self.inner.commands.lock().await;
|
let mut commands = self.inner.commands.lock().await;
|
||||||
if let Err(error) = self.ensure_open() {
|
if let Err(error) = self.ensure_open() {
|
||||||
|
let _ = cancel.send(true);
|
||||||
task.abort();
|
task.abort();
|
||||||
completion.notify_one();
|
completion.notify_one();
|
||||||
return Err(error);
|
return Err(error);
|
||||||
}
|
}
|
||||||
commands.insert(handle.0.clone(), LocalCommand::Running { task, completion });
|
commands.insert(
|
||||||
|
handle.0.clone(),
|
||||||
|
LocalCommand::Running {
|
||||||
|
task,
|
||||||
|
completion,
|
||||||
|
cancel,
|
||||||
|
},
|
||||||
|
);
|
||||||
Ok(handle)
|
Ok(handle)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -530,7 +676,13 @@ impl WorkdirSession for LocalWorkdirSession {
|
|||||||
.ok_or_else(|| WorkdirError::UnknownCommand(handle.0.clone()))?;
|
.ok_or_else(|| WorkdirError::UnknownCommand(handle.0.clone()))?;
|
||||||
Ok(match command {
|
Ok(match command {
|
||||||
LocalCommand::Running { task, .. } if !task.is_finished() => CommandStatus::Running,
|
LocalCommand::Running { task, .. } if !task.is_finished() => CommandStatus::Running,
|
||||||
LocalCommand::Running { .. } => CommandStatus::Completed,
|
LocalCommand::Running { .. } => self
|
||||||
|
.inner
|
||||||
|
.command_telemetry
|
||||||
|
.snapshot()
|
||||||
|
.into_iter()
|
||||||
|
.find(|snapshot| snapshot.command_id == handle.0)
|
||||||
|
.map_or(CommandStatus::Completed, |snapshot| snapshot.status),
|
||||||
LocalCommand::Completed(output) => output.status,
|
LocalCommand::Completed(output) => output.status,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -584,36 +736,75 @@ impl WorkdirSession for LocalWorkdirSession {
|
|||||||
if !self.inner.closed.load(Ordering::Acquire) {
|
if !self.inner.closed.load(Ordering::Acquire) {
|
||||||
commands.insert(request.handle.0, LocalCommand::Completed(output));
|
commands.insert(request.handle.0, LocalCommand::Completed(output));
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
self.inner.command_telemetry.remove(&request.handle.0);
|
||||||
}
|
}
|
||||||
Ok(page)
|
Ok(page)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError> {
|
async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError> {
|
||||||
self.ensure_capability(WorkdirSessionCapability::Command)?;
|
self.ensure_capability(WorkdirSessionCapability::Command)?;
|
||||||
let command = self
|
let cancel = {
|
||||||
.inner
|
let commands = self.inner.commands.lock().await;
|
||||||
.commands
|
let command = commands
|
||||||
.lock()
|
.get(&handle.0)
|
||||||
.await
|
.ok_or_else(|| WorkdirError::UnknownCommand(handle.0.clone()))?;
|
||||||
.remove(&handle.0)
|
match command {
|
||||||
.ok_or_else(|| WorkdirError::UnknownCommand(handle.0))?;
|
LocalCommand::Running { task, cancel, .. } if !task.is_finished() => {
|
||||||
if let LocalCommand::Running { task, completion } = command {
|
Some(cancel.clone())
|
||||||
task.abort();
|
}
|
||||||
completion.notify_one();
|
_ => None,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let Some(cancel) = cancel {
|
||||||
|
let _ = cancel.send(true);
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<CommandEvent>> {
|
||||||
|
self.inner
|
||||||
|
.capabilities
|
||||||
|
.supports(WorkdirSessionCapability::Command)
|
||||||
|
.then(|| self.inner.command_telemetry.subscribe())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn command_snapshot(&self) -> Vec<CommandSnapshot> {
|
||||||
|
if self
|
||||||
|
.inner
|
||||||
|
.capabilities
|
||||||
|
.supports(WorkdirSessionCapability::Command)
|
||||||
|
{
|
||||||
|
self.inner.command_telemetry.snapshot()
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn close(&self) -> Result<(), WorkdirError> {
|
async fn close(&self) -> Result<(), WorkdirError> {
|
||||||
let _close_guard = self.inner.close_lock.lock().await;
|
let _close_guard = self.inner.close_lock.lock().await;
|
||||||
if self.inner.closed.swap(true, Ordering::AcqRel) {
|
if self.inner.closed.swap(true, Ordering::AcqRel) {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
let mut commands = self.inner.commands.lock().await;
|
let commands = {
|
||||||
for (_, command) in commands.drain() {
|
let mut commands = self.inner.commands.lock().await;
|
||||||
if let LocalCommand::Running { task, completion } = command {
|
commands
|
||||||
task.abort();
|
.drain()
|
||||||
completion.notify_one();
|
.map(|(_, command)| command)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
};
|
||||||
|
for command in commands {
|
||||||
|
match command {
|
||||||
|
LocalCommand::Running {
|
||||||
|
task,
|
||||||
|
completion,
|
||||||
|
cancel,
|
||||||
|
} => {
|
||||||
|
let _ = cancel.send(true);
|
||||||
|
let _ = task.await;
|
||||||
|
completion.notify_one();
|
||||||
|
}
|
||||||
|
LocalCommand::Completed(_) => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -680,7 +871,13 @@ fn sanitize_error(error: WorkdirError, logical: &WorkdirPath) -> WorkdirError {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_command(cwd: PathBuf, request: CommandRequest) -> Result<CommandOutput, WorkdirError> {
|
async fn run_command(
|
||||||
|
cwd: PathBuf,
|
||||||
|
request: CommandRequest,
|
||||||
|
command_id: String,
|
||||||
|
telemetry: CommandTelemetry,
|
||||||
|
mut cancel: watch::Receiver<bool>,
|
||||||
|
) -> Result<CommandOutput, WorkdirError> {
|
||||||
let stdout = tempfile::NamedTempFile::new().map_err(|error| WorkdirError::io(&cwd, error))?;
|
let stdout = tempfile::NamedTempFile::new().map_err(|error| WorkdirError::io(&cwd, error))?;
|
||||||
let stderr = tempfile::NamedTempFile::new().map_err(|error| WorkdirError::io(&cwd, error))?;
|
let stderr = tempfile::NamedTempFile::new().map_err(|error| WorkdirError::io(&cwd, error))?;
|
||||||
let stdout_path = stdout.into_temp_path();
|
let stdout_path = stdout.into_temp_path();
|
||||||
@@ -690,7 +887,8 @@ async fn run_command(cwd: PathBuf, request: CommandRequest) -> Result<CommandOut
|
|||||||
let stderr_file = std::fs::File::create(&stderr_path)
|
let stderr_file = std::fs::File::create(&stderr_path)
|
||||||
.map_err(|error| WorkdirError::io(&stderr_path, error))?;
|
.map_err(|error| WorkdirError::io(&stderr_path, error))?;
|
||||||
|
|
||||||
let mut child = Command::new("bash")
|
telemetry.started(&command_id, request.tool_call_id.clone());
|
||||||
|
let mut child = match Command::new("bash")
|
||||||
.arg("-c")
|
.arg("-c")
|
||||||
.arg(&request.command)
|
.arg(&request.command)
|
||||||
.current_dir(&cwd)
|
.current_dir(&cwd)
|
||||||
@@ -699,45 +897,126 @@ async fn run_command(cwd: PathBuf, request: CommandRequest) -> Result<CommandOut
|
|||||||
.stderr(Stdio::from(stderr_file))
|
.stderr(Stdio::from(stderr_file))
|
||||||
.kill_on_drop(true)
|
.kill_on_drop(true)
|
||||||
.spawn()
|
.spawn()
|
||||||
.map_err(|error| WorkdirError::io(&cwd, error))?;
|
|
||||||
|
|
||||||
let timed_out = match tokio::time::timeout(
|
|
||||||
Duration::from_secs(request.timeout_secs.max(1)),
|
|
||||||
child.wait(),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
{
|
||||||
Ok(result) => {
|
Ok(child) => child,
|
||||||
let status = result.map_err(|error| WorkdirError::io(&cwd, error))?;
|
Err(error) => {
|
||||||
let (content, truncated) =
|
telemetry.terminal(&command_id, CommandStatus::Failed, None);
|
||||||
read_command_output_files(&stdout_path, &stderr_path, request.output_limit.max(1))?;
|
return Err(WorkdirError::io(&cwd, error));
|
||||||
return Ok(CommandOutput {
|
|
||||||
status: CommandStatus::Completed,
|
|
||||||
exit_code: status.code(),
|
|
||||||
timed_out: false,
|
|
||||||
content,
|
|
||||||
next_cursor: None,
|
|
||||||
truncated,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
let _ = child.kill().await;
|
|
||||||
true
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let mut stdout_reader =
|
||||||
|
std::fs::File::open(&stdout_path).map_err(|error| WorkdirError::io(&stdout_path, error))?;
|
||||||
|
let mut stderr_reader =
|
||||||
|
std::fs::File::open(&stderr_path).map_err(|error| WorkdirError::io(&stderr_path, error))?;
|
||||||
|
let mut stdout_offset = 0;
|
||||||
|
let mut stderr_offset = 0;
|
||||||
|
let mut timeout = Box::pin(tokio::time::sleep(Duration::from_secs(
|
||||||
|
request.timeout_secs.max(1),
|
||||||
|
)));
|
||||||
|
let mut interval = tokio::time::interval(Duration::from_millis(50));
|
||||||
|
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||||
|
interval.tick().await;
|
||||||
|
|
||||||
|
let (status, exit_code) = loop {
|
||||||
|
tokio::select! {
|
||||||
|
exit = child.wait() => {
|
||||||
|
let exit = exit.map_err(|error| WorkdirError::io(&cwd, error))?;
|
||||||
|
break (
|
||||||
|
if exit.success() { CommandStatus::Completed } else { CommandStatus::Failed },
|
||||||
|
exit.code(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
_ = &mut timeout => {
|
||||||
|
let _ = child.start_kill();
|
||||||
|
let exit_code = child.wait().await.ok().and_then(|status| status.code());
|
||||||
|
break (CommandStatus::TimedOut, exit_code);
|
||||||
|
}
|
||||||
|
changed = cancel.changed() => {
|
||||||
|
if changed.is_err() || *cancel.borrow() {
|
||||||
|
let _ = child.start_kill();
|
||||||
|
let exit_code = child.wait().await.ok().and_then(|status| status.code());
|
||||||
|
break (CommandStatus::Cancelled, exit_code);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = interval.tick() => {
|
||||||
|
publish_available_output(
|
||||||
|
&mut stdout_reader,
|
||||||
|
&mut stdout_offset,
|
||||||
|
&telemetry,
|
||||||
|
&command_id,
|
||||||
|
CommandStream::Stdout,
|
||||||
|
&stdout_path,
|
||||||
|
)?;
|
||||||
|
publish_available_output(
|
||||||
|
&mut stderr_reader,
|
||||||
|
&mut stderr_offset,
|
||||||
|
&telemetry,
|
||||||
|
&command_id,
|
||||||
|
CommandStream::Stderr,
|
||||||
|
&stderr_path,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
publish_available_output(
|
||||||
|
&mut stdout_reader,
|
||||||
|
&mut stdout_offset,
|
||||||
|
&telemetry,
|
||||||
|
&command_id,
|
||||||
|
CommandStream::Stdout,
|
||||||
|
&stdout_path,
|
||||||
|
)?;
|
||||||
|
publish_available_output(
|
||||||
|
&mut stderr_reader,
|
||||||
|
&mut stderr_offset,
|
||||||
|
&telemetry,
|
||||||
|
&command_id,
|
||||||
|
CommandStream::Stderr,
|
||||||
|
&stderr_path,
|
||||||
|
)?;
|
||||||
|
telemetry.terminal(&command_id, status, exit_code);
|
||||||
|
|
||||||
let (content, truncated) =
|
let (content, truncated) =
|
||||||
read_command_output_files(&stdout_path, &stderr_path, request.output_limit.max(1))?;
|
read_command_output_files(&stdout_path, &stderr_path, request.output_limit.max(1))?;
|
||||||
Ok(CommandOutput {
|
Ok(CommandOutput {
|
||||||
status: CommandStatus::Failed,
|
status,
|
||||||
exit_code: None,
|
exit_code,
|
||||||
timed_out,
|
timed_out: status == CommandStatus::TimedOut,
|
||||||
content,
|
content,
|
||||||
next_cursor: None,
|
next_cursor: None,
|
||||||
truncated,
|
truncated,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn publish_available_output(
|
||||||
|
file: &mut std::fs::File,
|
||||||
|
offset: &mut u64,
|
||||||
|
telemetry: &CommandTelemetry,
|
||||||
|
command_id: &str,
|
||||||
|
stream: CommandStream,
|
||||||
|
path: &Path,
|
||||||
|
) -> Result<(), WorkdirError> {
|
||||||
|
file.seek(SeekFrom::Start(*offset))
|
||||||
|
.map_err(|error| WorkdirError::io(path, error))?;
|
||||||
|
loop {
|
||||||
|
let mut buffer = vec![0; COMMAND_EVENT_CHUNK_BYTES];
|
||||||
|
let read = file
|
||||||
|
.read(&mut buffer)
|
||||||
|
.map_err(|error| WorkdirError::io(path, error))?;
|
||||||
|
if read == 0 {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
buffer.truncate(read);
|
||||||
|
telemetry.output(command_id, stream, *offset, &buffer);
|
||||||
|
*offset = offset.saturating_add(read as u64);
|
||||||
|
if read < COMMAND_EVENT_CHUNK_BYTES {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn read_command_output_files(
|
fn read_command_output_files(
|
||||||
stdout_path: &Path,
|
stdout_path: &Path,
|
||||||
stderr_path: &Path,
|
stderr_path: &Path,
|
||||||
@@ -1024,6 +1303,7 @@ mod tests {
|
|||||||
command: "sleep 30".to_owned(),
|
command: "sleep 30".to_owned(),
|
||||||
timeout_secs: 60,
|
timeout_secs: 60,
|
||||||
output_limit: 1024,
|
output_limit: 1024,
|
||||||
|
tool_call_id: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -1549,6 +1829,7 @@ mod tests {
|
|||||||
command: "pwd && printf provider-command".into(),
|
command: "pwd && printf provider-command".into(),
|
||||||
timeout_secs: 5,
|
timeout_secs: 5,
|
||||||
output_limit: 4096,
|
output_limit: 4096,
|
||||||
|
tool_call_id: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -1583,6 +1864,7 @@ mod tests {
|
|||||||
command: "printf 'aéz'".into(),
|
command: "printf 'aéz'".into(),
|
||||||
timeout_secs: 5,
|
timeout_secs: 5,
|
||||||
output_limit: 1024,
|
output_limit: 1024,
|
||||||
|
tool_call_id: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -1620,6 +1902,130 @@ mod tests {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn provider_streams_bounded_command_lifecycle_and_distinct_output() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let workdir = make_fs(&dir);
|
||||||
|
let mut events = WorkdirSession::subscribe_command_events(&workdir)
|
||||||
|
.expect("local command observation must be available");
|
||||||
|
let handle = WorkdirSession::start_command(
|
||||||
|
&workdir,
|
||||||
|
CommandRequest {
|
||||||
|
command: "printf ready; printf warning >&2; sleep 0.2; printf done".into(),
|
||||||
|
timeout_secs: 5,
|
||||||
|
output_limit: 1024,
|
||||||
|
tool_call_id: Some("tool-7".into()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let mut stdout = String::new();
|
||||||
|
let mut stderr = String::new();
|
||||||
|
let mut terminal = None;
|
||||||
|
while terminal.is_none() {
|
||||||
|
let event = tokio::time::timeout(Duration::from_secs(2), events.recv())
|
||||||
|
.await
|
||||||
|
.expect("command telemetry should not stall")
|
||||||
|
.unwrap();
|
||||||
|
match event {
|
||||||
|
CommandEvent::Started {
|
||||||
|
command_id,
|
||||||
|
tool_call_id,
|
||||||
|
} => {
|
||||||
|
assert_eq!(command_id, handle.0);
|
||||||
|
assert_eq!(tool_call_id.as_deref(), Some("tool-7"));
|
||||||
|
}
|
||||||
|
CommandEvent::Output {
|
||||||
|
command_id,
|
||||||
|
stream,
|
||||||
|
content,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
assert_eq!(command_id, handle.0);
|
||||||
|
match stream {
|
||||||
|
CommandStream::Stdout => stdout.push_str(&content),
|
||||||
|
CommandStream::Stderr => stderr.push_str(&content),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
CommandEvent::Terminal {
|
||||||
|
command_id,
|
||||||
|
status,
|
||||||
|
exit_code,
|
||||||
|
} => {
|
||||||
|
assert_eq!(command_id, handle.0);
|
||||||
|
terminal = Some((status, exit_code));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(terminal, Some((CommandStatus::Completed, Some(0))));
|
||||||
|
assert_eq!(stdout, "readydone");
|
||||||
|
assert_eq!(stderr, "warning");
|
||||||
|
let snapshot = WorkdirSession::command_snapshot(&workdir);
|
||||||
|
assert_eq!(snapshot.len(), 1);
|
||||||
|
assert_eq!(snapshot[0].status, CommandStatus::Completed);
|
||||||
|
assert_eq!(snapshot[0].stdout.content, "readydone");
|
||||||
|
assert_eq!(snapshot[0].stderr.content, "warning");
|
||||||
|
|
||||||
|
let output = WorkdirSession::command_output(
|
||||||
|
&workdir,
|
||||||
|
CommandOutputRequest {
|
||||||
|
handle,
|
||||||
|
cursor: 0,
|
||||||
|
limit: 1024,
|
||||||
|
wait: true,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(output.status, CommandStatus::Completed);
|
||||||
|
assert!(WorkdirSession::command_snapshot(&workdir).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn provider_distinguishes_timed_out_terminal_state() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let workdir = make_fs(&dir);
|
||||||
|
let mut events = WorkdirSession::subscribe_command_events(&workdir).unwrap();
|
||||||
|
let handle = WorkdirSession::start_command(
|
||||||
|
&workdir,
|
||||||
|
CommandRequest {
|
||||||
|
command: "sleep 30".into(),
|
||||||
|
timeout_secs: 1,
|
||||||
|
output_limit: 1024,
|
||||||
|
tool_call_id: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let output = WorkdirSession::command_output(
|
||||||
|
&workdir,
|
||||||
|
CommandOutputRequest {
|
||||||
|
handle: handle.clone(),
|
||||||
|
cursor: 0,
|
||||||
|
limit: 1024,
|
||||||
|
wait: true,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(output.status, CommandStatus::TimedOut);
|
||||||
|
assert!(output.timed_out);
|
||||||
|
|
||||||
|
let mut terminal = None;
|
||||||
|
while let Ok(event) = events.try_recv() {
|
||||||
|
if let CommandEvent::Terminal {
|
||||||
|
command_id,
|
||||||
|
status,
|
||||||
|
exit_code,
|
||||||
|
} = event
|
||||||
|
{
|
||||||
|
terminal = Some((command_id, status, exit_code));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(terminal, Some((handle.0, CommandStatus::TimedOut, None)));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn provider_cancels_active_command() {
|
async fn provider_cancels_active_command() {
|
||||||
let dir = TempDir::new().unwrap();
|
let dir = TempDir::new().unwrap();
|
||||||
@@ -1630,6 +2036,7 @@ mod tests {
|
|||||||
command: "sleep 30".into(),
|
command: "sleep 30".into(),
|
||||||
timeout_secs: 60,
|
timeout_secs: 60,
|
||||||
output_limit: 1024,
|
output_limit: 1024,
|
||||||
|
tool_call_id: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -1658,12 +2065,12 @@ mod tests {
|
|||||||
WorkdirSession::cancel_command(&workdir, handle.clone())
|
WorkdirSession::cancel_command(&workdir, handle.clone())
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let waiter_error = tokio::time::timeout(Duration::from_secs(1), waiter)
|
let output = tokio::time::timeout(Duration::from_secs(1), waiter)
|
||||||
.await
|
.await
|
||||||
.expect("cancel should wake command output waiters")
|
.expect("cancel should wake command output waiters")
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap_err();
|
.unwrap();
|
||||||
assert!(matches!(waiter_error, WorkdirError::UnknownCommand(_)));
|
assert_eq!(output.status, CommandStatus::Cancelled);
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
WorkdirSession::command_status(&workdir, handle).await,
|
WorkdirSession::command_status(&workdir, handle).await,
|
||||||
Err(WorkdirError::UnknownCommand(_))
|
Err(WorkdirError::UnknownCommand(_))
|
||||||
|
|||||||
@@ -9,6 +9,11 @@ pub struct CommandRequest {
|
|||||||
pub command: String,
|
pub command: String,
|
||||||
pub timeout_secs: u64,
|
pub timeout_secs: u64,
|
||||||
pub output_limit: usize,
|
pub output_limit: usize,
|
||||||
|
/// Optional caller-owned correlation id. Bash supplies its tool-call id so
|
||||||
|
/// user-facing command telemetry can update the corresponding Console row
|
||||||
|
/// without exposing provider/session handles.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub tool_call_id: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
@@ -24,8 +29,55 @@ pub struct CommandOutputRequest {
|
|||||||
pub enum CommandStatus {
|
pub enum CommandStatus {
|
||||||
Running,
|
Running,
|
||||||
Completed,
|
Completed,
|
||||||
Cancelled,
|
|
||||||
Failed,
|
Failed,
|
||||||
|
TimedOut,
|
||||||
|
Cancelled,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum CommandStream {
|
||||||
|
Stdout,
|
||||||
|
Stderr,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||||
|
pub struct CommandStreamSlice {
|
||||||
|
pub start_offset: u64,
|
||||||
|
pub end_offset: u64,
|
||||||
|
pub content: String,
|
||||||
|
pub truncated: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct CommandSnapshot {
|
||||||
|
pub command_id: String,
|
||||||
|
pub tool_call_id: Option<String>,
|
||||||
|
pub status: CommandStatus,
|
||||||
|
pub stdout: CommandStreamSlice,
|
||||||
|
pub stderr: CommandStreamSlice,
|
||||||
|
pub exit_code: Option<i32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||||
|
pub enum CommandEvent {
|
||||||
|
Started {
|
||||||
|
command_id: String,
|
||||||
|
tool_call_id: Option<String>,
|
||||||
|
},
|
||||||
|
Output {
|
||||||
|
command_id: String,
|
||||||
|
stream: CommandStream,
|
||||||
|
start_offset: u64,
|
||||||
|
end_offset: u64,
|
||||||
|
content: String,
|
||||||
|
},
|
||||||
|
Terminal {
|
||||||
|
command_id: String,
|
||||||
|
status: CommandStatus,
|
||||||
|
exit_code: Option<i32>,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
|||||||
@@ -1430,7 +1430,10 @@ impl Runtime {
|
|||||||
context_tokens: 0,
|
context_tokens: 0,
|
||||||
},
|
},
|
||||||
status: protocol::WorkerStatus::Idle,
|
status: protocol::WorkerStatus::Idle,
|
||||||
in_flight: protocol::InFlightSnapshot { blocks: Vec::new() },
|
in_flight: protocol::InFlightSnapshot {
|
||||||
|
blocks: Vec::new(),
|
||||||
|
commands: Vec::new(),
|
||||||
|
},
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -3765,7 +3768,10 @@ mod tests {
|
|||||||
context_tokens: 64,
|
context_tokens: 64,
|
||||||
},
|
},
|
||||||
status: protocol::WorkerStatus::Running,
|
status: protocol::WorkerStatus::Running,
|
||||||
in_flight: protocol::InFlightSnapshot { blocks: Vec::new() },
|
in_flight: protocol::InFlightSnapshot {
|
||||||
|
blocks: Vec::new(),
|
||||||
|
commands: Vec::new(),
|
||||||
|
},
|
||||||
internal_workers: Vec::new(),
|
internal_workers: Vec::new(),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -28,8 +28,14 @@ use crate::worker::{
|
|||||||
WorkerRunResult,
|
WorkerRunResult,
|
||||||
};
|
};
|
||||||
use protocol::{
|
use protocol::{
|
||||||
AlertLevel, AlertSource, ErrorCode, Event, Method, RewindTargetId, RunResult, Segment,
|
AlertLevel, AlertSource, CommandEvent as ProtocolCommandEvent,
|
||||||
TurnResult, WorkerStatus,
|
CommandSnapshot as ProtocolCommandSnapshot, CommandStatus as ProtocolCommandStatus,
|
||||||
|
CommandStream as ProtocolCommandStream, CommandStreamSlice as ProtocolCommandStreamSlice,
|
||||||
|
ErrorCode, Event, Method, RewindTargetId, RunResult, Segment, TurnResult, WorkerStatus,
|
||||||
|
};
|
||||||
|
use workdir::{
|
||||||
|
CommandEvent as WorkdirCommandEvent, CommandSnapshot as WorkdirCommandSnapshot,
|
||||||
|
CommandStatus as WorkdirCommandStatus, CommandStream as WorkdirCommandStream, WorkdirSession,
|
||||||
};
|
};
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -424,6 +430,9 @@ impl WorkerController {
|
|||||||
Some(method_tx.downgrade()),
|
Some(method_tx.downgrade()),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
if let Some(session) = fs_for_view.as_ref() {
|
||||||
|
wire_workdir_command_events(session, &in_flight);
|
||||||
|
}
|
||||||
|
|
||||||
// Intake role Workers self-terminate only after a successful
|
// Intake role Workers self-terminate only after a successful
|
||||||
// TicketIntakeReady turn has fully settled back to Idle. The request
|
// TicketIntakeReady turn has fully settled back to Idle. The request
|
||||||
@@ -498,6 +507,105 @@ impl WorkerController {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn wire_workdir_command_events(
|
||||||
|
session: &Arc<dyn WorkdirSession>,
|
||||||
|
in_flight: &InFlightEvents,
|
||||||
|
) {
|
||||||
|
in_flight.replace_command_snapshot(
|
||||||
|
session
|
||||||
|
.command_snapshot()
|
||||||
|
.into_iter()
|
||||||
|
.map(protocol_command_snapshot)
|
||||||
|
.collect(),
|
||||||
|
);
|
||||||
|
let Some(mut events) = session.subscribe_command_events() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let in_flight = in_flight.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
match events.recv().await {
|
||||||
|
Ok(event) => in_flight.publish_command_event(protocol_command_event(event)),
|
||||||
|
Err(broadcast::error::RecvError::Lagged(_)) => {
|
||||||
|
// Never retain stale command output after a provider-local
|
||||||
|
// observer lag. The next chunk reconstructs a bounded tail
|
||||||
|
// with its absolute offset and marks the gap truncated.
|
||||||
|
in_flight.replace_command_snapshot(Vec::new());
|
||||||
|
}
|
||||||
|
Err(broadcast::error::RecvError::Closed) => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn protocol_command_snapshot(snapshot: WorkdirCommandSnapshot) -> ProtocolCommandSnapshot {
|
||||||
|
ProtocolCommandSnapshot {
|
||||||
|
command_id: snapshot.command_id,
|
||||||
|
tool_call_id: snapshot.tool_call_id,
|
||||||
|
status: protocol_command_status(snapshot.status),
|
||||||
|
stdout: ProtocolCommandStreamSlice {
|
||||||
|
start_offset: snapshot.stdout.start_offset,
|
||||||
|
end_offset: snapshot.stdout.end_offset,
|
||||||
|
content: snapshot.stdout.content,
|
||||||
|
truncated: snapshot.stdout.truncated,
|
||||||
|
},
|
||||||
|
stderr: ProtocolCommandStreamSlice {
|
||||||
|
start_offset: snapshot.stderr.start_offset,
|
||||||
|
end_offset: snapshot.stderr.end_offset,
|
||||||
|
content: snapshot.stderr.content,
|
||||||
|
truncated: snapshot.stderr.truncated,
|
||||||
|
},
|
||||||
|
exit_code: snapshot.exit_code,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn protocol_command_event(event: WorkdirCommandEvent) -> ProtocolCommandEvent {
|
||||||
|
match event {
|
||||||
|
WorkdirCommandEvent::Started {
|
||||||
|
command_id,
|
||||||
|
tool_call_id,
|
||||||
|
} => ProtocolCommandEvent::Started {
|
||||||
|
command_id,
|
||||||
|
tool_call_id,
|
||||||
|
},
|
||||||
|
WorkdirCommandEvent::Output {
|
||||||
|
command_id,
|
||||||
|
stream,
|
||||||
|
start_offset,
|
||||||
|
end_offset,
|
||||||
|
content,
|
||||||
|
} => ProtocolCommandEvent::Output {
|
||||||
|
command_id,
|
||||||
|
stream: match stream {
|
||||||
|
WorkdirCommandStream::Stdout => ProtocolCommandStream::Stdout,
|
||||||
|
WorkdirCommandStream::Stderr => ProtocolCommandStream::Stderr,
|
||||||
|
},
|
||||||
|
start_offset,
|
||||||
|
end_offset,
|
||||||
|
content,
|
||||||
|
},
|
||||||
|
WorkdirCommandEvent::Terminal {
|
||||||
|
command_id,
|
||||||
|
status,
|
||||||
|
exit_code,
|
||||||
|
} => ProtocolCommandEvent::Terminal {
|
||||||
|
command_id,
|
||||||
|
status: protocol_command_status(status),
|
||||||
|
exit_code,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn protocol_command_status(status: WorkdirCommandStatus) -> ProtocolCommandStatus {
|
||||||
|
match status {
|
||||||
|
WorkdirCommandStatus::Running => ProtocolCommandStatus::Running,
|
||||||
|
WorkdirCommandStatus::Completed => ProtocolCommandStatus::Completed,
|
||||||
|
WorkdirCommandStatus::Failed => ProtocolCommandStatus::Failed,
|
||||||
|
WorkdirCommandStatus::TimedOut => ProtocolCommandStatus::TimedOut,
|
||||||
|
WorkdirCommandStatus::Cancelled => ProtocolCommandStatus::Cancelled,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Wire the per-event broadcast bridges on the Worker's Engine. Each callback
|
/// Wire the per-event broadcast bridges on the Worker's Engine. Each callback
|
||||||
/// re-publishes a worker-level signal as a `protocol::Event` on `event_tx`
|
/// re-publishes a worker-level signal as a `protocol::Event` on `event_tx`
|
||||||
/// so subscribers (TUI, socket clients) get a single typed stream.
|
/// so subscribers (TUI, socket clients) get a single typed stream.
|
||||||
|
|||||||
@@ -1,9 +1,14 @@
|
|||||||
use std::sync::{Arc, Mutex, MutexGuard};
|
use std::sync::{Arc, Mutex, MutexGuard};
|
||||||
|
|
||||||
use protocol::{Event, InFlightBlock, InFlightSnapshot, InFlightToolCallState};
|
use protocol::{
|
||||||
|
CommandEvent, CommandSnapshot, CommandStatus, CommandStream, CommandStreamSlice, Event,
|
||||||
|
InFlightBlock, InFlightSnapshot, InFlightToolCallState,
|
||||||
|
};
|
||||||
use session_store::{LoggedContentPart, LoggedItem};
|
use session_store::{LoggedContentPart, LoggedItem};
|
||||||
use tokio::sync::broadcast;
|
use tokio::sync::broadcast;
|
||||||
|
|
||||||
|
const COMMAND_SNAPSHOT_STREAM_BYTES: usize = 32 * 1024;
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub struct InFlightBlockId(u64);
|
pub struct InFlightBlockId(u64);
|
||||||
|
|
||||||
@@ -17,6 +22,7 @@ pub struct InFlightEvents {
|
|||||||
pub(crate) struct InFlightInner {
|
pub(crate) struct InFlightInner {
|
||||||
next_block_id: u64,
|
next_block_id: u64,
|
||||||
blocks: Vec<TrackedBlock>,
|
blocks: Vec<TrackedBlock>,
|
||||||
|
commands: Vec<CommandSnapshot>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
@@ -46,6 +52,7 @@ impl InFlightEvents {
|
|||||||
inner: Arc::new(Mutex::new(InFlightInner {
|
inner: Arc::new(Mutex::new(InFlightInner {
|
||||||
next_block_id: 1,
|
next_block_id: 1,
|
||||||
blocks: Vec::new(),
|
blocks: Vec::new(),
|
||||||
|
commands: Vec::new(),
|
||||||
})),
|
})),
|
||||||
event_tx,
|
event_tx,
|
||||||
}
|
}
|
||||||
@@ -201,6 +208,15 @@ impl InFlightEvents {
|
|||||||
f()
|
f()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn publish_command_event(&self, event: CommandEvent) {
|
||||||
|
self.lock().apply_command_event(&event);
|
||||||
|
let _ = self.event_tx.send(Event::Command { event });
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn replace_command_snapshot(&self, commands: Vec<CommandSnapshot>) {
|
||||||
|
self.lock().commands = commands;
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn clear(&self) {
|
pub(crate) fn clear(&self) {
|
||||||
let mut inner = self.lock();
|
let mut inner = self.lock();
|
||||||
inner.clear();
|
inner.clear();
|
||||||
@@ -224,6 +240,82 @@ impl InFlightInner {
|
|||||||
.find(|block| block.block_id() == block_id)
|
.find(|block| block.block_id() == block_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn apply_command_event(&mut self, event: &CommandEvent) {
|
||||||
|
match event {
|
||||||
|
CommandEvent::Started {
|
||||||
|
command_id,
|
||||||
|
tool_call_id,
|
||||||
|
} => {
|
||||||
|
self.commands
|
||||||
|
.retain(|command| command.command_id != *command_id);
|
||||||
|
self.commands.push(CommandSnapshot {
|
||||||
|
command_id: command_id.clone(),
|
||||||
|
tool_call_id: tool_call_id.clone(),
|
||||||
|
status: CommandStatus::Running,
|
||||||
|
stdout: CommandStreamSlice::default(),
|
||||||
|
stderr: CommandStreamSlice::default(),
|
||||||
|
exit_code: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
CommandEvent::Output {
|
||||||
|
command_id,
|
||||||
|
stream,
|
||||||
|
start_offset,
|
||||||
|
end_offset,
|
||||||
|
content,
|
||||||
|
} => {
|
||||||
|
let command = match self
|
||||||
|
.commands
|
||||||
|
.iter_mut()
|
||||||
|
.find(|command| command.command_id == *command_id)
|
||||||
|
{
|
||||||
|
Some(command) => command,
|
||||||
|
None => {
|
||||||
|
self.commands.push(CommandSnapshot {
|
||||||
|
command_id: command_id.clone(),
|
||||||
|
tool_call_id: None,
|
||||||
|
status: CommandStatus::Running,
|
||||||
|
stdout: CommandStreamSlice::default(),
|
||||||
|
stderr: CommandStreamSlice::default(),
|
||||||
|
exit_code: None,
|
||||||
|
});
|
||||||
|
self.commands.last_mut().expect("command was inserted")
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let target = match stream {
|
||||||
|
CommandStream::Stdout => &mut command.stdout,
|
||||||
|
CommandStream::Stderr => &mut command.stderr,
|
||||||
|
};
|
||||||
|
if target.end_offset != *start_offset {
|
||||||
|
target.content.clear();
|
||||||
|
target.start_offset = *start_offset;
|
||||||
|
target.truncated = *start_offset > 0;
|
||||||
|
}
|
||||||
|
target.content.push_str(content);
|
||||||
|
target.end_offset = *end_offset;
|
||||||
|
if target.content.len() > COMMAND_SNAPSHOT_STREAM_BYTES {
|
||||||
|
let mut cut = target.content.len() - COMMAND_SNAPSHOT_STREAM_BYTES;
|
||||||
|
while cut < target.content.len() && !target.content.is_char_boundary(cut) {
|
||||||
|
cut += 1;
|
||||||
|
}
|
||||||
|
target.content.drain(..cut);
|
||||||
|
target.start_offset = target
|
||||||
|
.end_offset
|
||||||
|
.saturating_sub(target.content.len() as u64);
|
||||||
|
target.truncated = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
CommandEvent::Terminal { command_id, .. } => {
|
||||||
|
// Terminal state is delivered as a live protocol event. It is
|
||||||
|
// no longer in-flight snapshot state, and removing it here
|
||||||
|
// also prevents queued output from an aborted turn from
|
||||||
|
// surviving the subsequent terminal event after `clear()`.
|
||||||
|
self.commands
|
||||||
|
.retain(|command| command.command_id != *command_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn clear_for_committed_item(&mut self, item: &LoggedItem) {
|
fn clear_for_committed_item(&mut self, item: &LoggedItem) {
|
||||||
match item {
|
match item {
|
||||||
LoggedItem::Message { role, content }
|
LoggedItem::Message { role, content }
|
||||||
@@ -273,14 +365,16 @@ impl InFlightInner {
|
|||||||
.iter()
|
.iter()
|
||||||
.filter_map(TrackedBlock::to_snapshot_block)
|
.filter_map(TrackedBlock::to_snapshot_block)
|
||||||
.collect(),
|
.collect(),
|
||||||
|
commands: self.commands.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn clear(&mut self) -> bool {
|
fn clear(&mut self) -> bool {
|
||||||
if self.blocks.is_empty() {
|
if self.blocks.is_empty() && self.commands.is_empty() {
|
||||||
false
|
false
|
||||||
} else {
|
} else {
|
||||||
self.blocks.clear();
|
self.blocks.clear();
|
||||||
|
self.commands.clear();
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -583,6 +677,52 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn command_events_are_bounded_and_recoverable_from_snapshot() {
|
||||||
|
let (event_tx, _) = broadcast::channel(16);
|
||||||
|
let mut rx = event_tx.subscribe();
|
||||||
|
let in_flight = InFlightEvents::new(event_tx);
|
||||||
|
in_flight.publish_command_event(CommandEvent::Started {
|
||||||
|
command_id: "command-1".into(),
|
||||||
|
tool_call_id: Some("tool-1".into()),
|
||||||
|
});
|
||||||
|
in_flight.publish_command_event(CommandEvent::Output {
|
||||||
|
command_id: "command-1".into(),
|
||||||
|
stream: CommandStream::Stdout,
|
||||||
|
start_offset: 0,
|
||||||
|
end_offset: 5,
|
||||||
|
content: "ready".into(),
|
||||||
|
});
|
||||||
|
|
||||||
|
let guard = in_flight.snapshot_guard();
|
||||||
|
let snapshot = snapshot_from_guard(&guard);
|
||||||
|
assert_eq!(snapshot.commands.len(), 1);
|
||||||
|
assert_eq!(snapshot.commands[0].tool_call_id.as_deref(), Some("tool-1"));
|
||||||
|
assert_eq!(snapshot.commands[0].stdout.content, "ready");
|
||||||
|
assert_eq!(snapshot.commands[0].status, CommandStatus::Running);
|
||||||
|
drop(guard);
|
||||||
|
assert!(matches!(
|
||||||
|
rx.try_recv().unwrap(),
|
||||||
|
Event::Command {
|
||||||
|
event: CommandEvent::Started { .. }
|
||||||
|
}
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
rx.try_recv().unwrap(),
|
||||||
|
Event::Command {
|
||||||
|
event: CommandEvent::Output { .. }
|
||||||
|
}
|
||||||
|
));
|
||||||
|
|
||||||
|
in_flight.publish_command_event(CommandEvent::Terminal {
|
||||||
|
command_id: "command-1".into(),
|
||||||
|
status: CommandStatus::TimedOut,
|
||||||
|
exit_code: None,
|
||||||
|
});
|
||||||
|
let guard = in_flight.snapshot_guard();
|
||||||
|
assert!(snapshot_from_guard(&guard).commands.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn clear_discards_uncommitted_blocks_without_protocol_event() {
|
fn clear_discards_uncommitted_blocks_without_protocol_event() {
|
||||||
let (event_tx, _) = broadcast::channel(16);
|
let (event_tx, _) = broadcast::channel(16);
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ use session_store::{LogEntry, SegmentId, SessionId, Store, StoreError, TraceEntr
|
|||||||
use tokio::sync::broadcast;
|
use tokio::sync::broadcast;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::controller::wire_event_bridges_on_engine;
|
use crate::controller::{wire_event_bridges_on_engine, wire_workdir_command_events};
|
||||||
use crate::feature::FeatureRegistryBuilder;
|
use crate::feature::FeatureRegistryBuilder;
|
||||||
use crate::in_flight::{InFlightEvents, snapshot_from_guard};
|
use crate::in_flight::{InFlightEvents, snapshot_from_guard};
|
||||||
use crate::ipc::alerter::Alerter;
|
use crate::ipc::alerter::Alerter;
|
||||||
@@ -555,6 +555,9 @@ pub(crate) async fn prepare_internal_worker_session(
|
|||||||
spawn_internal_log_event_bridge(sink.clone(), event_tx.clone());
|
spawn_internal_log_event_bridge(sink.clone(), event_tx.clone());
|
||||||
let alerter = Alerter::new(event_tx.clone());
|
let alerter = Alerter::new(event_tx.clone());
|
||||||
let in_flight = InFlightEvents::new(event_tx.clone());
|
let in_flight = InFlightEvents::new(event_tx.clone());
|
||||||
|
if let Some(session) = worker.workdir_session() {
|
||||||
|
wire_workdir_command_events(session, &in_flight);
|
||||||
|
}
|
||||||
let actor_in_flight = in_flight.clone();
|
let actor_in_flight = in_flight.clone();
|
||||||
worker.attach_alerter(alerter.clone());
|
worker.attach_alerter(alerter.clone());
|
||||||
worker.attach_event_tx(event_tx.clone());
|
worker.attach_event_tx(event_tx.clone());
|
||||||
|
|||||||
@@ -12,8 +12,8 @@ use llm_engine::tool::{Tool, ToolDefinition, ToolError, ToolMeta, ToolOutput};
|
|||||||
use session_store::{CombinedStore, FsWorkerStore};
|
use session_store::{CombinedStore, FsWorkerStore};
|
||||||
use session_store::{FsStore, LogEntry};
|
use session_store::{FsStore, LogEntry};
|
||||||
use workdir::{
|
use workdir::{
|
||||||
CommandRequest, LocalWorkdirSession, Workdir, WorkdirError, WorkdirSessionCapabilities,
|
CommandOutputRequest, CommandRequest, LocalWorkdirSession, Workdir, WorkdirError,
|
||||||
WorkdirSessionHandle,
|
WorkdirSessionCapabilities, WorkdirSessionHandle,
|
||||||
};
|
};
|
||||||
|
|
||||||
use worker::{
|
use worker::{
|
||||||
@@ -232,6 +232,7 @@ async fn shutdown_closes_bound_workdir_session() {
|
|||||||
command: "sleep 30".to_owned(),
|
command: "sleep 30".to_owned(),
|
||||||
timeout_secs: 60,
|
timeout_secs: 60,
|
||||||
output_limit: 1024,
|
output_limit: 1024,
|
||||||
|
tool_call_id: None,
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -253,6 +254,108 @@ async fn shutdown_closes_bound_workdir_session() {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn controller_projects_workdir_command_events_and_snapshot_state() {
|
||||||
|
let (mut worker, pwd) = make_worker_with_pwd(MockClient::new(simple_text_events())).await;
|
||||||
|
let session: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::materialized_bound(
|
||||||
|
Workdir::new("controller-command-observation-workdir"),
|
||||||
|
pwd.clone(),
|
||||||
|
pwd,
|
||||||
|
worker.scope().clone(),
|
||||||
|
WorkdirSessionCapabilities::ALL,
|
||||||
|
));
|
||||||
|
worker.bind_workdir_session(Some(Arc::clone(&session)));
|
||||||
|
let handle = spawn_controller(worker).await;
|
||||||
|
let mut events = handle.subscribe();
|
||||||
|
|
||||||
|
let command = session
|
||||||
|
.start_command(CommandRequest {
|
||||||
|
command: "printf ready; sleep 0.3; printf done".to_owned(),
|
||||||
|
timeout_secs: 5,
|
||||||
|
output_limit: 1024,
|
||||||
|
tool_call_id: Some("tool-command-1".into()),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let mut saw_started = false;
|
||||||
|
let mut saw_output = false;
|
||||||
|
while !saw_output {
|
||||||
|
let event = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
|
||||||
|
.await
|
||||||
|
.expect("command event should arrive")
|
||||||
|
.unwrap();
|
||||||
|
match event {
|
||||||
|
Event::Command {
|
||||||
|
event:
|
||||||
|
protocol::CommandEvent::Started {
|
||||||
|
command_id,
|
||||||
|
tool_call_id,
|
||||||
|
},
|
||||||
|
} => {
|
||||||
|
assert_eq!(command_id, command.0);
|
||||||
|
assert_eq!(tool_call_id.as_deref(), Some("tool-command-1"));
|
||||||
|
saw_started = true;
|
||||||
|
}
|
||||||
|
Event::Command {
|
||||||
|
event:
|
||||||
|
protocol::CommandEvent::Output {
|
||||||
|
command_id,
|
||||||
|
stream: protocol::CommandStream::Stdout,
|
||||||
|
content,
|
||||||
|
..
|
||||||
|
},
|
||||||
|
} if command_id == command.0 && content.contains("ready") => saw_output = true,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert!(saw_started);
|
||||||
|
|
||||||
|
let Event::Snapshot { in_flight, .. } = handle.snapshot_event() else {
|
||||||
|
panic!("worker snapshot expected");
|
||||||
|
};
|
||||||
|
assert_eq!(in_flight.commands.len(), 1);
|
||||||
|
assert_eq!(in_flight.commands[0].command_id, command.0);
|
||||||
|
assert_eq!(in_flight.commands[0].stdout.content, "ready");
|
||||||
|
assert_eq!(
|
||||||
|
in_flight.commands[0].status,
|
||||||
|
protocol::CommandStatus::Running
|
||||||
|
);
|
||||||
|
|
||||||
|
let saw_terminal = drain_until(&mut events, std::time::Duration::from_secs(2), |event| {
|
||||||
|
matches!(
|
||||||
|
event,
|
||||||
|
Event::Command {
|
||||||
|
event: protocol::CommandEvent::Terminal {
|
||||||
|
command_id,
|
||||||
|
status: protocol::CommandStatus::Completed,
|
||||||
|
exit_code: Some(0),
|
||||||
|
}
|
||||||
|
} if command_id == &command.0
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
assert!(saw_terminal, "completed command event should arrive");
|
||||||
|
|
||||||
|
let output = session
|
||||||
|
.command_output(CommandOutputRequest {
|
||||||
|
handle: command,
|
||||||
|
cursor: 0,
|
||||||
|
limit: 1024,
|
||||||
|
wait: true,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(output.status, workdir::CommandStatus::Completed);
|
||||||
|
let (entries, _) = handle.sink.subscribe_with_snapshot();
|
||||||
|
let durable_history = serde_json::to_string(&entries).unwrap();
|
||||||
|
assert!(
|
||||||
|
!durable_history.contains("ready") && !durable_history.contains("done"),
|
||||||
|
"operational command chunks must not be appended to Worker history: {durable_history}"
|
||||||
|
);
|
||||||
|
handle.send(Method::Shutdown).await.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn controller_startup_failure_closes_bound_workdir_session() {
|
async fn controller_startup_failure_closes_bound_workdir_session() {
|
||||||
let (mut worker, pwd) = make_worker_with_pwd(MockClient::new(simple_text_events())).await;
|
let (mut worker, pwd) = make_worker_with_pwd(MockClient::new(simple_text_events())).await;
|
||||||
@@ -279,6 +382,7 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
|
|||||||
command: "printf unreachable".to_owned(),
|
command: "printf unreachable".to_owned(),
|
||||||
timeout_secs: 5,
|
timeout_secs: 5,
|
||||||
output_limit: 1024,
|
output_limit: 1024,
|
||||||
|
tool_call_id: None,
|
||||||
})
|
})
|
||||||
.await,
|
.await,
|
||||||
Err(WorkdirError::Unavailable(_))
|
Err(WorkdirError::Unavailable(_))
|
||||||
|
|||||||
@@ -22,6 +22,16 @@ export type Permission = "read" | "write";
|
|||||||
|
|
||||||
export type InFlightToolCallState = "pending" | "streaming_args" | "done";
|
export type InFlightToolCallState = "pending" | "streaming_args" | "done";
|
||||||
|
|
||||||
|
export type CommandStatus = "running" | "completed" | "failed" | "timed_out" | "cancelled";
|
||||||
|
|
||||||
|
export type CommandStream = "stdout" | "stderr";
|
||||||
|
|
||||||
|
export type CommandStreamSlice = { start_offset: number, end_offset: number, content: string, truncated: boolean, };
|
||||||
|
|
||||||
|
export type CommandSnapshot = { command_id: string, tool_call_id: string | null, status: CommandStatus, stdout: CommandStreamSlice, stderr: CommandStreamSlice, exit_code: number | null, };
|
||||||
|
|
||||||
|
export type CommandEvent = { "kind": "started", command_id: string, tool_call_id: string | null, } | { "kind": "output", command_id: string, stream: CommandStream, start_offset: number, end_offset: number, content: string, } | { "kind": "terminal", command_id: string, status: CommandStatus, exit_code: number | null, };
|
||||||
|
|
||||||
export type ScopeRule = {
|
export type ScopeRule = {
|
||||||
/**
|
/**
|
||||||
* Target path. Must be absolute by the time a `Scope` is built from
|
* Target path. Must be absolute by the time a `Scope` is built from
|
||||||
@@ -51,7 +61,7 @@ export type RewindSummary = { truncated_to_entries: number, discarded_entries: n
|
|||||||
|
|
||||||
export type InFlightBlock = { "kind": "text", text: string, finished?: boolean, } | { "kind": "thinking", text: string, finished?: boolean, } | { "kind": "tool_call", id: string, name: string, args: string, state?: InFlightToolCallState, };
|
export type InFlightBlock = { "kind": "text", text: string, finished?: boolean, } | { "kind": "thinking", text: string, finished?: boolean, } | { "kind": "tool_call", id: string, name: string, args: string, state?: InFlightToolCallState, };
|
||||||
|
|
||||||
export type InFlightSnapshot = { blocks?: Array<InFlightBlock>, };
|
export type InFlightSnapshot = { blocks?: Array<InFlightBlock>, commands?: Array<CommandSnapshot>, };
|
||||||
|
|
||||||
export type InternalWorkerKind = "sub_worker";
|
export type InternalWorkerKind = "sub_worker";
|
||||||
|
|
||||||
@@ -178,4 +188,4 @@ in_flight?: InFlightSnapshot,
|
|||||||
* Parent-owned Internal Worker sessions visible to this client.
|
* Parent-owned Internal Worker sessions visible to this client.
|
||||||
* Service-private Internal Workers are deliberately excluded.
|
* Service-private Internal Workers are deliberately excluded.
|
||||||
*/
|
*/
|
||||||
internal_workers?: Array<InternalWorkerSnapshot>, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "segment_rotated", "data": { entry: unknown, } } | { "event": "status", "data": { status: WorkerStatus, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array<CompletionEntry>, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array<RewindTarget>, } } | { "event": "rewind_applied", "data": { entries: Array<unknown>, input: Array<Segment>, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start" } | { "event": "compact_done", "data": { new_segment_id: string, } } | { "event": "compact_failed", "data": { error: string, } } | { "event": "shutdown" };
|
internal_workers?: Array<InternalWorkerSnapshot>, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "segment_rotated", "data": { entry: unknown, } } | { "event": "status", "data": { status: WorkerStatus, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array<CompletionEntry>, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array<RewindTarget>, } } | { "event": "rewind_applied", "data": { entries: Array<unknown>, input: Array<Segment>, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start" } | { "event": "compact_done", "data": { new_segment_id: string, } } | { "event": "compact_failed", "data": { error: string, } } | { "event": "shutdown" };
|
||||||
|
|||||||
@@ -334,6 +334,123 @@ Deno.test("projectConsole groups tool call lifecycle into one Call block", () =>
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Deno.test("projectConsole streams distinct Bash stdout and stderr through terminal status", () => {
|
||||||
|
const projection = projectConsole([
|
||||||
|
{
|
||||||
|
eventId: "command-tool",
|
||||||
|
event: {
|
||||||
|
event: "tool_call_done",
|
||||||
|
data: {
|
||||||
|
id: "bash-stream",
|
||||||
|
name: "Bash",
|
||||||
|
arguments: JSON.stringify({ command: "long-command" }),
|
||||||
|
},
|
||||||
|
} satisfies Event,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventId: "command-started",
|
||||||
|
event: {
|
||||||
|
event: "command",
|
||||||
|
data: {
|
||||||
|
event: {
|
||||||
|
kind: "started",
|
||||||
|
command_id: "command-1",
|
||||||
|
tool_call_id: "bash-stream",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} satisfies Event,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventId: "command-stdout",
|
||||||
|
event: {
|
||||||
|
event: "command",
|
||||||
|
data: {
|
||||||
|
event: {
|
||||||
|
kind: "output",
|
||||||
|
command_id: "command-1",
|
||||||
|
stream: "stdout",
|
||||||
|
start_offset: 0,
|
||||||
|
end_offset: 6,
|
||||||
|
content: "ready\n",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} satisfies Event,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventId: "command-stderr",
|
||||||
|
event: {
|
||||||
|
event: "command",
|
||||||
|
data: {
|
||||||
|
event: {
|
||||||
|
kind: "output",
|
||||||
|
command_id: "command-1",
|
||||||
|
stream: "stderr",
|
||||||
|
start_offset: 0,
|
||||||
|
end_offset: 5,
|
||||||
|
content: "warn\n",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} satisfies Event,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
eventId: "command-terminal",
|
||||||
|
event: {
|
||||||
|
event: "command",
|
||||||
|
data: {
|
||||||
|
event: {
|
||||||
|
kind: "terminal",
|
||||||
|
command_id: "command-1",
|
||||||
|
status: "failed",
|
||||||
|
exit_code: 7,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} satisfies Event,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const [line] = projection.lines.filter((line) => line.kind === "tool");
|
||||||
|
assert(line.body.includes("Bash — failed (exit 7)"), line.body);
|
||||||
|
assert(line.body.includes("stdout:\nready\n"), line.body);
|
||||||
|
assert(line.body.includes("stderr:\nwarn\n"), line.body);
|
||||||
|
assertEquals(line.streaming, false);
|
||||||
|
assertEquals(line.error, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
Deno.test("snapshot restores bounded in-flight Bash command output", () => {
|
||||||
|
const snapshot = snapshotEvent("/repo");
|
||||||
|
if (snapshot.event !== "snapshot") throw new Error("snapshot fixture expected");
|
||||||
|
snapshot.data.status = "running";
|
||||||
|
snapshot.data.in_flight = {
|
||||||
|
blocks: [{
|
||||||
|
kind: "tool_call",
|
||||||
|
id: "bash-snapshot",
|
||||||
|
name: "Bash",
|
||||||
|
args: JSON.stringify({ command: "slow" }),
|
||||||
|
state: "done",
|
||||||
|
}],
|
||||||
|
commands: [{
|
||||||
|
command_id: "command-2",
|
||||||
|
tool_call_id: "bash-snapshot",
|
||||||
|
status: "running",
|
||||||
|
stdout: {
|
||||||
|
start_offset: 1024,
|
||||||
|
end_offset: 1031,
|
||||||
|
content: "tail\n",
|
||||||
|
truncated: true,
|
||||||
|
},
|
||||||
|
stderr: { start_offset: 0, end_offset: 0, content: "", truncated: false },
|
||||||
|
exit_code: null,
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
|
||||||
|
const projection = projectConsole([{ eventId: "snapshot-command", event: snapshot }]);
|
||||||
|
const [line] = projection.lines.filter((line) => line.kind === "tool");
|
||||||
|
assert(line.body.includes("Bash — running…"), line.body);
|
||||||
|
assert(line.body.includes("[stdout tail; earlier output omitted]"), line.body);
|
||||||
|
assert(line.body.includes("stdout:\ntail\n"), line.body);
|
||||||
|
assertEquals(line.streaming, true);
|
||||||
|
});
|
||||||
|
|
||||||
Deno.test("projectConsole caps default tool request and result previews", () => {
|
Deno.test("projectConsole caps default tool request and result previews", () => {
|
||||||
const projection = projectConsole([
|
const projection = projectConsole([
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
import type {
|
import type {
|
||||||
Alert,
|
Alert,
|
||||||
|
CommandEvent,
|
||||||
|
CommandSnapshot,
|
||||||
|
CommandStreamSlice,
|
||||||
Event as ProtocolEvent,
|
Event as ProtocolEvent,
|
||||||
InFlightBlock,
|
InFlightBlock,
|
||||||
InFlightToolCallState,
|
InFlightToolCallState,
|
||||||
@@ -42,6 +45,7 @@ type ToolCallView = {
|
|||||||
output?: string | null;
|
output?: string | null;
|
||||||
isError?: boolean;
|
isError?: boolean;
|
||||||
cwd?: string | null;
|
cwd?: string | null;
|
||||||
|
command?: CommandSnapshot;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type ConsoleDiffLine = {
|
export type ConsoleDiffLine = {
|
||||||
@@ -239,6 +243,126 @@ function appendSnapshotInFlightLines(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const COMMAND_STREAM_DISPLAY_BYTES = 32 * 1024;
|
||||||
|
|
||||||
|
function appendSnapshotCommands(
|
||||||
|
projection: ConsoleProjection,
|
||||||
|
commands: CommandSnapshot[],
|
||||||
|
eventId: string,
|
||||||
|
): void {
|
||||||
|
commands.forEach((command) => upsertCommandSnapshot(projection, eventId, command));
|
||||||
|
}
|
||||||
|
|
||||||
|
function upsertCommandSnapshot(
|
||||||
|
projection: ConsoleProjection,
|
||||||
|
eventId: string,
|
||||||
|
command: CommandSnapshot,
|
||||||
|
): void {
|
||||||
|
const toolCallId = command.tool_call_id ?? `command:${command.command_id}`;
|
||||||
|
const existingIndex = findToolCallLineIndex(projection, toolCallId);
|
||||||
|
const existing = existingIndex >= 0
|
||||||
|
? projection.lines[existingIndex].toolCall
|
||||||
|
: undefined;
|
||||||
|
upsertToolCall(projection, eventId, toolCallId, {
|
||||||
|
name: existing?.name ?? "Bash",
|
||||||
|
state: existing?.state ?? "running",
|
||||||
|
command,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyCommandEvent(
|
||||||
|
projection: ConsoleProjection,
|
||||||
|
eventId: string,
|
||||||
|
event: CommandEvent,
|
||||||
|
): void {
|
||||||
|
if (event.kind === "started") {
|
||||||
|
upsertCommandSnapshot(projection, eventId, {
|
||||||
|
command_id: event.command_id,
|
||||||
|
tool_call_id: event.tool_call_id,
|
||||||
|
status: "running",
|
||||||
|
stdout: emptyCommandStream(),
|
||||||
|
stderr: emptyCommandStream(),
|
||||||
|
exit_code: null,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const index = projection.lines.findIndex((line) =>
|
||||||
|
line.toolCall?.command?.command_id === event.command_id
|
||||||
|
);
|
||||||
|
if (index < 0) {
|
||||||
|
if (event.kind === "output") {
|
||||||
|
const stream = commandStreamFromEvent(event);
|
||||||
|
upsertCommandSnapshot(projection, eventId, {
|
||||||
|
command_id: event.command_id,
|
||||||
|
tool_call_id: null,
|
||||||
|
status: "running",
|
||||||
|
stdout: event.stream === "stdout" ? stream : emptyCommandStream(),
|
||||||
|
stderr: event.stream === "stderr" ? stream : emptyCommandStream(),
|
||||||
|
exit_code: null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = projection.lines[index].toolCall!.command!;
|
||||||
|
if (event.kind === "terminal") {
|
||||||
|
upsertCommandSnapshot(projection, eventId, {
|
||||||
|
...existing,
|
||||||
|
status: event.status,
|
||||||
|
exit_code: event.exit_code,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const updatedStream = appendCommandStream(
|
||||||
|
event.stream === "stdout" ? existing.stdout : existing.stderr,
|
||||||
|
event.start_offset,
|
||||||
|
event.end_offset,
|
||||||
|
event.content,
|
||||||
|
);
|
||||||
|
upsertCommandSnapshot(projection, eventId, {
|
||||||
|
...existing,
|
||||||
|
stdout: event.stream === "stdout" ? updatedStream : existing.stdout,
|
||||||
|
stderr: event.stream === "stderr" ? updatedStream : existing.stderr,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function emptyCommandStream(): CommandStreamSlice {
|
||||||
|
return { start_offset: 0, end_offset: 0, content: "", truncated: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
function commandStreamFromEvent(
|
||||||
|
event: Extract<CommandEvent, { kind: "output" }>,
|
||||||
|
): CommandStreamSlice {
|
||||||
|
return appendCommandStream(
|
||||||
|
emptyCommandStream(),
|
||||||
|
event.start_offset,
|
||||||
|
event.end_offset,
|
||||||
|
event.content,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function appendCommandStream(
|
||||||
|
existing: CommandStreamSlice,
|
||||||
|
startOffset: number,
|
||||||
|
endOffset: number,
|
||||||
|
content: string,
|
||||||
|
): CommandStreamSlice {
|
||||||
|
if (endOffset <= existing.end_offset) return existing;
|
||||||
|
const contiguous = startOffset === existing.end_offset;
|
||||||
|
const combined = contiguous ? `${existing.content}${content}` : content;
|
||||||
|
const tail = combined.length > COMMAND_STREAM_DISPLAY_BYTES
|
||||||
|
? combined.slice(-COMMAND_STREAM_DISPLAY_BYTES)
|
||||||
|
: combined;
|
||||||
|
return {
|
||||||
|
start_offset: endOffset - tail.length,
|
||||||
|
end_offset: endOffset,
|
||||||
|
content: tail,
|
||||||
|
truncated: existing.truncated || !contiguous || tail.length < combined.length ||
|
||||||
|
startOffset > 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function projectInternalWorkerSnapshot(
|
function projectInternalWorkerSnapshot(
|
||||||
snapshot: InternalWorkerSnapshot,
|
snapshot: InternalWorkerSnapshot,
|
||||||
eventId: string,
|
eventId: string,
|
||||||
@@ -256,6 +380,11 @@ function projectInternalWorkerSnapshot(
|
|||||||
`${eventId}:internal:${snapshot.worker.session_id}:in-flight`,
|
`${eventId}:internal:${snapshot.worker.session_id}:in-flight`,
|
||||||
cwd,
|
cwd,
|
||||||
);
|
);
|
||||||
|
appendSnapshotCommands(
|
||||||
|
console,
|
||||||
|
snapshot.in_flight?.commands ?? [],
|
||||||
|
`${eventId}:internal:${snapshot.worker.session_id}:command`,
|
||||||
|
);
|
||||||
if (snapshot.error) {
|
if (snapshot.error) {
|
||||||
console.lines.push({
|
console.lines.push({
|
||||||
id: `${eventId}:internal:${snapshot.worker.session_id}:error`,
|
id: `${eventId}:internal:${snapshot.worker.session_id}:error`,
|
||||||
@@ -403,6 +532,11 @@ export function applyProtocolEvent(
|
|||||||
`${envelope.eventId}:snapshot-in-flight`,
|
`${envelope.eventId}:snapshot-in-flight`,
|
||||||
next.cwd,
|
next.cwd,
|
||||||
);
|
);
|
||||||
|
appendSnapshotCommands(
|
||||||
|
next,
|
||||||
|
event.data.in_flight?.commands ?? [],
|
||||||
|
`${envelope.eventId}:snapshot-command`,
|
||||||
|
);
|
||||||
next.internalWorkers = (event.data.internal_workers ?? []).map((worker) =>
|
next.internalWorkers = (event.data.internal_workers ?? []).map((worker) =>
|
||||||
projectInternalWorkerSnapshot(worker, envelope.eventId, next.cwd)
|
projectInternalWorkerSnapshot(worker, envelope.eventId, next.cwd)
|
||||||
);
|
);
|
||||||
@@ -436,6 +570,9 @@ export function applyProtocolEvent(
|
|||||||
case "status":
|
case "status":
|
||||||
next.status = event.data.status;
|
next.status = event.data.status;
|
||||||
break;
|
break;
|
||||||
|
case "command":
|
||||||
|
applyCommandEvent(next, envelope.eventId, event.data.event);
|
||||||
|
break;
|
||||||
case "segment_rotated": {
|
case "segment_rotated": {
|
||||||
const retainedErrors = next.lines.filter((line) => line.kind === "error");
|
const retainedErrors = next.lines.filter((line) => line.kind === "error");
|
||||||
const segment = snapshotProjectionFromEntries(
|
const segment = snapshotProjectionFromEntries(
|
||||||
@@ -786,6 +923,10 @@ function refreshedToolLine(item: ConsoleLine): ConsoleLine {
|
|||||||
if (!toolCall) {
|
if (!toolCall) {
|
||||||
return item;
|
return item;
|
||||||
}
|
}
|
||||||
|
const commandTerminal = toolCall.command !== undefined &&
|
||||||
|
toolCall.command.status !== "running";
|
||||||
|
const commandError = toolCall.command !== undefined &&
|
||||||
|
["failed", "timed_out", "cancelled"].includes(toolCall.command.status);
|
||||||
return {
|
return {
|
||||||
...item,
|
...item,
|
||||||
title: item.title.startsWith("Call · Tool result")
|
title: item.title.startsWith("Call · Tool result")
|
||||||
@@ -794,8 +935,8 @@ function refreshedToolLine(item: ConsoleLine): ConsoleLine {
|
|||||||
body: renderToolCall(toolCall),
|
body: renderToolCall(toolCall),
|
||||||
detail: toolCallDetail(toolCall),
|
detail: toolCallDetail(toolCall),
|
||||||
diff: toolCall.name === "Edit" ? editDiff(toolCall) : undefined,
|
diff: toolCall.name === "Edit" ? editDiff(toolCall) : undefined,
|
||||||
streaming: !["done", "error"].includes(toolCall.state),
|
streaming: !["done", "error"].includes(toolCall.state) && !commandTerminal,
|
||||||
error: toolCall.state === "error",
|
error: toolCall.state === "error" || commandError,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1040,9 +1181,37 @@ function renderBashTool(toolCall: ToolCallView): string {
|
|||||||
const args = parsedArgs(toolCall);
|
const args = parsedArgs(toolCall);
|
||||||
const command = stringField(args, "command");
|
const command = stringField(args, "command");
|
||||||
return compactLines([
|
return compactLines([
|
||||||
`Bash — ${stateSuffix(toolCall.state)}`,
|
`Bash — ${commandStateSuffix(toolCall)}`,
|
||||||
command ? `$ ${command}` : argsText(toolCall),
|
command ? `$ ${command}` : argsText(toolCall),
|
||||||
cappedDisplaySection(resultText(toolCall), 10),
|
["done", "error"].includes(toolCall.state)
|
||||||
|
? cappedDisplaySection(resultText(toolCall), 10)
|
||||||
|
: renderLiveCommandOutput(toolCall.command),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
function commandStateSuffix(toolCall: ToolCallView): string {
|
||||||
|
const command = toolCall.command;
|
||||||
|
if (!command) return stateSuffix(toolCall.state);
|
||||||
|
if (command.status === "completed") {
|
||||||
|
return command.exit_code === null
|
||||||
|
? "completed"
|
||||||
|
: `completed (exit ${command.exit_code})`;
|
||||||
|
}
|
||||||
|
if (command.status === "failed") {
|
||||||
|
return command.exit_code === null ? "failed" : `failed (exit ${command.exit_code})`;
|
||||||
|
}
|
||||||
|
if (command.status === "timed_out") return "timed out";
|
||||||
|
if (command.status === "cancelled") return "cancelled";
|
||||||
|
return "running…";
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderLiveCommandOutput(command?: CommandSnapshot): string | undefined {
|
||||||
|
if (!command) return undefined;
|
||||||
|
return compactLines([
|
||||||
|
command.stdout.truncated ? "[stdout tail; earlier output omitted]" : undefined,
|
||||||
|
command.stdout.content ? `stdout:\n${command.stdout.content}` : undefined,
|
||||||
|
command.stderr.truncated ? "[stderr tail; earlier output omitted]" : undefined,
|
||||||
|
command.stderr.content ? `stderr:\n${command.stderr.content}` : undefined,
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user