fix: preserve console alert order and improve alert colors
This commit is contained in:
@@ -1497,6 +1497,11 @@ pub enum Event {
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
||||
pub struct Alert {
|
||||
/// Stable only within the emitting live Worker; not durable alert history.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub position: Option<AlertPosition>,
|
||||
pub level: AlertLevel,
|
||||
pub source: AlertSource,
|
||||
pub message: String,
|
||||
@@ -1504,6 +1509,18 @@ pub struct Alert {
|
||||
pub timestamp_ms: i64,
|
||||
}
|
||||
|
||||
/// Display anchor captured at emission, resolved when pending text is committed.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum AlertPosition {
|
||||
Start,
|
||||
AfterEntry { entry_id: String },
|
||||
AfterToolCall { call_id: String },
|
||||
AfterInFlight { block_id: u64 },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
||||
@@ -1704,6 +1721,9 @@ pub struct InFlightCompaction {
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
||||
pub struct InFlightSnapshot {
|
||||
/// Live-Worker identities parallel to the blocks actually included below.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub block_ids: Vec<u64>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub blocks: Vec<InFlightBlock>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
@@ -2721,6 +2741,7 @@ mod tests {
|
||||
},
|
||||
state: WorkerStatus::Running.into(),
|
||||
in_flight: InFlightSnapshot {
|
||||
block_ids: Vec::new(),
|
||||
blocks: vec![
|
||||
InFlightBlock::Text {
|
||||
text: "hel".into(),
|
||||
@@ -3048,9 +3069,42 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alert_ordering_metadata_is_optional_and_uses_tagged_wire_anchors() {
|
||||
let legacy = serde_json::json!({ "level": "warn", "source": "worker", "message": "old", "timestamp_ms": 42 });
|
||||
let alert: Alert = serde_json::from_value(legacy.clone()).unwrap();
|
||||
assert!(alert.id.is_none());
|
||||
assert!(alert.position.is_none());
|
||||
assert_eq!(serde_json::to_value(alert).unwrap(), legacy);
|
||||
for position in [
|
||||
serde_json::json!({ "kind": "start" }),
|
||||
serde_json::json!({ "kind": "after_entry", "entry_id": "entry-1" }),
|
||||
serde_json::json!({ "kind": "after_tool_call", "call_id": "call-1" }),
|
||||
serde_json::json!({ "kind": "after_in_flight", "block_id": 7 }),
|
||||
] {
|
||||
let mut wire = legacy.clone();
|
||||
wire["id"] = serde_json::json!("alert-1");
|
||||
wire["position"] = position;
|
||||
let alert: Alert = serde_json::from_value(wire.clone()).unwrap();
|
||||
assert_eq!(serde_json::to_value(alert).unwrap(), wire);
|
||||
}
|
||||
let old_snapshot: InFlightSnapshot = serde_json::from_value(serde_json::json!({})).unwrap();
|
||||
assert!(old_snapshot.block_ids.is_empty());
|
||||
assert_eq!(
|
||||
serde_json::to_value(old_snapshot).unwrap(),
|
||||
serde_json::json!({})
|
||||
);
|
||||
let wire =
|
||||
serde_json::json!({"blocks": [{"kind": "text", "text": "pending"}], "block_ids": [7]});
|
||||
let snapshot: InFlightSnapshot = serde_json::from_value(wire.clone()).unwrap();
|
||||
assert_eq!(serde_json::to_value(snapshot).unwrap(), wire);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn event_alert_format() {
|
||||
let event = Event::Alert(Alert {
|
||||
id: None,
|
||||
position: None,
|
||||
level: AlertLevel::Warn,
|
||||
source: AlertSource::Compactor,
|
||||
message: "compaction failed".into(),
|
||||
|
||||
@@ -4,19 +4,19 @@ use serde_json::{Map, Value};
|
||||
use ts_rs::{Config, TS};
|
||||
|
||||
use crate::{
|
||||
Alert, AlertLevel, AlertSource, CommandEvent, CommandSnapshot, CommandStatus, CommandStream,
|
||||
CommandStreamSlice, CompactionLifecycle, CompactionLifecycleState, CompactionPhase,
|
||||
CompactionTrigger, CompletionContext, CompletionEntry, CompletionKind, ContextTokenSource,
|
||||
ContextUsage, ErrorCode, Event, FeatureInvocation, FeatureInvocationDescriptor,
|
||||
FeatureInvocationIdentity, FeatureInvocationResult, FeatureInvocationStatus,
|
||||
FeatureInvocationSyntax, Greeting, InFlightBlock, InFlightCompaction, InFlightSnapshot,
|
||||
InFlightToolCallState, InternalWorkerKind, InternalWorkerRef, InternalWorkerSnapshot,
|
||||
InvocationArgumentDescriptor, InvocationArgumentType, InvocationArgumentValue,
|
||||
InvocationClientAdapter, InvocationCompletion, InvocationValue, InvokeKind, MemoryWorkerEvent,
|
||||
Method, PasteArtifactAvailability, PasteArtifactMediaType, PasteArtifactRef,
|
||||
PendingSubmissionSummary, PendingSubmissionsSnapshot, Permission, ReasoningConfig,
|
||||
RewindSummary, RunFailureKind, RunResult, RunResumeSource, RunYieldReason, ScopeRule, Segment,
|
||||
SessionContentPart, SessionConversationTurn, SessionEntryProvenance,
|
||||
Alert, AlertLevel, AlertPosition, AlertSource, CommandEvent, CommandSnapshot, CommandStatus,
|
||||
CommandStream, CommandStreamSlice, CompactionLifecycle, CompactionLifecycleState,
|
||||
CompactionPhase, CompactionTrigger, CompletionContext, CompletionEntry, CompletionKind,
|
||||
ContextTokenSource, ContextUsage, ErrorCode, Event, FeatureInvocation,
|
||||
FeatureInvocationDescriptor, FeatureInvocationIdentity, FeatureInvocationResult,
|
||||
FeatureInvocationStatus, FeatureInvocationSyntax, Greeting, InFlightBlock, InFlightCompaction,
|
||||
InFlightSnapshot, InFlightToolCallState, InternalWorkerKind, InternalWorkerRef,
|
||||
InternalWorkerSnapshot, InvocationArgumentDescriptor, InvocationArgumentType,
|
||||
InvocationArgumentValue, InvocationClientAdapter, InvocationCompletion, InvocationValue,
|
||||
InvokeKind, MemoryWorkerEvent, Method, PasteArtifactAvailability, PasteArtifactMediaType,
|
||||
PasteArtifactRef, PendingSubmissionSummary, PendingSubmissionsSnapshot, Permission,
|
||||
ReasoningConfig, RewindSummary, RunFailureKind, RunResult, RunResumeSource, RunYieldReason,
|
||||
ScopeRule, Segment, SessionContentPart, SessionConversationTurn, SessionEntryProvenance,
|
||||
SessionHistoryLineageBoundary, SessionHistoryPage, SessionMessageRole, SessionSnapshot,
|
||||
SessionSnapshotEntry, SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition,
|
||||
SymlinkPolicy, ToolResultDisposition, TurnResult, UploadedFileAvailability, UploadedFileRef,
|
||||
@@ -65,6 +65,7 @@ pub fn generated_protocol_types() -> String {
|
||||
);
|
||||
|
||||
push_decl::<AlertLevel>(&cfg, &mut output);
|
||||
push_decl::<AlertPosition>(&cfg, &mut output);
|
||||
push_decl::<AlertSource>(&cfg, &mut output);
|
||||
push_decl::<CompletionKind>(&cfg, &mut output);
|
||||
push_decl::<CompletionContext>(&cfg, &mut output);
|
||||
|
||||
@@ -619,6 +619,11 @@ async fn run_protocol_session(
|
||||
event = events.recv() => {
|
||||
match event {
|
||||
Ok(event) => {
|
||||
let Ok(event) = handle.alerter.freshen_event(event) else {
|
||||
// The retained anchor was evicted; reconnect must obtain
|
||||
// a paired session/Alert snapshot rather than a stale row.
|
||||
return;
|
||||
};
|
||||
if !send_protocol_event(&peer, event).await {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -4319,6 +4319,8 @@ mod completion_flow_tests {
|
||||
let mut app = App::new("test".into());
|
||||
let message = "mid-run compaction failed: summary unavailable";
|
||||
app.handle_worker_event(Event::Alert(protocol::Alert {
|
||||
id: None,
|
||||
position: None,
|
||||
level: AlertLevel::Error,
|
||||
source: AlertSource::Compactor,
|
||||
message: message.into(),
|
||||
@@ -4402,6 +4404,7 @@ mod completion_flow_tests {
|
||||
},
|
||||
state: test_worker_state(WorkerStatus::Running),
|
||||
in_flight: InFlightSnapshot {
|
||||
block_ids: Vec::new(),
|
||||
blocks: vec![
|
||||
InFlightBlock::Thinking {
|
||||
text: "why".into(),
|
||||
|
||||
@@ -4821,6 +4821,7 @@ mod ws_tests {
|
||||
},
|
||||
state: protocol::WorkerStateSnapshot::initial(),
|
||||
in_flight: protocol::InFlightSnapshot {
|
||||
block_ids: Vec::new(),
|
||||
blocks: Vec::new(),
|
||||
commands: Vec::new(),
|
||||
compaction: None,
|
||||
|
||||
@@ -11818,6 +11818,7 @@ mod tests {
|
||||
},
|
||||
state: protocol::WorkerStatus::Running.into(),
|
||||
in_flight: protocol::InFlightSnapshot {
|
||||
block_ids: Vec::new(),
|
||||
blocks: Vec::new(),
|
||||
commands: Vec::new(),
|
||||
compaction: None,
|
||||
|
||||
@@ -2901,12 +2901,33 @@ where
|
||||
let mut events = streams.events;
|
||||
let mut entry_events = streams.log_entries;
|
||||
let bridge_worker_state = worker_state.clone();
|
||||
let bridge_handle = handle.clone();
|
||||
let bridge_task = match self.spawn_on_adapter_runtime(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = events.recv() => {
|
||||
match event {
|
||||
Ok(mut event) => {
|
||||
Ok(event) => {
|
||||
let mut event = match bridge_handle.alerter.freshen_event(event) {
|
||||
Ok(event) => event,
|
||||
Err(_) => {
|
||||
// This shared bridge also carries Worker state: do
|
||||
// not terminate it when a bounded Alert was evicted.
|
||||
// Replace both receivers with the paired snapshot.
|
||||
let replacement = subscribe_worker_protocol_session(&bridge_handle);
|
||||
events = replacement.events;
|
||||
entry_events = replacement.log_entries;
|
||||
let mut snapshot = replacement.snapshot_event;
|
||||
if apply_protocol_worker_state(&bridge_worker_state, &mut snapshot).is_err() {
|
||||
break;
|
||||
}
|
||||
let _ = bridge_context.publish_protocol_event(snapshot);
|
||||
for alert in replacement.alert_snapshot {
|
||||
let _ = bridge_context.publish_protocol_event(Event::Alert(alert));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
match apply_protocol_worker_state(&bridge_worker_state, &mut event) {
|
||||
Ok(true) => {
|
||||
let _ = bridge_context.publish_protocol_event(event);
|
||||
@@ -4525,6 +4546,10 @@ where
|
||||
},
|
||||
};
|
||||
if let Some(event) = event {
|
||||
let Ok(event) = relay_handle.alerter.freshen_event(event) else {
|
||||
// An evicted Alert needs a fresh paired snapshot.
|
||||
break;
|
||||
};
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = relay_handle.protocol_closed() => break,
|
||||
@@ -5177,6 +5202,71 @@ mod tests {
|
||||
assert!(backend.stop_worker(&request.worker_ref).is_accepted());
|
||||
}
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn protocol_transport_reconnects_instead_of_forwarding_an_evicted_alert() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let runtime_store = root.path().join("runtime");
|
||||
let backend = Arc::new(
|
||||
WorkerRuntimeExecutionBackend::new(
|
||||
ProfileRuntimeWorkerFactory::new(root.path())
|
||||
.with_runtime_store_dir(&runtime_store)
|
||||
.with_runtime_id("runtime-subjektiv-test"),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
let request = backend
|
||||
.run_on_adapter_runtime(async move {
|
||||
Ok(saved_subjektiv_restore_fixture(&runtime_store, true).await)
|
||||
})
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
backend.restore_worker(request.clone()),
|
||||
WorkerExecutionSpawnResult::Connected { .. }
|
||||
));
|
||||
let execution = backend.workers.lock().unwrap()[&request.worker_ref].clone();
|
||||
let mut transport = Arc::clone(&backend)
|
||||
.attach_worker_protocol(&request.worker_ref)
|
||||
.unwrap();
|
||||
// A queued frame whose ID has left the bounded buffer must not be
|
||||
// forwarded with its now-unresolvable pending-text position.
|
||||
execution
|
||||
.handle
|
||||
.send_event(Event::Alert(protocol::Alert {
|
||||
id: Some("evicted-alert".into()),
|
||||
position: Some(protocol::AlertPosition::AfterInFlight { block_id: 99 }),
|
||||
level: protocol::AlertLevel::Warn,
|
||||
source: protocol::AlertSource::Worker,
|
||||
message: "stale alert".into(),
|
||||
timestamp_ms: 1,
|
||||
}))
|
||||
.unwrap();
|
||||
backend.run_on_adapter_runtime(async move {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while let Some(event) = transport.events.recv().await {
|
||||
assert!(!matches!(event, Event::Alert(alert) if alert.id.as_deref() == Some("evicted-alert")));
|
||||
}
|
||||
}).await.map_err(|_| "stale Alert did not close its transport".to_string())?;
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
execution.handle.alerter.alert(
|
||||
protocol::AlertLevel::Warn,
|
||||
protocol::AlertSource::Worker,
|
||||
"retained alert".into(),
|
||||
);
|
||||
let mut replacement = Arc::clone(&backend)
|
||||
.attach_worker_protocol(&request.worker_ref)
|
||||
.unwrap();
|
||||
backend.run_on_adapter_runtime(async move {
|
||||
let event = tokio::time::timeout(Duration::from_secs(5), replacement.events.recv())
|
||||
.await.map_err(|_| "replacement Alert replay timed out".to_string())?;
|
||||
assert!(matches!(event, Some(Event::Alert(alert)) if alert.message == "retained alert" && alert.id.is_some()));
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
assert!(backend.stop_worker(&request.worker_ref).is_accepted());
|
||||
}
|
||||
|
||||
fn profile_stop_fixture(
|
||||
root: &Path,
|
||||
) -> (WorkerRuntimeExecutionBackend, WorkerExecutionRestoreRequest) {
|
||||
|
||||
@@ -1532,6 +1532,8 @@ mod tests {
|
||||
let mut writer = JsonLineWriter::new(writer_half);
|
||||
writer
|
||||
.write(&Event::Alert(Alert {
|
||||
id: None,
|
||||
position: None,
|
||||
level: AlertLevel::Warn,
|
||||
source: AlertSource::Worker,
|
||||
message: "connect-time alert".into(),
|
||||
@@ -1763,6 +1765,8 @@ mod tests {
|
||||
let mut writer = JsonLineWriter::new(stream);
|
||||
writer
|
||||
.write(&Event::Alert(Alert {
|
||||
id: None,
|
||||
position: None,
|
||||
level: AlertLevel::Warn,
|
||||
source: AlertSource::Worker,
|
||||
message: "replayed alert".into(),
|
||||
|
||||
+142
-14
@@ -1,11 +1,11 @@
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
sync::{Arc, Mutex, MutexGuard},
|
||||
sync::{Arc, Mutex, MutexGuard, Weak},
|
||||
};
|
||||
|
||||
use protocol::{
|
||||
CommandEvent, CommandSnapshot, CommandStatus, CommandStream, CommandStreamSlice, Event,
|
||||
InFlightBlock, InFlightSnapshot, InFlightToolCallState,
|
||||
AlertPosition, CommandEvent, CommandSnapshot, CommandStatus, CommandStream, CommandStreamSlice,
|
||||
Event, InFlightBlock, InFlightSnapshot, InFlightToolCallState,
|
||||
};
|
||||
use session_store::{LoggedContentPart, LoggedItem};
|
||||
use tokio::sync::broadcast;
|
||||
@@ -38,6 +38,7 @@ enum TrackedBlock {
|
||||
block_id: InFlightBlockId,
|
||||
text: String,
|
||||
finished: bool,
|
||||
alert_positions: Vec<Weak<Mutex<AlertPosition>>>,
|
||||
},
|
||||
Thinking {
|
||||
block_id: InFlightBlockId,
|
||||
@@ -78,6 +79,7 @@ impl InFlightEvents {
|
||||
block_id,
|
||||
text: String::new(),
|
||||
finished: false,
|
||||
alert_positions: Vec::new(),
|
||||
});
|
||||
block_id
|
||||
}
|
||||
@@ -213,10 +215,11 @@ impl InFlightEvents {
|
||||
pub(crate) fn clear_for_committed_item_then<R>(
|
||||
&self,
|
||||
item: &LoggedItem,
|
||||
entry_id: Option<&str>,
|
||||
f: impl FnOnce() -> R,
|
||||
) -> R {
|
||||
let mut inner = self.lock();
|
||||
inner.clear_for_committed_item(item);
|
||||
inner.clear_for_committed_item(item, entry_id);
|
||||
f()
|
||||
}
|
||||
|
||||
@@ -378,7 +381,7 @@ impl InFlightInner {
|
||||
}
|
||||
}
|
||||
|
||||
fn clear_for_committed_item(&mut self, item: &LoggedItem) {
|
||||
fn clear_for_committed_item(&mut self, item: &LoggedItem, entry_id: Option<&str>) {
|
||||
match item {
|
||||
LoggedItem::Message { role, content }
|
||||
if matches!(role, session_store::LoggedRole::Assistant) =>
|
||||
@@ -391,7 +394,7 @@ impl InFlightInner {
|
||||
})
|
||||
.collect::<String>();
|
||||
if !text.is_empty() {
|
||||
self.remove_first_text_matching(&text);
|
||||
self.remove_first_text_matching(&text, entry_id);
|
||||
}
|
||||
}
|
||||
LoggedItem::Reasoning {
|
||||
@@ -420,13 +423,55 @@ impl InFlightInner {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn capture_alert_position(&mut self) -> Option<Arc<Mutex<AlertPosition>>> {
|
||||
// Prune evicted alerts even when their streamed block has not committed yet.
|
||||
for block in &mut self.blocks {
|
||||
if let TrackedBlock::Text {
|
||||
alert_positions, ..
|
||||
} = block
|
||||
{
|
||||
alert_positions.retain(|position| position.strong_count() > 0);
|
||||
}
|
||||
}
|
||||
for block in self.blocks.iter_mut().rev() {
|
||||
match block {
|
||||
TrackedBlock::Text {
|
||||
block_id,
|
||||
text,
|
||||
alert_positions,
|
||||
..
|
||||
} if !text.is_empty() => {
|
||||
let position = Arc::new(Mutex::new(AlertPosition::AfterInFlight {
|
||||
block_id: block_id.0,
|
||||
}));
|
||||
alert_positions.push(Arc::downgrade(&position));
|
||||
return Some(position);
|
||||
}
|
||||
TrackedBlock::ToolCall { id, .. } => {
|
||||
return Some(Arc::new(Mutex::new(AlertPosition::AfterToolCall {
|
||||
call_id: id.clone(),
|
||||
})));
|
||||
}
|
||||
// Hidden reasoning and empty text are not conversation rows.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> InFlightSnapshot {
|
||||
let (block_ids, blocks) = self
|
||||
.blocks
|
||||
.iter()
|
||||
.filter_map(|block| {
|
||||
block
|
||||
.to_snapshot_block()
|
||||
.map(|snapshot| (block.block_id().0, snapshot))
|
||||
})
|
||||
.unzip();
|
||||
InFlightSnapshot {
|
||||
blocks: self
|
||||
.blocks
|
||||
.iter()
|
||||
.filter_map(TrackedBlock::to_snapshot_block)
|
||||
.collect(),
|
||||
block_ids,
|
||||
blocks,
|
||||
commands: self.commands.clone(),
|
||||
compaction: self.compaction.clone(),
|
||||
}
|
||||
@@ -444,11 +489,25 @@ impl InFlightInner {
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_first_text_matching(&mut self, committed: &str) -> bool {
|
||||
fn remove_first_text_matching(&mut self, committed: &str, entry_id: Option<&str>) -> bool {
|
||||
if let Some(index) = self.blocks.iter().position(|block| match block {
|
||||
TrackedBlock::Text { text, .. } => text == committed,
|
||||
_ => false,
|
||||
}) {
|
||||
if let (
|
||||
TrackedBlock::Text {
|
||||
alert_positions, ..
|
||||
},
|
||||
Some(entry_id),
|
||||
) = (&self.blocks[index], entry_id)
|
||||
{
|
||||
for position in alert_positions.iter().filter_map(Weak::upgrade) {
|
||||
*position.lock().expect("alert position mutex poisoned") =
|
||||
AlertPosition::AfterEntry {
|
||||
entry_id: entry_id.to_owned(),
|
||||
};
|
||||
}
|
||||
}
|
||||
self.blocks.remove(index);
|
||||
true
|
||||
} else {
|
||||
@@ -559,6 +618,72 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alert_watchers_are_pruned_and_removed_with_their_text_block() {
|
||||
let (tx, _) = broadcast::channel(16);
|
||||
let in_flight = InFlightEvents::new(tx);
|
||||
let text = in_flight.start_text_block();
|
||||
in_flight.text_delta(text, "same".into());
|
||||
let first = in_flight.snapshot_guard().capture_alert_position().unwrap();
|
||||
drop(first);
|
||||
let second = in_flight.snapshot_guard().capture_alert_position().unwrap();
|
||||
{
|
||||
let guard = in_flight.snapshot_guard();
|
||||
let TrackedBlock::Text {
|
||||
alert_positions, ..
|
||||
} = &guard.blocks[0]
|
||||
else {
|
||||
panic!("expected text")
|
||||
};
|
||||
assert_eq!(alert_positions.len(), 1);
|
||||
}
|
||||
let later = in_flight.start_text_block();
|
||||
in_flight.text_delta(later, "same".into());
|
||||
let third = in_flight.snapshot_guard().capture_alert_position().unwrap();
|
||||
let item = LoggedItem::Message {
|
||||
role: session_store::LoggedRole::Assistant,
|
||||
content: vec![LoggedContentPart::Text {
|
||||
text: "same".into(),
|
||||
}],
|
||||
};
|
||||
in_flight.clear_for_committed_item_then(&item, Some("first-entry"), || ());
|
||||
assert_eq!(
|
||||
*second.lock().unwrap(),
|
||||
AlertPosition::AfterEntry {
|
||||
entry_id: "first-entry".into()
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
*third.lock().unwrap(),
|
||||
AlertPosition::AfterInFlight { block_id: later.0 }
|
||||
);
|
||||
in_flight.clear();
|
||||
assert!(in_flight.snapshot_guard().blocks.is_empty());
|
||||
assert_eq!(
|
||||
*third.lock().unwrap(),
|
||||
AlertPosition::AfterInFlight { block_id: later.0 }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_ids_are_parallel_to_emitted_blocks_not_filtered_placeholders() {
|
||||
let (tx, _) = broadcast::channel(16);
|
||||
let in_flight = InFlightEvents::new(tx);
|
||||
in_flight.start_text_block();
|
||||
let hidden = in_flight.thinking_start();
|
||||
in_flight.thinking_done(hidden, String::new());
|
||||
let visible = in_flight.start_text_block();
|
||||
in_flight.text_delta(visible, "shown".into());
|
||||
let tool = in_flight.tool_call_start("call-1".into(), "Read".into());
|
||||
let snapshot = snapshot_from_guard(&in_flight.snapshot_guard());
|
||||
assert_eq!(snapshot.blocks.len(), 2);
|
||||
assert_eq!(snapshot.block_ids, vec![visible.0, tool.0]);
|
||||
assert_eq!(
|
||||
snapshot.block_ids,
|
||||
snapshot_from_guard(&in_flight.snapshot_guard()).block_ids
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_boundary_does_not_duplicate_or_gap_delta_sent_after_subscribe() {
|
||||
let (working_event_tx, _) = broadcast::channel(16);
|
||||
@@ -621,7 +746,7 @@ mod tests {
|
||||
// This mirrors Worker::append_entry ordering: clear in-flight first,
|
||||
// then publish the finalized AssistantItem. Identity-aware clients
|
||||
// reconcile this durable commit with the preceding streamed block.
|
||||
in_flight_for_commit.clear_for_committed_item_then(&assistant_item, || {
|
||||
in_flight_for_commit.clear_for_committed_item_then(&assistant_item, None, || {
|
||||
sink_for_commit.publish(assistant_entry);
|
||||
});
|
||||
committed_tx.send(()).unwrap();
|
||||
@@ -670,7 +795,7 @@ mod tests {
|
||||
entry: crate::session_history::test_logged_history_entry(assistant_item.clone()),
|
||||
};
|
||||
|
||||
in_flight.clear_for_committed_item_then(&assistant_item, || {
|
||||
in_flight.clear_for_committed_item_then(&assistant_item, None, || {
|
||||
sink.publish(assistant_entry);
|
||||
});
|
||||
|
||||
@@ -698,6 +823,7 @@ mod tests {
|
||||
text: "done".into(),
|
||||
}],
|
||||
},
|
||||
None,
|
||||
|| (),
|
||||
);
|
||||
|
||||
@@ -723,6 +849,7 @@ mod tests {
|
||||
encrypted_content: Some("opaque".into()),
|
||||
signature: None,
|
||||
},
|
||||
None,
|
||||
|| (),
|
||||
);
|
||||
|
||||
@@ -746,6 +873,7 @@ mod tests {
|
||||
encrypted_content: Some("opaque".into()),
|
||||
signature: None,
|
||||
},
|
||||
None,
|
||||
|| (),
|
||||
);
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
//!
|
||||
//! Each alert is broadcast on the shared `Event` channel and
|
||||
//! also appended to an in-memory buffer so that clients connecting
|
||||
//! after the fact still see everything emitted during the session.
|
||||
//! after the fact can replay the bounded recent prefix from the same live Worker.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::{Arc, Mutex};
|
||||
@@ -14,7 +14,9 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
use protocol::{Alert, AlertLevel, AlertSource, Event};
|
||||
use protocol::{Alert, AlertLevel, AlertPosition, AlertSource, Event};
|
||||
|
||||
use crate::{in_flight::InFlightEvents, segment_log_sink::SegmentLogSink};
|
||||
|
||||
/// Upper bound on buffered alerts. When exceeded, the oldest
|
||||
/// entries are discarded so a long-running session cannot leak
|
||||
@@ -27,9 +29,53 @@ pub struct Alerter {
|
||||
inner: Arc<Inner>,
|
||||
}
|
||||
|
||||
/// A queued identified Alert is no longer retained by this live Worker.
|
||||
/// Transports must disconnect/resnapshot instead of forwarding its stale anchor.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
|
||||
#[error("alert is no longer retained; reconnect for a fresh snapshot")]
|
||||
pub struct AlertReplayRequired;
|
||||
|
||||
struct Inner {
|
||||
working_event_tx: broadcast::Sender<Event>,
|
||||
buffer: Mutex<VecDeque<Alert>>,
|
||||
buffer: Mutex<VecDeque<BufferedAlert>>,
|
||||
position_source: Mutex<Option<PositionSource>>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct PositionSource {
|
||||
sink: SegmentLogSink,
|
||||
in_flight: Option<InFlightEvents>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct BufferedAlert {
|
||||
alert: Alert,
|
||||
position: Option<Arc<Mutex<AlertPosition>>>,
|
||||
}
|
||||
|
||||
impl BufferedAlert {
|
||||
fn snapshot(&self) -> Alert {
|
||||
let mut alert = self.alert.clone();
|
||||
alert.position = self.position.as_ref().map(|position| {
|
||||
position
|
||||
.lock()
|
||||
.expect("alert position mutex poisoned")
|
||||
.clone()
|
||||
});
|
||||
alert
|
||||
}
|
||||
}
|
||||
|
||||
impl PositionSource {
|
||||
fn capture(&self) -> Arc<Mutex<AlertPosition>> {
|
||||
// Same order as commit/snapshot: in-flight -> sink. Never hold the
|
||||
// alerter buffer lock while acquiring either of these locks.
|
||||
let mut in_flight = self.in_flight.as_ref().map(InFlightEvents::snapshot_guard);
|
||||
in_flight
|
||||
.as_mut()
|
||||
.and_then(|guard| guard.capture_alert_position())
|
||||
.unwrap_or_else(|| Arc::new(Mutex::new(self.sink.alert_position())))
|
||||
}
|
||||
}
|
||||
|
||||
impl Alerter {
|
||||
@@ -38,10 +84,23 @@ impl Alerter {
|
||||
inner: Arc::new(Inner {
|
||||
working_event_tx,
|
||||
buffer: Mutex::new(VecDeque::with_capacity(MAX_BUFFERED_ALERTS)),
|
||||
position_source: Mutex::new(None),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn attach_position_source(
|
||||
&self,
|
||||
sink: SegmentLogSink,
|
||||
in_flight: Option<InFlightEvents>,
|
||||
) {
|
||||
*self
|
||||
.inner
|
||||
.position_source
|
||||
.lock()
|
||||
.expect("alert source mutex poisoned") = Some(PositionSource { sink, in_flight });
|
||||
}
|
||||
|
||||
/// Record and broadcast an alert.
|
||||
///
|
||||
/// The broadcast may have no subscribers (e.g. during Worker
|
||||
@@ -55,35 +114,88 @@ impl Alerter {
|
||||
/// exactly once: older ones from the snapshot, newer ones from
|
||||
/// the freshly-subscribed receiver.
|
||||
pub fn alert(&self, level: AlertLevel, source: AlertSource, message: String) {
|
||||
self.alert_at(level, source, message, now_ms());
|
||||
}
|
||||
|
||||
fn alert_at(&self, level: AlertLevel, source: AlertSource, message: String, timestamp_ms: i64) {
|
||||
let source_handle = self
|
||||
.inner
|
||||
.position_source
|
||||
.lock()
|
||||
.expect("alert source mutex poisoned")
|
||||
.clone();
|
||||
let position = source_handle.map(|source| source.capture());
|
||||
let alert = Alert {
|
||||
id: Some(uuid::Uuid::now_v7().to_string()),
|
||||
position: None,
|
||||
level,
|
||||
source,
|
||||
message,
|
||||
timestamp_ms: now_ms(),
|
||||
timestamp_ms,
|
||||
};
|
||||
if let Ok(mut buf) = self.inner.buffer.lock() {
|
||||
if buf.len() >= MAX_BUFFERED_ALERTS {
|
||||
buf.pop_front();
|
||||
}
|
||||
buf.push_back(alert.clone());
|
||||
let _ = self.inner.working_event_tx.send(Event::Alert(alert));
|
||||
let buffered = BufferedAlert { alert, position };
|
||||
let event = Event::Alert(buffered.snapshot());
|
||||
buf.push_back(buffered);
|
||||
let _ = self.inner.working_event_tx.send(event);
|
||||
}
|
||||
}
|
||||
|
||||
/// Refresh a queued Alert immediately before transport forwarding.
|
||||
///
|
||||
/// The independent log lane may have already delivered the text commit, so
|
||||
/// the broadcast's copied in-flight anchor is not authoritative anymore.
|
||||
/// Missing identified alerts require reconnect/resnapshot; legacy alerts
|
||||
/// without IDs and non-Alert events pass through unchanged.
|
||||
pub fn freshen_event(&self, event: Event) -> Result<Event, AlertReplayRequired> {
|
||||
let Event::Alert(ref alert) = event else {
|
||||
return Ok(event);
|
||||
};
|
||||
let Some(id) = alert.id.as_ref() else {
|
||||
return Ok(event);
|
||||
};
|
||||
let buffer = self
|
||||
.inner
|
||||
.buffer
|
||||
.lock()
|
||||
.expect("alerter buffer mutex poisoned");
|
||||
buffer
|
||||
.iter()
|
||||
.find(|buffered| buffered.alert.id.as_ref() == Some(id))
|
||||
.map(|buffered| Event::Alert(buffered.snapshot()))
|
||||
.ok_or(AlertReplayRequired)
|
||||
}
|
||||
|
||||
/// Subscribe and atomically snapshot the current buffer.
|
||||
///
|
||||
/// The returned snapshot contains alerts emitted before
|
||||
/// this call; the receiver will deliver alerts emitted
|
||||
/// after. An alert cannot appear in both.
|
||||
pub fn subscribe_with_snapshot(&self) -> (Vec<Alert>, broadcast::Receiver<Event>) {
|
||||
let (snapshot, rx) = self.subscribe_with_deferred_snapshot();
|
||||
(snapshot(), rx)
|
||||
}
|
||||
|
||||
// Resolve positions after the session snapshot has been captured. Otherwise
|
||||
// a text commit between the two snapshots can leave only its old block ID
|
||||
// in replay, although that block is already absent from the session snapshot.
|
||||
pub(crate) fn subscribe_with_deferred_snapshot(
|
||||
&self,
|
||||
) -> (impl FnOnce() -> Vec<Alert>, broadcast::Receiver<Event>) {
|
||||
let buf = self
|
||||
.inner
|
||||
.buffer
|
||||
.lock()
|
||||
.expect("alerter buffer mutex poisoned");
|
||||
let rx = self.inner.working_event_tx.subscribe();
|
||||
let snapshot: Vec<Alert> = buf.iter().cloned().collect();
|
||||
(snapshot, rx)
|
||||
let snapshot: Vec<BufferedAlert> = buf.iter().cloned().collect();
|
||||
(
|
||||
move || snapshot.iter().map(BufferedAlert::snapshot).collect(),
|
||||
rx,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,6 +210,266 @@ fn now_ms() -> i64 {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn positioned_alerter() -> (Alerter, SegmentLogSink, InFlightEvents) {
|
||||
let (tx, _) = broadcast::channel(1024);
|
||||
let in_flight = InFlightEvents::new(tx.clone());
|
||||
let sink = SegmentLogSink::new();
|
||||
let alerter = Alerter::new(tx);
|
||||
alerter.attach_position_source(sink.clone(), Some(in_flight.clone()));
|
||||
(alerter, sink, in_flight)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queued_alert_uses_committed_anchor_when_log_lane_was_delivered_first() {
|
||||
let (alerter, sink, in_flight) = positioned_alerter();
|
||||
let text = in_flight.start_text_block();
|
||||
in_flight.text_delta(text, "finished text".into());
|
||||
in_flight.text_done(text, "finished text".into());
|
||||
let (_, mut events) = alerter.subscribe_with_snapshot();
|
||||
let (_, mut log_entries) = sink.subscribe_with_snapshot();
|
||||
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "queued".into());
|
||||
let queued = events.try_recv().unwrap();
|
||||
let Event::Alert(ref original) = queued else {
|
||||
panic!("expected alert")
|
||||
};
|
||||
assert!(matches!(
|
||||
original.position,
|
||||
Some(AlertPosition::AfterInFlight { .. })
|
||||
));
|
||||
let original_id = original.id.clone();
|
||||
let item: session_store::LoggedItem = agen::Item::assistant_message("finished text").into();
|
||||
let entry = crate::session_history::test_logged_history_entry(item.clone());
|
||||
let entry_id = entry.metadata.entry_id.0.clone();
|
||||
in_flight.clear_for_committed_item_then(&item, Some(&entry_id), || {
|
||||
sink.publish(session_store::LogEntry::AnnotatedAssistantItem { ts: 42, entry });
|
||||
});
|
||||
let committed =
|
||||
crate::ipc::protocol_session::live_log_entry_event(log_entries.try_recv().unwrap())
|
||||
.unwrap();
|
||||
assert!(
|
||||
matches!(committed, Event::SessionEntryCommitted { entry } if entry.entry_id == entry_id)
|
||||
);
|
||||
let Event::Alert(forwarded) = alerter.freshen_event(queued).unwrap() else {
|
||||
panic!("expected alert")
|
||||
};
|
||||
assert_eq!(forwarded.id, original_id);
|
||||
assert_eq!(
|
||||
forwarded.position,
|
||||
Some(AlertPosition::AfterEntry { entry_id })
|
||||
);
|
||||
let replay = alerter.subscribe_with_snapshot().0;
|
||||
assert_eq!(
|
||||
serde_json::to_value(forwarded).unwrap(),
|
||||
serde_json::to_value(&replay[0]).unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn evicted_or_unknown_alert_ids_require_replay_but_legacy_events_pass_through() {
|
||||
let (alerter, _, _) = positioned_alerter();
|
||||
let (_, mut events) = alerter.subscribe_with_snapshot();
|
||||
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "old".into());
|
||||
let queued = events.try_recv().unwrap();
|
||||
for _ in 0..MAX_BUFFERED_ALERTS {
|
||||
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "later".into());
|
||||
}
|
||||
assert!(matches!(
|
||||
alerter.freshen_event(queued),
|
||||
Err(AlertReplayRequired)
|
||||
));
|
||||
let unknown = Event::Alert(Alert {
|
||||
id: Some("not-retained".into()),
|
||||
position: Some(AlertPosition::AfterInFlight { block_id: 1 }),
|
||||
level: AlertLevel::Warn,
|
||||
source: AlertSource::Worker,
|
||||
message: "unknown".into(),
|
||||
timestamp_ms: 42,
|
||||
});
|
||||
assert!(matches!(
|
||||
alerter.freshen_event(unknown),
|
||||
Err(AlertReplayRequired)
|
||||
));
|
||||
let legacy = Event::Alert(Alert {
|
||||
id: None,
|
||||
position: None,
|
||||
level: AlertLevel::Warn,
|
||||
source: AlertSource::Worker,
|
||||
message: "legacy".into(),
|
||||
timestamp_ms: 42,
|
||||
});
|
||||
let expected = serde_json::to_value(&legacy).unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(alerter.freshen_event(legacy).unwrap()).unwrap(),
|
||||
expected
|
||||
);
|
||||
assert!(matches!(
|
||||
alerter.freshen_event(Event::Shutdown),
|
||||
Ok(Event::Shutdown)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rowless_run_transitions_preserve_visible_alert_anchor_live_and_on_replay() {
|
||||
use session_store::LogEntry;
|
||||
let (alerter, sink, _) = positioned_alerter();
|
||||
let entry = crate::session_history::test_logged_history_entry(
|
||||
agen::Item::assistant_message("visible"),
|
||||
);
|
||||
let expected = Some(AlertPosition::AfterEntry {
|
||||
entry_id: entry.metadata.entry_id.0.clone(),
|
||||
});
|
||||
sink.publish(LogEntry::AnnotatedAssistantItem { ts: 1, entry });
|
||||
let (_, mut events) = alerter.subscribe_with_snapshot();
|
||||
let transitions = [
|
||||
LogEntry::RunCancelled {
|
||||
ts: 2,
|
||||
entry_id: None,
|
||||
},
|
||||
LogEntry::RunYielded {
|
||||
ts: 3,
|
||||
entry_id: None,
|
||||
reason: protocol::RunYieldReason::Compaction,
|
||||
active_run_turn_count: 1,
|
||||
},
|
||||
LogEntry::RunResumed {
|
||||
ts: 4,
|
||||
entry_id: None,
|
||||
source: protocol::RunResumeSource::Compaction,
|
||||
active_run_turn_count: 1,
|
||||
},
|
||||
];
|
||||
for mut transition in transitions {
|
||||
transition.ensure_session_entry_id();
|
||||
sink.publish(transition);
|
||||
alerter.alert(
|
||||
AlertLevel::Warn,
|
||||
AlertSource::Worker,
|
||||
"after transition".into(),
|
||||
);
|
||||
let Event::Alert(live) = alerter.freshen_event(events.try_recv().unwrap()).unwrap()
|
||||
else {
|
||||
panic!("expected alert")
|
||||
};
|
||||
assert_eq!(live.position, expected);
|
||||
}
|
||||
let replay = alerter.subscribe_with_snapshot().0;
|
||||
assert_eq!(replay.len(), 3);
|
||||
assert!(replay.iter().all(|alert| alert.position == expected));
|
||||
let (entries, _) = sink.subscribe_with_snapshot();
|
||||
// Rebuilding the sink cache from retained entries must use the same rule.
|
||||
sink.replace_silent(entries);
|
||||
alerter.alert(
|
||||
AlertLevel::Warn,
|
||||
AlertSource::Worker,
|
||||
"after cache rebuild".into(),
|
||||
);
|
||||
assert_eq!(
|
||||
alerter.subscribe_with_snapshot().0.last().unwrap().position,
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identical_alerts_in_same_millisecond_keep_distinct_ids_across_replay() {
|
||||
let (alerter, _, _) = positioned_alerter();
|
||||
let (_, mut rx) = alerter.subscribe_with_snapshot();
|
||||
for _ in 0..2 {
|
||||
alerter.alert_at(AlertLevel::Warn, AlertSource::Worker, "same".into(), 42);
|
||||
}
|
||||
let (snapshot, _) = alerter.subscribe_with_snapshot();
|
||||
assert_ne!(snapshot[0].id, snapshot[1].id);
|
||||
for expected in &snapshot {
|
||||
let Event::Alert(live) = rx.try_recv().unwrap() else {
|
||||
panic!("expected alert")
|
||||
};
|
||||
assert!(live.id.is_some());
|
||||
assert_eq!(live.id, expected.id);
|
||||
assert_eq!(live.timestamp_ms, 42);
|
||||
assert_eq!(live.position, Some(AlertPosition::Start));
|
||||
assert_eq!(live.position, expected.position);
|
||||
}
|
||||
let (replay, _) = alerter.subscribe_with_snapshot();
|
||||
assert_eq!(
|
||||
serde_json::to_value(snapshot).unwrap(),
|
||||
serde_json::to_value(replay).unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_call_anchor_survives_commit_and_new_text() {
|
||||
let (alerter, sink, in_flight) = positioned_alerter();
|
||||
in_flight.tool_call_start("call-1".into(), "Read".into());
|
||||
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "tool alert".into());
|
||||
let item: session_store::LoggedItem = agen::Item::tool_call("call-1", "Read", "{}").into();
|
||||
let entry = session_store::LogEntry::AnnotatedAssistantItem {
|
||||
ts: 1,
|
||||
entry: crate::session_history::test_logged_history_entry(item.clone()),
|
||||
};
|
||||
in_flight.clear_for_committed_item_then(&item, Some("entry-tool"), || sink.publish(entry));
|
||||
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "after commit".into());
|
||||
let text = in_flight.start_text_block();
|
||||
in_flight.text_delta(text, "next".into());
|
||||
let (snapshot, _) = alerter.subscribe_with_snapshot();
|
||||
for alert in snapshot {
|
||||
assert_eq!(
|
||||
alert.position,
|
||||
Some(AlertPosition::AfterToolCall {
|
||||
call_id: "call-1".into()
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eviction_drops_position_ownership_and_clear_does_not_retarget_old_alerts() {
|
||||
let (alerter, _, in_flight) = positioned_alerter();
|
||||
let text = in_flight.start_text_block();
|
||||
in_flight.text_delta(text, "discarded".into());
|
||||
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "old".into());
|
||||
let weak = {
|
||||
let buffer = alerter.inner.buffer.lock().unwrap();
|
||||
Arc::downgrade(buffer[0].position.as_ref().unwrap())
|
||||
};
|
||||
for _ in 0..MAX_BUFFERED_ALERTS {
|
||||
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "new".into());
|
||||
}
|
||||
assert!(weak.upgrade().is_none());
|
||||
let (before, _) = alerter.subscribe_with_snapshot();
|
||||
in_flight.clear();
|
||||
let next = in_flight.start_text_block();
|
||||
in_flight.text_delta(next, "discarded".into());
|
||||
let item = session_store::LoggedItem::Message {
|
||||
role: session_store::LoggedRole::Assistant,
|
||||
content: vec![session_store::LoggedContentPart::Text {
|
||||
text: "discarded".into(),
|
||||
}],
|
||||
};
|
||||
in_flight.clear_for_committed_item_then(&item, Some("unrelated-entry"), || ());
|
||||
let (after, _) = alerter.subscribe_with_snapshot();
|
||||
assert_eq!(before.len(), MAX_BUFFERED_ALERTS);
|
||||
assert_eq!(
|
||||
serde_json::to_value(before).unwrap(),
|
||||
serde_json::to_value(after).unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_does_not_acquire_in_flight_lock_under_buffer_lock() {
|
||||
let (alerter, _, in_flight) = positioned_alerter();
|
||||
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "before".into());
|
||||
let guard = in_flight.snapshot_guard();
|
||||
let cloned = alerter.clone();
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let thread = std::thread::spawn(move || {
|
||||
let (snapshot, _) = cloned.subscribe_with_snapshot();
|
||||
tx.send(snapshot).unwrap();
|
||||
});
|
||||
let result = rx.recv_timeout(std::time::Duration::from_secs(5));
|
||||
drop(guard);
|
||||
thread.join().unwrap();
|
||||
assert_eq!(result.unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alert_broadcasts_to_existing_subscriber() {
|
||||
let (tx, _keep) = broadcast::channel::<Event>(8);
|
||||
|
||||
@@ -30,12 +30,12 @@ fn subscribe_before_snapshot<Subscription, Snapshot>(
|
||||
|
||||
pub fn subscribe_worker_protocol_session(handle: &WorkerHandle) -> WorkerProtocolSessionStreams {
|
||||
let ((snapshot_event, log_entries), (alert_snapshot, events)) = subscribe_before_snapshot(
|
||||
|| handle.alerter.subscribe_with_snapshot(),
|
||||
|| handle.alerter.subscribe_with_deferred_snapshot(),
|
||||
|| handle.snapshot_event_with_entry_subscription(),
|
||||
);
|
||||
WorkerProtocolSessionStreams {
|
||||
snapshot_event,
|
||||
alert_snapshot,
|
||||
alert_snapshot: alert_snapshot(),
|
||||
log_entries,
|
||||
events,
|
||||
}
|
||||
|
||||
@@ -154,6 +154,11 @@ async fn handle_connection(stream: tokio::net::UnixStream, handle: WorkerHandle)
|
||||
event = streams.events.recv() => {
|
||||
match event {
|
||||
Ok(event) => {
|
||||
let Ok(event) = handle.alerter.freshen_event(event) else {
|
||||
// The queued Alert was evicted; reconnect must rebuild
|
||||
// the bounded prefix rather than forward a stale anchor.
|
||||
break;
|
||||
};
|
||||
if writer.write_event(&event).await.is_err() {
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
|
||||
use std::sync::{Arc, Mutex as StdMutex};
|
||||
|
||||
use protocol::{AlertPosition, SessionSnapshotEntryData};
|
||||
use session_store::LogEntry;
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
@@ -46,7 +47,7 @@ pub struct SegmentLogSink {
|
||||
struct SinkInner {
|
||||
/// Full session log mirror in commit order. Reset on session swap
|
||||
/// (compaction / fork) via [`SegmentLogSink::reset_with_initial`].
|
||||
mirror: StdMutex<Vec<LogEntry>>,
|
||||
mirror: StdMutex<Mirror>,
|
||||
/// Broadcast channel for live entry updates. The same `Sender`
|
||||
/// survives session swaps so existing subscribers keep their
|
||||
/// receiver — they observe the swap as a freshly broadcast
|
||||
@@ -54,6 +55,46 @@ struct SinkInner {
|
||||
session_entry_tx: broadcast::Sender<LogEntry>,
|
||||
}
|
||||
|
||||
struct Mirror {
|
||||
entries: Vec<LogEntry>,
|
||||
alert_position: AlertPosition,
|
||||
}
|
||||
|
||||
impl Mirror {
|
||||
fn new(entries: Vec<LogEntry>) -> Self {
|
||||
let alert_position = latest_alert_position(&entries).unwrap_or(AlertPosition::Start);
|
||||
Self {
|
||||
entries,
|
||||
alert_position,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Use the public projection so private reasoning/response boundaries cannot become
|
||||
// anchors. Results/accounting update existing rows, and non-error Run lifecycle
|
||||
// transitions have no conversation row of their own.
|
||||
fn latest_alert_position(entries: &[LogEntry]) -> Option<AlertPosition> {
|
||||
session_store::public_snapshot::project_current_session_snapshot(entries)
|
||||
.entries
|
||||
.into_iter()
|
||||
.rev()
|
||||
.find_map(|entry| match entry.data {
|
||||
SessionSnapshotEntryData::ToolCall { call_id, .. } => {
|
||||
Some(AlertPosition::AfterToolCall { call_id })
|
||||
}
|
||||
SessionSnapshotEntryData::ToolResult { .. }
|
||||
| SessionSnapshotEntryData::Invoke { .. }
|
||||
| SessionSnapshotEntryData::Usage { .. }
|
||||
| SessionSnapshotEntryData::RunStats { .. }
|
||||
| SessionSnapshotEntryData::RunYielded { .. }
|
||||
| SessionSnapshotEntryData::RunResumed { .. }
|
||||
| SessionSnapshotEntryData::RunCancelled => None,
|
||||
_ => Some(AlertPosition::AfterEntry {
|
||||
entry_id: entry.entry_id,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
impl SegmentLogSink {
|
||||
/// Create a fresh sink with an empty mirror. Used before any entry
|
||||
/// has been written (deferred SegmentStart) or as a placeholder in
|
||||
@@ -62,7 +103,7 @@ impl SegmentLogSink {
|
||||
let (session_entry_tx, _) = broadcast::channel(BROADCAST_CAPACITY);
|
||||
Self {
|
||||
inner: Arc::new(SinkInner {
|
||||
mirror: StdMutex::new(Vec::new()),
|
||||
mirror: StdMutex::new(Mirror::new(Vec::new())),
|
||||
session_entry_tx,
|
||||
}),
|
||||
}
|
||||
@@ -75,7 +116,7 @@ impl SegmentLogSink {
|
||||
let (session_entry_tx, _) = broadcast::channel(BROADCAST_CAPACITY);
|
||||
Self {
|
||||
inner: Arc::new(SinkInner {
|
||||
mirror: StdMutex::new(entries),
|
||||
mirror: StdMutex::new(Mirror::new(entries)),
|
||||
session_entry_tx,
|
||||
}),
|
||||
}
|
||||
@@ -103,7 +144,10 @@ impl SegmentLogSink {
|
||||
.mirror
|
||||
.lock()
|
||||
.expect("session log mirror mutex poisoned");
|
||||
mirror.push(entry.clone());
|
||||
if let Some(position) = latest_alert_position(std::slice::from_ref(&entry)) {
|
||||
mirror.alert_position = position;
|
||||
}
|
||||
mirror.entries.push(entry.clone());
|
||||
if Self::is_live_relevant(&entry) {
|
||||
// SendError means there are zero subscribers; harmless. The
|
||||
// mirror lock is held across `send` so subscribers cannot
|
||||
@@ -147,8 +191,7 @@ impl SegmentLogSink {
|
||||
.mirror
|
||||
.lock()
|
||||
.expect("session log mirror mutex poisoned");
|
||||
mirror.clear();
|
||||
mirror.push(initial.clone());
|
||||
*mirror = Mirror::new(vec![initial.clone()]);
|
||||
let _ = self.inner.session_entry_tx.send(initial);
|
||||
}
|
||||
|
||||
@@ -164,7 +207,7 @@ impl SegmentLogSink {
|
||||
.mirror
|
||||
.lock()
|
||||
.expect("session log mirror mutex poisoned");
|
||||
*mirror = entries;
|
||||
*mirror = Mirror::new(entries);
|
||||
if let Some(initial) = first {
|
||||
let _ = self.inner.session_entry_tx.send(initial);
|
||||
}
|
||||
@@ -181,7 +224,7 @@ impl SegmentLogSink {
|
||||
.mirror
|
||||
.lock()
|
||||
.expect("session log mirror mutex poisoned");
|
||||
*mirror = entries;
|
||||
*mirror = Mirror::new(entries);
|
||||
}
|
||||
|
||||
/// Truncate the mirror without broadcasting.
|
||||
@@ -191,7 +234,9 @@ impl SegmentLogSink {
|
||||
.mirror
|
||||
.lock()
|
||||
.expect("session log mirror mutex poisoned");
|
||||
mirror.truncate(entries_len);
|
||||
mirror.entries.truncate(entries_len);
|
||||
mirror.alert_position =
|
||||
latest_alert_position(&mirror.entries).unwrap_or(AlertPosition::Start);
|
||||
}
|
||||
|
||||
/// Atomically read the current mirror and subscribe to subsequent
|
||||
@@ -203,17 +248,27 @@ impl SegmentLogSink {
|
||||
.mirror
|
||||
.lock()
|
||||
.expect("session log mirror mutex poisoned");
|
||||
let snapshot = mirror.clone();
|
||||
let snapshot = mirror.entries.clone();
|
||||
let rx = self.inner.session_entry_tx.subscribe();
|
||||
(snapshot, rx)
|
||||
}
|
||||
|
||||
pub(crate) fn alert_position(&self) -> AlertPosition {
|
||||
self.inner
|
||||
.mirror
|
||||
.lock()
|
||||
.expect("session log mirror mutex poisoned")
|
||||
.alert_position
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Current entry count. Useful for tests / diagnostics.
|
||||
pub fn len(&self) -> usize {
|
||||
self.inner
|
||||
.mirror
|
||||
.lock()
|
||||
.expect("session log mirror mutex poisoned")
|
||||
.entries
|
||||
.len()
|
||||
}
|
||||
|
||||
@@ -267,6 +322,70 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alert_anchor_tracks_visible_rows_not_reasoning_results_or_accounting() {
|
||||
let sink = SegmentLogSink::new();
|
||||
assert_eq!(sink.alert_position(), AlertPosition::Start);
|
||||
sink.publish(session_start());
|
||||
assert_eq!(sink.alert_position(), AlertPosition::Start);
|
||||
let user = user_input("hello");
|
||||
sink.publish(user.clone());
|
||||
let expected = latest_alert_position(&[user.clone()]).unwrap();
|
||||
assert_eq!(sink.alert_position(), expected);
|
||||
sink.publish(turn_end(1));
|
||||
sink.publish(LogEntry::Invoke {
|
||||
ts: 2,
|
||||
trigger: protocol::InvokeKind::UserSend,
|
||||
});
|
||||
sink.publish(LogEntry::AnnotatedAssistantItem {
|
||||
ts: 3,
|
||||
entry: crate::session_history::test_logged_history_entry(
|
||||
session_store::LoggedItem::Reasoning {
|
||||
text: "private".into(),
|
||||
summary: vec![],
|
||||
encrypted_content: None,
|
||||
signature: None,
|
||||
},
|
||||
),
|
||||
});
|
||||
assert_eq!(sink.alert_position(), expected);
|
||||
let tool = LogEntry::AnnotatedAssistantItem {
|
||||
ts: 4,
|
||||
entry: crate::session_history::test_logged_history_entry(agen::Item::tool_call(
|
||||
"call-1", "Read", "{}",
|
||||
)),
|
||||
};
|
||||
sink.publish(tool.clone());
|
||||
assert_eq!(
|
||||
sink.alert_position(),
|
||||
AlertPosition::AfterToolCall {
|
||||
call_id: "call-1".into()
|
||||
}
|
||||
);
|
||||
sink.publish(user_input("newer row"));
|
||||
let newer = sink.alert_position();
|
||||
sink.publish(LogEntry::AnnotatedToolResult {
|
||||
ts: 5,
|
||||
entry: crate::session_history::test_logged_history_entry(agen::Item::tool_result(
|
||||
"call-1", "done",
|
||||
)),
|
||||
});
|
||||
assert_eq!(sink.alert_position(), newer);
|
||||
sink.replace_silent(vec![session_start(), user.clone(), tool]);
|
||||
assert_eq!(
|
||||
sink.alert_position(),
|
||||
AlertPosition::AfterToolCall {
|
||||
call_id: "call-1".into()
|
||||
}
|
||||
);
|
||||
sink.truncate_silent(2);
|
||||
assert_eq!(sink.alert_position(), expected);
|
||||
sink.reset_with_initial(session_start());
|
||||
assert_eq!(sink.alert_position(), AlertPosition::Start);
|
||||
sink.reset_with_initial_entries(vec![session_start(), user]);
|
||||
assert_eq!(sink.alert_position(), expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publish_then_subscribe_returns_history_in_snapshot() {
|
||||
let sink = SegmentLogSink::new();
|
||||
|
||||
@@ -131,6 +131,8 @@ mod tests {
|
||||
listener,
|
||||
vec![
|
||||
Event::Alert(Alert {
|
||||
id: None,
|
||||
position: None,
|
||||
level: AlertLevel::Warn,
|
||||
source: AlertSource::Worker,
|
||||
message: "replayed alert".into(),
|
||||
|
||||
@@ -1918,11 +1918,13 @@ where
|
||||
self.state.increment_entries();
|
||||
if let Some(in_flight) = &self.in_flight {
|
||||
let committed_item = match &entry {
|
||||
LogEntry::AnnotatedAssistantItem { entry, .. } => Some(entry.item.clone()),
|
||||
LogEntry::AnnotatedAssistantItem { entry, .. } => {
|
||||
Some((entry.item.clone(), entry.metadata.entry_id.0.clone()))
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
if let Some(item_for_clear) = committed_item {
|
||||
in_flight.clear_for_committed_item_then(&item_for_clear, || {
|
||||
if let Some((item_for_clear, entry_id)) = committed_item {
|
||||
in_flight.clear_for_committed_item_then(&item_for_clear, Some(&entry_id), || {
|
||||
self.sink.publish(entry);
|
||||
});
|
||||
return Ok(());
|
||||
@@ -4253,6 +4255,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
/// Worker-internal operations (compaction failures, AGENTS.md
|
||||
/// ingestion warnings) can surface messages to connected clients.
|
||||
pub fn attach_alerter(&mut self, alerter: Alerter) {
|
||||
alerter.attach_position_source(self.sink.clone(), self.in_flight.clone());
|
||||
self.alerter = Some(alerter);
|
||||
}
|
||||
|
||||
@@ -13313,6 +13316,77 @@ mod build_summary_prompt_tests {
|
||||
}
|
||||
}
|
||||
|
||||
// Real Worker wiring, filesystem Store, log writer and reconnect buffers;
|
||||
// only the model client is inert because this test never invokes the engine.
|
||||
#[tokio::test]
|
||||
async fn alert_pending_text_anchor_resolves_at_durable_commit_before_replay() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let cwd = dir.path().join("workspace");
|
||||
std::fs::create_dir_all(&cwd).unwrap();
|
||||
let store = session_store::FsStore::new(dir.path().join("sessions")).unwrap();
|
||||
let mut worker = Worker::new(
|
||||
minimal_manifest(),
|
||||
Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient),
|
||||
store,
|
||||
WorkerWorkspaceContext::no_workspace(),
|
||||
WorkerFilesystemAuthority::local(cwd.clone(), cwd.clone()),
|
||||
Scope::writable(&cwd).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (tx, _) = broadcast::channel(16);
|
||||
let in_flight = InFlightEvents::new(tx.clone());
|
||||
worker.in_flight = Some(in_flight.clone());
|
||||
let alerter = Alerter::new(tx);
|
||||
worker.attach_alerter(alerter.clone());
|
||||
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "before head".into());
|
||||
assert_eq!(
|
||||
alerter.subscribe_with_snapshot().0[0].position,
|
||||
Some(protocol::AlertPosition::Start)
|
||||
);
|
||||
worker.materialize_durable_session_head().await.unwrap();
|
||||
let text = in_flight.start_text_block();
|
||||
in_flight.text_delta(text, "streamed".into());
|
||||
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "during text".into());
|
||||
let (before, _) = alerter.subscribe_with_snapshot();
|
||||
let snapshot = crate::in_flight::snapshot_from_guard(&in_flight.snapshot_guard());
|
||||
assert_eq!(
|
||||
before[1].position,
|
||||
Some(protocol::AlertPosition::AfterInFlight {
|
||||
block_id: snapshot.block_ids[0]
|
||||
})
|
||||
);
|
||||
let (deferred, _) = alerter.subscribe_with_deferred_snapshot();
|
||||
let entry = crate::session_history::test_logged_history_entry(
|
||||
agen::Item::assistant_message("streamed"),
|
||||
);
|
||||
let entry_id = entry.metadata.entry_id.0.clone();
|
||||
worker
|
||||
.log_writer_handle()
|
||||
.append_entry(LogEntry::AnnotatedAssistantItem { ts: 42, entry })
|
||||
.unwrap();
|
||||
let (entries, _) = worker.sink.subscribe_with_snapshot();
|
||||
assert!(entries.iter().any(|entry| matches!(entry, LogEntry::AnnotatedAssistantItem { entry, .. } if entry.metadata.entry_id.0 == entry_id)));
|
||||
assert!(
|
||||
crate::in_flight::snapshot_from_guard(&in_flight.snapshot_guard())
|
||||
.blocks
|
||||
.is_empty()
|
||||
);
|
||||
let replay = deferred();
|
||||
assert_eq!(replay[1].id, before[1].id);
|
||||
assert_eq!(
|
||||
replay[1].position,
|
||||
Some(protocol::AlertPosition::AfterEntry {
|
||||
entry_id: entry_id.clone()
|
||||
})
|
||||
);
|
||||
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "after commit".into());
|
||||
assert_eq!(
|
||||
alerter.subscribe_with_snapshot().0[2].position,
|
||||
Some(protocol::AlertPosition::AfterEntry { entry_id })
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn materialize_durable_session_head_commits_without_model_run() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -46269,6 +46269,7 @@ mod tests {
|
||||
},
|
||||
state: self.state.lock().unwrap().clone(),
|
||||
in_flight: protocol::InFlightSnapshot {
|
||||
block_ids: Vec::new(),
|
||||
blocks: Vec::new(),
|
||||
commands: Vec::new(),
|
||||
compaction: None,
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"dev": "deno run -A npm:vite@8.3.2 dev",
|
||||
"dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787",
|
||||
"check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json",
|
||||
"test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/navigation/deferred.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/api/ticket-case.test.ts src/lib/workspace/api/ticket-operations.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/settings/workdir-checkout.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts src/lib/workspace/tickets/ticket-search.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts",
|
||||
"test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/navigation/deferred.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/api/ticket-case.test.ts src/lib/workspace/api/ticket-operations.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/alert-order.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/settings/workdir-checkout.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts src/lib/workspace/tickets/ticket-search.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts",
|
||||
"test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts",
|
||||
"test:browser:empty-workspace": "deno test --config ../../tools/web-ux/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/visual-review/empty-workspace.browser.test.ts",
|
||||
"test:browser:drive": "deno test --config test/drive-browser/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/drive-browser/drive.browser.test.ts",
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -4,6 +4,8 @@
|
||||
|
||||
export type AlertLevel = "warn" | "error";
|
||||
|
||||
export type AlertPosition = { "kind": "start" } | { "kind": "after_entry", entry_id: string, } | { "kind": "after_tool_call", call_id: string, } | { "kind": "after_in_flight", block_id: number, };
|
||||
|
||||
export type AlertSource = "worker" | "engine" | "compactor" | "agents_md";
|
||||
|
||||
export type CompletionKind = "file" | "feature" | "feature_argument";
|
||||
@@ -161,7 +163,11 @@ export type InFlightBlock = { "kind": "text", text: string, finished?: boolean,
|
||||
|
||||
export type InFlightCompaction = { phase: CompactionPhase, started_at_ms: number, trigger: CompactionTrigger, };
|
||||
|
||||
export type InFlightSnapshot = { blocks?: Array<InFlightBlock>, commands?: Array<CommandSnapshot>,
|
||||
export type InFlightSnapshot = {
|
||||
/**
|
||||
* Live-Worker identities parallel to the blocks actually included below.
|
||||
*/
|
||||
block_ids?: Array<number>, blocks?: Array<InFlightBlock>, commands?: Array<CommandSnapshot>,
|
||||
/**
|
||||
* The currently running compaction, if any.
|
||||
*
|
||||
@@ -284,7 +290,11 @@ export type InternalWorkerSnapshot = { worker: InternalWorkerRef, session: Sessi
|
||||
*/
|
||||
greeting?: Greeting | null, status: WorkerStatus, error?: string | null, in_flight?: InFlightSnapshot, internal_workers?: Array<InternalWorkerSnapshot>, };
|
||||
|
||||
export type Alert = { level: AlertLevel, source: AlertSource, message: string,
|
||||
export type Alert = {
|
||||
/**
|
||||
* Stable only within the emitting live Worker; not durable alert history.
|
||||
*/
|
||||
id?: string | null, position?: AlertPosition | null, level: AlertLevel, source: AlertSource, message: string,
|
||||
/**
|
||||
* Milliseconds since the Unix epoch.
|
||||
*/
|
||||
|
||||
@@ -1,13 +1,101 @@
|
||||
// @vitest-environment happy-dom
|
||||
|
||||
import type { Event } from "#lib/generated/protocol.ts";
|
||||
import type { AlertLevel, Event } from "#lib/generated/protocol.ts";
|
||||
import { cleanup, render, waitFor } from "@testing-library/svelte";
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
import { Window } from "happy-dom";
|
||||
import { compile } from "svelte/compiler";
|
||||
import ConsoleLineItem from "./ConsoleLineItem.svelte";
|
||||
import { createConsoleProjector } from "./model.ts";
|
||||
import componentSource from "./ConsoleLineItem.svelte?raw";
|
||||
import { type ConsoleLine, createConsoleProjector } from "./model.ts";
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
// Vitest does not attach extracted component CSS. Compile the real scoped CSS
|
||||
// into an isolated style document so the assertions exercise the cascade.
|
||||
function styledLine(container: HTMLElement, colorScheme: "light" | "dark") {
|
||||
const scope = [...container.querySelector("li")!.classList].find((name) =>
|
||||
name.startsWith("svelte-")
|
||||
)!;
|
||||
const componentCss = compile(componentSource, {
|
||||
filename: "ConsoleLineItem.svelte",
|
||||
cssHash: () => scope,
|
||||
}).css!.code;
|
||||
const browser = new Window({
|
||||
settings: { device: { prefersColorScheme: colorScheme } },
|
||||
});
|
||||
const style = browser.document.createElement("style");
|
||||
// Happy DOM cannot resolve app.css's @layer/oklch tokens. These sentinel
|
||||
// values verify that non-Alerts still use the unchanged semantic tokens.
|
||||
style.textContent =
|
||||
`:root { --danger: #cc3344; --text-muted: #777777; }\n${componentCss}`;
|
||||
browser.document.head.append(style);
|
||||
browser.document.body.innerHTML = container.innerHTML;
|
||||
return {
|
||||
heading: browser.getComputedStyle(
|
||||
browser.document.querySelector(".message-heading")!,
|
||||
).color,
|
||||
body: browser.getComputedStyle(
|
||||
browser.document.querySelector(".console-plain-text")!,
|
||||
).color,
|
||||
};
|
||||
}
|
||||
|
||||
for (const colorScheme of ["light", "dark"] as const) {
|
||||
for (const level of ["warn", "error"] satisfies AlertLevel[]) {
|
||||
test(`${level} Alert has an orange heading and readable grey body in ${colorScheme}`, () => {
|
||||
const item: ConsoleLine = {
|
||||
id: `alert-${level}`,
|
||||
kind: level === "error" ? "error" : "status",
|
||||
title: "Alert · engine",
|
||||
body: "The engine needs attention.",
|
||||
source: "event",
|
||||
error: level === "error",
|
||||
alert: {
|
||||
level,
|
||||
source: "engine",
|
||||
message: "The engine needs attention.",
|
||||
timestamp_ms: 1,
|
||||
},
|
||||
};
|
||||
const { container } = render(ConsoleLineItem, { item });
|
||||
expect(container.querySelector("li")?.classList.contains("alert-line"))
|
||||
.toBe(true);
|
||||
expect(container.querySelector(".message-heading")?.textContent).toBe(
|
||||
item.title,
|
||||
);
|
||||
expect(container.querySelector(".console-plain-text")?.textContent).toBe(
|
||||
item.body,
|
||||
);
|
||||
expect(styledLine(container, colorScheme)).toEqual({
|
||||
heading: colorScheme === "light" ? "#a74709" : "#f6a75c",
|
||||
body: colorScheme === "light" ? "#595959" : "#b8b8b8",
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test(`non-Alert errors retain error styling in ${colorScheme}, even with an Alert-like title`, () => {
|
||||
const { container } = render(ConsoleLineItem, {
|
||||
item: {
|
||||
id: "ordinary-error",
|
||||
kind: "error",
|
||||
title: "Alert · engine",
|
||||
body: "An ordinary error, not a protocol Alert.",
|
||||
source: "event",
|
||||
error: true,
|
||||
},
|
||||
});
|
||||
expect(container.querySelector("li")?.classList.contains("alert-line"))
|
||||
.toBe(false);
|
||||
expect(container.querySelector("li")?.classList.contains("error-line"))
|
||||
.toBe(true);
|
||||
expect(styledLine(container, colorScheme)).toEqual({
|
||||
heading: "#777777",
|
||||
body: "#cc3344",
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function reconnectSnapshot(text: string): Event {
|
||||
return {
|
||||
event: "snapshot",
|
||||
@@ -15,7 +103,6 @@ function reconnectSnapshot(text: string): Event {
|
||||
session: {
|
||||
entries: [],
|
||||
pending_submissions: {
|
||||
|
||||
head_id: null,
|
||||
notification_count: 0,
|
||||
submissions: [],
|
||||
|
||||
@@ -85,6 +85,7 @@
|
||||
<li
|
||||
class={`console-line ${lineClass(item)} ${toolClass(item)}`}
|
||||
class:error-line={item.error}
|
||||
class:alert-line={!!item.alert}
|
||||
class:rewind-selected={rewindSelected}
|
||||
aria-current={rewindSelected ? "true" : undefined}
|
||||
data-console-line-id={item.id}
|
||||
@@ -185,6 +186,23 @@
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.console-line.alert-line {
|
||||
--alert-heading: #a74709;
|
||||
--alert-body: #595959;
|
||||
color: var(--alert-body);
|
||||
}
|
||||
|
||||
.console-line.alert-line .message-heading {
|
||||
color: var(--alert-heading);
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: dark) {
|
||||
.console-line.alert-line {
|
||||
--alert-heading: #f6a75c;
|
||||
--alert-body: #b8b8b8;
|
||||
}
|
||||
}
|
||||
|
||||
.console-line.user {
|
||||
color: var(--tui-green);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,687 @@
|
||||
import type {
|
||||
Alert,
|
||||
AlertPosition,
|
||||
Event,
|
||||
InFlightSnapshot,
|
||||
SessionSnapshotEntry,
|
||||
} from "#lib/generated/protocol.ts";
|
||||
import {
|
||||
applyProtocolEvent,
|
||||
type ConsoleLine,
|
||||
type ConsoleProjection,
|
||||
emptyConsoleProjection,
|
||||
mergeCommittedHistoryLines,
|
||||
projectConsoleLines,
|
||||
projectSessionHistoryEntries,
|
||||
projectVisibleConsole,
|
||||
} from "./model.ts";
|
||||
|
||||
declare const Deno: { test(name: string, fn: () => void): void };
|
||||
|
||||
function assertEquals(actual: unknown, expected: unknown): void {
|
||||
if (JSON.stringify(actual) !== JSON.stringify(expected)) {
|
||||
throw new Error(
|
||||
`Expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let sequence = 0;
|
||||
function apply(
|
||||
projection: ConsoleProjection,
|
||||
event: Event,
|
||||
observedAtMs = 10000,
|
||||
): ConsoleProjection {
|
||||
return applyProtocolEvent(projection, {
|
||||
eventId: `event-${sequence++}`,
|
||||
event,
|
||||
observedAtMs,
|
||||
});
|
||||
}
|
||||
|
||||
function message(
|
||||
id: string,
|
||||
timestamp: number,
|
||||
body = id,
|
||||
): SessionSnapshotEntry {
|
||||
return {
|
||||
kind: "message",
|
||||
entry_id: id,
|
||||
timestamp,
|
||||
provenance: "model_output",
|
||||
derived_from: [],
|
||||
role: "assistant",
|
||||
content: [{ kind: "text", text: body }],
|
||||
};
|
||||
}
|
||||
|
||||
function readCall(id: string, timestamp: number): SessionSnapshotEntry {
|
||||
return {
|
||||
kind: "tool_call",
|
||||
entry_id: `entry-${id}`,
|
||||
timestamp,
|
||||
provenance: "model_output",
|
||||
derived_from: [],
|
||||
call_id: id,
|
||||
name: "Read",
|
||||
arguments: JSON.stringify({ file_path: `${id}.ts` }),
|
||||
};
|
||||
}
|
||||
|
||||
function snapshot(
|
||||
entries: SessionSnapshotEntry[],
|
||||
inFlight?: InFlightSnapshot,
|
||||
): Event {
|
||||
return {
|
||||
event: "snapshot",
|
||||
data: {
|
||||
greeting: {
|
||||
worker_name: "Worker",
|
||||
cwd: "/workspace",
|
||||
provider: "provider",
|
||||
model: "model",
|
||||
scope_summary: "bounded",
|
||||
tools: [],
|
||||
context_window: 100,
|
||||
context_tokens: 20,
|
||||
},
|
||||
state: { last_command_id: 0, state: { kind: "idle" } },
|
||||
session: {
|
||||
entries,
|
||||
pending_submissions: {
|
||||
notification_count: 0,
|
||||
head_id: null,
|
||||
submissions: [],
|
||||
},
|
||||
},
|
||||
in_flight: inFlight,
|
||||
internal_workers: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function alert(
|
||||
id: string | undefined,
|
||||
position?: AlertPosition,
|
||||
timestamp = 150,
|
||||
): Alert {
|
||||
return {
|
||||
id,
|
||||
position,
|
||||
level: "warn",
|
||||
source: "worker",
|
||||
message: "same warning",
|
||||
timestamp_ms: timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
function emit(projection: ConsoleProjection, data: Alert): ConsoleProjection {
|
||||
return apply(projection, { event: "alert", data });
|
||||
}
|
||||
|
||||
function commit(
|
||||
projection: ConsoleProjection,
|
||||
entry: SessionSnapshotEntry,
|
||||
): ConsoleProjection {
|
||||
return apply(projection, {
|
||||
event: "session_entry_committed",
|
||||
data: { entry },
|
||||
});
|
||||
}
|
||||
|
||||
function labels(lines: ConsoleLine[]): string[] {
|
||||
return lines.map((line) =>
|
||||
line.alert
|
||||
? `alert:${line.alert.id ?? "legacy"}`
|
||||
: line.toolCall?.id ?? line.body
|
||||
);
|
||||
}
|
||||
|
||||
function visible(projection: ConsoleProjection): string[] {
|
||||
return labels(projectVisibleConsole(projection).lines);
|
||||
}
|
||||
|
||||
Deno.test("live Alert chronology equals fresh snapshot plus replay without timestamp sorting", () => {
|
||||
const entries = [
|
||||
message("first", 300),
|
||||
message("second", 100),
|
||||
message("third", 200),
|
||||
];
|
||||
const warnings = [
|
||||
alert("start", { kind: "start" }),
|
||||
alert("a", { kind: "after_entry", entry_id: "first" }),
|
||||
alert("b", { kind: "after_entry", entry_id: "first" }),
|
||||
alert("c", { kind: "after_entry", entry_id: "second" }),
|
||||
];
|
||||
let live = emit(emptyConsoleProjection(), warnings[0]);
|
||||
live = commit(live, entries[0]);
|
||||
live = emit(emit(live, warnings[1]), warnings[2]);
|
||||
live = emit(commit(live, entries[1]), warnings[3]);
|
||||
live = commit(live, entries[2]);
|
||||
let replay = apply(emptyConsoleProjection(), snapshot(entries));
|
||||
for (const warning of warnings) replay = emit(replay, warning);
|
||||
assertEquals(visible(live), [
|
||||
"alert:start",
|
||||
"first",
|
||||
"alert:a",
|
||||
"alert:b",
|
||||
"second",
|
||||
"alert:c",
|
||||
"third",
|
||||
]);
|
||||
assertEquals(visible(replay), visible(live));
|
||||
});
|
||||
|
||||
Deno.test("same-id replay updates metadata; distinct same-ms same-body IDs remain distinct", () => {
|
||||
const entries = [message("one", 100), message("two", 200)];
|
||||
let projection = apply(emptyConsoleProjection(), snapshot(entries));
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("a", { kind: "after_entry", entry_id: "one" }),
|
||||
);
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("b", { kind: "after_entry", entry_id: "one" }),
|
||||
);
|
||||
const originalId =
|
||||
projection.lines.find((line) => line.alert?.id === "a")!.id;
|
||||
projection = apply(projection, snapshot(entries));
|
||||
assertEquals(visible(projection), ["one", "alert:a", "alert:b", "two"]);
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("a", { kind: "after_entry", entry_id: "two" }),
|
||||
);
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("a", { kind: "after_entry", entry_id: "two" }),
|
||||
);
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("b", { kind: "after_entry", entry_id: "one" }),
|
||||
);
|
||||
assertEquals(visible(projection), ["one", "alert:b", "two", "alert:a"]);
|
||||
assertEquals(
|
||||
projection.lines.find((line) => line.alert?.id === "a")!.id,
|
||||
originalId,
|
||||
);
|
||||
assertEquals(projection.lines.filter((line) => line.alert).length, 2);
|
||||
});
|
||||
|
||||
Deno.test("live pending Alert anchor follows the actual assistant through commit and reconnect", () => {
|
||||
let live = commit(emptyConsoleProjection(), message("old", 10));
|
||||
live = apply(live, { event: "text_delta", data: { text: "pending" } }, 100);
|
||||
live = emit(
|
||||
live,
|
||||
alert("pending-alert", { kind: "after_in_flight", block_id: 42 }),
|
||||
);
|
||||
assertEquals(visible(live), ["old", "pending", "alert:pending-alert"]);
|
||||
assertEquals(
|
||||
live.lines.find((line) => line.body === "pending")?.inFlightBlockId,
|
||||
42,
|
||||
);
|
||||
live = apply(
|
||||
live,
|
||||
{ event: "text_done", data: { text: "pending answer" } },
|
||||
180,
|
||||
);
|
||||
live = commit(live, message("answer", 190, "pending answer"));
|
||||
live = commit(live, message("later", 200));
|
||||
assertEquals(visible(live), [
|
||||
"old",
|
||||
"pending answer",
|
||||
"alert:pending-alert",
|
||||
"later",
|
||||
]);
|
||||
assertEquals(live.lines.find((line) => line.alert)?.alert?.position, {
|
||||
kind: "after_entry",
|
||||
entry_id: "answer",
|
||||
});
|
||||
assertEquals(
|
||||
live.lines.find((line) => line.entryId === "answer")?.inFlightBlockId,
|
||||
42,
|
||||
);
|
||||
// Even a repeated pending-anchor payload cannot bind to a later assistant.
|
||||
live = emit(
|
||||
live,
|
||||
alert("pending-alert", { kind: "after_in_flight", block_id: 42 }),
|
||||
);
|
||||
assertEquals(visible(live), [
|
||||
"old",
|
||||
"pending answer",
|
||||
"alert:pending-alert",
|
||||
"later",
|
||||
]);
|
||||
const entries = [
|
||||
message("old", 10),
|
||||
message("answer", 190, "pending answer"),
|
||||
message("later", 200),
|
||||
];
|
||||
live = apply(live, snapshot(entries));
|
||||
assertEquals(visible(live), [
|
||||
"old",
|
||||
"pending answer",
|
||||
"alert:pending-alert",
|
||||
"later",
|
||||
]);
|
||||
const resolved = alert("pending-alert", {
|
||||
kind: "after_entry",
|
||||
entry_id: "answer",
|
||||
});
|
||||
live = emit(live, resolved);
|
||||
const fresh = emit(
|
||||
apply(emptyConsoleProjection(), snapshot(entries)),
|
||||
resolved,
|
||||
);
|
||||
assertEquals(visible(live), visible(fresh));
|
||||
});
|
||||
|
||||
Deno.test("snapshot block IDs restore pending anchors and survive text continuation", () => {
|
||||
const entries = [message("old", 10)];
|
||||
const inFlight: InFlightSnapshot = {
|
||||
block_ids: [7, 8, 9],
|
||||
blocks: [
|
||||
{ kind: "text", text: "first pending", finished: true },
|
||||
{
|
||||
kind: "tool_call",
|
||||
id: "read",
|
||||
name: "Read",
|
||||
args: "{}",
|
||||
state: "done",
|
||||
},
|
||||
{ kind: "text", text: "second", finished: false },
|
||||
],
|
||||
};
|
||||
let projection = apply(emptyConsoleProjection(), snapshot(entries, inFlight));
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("a", { kind: "after_in_flight", block_id: 7 }),
|
||||
);
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("b", { kind: "after_in_flight", block_id: 9 }),
|
||||
);
|
||||
const before = visible(projection);
|
||||
projection = apply(projection, snapshot(entries, inFlight));
|
||||
assertEquals(visible(projection), before);
|
||||
assertEquals(before, [
|
||||
"old",
|
||||
"first pending",
|
||||
"alert:a",
|
||||
"read",
|
||||
"second",
|
||||
"alert:b",
|
||||
]);
|
||||
projection = commit(projection, message("first", 100, "first pending"));
|
||||
projection = apply(projection, {
|
||||
event: "text_delta",
|
||||
data: { text: " pending" },
|
||||
});
|
||||
projection = apply(projection, {
|
||||
event: "text_done",
|
||||
data: { text: "second pending" },
|
||||
});
|
||||
projection = commit(projection, message("second", 200, "second pending"));
|
||||
assertEquals(visible(projection), [
|
||||
"old",
|
||||
"first pending",
|
||||
"alert:a",
|
||||
"read",
|
||||
"second pending",
|
||||
"alert:b",
|
||||
]);
|
||||
assertEquals(
|
||||
projection.lines.filter((line) => line.alert).map((line) =>
|
||||
line.alert!.position
|
||||
),
|
||||
[
|
||||
{ kind: "after_entry", entry_id: "first" },
|
||||
{ kind: "after_entry", entry_id: "second" },
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("snapshot with committed pending text retains Alerts until replay resolves metadata", () => {
|
||||
let projection = apply(emptyConsoleProjection(), {
|
||||
event: "text_delta",
|
||||
data: { text: "answer" },
|
||||
});
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("a", { kind: "after_in_flight", block_id: 15 }),
|
||||
);
|
||||
projection = apply(
|
||||
projection,
|
||||
snapshot([message("answer", 100), message("later", 200)]),
|
||||
);
|
||||
assertEquals(projection.lines.filter((line) => line.alert).length, 1);
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("a", { kind: "after_entry", entry_id: "answer" }),
|
||||
);
|
||||
assertEquals(visible(projection), ["answer", "alert:a", "later"]);
|
||||
});
|
||||
|
||||
Deno.test("Alert anchors split Read and overview aggregation before and after history merge", () => {
|
||||
const entries = [
|
||||
readCall("r1", 100),
|
||||
readCall("r2", 200),
|
||||
readCall("r3", 300),
|
||||
];
|
||||
let projection = apply(emptyConsoleProjection(), snapshot(entries));
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("boundary", { kind: "after_tool_call", call_id: "r1" }),
|
||||
);
|
||||
const lines = projectVisibleConsole(projection).lines;
|
||||
assertEquals(
|
||||
lines.map((line) => line.alert ? "alert" : line.toolCallCount ?? 1),
|
||||
[1, "alert", 2],
|
||||
);
|
||||
assertEquals(
|
||||
projectConsoleLines(lines, "overview").map((line) => line.kind),
|
||||
["activity", "status", "activity"],
|
||||
);
|
||||
const history = projectSessionHistoryEntries(entries, null);
|
||||
const merged = mergeCommittedHistoryLines(history, lines);
|
||||
assertEquals(
|
||||
merged.map((line) => line.alert ? "alert" : line.toolCallCount ?? 1),
|
||||
[1, "alert", 2],
|
||||
);
|
||||
assertEquals(
|
||||
projectConsoleLines(merged, "overview").map((line) => line.kind),
|
||||
["activity", "status", "activity"],
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("missing older anchors stay before the window and resolve when history arrives", () => {
|
||||
let projection = apply(
|
||||
emptyConsoleProjection(),
|
||||
snapshot([message("current", 300)]),
|
||||
);
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("older", { kind: "after_entry", entry_id: "older" }),
|
||||
);
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("same-anchor", { kind: "after_entry", entry_id: "older" }),
|
||||
);
|
||||
projection = emit(projection, alert("start", { kind: "start" }));
|
||||
const current = projectVisibleConsole(projection).lines;
|
||||
assertEquals(labels(current), [
|
||||
"alert:older",
|
||||
"alert:same-anchor",
|
||||
"alert:start",
|
||||
"current",
|
||||
]);
|
||||
const history = projectSessionHistoryEntries([
|
||||
message("first", 50),
|
||||
message("older", 100),
|
||||
message("middle", 200),
|
||||
message("current", 300),
|
||||
], null);
|
||||
const merged = mergeCommittedHistoryLines(history, current);
|
||||
assertEquals(labels(merged), [
|
||||
"alert:start",
|
||||
"first",
|
||||
"older",
|
||||
"alert:older",
|
||||
"alert:same-anchor",
|
||||
"middle",
|
||||
"current",
|
||||
]);
|
||||
assertEquals(
|
||||
labels(mergeCommittedHistoryLines(history, merged)),
|
||||
labels(merged),
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("legacy Alerts use original emission timestamps for live and replay chronology", () => {
|
||||
let live = apply(emptyConsoleProjection(), {
|
||||
event: "text_done",
|
||||
data: { text: "first" },
|
||||
}, 100);
|
||||
live = emit(live, alert(undefined, undefined, 150));
|
||||
live = apply(live, { event: "text_done", data: { text: "second" } }, 200);
|
||||
assertEquals(visible(live), ["first", "alert:legacy", "second"]);
|
||||
let replay = apply(
|
||||
emptyConsoleProjection(),
|
||||
snapshot([message("first", 100), message("second", 200)]),
|
||||
);
|
||||
replay = emit(replay, alert(undefined, undefined, 150));
|
||||
replay = emit(replay, alert(undefined, undefined, 150));
|
||||
assertEquals(visible(replay), visible(live));
|
||||
assertEquals(replay.lines.map((line) => line.timestampMs), [100, 200, 150]);
|
||||
const merged = mergeCommittedHistoryLines(
|
||||
projectSessionHistoryEntries([message("earliest", 50)], null),
|
||||
projectVisibleConsole(replay).lines,
|
||||
);
|
||||
assertEquals(labels(merged), ["earliest", "first", "alert:legacy", "second"]);
|
||||
});
|
||||
|
||||
Deno.test("appended entry metadata does not rescan prior event IDs or mutate retained rows", () => {
|
||||
const projection = apply(
|
||||
emptyConsoleProjection(),
|
||||
snapshot(
|
||||
Array.from(
|
||||
{ length: 128 },
|
||||
(_, index) => message(`entry-${index}`, index),
|
||||
),
|
||||
),
|
||||
);
|
||||
let priorEventIdReads = 0;
|
||||
projection.lines = projection.lines.map((line) =>
|
||||
Object.freeze({
|
||||
...line,
|
||||
get eventId() {
|
||||
priorEventIdReads += 1;
|
||||
return line.eventId;
|
||||
},
|
||||
})
|
||||
);
|
||||
const next = commit(projection, message("appended", 200));
|
||||
assertEquals(priorEventIdReads, 0);
|
||||
assertEquals(next.lines.length, projection.lines.length + 1);
|
||||
assertEquals(
|
||||
next.lines.slice(0, -1).every((line, index) =>
|
||||
line === projection.lines[index]
|
||||
),
|
||||
true,
|
||||
);
|
||||
assertEquals(next.lines.at(-1)?.entryId, "appended");
|
||||
assertEquals(next.lines.at(-1)?.sourceEntryIds, ["appended"]);
|
||||
assertEquals(next.lines.at(-1)?.timestampMs, 200);
|
||||
});
|
||||
|
||||
Deno.test("snapshot retention indexes previous entry metadata in one pass", () => {
|
||||
const entries = Array.from(
|
||||
{ length: 128 },
|
||||
(_, index) => message(`entry-${index}`, index),
|
||||
);
|
||||
const projection = apply(emptyConsoleProjection(), snapshot(entries));
|
||||
let priorEntryIdReads = 0;
|
||||
projection.lines = projection.lines.map((line, index) =>
|
||||
Object.freeze({
|
||||
...line,
|
||||
inFlightBlockId: index,
|
||||
get entryId() {
|
||||
priorEntryIdReads += 1;
|
||||
return line.entryId;
|
||||
},
|
||||
})
|
||||
);
|
||||
const next = apply(projection, snapshot(entries));
|
||||
assertEquals(priorEntryIdReads, entries.length);
|
||||
assertEquals(
|
||||
next.lines.map((line) => line.inFlightBlockId),
|
||||
entries.map((_, index) => index),
|
||||
);
|
||||
assertEquals(
|
||||
next.lines.map((line) => line.timestampMs),
|
||||
entries.map((_, index) => index),
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("tool-result reconciliation retains call-entry anchors and creation timestamp", () => {
|
||||
let projection = commit(emptyConsoleProjection(), readCall("r1", 100));
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("a", { kind: "after_entry", entry_id: "entry-r1" }),
|
||||
);
|
||||
projection = commit(projection, readCall("r2", 200));
|
||||
const before = projection;
|
||||
const originalCall = before.lines[0];
|
||||
Object.freeze(originalCall.sourceEntryIds);
|
||||
Object.freeze(originalCall);
|
||||
projection = commit(projection, {
|
||||
kind: "tool_result",
|
||||
entry_id: "result-r1",
|
||||
timestamp: 250,
|
||||
provenance: "tool_output",
|
||||
derived_from: [],
|
||||
call_id: "r1",
|
||||
summary: "read",
|
||||
content: "text",
|
||||
is_error: false,
|
||||
});
|
||||
assertEquals(visible(projection), ["r1", "alert:a", "r2"]);
|
||||
assertEquals(originalCall.sourceEntryIds, ["entry-r1"]);
|
||||
assertEquals(originalCall.entryId, "entry-r1");
|
||||
assertEquals(originalCall.toolCall?.state, "running");
|
||||
assertEquals(projection.lines[0] === originalCall, false);
|
||||
assertEquals(projection.lines[2] === before.lines[2], true);
|
||||
assertEquals(projection.lines[0].sourceEntryIds, ["entry-r1", "result-r1"]);
|
||||
assertEquals(
|
||||
projection.lines.find((line) => line.toolCall?.id === "r1")?.timestampMs,
|
||||
100,
|
||||
);
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("a", { kind: "after_entry", entry_id: "result-r1" }),
|
||||
);
|
||||
assertEquals(visible(projection), ["r1", "alert:a", "r2"]);
|
||||
});
|
||||
|
||||
Deno.test("history enriches a live tool row with durable anchors before placing Alerts", () => {
|
||||
let projection = apply(emptyConsoleProjection(), {
|
||||
event: "tool_call_start",
|
||||
data: { id: "r1", name: "Read" },
|
||||
}, 110);
|
||||
projection = emit(
|
||||
projection,
|
||||
alert("a", { kind: "after_entry", entry_id: "entry-r1" }),
|
||||
);
|
||||
projection = apply(projection, {
|
||||
event: "tool_call_start",
|
||||
data: { id: "r2", name: "Read" },
|
||||
}, 210);
|
||||
const merged = mergeCommittedHistoryLines(
|
||||
projectSessionHistoryEntries(
|
||||
[readCall("r1", 100), readCall("r2", 200)],
|
||||
null,
|
||||
),
|
||||
projectVisibleConsole(projection).lines,
|
||||
);
|
||||
assertEquals(labels(merged), ["r1", "alert:a", "r2"]);
|
||||
assertEquals(merged[0].toolCallLines?.[0].timestampMs, 100);
|
||||
assertEquals(merged[0].toolCallLines?.[0].toolCall?.state, "pending");
|
||||
});
|
||||
|
||||
Deno.test("live pending metadata survives reconnect before commit without changing first observation", () => {
|
||||
let projection = apply(emptyConsoleProjection(), {
|
||||
event: "text_delta",
|
||||
data: { text: "pending" },
|
||||
}, 100);
|
||||
const warning = alert("a", { kind: "after_in_flight", block_id: 27 });
|
||||
projection = emit(projection, warning);
|
||||
projection = apply(
|
||||
projection,
|
||||
snapshot([], {
|
||||
block_ids: [27],
|
||||
blocks: [{ kind: "text", text: "pending", finished: false }],
|
||||
}),
|
||||
5000,
|
||||
);
|
||||
assertEquals(visible(projection), ["pending", "alert:a"]);
|
||||
assertEquals(projection.lines[0].timestampMs, 100);
|
||||
projection = emit(projection, warning);
|
||||
assertEquals(visible(projection), ["pending", "alert:a"]);
|
||||
projection = apply(projection, {
|
||||
event: "text_done",
|
||||
data: { text: "pending" },
|
||||
});
|
||||
projection = commit(projection, message("answer", 200, "pending"));
|
||||
assertEquals(projection.lines.find((line) => line.alert)?.alert?.position, {
|
||||
kind: "after_entry",
|
||||
entry_id: "answer",
|
||||
});
|
||||
});
|
||||
|
||||
Deno.test("parent snapshot retains child Alerts in their own console only", () => {
|
||||
const worker = {
|
||||
kind: "sub_worker",
|
||||
name: "child",
|
||||
session_id: "child-session",
|
||||
} as const;
|
||||
const childEvent = (event: Event): Event => ({
|
||||
event: "internal_worker",
|
||||
data: { worker, event },
|
||||
});
|
||||
let projection = apply(
|
||||
emptyConsoleProjection(),
|
||||
childEvent(snapshot([message("child-answer", 100)])),
|
||||
);
|
||||
projection = apply(
|
||||
projection,
|
||||
childEvent({
|
||||
event: "alert",
|
||||
data: alert("a", { kind: "after_entry", entry_id: "child-answer" }),
|
||||
}),
|
||||
);
|
||||
const parentSnapshot = snapshot([message("parent", 300)]);
|
||||
if (parentSnapshot.event !== "snapshot") throw new Error("snapshot expected");
|
||||
const childSnapshot = snapshot([message("child-answer", 100)]);
|
||||
if (childSnapshot.event !== "snapshot") throw new Error("snapshot expected");
|
||||
parentSnapshot.data.internal_workers = [{
|
||||
worker,
|
||||
status: "idle",
|
||||
error: null,
|
||||
session: childSnapshot.data.session,
|
||||
}];
|
||||
projection = apply(projection, parentSnapshot);
|
||||
assertEquals(visible(projection), ["parent"]);
|
||||
assertEquals(visible(projection.internalWorkers[0].console), [
|
||||
"child-answer",
|
||||
"alert:a",
|
||||
]);
|
||||
});
|
||||
|
||||
Deno.test("snapshot retains warning Alerts but reconciles durable run errors", () => {
|
||||
let projection = emit(
|
||||
emptyConsoleProjection(),
|
||||
alert("warn", { kind: "start" }),
|
||||
);
|
||||
projection = emit(projection, {
|
||||
...alert("error", { kind: "start" }),
|
||||
level: "error",
|
||||
message: "failed",
|
||||
});
|
||||
projection = apply(
|
||||
projection,
|
||||
snapshot([{
|
||||
kind: "run_error",
|
||||
entry_id: "failure",
|
||||
timestamp: 300,
|
||||
provenance: "legacy_unknown",
|
||||
derived_from: [],
|
||||
message: "failed",
|
||||
failure: "compaction",
|
||||
}]),
|
||||
);
|
||||
assertEquals(
|
||||
projectVisibleConsole(projection).lines.map((line) =>
|
||||
line.alert?.id ?? line.title
|
||||
),
|
||||
["warn", "Compaction failed"],
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import type { Alert } from "#lib/generated/protocol.ts";
|
||||
import type { ConsoleLine } from "./model.ts";
|
||||
|
||||
/** Stable IDs are authoritative; legacy replay can only use the original payload. */
|
||||
export function alertIdentity(alert: Alert): string {
|
||||
return alert.id != null ? `id:${alert.id}` : `legacy:${
|
||||
JSON.stringify([
|
||||
alert.timestamp_ms,
|
||||
alert.level,
|
||||
alert.source,
|
||||
alert.message,
|
||||
])
|
||||
}`;
|
||||
}
|
||||
|
||||
/** Keep first-seen order while allowing replay to refresh anchor metadata. */
|
||||
export function deduplicateAlertLines(lines: ConsoleLine[]): ConsoleLine[] {
|
||||
const result: ConsoleLine[] = [];
|
||||
const positions = new Map<string, number>();
|
||||
for (const line of lines) {
|
||||
if (!line.alert) {
|
||||
result.push(line);
|
||||
continue;
|
||||
}
|
||||
const key = alertIdentity(line.alert);
|
||||
const previous = positions.get(key);
|
||||
if (previous === undefined) {
|
||||
positions.set(key, result.length);
|
||||
result.push(line);
|
||||
} else {
|
||||
result[previous] = { ...line, id: result[previous]!.id };
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert Alerts between source rows BEFORE any Read/activity aggregation.
|
||||
* Conversation order remains authoritative, even when timestamps are skewed.
|
||||
* An absent explicit anchor belongs before the loaded window, not at its tail;
|
||||
* its metadata remains intact so loading older history can resolve it later.
|
||||
*/
|
||||
export function orderAlertLines(lines: ConsoleLine[]): ConsoleLine[] {
|
||||
const ungrouped = lines.flatMap((line) => line.toolCallLines ?? [line]);
|
||||
const unique = deduplicateAlertLines(ungrouped);
|
||||
const source = unique.filter((line) => !line.alert);
|
||||
const slots = new Map<number, ConsoleLine[]>();
|
||||
for (const line of unique) {
|
||||
const alert = line.alert;
|
||||
if (!alert) continue;
|
||||
const position = alert.position;
|
||||
let index: number;
|
||||
switch (position?.kind) {
|
||||
case "start":
|
||||
index = 0;
|
||||
break;
|
||||
case "after_entry":
|
||||
index = source.findIndex((row) =>
|
||||
row.entryId === position.entry_id ||
|
||||
row.sourceEntryIds?.includes(position.entry_id)
|
||||
) + 1;
|
||||
break;
|
||||
case "after_tool_call":
|
||||
index = source.findIndex((row) =>
|
||||
row.toolCall?.id === position.call_id
|
||||
) + 1;
|
||||
break;
|
||||
case "after_in_flight":
|
||||
index =
|
||||
source.findIndex((row) => row.inFlightBlockId === position.block_id) +
|
||||
1;
|
||||
break;
|
||||
default: {
|
||||
// Legacy senders have no identity/anchor. Use emission time, never the
|
||||
// reconnect observation time, without sorting the conversation itself.
|
||||
const later = source.findIndex((row) =>
|
||||
row.timestampMs !== undefined && row.timestampMs > alert.timestamp_ms
|
||||
);
|
||||
index = later < 0 ? source.length : later;
|
||||
}
|
||||
}
|
||||
const slot = slots.get(index) ?? [];
|
||||
slot.push(line);
|
||||
slots.set(index, slot);
|
||||
}
|
||||
const result = [...(slots.get(0) ?? [])];
|
||||
source.forEach((line, index) => {
|
||||
result.push(line, ...(slots.get(index + 1) ?? []));
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Live deltas lack block IDs; the emitting Alert identifies the pending row. */
|
||||
export function attachAlertInFlightAnchor(
|
||||
lines: ConsoleLine[],
|
||||
alert: Alert,
|
||||
): ConsoleLine[] {
|
||||
const position = alert.position;
|
||||
if (position?.kind !== "after_in_flight") return lines;
|
||||
if (lines.some((line) => line.inFlightBlockId === position.block_id)) {
|
||||
return lines;
|
||||
}
|
||||
const index = lines.findLastIndex((line) =>
|
||||
line.kind === "assistant" && !line.entryId
|
||||
);
|
||||
if (index < 0 || lines[index].inFlightBlockId !== undefined) return lines;
|
||||
return lines.map((line, rowIndex) =>
|
||||
rowIndex === index ? { ...line, inFlightBlockId: position.block_id } : line
|
||||
);
|
||||
}
|
||||
|
||||
/** Resolve before the pending row is removed; replay may repeat either anchor. */
|
||||
export function resolveCommittedAlertAnchors(
|
||||
lines: ConsoleLine[],
|
||||
blockId: number | undefined,
|
||||
entryId: string,
|
||||
): ConsoleLine[] {
|
||||
if (blockId === undefined) return lines;
|
||||
return lines.map((line) =>
|
||||
line.alert?.position?.kind === "after_in_flight" &&
|
||||
line.alert.position.block_id === blockId
|
||||
? {
|
||||
...line,
|
||||
alert: {
|
||||
...line.alert,
|
||||
position: { kind: "after_entry", entry_id: entryId },
|
||||
},
|
||||
}
|
||||
: line
|
||||
);
|
||||
}
|
||||
@@ -24,6 +24,12 @@ import type {
|
||||
WorkerStatus,
|
||||
} from "#lib/generated/protocol.ts";
|
||||
import { invocationInput } from "./composer-invocation.ts";
|
||||
import {
|
||||
attachAlertInFlightAnchor,
|
||||
deduplicateAlertLines,
|
||||
orderAlertLines,
|
||||
resolveCommittedAlertAnchors,
|
||||
} from "./alert-order.ts";
|
||||
import { stringify as stringifyYaml } from "yaml";
|
||||
import {
|
||||
applyRunActivityEvent,
|
||||
@@ -103,6 +109,12 @@ export type ConsoleLine = {
|
||||
eventId?: string | null;
|
||||
/** Stable durable identity when this line originates from a committed entry. */
|
||||
entryId?: string;
|
||||
/** Calls/results can share a row while retaining distinct durable anchors. */
|
||||
sourceEntryIds?: string[];
|
||||
/** Durable creation time or first live observation; never reconnect time for Alerts. */
|
||||
timestampMs?: number;
|
||||
/** Live-Worker block identity, retained through committed-row replacement. */
|
||||
inFlightBlockId?: number;
|
||||
source: "event";
|
||||
streaming?: boolean;
|
||||
error?: boolean;
|
||||
@@ -114,6 +126,8 @@ export type ConsoleLine = {
|
||||
toolCallLines?: ConsoleLine[];
|
||||
/** Typed `SystemItem.kind` used by presentation-only projections. */
|
||||
systemItemKind?: string;
|
||||
/** Original protocol Alert, retained for chronology and Alert-specific presentation. */
|
||||
alert?: Alert;
|
||||
};
|
||||
|
||||
export type InternalWorkerProjection = {
|
||||
@@ -340,18 +354,46 @@ export function mergeCommittedHistoryLines(
|
||||
history: ConsoleLine[],
|
||||
current: ConsoleLine[],
|
||||
): ConsoleLine[] {
|
||||
const durableStats = new Map(
|
||||
history.filter((line) => line.kind === "run_stats" && line.entryId)
|
||||
.map((line) => [line.entryId!, line]),
|
||||
const historySource = history.flatMap((line) => line.toolCallLines ?? [line]);
|
||||
const currentSource = current.flatMap((line) => line.toolCallLines ?? [line]);
|
||||
const alerts = [...historySource, ...currentSource].filter((line) =>
|
||||
line.alert
|
||||
);
|
||||
return aggregateReadToolLines(mergeHistorySourceLines(
|
||||
history.flatMap((line) => line.toolCallLines ?? [line]),
|
||||
current.flatMap((line) => line.toolCallLines ?? [line]).map((line) =>
|
||||
line.kind === "run_stats" && line.entryId
|
||||
? durableStats.get(line.entryId) ?? line
|
||||
: line
|
||||
),
|
||||
));
|
||||
const durableRows = new Map(
|
||||
historySource.filter((line) => !line.alert)
|
||||
.map((line) => [historyLineKey(line), line]),
|
||||
);
|
||||
const merged = mergeHistorySourceLines(
|
||||
historySource.filter((line) => !line.alert),
|
||||
currentSource.filter((line) => !line.alert).map((line) => {
|
||||
const key = historyLineKey(line);
|
||||
const durable = key ? durableRows.get(key) : undefined;
|
||||
if (!durable) return line;
|
||||
if (line.kind === "run_stats") return durable;
|
||||
// Keep the current row authoritative; borrow durable metadata only when
|
||||
// needed to place Alerts against a live or partially restored tool row.
|
||||
if (alerts.length === 0) return line;
|
||||
return {
|
||||
...line,
|
||||
entryId: line.entryId ?? durable.entryId,
|
||||
sourceEntryIds: [
|
||||
...new Set([
|
||||
...(durable.sourceEntryIds ?? []),
|
||||
...(durable.entryId ? [durable.entryId] : []),
|
||||
...(line.sourceEntryIds ?? []),
|
||||
...(line.entryId ? [line.entryId] : []),
|
||||
]),
|
||||
],
|
||||
timestampMs: line.entryId
|
||||
? line.timestampMs ?? durable.timestampMs
|
||||
: durable.timestampMs ?? line.timestampMs,
|
||||
};
|
||||
}),
|
||||
);
|
||||
return aggregateReadToolLines(orderAlertLines([
|
||||
...merged,
|
||||
...alerts,
|
||||
]));
|
||||
}
|
||||
|
||||
function historyLineKey(line: ConsoleLine): string | undefined {
|
||||
@@ -440,12 +482,12 @@ export function projectSessionHistoryEntries(
|
||||
).lines;
|
||||
}
|
||||
|
||||
function projectVisibleConsole(
|
||||
export function projectVisibleConsole(
|
||||
projection: ConsoleProjection,
|
||||
): ConsoleProjection {
|
||||
return {
|
||||
...projection,
|
||||
lines: aggregateReadToolLines(projection.lines),
|
||||
lines: aggregateReadToolLines(orderAlertLines(projection.lines)),
|
||||
internalWorkers: projection.internalWorkers.map((worker) => ({
|
||||
...worker,
|
||||
console: projectVisibleConsole(worker.console),
|
||||
@@ -650,6 +692,8 @@ function appendSnapshotInFlightLines(
|
||||
blocks: InFlightBlock[],
|
||||
eventId: string,
|
||||
cwd: string | null,
|
||||
blockIds: number[] = [],
|
||||
observedAtMs?: number,
|
||||
): void {
|
||||
blocks.forEach((block, index) => {
|
||||
const blockEventId = `${eventId}:${index}`;
|
||||
@@ -703,6 +747,16 @@ function appendSnapshotInFlightLines(
|
||||
});
|
||||
break;
|
||||
}
|
||||
const rowIndex = projection.lines.findLastIndex((line) =>
|
||||
line.eventId === blockEventId
|
||||
);
|
||||
if (rowIndex >= 0) {
|
||||
projection.lines[rowIndex] = {
|
||||
...projection.lines[rowIndex],
|
||||
inFlightBlockId: blockIds[index],
|
||||
timestampMs: projection.lines[rowIndex].timestampMs ?? observedAtMs,
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -862,6 +916,65 @@ function appendCommandStream(
|
||||
};
|
||||
}
|
||||
|
||||
function retainSnapshotAlerts(
|
||||
previous: ConsoleProjection,
|
||||
restored: ConsoleProjection,
|
||||
): ConsoleProjection {
|
||||
const entryIndexes = new Map<string, number>();
|
||||
const blockIndexes = new Map<number, number>();
|
||||
previous.lines.forEach((line, index) => {
|
||||
const { entryId, inFlightBlockId } = line;
|
||||
if (entryId !== undefined && !entryIndexes.has(entryId)) {
|
||||
entryIndexes.set(entryId, index);
|
||||
}
|
||||
if (inFlightBlockId !== undefined && !blockIndexes.has(inFlightBlockId)) {
|
||||
blockIndexes.set(inFlightBlockId, index);
|
||||
}
|
||||
});
|
||||
const lines = restored.lines.map((line) => {
|
||||
// Preserve the first matching source row, including when both keys match.
|
||||
const entryIndex = line.entryId === undefined
|
||||
? undefined
|
||||
: entryIndexes.get(line.entryId);
|
||||
const blockIndex = line.inFlightBlockId === undefined
|
||||
? undefined
|
||||
: blockIndexes.get(line.inFlightBlockId);
|
||||
const known = previous.lines[
|
||||
Math.min(
|
||||
entryIndex ?? previous.lines.length,
|
||||
blockIndex ?? previous.lines.length,
|
||||
)
|
||||
];
|
||||
return known
|
||||
? {
|
||||
...line,
|
||||
inFlightBlockId: line.inFlightBlockId ?? known.inFlightBlockId,
|
||||
timestampMs: line.entryId
|
||||
? line.timestampMs
|
||||
: known.timestampMs ?? line.timestampMs,
|
||||
}
|
||||
: line;
|
||||
});
|
||||
const alerts = previous.lines.filter((line) =>
|
||||
line.alert && !(line.error && hasDurableRunFailure(restored, line.body))
|
||||
);
|
||||
return {
|
||||
...restored,
|
||||
lines: deduplicateAlertLines([...lines, ...alerts]),
|
||||
internalWorkers: restored.internalWorkers.map((worker) => {
|
||||
const known = previous.internalWorkers.find((old) =>
|
||||
old.worker.session_id === worker.worker.session_id
|
||||
);
|
||||
return known
|
||||
? {
|
||||
...worker,
|
||||
console: retainSnapshotAlerts(known.console, worker.console),
|
||||
}
|
||||
: worker;
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function projectInternalWorkerSnapshot(
|
||||
snapshot: InternalWorkerSnapshot,
|
||||
eventId: string,
|
||||
@@ -882,6 +995,7 @@ function projectInternalWorkerSnapshot(
|
||||
snapshot.in_flight?.blocks ?? [],
|
||||
`${eventId}:internal:${snapshot.worker.session_id}:in-flight`,
|
||||
cwd,
|
||||
snapshot.in_flight?.block_ids,
|
||||
);
|
||||
appendSnapshotCommands(
|
||||
console,
|
||||
@@ -1057,7 +1171,10 @@ export function applyProtocolEvent(
|
||||
|
||||
switch (event.event) {
|
||||
case "user_message":
|
||||
if (event.data.entry_id && next.lines.some((line) => line.entryId === event.data.entry_id)) break;
|
||||
if (
|
||||
event.data.entry_id &&
|
||||
next.lines.some((line) => line.entryId === event.data.entry_id)
|
||||
) break;
|
||||
next.lines.push({
|
||||
...line(
|
||||
envelope.eventId,
|
||||
@@ -1072,17 +1189,19 @@ export function applyProtocolEvent(
|
||||
// A snapshot may already include this commit. Check before reconciling so
|
||||
// replay cannot consume a later, unrelated live assistant block.
|
||||
if (isRepeatedSessionEntry(next, event.data.entry)) break;
|
||||
const replacementIndex = reconcileCommittedSessionEntry(
|
||||
const replacement = reconcileCommittedSessionEntry(
|
||||
next,
|
||||
event.data.entry,
|
||||
);
|
||||
applySessionEntry(next, envelope.eventId, event.data.entry);
|
||||
if (replacementIndex !== undefined) {
|
||||
if (replacement !== undefined) {
|
||||
const replacementIndex = replacement.index;
|
||||
const committedIndex = next.lines.findIndex((line) =>
|
||||
line.entryId === event.data.entry.entry_id
|
||||
);
|
||||
if (committedIndex >= 0) {
|
||||
const [committed] = next.lines.splice(committedIndex, 1);
|
||||
committed.inFlightBlockId = replacement.inFlightBlockId;
|
||||
next.lines.splice(replacementIndex, 0, committed);
|
||||
const stats = next.lines[replacementIndex + 1];
|
||||
if (stats?.kind === "run_stats" && !stats.entryId) {
|
||||
@@ -1212,6 +1331,8 @@ export function applyProtocolEvent(
|
||||
event.data.in_flight?.blocks ?? [],
|
||||
`${envelope.eventId}:snapshot-in-flight`,
|
||||
next.cwd,
|
||||
event.data.in_flight?.block_ids,
|
||||
envelope.observedAtMs,
|
||||
);
|
||||
appendSnapshotCommands(
|
||||
next,
|
||||
@@ -1221,6 +1342,9 @@ export function applyProtocolEvent(
|
||||
next.internalWorkers = (event.data.internal_workers ?? []).map((worker) =>
|
||||
projectInternalWorkerSnapshot(worker, envelope.eventId, next.cwd)
|
||||
);
|
||||
const retained = retainSnapshotAlerts(projection, next);
|
||||
next.lines = retained.lines;
|
||||
next.internalWorkers = retained.internalWorkers;
|
||||
next.removedInternalWorkers = {};
|
||||
const withCompaction = applyInFlightCompaction(
|
||||
next,
|
||||
@@ -1278,7 +1402,9 @@ export function applyProtocolEvent(
|
||||
applyCommandEvent(next, envelope.eventId, event.data.event);
|
||||
break;
|
||||
case "segment_rotated": {
|
||||
const retainedErrors = next.lines.filter((line) => line.kind === "error");
|
||||
const retainedErrors = next.lines.filter((line) =>
|
||||
line.kind === "error" || line.alert
|
||||
);
|
||||
const retainedCompaction = next.lines.filter((line) =>
|
||||
line.id === RUNTIME_COMPACTION_LINE_ID
|
||||
);
|
||||
@@ -1348,6 +1474,11 @@ export function applyProtocolEvent(
|
||||
break;
|
||||
}
|
||||
|
||||
next.lines = next.lines.map((line) =>
|
||||
line.timestampMs === undefined && line.eventId === envelope.eventId
|
||||
? { ...line, timestampMs: envelope.observedAtMs }
|
||||
: line
|
||||
);
|
||||
return next;
|
||||
}
|
||||
|
||||
@@ -1509,8 +1640,9 @@ function appendAlertLine(
|
||||
): void {
|
||||
const isError = alert.level === "error";
|
||||
if (isError && hasDurableRunFailure(projection, alert.message)) return;
|
||||
projection.lines.push(
|
||||
line(
|
||||
projection.lines = attachAlertInFlightAnchor(projection.lines, alert);
|
||||
projection.lines = deduplicateAlertLines([...projection.lines, {
|
||||
...line(
|
||||
eventId,
|
||||
isError ? "error" : "status",
|
||||
`Alert · ${alert.source}`,
|
||||
@@ -1519,7 +1651,9 @@ function appendAlertLine(
|
||||
false,
|
||||
isError,
|
||||
),
|
||||
);
|
||||
alert,
|
||||
timestampMs: alert.timestamp_ms,
|
||||
}]);
|
||||
}
|
||||
|
||||
function findLastLineIndex(
|
||||
@@ -2335,7 +2469,7 @@ function snapshotProjectionFromSession(
|
||||
function reconcileCommittedSessionEntry(
|
||||
projection: ConsoleProjection,
|
||||
entry: SessionSnapshotEntry,
|
||||
): number | undefined {
|
||||
): { index: number; inFlightBlockId?: number } | undefined {
|
||||
if (entry.kind !== "message" || entry.role !== "assistant") return;
|
||||
const turnStart = findLastLineIndex(
|
||||
projection.lines,
|
||||
@@ -2348,8 +2482,13 @@ function reconcileCommittedSessionEntry(
|
||||
!line.entryId &&
|
||||
line.streaming !== true
|
||||
) {
|
||||
projection.lines = resolveCommittedAlertAnchors(
|
||||
projection.lines,
|
||||
line.inFlightBlockId,
|
||||
entry.entry_id,
|
||||
);
|
||||
projection.lines.splice(index, 1);
|
||||
return index;
|
||||
return { index, inFlightBlockId: line.inFlightBlockId };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2375,7 +2514,8 @@ function applySessionEntry(
|
||||
if (!isRecord(value) || isRepeatedSessionEntry(projection, value)) return;
|
||||
const eventId = stringField(value, "entry_id") ?? fallbackEventId;
|
||||
const lineCountBefore = projection.lines.length;
|
||||
switch (stringField(value, "kind")) {
|
||||
const entryKind = stringField(value, "kind");
|
||||
switch (entryKind) {
|
||||
case "user_input":
|
||||
projection.lines.push(
|
||||
line(
|
||||
@@ -2422,7 +2562,10 @@ function applySessionEntry(
|
||||
const requests = safeTokenCount(value["requests"]);
|
||||
const upload = safeTokenCount(value["upload_tokens"]);
|
||||
const output = safeTokenCount(value["output_tokens"]);
|
||||
if (elapsed !== null && requests !== null && upload !== null && output !== null) {
|
||||
if (
|
||||
elapsed !== null && requests !== null && upload !== null &&
|
||||
output !== null
|
||||
) {
|
||||
projection.lines.push(runStatsLine(eventId, {
|
||||
startedAtMs: 0,
|
||||
requests,
|
||||
@@ -2451,18 +2594,41 @@ function applySessionEntry(
|
||||
break;
|
||||
}
|
||||
const entryId = stringField(value, "entry_id");
|
||||
if (entryId) {
|
||||
const timestampMs = typeof value["timestamp"] === "number"
|
||||
? value["timestamp"]
|
||||
: undefined;
|
||||
const stampLine = (index: number) => {
|
||||
const line = projection.lines[index];
|
||||
if (!line || line.eventId !== eventId) return;
|
||||
const sourceEntryIds = new Set(line.sourceEntryIds ?? []);
|
||||
if (line.entryId) sourceEntryIds.add(line.entryId);
|
||||
if (entryId) sourceEntryIds.add(entryId);
|
||||
projection.lines[index] = {
|
||||
...line,
|
||||
entryId: entryId ?? line.entryId,
|
||||
sourceEntryIds: [...sourceEntryIds],
|
||||
timestampMs: entryKind === "tool_result"
|
||||
? line.timestampMs ?? timestampMs
|
||||
: timestampMs ?? line.timestampMs,
|
||||
};
|
||||
};
|
||||
if (entryKind === "run_error") {
|
||||
// Failure reconciliation can remove transient rows before appending.
|
||||
stampLine(projection.lines.length - 1);
|
||||
} else if (
|
||||
(entryKind === "tool_call" || entryKind === "tool_result") &&
|
||||
projection.lines.length === lineCountBefore
|
||||
) {
|
||||
stampLine(
|
||||
projection.lines.findLastIndex((line) => line.eventId === eventId),
|
||||
);
|
||||
} else {
|
||||
for (
|
||||
let index = Math.max(
|
||||
0,
|
||||
Math.min(lineCountBefore, projection.lines.length - 1),
|
||||
);
|
||||
let index = lineCountBefore;
|
||||
index < projection.lines.length;
|
||||
index += 1
|
||||
) {
|
||||
if (projection.lines[index].eventId === eventId) {
|
||||
projection.lines[index].entryId = entryId;
|
||||
}
|
||||
stampLine(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+49
@@ -1268,6 +1268,55 @@ test("preserves rendered content while reconnecting", async () => {
|
||||
).not.toBeNull();
|
||||
});
|
||||
|
||||
test("Alert keeps its transcript position on reconnect and a fresh Console mount", async () => {
|
||||
vi.stubGlobal("fetch", vi.fn((input: RequestInfo | URL) =>
|
||||
Promise.resolve(Response.json(String(input).includes("/session/history")
|
||||
? emptyHistoryPage()
|
||||
: { availability: "live_protocol" }))
|
||||
));
|
||||
const first = sessionWithUserMessage("before the alert");
|
||||
const second = sessionWithUserMessage("after the alert").entries[0];
|
||||
const full = { ...first, entries: [...first.entries, second] };
|
||||
const alert: ProtocolEvent = {
|
||||
event: "alert",
|
||||
data: {
|
||||
id: "alert-reopen-test",
|
||||
position: { kind: "after_entry", entry_id: first.entries[0].entry_id },
|
||||
level: "warn", source: "compactor", message: "retained alert body", timestamp_ms: 2,
|
||||
},
|
||||
};
|
||||
const expectOrder = () => {
|
||||
const article = screen.getByRole("article", { name: "main transcript" });
|
||||
const rows = Array.from(article.querySelectorAll(".console-line"));
|
||||
expect(rows).toHaveLength(3);
|
||||
expect(rows[0].textContent).toContain("before the alert");
|
||||
expect(rows[1].textContent).toContain("retained alert body");
|
||||
expect(rows[1].classList.contains("alert-line")).toBe(true);
|
||||
expect(rows[2].textContent).toContain("after the alert");
|
||||
};
|
||||
const initial = render(ConsolePage, { data: pageData() });
|
||||
await waitFor(() => expect(multiplexer.subscribe).toHaveBeenCalledOnce());
|
||||
latestListener().onFrame(subscribedFrame(first));
|
||||
protocolEvent(alert);
|
||||
protocolEvent({ event: "user_message", data: {
|
||||
entry_id: second.entry_id, segments: [{ kind: "text", content: "after the alert" }],
|
||||
} });
|
||||
await waitFor(expectOrder);
|
||||
|
||||
latestListener().onStatus?.("closed");
|
||||
latestListener().onStatus?.("connecting");
|
||||
const replay = subscribedFrame(full);
|
||||
replay.message.payload.snapshot.data.events.push(alert, alert);
|
||||
latestListener().onFrame(replay);
|
||||
await waitFor(expectOrder);
|
||||
|
||||
initial.unmount();
|
||||
render(ConsolePage, { data: pageData() });
|
||||
await waitFor(() => expect(multiplexer.subscribe).toHaveBeenCalledTimes(2));
|
||||
latestListener().onFrame(replay);
|
||||
await waitFor(expectOrder);
|
||||
});
|
||||
|
||||
test("reconnect snapshot refreshes lineage and removes stale sibling history", async () => {
|
||||
let historyRequests = 0;
|
||||
vi.stubGlobal(
|
||||
|
||||
Reference in New Issue
Block a user