fix: preserve console alert order and improve alert colors

This commit is contained in:
2026-10-16 05:28:17 +09:00
parent b67aa2de7d
commit a2cec5afd1
25 changed files with 2098 additions and 90 deletions
+54
View File
@@ -1497,6 +1497,11 @@ pub enum Event {
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
pub struct Alert {
/// Stable only within the emitting live Worker; not durable alert history.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub position: Option<AlertPosition>,
pub level: AlertLevel,
pub source: AlertSource,
pub message: String,
@@ -1504,6 +1509,18 @@ pub struct Alert {
pub timestamp_ms: i64,
}
/// Display anchor captured at emission, resolved when pending text is committed.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum AlertPosition {
Start,
AfterEntry { entry_id: String },
AfterToolCall { call_id: String },
AfterInFlight { block_id: u64 },
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
@@ -1704,6 +1721,9 @@ pub struct InFlightCompaction {
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
pub struct InFlightSnapshot {
/// Live-Worker identities parallel to the blocks actually included below.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub block_ids: Vec<u64>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub blocks: Vec<InFlightBlock>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
@@ -2721,6 +2741,7 @@ mod tests {
},
state: WorkerStatus::Running.into(),
in_flight: InFlightSnapshot {
block_ids: Vec::new(),
blocks: vec![
InFlightBlock::Text {
text: "hel".into(),
@@ -3048,9 +3069,42 @@ mod tests {
}
}
#[test]
fn alert_ordering_metadata_is_optional_and_uses_tagged_wire_anchors() {
let legacy = serde_json::json!({ "level": "warn", "source": "worker", "message": "old", "timestamp_ms": 42 });
let alert: Alert = serde_json::from_value(legacy.clone()).unwrap();
assert!(alert.id.is_none());
assert!(alert.position.is_none());
assert_eq!(serde_json::to_value(alert).unwrap(), legacy);
for position in [
serde_json::json!({ "kind": "start" }),
serde_json::json!({ "kind": "after_entry", "entry_id": "entry-1" }),
serde_json::json!({ "kind": "after_tool_call", "call_id": "call-1" }),
serde_json::json!({ "kind": "after_in_flight", "block_id": 7 }),
] {
let mut wire = legacy.clone();
wire["id"] = serde_json::json!("alert-1");
wire["position"] = position;
let alert: Alert = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(serde_json::to_value(alert).unwrap(), wire);
}
let old_snapshot: InFlightSnapshot = serde_json::from_value(serde_json::json!({})).unwrap();
assert!(old_snapshot.block_ids.is_empty());
assert_eq!(
serde_json::to_value(old_snapshot).unwrap(),
serde_json::json!({})
);
let wire =
serde_json::json!({"blocks": [{"kind": "text", "text": "pending"}], "block_ids": [7]});
let snapshot: InFlightSnapshot = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(serde_json::to_value(snapshot).unwrap(), wire);
}
#[test]
fn event_alert_format() {
let event = Event::Alert(Alert {
id: None,
position: None,
level: AlertLevel::Warn,
source: AlertSource::Compactor,
message: "compaction failed".into(),
+14 -13
View File
@@ -4,19 +4,19 @@ use serde_json::{Map, Value};
use ts_rs::{Config, TS};
use crate::{
Alert, AlertLevel, AlertSource, CommandEvent, CommandSnapshot, CommandStatus, CommandStream,
CommandStreamSlice, CompactionLifecycle, CompactionLifecycleState, CompactionPhase,
CompactionTrigger, CompletionContext, CompletionEntry, CompletionKind, ContextTokenSource,
ContextUsage, ErrorCode, Event, FeatureInvocation, FeatureInvocationDescriptor,
FeatureInvocationIdentity, FeatureInvocationResult, FeatureInvocationStatus,
FeatureInvocationSyntax, Greeting, InFlightBlock, InFlightCompaction, InFlightSnapshot,
InFlightToolCallState, InternalWorkerKind, InternalWorkerRef, InternalWorkerSnapshot,
InvocationArgumentDescriptor, InvocationArgumentType, InvocationArgumentValue,
InvocationClientAdapter, InvocationCompletion, InvocationValue, InvokeKind, MemoryWorkerEvent,
Method, PasteArtifactAvailability, PasteArtifactMediaType, PasteArtifactRef,
PendingSubmissionSummary, PendingSubmissionsSnapshot, Permission, ReasoningConfig,
RewindSummary, RunFailureKind, RunResult, RunResumeSource, RunYieldReason, ScopeRule, Segment,
SessionContentPart, SessionConversationTurn, SessionEntryProvenance,
Alert, AlertLevel, AlertPosition, AlertSource, CommandEvent, CommandSnapshot, CommandStatus,
CommandStream, CommandStreamSlice, CompactionLifecycle, CompactionLifecycleState,
CompactionPhase, CompactionTrigger, CompletionContext, CompletionEntry, CompletionKind,
ContextTokenSource, ContextUsage, ErrorCode, Event, FeatureInvocation,
FeatureInvocationDescriptor, FeatureInvocationIdentity, FeatureInvocationResult,
FeatureInvocationStatus, FeatureInvocationSyntax, Greeting, InFlightBlock, InFlightCompaction,
InFlightSnapshot, InFlightToolCallState, InternalWorkerKind, InternalWorkerRef,
InternalWorkerSnapshot, InvocationArgumentDescriptor, InvocationArgumentType,
InvocationArgumentValue, InvocationClientAdapter, InvocationCompletion, InvocationValue,
InvokeKind, MemoryWorkerEvent, Method, PasteArtifactAvailability, PasteArtifactMediaType,
PasteArtifactRef, PendingSubmissionSummary, PendingSubmissionsSnapshot, Permission,
ReasoningConfig, RewindSummary, RunFailureKind, RunResult, RunResumeSource, RunYieldReason,
ScopeRule, Segment, SessionContentPart, SessionConversationTurn, SessionEntryProvenance,
SessionHistoryLineageBoundary, SessionHistoryPage, SessionMessageRole, SessionSnapshot,
SessionSnapshotEntry, SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition,
SymlinkPolicy, ToolResultDisposition, TurnResult, UploadedFileAvailability, UploadedFileRef,
@@ -65,6 +65,7 @@ pub fn generated_protocol_types() -> String {
);
push_decl::<AlertLevel>(&cfg, &mut output);
push_decl::<AlertPosition>(&cfg, &mut output);
push_decl::<AlertSource>(&cfg, &mut output);
push_decl::<CompletionKind>(&cfg, &mut output);
push_decl::<CompletionContext>(&cfg, &mut output);
+5
View File
@@ -619,6 +619,11 @@ async fn run_protocol_session(
event = events.recv() => {
match event {
Ok(event) => {
let Ok(event) = handle.alerter.freshen_event(event) else {
// The retained anchor was evicted; reconnect must obtain
// a paired session/Alert snapshot rather than a stale row.
return;
};
if !send_protocol_event(&peer, event).await {
return;
}
+3
View File
@@ -4319,6 +4319,8 @@ mod completion_flow_tests {
let mut app = App::new("test".into());
let message = "mid-run compaction failed: summary unavailable";
app.handle_worker_event(Event::Alert(protocol::Alert {
id: None,
position: None,
level: AlertLevel::Error,
source: AlertSource::Compactor,
message: message.into(),
@@ -4402,6 +4404,7 @@ mod completion_flow_tests {
},
state: test_worker_state(WorkerStatus::Running),
in_flight: InFlightSnapshot {
block_ids: Vec::new(),
blocks: vec![
InFlightBlock::Thinking {
text: "why".into(),
+1
View File
@@ -4821,6 +4821,7 @@ mod ws_tests {
},
state: protocol::WorkerStateSnapshot::initial(),
in_flight: protocol::InFlightSnapshot {
block_ids: Vec::new(),
blocks: Vec::new(),
commands: Vec::new(),
compaction: None,
+1
View File
@@ -11818,6 +11818,7 @@ mod tests {
},
state: protocol::WorkerStatus::Running.into(),
in_flight: protocol::InFlightSnapshot {
block_ids: Vec::new(),
blocks: Vec::new(),
commands: Vec::new(),
compaction: None,
+91 -1
View File
@@ -2901,12 +2901,33 @@ where
let mut events = streams.events;
let mut entry_events = streams.log_entries;
let bridge_worker_state = worker_state.clone();
let bridge_handle = handle.clone();
let bridge_task = match self.spawn_on_adapter_runtime(async move {
loop {
tokio::select! {
event = events.recv() => {
match event {
Ok(mut event) => {
Ok(event) => {
let mut event = match bridge_handle.alerter.freshen_event(event) {
Ok(event) => event,
Err(_) => {
// This shared bridge also carries Worker state: do
// not terminate it when a bounded Alert was evicted.
// Replace both receivers with the paired snapshot.
let replacement = subscribe_worker_protocol_session(&bridge_handle);
events = replacement.events;
entry_events = replacement.log_entries;
let mut snapshot = replacement.snapshot_event;
if apply_protocol_worker_state(&bridge_worker_state, &mut snapshot).is_err() {
break;
}
let _ = bridge_context.publish_protocol_event(snapshot);
for alert in replacement.alert_snapshot {
let _ = bridge_context.publish_protocol_event(Event::Alert(alert));
}
continue;
}
};
match apply_protocol_worker_state(&bridge_worker_state, &mut event) {
Ok(true) => {
let _ = bridge_context.publish_protocol_event(event);
@@ -4525,6 +4546,10 @@ where
},
};
if let Some(event) = event {
let Ok(event) = relay_handle.alerter.freshen_event(event) else {
// An evicted Alert needs a fresh paired snapshot.
break;
};
tokio::select! {
biased;
_ = relay_handle.protocol_closed() => break,
@@ -5177,6 +5202,71 @@ mod tests {
assert!(backend.stop_worker(&request.worker_ref).is_accepted());
}
#[cfg(feature = "ws-server")]
#[test]
#[serial_test::serial(worker_allocation)]
fn protocol_transport_reconnects_instead_of_forwarding_an_evicted_alert() {
let root = tempfile::tempdir().unwrap();
let runtime_store = root.path().join("runtime");
let backend = Arc::new(
WorkerRuntimeExecutionBackend::new(
ProfileRuntimeWorkerFactory::new(root.path())
.with_runtime_store_dir(&runtime_store)
.with_runtime_id("runtime-subjektiv-test"),
)
.unwrap(),
);
let request = backend
.run_on_adapter_runtime(async move {
Ok(saved_subjektiv_restore_fixture(&runtime_store, true).await)
})
.unwrap();
assert!(matches!(
backend.restore_worker(request.clone()),
WorkerExecutionSpawnResult::Connected { .. }
));
let execution = backend.workers.lock().unwrap()[&request.worker_ref].clone();
let mut transport = Arc::clone(&backend)
.attach_worker_protocol(&request.worker_ref)
.unwrap();
// A queued frame whose ID has left the bounded buffer must not be
// forwarded with its now-unresolvable pending-text position.
execution
.handle
.send_event(Event::Alert(protocol::Alert {
id: Some("evicted-alert".into()),
position: Some(protocol::AlertPosition::AfterInFlight { block_id: 99 }),
level: protocol::AlertLevel::Warn,
source: protocol::AlertSource::Worker,
message: "stale alert".into(),
timestamp_ms: 1,
}))
.unwrap();
backend.run_on_adapter_runtime(async move {
tokio::time::timeout(Duration::from_secs(5), async {
while let Some(event) = transport.events.recv().await {
assert!(!matches!(event, Event::Alert(alert) if alert.id.as_deref() == Some("evicted-alert")));
}
}).await.map_err(|_| "stale Alert did not close its transport".to_string())?;
Ok(())
}).unwrap();
execution.handle.alerter.alert(
protocol::AlertLevel::Warn,
protocol::AlertSource::Worker,
"retained alert".into(),
);
let mut replacement = Arc::clone(&backend)
.attach_worker_protocol(&request.worker_ref)
.unwrap();
backend.run_on_adapter_runtime(async move {
let event = tokio::time::timeout(Duration::from_secs(5), replacement.events.recv())
.await.map_err(|_| "replacement Alert replay timed out".to_string())?;
assert!(matches!(event, Some(Event::Alert(alert)) if alert.message == "retained alert" && alert.id.is_some()));
Ok(())
}).unwrap();
assert!(backend.stop_worker(&request.worker_ref).is_accepted());
}
fn profile_stop_fixture(
root: &Path,
) -> (WorkerRuntimeExecutionBackend, WorkerExecutionRestoreRequest) {
+4
View File
@@ -1532,6 +1532,8 @@ mod tests {
let mut writer = JsonLineWriter::new(writer_half);
writer
.write(&Event::Alert(Alert {
id: None,
position: None,
level: AlertLevel::Warn,
source: AlertSource::Worker,
message: "connect-time alert".into(),
@@ -1763,6 +1765,8 @@ mod tests {
let mut writer = JsonLineWriter::new(stream);
writer
.write(&Event::Alert(Alert {
id: None,
position: None,
level: AlertLevel::Warn,
source: AlertSource::Worker,
message: "replayed alert".into(),
+142 -14
View File
@@ -1,11 +1,11 @@
use std::{
collections::HashSet,
sync::{Arc, Mutex, MutexGuard},
sync::{Arc, Mutex, MutexGuard, Weak},
};
use protocol::{
CommandEvent, CommandSnapshot, CommandStatus, CommandStream, CommandStreamSlice, Event,
InFlightBlock, InFlightSnapshot, InFlightToolCallState,
AlertPosition, CommandEvent, CommandSnapshot, CommandStatus, CommandStream, CommandStreamSlice,
Event, InFlightBlock, InFlightSnapshot, InFlightToolCallState,
};
use session_store::{LoggedContentPart, LoggedItem};
use tokio::sync::broadcast;
@@ -38,6 +38,7 @@ enum TrackedBlock {
block_id: InFlightBlockId,
text: String,
finished: bool,
alert_positions: Vec<Weak<Mutex<AlertPosition>>>,
},
Thinking {
block_id: InFlightBlockId,
@@ -78,6 +79,7 @@ impl InFlightEvents {
block_id,
text: String::new(),
finished: false,
alert_positions: Vec::new(),
});
block_id
}
@@ -213,10 +215,11 @@ impl InFlightEvents {
pub(crate) fn clear_for_committed_item_then<R>(
&self,
item: &LoggedItem,
entry_id: Option<&str>,
f: impl FnOnce() -> R,
) -> R {
let mut inner = self.lock();
inner.clear_for_committed_item(item);
inner.clear_for_committed_item(item, entry_id);
f()
}
@@ -378,7 +381,7 @@ impl InFlightInner {
}
}
fn clear_for_committed_item(&mut self, item: &LoggedItem) {
fn clear_for_committed_item(&mut self, item: &LoggedItem, entry_id: Option<&str>) {
match item {
LoggedItem::Message { role, content }
if matches!(role, session_store::LoggedRole::Assistant) =>
@@ -391,7 +394,7 @@ impl InFlightInner {
})
.collect::<String>();
if !text.is_empty() {
self.remove_first_text_matching(&text);
self.remove_first_text_matching(&text, entry_id);
}
}
LoggedItem::Reasoning {
@@ -420,13 +423,55 @@ impl InFlightInner {
}
}
pub(crate) fn capture_alert_position(&mut self) -> Option<Arc<Mutex<AlertPosition>>> {
// Prune evicted alerts even when their streamed block has not committed yet.
for block in &mut self.blocks {
if let TrackedBlock::Text {
alert_positions, ..
} = block
{
alert_positions.retain(|position| position.strong_count() > 0);
}
}
for block in self.blocks.iter_mut().rev() {
match block {
TrackedBlock::Text {
block_id,
text,
alert_positions,
..
} if !text.is_empty() => {
let position = Arc::new(Mutex::new(AlertPosition::AfterInFlight {
block_id: block_id.0,
}));
alert_positions.push(Arc::downgrade(&position));
return Some(position);
}
TrackedBlock::ToolCall { id, .. } => {
return Some(Arc::new(Mutex::new(AlertPosition::AfterToolCall {
call_id: id.clone(),
})));
}
// Hidden reasoning and empty text are not conversation rows.
_ => {}
}
}
None
}
fn snapshot(&self) -> InFlightSnapshot {
let (block_ids, blocks) = self
.blocks
.iter()
.filter_map(|block| {
block
.to_snapshot_block()
.map(|snapshot| (block.block_id().0, snapshot))
})
.unzip();
InFlightSnapshot {
blocks: self
.blocks
.iter()
.filter_map(TrackedBlock::to_snapshot_block)
.collect(),
block_ids,
blocks,
commands: self.commands.clone(),
compaction: self.compaction.clone(),
}
@@ -444,11 +489,25 @@ impl InFlightInner {
}
}
fn remove_first_text_matching(&mut self, committed: &str) -> bool {
fn remove_first_text_matching(&mut self, committed: &str, entry_id: Option<&str>) -> bool {
if let Some(index) = self.blocks.iter().position(|block| match block {
TrackedBlock::Text { text, .. } => text == committed,
_ => false,
}) {
if let (
TrackedBlock::Text {
alert_positions, ..
},
Some(entry_id),
) = (&self.blocks[index], entry_id)
{
for position in alert_positions.iter().filter_map(Weak::upgrade) {
*position.lock().expect("alert position mutex poisoned") =
AlertPosition::AfterEntry {
entry_id: entry_id.to_owned(),
};
}
}
self.blocks.remove(index);
true
} else {
@@ -559,6 +618,72 @@ mod tests {
}
}
#[test]
fn alert_watchers_are_pruned_and_removed_with_their_text_block() {
let (tx, _) = broadcast::channel(16);
let in_flight = InFlightEvents::new(tx);
let text = in_flight.start_text_block();
in_flight.text_delta(text, "same".into());
let first = in_flight.snapshot_guard().capture_alert_position().unwrap();
drop(first);
let second = in_flight.snapshot_guard().capture_alert_position().unwrap();
{
let guard = in_flight.snapshot_guard();
let TrackedBlock::Text {
alert_positions, ..
} = &guard.blocks[0]
else {
panic!("expected text")
};
assert_eq!(alert_positions.len(), 1);
}
let later = in_flight.start_text_block();
in_flight.text_delta(later, "same".into());
let third = in_flight.snapshot_guard().capture_alert_position().unwrap();
let item = LoggedItem::Message {
role: session_store::LoggedRole::Assistant,
content: vec![LoggedContentPart::Text {
text: "same".into(),
}],
};
in_flight.clear_for_committed_item_then(&item, Some("first-entry"), || ());
assert_eq!(
*second.lock().unwrap(),
AlertPosition::AfterEntry {
entry_id: "first-entry".into()
}
);
assert_eq!(
*third.lock().unwrap(),
AlertPosition::AfterInFlight { block_id: later.0 }
);
in_flight.clear();
assert!(in_flight.snapshot_guard().blocks.is_empty());
assert_eq!(
*third.lock().unwrap(),
AlertPosition::AfterInFlight { block_id: later.0 }
);
}
#[test]
fn snapshot_ids_are_parallel_to_emitted_blocks_not_filtered_placeholders() {
let (tx, _) = broadcast::channel(16);
let in_flight = InFlightEvents::new(tx);
in_flight.start_text_block();
let hidden = in_flight.thinking_start();
in_flight.thinking_done(hidden, String::new());
let visible = in_flight.start_text_block();
in_flight.text_delta(visible, "shown".into());
let tool = in_flight.tool_call_start("call-1".into(), "Read".into());
let snapshot = snapshot_from_guard(&in_flight.snapshot_guard());
assert_eq!(snapshot.blocks.len(), 2);
assert_eq!(snapshot.block_ids, vec![visible.0, tool.0]);
assert_eq!(
snapshot.block_ids,
snapshot_from_guard(&in_flight.snapshot_guard()).block_ids
);
}
#[test]
fn snapshot_boundary_does_not_duplicate_or_gap_delta_sent_after_subscribe() {
let (working_event_tx, _) = broadcast::channel(16);
@@ -621,7 +746,7 @@ mod tests {
// This mirrors Worker::append_entry ordering: clear in-flight first,
// then publish the finalized AssistantItem. Identity-aware clients
// reconcile this durable commit with the preceding streamed block.
in_flight_for_commit.clear_for_committed_item_then(&assistant_item, || {
in_flight_for_commit.clear_for_committed_item_then(&assistant_item, None, || {
sink_for_commit.publish(assistant_entry);
});
committed_tx.send(()).unwrap();
@@ -670,7 +795,7 @@ mod tests {
entry: crate::session_history::test_logged_history_entry(assistant_item.clone()),
};
in_flight.clear_for_committed_item_then(&assistant_item, || {
in_flight.clear_for_committed_item_then(&assistant_item, None, || {
sink.publish(assistant_entry);
});
@@ -698,6 +823,7 @@ mod tests {
text: "done".into(),
}],
},
None,
|| (),
);
@@ -723,6 +849,7 @@ mod tests {
encrypted_content: Some("opaque".into()),
signature: None,
},
None,
|| (),
);
@@ -746,6 +873,7 @@ mod tests {
encrypted_content: Some("opaque".into()),
signature: None,
},
None,
|| (),
);
+380 -8
View File
@@ -6,7 +6,7 @@
//!
//! Each alert is broadcast on the shared `Event` channel and
//! also appended to an in-memory buffer so that clients connecting
//! after the fact still see everything emitted during the session.
//! after the fact can replay the bounded recent prefix from the same live Worker.
use std::collections::VecDeque;
use std::sync::{Arc, Mutex};
@@ -14,7 +14,9 @@ use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::broadcast;
use protocol::{Alert, AlertLevel, AlertSource, Event};
use protocol::{Alert, AlertLevel, AlertPosition, AlertSource, Event};
use crate::{in_flight::InFlightEvents, segment_log_sink::SegmentLogSink};
/// Upper bound on buffered alerts. When exceeded, the oldest
/// entries are discarded so a long-running session cannot leak
@@ -27,9 +29,53 @@ pub struct Alerter {
inner: Arc<Inner>,
}
/// A queued identified Alert is no longer retained by this live Worker.
/// Transports must disconnect/resnapshot instead of forwarding its stale anchor.
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
#[error("alert is no longer retained; reconnect for a fresh snapshot")]
pub struct AlertReplayRequired;
struct Inner {
working_event_tx: broadcast::Sender<Event>,
buffer: Mutex<VecDeque<Alert>>,
buffer: Mutex<VecDeque<BufferedAlert>>,
position_source: Mutex<Option<PositionSource>>,
}
#[derive(Clone)]
struct PositionSource {
sink: SegmentLogSink,
in_flight: Option<InFlightEvents>,
}
#[derive(Clone)]
struct BufferedAlert {
alert: Alert,
position: Option<Arc<Mutex<AlertPosition>>>,
}
impl BufferedAlert {
fn snapshot(&self) -> Alert {
let mut alert = self.alert.clone();
alert.position = self.position.as_ref().map(|position| {
position
.lock()
.expect("alert position mutex poisoned")
.clone()
});
alert
}
}
impl PositionSource {
fn capture(&self) -> Arc<Mutex<AlertPosition>> {
// Same order as commit/snapshot: in-flight -> sink. Never hold the
// alerter buffer lock while acquiring either of these locks.
let mut in_flight = self.in_flight.as_ref().map(InFlightEvents::snapshot_guard);
in_flight
.as_mut()
.and_then(|guard| guard.capture_alert_position())
.unwrap_or_else(|| Arc::new(Mutex::new(self.sink.alert_position())))
}
}
impl Alerter {
@@ -38,10 +84,23 @@ impl Alerter {
inner: Arc::new(Inner {
working_event_tx,
buffer: Mutex::new(VecDeque::with_capacity(MAX_BUFFERED_ALERTS)),
position_source: Mutex::new(None),
}),
}
}
pub(crate) fn attach_position_source(
&self,
sink: SegmentLogSink,
in_flight: Option<InFlightEvents>,
) {
*self
.inner
.position_source
.lock()
.expect("alert source mutex poisoned") = Some(PositionSource { sink, in_flight });
}
/// Record and broadcast an alert.
///
/// The broadcast may have no subscribers (e.g. during Worker
@@ -55,35 +114,88 @@ impl Alerter {
/// exactly once: older ones from the snapshot, newer ones from
/// the freshly-subscribed receiver.
pub fn alert(&self, level: AlertLevel, source: AlertSource, message: String) {
self.alert_at(level, source, message, now_ms());
}
fn alert_at(&self, level: AlertLevel, source: AlertSource, message: String, timestamp_ms: i64) {
let source_handle = self
.inner
.position_source
.lock()
.expect("alert source mutex poisoned")
.clone();
let position = source_handle.map(|source| source.capture());
let alert = Alert {
id: Some(uuid::Uuid::now_v7().to_string()),
position: None,
level,
source,
message,
timestamp_ms: now_ms(),
timestamp_ms,
};
if let Ok(mut buf) = self.inner.buffer.lock() {
if buf.len() >= MAX_BUFFERED_ALERTS {
buf.pop_front();
}
buf.push_back(alert.clone());
let _ = self.inner.working_event_tx.send(Event::Alert(alert));
let buffered = BufferedAlert { alert, position };
let event = Event::Alert(buffered.snapshot());
buf.push_back(buffered);
let _ = self.inner.working_event_tx.send(event);
}
}
/// Refresh a queued Alert immediately before transport forwarding.
///
/// The independent log lane may have already delivered the text commit, so
/// the broadcast's copied in-flight anchor is not authoritative anymore.
/// Missing identified alerts require reconnect/resnapshot; legacy alerts
/// without IDs and non-Alert events pass through unchanged.
pub fn freshen_event(&self, event: Event) -> Result<Event, AlertReplayRequired> {
let Event::Alert(ref alert) = event else {
return Ok(event);
};
let Some(id) = alert.id.as_ref() else {
return Ok(event);
};
let buffer = self
.inner
.buffer
.lock()
.expect("alerter buffer mutex poisoned");
buffer
.iter()
.find(|buffered| buffered.alert.id.as_ref() == Some(id))
.map(|buffered| Event::Alert(buffered.snapshot()))
.ok_or(AlertReplayRequired)
}
/// Subscribe and atomically snapshot the current buffer.
///
/// The returned snapshot contains alerts emitted before
/// this call; the receiver will deliver alerts emitted
/// after. An alert cannot appear in both.
pub fn subscribe_with_snapshot(&self) -> (Vec<Alert>, broadcast::Receiver<Event>) {
let (snapshot, rx) = self.subscribe_with_deferred_snapshot();
(snapshot(), rx)
}
// Resolve positions after the session snapshot has been captured. Otherwise
// a text commit between the two snapshots can leave only its old block ID
// in replay, although that block is already absent from the session snapshot.
pub(crate) fn subscribe_with_deferred_snapshot(
&self,
) -> (impl FnOnce() -> Vec<Alert>, broadcast::Receiver<Event>) {
let buf = self
.inner
.buffer
.lock()
.expect("alerter buffer mutex poisoned");
let rx = self.inner.working_event_tx.subscribe();
let snapshot: Vec<Alert> = buf.iter().cloned().collect();
(snapshot, rx)
let snapshot: Vec<BufferedAlert> = buf.iter().cloned().collect();
(
move || snapshot.iter().map(BufferedAlert::snapshot).collect(),
rx,
)
}
}
@@ -98,6 +210,266 @@ fn now_ms() -> i64 {
mod tests {
use super::*;
fn positioned_alerter() -> (Alerter, SegmentLogSink, InFlightEvents) {
let (tx, _) = broadcast::channel(1024);
let in_flight = InFlightEvents::new(tx.clone());
let sink = SegmentLogSink::new();
let alerter = Alerter::new(tx);
alerter.attach_position_source(sink.clone(), Some(in_flight.clone()));
(alerter, sink, in_flight)
}
#[test]
fn queued_alert_uses_committed_anchor_when_log_lane_was_delivered_first() {
let (alerter, sink, in_flight) = positioned_alerter();
let text = in_flight.start_text_block();
in_flight.text_delta(text, "finished text".into());
in_flight.text_done(text, "finished text".into());
let (_, mut events) = alerter.subscribe_with_snapshot();
let (_, mut log_entries) = sink.subscribe_with_snapshot();
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "queued".into());
let queued = events.try_recv().unwrap();
let Event::Alert(ref original) = queued else {
panic!("expected alert")
};
assert!(matches!(
original.position,
Some(AlertPosition::AfterInFlight { .. })
));
let original_id = original.id.clone();
let item: session_store::LoggedItem = agen::Item::assistant_message("finished text").into();
let entry = crate::session_history::test_logged_history_entry(item.clone());
let entry_id = entry.metadata.entry_id.0.clone();
in_flight.clear_for_committed_item_then(&item, Some(&entry_id), || {
sink.publish(session_store::LogEntry::AnnotatedAssistantItem { ts: 42, entry });
});
let committed =
crate::ipc::protocol_session::live_log_entry_event(log_entries.try_recv().unwrap())
.unwrap();
assert!(
matches!(committed, Event::SessionEntryCommitted { entry } if entry.entry_id == entry_id)
);
let Event::Alert(forwarded) = alerter.freshen_event(queued).unwrap() else {
panic!("expected alert")
};
assert_eq!(forwarded.id, original_id);
assert_eq!(
forwarded.position,
Some(AlertPosition::AfterEntry { entry_id })
);
let replay = alerter.subscribe_with_snapshot().0;
assert_eq!(
serde_json::to_value(forwarded).unwrap(),
serde_json::to_value(&replay[0]).unwrap()
);
}
#[test]
fn evicted_or_unknown_alert_ids_require_replay_but_legacy_events_pass_through() {
let (alerter, _, _) = positioned_alerter();
let (_, mut events) = alerter.subscribe_with_snapshot();
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "old".into());
let queued = events.try_recv().unwrap();
for _ in 0..MAX_BUFFERED_ALERTS {
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "later".into());
}
assert!(matches!(
alerter.freshen_event(queued),
Err(AlertReplayRequired)
));
let unknown = Event::Alert(Alert {
id: Some("not-retained".into()),
position: Some(AlertPosition::AfterInFlight { block_id: 1 }),
level: AlertLevel::Warn,
source: AlertSource::Worker,
message: "unknown".into(),
timestamp_ms: 42,
});
assert!(matches!(
alerter.freshen_event(unknown),
Err(AlertReplayRequired)
));
let legacy = Event::Alert(Alert {
id: None,
position: None,
level: AlertLevel::Warn,
source: AlertSource::Worker,
message: "legacy".into(),
timestamp_ms: 42,
});
let expected = serde_json::to_value(&legacy).unwrap();
assert_eq!(
serde_json::to_value(alerter.freshen_event(legacy).unwrap()).unwrap(),
expected
);
assert!(matches!(
alerter.freshen_event(Event::Shutdown),
Ok(Event::Shutdown)
));
}
#[test]
fn rowless_run_transitions_preserve_visible_alert_anchor_live_and_on_replay() {
use session_store::LogEntry;
let (alerter, sink, _) = positioned_alerter();
let entry = crate::session_history::test_logged_history_entry(
agen::Item::assistant_message("visible"),
);
let expected = Some(AlertPosition::AfterEntry {
entry_id: entry.metadata.entry_id.0.clone(),
});
sink.publish(LogEntry::AnnotatedAssistantItem { ts: 1, entry });
let (_, mut events) = alerter.subscribe_with_snapshot();
let transitions = [
LogEntry::RunCancelled {
ts: 2,
entry_id: None,
},
LogEntry::RunYielded {
ts: 3,
entry_id: None,
reason: protocol::RunYieldReason::Compaction,
active_run_turn_count: 1,
},
LogEntry::RunResumed {
ts: 4,
entry_id: None,
source: protocol::RunResumeSource::Compaction,
active_run_turn_count: 1,
},
];
for mut transition in transitions {
transition.ensure_session_entry_id();
sink.publish(transition);
alerter.alert(
AlertLevel::Warn,
AlertSource::Worker,
"after transition".into(),
);
let Event::Alert(live) = alerter.freshen_event(events.try_recv().unwrap()).unwrap()
else {
panic!("expected alert")
};
assert_eq!(live.position, expected);
}
let replay = alerter.subscribe_with_snapshot().0;
assert_eq!(replay.len(), 3);
assert!(replay.iter().all(|alert| alert.position == expected));
let (entries, _) = sink.subscribe_with_snapshot();
// Rebuilding the sink cache from retained entries must use the same rule.
sink.replace_silent(entries);
alerter.alert(
AlertLevel::Warn,
AlertSource::Worker,
"after cache rebuild".into(),
);
assert_eq!(
alerter.subscribe_with_snapshot().0.last().unwrap().position,
expected
);
}
#[test]
fn identical_alerts_in_same_millisecond_keep_distinct_ids_across_replay() {
let (alerter, _, _) = positioned_alerter();
let (_, mut rx) = alerter.subscribe_with_snapshot();
for _ in 0..2 {
alerter.alert_at(AlertLevel::Warn, AlertSource::Worker, "same".into(), 42);
}
let (snapshot, _) = alerter.subscribe_with_snapshot();
assert_ne!(snapshot[0].id, snapshot[1].id);
for expected in &snapshot {
let Event::Alert(live) = rx.try_recv().unwrap() else {
panic!("expected alert")
};
assert!(live.id.is_some());
assert_eq!(live.id, expected.id);
assert_eq!(live.timestamp_ms, 42);
assert_eq!(live.position, Some(AlertPosition::Start));
assert_eq!(live.position, expected.position);
}
let (replay, _) = alerter.subscribe_with_snapshot();
assert_eq!(
serde_json::to_value(snapshot).unwrap(),
serde_json::to_value(replay).unwrap()
);
}
#[test]
fn tool_call_anchor_survives_commit_and_new_text() {
let (alerter, sink, in_flight) = positioned_alerter();
in_flight.tool_call_start("call-1".into(), "Read".into());
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "tool alert".into());
let item: session_store::LoggedItem = agen::Item::tool_call("call-1", "Read", "{}").into();
let entry = session_store::LogEntry::AnnotatedAssistantItem {
ts: 1,
entry: crate::session_history::test_logged_history_entry(item.clone()),
};
in_flight.clear_for_committed_item_then(&item, Some("entry-tool"), || sink.publish(entry));
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "after commit".into());
let text = in_flight.start_text_block();
in_flight.text_delta(text, "next".into());
let (snapshot, _) = alerter.subscribe_with_snapshot();
for alert in snapshot {
assert_eq!(
alert.position,
Some(AlertPosition::AfterToolCall {
call_id: "call-1".into()
})
);
}
}
#[test]
fn eviction_drops_position_ownership_and_clear_does_not_retarget_old_alerts() {
let (alerter, _, in_flight) = positioned_alerter();
let text = in_flight.start_text_block();
in_flight.text_delta(text, "discarded".into());
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "old".into());
let weak = {
let buffer = alerter.inner.buffer.lock().unwrap();
Arc::downgrade(buffer[0].position.as_ref().unwrap())
};
for _ in 0..MAX_BUFFERED_ALERTS {
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "new".into());
}
assert!(weak.upgrade().is_none());
let (before, _) = alerter.subscribe_with_snapshot();
in_flight.clear();
let next = in_flight.start_text_block();
in_flight.text_delta(next, "discarded".into());
let item = session_store::LoggedItem::Message {
role: session_store::LoggedRole::Assistant,
content: vec![session_store::LoggedContentPart::Text {
text: "discarded".into(),
}],
};
in_flight.clear_for_committed_item_then(&item, Some("unrelated-entry"), || ());
let (after, _) = alerter.subscribe_with_snapshot();
assert_eq!(before.len(), MAX_BUFFERED_ALERTS);
assert_eq!(
serde_json::to_value(before).unwrap(),
serde_json::to_value(after).unwrap()
);
}
#[test]
fn snapshot_does_not_acquire_in_flight_lock_under_buffer_lock() {
let (alerter, _, in_flight) = positioned_alerter();
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "before".into());
let guard = in_flight.snapshot_guard();
let cloned = alerter.clone();
let (tx, rx) = std::sync::mpsc::channel();
let thread = std::thread::spawn(move || {
let (snapshot, _) = cloned.subscribe_with_snapshot();
tx.send(snapshot).unwrap();
});
let result = rx.recv_timeout(std::time::Duration::from_secs(5));
drop(guard);
thread.join().unwrap();
assert_eq!(result.unwrap().len(), 1);
}
#[test]
fn alert_broadcasts_to_existing_subscriber() {
let (tx, _keep) = broadcast::channel::<Event>(8);
+2 -2
View File
@@ -30,12 +30,12 @@ fn subscribe_before_snapshot<Subscription, Snapshot>(
pub fn subscribe_worker_protocol_session(handle: &WorkerHandle) -> WorkerProtocolSessionStreams {
let ((snapshot_event, log_entries), (alert_snapshot, events)) = subscribe_before_snapshot(
|| handle.alerter.subscribe_with_snapshot(),
|| handle.alerter.subscribe_with_deferred_snapshot(),
|| handle.snapshot_event_with_entry_subscription(),
);
WorkerProtocolSessionStreams {
snapshot_event,
alert_snapshot,
alert_snapshot: alert_snapshot(),
log_entries,
events,
}
+5
View File
@@ -154,6 +154,11 @@ async fn handle_connection(stream: tokio::net::UnixStream, handle: WorkerHandle)
event = streams.events.recv() => {
match event {
Ok(event) => {
let Ok(event) = handle.alerter.freshen_event(event) else {
// The queued Alert was evicted; reconnect must rebuild
// the bounded prefix rather than forward a stale anchor.
break;
};
if writer.write_event(&event).await.is_err() {
break;
}
+129 -10
View File
@@ -24,6 +24,7 @@
use std::sync::{Arc, Mutex as StdMutex};
use protocol::{AlertPosition, SessionSnapshotEntryData};
use session_store::LogEntry;
use tokio::sync::broadcast;
@@ -46,7 +47,7 @@ pub struct SegmentLogSink {
struct SinkInner {
/// Full session log mirror in commit order. Reset on session swap
/// (compaction / fork) via [`SegmentLogSink::reset_with_initial`].
mirror: StdMutex<Vec<LogEntry>>,
mirror: StdMutex<Mirror>,
/// Broadcast channel for live entry updates. The same `Sender`
/// survives session swaps so existing subscribers keep their
/// receiver — they observe the swap as a freshly broadcast
@@ -54,6 +55,46 @@ struct SinkInner {
session_entry_tx: broadcast::Sender<LogEntry>,
}
struct Mirror {
entries: Vec<LogEntry>,
alert_position: AlertPosition,
}
impl Mirror {
fn new(entries: Vec<LogEntry>) -> Self {
let alert_position = latest_alert_position(&entries).unwrap_or(AlertPosition::Start);
Self {
entries,
alert_position,
}
}
}
// Use the public projection so private reasoning/response boundaries cannot become
// anchors. Results/accounting update existing rows, and non-error Run lifecycle
// transitions have no conversation row of their own.
fn latest_alert_position(entries: &[LogEntry]) -> Option<AlertPosition> {
session_store::public_snapshot::project_current_session_snapshot(entries)
.entries
.into_iter()
.rev()
.find_map(|entry| match entry.data {
SessionSnapshotEntryData::ToolCall { call_id, .. } => {
Some(AlertPosition::AfterToolCall { call_id })
}
SessionSnapshotEntryData::ToolResult { .. }
| SessionSnapshotEntryData::Invoke { .. }
| SessionSnapshotEntryData::Usage { .. }
| SessionSnapshotEntryData::RunStats { .. }
| SessionSnapshotEntryData::RunYielded { .. }
| SessionSnapshotEntryData::RunResumed { .. }
| SessionSnapshotEntryData::RunCancelled => None,
_ => Some(AlertPosition::AfterEntry {
entry_id: entry.entry_id,
}),
})
}
impl SegmentLogSink {
/// Create a fresh sink with an empty mirror. Used before any entry
/// has been written (deferred SegmentStart) or as a placeholder in
@@ -62,7 +103,7 @@ impl SegmentLogSink {
let (session_entry_tx, _) = broadcast::channel(BROADCAST_CAPACITY);
Self {
inner: Arc::new(SinkInner {
mirror: StdMutex::new(Vec::new()),
mirror: StdMutex::new(Mirror::new(Vec::new())),
session_entry_tx,
}),
}
@@ -75,7 +116,7 @@ impl SegmentLogSink {
let (session_entry_tx, _) = broadcast::channel(BROADCAST_CAPACITY);
Self {
inner: Arc::new(SinkInner {
mirror: StdMutex::new(entries),
mirror: StdMutex::new(Mirror::new(entries)),
session_entry_tx,
}),
}
@@ -103,7 +144,10 @@ impl SegmentLogSink {
.mirror
.lock()
.expect("session log mirror mutex poisoned");
mirror.push(entry.clone());
if let Some(position) = latest_alert_position(std::slice::from_ref(&entry)) {
mirror.alert_position = position;
}
mirror.entries.push(entry.clone());
if Self::is_live_relevant(&entry) {
// SendError means there are zero subscribers; harmless. The
// mirror lock is held across `send` so subscribers cannot
@@ -147,8 +191,7 @@ impl SegmentLogSink {
.mirror
.lock()
.expect("session log mirror mutex poisoned");
mirror.clear();
mirror.push(initial.clone());
*mirror = Mirror::new(vec![initial.clone()]);
let _ = self.inner.session_entry_tx.send(initial);
}
@@ -164,7 +207,7 @@ impl SegmentLogSink {
.mirror
.lock()
.expect("session log mirror mutex poisoned");
*mirror = entries;
*mirror = Mirror::new(entries);
if let Some(initial) = first {
let _ = self.inner.session_entry_tx.send(initial);
}
@@ -181,7 +224,7 @@ impl SegmentLogSink {
.mirror
.lock()
.expect("session log mirror mutex poisoned");
*mirror = entries;
*mirror = Mirror::new(entries);
}
/// Truncate the mirror without broadcasting.
@@ -191,7 +234,9 @@ impl SegmentLogSink {
.mirror
.lock()
.expect("session log mirror mutex poisoned");
mirror.truncate(entries_len);
mirror.entries.truncate(entries_len);
mirror.alert_position =
latest_alert_position(&mirror.entries).unwrap_or(AlertPosition::Start);
}
/// Atomically read the current mirror and subscribe to subsequent
@@ -203,17 +248,27 @@ impl SegmentLogSink {
.mirror
.lock()
.expect("session log mirror mutex poisoned");
let snapshot = mirror.clone();
let snapshot = mirror.entries.clone();
let rx = self.inner.session_entry_tx.subscribe();
(snapshot, rx)
}
pub(crate) fn alert_position(&self) -> AlertPosition {
self.inner
.mirror
.lock()
.expect("session log mirror mutex poisoned")
.alert_position
.clone()
}
/// Current entry count. Useful for tests / diagnostics.
pub fn len(&self) -> usize {
self.inner
.mirror
.lock()
.expect("session log mirror mutex poisoned")
.entries
.len()
}
@@ -267,6 +322,70 @@ mod tests {
}
}
#[test]
fn alert_anchor_tracks_visible_rows_not_reasoning_results_or_accounting() {
let sink = SegmentLogSink::new();
assert_eq!(sink.alert_position(), AlertPosition::Start);
sink.publish(session_start());
assert_eq!(sink.alert_position(), AlertPosition::Start);
let user = user_input("hello");
sink.publish(user.clone());
let expected = latest_alert_position(&[user.clone()]).unwrap();
assert_eq!(sink.alert_position(), expected);
sink.publish(turn_end(1));
sink.publish(LogEntry::Invoke {
ts: 2,
trigger: protocol::InvokeKind::UserSend,
});
sink.publish(LogEntry::AnnotatedAssistantItem {
ts: 3,
entry: crate::session_history::test_logged_history_entry(
session_store::LoggedItem::Reasoning {
text: "private".into(),
summary: vec![],
encrypted_content: None,
signature: None,
},
),
});
assert_eq!(sink.alert_position(), expected);
let tool = LogEntry::AnnotatedAssistantItem {
ts: 4,
entry: crate::session_history::test_logged_history_entry(agen::Item::tool_call(
"call-1", "Read", "{}",
)),
};
sink.publish(tool.clone());
assert_eq!(
sink.alert_position(),
AlertPosition::AfterToolCall {
call_id: "call-1".into()
}
);
sink.publish(user_input("newer row"));
let newer = sink.alert_position();
sink.publish(LogEntry::AnnotatedToolResult {
ts: 5,
entry: crate::session_history::test_logged_history_entry(agen::Item::tool_result(
"call-1", "done",
)),
});
assert_eq!(sink.alert_position(), newer);
sink.replace_silent(vec![session_start(), user.clone(), tool]);
assert_eq!(
sink.alert_position(),
AlertPosition::AfterToolCall {
call_id: "call-1".into()
}
);
sink.truncate_silent(2);
assert_eq!(sink.alert_position(), expected);
sink.reset_with_initial(session_start());
assert_eq!(sink.alert_position(), AlertPosition::Start);
sink.reset_with_initial_entries(vec![session_start(), user]);
assert_eq!(sink.alert_position(), expected);
}
#[test]
fn publish_then_subscribe_returns_history_in_snapshot() {
let sink = SegmentLogSink::new();
+2
View File
@@ -131,6 +131,8 @@ mod tests {
listener,
vec![
Event::Alert(Alert {
id: None,
position: None,
level: AlertLevel::Warn,
source: AlertSource::Worker,
message: "replayed alert".into(),
+77 -3
View File
@@ -1918,11 +1918,13 @@ where
self.state.increment_entries();
if let Some(in_flight) = &self.in_flight {
let committed_item = match &entry {
LogEntry::AnnotatedAssistantItem { entry, .. } => Some(entry.item.clone()),
LogEntry::AnnotatedAssistantItem { entry, .. } => {
Some((entry.item.clone(), entry.metadata.entry_id.0.clone()))
}
_ => None,
};
if let Some(item_for_clear) = committed_item {
in_flight.clear_for_committed_item_then(&item_for_clear, || {
if let Some((item_for_clear, entry_id)) = committed_item {
in_flight.clear_for_committed_item_then(&item_for_clear, Some(&entry_id), || {
self.sink.publish(entry);
});
return Ok(());
@@ -4253,6 +4255,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
/// Worker-internal operations (compaction failures, AGENTS.md
/// ingestion warnings) can surface messages to connected clients.
pub fn attach_alerter(&mut self, alerter: Alerter) {
alerter.attach_position_source(self.sink.clone(), self.in_flight.clone());
self.alerter = Some(alerter);
}
@@ -13313,6 +13316,77 @@ mod build_summary_prompt_tests {
}
}
// Real Worker wiring, filesystem Store, log writer and reconnect buffers;
// only the model client is inert because this test never invokes the engine.
#[tokio::test]
async fn alert_pending_text_anchor_resolves_at_durable_commit_before_replay() {
let dir = tempfile::tempdir().unwrap();
let cwd = dir.path().join("workspace");
std::fs::create_dir_all(&cwd).unwrap();
let store = session_store::FsStore::new(dir.path().join("sessions")).unwrap();
let mut worker = Worker::new(
minimal_manifest(),
Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient),
store,
WorkerWorkspaceContext::no_workspace(),
WorkerFilesystemAuthority::local(cwd.clone(), cwd.clone()),
Scope::writable(&cwd).unwrap(),
)
.await
.unwrap();
let (tx, _) = broadcast::channel(16);
let in_flight = InFlightEvents::new(tx.clone());
worker.in_flight = Some(in_flight.clone());
let alerter = Alerter::new(tx);
worker.attach_alerter(alerter.clone());
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "before head".into());
assert_eq!(
alerter.subscribe_with_snapshot().0[0].position,
Some(protocol::AlertPosition::Start)
);
worker.materialize_durable_session_head().await.unwrap();
let text = in_flight.start_text_block();
in_flight.text_delta(text, "streamed".into());
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "during text".into());
let (before, _) = alerter.subscribe_with_snapshot();
let snapshot = crate::in_flight::snapshot_from_guard(&in_flight.snapshot_guard());
assert_eq!(
before[1].position,
Some(protocol::AlertPosition::AfterInFlight {
block_id: snapshot.block_ids[0]
})
);
let (deferred, _) = alerter.subscribe_with_deferred_snapshot();
let entry = crate::session_history::test_logged_history_entry(
agen::Item::assistant_message("streamed"),
);
let entry_id = entry.metadata.entry_id.0.clone();
worker
.log_writer_handle()
.append_entry(LogEntry::AnnotatedAssistantItem { ts: 42, entry })
.unwrap();
let (entries, _) = worker.sink.subscribe_with_snapshot();
assert!(entries.iter().any(|entry| matches!(entry, LogEntry::AnnotatedAssistantItem { entry, .. } if entry.metadata.entry_id.0 == entry_id)));
assert!(
crate::in_flight::snapshot_from_guard(&in_flight.snapshot_guard())
.blocks
.is_empty()
);
let replay = deferred();
assert_eq!(replay[1].id, before[1].id);
assert_eq!(
replay[1].position,
Some(protocol::AlertPosition::AfterEntry {
entry_id: entry_id.clone()
})
);
alerter.alert(AlertLevel::Warn, AlertSource::Worker, "after commit".into());
assert_eq!(
alerter.subscribe_with_snapshot().0[2].position,
Some(protocol::AlertPosition::AfterEntry { entry_id })
);
}
#[tokio::test]
async fn materialize_durable_session_head_commits_without_model_run() {
let dir = tempfile::tempdir().unwrap();
+1
View File
@@ -46269,6 +46269,7 @@ mod tests {
},
state: self.state.lock().unwrap().clone(),
in_flight: protocol::InFlightSnapshot {
block_ids: Vec::new(),
blocks: Vec::new(),
commands: Vec::new(),
compaction: None,