fix: deduplicate compaction terminal failures
This commit is contained in:
@@ -2459,7 +2459,9 @@ async fn controller_loop<C, St>(
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
if let Err(error) = result {
|
if let Err(error) = result
|
||||||
|
&& !matches!(&error, WorkerError::CompactCancelled)
|
||||||
|
{
|
||||||
let _ = working_event_tx.send(Event::Error {
|
let _ = working_event_tx.send(Event::Error {
|
||||||
code: worker_error_code(&error),
|
code: worker_error_code(&error),
|
||||||
message: error.to_string(),
|
message: error.to_string(),
|
||||||
@@ -2883,11 +2885,7 @@ where
|
|||||||
(WorkerStatus::Paused, shutdown_requested, false)
|
(WorkerStatus::Paused, shutdown_requested, false)
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let status = if matches!(
|
let status = if worker_error_requires_attention(&e) {
|
||||||
&e,
|
|
||||||
WorkerError::SegmentActivationIncomplete { .. }
|
|
||||||
| WorkerError::CompactionCleanupPending { .. }
|
|
||||||
) {
|
|
||||||
// Metadata has already committed the replacement, but
|
// Metadata has already committed the replacement, but
|
||||||
// one live authority failed to publish. Keep the
|
// one live authority failed to publish. Keep the
|
||||||
// controller visibly busy/fail-closed; publishing Idle
|
// controller visibly busy/fail-closed; publishing Idle
|
||||||
@@ -2896,6 +2894,7 @@ where
|
|||||||
} else {
|
} else {
|
||||||
WorkerStatus::Idle
|
WorkerStatus::Idle
|
||||||
};
|
};
|
||||||
|
if !matches!(&e, WorkerError::RunFailureAlreadyDelivered { .. }) {
|
||||||
let code = worker_error_code(&e);
|
let code = worker_error_code(&e);
|
||||||
let message = e.to_string();
|
let message = e.to_string();
|
||||||
let _ = working_event_tx.send(Event::Error {
|
let _ = working_event_tx.send(Event::Error {
|
||||||
@@ -2911,6 +2910,7 @@ where
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
(status, shutdown_requested, false)
|
(status, shutdown_requested, false)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -3375,8 +3375,22 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn worker_error_requires_attention(error: &WorkerError) -> bool {
|
||||||
|
match error {
|
||||||
|
WorkerError::SegmentActivationIncomplete { .. }
|
||||||
|
| WorkerError::CompactionCleanupPending { .. } => true,
|
||||||
|
WorkerError::RunFailureAlreadyDelivered { source }
|
||||||
|
| WorkerError::RunTerminalPersistence {
|
||||||
|
run_failure: source,
|
||||||
|
..
|
||||||
|
} => worker_error_requires_attention(source),
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn worker_error_code(e: &WorkerError) -> ErrorCode {
|
fn worker_error_code(e: &WorkerError) -> ErrorCode {
|
||||||
match e {
|
match e {
|
||||||
|
WorkerError::RunFailureAlreadyDelivered { source } => worker_error_code(source),
|
||||||
WorkerError::Engine(we) => match we {
|
WorkerError::Engine(we) => match we {
|
||||||
EngineError::Tool(_) => ErrorCode::ToolError,
|
EngineError::Tool(_) => ErrorCode::ToolError,
|
||||||
EngineError::Client(_) => ErrorCode::ProviderError,
|
EngineError::Client(_) => ErrorCode::ProviderError,
|
||||||
@@ -3985,6 +3999,87 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn delivered_run_failure_is_not_reported_again_by_controller() {
|
||||||
|
let mut env = make_env().await;
|
||||||
|
let listener = UnixListener::bind(&env.parent_socket_path).expect("bind listener");
|
||||||
|
let mut events = env.working_event_tx.subscribe();
|
||||||
|
let worker_future = async {
|
||||||
|
Err::<WorkerRunResult, _>(WorkerError::RunFailureAlreadyDelivered {
|
||||||
|
source: Box::new(WorkerError::CompactSummaryMissing),
|
||||||
|
})
|
||||||
|
};
|
||||||
|
|
||||||
|
let (status, _, may_drain_pending) = drive_turn(
|
||||||
|
worker_future,
|
||||||
|
&mut env.method_rx,
|
||||||
|
&env.working_event_tx,
|
||||||
|
&env.cancel_tx,
|
||||||
|
&env.pause_tx,
|
||||||
|
&env.shared_state,
|
||||||
|
&env.runtime_dir,
|
||||||
|
None,
|
||||||
|
&env.notify_buffer,
|
||||||
|
&env.pending_submissions,
|
||||||
|
Some(&env.parent_socket_path),
|
||||||
|
"child-worker",
|
||||||
|
&env.spawned_registry,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(status, WorkerStatus::Idle);
|
||||||
|
assert!(!may_drain_pending, "failed runs must not become successful");
|
||||||
|
assert!(events.try_recv().is_err(), "must not emit a generic Error");
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(200), listener.accept())
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"must not send a second parent Errored event"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn terminal_persistence_failure_reports_both_causes_once() {
|
||||||
|
let mut env = make_env().await;
|
||||||
|
let mut events = env.working_event_tx.subscribe();
|
||||||
|
let worker_future = async {
|
||||||
|
Err::<WorkerRunResult, _>(WorkerError::RunTerminalPersistence {
|
||||||
|
run_failure: Box::new(WorkerError::CompactSummaryMissing),
|
||||||
|
persist_error: Box::new(WorkerError::Store(session_store::StoreError::Io(
|
||||||
|
std::io::Error::other("terminal append failed"),
|
||||||
|
))),
|
||||||
|
})
|
||||||
|
};
|
||||||
|
|
||||||
|
let (status, _, may_drain_pending) = drive_turn(
|
||||||
|
worker_future,
|
||||||
|
&mut env.method_rx,
|
||||||
|
&env.working_event_tx,
|
||||||
|
&env.cancel_tx,
|
||||||
|
&env.pause_tx,
|
||||||
|
&env.shared_state,
|
||||||
|
&env.runtime_dir,
|
||||||
|
None,
|
||||||
|
&env.notify_buffer,
|
||||||
|
&env.pending_submissions,
|
||||||
|
None,
|
||||||
|
"child-worker",
|
||||||
|
&env.spawned_registry,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(status, WorkerStatus::Idle);
|
||||||
|
assert!(!may_drain_pending);
|
||||||
|
let Event::Error { message, .. } = events.try_recv().expect("one error event") else {
|
||||||
|
panic!("expected Error event");
|
||||||
|
};
|
||||||
|
assert!(message.contains("compact worker did not produce a summary"));
|
||||||
|
assert!(message.contains("terminal append failed"));
|
||||||
|
assert!(events.try_recv().is_err(), "must report once");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn non_parent_originated_worker_error_stays_silent() {
|
async fn non_parent_originated_worker_error_stays_silent() {
|
||||||
let mut env = make_env().await;
|
let mut env = make_env().await;
|
||||||
|
|||||||
@@ -271,6 +271,17 @@ pub(crate) enum InternalWorkerSessionStatus {
|
|||||||
Failed,
|
Failed,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Typed result of the most recent reusable Internal Worker turn.
|
||||||
|
///
|
||||||
|
/// Session status intentionally remains a compact presentation state. Owners that
|
||||||
|
/// make domain decisions (such as Compaction) must use this result instead of
|
||||||
|
/// guessing a stop cause from `Stopped` or parsing the presentation error string.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub(crate) enum InternalWorkerTurnOutcome {
|
||||||
|
Completed(WorkerRunResult),
|
||||||
|
Failed { message: String },
|
||||||
|
}
|
||||||
|
|
||||||
impl InternalWorkerSessionStatus {
|
impl InternalWorkerSessionStatus {
|
||||||
fn encode(self) -> u8 {
|
fn encode(self) -> u8 {
|
||||||
match self {
|
match self {
|
||||||
@@ -323,26 +334,53 @@ fn send_internal_worker_state(
|
|||||||
|
|
||||||
fn classify_internal_turn_result(
|
fn classify_internal_turn_result(
|
||||||
result: Result<WorkerRunResult, WorkerError>,
|
result: Result<WorkerRunResult, WorkerError>,
|
||||||
) -> (InternalWorkerSessionStatus, Option<String>) {
|
) -> (
|
||||||
|
InternalWorkerSessionStatus,
|
||||||
|
Option<String>,
|
||||||
|
InternalWorkerTurnOutcome,
|
||||||
|
) {
|
||||||
match result {
|
match result {
|
||||||
Ok(WorkerRunResult::Finished) => (InternalWorkerSessionStatus::Idle, None),
|
Ok(result @ WorkerRunResult::Finished) => (
|
||||||
Ok(WorkerRunResult::Paused) => (InternalWorkerSessionStatus::Paused, None),
|
InternalWorkerSessionStatus::Idle,
|
||||||
Ok(WorkerRunResult::LimitReached) => (
|
None,
|
||||||
InternalWorkerSessionStatus::Stopped,
|
InternalWorkerTurnOutcome::Completed(result),
|
||||||
Some("internal Worker reached its turn limit".to_string()),
|
|
||||||
),
|
),
|
||||||
Ok(WorkerRunResult::Interrupted { message, .. }) => {
|
Ok(result @ WorkerRunResult::Paused) => (
|
||||||
(InternalWorkerSessionStatus::Stopped, Some(message))
|
InternalWorkerSessionStatus::Paused,
|
||||||
|
None,
|
||||||
|
InternalWorkerTurnOutcome::Completed(result),
|
||||||
|
),
|
||||||
|
Ok(result @ WorkerRunResult::LimitReached) => {
|
||||||
|
let message = "internal Worker reached its turn limit".to_string();
|
||||||
|
(
|
||||||
|
InternalWorkerSessionStatus::Stopped,
|
||||||
|
Some(message),
|
||||||
|
InternalWorkerTurnOutcome::Completed(result),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
Ok(WorkerRunResult::Cancelled) => (
|
Ok(WorkerRunResult::Interrupted { code, message }) => (
|
||||||
|
InternalWorkerSessionStatus::Stopped,
|
||||||
|
Some(message.clone()),
|
||||||
|
InternalWorkerTurnOutcome::Completed(WorkerRunResult::Interrupted { code, message }),
|
||||||
|
),
|
||||||
|
Ok(result @ WorkerRunResult::Cancelled) => (
|
||||||
InternalWorkerSessionStatus::Stopped,
|
InternalWorkerSessionStatus::Stopped,
|
||||||
Some("internal Worker run was cancelled".to_string()),
|
Some("internal Worker run was cancelled".to_string()),
|
||||||
|
InternalWorkerTurnOutcome::Completed(result),
|
||||||
),
|
),
|
||||||
Ok(WorkerRunResult::RolledBack) => (
|
Ok(result @ WorkerRunResult::RolledBack) => (
|
||||||
InternalWorkerSessionStatus::Stopped,
|
InternalWorkerSessionStatus::Stopped,
|
||||||
Some("internal Worker run was cancelled before AI output".to_string()),
|
Some("internal Worker run was cancelled before AI output".to_string()),
|
||||||
|
InternalWorkerTurnOutcome::Completed(result),
|
||||||
),
|
),
|
||||||
Err(error) => (InternalWorkerSessionStatus::Failed, Some(error.to_string())),
|
Err(error) => {
|
||||||
|
let message = error.to_string();
|
||||||
|
(
|
||||||
|
InternalWorkerSessionStatus::Failed,
|
||||||
|
Some(message.clone()),
|
||||||
|
InternalWorkerTurnOutcome::Failed { message },
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -389,6 +427,7 @@ pub(crate) struct InternalWorkerSessionHandle {
|
|||||||
event_tx: broadcast::Sender<Event>,
|
event_tx: broadcast::Sender<Event>,
|
||||||
visibility: InternalWorkerVisibility,
|
visibility: InternalWorkerVisibility,
|
||||||
last_error: Arc<Mutex<Option<String>>>,
|
last_error: Arc<Mutex<Option<String>>>,
|
||||||
|
last_outcome: Arc<Mutex<Option<InternalWorkerTurnOutcome>>>,
|
||||||
child_registry: Option<Arc<SpawnedWorkerRegistry>>,
|
child_registry: Option<Arc<SpawnedWorkerRegistry>>,
|
||||||
sink: SegmentLogSink,
|
sink: SegmentLogSink,
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -488,6 +527,8 @@ impl InternalWorkerSessionHandle {
|
|||||||
InternalWorkerSessionStatus::Idle => InternalWorkerSessionError::Unavailable,
|
InternalWorkerSessionStatus::Idle => InternalWorkerSessionError::Unavailable,
|
||||||
},
|
},
|
||||||
)?;
|
)?;
|
||||||
|
*self.last_error.lock().unwrap() = None;
|
||||||
|
*self.last_outcome.lock().unwrap() = None;
|
||||||
if self
|
if self
|
||||||
.command_tx
|
.command_tx
|
||||||
.send(InternalWorkerSessionCommand::Run(input.into()))
|
.send(InternalWorkerSessionCommand::Run(input.into()))
|
||||||
@@ -501,6 +542,9 @@ impl InternalWorkerSessionHandle {
|
|||||||
self.state_changed.notify_waiters();
|
self.state_changed.notify_waiters();
|
||||||
let message = "internal Worker session actor is unavailable".to_owned();
|
let message = "internal Worker session actor is unavailable".to_owned();
|
||||||
*self.last_error.lock().unwrap() = Some(message.clone());
|
*self.last_error.lock().unwrap() = Some(message.clone());
|
||||||
|
*self.last_outcome.lock().unwrap() = Some(InternalWorkerTurnOutcome::Failed {
|
||||||
|
message: message.clone(),
|
||||||
|
});
|
||||||
let _ = self.event_tx.send(Event::Error {
|
let _ = self.event_tx.send(Event::Error {
|
||||||
code: protocol::ErrorCode::Internal,
|
code: protocol::ErrorCode::Internal,
|
||||||
message,
|
message,
|
||||||
@@ -524,6 +568,15 @@ impl InternalWorkerSessionHandle {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Wait for the active turn and return its typed outcome.
|
||||||
|
///
|
||||||
|
/// `None` means the actor stopped without publishing a turn result; callers
|
||||||
|
/// must treat that as unknown rather than manufacturing a cancellation.
|
||||||
|
pub(crate) async fn wait_for_turn_outcome(&self) -> Option<InternalWorkerTurnOutcome> {
|
||||||
|
self.wait_until_idle().await;
|
||||||
|
self.last_outcome.lock().unwrap().clone()
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) fn force_status(&self, status: InternalWorkerSessionStatus) {
|
pub(crate) fn force_status(&self, status: InternalWorkerSessionStatus) {
|
||||||
self.status
|
self.status
|
||||||
@@ -808,6 +861,7 @@ pub(crate) async fn prepare_internal_worker_session(
|
|||||||
));
|
));
|
||||||
let state_changed = Arc::new(tokio::sync::Notify::new());
|
let state_changed = Arc::new(tokio::sync::Notify::new());
|
||||||
let last_error = Arc::new(Mutex::new(None));
|
let last_error = Arc::new(Mutex::new(None));
|
||||||
|
let last_outcome = Arc::new(Mutex::new(None));
|
||||||
let handle = InternalWorkerSessionHandle {
|
let handle = InternalWorkerSessionHandle {
|
||||||
command_tx,
|
command_tx,
|
||||||
status: status.clone(),
|
status: status.clone(),
|
||||||
@@ -819,6 +873,7 @@ pub(crate) async fn prepare_internal_worker_session(
|
|||||||
event_tx: event_tx.clone(),
|
event_tx: event_tx.clone(),
|
||||||
visibility,
|
visibility,
|
||||||
last_error: last_error.clone(),
|
last_error: last_error.clone(),
|
||||||
|
last_outcome: last_outcome.clone(),
|
||||||
child_registry,
|
child_registry,
|
||||||
sink,
|
sink,
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -836,8 +891,10 @@ pub(crate) async fn prepare_internal_worker_session(
|
|||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
result = &mut run => {
|
result = &mut run => {
|
||||||
let (turn_status, error) = classify_internal_turn_result(result);
|
let (turn_status, error, outcome) =
|
||||||
|
classify_internal_turn_result(result);
|
||||||
actor_in_flight.clear();
|
actor_in_flight.clear();
|
||||||
|
*last_outcome.lock().unwrap() = Some(outcome);
|
||||||
status.store(turn_status.encode(), std::sync::atomic::Ordering::Release);
|
status.store(turn_status.encode(), std::sync::atomic::Ordering::Release);
|
||||||
if let Some(message) = error {
|
if let Some(message) = error {
|
||||||
*last_error.lock().unwrap() = Some(message.clone());
|
*last_error.lock().unwrap() = Some(message.clone());
|
||||||
@@ -1148,6 +1205,7 @@ pub(crate) fn test_internal_worker_session(
|
|||||||
event_tx: event_tx.clone(),
|
event_tx: event_tx.clone(),
|
||||||
visibility,
|
visibility,
|
||||||
last_error: Arc::new(Mutex::new(None)),
|
last_error: Arc::new(Mutex::new(None)),
|
||||||
|
last_outcome: Arc::new(Mutex::new(None)),
|
||||||
child_registry: None,
|
child_registry: None,
|
||||||
sink,
|
sink,
|
||||||
fail_stop: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
fail_stop: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||||
@@ -1354,6 +1412,11 @@ permission = "write"
|
|||||||
InternalWorkerSessionStatus::Stopped,
|
InternalWorkerSessionStatus::Stopped,
|
||||||
true,
|
true,
|
||||||
),
|
),
|
||||||
|
(
|
||||||
|
WorkerRunResult::Cancelled,
|
||||||
|
InternalWorkerSessionStatus::Stopped,
|
||||||
|
true,
|
||||||
|
),
|
||||||
(
|
(
|
||||||
WorkerRunResult::RolledBack,
|
WorkerRunResult::RolledBack,
|
||||||
InternalWorkerSessionStatus::Stopped,
|
InternalWorkerSessionStatus::Stopped,
|
||||||
@@ -1362,16 +1425,22 @@ permission = "write"
|
|||||||
];
|
];
|
||||||
|
|
||||||
for (result, expected_status, expects_error) in cases {
|
for (result, expected_status, expects_error) in cases {
|
||||||
let (status, error) = classify_internal_turn_result(Ok(result));
|
let expected_outcome = InternalWorkerTurnOutcome::Completed(result.clone());
|
||||||
|
let (status, error, outcome) = classify_internal_turn_result(Ok(result));
|
||||||
assert_eq!(status, expected_status);
|
assert_eq!(status, expected_status);
|
||||||
assert_eq!(error.is_some(), expects_error);
|
assert_eq!(error.is_some(), expects_error);
|
||||||
|
assert_eq!(outcome, expected_outcome);
|
||||||
}
|
}
|
||||||
|
|
||||||
let (status, error) = classify_internal_turn_result(Err(WorkerError::Engine(
|
let (status, error, outcome) = classify_internal_turn_result(Err(WorkerError::Engine(
|
||||||
EngineError::Aborted("fatal".to_string()),
|
EngineError::Aborted("fatal".to_string()),
|
||||||
)));
|
)));
|
||||||
assert_eq!(status, InternalWorkerSessionStatus::Failed);
|
assert_eq!(status, InternalWorkerSessionStatus::Failed);
|
||||||
assert!(error.is_some_and(|message| message.contains("fatal")));
|
assert!(error.is_some_and(|message| message.contains("fatal")));
|
||||||
|
assert!(matches!(
|
||||||
|
outcome,
|
||||||
|
InternalWorkerTurnOutcome::Failed { message } if message.contains("fatal")
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
+257
-84
@@ -4941,11 +4941,19 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
|
|
||||||
if let Some(block) = request_block {
|
if let Some(block) = request_block {
|
||||||
let error = automatic_compact_block_error(block);
|
let error = automatic_compact_block_error(block);
|
||||||
if matches!(result, EngineRunExit::Yielded) {
|
let error = if matches!(result, EngineRunExit::Yielded) {
|
||||||
self.persist_compaction_run_failure(
|
self.terminalize_compaction_run_failure(
|
||||||
request_block_message.expect("blocked request has a terminal reason"),
|
request_block_message.expect("blocked request has a terminal reason"),
|
||||||
);
|
error,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
// `persist_turn` above already committed this non-yielded Run's
|
||||||
|
// compaction terminal. Preserve Err semantics while preventing a
|
||||||
|
// second Controller Error for the same durable entry.
|
||||||
|
WorkerError::RunFailureAlreadyDelivered {
|
||||||
|
source: Box::new(error),
|
||||||
}
|
}
|
||||||
|
};
|
||||||
if let Some(state) = &self.compact_state {
|
if let Some(state) = &self.compact_state {
|
||||||
state.finish_logical_run();
|
state.finish_logical_run();
|
||||||
}
|
}
|
||||||
@@ -4999,33 +5007,33 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn persist_compaction_run_failure(&mut self, message: String) {
|
fn persist_compaction_run_failure(&mut self, message: String) -> Result<(), WorkerError> {
|
||||||
// Keep the original compaction/preparation error as controller authority.
|
// The Store is the terminal authority. Only a successful append can be
|
||||||
// A terminal append failure is diagnosed separately and cannot publish a
|
// marked as already delivered to clients through the session-log sink.
|
||||||
// false persisted result through the session-log sink.
|
self.commit_entry(LogEntry::RunErrored {
|
||||||
match self.commit_entry(LogEntry::RunErrored {
|
|
||||||
ts: segment_log::now_millis(),
|
ts: segment_log::now_millis(),
|
||||||
interrupted: false,
|
interrupted: false,
|
||||||
message,
|
message,
|
||||||
failure: Some(RunFailureKind::Compaction),
|
failure: Some(RunFailureKind::Compaction),
|
||||||
}) {
|
})?;
|
||||||
Ok(()) => {
|
|
||||||
self.last_run_interrupted = false;
|
self.last_run_interrupted = false;
|
||||||
self.engine_mut().set_active_run_turn_count(None);
|
self.engine_mut().set_active_run_turn_count(None);
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
Err(persist_error) => {
|
|
||||||
warn!(
|
fn terminalize_compaction_run_failure(
|
||||||
error = %persist_error,
|
&mut self,
|
||||||
"failed to persist mid-run compaction terminal record"
|
message: String,
|
||||||
);
|
run_failure: WorkerError,
|
||||||
self.alert(
|
) -> WorkerError {
|
||||||
AlertLevel::Error,
|
match self.persist_compaction_run_failure(message) {
|
||||||
AlertSource::Worker,
|
Ok(()) => WorkerError::RunFailureAlreadyDelivered {
|
||||||
format!(
|
source: Box::new(run_failure),
|
||||||
"mid-run compaction failed and its Run terminal record could not be saved: {persist_error}"
|
},
|
||||||
),
|
Err(persist_error) => WorkerError::RunTerminalPersistence {
|
||||||
);
|
run_failure: Box::new(run_failure),
|
||||||
}
|
persist_error: Box::new(persist_error),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -5078,9 +5086,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
}
|
}
|
||||||
let message =
|
let message =
|
||||||
format!("mid-run resume preparation failed after compaction: {error}");
|
format!("mid-run resume preparation failed after compaction: {error}");
|
||||||
self.persist_compaction_run_failure(message.clone());
|
return Err(self.terminalize_compaction_run_failure(message, error));
|
||||||
self.alert(AlertLevel::Error, AlertSource::Compactor, message);
|
|
||||||
return Err(error);
|
|
||||||
}
|
}
|
||||||
self.resume_prepared(RunResumeSource::Compaction).await
|
self.resume_prepared(RunResumeSource::Compaction).await
|
||||||
}
|
}
|
||||||
@@ -5102,14 +5108,11 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The yielded logical Run is no longer resumable. Commit its
|
// The yielded logical Run is no longer resumable. Commit its
|
||||||
// durable failure before any log-derived live event can claim
|
// durable failure before returning a typed handled error. The
|
||||||
// the terminal transition. If storage itself is unavailable,
|
// Controller suppresses only that exact handled variant; if
|
||||||
// the helper retains the original compaction error (and its
|
// persistence fails it instead reports both failures through
|
||||||
// fail-closed controller classification) while diagnosing the
|
// the non-durable error channel.
|
||||||
// missing terminal record separately.
|
Err(self.terminalize_compaction_run_failure(message, e))
|
||||||
self.persist_compaction_run_failure(message.clone());
|
|
||||||
self.alert(AlertLevel::Error, AlertSource::Compactor, message);
|
|
||||||
Err(e)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -5951,25 +5954,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
// outer compaction boundary performs the single stop attempt.
|
// outer compaction boundary performs the single stop attempt.
|
||||||
return Err(WorkerError::InvalidState(error.to_string()));
|
return Err(WorkerError::InvalidState(error.to_string()));
|
||||||
}
|
}
|
||||||
match handle.wait_until_idle().await {
|
ensure_compaction_internal_turn_succeeded(handle.wait_for_turn_outcome().await)?;
|
||||||
crate::internal_worker::InternalWorkerSessionStatus::Idle => {}
|
|
||||||
crate::internal_worker::InternalWorkerSessionStatus::Stopped => {
|
|
||||||
return Err(WorkerError::CompactCancelled);
|
|
||||||
}
|
|
||||||
crate::internal_worker::InternalWorkerSessionStatus::Failed => {
|
|
||||||
return Err(WorkerError::InvalidState(
|
|
||||||
handle
|
|
||||||
.protocol_snapshot()
|
|
||||||
.error
|
|
||||||
.unwrap_or_else(|| "compactor Internal Worker failed".into()),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
status => {
|
|
||||||
return Err(WorkerError::InvalidState(format!(
|
|
||||||
"compactor Internal Worker ended in {status:?}"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Guard: nudge the worker once more if the expected outputs
|
// Guard: nudge the worker once more if the expected outputs
|
||||||
// (summary, and any auto-read nominations when default refs
|
// (summary, and any auto-read nominations when default refs
|
||||||
@@ -6003,20 +5988,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
.send(prompt)
|
.send(prompt)
|
||||||
.await
|
.await
|
||||||
.map_err(|error| WorkerError::InvalidState(error.to_string()))?;
|
.map_err(|error| WorkerError::InvalidState(error.to_string()))?;
|
||||||
match handle.wait_until_idle().await {
|
ensure_compaction_internal_turn_succeeded(handle.wait_for_turn_outcome().await)?;
|
||||||
crate::internal_worker::InternalWorkerSessionStatus::Idle => {}
|
|
||||||
crate::internal_worker::InternalWorkerSessionStatus::Stopped => {
|
|
||||||
return Err(WorkerError::CompactCancelled);
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
return Err(WorkerError::InvalidState(
|
|
||||||
handle
|
|
||||||
.protocol_snapshot()
|
|
||||||
.error
|
|
||||||
.unwrap_or_else(|| "compactor Internal Worker failed".into()),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut final_ctx = ctx.lock().expect("compact ctx poisoned").clone();
|
let mut final_ctx = ctx.lock().expect("compact ctx poisoned").clone();
|
||||||
@@ -6035,20 +6007,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
.send(prompt)
|
.send(prompt)
|
||||||
.await
|
.await
|
||||||
.map_err(|error| WorkerError::InvalidState(error.to_string()))?;
|
.map_err(|error| WorkerError::InvalidState(error.to_string()))?;
|
||||||
match handle.wait_until_idle().await {
|
ensure_compaction_internal_turn_succeeded(handle.wait_for_turn_outcome().await)?;
|
||||||
crate::internal_worker::InternalWorkerSessionStatus::Idle => {}
|
|
||||||
crate::internal_worker::InternalWorkerSessionStatus::Stopped => {
|
|
||||||
return Err(WorkerError::CompactCancelled);
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
return Err(WorkerError::InvalidState(
|
|
||||||
handle
|
|
||||||
.protocol_snapshot()
|
|
||||||
.error
|
|
||||||
.unwrap_or_else(|| "compactor Internal Worker failed".into()),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
final_ctx = ctx.lock().expect("compact ctx poisoned").clone();
|
final_ctx = ctx.lock().expect("compact ctx poisoned").clone();
|
||||||
summary_text = final_ctx
|
summary_text = final_ctx
|
||||||
.summary
|
.summary
|
||||||
@@ -7904,6 +7863,37 @@ fn restored_flow_runtime_state(
|
|||||||
.transpose()
|
.transpose()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn ensure_compaction_internal_turn_succeeded(
|
||||||
|
outcome: Option<crate::internal_worker::InternalWorkerTurnOutcome>,
|
||||||
|
) -> Result<(), WorkerError> {
|
||||||
|
use crate::internal_worker::InternalWorkerTurnOutcome;
|
||||||
|
|
||||||
|
match outcome {
|
||||||
|
Some(InternalWorkerTurnOutcome::Completed(WorkerRunResult::Finished)) => Ok(()),
|
||||||
|
Some(InternalWorkerTurnOutcome::Completed(WorkerRunResult::LimitReached)) => {
|
||||||
|
Err(WorkerError::Engine(EngineError::Aborted(
|
||||||
|
"compactor Internal Worker reached its turn limit".to_string(),
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
Some(InternalWorkerTurnOutcome::Completed(WorkerRunResult::Interrupted {
|
||||||
|
message,
|
||||||
|
..
|
||||||
|
})) => Err(WorkerError::Engine(EngineError::Aborted(message))),
|
||||||
|
Some(InternalWorkerTurnOutcome::Completed(
|
||||||
|
WorkerRunResult::Cancelled | WorkerRunResult::RolledBack,
|
||||||
|
)) => Err(WorkerError::CompactCancelled),
|
||||||
|
Some(InternalWorkerTurnOutcome::Completed(WorkerRunResult::Paused)) => Err(
|
||||||
|
WorkerError::InvalidState("compactor Internal Worker paused unexpectedly".to_string()),
|
||||||
|
),
|
||||||
|
Some(InternalWorkerTurnOutcome::Failed { message }) => {
|
||||||
|
Err(WorkerError::InvalidState(message))
|
||||||
|
}
|
||||||
|
None => Err(WorkerError::InvalidState(
|
||||||
|
"compactor Internal Worker stopped without a typed turn outcome".to_string(),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn automatic_compact_block_error(block: AutomaticCompactBlock) -> WorkerError {
|
fn automatic_compact_block_error(block: AutomaticCompactBlock) -> WorkerError {
|
||||||
match block {
|
match block {
|
||||||
AutomaticCompactBlock::Thrash => WorkerError::CompactThrash,
|
AutomaticCompactBlock::Thrash => WorkerError::CompactThrash,
|
||||||
@@ -8019,6 +8009,26 @@ pub enum WorkerError {
|
|||||||
#[error("compaction was cancelled")]
|
#[error("compaction was cancelled")]
|
||||||
CompactCancelled,
|
CompactCancelled,
|
||||||
|
|
||||||
|
/// The logical Run failed, and its terminal entry was already committed and
|
||||||
|
/// published through the session-log authority. Controllers must preserve the
|
||||||
|
/// error outcome without emitting a second generic error event.
|
||||||
|
#[error("{source}")]
|
||||||
|
RunFailureAlreadyDelivered {
|
||||||
|
#[source]
|
||||||
|
source: Box<WorkerError>,
|
||||||
|
},
|
||||||
|
|
||||||
|
/// The logical Run failed and its terminal entry could not be committed.
|
||||||
|
/// This remains unhandled so the Controller reports the run and persistence
|
||||||
|
/// failures together through its non-durable error channel.
|
||||||
|
#[error(
|
||||||
|
"run failed ({run_failure}) and its terminal record could not be persisted: {persist_error}"
|
||||||
|
)]
|
||||||
|
RunTerminalPersistence {
|
||||||
|
run_failure: Box<WorkerError>,
|
||||||
|
persist_error: Box<WorkerError>,
|
||||||
|
},
|
||||||
|
|
||||||
#[error("compact summary too large: {tokens} tokens exceeds max {max}")]
|
#[error("compact summary too large: {tokens} tokens exceeds max {max}")]
|
||||||
CompactSummaryTooLarge { tokens: u64, max: u64 },
|
CompactSummaryTooLarge { tokens: u64, max: u64 },
|
||||||
|
|
||||||
@@ -9104,6 +9114,83 @@ mod build_summary_prompt_tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct FailRunTerminalStore {
|
||||||
|
inner: session_store::FsStore,
|
||||||
|
fail_run_terminal: Arc<AtomicBool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Store for FailRunTerminalStore {
|
||||||
|
fn append(
|
||||||
|
&self,
|
||||||
|
session_id: SessionId,
|
||||||
|
segment_id: SegmentId,
|
||||||
|
entry: &LogEntry,
|
||||||
|
) -> Result<(), StoreError> {
|
||||||
|
if matches!(entry, LogEntry::RunErrored { .. })
|
||||||
|
&& self.fail_run_terminal.swap(false, Ordering::SeqCst)
|
||||||
|
{
|
||||||
|
return Err(StoreError::Io(std::io::Error::other(
|
||||||
|
"synthetic terminal append failure",
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
self.inner.append(session_id, segment_id, entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_all(
|
||||||
|
&self,
|
||||||
|
session_id: SessionId,
|
||||||
|
segment_id: SegmentId,
|
||||||
|
) -> Result<Vec<LogEntry>, StoreError> {
|
||||||
|
self.inner.read_all(session_id, segment_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn list_sessions(&self) -> Result<Vec<SessionId>, StoreError> {
|
||||||
|
self.inner.list_sessions()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn list_segments(&self, session_id: SessionId) -> Result<Vec<SegmentId>, StoreError> {
|
||||||
|
self.inner.list_segments(session_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lookup_session_of(
|
||||||
|
&self,
|
||||||
|
segment_id: SegmentId,
|
||||||
|
) -> Result<Option<SessionId>, StoreError> {
|
||||||
|
self.inner.lookup_session_of(segment_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_segment(
|
||||||
|
&self,
|
||||||
|
session_id: SessionId,
|
||||||
|
segment_id: SegmentId,
|
||||||
|
entries: &[LogEntry],
|
||||||
|
) -> Result<(), StoreError> {
|
||||||
|
self.inner.create_segment(session_id, segment_id, entries)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exists(&self, session_id: SessionId, segment_id: SegmentId) -> Result<bool, StoreError> {
|
||||||
|
self.inner.exists(session_id, segment_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_entry_count(
|
||||||
|
&self,
|
||||||
|
session_id: SessionId,
|
||||||
|
segment_id: SegmentId,
|
||||||
|
) -> Result<usize, StoreError> {
|
||||||
|
self.inner.read_entry_count(session_id, segment_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn append_trace(
|
||||||
|
&self,
|
||||||
|
session_id: SessionId,
|
||||||
|
segment_id: SegmentId,
|
||||||
|
entry: &session_store::TraceEntry,
|
||||||
|
) -> Result<(), StoreError> {
|
||||||
|
self.inner.append_trace(session_id, segment_id, entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct NoopClient;
|
struct NoopClient;
|
||||||
|
|
||||||
@@ -9314,9 +9401,9 @@ mod build_summary_prompt_tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
worker.last_run_interrupted = true;
|
worker.last_run_interrupted = true;
|
||||||
|
|
||||||
worker.persist_compaction_run_failure(
|
worker
|
||||||
"mid-run compaction failed: summary unavailable".into(),
|
.persist_compaction_run_failure("mid-run compaction failed: summary unavailable".into())
|
||||||
);
|
.unwrap();
|
||||||
|
|
||||||
let entries = store
|
let entries = store
|
||||||
.read_all(worker.session_id(), worker.segment_id())
|
.read_all(worker.session_id(), worker.segment_id())
|
||||||
@@ -9348,6 +9435,92 @@ mod build_summary_prompt_tests {
|
|||||||
assert_eq!(worker.engine().active_run_turn_count(), None);
|
assert_eq!(worker.engine().active_run_turn_count(), None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn failed_compaction_terminal_append_stays_unhandled_and_unpublished() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let store = FailRunTerminalStore {
|
||||||
|
inner: session_store::FsStore::new(dir.path().join("sessions")).unwrap(),
|
||||||
|
fail_run_terminal: Arc::new(AtomicBool::new(false)),
|
||||||
|
};
|
||||||
|
let mut worker = Worker::new(
|
||||||
|
minimal_manifest(),
|
||||||
|
Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient),
|
||||||
|
store.clone(),
|
||||||
|
WorkerWorkspaceContext::no_workspace(),
|
||||||
|
WorkerFilesystemAuthority::None,
|
||||||
|
Scope::empty(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
worker.ensure_segment_head().await.unwrap();
|
||||||
|
store.fail_run_terminal.store(true, Ordering::SeqCst);
|
||||||
|
|
||||||
|
let error = worker.terminalize_compaction_run_failure(
|
||||||
|
"mid-run compaction failed: summary unavailable".into(),
|
||||||
|
WorkerError::CompactSummaryMissing,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
error,
|
||||||
|
WorkerError::RunTerminalPersistence {
|
||||||
|
run_failure,
|
||||||
|
persist_error,
|
||||||
|
} if matches!(*run_failure, WorkerError::CompactSummaryMissing)
|
||||||
|
&& persist_error.to_string().contains("synthetic terminal append failure")
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.read_all(worker.session_id(), worker.segment_id())
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.all(|entry| !matches!(entry, LogEntry::RunErrored { .. }))
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
worker
|
||||||
|
.sink()
|
||||||
|
.subscribe_with_snapshot()
|
||||||
|
.0
|
||||||
|
.iter()
|
||||||
|
.all(|entry| !matches!(entry, LogEntry::RunErrored { .. })),
|
||||||
|
"a failed append must not be published as durable"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn compaction_internal_turn_preserves_typed_stop_causes() {
|
||||||
|
use crate::internal_worker::InternalWorkerTurnOutcome;
|
||||||
|
|
||||||
|
let limit = ensure_compaction_internal_turn_succeeded(Some(
|
||||||
|
InternalWorkerTurnOutcome::Completed(WorkerRunResult::LimitReached),
|
||||||
|
))
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
matches!(limit, WorkerError::Engine(EngineError::Aborted(message)) if message.contains("turn limit"))
|
||||||
|
);
|
||||||
|
|
||||||
|
let interrupted = ensure_compaction_internal_turn_succeeded(Some(
|
||||||
|
InternalWorkerTurnOutcome::Completed(WorkerRunResult::Interrupted {
|
||||||
|
code: ErrorCode::ProviderError,
|
||||||
|
message: "summary provider failed".to_string(),
|
||||||
|
}),
|
||||||
|
))
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
matches!(interrupted, WorkerError::Engine(EngineError::Aborted(message)) if message == "summary provider failed")
|
||||||
|
);
|
||||||
|
|
||||||
|
let cancelled = ensure_compaction_internal_turn_succeeded(Some(
|
||||||
|
InternalWorkerTurnOutcome::Completed(WorkerRunResult::Cancelled),
|
||||||
|
))
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(cancelled, WorkerError::CompactCancelled));
|
||||||
|
|
||||||
|
let unknown = ensure_compaction_internal_turn_succeeded(None).unwrap_err();
|
||||||
|
assert!(
|
||||||
|
matches!(unknown, WorkerError::InvalidState(message) if message.contains("without a typed turn outcome"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn intentional_cancel_commits_a_non_failure_terminal() {
|
async fn intentional_cancel_commits_a_non_failure_terminal() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
@@ -991,12 +991,16 @@ async fn compact_failure_and_cancellation_emit_bounded_categories() {
|
|||||||
let encoded = serde_json::to_string(&failure.metric).unwrap();
|
let encoded = serde_json::to_string(&failure.metric).unwrap();
|
||||||
assert!(!encoded.contains("missing summary"));
|
assert!(!encoded.contains("missing summary"));
|
||||||
|
|
||||||
let (_cancel_tx, cancel_rx) = tokio::sync::watch::channel(true);
|
let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false);
|
||||||
|
cancel_tx.send(true).unwrap();
|
||||||
let error = worker
|
let error = worker
|
||||||
.manual_compact_with_cancel(cancel_rx)
|
.manual_compact_with_cancel(cancel_rx)
|
||||||
.await
|
.await
|
||||||
.unwrap_err();
|
.unwrap_err();
|
||||||
assert!(matches!(error, worker::WorkerError::CompactCancelled));
|
assert!(
|
||||||
|
matches!(error, worker::WorkerError::CompactCancelled),
|
||||||
|
"unexpected cancellation error: {error:?}"
|
||||||
|
);
|
||||||
let metrics = session_metrics::read_session_metrics(worker.store(), session_id).unwrap();
|
let metrics = session_metrics::read_session_metrics(worker.store(), session_id).unwrap();
|
||||||
let cancelled = metrics
|
let cancelled = metrics
|
||||||
.iter()
|
.iter()
|
||||||
@@ -1503,7 +1507,14 @@ async fn compacted_context_above_request_threshold_fails_before_provider_request
|
|||||||
.await
|
.await
|
||||||
.expect_err("unsafe compacted context must fail closed");
|
.expect_err("unsafe compacted context must fail closed");
|
||||||
|
|
||||||
assert!(matches!(error, WorkerError::CompactThrash));
|
assert!(
|
||||||
|
matches!(
|
||||||
|
error,
|
||||||
|
WorkerError::RunFailureAlreadyDelivered { ref source }
|
||||||
|
if matches!(**source, WorkerError::CompactThrash)
|
||||||
|
),
|
||||||
|
"unexpected error: {error:?}"
|
||||||
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
call_count.load(Ordering::SeqCst),
|
call_count.load(Ordering::SeqCst),
|
||||||
3,
|
3,
|
||||||
@@ -1526,6 +1537,100 @@ async fn compacted_context_above_request_threshold_fails_before_provider_request
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn controller_does_not_repeat_a_persisted_mid_run_compaction_failure() {
|
||||||
|
let client = MockClient::new(vec![
|
||||||
|
grow_tool_use_events("grow-before-compaction", 1_000),
|
||||||
|
single_text_events("missing summary"),
|
||||||
|
single_text_events("still missing summary"),
|
||||||
|
]);
|
||||||
|
let (worker, store) = make_worker_with_manifest_and_store(MID_TURN_MANIFEST_TOML, client).await;
|
||||||
|
let session_id = worker.session_id();
|
||||||
|
let runtime_tmp = tempfile::tempdir().unwrap();
|
||||||
|
let bash_output_dir = runtime_tmp.path().join("bash-output");
|
||||||
|
let (handle, shutdown_receiver) =
|
||||||
|
WorkerController::spawn(worker, runtime_tmp.path(), &bash_output_dir)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut rx = handle.subscribe();
|
||||||
|
|
||||||
|
handle
|
||||||
|
.send(Method::submit_text(
|
||||||
|
protocol::new_submission_request_id(),
|
||||||
|
"trigger failing compaction",
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut observed = Vec::new();
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||||
|
loop {
|
||||||
|
let event = rx.recv().await.expect("event");
|
||||||
|
let idle = matches!(
|
||||||
|
event,
|
||||||
|
Event::WorkerState {
|
||||||
|
snapshot: protocol::WorkerStateSnapshot {
|
||||||
|
state: protocol::WorkerState::Idle,
|
||||||
|
..
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
observed.push(event);
|
||||||
|
if idle {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("timeout waiting for failed compaction and Idle");
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
while let Ok(event) = rx.try_recv() {
|
||||||
|
observed.push(event);
|
||||||
|
}
|
||||||
|
|
||||||
|
let delivered = handle
|
||||||
|
.committed_entries()
|
||||||
|
.into_iter()
|
||||||
|
.filter(|entry| matches!(entry, LogEntry::RunErrored { .. }))
|
||||||
|
.count();
|
||||||
|
assert_eq!(delivered, 1, "the durable terminal is the only delivery");
|
||||||
|
assert!(
|
||||||
|
observed
|
||||||
|
.iter()
|
||||||
|
.all(|event| !matches!(event, Event::Error { .. })),
|
||||||
|
"Controller must not add a generic Error after durable delivery"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
observed.iter().all(|event| !matches!(
|
||||||
|
event,
|
||||||
|
Event::Alert(protocol::Alert {
|
||||||
|
level: protocol::AlertLevel::Error,
|
||||||
|
..
|
||||||
|
})
|
||||||
|
)),
|
||||||
|
"mid-run terminal failure must not add an Error Alert"
|
||||||
|
);
|
||||||
|
|
||||||
|
let terminal_count = store
|
||||||
|
.list_segments(session_id)
|
||||||
|
.unwrap()
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|segment_id| store.read_all(session_id, segment_id).unwrap())
|
||||||
|
.filter(|entry| matches!(entry, LogEntry::RunErrored { .. }))
|
||||||
|
.count();
|
||||||
|
assert_eq!(terminal_count, 1);
|
||||||
|
|
||||||
|
handle
|
||||||
|
.send(Method::Shutdown {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(1),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(2), shutdown_receiver)
|
||||||
|
.await
|
||||||
|
.expect("controller shutdown timeout")
|
||||||
|
.expect("shutdown confirmation");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn pre_run_compact_failure_clears_runtime_progress() {
|
async fn pre_run_compact_failure_clears_runtime_progress() {
|
||||||
// Only the first run has a response. Compaction will run the
|
// Only the first run has a response. Compaction will run the
|
||||||
@@ -1621,6 +1726,7 @@ async fn manual_compact_cancel_clears_progress_before_returning_idle() {
|
|||||||
.expect("send compact cancel");
|
.expect("send compact cancel");
|
||||||
let mut saw_interrupted = false;
|
let mut saw_interrupted = false;
|
||||||
let mut saw_idle = false;
|
let mut saw_idle = false;
|
||||||
|
let mut saw_cancel_error = false;
|
||||||
while !(saw_interrupted && saw_idle) {
|
while !(saw_interrupted && saw_idle) {
|
||||||
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv())
|
||||||
.await
|
.await
|
||||||
@@ -1630,6 +1736,7 @@ async fn manual_compact_cancel_clears_progress_before_returning_idle() {
|
|||||||
Event::CompactionProgress { compaction: None } => {
|
Event::CompactionProgress { compaction: None } => {
|
||||||
saw_interrupted = true;
|
saw_interrupted = true;
|
||||||
}
|
}
|
||||||
|
Event::Error { .. } => saw_cancel_error = true,
|
||||||
Event::WorkerState { snapshot }
|
Event::WorkerState { snapshot }
|
||||||
if snapshot.catalog_status() == protocol::WorkerStatus::Idle =>
|
if snapshot.catalog_status() == protocol::WorkerStatus::Idle =>
|
||||||
{
|
{
|
||||||
@@ -1642,6 +1749,10 @@ async fn manual_compact_cancel_clears_progress_before_returning_idle() {
|
|||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
assert!(
|
||||||
|
!saw_cancel_error,
|
||||||
|
"intentional manual cancellation must not be reported as an Error"
|
||||||
|
);
|
||||||
|
|
||||||
let compact = protocol::WorkerCommandEnvelope::new(3);
|
let compact = protocol::WorkerCommandEnvelope::new(3);
|
||||||
handle
|
handle
|
||||||
|
|||||||
@@ -338,6 +338,13 @@ Deno.test("durable compaction failure reconciles live alert and error in both vi
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
eventId: "controller-error",
|
||||||
|
event: {
|
||||||
|
event: "error",
|
||||||
|
data: { code: "internal", message: "summary unavailable" },
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
eventId: "failed",
|
eventId: "failed",
|
||||||
event: transition("failed-entry", {
|
event: transition("failed-entry", {
|
||||||
@@ -346,13 +353,6 @@ Deno.test("durable compaction failure reconciles live alert and error in both vi
|
|||||||
failure: "compaction",
|
failure: "compaction",
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
{
|
|
||||||
eventId: "controller-error",
|
|
||||||
event: {
|
|
||||||
event: "error",
|
|
||||||
data: { code: "internal", message: "summary unavailable" },
|
|
||||||
},
|
|
||||||
},
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const errors = projection.lines.filter((line) => line.kind === "error");
|
const errors = projection.lines.filter((line) => line.kind === "error");
|
||||||
@@ -362,6 +362,18 @@ Deno.test("durable compaction failure reconciles live alert and error in both vi
|
|||||||
assertEquals(errors[0].entryId, "failed-entry");
|
assertEquals(errors[0].entryId, "failed-entry");
|
||||||
assertEquals(projectConsoleLines(projection.lines, "normal"), errors);
|
assertEquals(projectConsoleLines(projection.lines, "normal"), errors);
|
||||||
assertEquals(projectConsoleLines(projection.lines, "overview"), errors);
|
assertEquals(projectConsoleLines(projection.lines, "overview"), errors);
|
||||||
|
|
||||||
|
const history = projectSessionHistoryEntries([{
|
||||||
|
entry_id: "failed-entry",
|
||||||
|
timestamp: 10,
|
||||||
|
provenance: "legacy_unknown",
|
||||||
|
kind: "run_error",
|
||||||
|
message,
|
||||||
|
failure: "compaction",
|
||||||
|
}], null);
|
||||||
|
const merged = mergeCommittedHistoryLines(history, projection.lines);
|
||||||
|
assertEquals(merged.filter((line) => line.kind === "error").length, 1);
|
||||||
|
assertEquals(merged.find((line) => line.kind === "error")?.entryId, "failed-entry");
|
||||||
});
|
});
|
||||||
|
|
||||||
Deno.test("snapshot restores compaction terminal without reviving transition progress", () => {
|
Deno.test("snapshot restores compaction terminal without reviving transition progress", () => {
|
||||||
@@ -3232,6 +3244,26 @@ Deno.test("committed history reconciles overlapping Read groups before counting
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Deno.test("committed history keeps identical run failures with distinct entry identities", () => {
|
||||||
|
const message = "the same failure happened again";
|
||||||
|
const runError = (entryId: string): SessionSnapshotEntry => ({
|
||||||
|
kind: "run_error",
|
||||||
|
entry_id: entryId,
|
||||||
|
timestamp: 1,
|
||||||
|
provenance: "legacy_unknown",
|
||||||
|
message,
|
||||||
|
failure: "compaction",
|
||||||
|
});
|
||||||
|
const retained = projectSessionHistoryEntries([runError("run-error-1")], null);
|
||||||
|
const current = projectSessionHistoryEntries([runError("run-error-2")], null);
|
||||||
|
|
||||||
|
const merged = mergeCommittedHistoryLines(retained, current);
|
||||||
|
assertEquals(merged.map((line) => line.entryId), ["run-error-1", "run-error-2"]);
|
||||||
|
assertEquals(merged.map((line) => line.body), [message, message]);
|
||||||
|
assertEquals(projectConsoleLines(merged, "normal").length, 2);
|
||||||
|
assertEquals(projectConsoleLines(merged, "overview").length, 2);
|
||||||
|
});
|
||||||
|
|
||||||
Deno.test("committed history matches tool calls across live and durable result identities", () => {
|
Deno.test("committed history matches tool calls across live and durable result identities", () => {
|
||||||
const retained: ConsoleLine = {
|
const retained: ConsoleLine = {
|
||||||
...consoleLine("tool-call-a", "tool"), entryId: "call-entry-a",
|
...consoleLine("tool-call-a", "tool"), entryId: "call-entry-a",
|
||||||
|
|||||||
@@ -2208,7 +2208,10 @@ function applySessionEntry(
|
|||||||
const entryId = stringField(value, "entry_id");
|
const entryId = stringField(value, "entry_id");
|
||||||
if (entryId) {
|
if (entryId) {
|
||||||
for (
|
for (
|
||||||
let index = Math.max(0, lineCountBefore - 1);
|
let index = Math.max(
|
||||||
|
0,
|
||||||
|
Math.min(lineCountBefore, projection.lines.length - 1),
|
||||||
|
);
|
||||||
index < projection.lines.length;
|
index < projection.lines.length;
|
||||||
index += 1
|
index += 1
|
||||||
) {
|
) {
|
||||||
|
|||||||
Reference in New Issue
Block a user