feat: merge develop into hare/develop
This commit is contained in:
@@ -344,7 +344,10 @@ target = "./"
|
||||
permission = "write"
|
||||
"#;
|
||||
|
||||
async fn make_worker_with_manifest<C>(manifest_toml: &str, client: C) -> Worker<C, TestStore>
|
||||
async fn make_worker_with_manifest_and_store<C>(
|
||||
manifest_toml: &str,
|
||||
client: C,
|
||||
) -> (Worker<C, TestStore>, TestStore)
|
||||
where
|
||||
C: LlmClient + Clone + Send + Sync + 'static,
|
||||
{
|
||||
@@ -364,6 +367,7 @@ where
|
||||
|
||||
let worker =
|
||||
Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(client);
|
||||
let observed_store = store.clone();
|
||||
let mut worker = Worker::new(
|
||||
manifest,
|
||||
worker,
|
||||
@@ -375,7 +379,16 @@ where
|
||||
.await
|
||||
.unwrap();
|
||||
worker.enable_worker_metadata_write_through().unwrap();
|
||||
worker
|
||||
(worker, observed_store)
|
||||
}
|
||||
|
||||
async fn make_worker_with_manifest<C>(manifest_toml: &str, client: C) -> Worker<C, TestStore>
|
||||
where
|
||||
C: LlmClient + Clone + Send + Sync + 'static,
|
||||
{
|
||||
make_worker_with_manifest_and_store(manifest_toml, client)
|
||||
.await
|
||||
.0
|
||||
}
|
||||
|
||||
async fn make_worker(client: MockClient) -> Worker<MockClient, TestStore> {
|
||||
@@ -922,7 +935,7 @@ async fn pre_run_compact_publishes_runtime_progress_phases() {
|
||||
|
||||
let session_before = worker.session_id();
|
||||
let segment_before = worker.segment_id();
|
||||
worker.try_pre_run_compact().await;
|
||||
worker.try_pre_run_compact().await.unwrap();
|
||||
assert_eq!(worker.session_id(), session_before);
|
||||
assert_ne!(worker.segment_id(), segment_before);
|
||||
|
||||
@@ -1083,7 +1096,7 @@ async fn pre_run_compact_failure_clears_runtime_progress() {
|
||||
let _ = drain(&mut rx);
|
||||
|
||||
// Best-effort: returns Ok(()) even on failure and clears runtime progress.
|
||||
worker.try_pre_run_compact().await;
|
||||
worker.try_pre_run_compact().await.unwrap();
|
||||
|
||||
let events = drain(&mut rx);
|
||||
assert!(events.iter().any(|event| matches!(
|
||||
@@ -1104,8 +1117,11 @@ async fn pre_run_compact_failure_clears_runtime_progress() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_compact_cancel_clears_progress_before_returning_idle() {
|
||||
let worker =
|
||||
make_worker_with_manifest(POST_RUN_MANIFEST_TOML, BlockingCompactClient::new()).await;
|
||||
let (worker, observed_store) =
|
||||
make_worker_with_manifest_and_store(POST_RUN_MANIFEST_TOML, BlockingCompactClient::new())
|
||||
.await;
|
||||
let source_session = worker.session_id();
|
||||
let source_segment = worker.segment_id();
|
||||
let runtime_tmp = tempfile::tempdir().unwrap();
|
||||
let bash_output_dir = runtime_tmp.path().join("bash-output");
|
||||
let (handle, shutdown_receiver) =
|
||||
@@ -1201,6 +1217,15 @@ async fn manual_compact_cancel_clears_progress_before_returning_idle() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let completion_request_id = "subworker-completion-during-compact";
|
||||
handle
|
||||
.send(Method::NotifyTracked {
|
||||
notification_request_id: completion_request_id.into(),
|
||||
message: "subworker completed".into(),
|
||||
source: protocol::AuthenticatedInputSource::UntrustedWire,
|
||||
})
|
||||
.await
|
||||
.expect("send completion notification during compact");
|
||||
let shutdown = protocol::WorkerCommandEnvelope::new(4);
|
||||
handle
|
||||
.send(Method::Shutdown { command: shutdown })
|
||||
@@ -1230,6 +1255,18 @@ async fn manual_compact_cancel_clears_progress_before_returning_idle() {
|
||||
.await
|
||||
.expect("controller shutdown timeout")
|
||||
.expect("shutdown confirmation");
|
||||
|
||||
let persisted = observed_store
|
||||
.read_all(source_session, source_segment)
|
||||
.expect("read source Segment after shutdown");
|
||||
assert!(persisted.iter().any(|entry| {
|
||||
matches!(
|
||||
entry,
|
||||
LogEntry::Extension { domain, payload, .. }
|
||||
if domain == "worker.pending_activations.v1"
|
||||
&& payload.to_string().contains(completion_request_id)
|
||||
)
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1783,7 +1783,9 @@ async fn run_with_unresolved_segment_emits_alert_and_placeholder() {
|
||||
async fn notify_while_idle_auto_starts_turn_and_injects_system_message() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let client_for_assert = client.clone();
|
||||
let worker = make_worker(client).await;
|
||||
let worker = make_worker(client)
|
||||
.await
|
||||
.with_notification_coalesce_delay(std::time::Duration::from_millis(20));
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
@@ -1791,7 +1793,6 @@ async fn notify_while_idle_auto_starts_turn_and_injects_system_message() {
|
||||
.send(Method::Notify {
|
||||
notification_request_id: protocol::new_submission_request_id(),
|
||||
message: "turn finished".into(),
|
||||
auto_run: true,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1883,10 +1884,12 @@ async fn notify_while_idle_auto_starts_turn_and_injects_system_message() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() {
|
||||
async fn repeated_notify_while_idle_coalesces_and_auto_starts_one_turn() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let client_for_assert = client.clone();
|
||||
let worker = make_worker(client).await;
|
||||
let worker = make_worker(client)
|
||||
.await
|
||||
.with_notification_coalesce_delay(std::time::Duration::from_millis(50));
|
||||
let handle = spawn_controller(worker).await;
|
||||
let notification_request_id = protocol::new_submission_request_id();
|
||||
|
||||
@@ -1895,26 +1898,18 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() {
|
||||
.send(Method::Notify {
|
||||
notification_request_id: notification_request_id.clone(),
|
||||
message: "progress snapshot".into(),
|
||||
auto_run: false,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||
assert!(
|
||||
client_for_assert.captured_requests().is_empty(),
|
||||
"weak Notify must not stage RunForNotification while idle"
|
||||
);
|
||||
|
||||
handle
|
||||
.send(Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"continue",
|
||||
))
|
||||
.send(Method::Notify {
|
||||
notification_request_id: "notify-coalesced-2".into(),
|
||||
message: "second coalesced notification".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||
loop {
|
||||
if !client_for_assert.captured_requests().is_empty() {
|
||||
@@ -1922,30 +1917,26 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() {
|
||||
}
|
||||
assert!(
|
||||
tokio::time::Instant::now() < deadline,
|
||||
"explicit run did not reach the mock LLM"
|
||||
"coalescing deadline did not start a notification turn"
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
wait_for_status(&handle, WorkerStatus::Idle).await;
|
||||
let requests = client_for_assert.captured_requests();
|
||||
assert_eq!(requests.len(), 1, "coalesced notifications need one turn");
|
||||
let notifications = requests[0]
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(|item| item.as_text())
|
||||
.filter(|text| text.contains("[Notification]"))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
requests.len(),
|
||||
1,
|
||||
"explicit run should drain the queued notification"
|
||||
);
|
||||
let notify_in_request = requests[0].items.iter().any(|i| {
|
||||
i.as_text()
|
||||
.is_some_and(|t| t.contains("[Notification]") && t.contains("progress snapshot"))
|
||||
});
|
||||
assert!(
|
||||
notify_in_request,
|
||||
"queued weak notification must be history-backed on the next explicit run; got items: {:?}",
|
||||
requests[0]
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(|i| i.as_text())
|
||||
.collect::<Vec<_>>()
|
||||
notifications.len(),
|
||||
2,
|
||||
"duplicate receipt must not duplicate content"
|
||||
);
|
||||
assert!(notifications[0].contains("progress snapshot"));
|
||||
assert!(notifications[1].contains("second coalesced notification"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2105,7 +2096,6 @@ async fn notify_while_running_does_not_emit_already_running_error() {
|
||||
.send(Method::Notify {
|
||||
notification_request_id: protocol::new_submission_request_id(),
|
||||
message: "ping".into(),
|
||||
auto_run: true,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -2137,7 +2127,7 @@ async fn notify_while_running_does_not_emit_already_running_error() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn weak_notify_while_running_is_deduped_and_survives_until_next_submit() {
|
||||
async fn notify_while_running_is_deduped_and_survives_until_next_model_boundary() {
|
||||
let client = MockClient::sequential(vec![
|
||||
MockResponse::Hang(Vec::new()),
|
||||
MockResponse::Complete(simple_text_events()),
|
||||
@@ -2160,7 +2150,6 @@ async fn weak_notify_while_running_is_deduped_and_survives_until_next_submit() {
|
||||
.send(Method::Notify {
|
||||
notification_request_id: notification_request_id.clone(),
|
||||
message: "durable weak notice".into(),
|
||||
auto_run: false,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -2586,11 +2575,11 @@ async fn drain_until<F: FnMut(&Event) -> bool>(
|
||||
}
|
||||
}
|
||||
|
||||
/// Pause mid-stream, then Resume: status round-trips Running →
|
||||
/// Paused → Running → Idle, and the final history contains exactly
|
||||
/// one user turn plus the assistant reply produced by the resume call.
|
||||
/// Paused → Running → Idle. A notification accepted while Paused must not
|
||||
/// auto-resume the Worker, and explicit Resume must inject it while preserving
|
||||
/// the interrupted turn's history consistency.
|
||||
#[tokio::test]
|
||||
async fn pause_then_resume_transitions_and_preserves_history_consistency() {
|
||||
async fn pause_then_resume_preserves_notifications_and_history_consistency() {
|
||||
// Response 1: hang after opening a text block (no stop / completed),
|
||||
// so the Engine is parked inside the stream read and `cancel_rx`
|
||||
// races it cleanly on Method::Pause.
|
||||
@@ -2608,7 +2597,10 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
|
||||
}),
|
||||
]);
|
||||
let client = MockClient::sequential(vec![hang, ok]);
|
||||
let worker = make_worker(client).await;
|
||||
let client_for_assert = client.clone();
|
||||
let worker = make_worker(client)
|
||||
.await
|
||||
.with_notification_coalesce_delay(std::time::Duration::from_millis(20));
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut rx = handle.subscribe();
|
||||
|
||||
@@ -2654,6 +2646,25 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused);
|
||||
|
||||
handle
|
||||
.send(Method::Notify {
|
||||
notification_request_id: "notify-while-paused".into(),
|
||||
message: "resume context".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(80)).await;
|
||||
assert_eq!(
|
||||
handle.shared_state.catalog_status(),
|
||||
WorkerStatus::Paused,
|
||||
"notification deadline must not auto-resume a paused Worker"
|
||||
);
|
||||
assert_eq!(
|
||||
client_for_assert.captured_requests().len(),
|
||||
1,
|
||||
"paused notification must wait for an explicit resume"
|
||||
);
|
||||
|
||||
handle
|
||||
.send(Method::Resume {
|
||||
command: worker_command(&handle),
|
||||
@@ -2674,10 +2685,15 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||
let requests = client_for_assert.captured_requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert!(requests[1].items.iter().any(|item| {
|
||||
item.as_text()
|
||||
.is_some_and(|text| text.contains("[Notification]") && text.contains("resume context"))
|
||||
}));
|
||||
|
||||
// History consistency: exactly [user "hello", assistant
|
||||
// "resumed output"]. No artifacts from the aborted stream
|
||||
// (partial text is not committed), no orphan tool_use.
|
||||
// History consistency: the interrupted partial response is absent, while
|
||||
// the queued notification is committed on the explicit resume.
|
||||
let history = history_from_sink(&handle);
|
||||
let roles: Vec<&str> = history
|
||||
.iter()
|
||||
@@ -2692,8 +2708,8 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() {
|
||||
.collect();
|
||||
assert_eq!(
|
||||
roles,
|
||||
vec!["user", "assistant"],
|
||||
"history = user + assistant only; got {history:?}"
|
||||
vec!["user", "system", "assistant"],
|
||||
"history should retain user input, queued notification, and resumed output; got {history:?}"
|
||||
);
|
||||
let assistant_text = history
|
||||
.iter()
|
||||
|
||||
Reference in New Issue
Block a user