fix: preserve test completion and fixture invariants under concurrency

This commit is contained in:
2026-10-16 15:04:42 +09:00
parent 22dcd8a29f
commit 6667985883
7 changed files with 168 additions and 30 deletions
@@ -1,6 +1,15 @@
use super::*;
// Keep the short transport/session-lock liveness bounds separate from full
// lifecycle cleanup. The 10s output/Close deadlines below are asserted exactly
// with paused time; these wall-clock limits are deadlock watchdogs, not the
// normal-suite performance budget.
const PROTOCOL_TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);
// Stop also runs Host admission, Runtime persistence and linked-Workdir storage
// publication. The parallel normal run exceeded 2s there after transport had
// already terminated. Check lock release separately at 2s, but allow this real
// lifecycle path scheduling/I/O headroom without replacing it with a stub.
const LIFECYCLE_CLEANUP_WATCHDOG: std::time::Duration = std::time::Duration::from_secs(15);
// Completion replies are produced synchronously by the actual attached execution.
// Unlike flooding the fixture's 32-entry broadcast bus, this reaches the Server's
@@ -289,8 +298,13 @@ async fn browser_method_queue_overflow_fails_fast_and_does_not_pin_session_clean
);
assert_eq!(receiver.len(), WORKER_PROTOCOL_METHOD_CAPACITY - 1);
let session_lock = current_worker_session_lock(&api, &identity);
let guard = tokio::time::timeout(PROTOCOL_TEST_TIMEOUT, session_lock.lock())
.await
.expect("full method queue retained the session guard");
drop(guard);
let stopped = tokio::time::timeout(
PROTOCOL_TEST_TIMEOUT,
LIFECYCLE_CLEANUP_WATCHDOG,
worker.stop(
&WorkerOperationContext::Backend,
WorkerLifecycleRequest {
@@ -636,6 +650,7 @@ fn assert_bounded_unknown_delivery_close(messages: &[WsMessage]) {
}
async fn assert_same_worker_reconnect_and_stop(
api: &WorkspaceApi,
worker: &WorkspaceWorker,
context: &WorkerOperationContext,
) {
@@ -649,8 +664,13 @@ async fn assert_same_worker_reconnect_and_stop(
.expect("same Worker reconnect did not deliver a fresh snapshot"),
Some(protocol::Event::Snapshot { .. })
));
let session_lock = current_worker_session_lock(api, worker.identity());
let guard = tokio::time::timeout(PROTOCOL_TEST_TIMEOUT, session_lock.lock())
.await
.expect("terminated transport retained same Worker's session guard");
drop(guard);
let stopped = tokio::time::timeout(
PROTOCOL_TEST_TIMEOUT,
LIFECYCLE_CLEANUP_WATCHDOG,
worker.stop(
&WorkerOperationContext::Backend,
WorkerLifecycleRequest {
@@ -780,7 +800,7 @@ async fn assert_stalled_browser_output_reclaimed(explicit_termination: bool) {
// SQLite jobs run on external threads; paused Tokio time cannot observe their
// progress and may auto-advance a reconnect timeout before SQL completes.
tokio::time::resume();
assert_same_worker_reconnect_and_stop(&worker, &context).await;
assert_same_worker_reconnect_and_stop(&api, &worker, &context).await;
}
#[tokio::test(start_paused = true)]
@@ -930,5 +950,5 @@ async fn stalled_remote_bridge_browser_cancel_overflow_reclaims_wire_and_allows_
));
}
assert!(execution.protocol_methods().is_empty());
assert_same_worker_reconnect_and_stop(&worker, &context).await;
assert_same_worker_reconnect_and_stop(&api, &worker, &context).await;
}
@@ -155,8 +155,17 @@ async fn orchestrator_attention_barrier_callers_share_unsent_reservation() {
#[tokio::test]
async fn orchestrator_attention_input_rechecks_reserved_noop_before_and_after_comparison() {
// This handshake holds a reservation across the real guarded Restore path
// (including durable intent/config/Runtime writes). Its release deadline is
// a deadlock watchdog, not a 5s Restore performance contract: the parallel
// normal run exceeded that old bound before it could release the caller.
let restore_watchdog = std::time::Duration::from_secs(15);
for pause_before_comparison in [true, false] {
let (_dir, api, execution) = attention_test_api().await;
// This case only needs assigned Ticket metadata, not a checkout. Keep
// the real API/storage and configured target, but skip Git init/commit.
let _dir = tempfile::tempdir().unwrap();
let (api, execution) = test_api_with_recording_backend(_dir.path()).await;
attention_assign_ticket(&api, "Attention backlog");
let started = start_attention_test_orchestrator(&api).await;
let worker = RuntimeWorkerRef::new(
EMBEDDED_WORKER_RUNTIME_ID,
@@ -192,8 +201,8 @@ async fn orchestrator_attention_input_rechecks_reserved_noop_before_and_after_co
if checks == 0 && pause_before_comparison {
entered_tx.send(()).unwrap();
release_rx
.recv_timeout(std::time::Duration::from_secs(5))
.unwrap();
.recv_timeout(restore_watchdog)
.expect("guarded Restore did not release the reserved no-op");
}
check_orchestrator_assignment_attention(
&api_for_caller,
@@ -210,8 +219,8 @@ async fn orchestrator_attention_input_rechecks_reserved_noop_before_and_after_co
);
entered_tx.send(()).unwrap();
release_rx
.recv_timeout(std::time::Duration::from_secs(5))
.unwrap();
.recv_timeout(restore_watchdog)
.expect("guarded Restore did not release the reserved no-op");
}
checks += 1;
assert!(attention_pending(&api_for_caller).is_empty());