test: prove real postcommit activation timeout ownership and Stop fencing
This commit is contained in:
@@ -847,10 +847,43 @@ struct RestoreFaults {
|
|||||||
factory_entered: Mutex<Option<mpsc::Sender<()>>>,
|
factory_entered: Mutex<Option<mpsc::Sender<()>>>,
|
||||||
prepared_gate: Mutex<Option<Arc<tokio::sync::Notify>>>,
|
prepared_gate: Mutex<Option<Arc<tokio::sync::Notify>>>,
|
||||||
prepared_entered: Mutex<Option<mpsc::Sender<protocol::Event>>>,
|
prepared_entered: Mutex<Option<mpsc::Sender<protocol::Event>>>,
|
||||||
|
activation_effects_gate: Mutex<Option<RestoreActivationGate>>,
|
||||||
|
activation_started_gate: Mutex<Option<RestoreActivationGate>>,
|
||||||
cleanup_gate: Mutex<Option<Arc<tokio::sync::Notify>>>,
|
cleanup_gate: Mutex<Option<Arc<tokio::sync::Notify>>>,
|
||||||
cleanup_entered: Mutex<Option<std::sync::mpsc::Sender<()>>>,
|
cleanup_entered: Mutex<Option<std::sync::mpsc::Sender<()>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Test-only synchronization inside the real postcommit candidate, not around
|
||||||
|
// factory preparation. The receiver is moved into the operation-owned task;
|
||||||
|
// dropping the one-shot sender releases it even during test unwinding.
|
||||||
|
#[cfg(test)]
|
||||||
|
struct RestoreActivationGate {
|
||||||
|
entered: mpsc::Sender<RestoreActivationEvidence>,
|
||||||
|
release: tokio::sync::oneshot::Receiver<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
struct RestoreActivationEvidence {
|
||||||
|
operation_id: crate::execution::WorkerLifecycleOperationId,
|
||||||
|
candidate: usize,
|
||||||
|
snapshot: Event,
|
||||||
|
handle: Option<WorkerHandle>,
|
||||||
|
allocation_retained: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl RestoreActivationGate {
|
||||||
|
async fn wait(self, evidence: RestoreActivationEvidence) {
|
||||||
|
let _ = self.entered.send(evidence);
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_secs(30), self.release)
|
||||||
|
.await
|
||||||
|
.is_ok(),
|
||||||
|
"test did not release the internal restore activation gate"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl ProfileRuntimeWorkerFactory {
|
impl ProfileRuntimeWorkerFactory {
|
||||||
fn retain_construction_failure(
|
fn retain_construction_failure(
|
||||||
&self,
|
&self,
|
||||||
@@ -2631,6 +2664,8 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn activate(&mut self) -> Result<RuntimeWorkerController, RuntimeRestoreFailure> {
|
async fn activate(&mut self) -> Result<RuntimeWorkerController, RuntimeRestoreFailure> {
|
||||||
|
#[cfg(test)]
|
||||||
|
let candidate_identity = self as *const Self as usize;
|
||||||
let prepared = self.prepared.as_mut().ok_or_else(|| {
|
let prepared = self.prepared.as_mut().ok_or_else(|| {
|
||||||
RuntimeRestoreFailure::unproven(
|
RuntimeRestoreFailure::unproven(
|
||||||
RuntimeRestoreStage::Controller,
|
RuntimeRestoreStage::Controller,
|
||||||
@@ -2668,6 +2703,30 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate {
|
|||||||
.expect("retained restore owner")
|
.expect("retained restore owner")
|
||||||
.allocation = prepared.allocation_cleanup();
|
.allocation = prepared.allocation_cleanup();
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
let gate = self
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.activation_effects_gate
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.take();
|
||||||
|
if let Some(gate) = gate {
|
||||||
|
let allocation_retained = self.factory.failed_restore_resources.lock().unwrap()
|
||||||
|
[&self.request.worker_ref]
|
||||||
|
.allocation
|
||||||
|
.is_some();
|
||||||
|
gate.wait(RestoreActivationEvidence {
|
||||||
|
operation_id: self.request.operation_id,
|
||||||
|
candidate: candidate_identity,
|
||||||
|
snapshot: prepared.snapshot().clone(),
|
||||||
|
handle: None,
|
||||||
|
allocation_retained,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[cfg(test)]
|
||||||
if self
|
if self
|
||||||
.factory
|
.factory
|
||||||
.restore_checkpoint(RuntimeRestoreStage::ActivationEffects)
|
.restore_checkpoint(RuntimeRestoreStage::ActivationEffects)
|
||||||
@@ -2703,6 +2762,30 @@ impl RuntimeRestoreCandidate for ProfileRestoreCandidate {
|
|||||||
}
|
}
|
||||||
.await
|
.await
|
||||||
.map_err(map_error)?;
|
.map_err(map_error)?;
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
let gate = self
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.activation_started_gate
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.take();
|
||||||
|
if let Some(gate) = gate {
|
||||||
|
let allocation_retained = self.factory.failed_restore_resources.lock().unwrap()
|
||||||
|
[&self.request.worker_ref]
|
||||||
|
.allocation
|
||||||
|
.is_some();
|
||||||
|
gate.wait(RestoreActivationEvidence {
|
||||||
|
operation_id: self.request.operation_id,
|
||||||
|
candidate: candidate_identity,
|
||||||
|
snapshot: started.streams.snapshot_event.clone(),
|
||||||
|
handle: Some(started.controller.handle.clone()),
|
||||||
|
allocation_retained,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
self.prepared = None;
|
self.prepared = None;
|
||||||
self.factory
|
self.factory
|
||||||
.failed_restore_resources
|
.failed_restore_resources
|
||||||
|
|||||||
@@ -94,6 +94,647 @@ mod publication {
|
|||||||
(release, received)
|
(release, received)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One-shot sender Drop is the RAII release, including a panic before the
|
||||||
|
// explicit release. The task also has a 30s fail-safe independent of Drop.
|
||||||
|
struct ActivationRelease(tokio::sync::oneshot::Sender<()>);
|
||||||
|
|
||||||
|
impl ActivationRelease {
|
||||||
|
fn release(self) {
|
||||||
|
let _ = self.0.send(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn activation_gate(
|
||||||
|
fixture: &Fixture,
|
||||||
|
started: bool,
|
||||||
|
) -> (ActivationRelease, mpsc::Receiver<RestoreActivationEvidence>) {
|
||||||
|
let (release, receiver) = tokio::sync::oneshot::channel();
|
||||||
|
let (entered, evidence) = mpsc::channel();
|
||||||
|
let faults = &fixture.backend.factory.restore_faults;
|
||||||
|
let slot = if started {
|
||||||
|
&faults.activation_started_gate
|
||||||
|
} else {
|
||||||
|
&faults.activation_effects_gate
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
slot.lock()
|
||||||
|
.unwrap()
|
||||||
|
.replace(RestoreActivationGate {
|
||||||
|
entered,
|
||||||
|
release: receiver
|
||||||
|
})
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
(ActivationRelease(release), evidence)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn restore_owner(
|
||||||
|
runtime: &EmbeddedRuntime,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
request_id: &str,
|
||||||
|
) -> serde_json::Value {
|
||||||
|
let record = runtime
|
||||||
|
.test_runtime_store()
|
||||||
|
.read_worker_record(worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
let guard = serde_json::to_value(&record.restore_guard).unwrap();
|
||||||
|
assert_eq!(guard["owners"].as_object().unwrap().len(), 1);
|
||||||
|
assert!(
|
||||||
|
guard["owners"]
|
||||||
|
.as_object()
|
||||||
|
.unwrap()
|
||||||
|
.contains_key(request_id)
|
||||||
|
);
|
||||||
|
guard["owners"][request_id].clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_activation_owned(
|
||||||
|
fixture: &Fixture,
|
||||||
|
runtime: &EmbeddedRuntime,
|
||||||
|
request_id: &str,
|
||||||
|
evidence: &RestoreActivationEvidence,
|
||||||
|
) -> Arc<PendingRuntimeRestore> {
|
||||||
|
let worker_ref = &fixture.request.worker_ref;
|
||||||
|
let durable = durable_operation(runtime, worker_ref).unwrap();
|
||||||
|
assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed);
|
||||||
|
assert_eq!(durable.operation_id, evidence.operation_id);
|
||||||
|
let owner = restore_owner(runtime, worker_ref, request_id);
|
||||||
|
assert_eq!(
|
||||||
|
owner["operation_id"],
|
||||||
|
serde_json::to_value(evidence.operation_id).unwrap()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
owner["receipt"].is_null(),
|
||||||
|
"committed activation cannot have a terminal receipt yet"
|
||||||
|
);
|
||||||
|
let pending = fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(pending.request.operation_id, evidence.operation_id);
|
||||||
|
assert!(pending.activation_started.load(Ordering::Acquire));
|
||||||
|
assert!(pending.task_in_flight.load(Ordering::Acquire));
|
||||||
|
assert!(pending.task.try_lock().unwrap().is_some());
|
||||||
|
assert!(
|
||||||
|
pending.result.lock().unwrap().is_none(),
|
||||||
|
"the task, not another caller, owns the candidate"
|
||||||
|
);
|
||||||
|
assert!(pending.activated.lock().unwrap().is_none());
|
||||||
|
assert!(evidence.allocation_retained);
|
||||||
|
assert!(fixture.backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(worker_ref)
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
fixture.assert_quiet();
|
||||||
|
pending
|
||||||
|
}
|
||||||
|
|
||||||
|
fn alert_marker(handle: &WorkerHandle, marker: &str) {
|
||||||
|
handle.alerter.alert(
|
||||||
|
protocol::AlertLevel::Warn,
|
||||||
|
protocol::AlertSource::Worker,
|
||||||
|
marker.into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_marker(event: &Event, marker: &str) -> bool {
|
||||||
|
matches!(event, Event::Alert(alert) if alert.message == marker)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn postcommit_activation_timeout_autonomously_publishes_same_controller_and_paired_streams() {
|
||||||
|
let mut fixture = Fixture::no_model();
|
||||||
|
Arc::get_mut(&mut fixture.backend)
|
||||||
|
.unwrap()
|
||||||
|
.spawn_restore_timeout = Duration::from_millis(250);
|
||||||
|
let runtime = stopped_runtime(&fixture);
|
||||||
|
let worker_ref = fixture.request.worker_ref.clone();
|
||||||
|
let scope = fixture.request.workspace_scope.as_ref().unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker_ref);
|
||||||
|
let (effects_release, effects_entered) = activation_gate(&fixture, false);
|
||||||
|
let (started_release, started_entered) = activation_gate(&fixture, true);
|
||||||
|
let observations = runtime.subscribe_worker_observation().unwrap();
|
||||||
|
let first = runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
first.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let effects = effects_entered.recv_timeout(WAIT).unwrap();
|
||||||
|
assert!(effects.handle.is_none());
|
||||||
|
let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &effects);
|
||||||
|
let canonical = fixture.canonical_files();
|
||||||
|
let owner = restore_owner(&runtime, &worker_ref, &request.request_id);
|
||||||
|
effects_release.release();
|
||||||
|
let started = started_entered.recv_timeout(WAIT).unwrap();
|
||||||
|
let same_pending =
|
||||||
|
assert_activation_owned(&fixture, &runtime, &request.request_id, &started);
|
||||||
|
assert!(Arc::ptr_eq(&pending, &same_pending));
|
||||||
|
assert_eq!(effects.candidate, started.candidate);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&effects.snapshot).unwrap(),
|
||||||
|
serde_json::to_value(&started.snapshot).unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
restore_owner(&runtime, &worker_ref, &request.request_id),
|
||||||
|
owner
|
||||||
|
);
|
||||||
|
let old = started.handle.unwrap();
|
||||||
|
assert!(!old.protocol_is_closed());
|
||||||
|
// This live event is queued on the precommit paired receiver. A newly
|
||||||
|
// subscribed bridge would miss it; it is not in its original snapshot.
|
||||||
|
alert_marker(&old, "activation-paired-before-publication");
|
||||||
|
started_release.release();
|
||||||
|
runtime.test_wait_restore_continuation(&worker_ref, started.operation_id, WAIT);
|
||||||
|
assert!(durable_operation(&runtime, &worker_ref).is_none());
|
||||||
|
let settled = restore_owner(&runtime, &worker_ref, &request.request_id);
|
||||||
|
assert_eq!(settled["operation_id"], owner["operation_id"]);
|
||||||
|
assert_eq!(settled["receipt"]["state"], "accepted");
|
||||||
|
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(old.same_controller(&execution.handle));
|
||||||
|
assert_eq!(execution.restore_operation_id, Some(started.operation_id));
|
||||||
|
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.publication_attempts
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
let transport = fixture
|
||||||
|
.backend
|
||||||
|
.clone()
|
||||||
|
.attach_worker_protocol(&worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
transport.validate().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&transport.snapshot).unwrap(),
|
||||||
|
serde_json::to_value(&effects.snapshot).unwrap()
|
||||||
|
);
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
let mut observations = observations;
|
||||||
|
tokio::time::timeout(WAIT, async {
|
||||||
|
loop {
|
||||||
|
let observed = observations.recv().await.unwrap();
|
||||||
|
if is_marker(&observed.payload, "activation-paired-before-publication") {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|_| "paired receiver lost its prepublication event".to_string())?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
fixture.canonical_files() == canonical,
|
||||||
|
"canonical files changed unexpectedly"
|
||||||
|
);
|
||||||
|
drop(transport);
|
||||||
|
drop(execution);
|
||||||
|
drop(pending);
|
||||||
|
drop(same_pending);
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(scope, &worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
assert!(old.protocol_is_closed());
|
||||||
|
fixture.assert_released();
|
||||||
|
fixture.assert_quiet();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn postcommit_activation_timeout_late_nonretryable_failure_never_fakes_rollback() {
|
||||||
|
for cleanup_fails in [false, true] {
|
||||||
|
let mut fixture = Fixture::no_model();
|
||||||
|
Arc::get_mut(&mut fixture.backend)
|
||||||
|
.unwrap()
|
||||||
|
.spawn_restore_timeout = Duration::from_millis(250);
|
||||||
|
let runtime = stopped_runtime(&fixture);
|
||||||
|
let worker_ref = fixture.request.worker_ref.clone();
|
||||||
|
let scope = fixture.request.workspace_scope.as_ref().unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker_ref);
|
||||||
|
let (release, entered) = activation_gate(&fixture, false);
|
||||||
|
let faults = &fixture.backend.factory.restore_faults;
|
||||||
|
*faults.fail_at.lock().unwrap() = Some(RuntimeRestoreStage::ActivationEffects);
|
||||||
|
faults.cleanup_fail.store(cleanup_fails, Ordering::SeqCst);
|
||||||
|
let first = runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
first.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let effects = entered.recv_timeout(WAIT).unwrap();
|
||||||
|
let pending =
|
||||||
|
assert_activation_owned(&fixture, &runtime, &request.request_id, &effects);
|
||||||
|
let canonical = fixture.canonical_files();
|
||||||
|
release.release();
|
||||||
|
if cleanup_fails {
|
||||||
|
let reason = runtime.test_wait_restore_continuation_parked(
|
||||||
|
&worker_ref,
|
||||||
|
effects.operation_id,
|
||||||
|
WAIT,
|
||||||
|
);
|
||||||
|
assert!(!reason.is_empty());
|
||||||
|
let failure = pending
|
||||||
|
.activated
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.as_ref()
|
||||||
|
.err()
|
||||||
|
.unwrap()
|
||||||
|
.clone();
|
||||||
|
assert_eq!(failure.stage, RuntimeRestoreStage::ActivationEffects);
|
||||||
|
assert!(failure.durable_effects_possible);
|
||||||
|
assert!(failure.needs_cleanup());
|
||||||
|
assert!(matches!(failure.cleanup, RuntimeRestoreCleanup::Failed(_)));
|
||||||
|
let durable = durable_operation(&runtime, &worker_ref).unwrap();
|
||||||
|
assert_eq!(durable.operation_id, effects.operation_id);
|
||||||
|
assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed);
|
||||||
|
assert!(
|
||||||
|
restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null()
|
||||||
|
);
|
||||||
|
assert!(Arc::ptr_eq(
|
||||||
|
&pending,
|
||||||
|
&fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
));
|
||||||
|
assert!(fixture.backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&worker_ref)
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.completed_restore_failure(effects.operation_id, &worker_ref)
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(faults.executions.load(Ordering::SeqCst), 1);
|
||||||
|
// Obstruction proof ends here. Stop below is fixture teardown,
|
||||||
|
// not an autonomous terminal-failure/success claim.
|
||||||
|
faults.cleanup_fail.store(false, Ordering::SeqCst);
|
||||||
|
} else {
|
||||||
|
// A nonretryable candidate must be discarded, not reactivated.
|
||||||
|
// Reconstruction belongs to the same committed operation.
|
||||||
|
runtime.test_wait_restore_continuation(&worker_ref, effects.operation_id, WAIT);
|
||||||
|
let settled = restore_owner(&runtime, &worker_ref, &request.request_id);
|
||||||
|
assert_eq!(settled["receipt"]["state"], "accepted");
|
||||||
|
assert_eq!(
|
||||||
|
fixture.backend.workers.lock().unwrap()[&worker_ref].restore_operation_id,
|
||||||
|
Some(effects.operation_id)
|
||||||
|
);
|
||||||
|
assert_eq!(faults.executions.load(Ordering::SeqCst), 2);
|
||||||
|
assert_eq!(faults.publication_attempts.load(Ordering::SeqCst), 1);
|
||||||
|
let transport = fixture
|
||||||
|
.backend
|
||||||
|
.clone()
|
||||||
|
.attach_worker_protocol(&worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
transport.validate().unwrap();
|
||||||
|
drop(transport);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
fixture.canonical_files() == canonical,
|
||||||
|
"canonical files changed unexpectedly"
|
||||||
|
);
|
||||||
|
drop(pending);
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(scope, &worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
fixture.assert_released();
|
||||||
|
fixture.assert_quiet();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn postcommit_activation_timeout_stop_remove_fence_late_cleanup_then_fresh_replacement() {
|
||||||
|
let mut fixture = Fixture::no_model();
|
||||||
|
Arc::get_mut(&mut fixture.backend)
|
||||||
|
.unwrap()
|
||||||
|
.spawn_restore_timeout = Duration::from_millis(250);
|
||||||
|
let runtime = stopped_runtime(&fixture);
|
||||||
|
let pause = runtime.test_pause_restore_continuation();
|
||||||
|
let worker_ref = fixture.request.worker_ref.clone();
|
||||||
|
let scope = fixture.request.workspace_scope.as_ref().unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker_ref);
|
||||||
|
let (release, entered) = activation_gate(&fixture, true);
|
||||||
|
let first = runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
first.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let started = entered.recv_timeout(WAIT).unwrap();
|
||||||
|
let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started);
|
||||||
|
let old = started.handle.unwrap();
|
||||||
|
let canonical = fixture.canonical_files();
|
||||||
|
alert_marker(&old, "superseded-old-buffered");
|
||||||
|
let stop_error = runtime
|
||||||
|
.stop_worker_scoped(scope, &worker_ref, None)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
stop_error
|
||||||
|
.to_string()
|
||||||
|
.contains("restore factory is still pending"),
|
||||||
|
"Stop must time out joining the activation task"
|
||||||
|
);
|
||||||
|
let remove_error = runtime
|
||||||
|
.delete_worker_scoped(scope, &worker_ref)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
remove_error
|
||||||
|
.to_string()
|
||||||
|
.contains("pending stop reconciliation"),
|
||||||
|
"Remove must be fenced by the unresolved Stop"
|
||||||
|
);
|
||||||
|
assert!(Arc::ptr_eq(
|
||||||
|
&pending,
|
||||||
|
&fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
));
|
||||||
|
assert_ne!(
|
||||||
|
runtime.worker_detail(&worker_ref).unwrap().status,
|
||||||
|
crate::catalog::WorkerStatus::Stopped
|
||||||
|
);
|
||||||
|
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
|
||||||
|
assert!(pending.task_in_flight.load(Ordering::Acquire));
|
||||||
|
assert!(!old.protocol_is_closed());
|
||||||
|
assert!(fixture.backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&worker_ref)
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
release.release();
|
||||||
|
drop(pause);
|
||||||
|
runtime.test_wait_restore_continuation(
|
||||||
|
&worker_ref,
|
||||||
|
started.operation_id,
|
||||||
|
Duration::from_secs(10),
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime.worker_detail(&worker_ref).unwrap().status,
|
||||||
|
crate::catalog::WorkerStatus::Stopped
|
||||||
|
);
|
||||||
|
assert!(old.protocol_is_closed());
|
||||||
|
assert!(!pending.task_in_flight.load(Ordering::Acquire));
|
||||||
|
assert!(
|
||||||
|
pending.task.try_lock().unwrap().is_none(),
|
||||||
|
"late task must have been joined"
|
||||||
|
);
|
||||||
|
fixture.assert_released();
|
||||||
|
// Confirmed Stop deletes the fixture orphan upload, not its Session,
|
||||||
|
// metadata, or committed entries. Keep the comparison content-private.
|
||||||
|
let mut stopped_canonical = canonical.clone();
|
||||||
|
let orphans = canonical
|
||||||
|
.keys()
|
||||||
|
.filter(|path| {
|
||||||
|
path.parent() == Some(std::path::Path::new("session/artifacts/paste"))
|
||||||
|
&& path.to_string_lossy().ends_with(".file.json")
|
||||||
|
})
|
||||||
|
.cloned()
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_eq!(orphans.len(), 1);
|
||||||
|
stopped_canonical.remove(&orphans[0]);
|
||||||
|
assert!(
|
||||||
|
fixture.canonical_files() == stopped_canonical,
|
||||||
|
"Stop must remove only the disposable orphan upload"
|
||||||
|
);
|
||||||
|
let superseded = restore_owner(&runtime, &worker_ref, &request.request_id);
|
||||||
|
assert_eq!(superseded["receipt"]["state"], "superseded");
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.read_worker_observation_events(
|
||||||
|
&worker_ref,
|
||||||
|
crate::observation::WorkerObservationCursor::zero()
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
// Only a fresh request from a fresh observation may create a replacement.
|
||||||
|
let replacement_request = runtime.test_restore_request(&worker_ref);
|
||||||
|
assert_ne!(replacement_request.request_id, request.request_id);
|
||||||
|
let accepted = runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, replacement_request)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(accepted.state, server_api::WorkerRestoreState::Accepted);
|
||||||
|
let replacement = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(!old.same_controller(&replacement.handle));
|
||||||
|
{
|
||||||
|
// Even a late retirement callback identifies the actual old
|
||||||
|
// Controller, never whichever execution now has this WorkerRef.
|
||||||
|
let lock = fixture.backend.worker_lock(&worker_ref).unwrap();
|
||||||
|
let _guard = lock.lock().unwrap();
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.deactivate_stopped_controller(&worker_ref, &old)
|
||||||
|
.unwrap();
|
||||||
|
let (_, observed_state, _) = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
assert!(Arc::ptr_eq(
|
||||||
|
&observed_state,
|
||||||
|
&replacement.handle.shared_state
|
||||||
|
));
|
||||||
|
}
|
||||||
|
assert_ne!(replacement.restore_operation_id, Some(started.operation_id));
|
||||||
|
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
// A diagnostic event on the retained old endpoint must never enter the
|
||||||
|
// new bridge, and a command on that closed endpoint must not be rerouted.
|
||||||
|
alert_marker(&old, "superseded-old-after-replacement");
|
||||||
|
let old_command = old.clone();
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
old_command
|
||||||
|
.send(Method::Pause {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(77111),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())
|
||||||
|
})
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
replacement.handle.shared_state.snapshot().state,
|
||||||
|
protocol::WorkerState::Idle
|
||||||
|
);
|
||||||
|
let transport = fixture
|
||||||
|
.backend
|
||||||
|
.clone()
|
||||||
|
.attach_worker_protocol(&worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
transport.validate().unwrap();
|
||||||
|
let mut observations = runtime.subscribe_worker_observation().unwrap();
|
||||||
|
alert_marker(&replacement.handle, "replacement-live-barrier");
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
tokio::time::timeout(WAIT, async {
|
||||||
|
loop {
|
||||||
|
let observed = observations.recv().await.unwrap();
|
||||||
|
assert!(!is_marker(&observed.payload, "superseded-old-buffered"));
|
||||||
|
assert!(!is_marker(
|
||||||
|
&observed.payload,
|
||||||
|
"superseded-old-after-replacement"
|
||||||
|
));
|
||||||
|
if is_marker(&observed.payload, "replacement-live-barrier") {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|_| "replacement bridge did not progress".to_string())?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let events = runtime
|
||||||
|
.read_worker_observation_events(
|
||||||
|
&worker_ref,
|
||||||
|
crate::observation::WorkerObservationCursor::zero(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
!events
|
||||||
|
.iter()
|
||||||
|
.any(|event| is_marker(&event.payload, "superseded-old-buffered")
|
||||||
|
|| is_marker(&event.payload, "superseded-old-after-replacement"))
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
fixture.canonical_files() == stopped_canonical,
|
||||||
|
"replacement changed canonical Session content"
|
||||||
|
);
|
||||||
|
drop(transport);
|
||||||
|
drop(replacement);
|
||||||
|
drop(pending);
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(scope, &worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
fixture.assert_released();
|
||||||
|
fixture.assert_quiet();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn postcommit_activation_timeout_gate_release_on_panic_keeps_owned_task_bounded() {
|
||||||
|
let mut fixture = Fixture::no_model();
|
||||||
|
Arc::get_mut(&mut fixture.backend)
|
||||||
|
.unwrap()
|
||||||
|
.spawn_restore_timeout = Duration::from_millis(250);
|
||||||
|
let runtime = stopped_runtime(&fixture);
|
||||||
|
let worker_ref = fixture.request.worker_ref.clone();
|
||||||
|
let scope = fixture.request.workspace_scope.as_ref().unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker_ref);
|
||||||
|
let (release, entered) = activation_gate(&fixture, true);
|
||||||
|
let first = runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
first.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let started = entered.recv_timeout(WAIT).unwrap();
|
||||||
|
let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started);
|
||||||
|
let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || {
|
||||||
|
let _release_on_unwind = release;
|
||||||
|
panic!("intentional test-owner unwind at activation gate");
|
||||||
|
}));
|
||||||
|
assert!(panic.is_err());
|
||||||
|
runtime.test_wait_restore_continuation(&worker_ref, started.operation_id, WAIT);
|
||||||
|
assert_eq!(
|
||||||
|
restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"],
|
||||||
|
"accepted"
|
||||||
|
);
|
||||||
|
assert!(!pending.task_in_flight.load(Ordering::Acquire));
|
||||||
|
assert!(pending.task.try_lock().unwrap().is_none());
|
||||||
|
assert!(
|
||||||
|
started
|
||||||
|
.handle
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.same_controller(&fixture.backend.workers.lock().unwrap()[&worker_ref].handle)
|
||||||
|
);
|
||||||
|
drop(pending);
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(scope, &worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
assert!(started.handle.unwrap().protocol_is_closed());
|
||||||
|
fixture.assert_released();
|
||||||
|
fixture.assert_quiet();
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial_test::serial(worker_allocation)]
|
#[serial_test::serial(worker_allocation)]
|
||||||
fn startup_fault_recovery_retains_cleanup_or_committed_reconstruction_without_repost() {
|
fn startup_fault_recovery_retains_cleanup_or_committed_reconstruction_without_repost() {
|
||||||
|
|||||||
Reference in New Issue
Block a user