fix(runtime): consume original Stop receipts for worker deletion

This commit is contained in:
2026-11-05 01:58:24 +09:00
parent 7181287dfc
commit 0f86465929
6 changed files with 419 additions and 50 deletions
+24 -3
View File
@@ -3266,6 +3266,14 @@ impl Runtime {
let _guard = operation_lock let _guard = operation_lock
.lock() .lock()
.map_err(|_| RuntimeError::StatePoisoned)?; .map_err(|_| RuntimeError::StatePoisoned)?;
self.execution_termination_under_lock(worker_ref)
}
// Caller holds the Runtime Worker operation lock, including deletion callers.
fn execution_termination_under_lock(
&self,
worker_ref: &WorkerRef,
) -> Result<Option<crate::execution::WorkerExecutionTermination>, RuntimeError> {
let (backend, saved) = { let (backend, saved) = {
let mut state = self.lock()?; let mut state = self.lock()?;
state.ensure_worker_ref(worker_ref)?; state.ensure_worker_ref(worker_ref)?;
@@ -4630,7 +4638,7 @@ impl Runtime {
if let Some(scope) = scope { if let Some(scope) = scope {
self.ensure_worker_in_workspace(scope, worker_ref)?; self.ensure_worker_in_workspace(scope, worker_ref)?;
} }
let backend = { let (backend, has_termination) = {
let state = self.lock()?; let state = self.lock()?;
state.ensure_running()?; state.ensure_running()?;
state.ensure_worker_ref(worker_ref)?; state.ensure_worker_ref(worker_ref)?;
@@ -4648,9 +4656,22 @@ impl Runtime {
worker_ref.worker_id worker_ref.worker_id
))); )));
} }
state.execution_backend.clone() (
state.execution_backend.clone(),
worker.restore_guard.execution_termination.is_some(),
)
}; };
if let Some(backend) = backend { if has_termination {
// Consume the canonical Stop's exact receipt. A new Stop could
// target a replacement Controller or discard a later candidate;
// neither is authorized by the original scope's termination.
if self.execution_termination_under_lock(worker_ref)?.is_none() {
return Err(RuntimeError::WorkerExecutionUnavailable {
worker_id: worker_ref.worker_id,
message: "canonical Stop receipt no longer matches the execution owner; deletion remains pending".into(),
});
}
} else if let Some(backend) = backend {
let result = backend.stop_worker(worker_ref); let result = backend.stop_worker(worker_ref);
if !result.is_accepted() { if !result.is_accepted() {
return Err(RuntimeError::WorkerExecutionRejected { return Err(RuntimeError::WorkerExecutionRejected {
@@ -196,8 +196,21 @@ fn subject_delete_unregistered_profile_controller_retries_unknown_shutdown_witho
backend.restore_worker(request.clone()), backend.restore_worker(request.clone()),
WorkerExecutionSpawnResult::Rejected(_) WorkerExecutionSpawnResult::Rejected(_)
)); ));
let stop_operation_id = backend.admissions.lock().unwrap()[&request.worker_ref]
.deletion_stop_operation_id
.expect("the original rejected Stop keeps its identity for retry");
backend.subject_delete_worker(&deletion).unwrap(); backend.subject_delete_worker(&deletion).unwrap();
let original = backend
.termination_receipt(&request.worker_ref)
.unwrap()
.unwrap();
assert_eq!(original.stop_operation_id, stop_operation_id);
assert_eq!(original.execution_id, execution.tasks.execution_id);
backend.subject_delete_worker(&deletion).unwrap(); backend.subject_delete_worker(&deletion).unwrap();
assert_eq!(
backend.termination_receipt(&request.worker_ref).unwrap(),
Some(original)
);
assert!(execution.handle.protocol_is_closed()); assert!(execution.handle.protocol_is_closed());
assert!(execution.tasks.tasks.lock().unwrap().is_empty()); assert!(execution.tasks.tasks.lock().unwrap().is_empty());
assert!(backend.workers.lock().unwrap().is_empty()); assert!(backend.workers.lock().unwrap().is_empty());
@@ -283,3 +296,111 @@ fn subject_delete_pending_restore_fences_late_controller_and_never_restarts_it()
WorkerExecutionSpawnResult::Rejected(_) WorkerExecutionSpawnResult::Rejected(_)
)); ));
} }
#[test]
fn subject_delete_without_original_owner_retains_child_identity_but_never_issues_receipt() {
let root = tempfile::tempdir().unwrap();
let backend = WorkerRuntimeExecutionBackend::new(MockFactory {
client: MockClient::new(vec![]),
runtime_base: root.path().join("must-not-launch"),
cwd: root.path().to_path_buf(),
store_dir: root.path().join("sessions"),
worker_metadata_dir: root.path().join("metadata"),
observed_cwds: Arc::default(),
observed_workspace_clients: Arc::default(),
})
.unwrap();
let worker_ref = WorkerRef::new(WorkerId::now_v7());
let deletion = runtime_api::SubjectDeleteWorkerRequest {
workspace_id: "ws-test".into(),
worker_id: worker_ref.worker_id,
create_fingerprint: "not-admitted".into(),
operation_id: "delete-without-owner".into(),
};
assert!(
backend
.subject_delete_worker(&deletion)
.unwrap_err()
.contains("map absence")
);
let child = backend.admissions.lock().unwrap()[&worker_ref]
.deletion_stop_operation_id
.unwrap();
assert!(
backend
.subject_delete_worker(&deletion)
.unwrap_err()
.contains("map absence")
);
assert_eq!(
backend.admissions.lock().unwrap()[&worker_ref].deletion_stop_operation_id,
Some(child)
);
assert!(backend.completed_stops.lock().unwrap().is_empty());
assert!(backend.termination_receipt(&worker_ref).unwrap().is_none());
assert!(!root.path().join("must-not-launch").exists());
}
#[test]
#[cfg(feature = "fs-store")]
fn subject_delete_consumes_prior_canonical_stop_without_retagging_original_scope() {
let temp = tempfile::tempdir_in(".").unwrap();
let root = fs::canonicalize(temp.path()).unwrap();
let backend = Arc::new(
WorkerRuntimeExecutionBackend::new(MockFactory {
client: MockClient::new(vec![]),
runtime_base: PathBuf::from(temp.path().file_name().unwrap()).join("c"),
cwd: root.clone(),
store_dir: root.join("sessions"),
worker_metadata_dir: root.join("metadata"),
observed_cwds: Arc::default(),
observed_workspace_clients: Arc::default(),
})
.unwrap(),
);
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
crate::fs_store::FsRuntimeStoreOptions::new(root.join("catalog"))
.with_runtime_id("delete-prior-stop"),
backend.clone(),
)
.unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let scope = crate::runtime::RuntimeWorkspaceScope::new("ws-test", "server-test");
let mut create = create_request("prior Stop Subject deletion");
create.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: scope.workspace_id.clone(),
language: "English".into(),
});
create.workspace_api = Some(WorkspaceApiRef {
workspace_id: scope.workspace_id.clone(),
base_url: "https://workspace.invalid".into(),
});
let worker = runtime
.create_worker_scoped(&scope, create.clone())
.unwrap();
runtime
.stop_worker_scoped(&scope, &worker.worker_ref, None)
.unwrap();
let original = runtime
.execution_termination(&worker.worker_ref)
.unwrap()
.unwrap();
let deletion = runtime_api::SubjectDeleteWorkerRequest {
workspace_id: scope.workspace_id.clone(),
worker_id: create.worker_id,
create_fingerprint: create.create_fingerprint,
operation_id: "delete-prior-stop".into(),
};
runtime
.subject_delete_worker_scoped(&scope, deletion.clone())
.unwrap();
backend.subject_delete_worker(&deletion).unwrap();
assert_eq!(
backend.termination_receipt(&worker.worker_ref).unwrap(),
Some(original.clone())
);
assert_eq!(
backend.admissions.lock().unwrap()[&worker.worker_ref].deletion_stop_operation_id,
Some(original.stop_operation_id)
);
}
+146 -35
View File
@@ -3903,6 +3903,9 @@ struct WorkerAdmission {
workspace_id: Option<String>, workspace_id: Option<String>,
create_fingerprint: String, create_fingerprint: String,
deletion: Option<runtime_api::SubjectDeleteWorkerRequest>, deletion: Option<runtime_api::SubjectDeleteWorkerRequest>,
// Stable child Stop for this exact deletion. This is an operation identity,
// not termination proof; only the original joined owner can issue a receipt.
deletion_stop_operation_id: Option<crate::execution::WorkerLifecycleOperationId>,
} }
/// Worker execution backend backed by real worker crate Workers. /// Worker execution backend backed by real worker crate Workers.
@@ -4007,6 +4010,7 @@ where
workspace_id: workspace.map(str::to_string), workspace_id: workspace.map(str::to_string),
create_fingerprint: fingerprint.into(), create_fingerprint: fingerprint.into(),
deletion: None, deletion: None,
deletion_stop_operation_id: None,
}, },
); );
} }
@@ -5825,18 +5829,42 @@ where
let lock = self.worker_lock(&worker)?; let lock = self.worker_lock(&worker)?;
let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?; let _operation = lock.lock().map_err(|_| "worker operation lock poisoned")?;
self.validate_subject_delete_identity(request)?; self.validate_subject_delete_identity(request)?;
self.admissions let operation_id = {
.lock() let mut admissions = self
.map_err(|_| "worker admission lock poisoned")? .admissions
.insert( .lock()
worker.clone(), .map_err(|_| "worker admission lock poisoned")?;
WorkerAdmission { let admission = admissions
.entry(worker.clone())
.or_insert_with(|| WorkerAdmission {
workspace_id: Some(request.workspace_id.clone()), workspace_id: Some(request.workspace_id.clone()),
create_fingerprint: request.create_fingerprint.clone(), create_fingerprint: request.create_fingerprint.clone(),
deletion: Some(request.clone()), deletion: None,
}, deletion_stop_operation_id: None,
); });
let result = self.stop_worker_under_lock(&worker, None); admission.deletion = Some(request.clone());
match admission.deletion_stop_operation_id {
Some(operation_id) => operation_id,
None => {
// Reuse a completed original Stop, never retag its receipt
// as a newly generated deletion child. The Stop consumer
// below rejects replay if any replacement owner is present.
let operation_id = self
.completed_stops
.lock()
.map_err(|_| "stop completion receipts lock poisoned")?
.get(&worker)
.map(|receipt| receipt.stop_operation_id)
.unwrap_or_default();
admission.deletion_stop_operation_id = Some(operation_id);
operation_id
}
}
};
let result = self.stop_worker_operation_under_lock(WorkerExecutionStopRequest {
worker_ref: worker.clone(),
operation_id,
});
if !result.is_accepted() { if !result.is_accepted() {
return Err(result.message_or_default()); return Err(result.message_or_default());
} }
@@ -8737,12 +8765,17 @@ mod tests {
}) })
.await .await
.unwrap(); .unwrap();
if let Some(receiver) = controller.shutdown.lock().await.take() { let receiver = controller
tokio::time::timeout(Duration::from_secs(5), receiver) .shutdown
.await .lock()
.unwrap() .await
.unwrap(); .take()
} .expect("fixture must retain the original Controller shutdown report");
let report = tokio::time::timeout(Duration::from_secs(5), receiver)
.await
.unwrap()
.unwrap();
assert!(report.feature_failures.is_empty(), "{report:?}");
tokio::time::timeout(Duration::from_secs(5), controller.controller_task) tokio::time::timeout(Duration::from_secs(5), controller.controller_task)
.await .await
.unwrap() .unwrap()
@@ -9795,11 +9828,22 @@ mod tests {
.contains("still present"), .contains("still present"),
"closed Controller is still residual execution" "closed Controller is still residual execution"
); );
assert!(backend.stop_worker(&request.worker_ref).is_accepted()); let stop = WorkerExecutionStopRequest {
worker_ref: request.worker_ref.clone(),
operation_id: crate::execution::WorkerLifecycleOperationId::new(),
};
let original = backend.stop_worker_operation(stop.clone());
assert!(original.is_accepted(), "{original:?}");
let receipt = original.termination.as_ref().unwrap();
assert_eq!(receipt.execution_id, execution.tasks.execution_id);
assert!(backend.workers.lock().unwrap().is_empty()); assert!(backend.workers.lock().unwrap().is_empty());
backend.recover_subject_create(&recovery).unwrap(); backend.recover_subject_create(&recovery).unwrap();
assert!(execution.tasks.tasks.lock().unwrap().is_empty()); assert!(execution.tasks.tasks.lock().unwrap().is_empty());
assert!(backend.stop_worker(&request.worker_ref).is_accepted()); assert_eq!(backend.stop_worker_operation(stop), original);
assert!(
!backend.stop_worker(&request.worker_ref).is_accepted(),
"identity-less absence is not a replay"
);
} }
#[test] #[test]
@@ -11263,6 +11307,8 @@ mod tests {
struct PendingCreateFactory { struct PendingCreateFactory {
started: mpsc::Sender<()>, started: mpsc::Sender<()>,
dropped: mpsc::Sender<()>, dropped: mpsc::Sender<()>,
release: Arc<tokio::sync::Notify>,
inner: MockFactory,
} }
struct FactoryFutureDrop(mpsc::Sender<()>); struct FactoryFutureDrop(mpsc::Sender<()>);
@@ -11286,11 +11332,12 @@ mod tests {
async fn spawn_controller( async fn spawn_controller(
&self, &self,
_request: WorkerExecutionSpawnRequest, request: WorkerExecutionSpawnRequest,
) -> Result<RuntimeWorkerController, RuntimeRestoreFailure> { ) -> Result<RuntimeWorkerController, RuntimeRestoreFailure> {
let _drop = FactoryFutureDrop(self.dropped.clone()); let _drop = FactoryFutureDrop(self.dropped.clone());
self.started.send(()).unwrap(); self.started.send(()).unwrap();
std::future::pending().await self.release.notified().await;
self.inner.spawn_controller(request).await
} }
async fn restore_controller( async fn restore_controller(
@@ -11303,15 +11350,26 @@ mod tests {
#[cfg(feature = "fs-store")] #[cfg(feature = "fs-store")]
#[test] #[test]
fn create_timeout_cancels_factory_and_removes_persisted_worker() { fn create_timeout_retains_factory_and_persisted_worker_until_original_controller_is_joined() {
let root = tempfile::tempdir().unwrap(); let root = tempfile::tempdir().unwrap();
let runtime_store_dir = root.path().join("runtime"); let runtime_store_dir = root.path().join("runtime");
let (started_tx, started) = mpsc::channel(); let (started_tx, started) = mpsc::channel();
let (dropped_tx, dropped) = mpsc::channel(); let (dropped_tx, dropped) = mpsc::channel();
let release = Arc::new(tokio::sync::Notify::new());
let backend = Arc::new( let backend = Arc::new(
WorkerRuntimeExecutionBackend::new(PendingCreateFactory { WorkerRuntimeExecutionBackend::new(PendingCreateFactory {
started: started_tx, started: started_tx,
dropped: dropped_tx, dropped: dropped_tx,
release: release.clone(),
inner: MockFactory {
client: MockClient::new(vec![]),
runtime_base: root.path().join("control"),
cwd: root.path().to_path_buf(),
store_dir: root.path().join("sessions"),
worker_metadata_dir: root.path().join("metadata"),
observed_cwds: Arc::default(),
observed_workspace_clients: Arc::default(),
},
}) })
.unwrap() .unwrap()
.with_spawn_restore_timeout(Duration::from_millis(20)), .with_spawn_restore_timeout(Duration::from_millis(20)),
@@ -11339,23 +11397,46 @@ mod tests {
.unwrap_err(); .unwrap_err();
let error = create.join().unwrap().unwrap_err(); let error = create.join().unwrap().unwrap_err();
let worker_ref = WorkerRef::new(worker_id);
assert!(matches!( assert!(matches!(
delete_error, delete_error,
crate::error::RuntimeError::WorkerNotFound { worker_id: missing } if missing == worker_id crate::error::RuntimeError::InvalidRequest(_)
)); ));
assert!(error.to_string().contains("was cancelled")); assert!(error.to_string().contains("pending"), "{error}");
assert!(matches!(dropped.try_recv(), Err(mpsc::TryRecvError::Empty)));
assert!(backend.pending_spawn_exists(&worker_ref).unwrap());
assert!(
runtime
.execution_termination(&worker_ref)
.unwrap()
.is_none()
);
let worker_path = runtime_store_dir
.join("workers")
.join(worker_id.to_string());
assert!(
worker_path.exists(),
"timeout must retain the unresolved Worker"
);
assert_eq!(runtime.list_workers().unwrap().len(), 1);
// Release the original factory, not a replacement. Stop must transfer
// its actual late Controller into the owned scope and join/report it.
release.notify_one();
runtime.stop_worker(&worker_ref, None).unwrap();
dropped dropped
.try_recv() .try_recv()
.expect("cancelled factory future must be dropped before create returns"); .expect("factory completed normally before Stop settled");
assert!(runtime.list_workers().unwrap().is_empty()); assert!(!backend.pending_spawn_exists(&worker_ref).unwrap());
assert!(backend.workers.lock().unwrap().is_empty()); let original = runtime.execution_termination(&worker_ref).unwrap().unwrap();
assert!( assert_eq!(
!runtime_store_dir original.scope,
.join("workers") crate::execution::WorkerExecutionTerminationScope::Controller
.join(worker_id.to_string())
.exists(),
"failed create must remove its persisted Worker aggregate"
); );
assert!(original.shutdown_command_id.is_some());
assert!(backend.workers.lock().unwrap().is_empty());
runtime.delete_worker(&worker_ref).unwrap();
assert!(!worker_path.exists());
} }
// Generic controller tests explicitly substitute the Host's non-Subject // Generic controller tests explicitly substitute the Host's non-Subject
@@ -11922,7 +12003,13 @@ mod tests {
&self, &self,
_request: WorkerExecutionSpawnRequest, _request: WorkerExecutionSpawnRequest,
) -> Result<RuntimeWorkerController, RuntimeRestoreFailure> { ) -> Result<RuntimeWorkerController, RuntimeRestoreFailure> {
Err("spawn failed".into()) // This fixture returns before constructing a Worker, Controller or
// background resource. Unlike a string failure, these are known
// owner facts and allow only the adapter-created workdir rollback.
Err(RuntimeRestoreFailure::refused(
RuntimeRestoreStage::Preflight,
"spawn failed",
))
} }
async fn restore_controller( async fn restore_controller(
@@ -14104,7 +14191,19 @@ mod tests {
fs::read_to_string(checkout.join("README.md")).unwrap(), fs::read_to_string(checkout.join("README.md")).unwrap(),
"owner recovery in progress" "owner recovery in progress"
); );
assert!(backend.stop_worker(&request.worker_ref).is_accepted()); let stop = WorkerExecutionStopRequest {
worker_ref: request.worker_ref.clone(),
operation_id: crate::execution::WorkerLifecycleOperationId::new(),
};
let result = backend.stop_worker_operation(stop.clone());
assert!(result.is_accepted(), "{result:?}");
assert!(
result
.termination
.as_ref()
.unwrap()
.confirms_scope_shutdown(&stop.worker_ref, stop.operation_id)
);
} }
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
@@ -14176,7 +14275,19 @@ mod tests {
before, before,
"damaged checkout must be retained" "damaged checkout must be retained"
); );
assert!(backend.stop_worker(&request.worker_ref).is_accepted()); let stop = WorkerExecutionStopRequest {
worker_ref: request.worker_ref.clone(),
operation_id: crate::execution::WorkerLifecycleOperationId::new(),
};
let result = backend.stop_worker_operation(stop.clone());
assert!(result.is_accepted(), "{result:?}");
assert!(
result
.termination
.as_ref()
.unwrap()
.confirms_scope_shutdown(&stop.worker_ref, stop.operation_id)
);
println!( println!(
"damage={damage}; reason={:?}; classification={:?}; cwd_bind_count={}; retained_files={}; retained_data_unchanged=true; stopped=true", "damage={damage}; reason={:?}; classification={:?}; cwd_bind_count={}; retained_files={}; retained_data_unchanged=true; stopped=true",
error.failure_reason(), error.failure_reason(),
@@ -253,9 +253,15 @@ mod tests {
#[test] #[test]
fn typed_factory_failure_survives_adapter_and_protocol_without_changing_disposition() { fn typed_factory_failure_survives_adapter_and_protocol_without_changing_disposition() {
let backend = WorkerRuntimeExecutionBackend::new(TypedFailingFactory).unwrap(); let backend = WorkerRuntimeExecutionBackend::new(TypedFailingFactory).unwrap();
let WorkerExecutionSpawnResult::Errored(result) = backend.spawn_worker(request()) else { let request = request();
panic!("cause attachment must not reclassify the existing spawn result") let worker_ref = request.worker_ref.clone();
let WorkerExecutionSpawnResult::ReconciliationRequired { result, .. } =
backend.spawn_worker(request)
else {
panic!("unproven resources and failed cleanup must retain the original factory owner")
}; };
assert!(backend.pending_spawn_exists(&worker_ref).unwrap());
assert!(backend.termination_receipt(&worker_ref).unwrap().is_none());
assert_primary(&result); assert_primary(&result);
let transferred: WorkerExecutionResult = let transferred: WorkerExecutionResult =
serde_json::from_slice(&serde_json::to_vec(&result).unwrap()).unwrap(); serde_json::from_slice(&serde_json::to_vec(&result).unwrap()).unwrap();
@@ -97,6 +97,15 @@ fn stop_termination_real_join_survives_fs_reopen_rejects_replacement_and_legacy_
.unwrap() .unwrap()
.is_none() .is_none()
); );
assert!(reopened.delete_worker(&worker.worker_ref).is_err());
assert!(worker_path.exists());
assert!(
backend
.pending_restore(&worker.worker_ref)
.unwrap()
.is_some(),
"deletion must not stop/discard a replacement candidate with the old receipt"
);
let replacement_stop = backend.stop_worker_operation(WorkerExecutionStopRequest { let replacement_stop = backend.stop_worker_operation(WorkerExecutionStopRequest {
worker_ref: worker.worker_ref.clone(), worker_ref: worker.worker_ref.clone(),
operation_id: crate::execution::WorkerLifecycleOperationId::new(), operation_id: crate::execution::WorkerLifecycleOperationId::new(),
@@ -110,6 +119,8 @@ fn stop_termination_real_join_survives_fs_reopen_rejects_replacement_and_legacy_
reopened.execution_termination(&worker.worker_ref).is_err(), reopened.execution_termination(&worker.worker_ref).is_err(),
"a different run/stop-operation receipt must not replace canonical evidence" "a different run/stop-operation receipt must not replace canonical evidence"
); );
assert!(reopened.delete_worker(&worker.worker_ref).is_err());
assert!(worker_path.exists());
drop(reopened); drop(reopened);
drop(backend); drop(backend);
@@ -155,3 +166,97 @@ fn stop_termination_real_join_survives_fs_reopen_rejects_replacement_and_legacy_
.is_none() .is_none()
); );
} }
#[cfg(feature = "fs-store")]
#[test]
fn delete_after_reopen_consumes_original_stop_without_starting_or_restopping_controller() {
let temp = tempfile::tempdir_in(".").unwrap();
let root = fs::canonicalize(temp.path()).unwrap();
let options = crate::fs_store::FsRuntimeStoreOptions::new(root.join("runtime"))
.with_runtime_id("delete-original-stop");
let backend = Arc::new(
WorkerRuntimeExecutionBackend::new(MockFactory {
client: MockClient::new(vec![]),
runtime_base: PathBuf::from(temp.path().file_name().unwrap()).join("c"),
cwd: root.clone(),
store_dir: root.join("sessions"),
worker_metadata_dir: root.join("metadata"),
observed_cwds: Arc::default(),
observed_workspace_clients: Arc::default(),
})
.unwrap(),
);
let runtime =
EmbeddedRuntime::with_fs_store_and_execution_backend(options.clone(), backend.clone())
.unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let worker = runtime
.create_worker(create_request("delete original stop"))
.unwrap();
runtime.stop_worker(&worker.worker_ref, None).unwrap();
let original = runtime
.execution_termination(&worker.worker_ref)
.unwrap()
.unwrap();
// Inject mismatched observations, never positive termination evidence.
// Deletion must keep the canonical original and refuse every substitution.
for mismatch in ["worker", "execution", "operation"] {
let mut observed = original.clone();
match mismatch {
"worker" => observed.worker_ref = WorkerRef::new(WorkerId::now_v7()),
"execution" => observed.execution_id = uuid::Uuid::now_v7(),
"operation" => {
observed.stop_operation_id = crate::execution::WorkerLifecycleOperationId::new()
}
_ => unreachable!(),
}
backend
.completed_stops
.lock()
.unwrap()
.insert(worker.worker_ref.clone(), observed);
assert!(
runtime.delete_worker(&worker.worker_ref).is_err(),
"{mismatch}"
);
assert!(
options
.root
.join("workers")
.join(worker.worker_id.to_string())
.exists()
);
}
backend
.completed_stops
.lock()
.unwrap()
.insert(worker.worker_ref.clone(), original.clone());
drop(runtime);
drop(backend);
let no_launch = root.join("must-not-launch");
let backend = Arc::new(
WorkerRuntimeExecutionBackend::new(ProfileRuntimeWorkerFactory::new(&no_launch)).unwrap(),
);
let reopened =
EmbeddedRuntime::with_fs_store_and_execution_backend(options.clone(), backend.clone())
.unwrap();
assert!(backend.completed_stops.lock().unwrap().is_empty());
assert_eq!(
reopened.execution_termination(&worker.worker_ref).unwrap(),
Some(original)
);
reopened.delete_worker(&worker.worker_ref).unwrap();
assert!(
!options
.root
.join("workers")
.join(worker.worker_id.to_string())
.exists()
);
assert!(
backend.completed_stops.lock().unwrap().is_empty(),
"no new Stop was issued"
);
assert!(!no_launch.exists());
}
@@ -1246,11 +1246,18 @@ mod restore_candidate {
.is_some(), .is_some(),
"retain the real prepared Worker's close-only resource owner" "retain the real prepared Worker's close-only resource owner"
); );
let stop = WorkerExecutionStopRequest {
worker_ref: fixture.request.worker_ref.clone(),
operation_id: crate::execution::WorkerLifecycleOperationId::new(),
};
let result = fixture.backend.stop_worker_operation(stop.clone());
assert!(result.is_accepted(), "{result:?}");
assert!( assert!(
fixture result
.backend .termination
.stop_worker(&fixture.request.worker_ref) .as_ref()
.is_accepted() .unwrap()
.confirms_scope_shutdown(&stop.worker_ref, stop.operation_id)
); );
fixture.assert_released(); fixture.assert_released();
@@ -1285,12 +1292,10 @@ mod restore_candidate {
assert_eq!(opens.load(Ordering::SeqCst), 0); assert_eq!(opens.load(Ordering::SeqCst), 0);
fixture.assert_quiet(); fixture.assert_quiet();
assert_eq!(fixture.canonical_files(), before); assert_eq!(fixture.canonical_files(), before);
assert!( let replay = fixture.backend.stop_worker_operation(stop);
fixture assert!(replay.is_accepted(), "{replay:?}");
.backend assert_eq!(replay.termination, result.termination);
.stop_worker(&fixture.request.worker_ref) assert_eq!(replay.execution_id, result.execution_id);
.is_accepted()
);
assert_eq!(fixture.canonical_files(), before); assert_eq!(fixture.canonical_files(), before);
} }