fix: recover confirmed worker restore failures
This commit is contained in:
@@ -630,6 +630,16 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Completed resource-owner evidence retained while Runtime settlement may
|
||||||
|
/// still need retry. This is not a new request or execution authorization.
|
||||||
|
fn completed_restore_failure(
|
||||||
|
&self,
|
||||||
|
_operation_id: WorkerLifecycleOperationId,
|
||||||
|
_worker_ref: &WorkerRef,
|
||||||
|
) -> Option<WorkerExecutionSpawnResult> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
fn reconcile_restore(
|
fn reconcile_restore(
|
||||||
&self,
|
&self,
|
||||||
request: WorkerExecutionRestoreRequest,
|
request: WorkerExecutionRestoreRequest,
|
||||||
@@ -898,6 +908,15 @@ impl WorkerExecutionBackendRef {
|
|||||||
self.backend.preflight_restore(request)
|
self.backend.preflight_restore(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn completed_restore_failure(
|
||||||
|
&self,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> Option<WorkerExecutionSpawnResult> {
|
||||||
|
self.backend
|
||||||
|
.completed_restore_failure(operation_id, worker_ref)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn reconcile_restore(
|
pub(crate) fn reconcile_restore(
|
||||||
&self,
|
&self,
|
||||||
request: WorkerExecutionRestoreRequest,
|
request: WorkerExecutionRestoreRequest,
|
||||||
|
|||||||
@@ -2489,6 +2489,14 @@ impl Runtime {
|
|||||||
.map(|result| (Some(result), None))
|
.map(|result| (Some(result), None))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(all(test, feature = "fs-store"))]
|
||||||
|
pub(crate) fn test_runtime_store(&self) -> FsRuntimeStore {
|
||||||
|
match &self.lock().unwrap().persistence {
|
||||||
|
RuntimePersistence::Fs(store) => store.clone(),
|
||||||
|
RuntimePersistence::Memory => panic!("expected filesystem Runtime"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) fn test_restore_request(
|
pub(crate) fn test_restore_request(
|
||||||
&self,
|
&self,
|
||||||
@@ -2944,7 +2952,11 @@ impl Runtime {
|
|||||||
return Err(error);
|
return Err(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
let restore_result = if reconcile {
|
let completed_failure = backend.completed_restore_failure(operation_id, worker_ref);
|
||||||
|
let has_completed_failure = completed_failure.is_some();
|
||||||
|
let restore_result = if let Some(result) = completed_failure {
|
||||||
|
result
|
||||||
|
} else if reconcile {
|
||||||
backend.reconcile_restore(request)
|
backend.reconcile_restore(request)
|
||||||
} else {
|
} else {
|
||||||
backend.restore_worker(request)
|
backend.restore_worker(request)
|
||||||
@@ -3069,7 +3081,7 @@ impl Runtime {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
WorkerExecutionSpawnResult::Rejected(result) => {
|
WorkerExecutionSpawnResult::Rejected(result) => {
|
||||||
if reconcile {
|
if reconcile && !has_completed_failure {
|
||||||
return Ok(RuntimeWorkerRestoreResult::failed(
|
return Ok(RuntimeWorkerRestoreResult::failed(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"worker_restore_reconciliation_rejected",
|
"worker_restore_reconciliation_rejected",
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,945 @@
|
|||||||
|
// Integration coverage through EmbeddedRuntime -> real Profile factory -> real
|
||||||
|
// Worker/bootstrap, local Git Workdir provider, and allocation table. Model I/O
|
||||||
|
// is not invoked; this is a deterministic ordinary suite, not product E2E.
|
||||||
|
|
||||||
|
struct ProfileFailureFixture {
|
||||||
|
runtime: EmbeddedRuntime,
|
||||||
|
backend: Arc<WorkerRuntimeExecutionBackend>,
|
||||||
|
worker: crate::catalog::WorkerDetail,
|
||||||
|
_repo: tempfile::TempDir,
|
||||||
|
root: tempfile::TempDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProfileFailureFixture {
|
||||||
|
fn new(timeout: Duration) -> Self {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let repo = create_clean_repo();
|
||||||
|
let store = root.path().join("private-restore-test/runtime");
|
||||||
|
let backend = Arc::new(
|
||||||
|
WorkerRuntimeExecutionBackend::new(
|
||||||
|
ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.with_spawn_restore_timeout(timeout)
|
||||||
|
.with_working_directory_materializer(RuntimeGitMaterializer::new(
|
||||||
|
root.path().join("workdirs"),
|
||||||
|
)),
|
||||||
|
);
|
||||||
|
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
|
||||||
|
crate::fs_store::FsRuntimeStoreOptions {
|
||||||
|
root: store,
|
||||||
|
runtime_id: "restore-failure-test".into(),
|
||||||
|
display_name: None,
|
||||||
|
},
|
||||||
|
backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let mut request = create_request("restore failure recovery");
|
||||||
|
request.profile = ProfileSelector::Builtin("default".into());
|
||||||
|
request.workdir_attachment_requests = vec![WorkingDirectoryAttachmentRequest {
|
||||||
|
alias: WorkdirAttachmentAlias::new("checkout").unwrap(),
|
||||||
|
working_directory: working_directory_request(repo.path()),
|
||||||
|
}];
|
||||||
|
let worker = runtime.create_worker(request).unwrap();
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
Self {
|
||||||
|
root,
|
||||||
|
_repo: repo,
|
||||||
|
runtime,
|
||||||
|
backend,
|
||||||
|
worker,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fail(&self, stage: RuntimeRestoreStage) {
|
||||||
|
*self.backend.factory.restore_faults.fail_at.lock().unwrap() = Some(stage);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_released(&self) {
|
||||||
|
assert!(self.backend.pending_restores.lock().unwrap().is_empty());
|
||||||
|
assert!(self.backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert!(
|
||||||
|
self.backend
|
||||||
|
.factory
|
||||||
|
.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
self.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.workers
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
let table = worker::runtime::worker_allocation::LockFileGuard::open(
|
||||||
|
&worker::runtime::worker_allocation::default_allocation_path().unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
table
|
||||||
|
.data()
|
||||||
|
.find(&ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(
|
||||||
|
&self.worker.worker_ref
|
||||||
|
))
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
let aggregate = self
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.worker_aggregate_dir(&self.worker.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
assert!(fs::read_dir(aggregate.join("runs")).unwrap().all(|entry| {
|
||||||
|
!entry
|
||||||
|
.unwrap()
|
||||||
|
.file_name()
|
||||||
|
.to_string_lossy()
|
||||||
|
.starts_with("restore-")
|
||||||
|
}));
|
||||||
|
assert_eq!(
|
||||||
|
self.runtime
|
||||||
|
.worker_detail(&self.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.status,
|
||||||
|
crate::catalog::WorkerStatus::Stopped
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fresh_restore_and_old_replay(
|
||||||
|
&self,
|
||||||
|
request: runtime_api::WorkerRestoreRequest,
|
||||||
|
failed: crate::runtime::RuntimeWorkerRestoreResult,
|
||||||
|
) {
|
||||||
|
let calls = self
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst);
|
||||||
|
let replay = self
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&self.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&replay).unwrap(),
|
||||||
|
serde_json::to_value(&failed).unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
self.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
calls
|
||||||
|
);
|
||||||
|
let fresh = self.runtime.test_restore_request(&self.worker.worker_ref);
|
||||||
|
assert_ne!(fresh.request_id, request.request_id);
|
||||||
|
assert_ne!(
|
||||||
|
fresh.expected_observation_token,
|
||||||
|
request.expected_observation_token
|
||||||
|
);
|
||||||
|
let restored = self
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&self.worker.worker_ref, fresh)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
restored.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted,
|
||||||
|
"{restored:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
self.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
calls + 1
|
||||||
|
);
|
||||||
|
assert_eq!(self.backend.workers.lock().unwrap().len(), 1);
|
||||||
|
let replay = self
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&self.worker.worker_ref, request)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&replay).unwrap(),
|
||||||
|
serde_json::to_value(&failed).unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
self.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
calls + 1
|
||||||
|
);
|
||||||
|
// Only tear down after the fresh explicit Restore has already succeeded.
|
||||||
|
self.runtime
|
||||||
|
.stop_worker(&self.worker.worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() {
|
||||||
|
for stage in [
|
||||||
|
RuntimeRestoreStage::Reconstruction,
|
||||||
|
RuntimeRestoreStage::Binding,
|
||||||
|
RuntimeRestoreStage::Feature,
|
||||||
|
RuntimeRestoreStage::Controller,
|
||||||
|
] {
|
||||||
|
let fixture = ProfileFailureFixture::new(Duration::from_secs(5));
|
||||||
|
fixture.fail(stage);
|
||||||
|
let request = fixture
|
||||||
|
.runtime
|
||||||
|
.test_restore_request(&fixture.worker.worker_ref);
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::RolledBack,
|
||||||
|
"{stage:?}: {result:?}"
|
||||||
|
);
|
||||||
|
let diagnostic = serde_json::to_string(&result).unwrap();
|
||||||
|
assert!(
|
||||||
|
diagnostic.contains("injected restore boundary failure"),
|
||||||
|
"{diagnostic}"
|
||||||
|
);
|
||||||
|
assert!(diagnostic.contains(&format!("{stage:?}")), "{diagnostic}");
|
||||||
|
assert!(!diagnostic.contains(&fixture.root.path().to_string_lossy().to_string()));
|
||||||
|
assert!(diagnostic.len() < 1024);
|
||||||
|
fixture.assert_released();
|
||||||
|
eprintln!(
|
||||||
|
"restore-fault stage={stage:?} outcome=RolledBack pending=0 controllers=0 allocations=0"
|
||||||
|
);
|
||||||
|
fixture.fresh_restore_and_old_replay(request, result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn profile_restore_cleanup_failure_retains_owner_and_original_cause_until_same_operation_recovers()
|
||||||
|
{
|
||||||
|
let fixture = ProfileFailureFixture::new(Duration::from_secs(5));
|
||||||
|
fixture.fail(RuntimeRestoreStage::Feature);
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.cleanup_fail
|
||||||
|
.store(true, Ordering::SeqCst);
|
||||||
|
let request = fixture
|
||||||
|
.runtime
|
||||||
|
.test_restore_request(&fixture.worker.worker_ref);
|
||||||
|
let failed = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
failed.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let pending = fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let message = pending
|
||||||
|
.result
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.as_ref()
|
||||||
|
.err()
|
||||||
|
.unwrap()
|
||||||
|
.to_string();
|
||||||
|
assert!(
|
||||||
|
message.contains("injected restore boundary failure"),
|
||||||
|
"{message}"
|
||||||
|
);
|
||||||
|
assert!(message.contains("Workdir"), "{message}");
|
||||||
|
let owned = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.get(&fixture.worker.worker_ref)
|
||||||
|
.cloned()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(owned.operation_id, pending.request.operation_id);
|
||||||
|
assert!(owned.allocation.is_some());
|
||||||
|
assert!(
|
||||||
|
owned
|
||||||
|
.sessions
|
||||||
|
.iter()
|
||||||
|
.any(|sessions| !sessions.aliases().is_empty())
|
||||||
|
);
|
||||||
|
let repeat = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
repeat.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
assert!(Arc::ptr_eq(
|
||||||
|
&pending,
|
||||||
|
&fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.cleanup_fail
|
||||||
|
.store(false, Ordering::SeqCst);
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::RolledBack,
|
||||||
|
"{result:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
serde_json::to_string(&result)
|
||||||
|
.unwrap()
|
||||||
|
.contains("injected restore boundary failure")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
owned
|
||||||
|
.sessions
|
||||||
|
.iter()
|
||||||
|
.all(|sessions| sessions.aliases().is_empty())
|
||||||
|
);
|
||||||
|
fixture.assert_released();
|
||||||
|
eprintln!(
|
||||||
|
"restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=RolledBack"
|
||||||
|
);
|
||||||
|
fixture.fresh_restore_and_old_replay(request, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn profile_restore_cleanup_timeout_keeps_running_join_and_late_failure_reconciles_without_execution()
|
||||||
|
{
|
||||||
|
let fixture = ProfileFailureFixture::new(Duration::from_secs(2));
|
||||||
|
fixture.fail(RuntimeRestoreStage::Feature);
|
||||||
|
let release = Arc::new(tokio::sync::Notify::new());
|
||||||
|
let (entered_tx, entered) = mpsc::channel();
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.cleanup_gate
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(release.clone());
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.cleanup_entered
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(entered_tx);
|
||||||
|
let request = fixture
|
||||||
|
.runtime
|
||||||
|
.test_restore_request(&fixture.worker.worker_ref);
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
entered.recv_timeout(Duration::from_secs(5)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let pending = fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
pending
|
||||||
|
.result
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.as_ref()
|
||||||
|
.err()
|
||||||
|
.unwrap()
|
||||||
|
.to_string()
|
||||||
|
.contains("injected restore boundary failure")
|
||||||
|
);
|
||||||
|
assert!(pending.task.try_lock().unwrap().is_some());
|
||||||
|
for _ in 0..2 {
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
assert!(Arc::ptr_eq(
|
||||||
|
&pending,
|
||||||
|
&fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
release.notify_one();
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::RolledBack,
|
||||||
|
"{result:?}"
|
||||||
|
);
|
||||||
|
fixture.assert_released();
|
||||||
|
eprintln!(
|
||||||
|
"restore-fault cleanup=timeout retained-task=true late-completion=joined executions=1"
|
||||||
|
);
|
||||||
|
fixture.fresh_restore_and_old_replay(request, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation() {
|
||||||
|
let fixture = ProfileFailureFixture::new(Duration::from_secs(5));
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.socket_failure
|
||||||
|
.store(true, Ordering::SeqCst);
|
||||||
|
let request = fixture
|
||||||
|
.runtime
|
||||||
|
.test_restore_request(&fixture.worker.worker_ref);
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::RolledBack,
|
||||||
|
"{result:?}"
|
||||||
|
);
|
||||||
|
let diagnostic = serde_json::to_string(&result).unwrap();
|
||||||
|
assert!(
|
||||||
|
diagnostic.contains("controller startup failed"),
|
||||||
|
"{diagnostic}"
|
||||||
|
);
|
||||||
|
assert!(diagnostic.contains("InvalidInput"), "{diagnostic}");
|
||||||
|
assert!(!diagnostic.contains("private-restore-test"));
|
||||||
|
fixture.assert_released();
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.socket_failure
|
||||||
|
.store(false, Ordering::SeqCst);
|
||||||
|
eprintln!(
|
||||||
|
"restore-fault stage=socket-start real-bootstrap=true partial-run=removed allocation=released"
|
||||||
|
);
|
||||||
|
fixture.fresh_restore_and_old_replay(request, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn profile_restore_read_only_refusal_replays_and_fresh_observation_can_restore() {
|
||||||
|
let fixture = ProfileFailureFixture::new(Duration::from_secs(5));
|
||||||
|
let aggregate = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.worker_aggregate_dir(&fixture.worker.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
fs::rename(aggregate.join("session"), aggregate.join("saved-session")).unwrap();
|
||||||
|
let before = persisted_files(&aggregate);
|
||||||
|
let request = fixture
|
||||||
|
.runtime
|
||||||
|
.test_restore_request(&fixture.worker.worker_ref);
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::Rejected,
|
||||||
|
"{result:?}"
|
||||||
|
);
|
||||||
|
let mut after = persisted_files(&aggregate);
|
||||||
|
let mut before = before;
|
||||||
|
// The Runtime rejection receipt is expected to commit; source authority is not.
|
||||||
|
before.remove(Path::new("worker.json"));
|
||||||
|
after.remove(Path::new("worker.json"));
|
||||||
|
assert_eq!(after, before);
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
fs::rename(aggregate.join("saved-session"), aggregate.join("session")).unwrap();
|
||||||
|
// Creation's stopped observation entry is weak; querying it evicts it.
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&fixture.worker.worker_ref)
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
fixture.assert_released();
|
||||||
|
fixture.fresh_restore_and_old_replay(request, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn profile_restore_real_artifact_cleanup_failure_retains_same_owner_and_cause() {
|
||||||
|
let fixture = ProfileFailureFixture::new(Duration::from_secs(2));
|
||||||
|
fixture.fail(RuntimeRestoreStage::Binding);
|
||||||
|
let release = Arc::new(tokio::sync::Notify::new());
|
||||||
|
let (entered_tx, entered) = mpsc::channel();
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.factory_gate
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(release.clone());
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.factory_entered
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(entered_tx);
|
||||||
|
let request = fixture
|
||||||
|
.runtime
|
||||||
|
.test_restore_request(&fixture.worker.worker_ref);
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
entered.recv_timeout(Duration::from_secs(5)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let pending = fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let run = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.worker_restore_run_dir(&fixture.worker.worker_ref, pending.request.operation_id)
|
||||||
|
.unwrap();
|
||||||
|
fs::write(&run, "invalid disposable run artifact").unwrap();
|
||||||
|
release.notify_one();
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"{result:?}"
|
||||||
|
);
|
||||||
|
let cause = pending
|
||||||
|
.result
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.as_ref()
|
||||||
|
.err()
|
||||||
|
.unwrap()
|
||||||
|
.clone();
|
||||||
|
assert_eq!(cause.stage, RuntimeRestoreStage::Binding);
|
||||||
|
assert_eq!(
|
||||||
|
cause.cleanup_failure,
|
||||||
|
Some(RuntimeRestoreCleanupFailure::Artifacts)
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.contains_key(&fixture.worker.worker_ref)
|
||||||
|
);
|
||||||
|
assert!(fixture.backend.workers.lock().unwrap().is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
fs::remove_file(&run).unwrap();
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::RolledBack,
|
||||||
|
"{result:?}"
|
||||||
|
);
|
||||||
|
assert!(result.message.as_ref().unwrap().contains("Artifacts"));
|
||||||
|
fixture.assert_released();
|
||||||
|
eprintln!(
|
||||||
|
"restore-fault cleanup=real-artifact-failure late-factory=true owner=retained original-cause=preserved"
|
||||||
|
);
|
||||||
|
fixture.fresh_restore_and_old_replay(request, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn profile_restore_stop_race_joins_late_factory_failure_without_second_controller() {
|
||||||
|
let fixture = ProfileFailureFixture::new(Duration::from_secs(2));
|
||||||
|
fixture.fail(RuntimeRestoreStage::Binding);
|
||||||
|
let release = Arc::new(tokio::sync::Notify::new());
|
||||||
|
let (entered_tx, entered) = mpsc::channel();
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.factory_gate
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(release.clone());
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.factory_entered
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(entered_tx);
|
||||||
|
let request = fixture
|
||||||
|
.runtime
|
||||||
|
.test_restore_request(&fixture.worker.worker_ref);
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request)
|
||||||
|
.unwrap();
|
||||||
|
entered.recv_timeout(Duration::from_secs(5)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let pending = fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(pending.result.lock().unwrap().is_none());
|
||||||
|
let runtime = fixture.runtime.clone();
|
||||||
|
let worker_ref = fixture.worker.worker_ref.clone();
|
||||||
|
let (started_tx, started) = mpsc::channel();
|
||||||
|
let stopping = std::thread::spawn(move || {
|
||||||
|
started_tx.send(()).unwrap();
|
||||||
|
runtime.stop_worker(&worker_ref, None)
|
||||||
|
});
|
||||||
|
started.recv_timeout(Duration::from_secs(5)).unwrap();
|
||||||
|
// The factory still owns the allocation and cannot attest stopped yet.
|
||||||
|
assert!(
|
||||||
|
!pending
|
||||||
|
.task
|
||||||
|
.try_lock()
|
||||||
|
.map(|task| task.as_ref().is_some_and(|task| task.is_finished()))
|
||||||
|
.unwrap_or(false)
|
||||||
|
);
|
||||||
|
release.notify_one();
|
||||||
|
stopping.join().unwrap().unwrap();
|
||||||
|
fixture.assert_released();
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
let fresh = fixture
|
||||||
|
.runtime
|
||||||
|
.test_restore_request(&fixture.worker.worker_ref);
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, fresh)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted,
|
||||||
|
"{result:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
|
||||||
|
fixture
|
||||||
|
.runtime
|
||||||
|
.stop_worker(&fixture.worker.worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
eprintln!("restore-fault race=Stop late-factory=joined controllers-at-most=1");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn profile_restore_failure_settlement_write_retry_preserves_completed_cause_without_execution() {
|
||||||
|
for fault in [
|
||||||
|
crate::fs_store::AtomicWriteFault::BeforeRename,
|
||||||
|
crate::fs_store::AtomicWriteFault::AfterRename,
|
||||||
|
] {
|
||||||
|
let fixture = ProfileFailureFixture::new(Duration::from_secs(2));
|
||||||
|
fixture.fail(RuntimeRestoreStage::Binding);
|
||||||
|
let release = Arc::new(tokio::sync::Notify::new());
|
||||||
|
let (entered_tx, entered) = mpsc::channel();
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.factory_gate
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(release.clone());
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.factory_entered
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(entered_tx);
|
||||||
|
let request = fixture
|
||||||
|
.runtime
|
||||||
|
.test_restore_request(&fixture.worker.worker_ref);
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
entered.recv_timeout(Duration::from_secs(5)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let operation_id = fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.request
|
||||||
|
.operation_id;
|
||||||
|
fixture
|
||||||
|
.runtime
|
||||||
|
.test_runtime_store()
|
||||||
|
.fail_next_worker_write(fault);
|
||||||
|
release.notify_one();
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"{result:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
result.reason_code.as_deref(),
|
||||||
|
Some("worker_restore_rollback_commit_failed")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&fixture.worker.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
let saved = fixture
|
||||||
|
.backend
|
||||||
|
.completed_restore_failure(operation_id, &fixture.worker.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
matches!(saved, WorkerExecutionSpawnResult::RolledBack(ref failure)
|
||||||
|
if failure.message.as_ref().unwrap().contains("injected restore boundary failure"))
|
||||||
|
);
|
||||||
|
let result = fixture
|
||||||
|
.runtime
|
||||||
|
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
result.state,
|
||||||
|
server_api::WorkerRestoreState::RolledBack,
|
||||||
|
"{result:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
result
|
||||||
|
.message
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.contains("injected restore boundary failure")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.restore_faults
|
||||||
|
.executions
|
||||||
|
.load(Ordering::SeqCst),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
fixture.assert_released();
|
||||||
|
eprintln!(
|
||||||
|
"restore-fault settlement={fault:?} completed-cause=retained replay-executions=1"
|
||||||
|
);
|
||||||
|
fixture.fresh_restore_and_old_replay(request, result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct RacedBindingFactory {
|
||||||
|
changed_checkout: PathBuf,
|
||||||
|
preflights: AtomicUsize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl RuntimeWorkerFactory for RacedBindingFactory {
|
||||||
|
async fn spawn_controller(
|
||||||
|
&self,
|
||||||
|
_: WorkerExecutionSpawnRequest,
|
||||||
|
) -> Result<RuntimeWorkerController, String> {
|
||||||
|
panic!("binding-only failure must not create a controller")
|
||||||
|
}
|
||||||
|
async fn preflight_restore(&self, _: &WorkerExecutionRestoreRequest) -> Result<(), String> {
|
||||||
|
// Inject another owner's checkout change AFTER the real materializer's
|
||||||
|
// read-only checks, but before its mutable bind. Recovery stays real.
|
||||||
|
if self.preflights.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||||
|
fs::remove_dir_all(&self.changed_checkout).unwrap();
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
async fn restore_controller(
|
||||||
|
&self,
|
||||||
|
_: WorkerExecutionRestoreRequest,
|
||||||
|
) -> Result<RuntimeWorkerController, RuntimeRestoreFailure> {
|
||||||
|
panic!("factory execution must not run after partial binding failed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn partial_restore_binding_failure_keeps_exact_operation_without_factory_cleanup_claim() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let repo = create_clean_repo();
|
||||||
|
let materializer = RuntimeGitMaterializer::new(root.path());
|
||||||
|
let first = materializer
|
||||||
|
.create(&working_directory_request(repo.path()))
|
||||||
|
.unwrap();
|
||||||
|
let second = materializer
|
||||||
|
.create(&working_directory_request(repo.path()))
|
||||||
|
.unwrap();
|
||||||
|
let mut request = direct_restore_request(WorkerRef::new(WorkerId::now_v7()));
|
||||||
|
request.previous_workdir_attachments = vec![
|
||||||
|
WorkingDirectoryAttachmentStatus {
|
||||||
|
alias: WorkdirAttachmentAlias::new("first").unwrap(),
|
||||||
|
working_directory: first.status(),
|
||||||
|
},
|
||||||
|
WorkingDirectoryAttachmentStatus {
|
||||||
|
alias: WorkdirAttachmentAlias::new("second").unwrap(),
|
||||||
|
working_directory: second.status(),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
let backend = WorkerRuntimeExecutionBackend::new(RacedBindingFactory {
|
||||||
|
changed_checkout: second.root().to_path_buf(),
|
||||||
|
preflights: AtomicUsize::new(0),
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
.with_working_directory_materializer(materializer);
|
||||||
|
assert!(matches!(
|
||||||
|
backend.restore_worker(request.clone()),
|
||||||
|
WorkerExecutionSpawnResult::ReconciliationRequired { .. }
|
||||||
|
));
|
||||||
|
let pending = backend
|
||||||
|
.pending_restore(&request.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(pending.request.operation_id, request.operation_id);
|
||||||
|
assert_eq!(pending.request.workdir_attachments.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
pending
|
||||||
|
.request
|
||||||
|
.workdir_attachments
|
||||||
|
.values()
|
||||||
|
.next()
|
||||||
|
.unwrap()
|
||||||
|
.root(),
|
||||||
|
first.root()
|
||||||
|
);
|
||||||
|
for _ in 0..2 {
|
||||||
|
assert!(matches!(
|
||||||
|
backend.reconcile_restore(request.clone()),
|
||||||
|
WorkerExecutionSpawnResult::ReconciliationRequired { .. }
|
||||||
|
));
|
||||||
|
}
|
||||||
|
assert_eq!(backend.factory.preflights.load(Ordering::SeqCst), 1);
|
||||||
|
assert!(!backend.stop_worker(&request.worker_ref).is_accepted());
|
||||||
|
assert!(Arc::ptr_eq(
|
||||||
|
&pending,
|
||||||
|
&backend
|
||||||
|
.pending_restore(&request.worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
));
|
||||||
|
assert!(backend.workers.lock().unwrap().is_empty());
|
||||||
|
eprintln!(
|
||||||
|
"restore-fault stage=mutable-binding outcome=ReconciliationRequired partial-bindings=retained factory-executions=0"
|
||||||
|
);
|
||||||
|
// Fixture teardown drops the retained passive local binding. This is not
|
||||||
|
// evidence that credential/policy materialization has been rolled back.
|
||||||
|
}
|
||||||
@@ -271,6 +271,7 @@ where
|
|||||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
let cleanup_sessions = worker.workdir_sessions();
|
let cleanup_sessions = worker.workdir_sessions();
|
||||||
|
let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some();
|
||||||
let controller = match layout {
|
let controller = match layout {
|
||||||
WorkerBootstrapLayout::Direct {
|
WorkerBootstrapLayout::Direct {
|
||||||
runtime_base,
|
runtime_base,
|
||||||
@@ -305,7 +306,10 @@ where
|
|||||||
controller_task,
|
controller_task,
|
||||||
}),
|
}),
|
||||||
Err(source) => {
|
Err(source) => {
|
||||||
let cleanup_failed = cleanup_sessions.close_all().await.is_err();
|
// Factory-owned task scopes must settle before sessions close.
|
||||||
|
// Cleanup remains unconfirmed here; preserve the startup source.
|
||||||
|
let cleanup_failed =
|
||||||
|
factory_owned_cleanup || cleanup_sessions.close_all().await.is_err();
|
||||||
Err(WorkerBootstrapError::Controller {
|
Err(WorkerBootstrapError::Controller {
|
||||||
source,
|
source,
|
||||||
cleanup_failed,
|
cleanup_failed,
|
||||||
|
|||||||
@@ -854,6 +854,7 @@ impl WorkerController {
|
|||||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
let sessions = worker.workdir_sessions();
|
let sessions = worker.workdir_sessions();
|
||||||
|
let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some();
|
||||||
let result = Self::spawn_initialized(
|
let result = Self::spawn_initialized(
|
||||||
worker,
|
worker,
|
||||||
runtime_base,
|
runtime_base,
|
||||||
@@ -863,7 +864,10 @@ impl WorkerController {
|
|||||||
transport,
|
transport,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
// An opted-in factory must quiesce its captured Feature tasks before
|
||||||
|
// closing their Workdir sessions; this wrapper cannot confirm that order.
|
||||||
if result.is_err()
|
if result.is_err()
|
||||||
|
&& !factory_owned_cleanup
|
||||||
&& let Err(error) = sessions.close_all().await
|
&& let Err(error) = sessions.close_all().await
|
||||||
{
|
{
|
||||||
tracing::warn!(%error, "Workdir sessions close after controller startup failure failed");
|
tracing::warn!(%error, "Workdir sessions close after controller startup failure failed");
|
||||||
@@ -1092,6 +1096,7 @@ impl WorkerController {
|
|||||||
let pause_tx = worker.engine_mut().pause_sender();
|
let pause_tx = worker.engine_mut().pause_sender();
|
||||||
let notify_buffer = worker.notify_buffer_handle();
|
let notify_buffer = worker.notify_buffer_handle();
|
||||||
|
|
||||||
|
worker.finish_startup_feature_cleanup_capture();
|
||||||
let controller_task = tokio::spawn(controller_loop(
|
let controller_task = tokio::spawn(controller_loop(
|
||||||
worker,
|
worker,
|
||||||
method_rx,
|
method_rx,
|
||||||
@@ -4424,21 +4429,6 @@ mod tests {
|
|||||||
assert!(feature_install < materialize_gate);
|
assert!(feature_install < materialize_gate);
|
||||||
assert!(materialize_gate < materialize);
|
assert!(materialize_gate < materialize);
|
||||||
assert!(materialize < exposure);
|
assert!(materialize < exposure);
|
||||||
let observer = startup.find("let command_observer = fs_for_view").unwrap();
|
|
||||||
let recover = startup.find(".recover_unfinished_compaction()").unwrap();
|
|
||||||
let files = startup
|
|
||||||
.find("runtime_dir.write_status(&shared_state).await?")
|
|
||||||
.unwrap();
|
|
||||||
let loop_start = startup.find("tokio::spawn(controller_loop(").unwrap();
|
|
||||||
assert!(recover < observer && files < observer && observer < loop_start);
|
|
||||||
let socket_startup = &startup[observer..loop_start];
|
|
||||||
let abort = socket_startup.find("observer.abort()").unwrap();
|
|
||||||
let join = socket_startup.find("observer.await").unwrap();
|
|
||||||
let failure = socket_startup.find("return Err(error)").unwrap();
|
|
||||||
assert!(
|
|
||||||
abort < join && join < failure,
|
|
||||||
"startup cannot detach its observer on error"
|
|
||||||
);
|
|
||||||
assert!(
|
assert!(
|
||||||
startup[materialize_gate..materialize]
|
startup[materialize_gate..materialize]
|
||||||
.contains("needs_initial_session_head_materialization")
|
.contains("needs_initial_session_head_materialization")
|
||||||
|
|||||||
@@ -69,9 +69,10 @@ pub use session_history::{
|
|||||||
pub use shared_state::WorkerSharedState;
|
pub use shared_state::WorkerSharedState;
|
||||||
pub use worker::{
|
pub use worker::{
|
||||||
LocalWorkingDirectory, SubjektivSessionAttributionLifecycle, Worker, WorkerError,
|
LocalWorkingDirectory, SubjektivSessionAttributionLifecycle, Worker, WorkerError,
|
||||||
WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, WorkspaceBinaryRequest,
|
WorkerFilesystemAuthority, WorkerRunResult, WorkerStartupFeatureCleanup,
|
||||||
WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
|
WorkerWorkspaceContext, WorkspaceBinaryRequest, WorkspaceBinaryResponse, WorkspaceClient,
|
||||||
WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse,
|
WorkspaceClientError, WorkspaceId, WorkspaceIdError, WorkspacePromptCatalogResolution,
|
||||||
WorkspaceServerOperation, WorkspaceWorkerDiscoveryRequest, apply_worker_manifest,
|
WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse, WorkspaceServerOperation,
|
||||||
marker_workspace_client, unavailable_workspace_client,
|
WorkspaceWorkerDiscoveryRequest, apply_worker_manifest, marker_workspace_client,
|
||||||
|
unavailable_workspace_client,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -21,8 +21,9 @@ pub use error::ScopeLockError;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use lifecycle::install_top_level_at_for_test;
|
pub(crate) use lifecycle::install_top_level_at_for_test;
|
||||||
pub use lifecycle::{
|
pub use lifecycle::{
|
||||||
ScopeAllocationGuard, SegmentActivationCommitError, SegmentActivationGuard, SegmentLockInfo,
|
ScopeAllocationCleanup, ScopeAllocationGuard, SegmentActivationCommitError,
|
||||||
adopt_allocation, install_top_level, install_top_level_with_deny, lookup_segment,
|
SegmentActivationGuard, SegmentLockInfo, adopt_allocation, install_top_level,
|
||||||
|
install_top_level_with_deny, lookup_segment,
|
||||||
};
|
};
|
||||||
pub use mutate::{
|
pub use mutate::{
|
||||||
delegate_scope, reclaim_delegated_scope, reclaim_stale, reclaim_stale_with, register_worker,
|
delegate_scope, reclaim_delegated_scope, reclaim_stale, reclaim_stale_with, register_worker,
|
||||||
|
|||||||
@@ -11,6 +11,14 @@ use session_store::SegmentId;
|
|||||||
pub enum ScopeLockError {
|
pub enum ScopeLockError {
|
||||||
#[error("I/O error on workers.json: {0}")]
|
#[error("I/O error on workers.json: {0}")]
|
||||||
Io(#[from] io::Error),
|
Io(#[from] io::Error),
|
||||||
|
#[error("worker `{worker_name}` allocation still has a live guard")]
|
||||||
|
AllocationGuardAttached { worker_name: String },
|
||||||
|
#[error("allocation acquisition was published but its save failed: {source}")]
|
||||||
|
AcquisitionSaveFailed {
|
||||||
|
#[source]
|
||||||
|
source: io::Error,
|
||||||
|
cleanup: super::lifecycle::ScopeAllocationCleanup,
|
||||||
|
},
|
||||||
#[error("worker `{0}` is already allocated")]
|
#[error("worker `{0}` is already allocated")]
|
||||||
DuplicateWorkerName(String),
|
DuplicateWorkerName(String),
|
||||||
#[error("requested scope `{}` conflicts with worker allocation `{competitor}` rule `{}`", .rule.target.display(), .competitor_rule.target.display())]
|
#[error("requested scope `{}` conflicts with worker allocation `{competitor}` rule `{}`", .rule.target.display(), .competitor_rule.target.display())]
|
||||||
@@ -46,3 +54,15 @@ pub enum ScopeLockError {
|
|||||||
actual: Option<SegmentId>,
|
actual: Option<SegmentId>,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl ScopeLockError {
|
||||||
|
/// Retain the release authority for an owned acquisition whose table rename
|
||||||
|
/// succeeded before save returned an error. Refusals and unpublished saves
|
||||||
|
/// do not confer ownership of any preexisting allocation.
|
||||||
|
pub fn allocation_cleanup_handle(&self) -> Option<super::lifecycle::ScopeAllocationCleanup> {
|
||||||
|
match self {
|
||||||
|
Self::AcquisitionSaveFailed { cleanup, .. } => Some(cleanup.clone()),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
//! up on drop.
|
//! up on drop.
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
use manifest::ScopeRule;
|
use manifest::ScopeRule;
|
||||||
use session_store::SegmentId;
|
use session_store::SegmentId;
|
||||||
@@ -11,13 +12,82 @@ use super::error::ScopeLockError;
|
|||||||
use super::mutate::release_worker;
|
use super::mutate::release_worker;
|
||||||
use super::table::{LockFileGuard, default_allocation_path};
|
use super::table::{LockFileGuard, default_allocation_path};
|
||||||
|
|
||||||
/// Owned allocation: on drop, opens the lock file and releases this
|
/// Owned allocation shared with any retained cleanup handles. Only the final
|
||||||
/// Worker's entry. The guard keeps only the name + lock-file path; it
|
/// owner's drop attempts best-effort release; no `flock` is held for its lifetime.
|
||||||
/// does not hold the `flock` for the Worker's lifetime.
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct ScopeAllocationGuard {
|
pub struct ScopeAllocationGuard {
|
||||||
|
owner: Arc<ScopeAllocationOwner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retains an allocation's release authority even after its Worker is consumed
|
||||||
|
/// or dropped. A successful controller transfer should drop this handle without
|
||||||
|
/// explicitly releasing the live Worker's allocation.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ScopeAllocationCleanup {
|
||||||
|
owner: Arc<ScopeAllocationOwner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct ScopeAllocationOwner {
|
||||||
worker_name: String,
|
worker_name: String,
|
||||||
lock_path: PathBuf,
|
lock_path: PathBuf,
|
||||||
|
state: Mutex<ScopeAllocationOwnerState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct ScopeAllocationOwnerState {
|
||||||
|
guard_attached: bool,
|
||||||
|
released: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScopeAllocationOwner {
|
||||||
|
fn release(&self) -> Result<(), ScopeLockError> {
|
||||||
|
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||||
|
if state.guard_attached {
|
||||||
|
return Err(ScopeLockError::AllocationGuardAttached {
|
||||||
|
worker_name: self.worker_name.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if state.released {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let mut guard = LockFileGuard::open(&self.lock_path)?;
|
||||||
|
if guard.data().find(&self.worker_name).is_some() {
|
||||||
|
release_worker(&mut guard, &self.worker_name)?;
|
||||||
|
} else {
|
||||||
|
// A previous save may have removed the entry but failed after rename.
|
||||||
|
// Complete the durable save before reporting release as confirmed.
|
||||||
|
guard.save()?;
|
||||||
|
}
|
||||||
|
state.released = true;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for ScopeAllocationOwner {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = self.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for ScopeAllocationGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.owner
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner())
|
||||||
|
.guard_attached = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScopeAllocationCleanup {
|
||||||
|
/// Release only after the factory has returned an error and no live
|
||||||
|
/// Controller can still use this allocation. An attached guard rejects
|
||||||
|
/// release. Success is idempotent across all clones; failure retains the
|
||||||
|
/// authority for an explicit retry.
|
||||||
|
pub fn release(&self) -> Result<(), ScopeLockError> {
|
||||||
|
self.owner.release()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Exclusive authority to activate a replacement Segment for one allocated Worker.
|
/// Exclusive authority to activate a replacement Segment for one allocated Worker.
|
||||||
@@ -56,12 +126,58 @@ impl SegmentActivationCommitError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ScopeAllocationGuard {
|
impl ScopeAllocationGuard {
|
||||||
|
fn new(worker_name: String, lock_path: PathBuf) -> Self {
|
||||||
|
Self {
|
||||||
|
owner: Arc::new(ScopeAllocationOwner {
|
||||||
|
worker_name,
|
||||||
|
lock_path,
|
||||||
|
state: Mutex::new(ScopeAllocationOwnerState {
|
||||||
|
guard_attached: true,
|
||||||
|
released: false,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn commit_acquisition(self, table: &mut LockFileGuard) -> Result<Self, ScopeLockError> {
|
||||||
|
let mut published = false;
|
||||||
|
match table.save_with_publication(&mut published) {
|
||||||
|
Ok(()) => Ok(self),
|
||||||
|
Err(source) => {
|
||||||
|
let cleanup = if published {
|
||||||
|
Some(self.cleanup_handle())
|
||||||
|
} else {
|
||||||
|
// A staged entry is not ours until rename. Disarm before
|
||||||
|
// detaching so Drop cannot remove a preexisting allocation
|
||||||
|
// (or reopen the table lock still held by this caller).
|
||||||
|
self.owner
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner())
|
||||||
|
.released = true;
|
||||||
|
None
|
||||||
|
};
|
||||||
|
drop(self);
|
||||||
|
match cleanup {
|
||||||
|
Some(cleanup) => Err(ScopeLockError::AcquisitionSaveFailed { source, cleanup }),
|
||||||
|
None => Err(ScopeLockError::Io(source)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn worker_name(&self) -> &str {
|
pub fn worker_name(&self) -> &str {
|
||||||
&self.worker_name
|
&self.owner.worker_name
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn lock_path(&self) -> &Path {
|
pub fn lock_path(&self) -> &Path {
|
||||||
&self.lock_path
|
&self.owner.lock_path
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn cleanup_handle(&self) -> ScopeAllocationCleanup {
|
||||||
|
ScopeAllocationCleanup {
|
||||||
|
owner: Arc::clone(&self.owner),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Lock the machine-wide table and validate a Segment replacement before
|
/// Lock the machine-wide table and validate a Segment replacement before
|
||||||
@@ -71,24 +187,25 @@ impl ScopeAllocationGuard {
|
|||||||
expected: SegmentId,
|
expected: SegmentId,
|
||||||
replacement: SegmentId,
|
replacement: SegmentId,
|
||||||
) -> Result<SegmentActivationGuard, ScopeLockError> {
|
) -> Result<SegmentActivationGuard, ScopeLockError> {
|
||||||
let guard = LockFileGuard::open(&self.lock_path)?;
|
let guard = LockFileGuard::open(self.lock_path())?;
|
||||||
let actual = guard
|
let actual = guard
|
||||||
.data()
|
.data()
|
||||||
.allocations
|
.allocations
|
||||||
.iter()
|
.iter()
|
||||||
.find(|allocation| allocation.worker_name == self.worker_name)
|
.find(|allocation| allocation.worker_name == self.worker_name())
|
||||||
.and_then(|allocation| allocation.segment_id);
|
.and_then(|allocation| allocation.segment_id);
|
||||||
|
|
||||||
if actual != Some(expected) {
|
if actual != Some(expected) {
|
||||||
return Err(ScopeLockError::SegmentChanged {
|
return Err(ScopeLockError::SegmentChanged {
|
||||||
worker_name: self.worker_name.clone(),
|
worker_name: self.worker_name().to_owned(),
|
||||||
expected,
|
expected,
|
||||||
actual,
|
actual,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(existing) = guard.data().allocations.iter().find(|allocation| {
|
if let Some(existing) = guard.data().allocations.iter().find(|allocation| {
|
||||||
allocation.segment_id == Some(replacement) && allocation.worker_name != self.worker_name
|
allocation.segment_id == Some(replacement)
|
||||||
|
&& allocation.worker_name != self.worker_name()
|
||||||
}) {
|
}) {
|
||||||
return Err(ScopeLockError::SegmentConflict {
|
return Err(ScopeLockError::SegmentConflict {
|
||||||
segment_id: replacement,
|
segment_id: replacement,
|
||||||
@@ -99,7 +216,7 @@ impl ScopeAllocationGuard {
|
|||||||
|
|
||||||
Ok(SegmentActivationGuard {
|
Ok(SegmentActivationGuard {
|
||||||
guard,
|
guard,
|
||||||
worker_name: self.worker_name.clone(),
|
worker_name: self.worker_name().to_owned(),
|
||||||
expected,
|
expected,
|
||||||
replacement,
|
replacement,
|
||||||
})
|
})
|
||||||
@@ -145,14 +262,6 @@ impl SegmentActivationGuard {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for ScopeAllocationGuard {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
if let Ok(mut guard) = LockFileGuard::open(&self.lock_path) {
|
|
||||||
let _ = release_worker(&mut guard, &self.worker_name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Open the default lock file, register a top-level Worker, and return a
|
/// Open the default lock file, register a top-level Worker, and return a
|
||||||
/// guard that will release the allocation on drop.
|
/// guard that will release the allocation on drop.
|
||||||
pub fn install_top_level(
|
pub fn install_top_level(
|
||||||
@@ -185,7 +294,7 @@ pub fn install_top_level_with_deny(
|
|||||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||||
let lock_path = default_allocation_path()?;
|
let lock_path = default_allocation_path()?;
|
||||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||||
super::mutate::register_worker_with_deny(
|
super::mutate::prepare_worker_with_deny(
|
||||||
&mut guard,
|
&mut guard,
|
||||||
worker_name.clone(),
|
worker_name.clone(),
|
||||||
pid,
|
pid,
|
||||||
@@ -194,10 +303,7 @@ pub fn install_top_level_with_deny(
|
|||||||
scope_deny,
|
scope_deny,
|
||||||
segment_id,
|
segment_id,
|
||||||
)?;
|
)?;
|
||||||
Ok(ScopeAllocationGuard {
|
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||||
worker_name,
|
|
||||||
lock_path,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -210,7 +316,7 @@ pub(crate) fn install_top_level_at_for_test(
|
|||||||
segment_id: SegmentId,
|
segment_id: SegmentId,
|
||||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||||
super::mutate::register_worker_with_deny(
|
super::mutate::prepare_worker_with_deny(
|
||||||
&mut guard,
|
&mut guard,
|
||||||
worker_name.clone(),
|
worker_name.clone(),
|
||||||
pid,
|
pid,
|
||||||
@@ -219,10 +325,7 @@ pub(crate) fn install_top_level_at_for_test(
|
|||||||
Vec::new(),
|
Vec::new(),
|
||||||
segment_id,
|
segment_id,
|
||||||
)?;
|
)?;
|
||||||
Ok(ScopeAllocationGuard {
|
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||||
worker_name,
|
|
||||||
lock_path,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Take ownership of an existing allocation that was pre-registered by
|
/// Take ownership of an existing allocation that was pre-registered by
|
||||||
@@ -240,6 +343,15 @@ pub fn adopt_allocation(
|
|||||||
segment_id: SegmentId,
|
segment_id: SegmentId,
|
||||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||||
let lock_path = default_allocation_path()?;
|
let lock_path = default_allocation_path()?;
|
||||||
|
adopt_allocation_at(lock_path, worker_name, new_pid, segment_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn adopt_allocation_at(
|
||||||
|
lock_path: PathBuf,
|
||||||
|
worker_name: String,
|
||||||
|
new_pid: u32,
|
||||||
|
segment_id: SegmentId,
|
||||||
|
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||||
let alloc = guard
|
let alloc = guard
|
||||||
.data_mut()
|
.data_mut()
|
||||||
@@ -247,11 +359,7 @@ pub fn adopt_allocation(
|
|||||||
.ok_or_else(|| ScopeLockError::UnknownWorker(worker_name.clone()))?;
|
.ok_or_else(|| ScopeLockError::UnknownWorker(worker_name.clone()))?;
|
||||||
alloc.pid = new_pid;
|
alloc.pid = new_pid;
|
||||||
alloc.segment_id = Some(segment_id);
|
alloc.segment_id = Some(segment_id);
|
||||||
guard.save()?;
|
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||||
Ok(ScopeAllocationGuard {
|
|
||||||
worker_name,
|
|
||||||
lock_path,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Information about a Worker that currently holds an allocation for a
|
/// Information about a Worker that currently holds an allocation for a
|
||||||
@@ -329,6 +437,388 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn retained_cleanup_handle_defers_release_until_final_owner_drop() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let guard = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"retained".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("retained"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let cleanup = guard.cleanup_handle();
|
||||||
|
let retained = cleanup.clone();
|
||||||
|
|
||||||
|
drop(guard);
|
||||||
|
drop(cleanup);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("retained").is_some());
|
||||||
|
}
|
||||||
|
drop(retained);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("retained").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dropping_cleanup_handle_after_transfer_preserves_live_guard() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let segment = sid();
|
||||||
|
let guard = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"transferred".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("transferred"),
|
||||||
|
Vec::new(),
|
||||||
|
segment,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
drop(guard.cleanup_handle());
|
||||||
|
// A successful controller transfer leaves the guard usable, including
|
||||||
|
// the existing activation protocol, without retaining a cleanup handle.
|
||||||
|
let replacement = sid();
|
||||||
|
guard
|
||||||
|
.begin_segment_activation(segment, replacement)
|
||||||
|
.unwrap()
|
||||||
|
.commit()
|
||||||
|
.unwrap();
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("transferred").unwrap().segment_id,
|
||||||
|
Some(replacement)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
drop(guard);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("transferred").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cleanup_rejects_live_guard_then_releases_only_detached_allocation() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let guard = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"live".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("live"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let _competitor = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"competitor".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("competitor"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let cleanup = guard.cleanup_handle();
|
||||||
|
assert!(matches!(
|
||||||
|
cleanup.release(),
|
||||||
|
Err(ScopeLockError::AllocationGuardAttached { worker_name }) if worker_name == "live"
|
||||||
|
));
|
||||||
|
drop(cleanup.clone());
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("live").is_some());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
drop(guard);
|
||||||
|
cleanup.release().unwrap();
|
||||||
|
cleanup.clone().release().unwrap();
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("live").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn published_install_failure_retains_cleanup_across_error_drop_and_retry() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let _competitor = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"competitor".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("competitor"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let segment = sid();
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||||
|
let error = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"failed".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("failed"),
|
||||||
|
Vec::new(),
|
||||||
|
segment,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
&error,
|
||||||
|
ScopeLockError::AcquisitionSaveFailed { .. }
|
||||||
|
));
|
||||||
|
let cleanup = error.allocation_cleanup_handle().unwrap();
|
||||||
|
let retry = error.allocation_cleanup_handle().unwrap();
|
||||||
|
drop(error);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("failed").unwrap().segment_id,
|
||||||
|
Some(segment)
|
||||||
|
);
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||||
|
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||||
|
drop(cleanup);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("failed").is_some());
|
||||||
|
}
|
||||||
|
retry.release().unwrap();
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("failed").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn refused_install_has_no_cleanup_authority_over_existing_allocation() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let segment = sid();
|
||||||
|
let _competitor = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"competitor".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("competitor"),
|
||||||
|
vec![write_rule("/shared", true)],
|
||||||
|
segment,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let cases = [
|
||||||
|
("competitor", sid(), Vec::new()),
|
||||||
|
("other", segment, Vec::new()),
|
||||||
|
("other", sid(), vec![write_rule("/shared", true)]),
|
||||||
|
];
|
||||||
|
for (name, requested_segment, scope) in cases {
|
||||||
|
let error = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
name.into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("refused"),
|
||||||
|
scope,
|
||||||
|
requested_segment,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
&error,
|
||||||
|
ScopeLockError::DuplicateWorkerName(_)
|
||||||
|
| ScopeLockError::SegmentConflict { .. }
|
||||||
|
| ScopeLockError::WriteConflict { .. }
|
||||||
|
));
|
||||||
|
assert!(error.allocation_cleanup_handle().is_none());
|
||||||
|
drop(error);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(table.data().allocations.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("competitor").unwrap().socket,
|
||||||
|
sock("competitor")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("competitor").unwrap().segment_id,
|
||||||
|
Some(segment)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unpublished_install_failure_has_no_cleanup_authority() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let _competitor = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"competitor".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("competitor"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||||
|
let error = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"failed".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("failed"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(&error, ScopeLockError::Io(_)));
|
||||||
|
assert!(error.allocation_cleanup_handle().is_none());
|
||||||
|
drop(error);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("failed").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn published_adoption_failure_retains_cleanup_for_retry() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
{
|
||||||
|
let mut table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
delegate_placeholder(&mut table, "child", std::process::id());
|
||||||
|
delegate_placeholder(&mut table, "competitor", std::process::id());
|
||||||
|
}
|
||||||
|
let segment = sid();
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||||
|
let error = adopt_allocation_at(
|
||||||
|
lock_path.clone(),
|
||||||
|
"child".into(),
|
||||||
|
std::process::id(),
|
||||||
|
segment,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
let cleanup = error.allocation_cleanup_handle().unwrap();
|
||||||
|
drop(error);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("child").unwrap().segment_id,
|
||||||
|
Some(segment)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||||
|
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("child").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
cleanup.release().unwrap();
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("child").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unpublished_adoption_failure_preserves_placeholder_without_cleanup_authority() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let placeholder_pid = std::process::id();
|
||||||
|
{
|
||||||
|
let mut table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
delegate_placeholder(&mut table, "child", placeholder_pid);
|
||||||
|
}
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||||
|
let error = adopt_allocation_at(
|
||||||
|
lock_path.clone(),
|
||||||
|
"child".into(),
|
||||||
|
placeholder_pid.wrapping_add(1),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(&error, ScopeLockError::Io(_)));
|
||||||
|
assert!(error.allocation_cleanup_handle().is_none());
|
||||||
|
drop(error);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
let allocation = table.data().find("child").unwrap();
|
||||||
|
assert_eq!(allocation.pid, placeholder_pid);
|
||||||
|
assert_eq!(allocation.segment_id, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn failed_cleanup_release_retains_authority_for_retry() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
for point in [SaveFaultPoint::BeforeRename, SaveFaultPoint::AfterRename] {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let guard = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"failed".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("failed"),
|
||||||
|
vec![read_rule("/shared", true)],
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let _other = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"other".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("other"),
|
||||||
|
vec![read_rule("/shared", true)],
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let cleanup = guard.cleanup_handle();
|
||||||
|
let retry = cleanup.clone();
|
||||||
|
drop(guard);
|
||||||
|
fail_next_save_at_for_test(&lock_path, point);
|
||||||
|
|
||||||
|
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||||
|
drop(cleanup);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("failed").is_some(),
|
||||||
|
point == SaveFaultPoint::BeforeRename
|
||||||
|
);
|
||||||
|
assert!(table.data().find("other").is_some());
|
||||||
|
}
|
||||||
|
retry.release().unwrap();
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("failed").is_none());
|
||||||
|
assert!(table.data().find("other").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Success disarms every clone and final Drop, even if the name is
|
||||||
|
// subsequently allocated to a different Worker.
|
||||||
|
let _replacement = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"failed".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("replacement"),
|
||||||
|
vec![read_rule("/shared", true)],
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
retry.clone().release().unwrap();
|
||||||
|
drop(retry);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("failed").unwrap().socket,
|
||||||
|
sock("replacement")
|
||||||
|
);
|
||||||
|
assert!(table.data().find("other").is_some());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn adopt_allocation_rewrites_pid_and_releases_on_drop() {
|
fn adopt_allocation_rewrites_pid_and_releases_on_drop() {
|
||||||
let dir = TempDir::new().unwrap();
|
let dir = TempDir::new().unwrap();
|
||||||
@@ -358,6 +848,7 @@ mod tests {
|
|||||||
let dir = TempDir::new().unwrap();
|
let dir = TempDir::new().unwrap();
|
||||||
let _sandbox = RuntimeDirSandbox::new(dir.path());
|
let _sandbox = RuntimeDirSandbox::new(dir.path());
|
||||||
let err = adopt_allocation("ghost".into(), 42, sid()).unwrap_err();
|
let err = adopt_allocation("ghost".into(), 42, sid()).unwrap_err();
|
||||||
|
assert!(err.allocation_cleanup_handle().is_none());
|
||||||
assert!(matches!(err, ScopeLockError::UnknownWorker(ref n) if n == "ghost"));
|
assert!(matches!(err, ScopeLockError::UnknownWorker(ref n) if n == "ghost"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -57,6 +57,30 @@ pub fn register_worker_with_deny(
|
|||||||
scope_allow: Vec<ScopeRule>,
|
scope_allow: Vec<ScopeRule>,
|
||||||
scope_deny: Vec<ScopeRule>,
|
scope_deny: Vec<ScopeRule>,
|
||||||
segment_id: SegmentId,
|
segment_id: SegmentId,
|
||||||
|
) -> Result<(), ScopeLockError> {
|
||||||
|
prepare_worker_with_deny(
|
||||||
|
guard,
|
||||||
|
worker_name,
|
||||||
|
pid,
|
||||||
|
socket,
|
||||||
|
scope_allow,
|
||||||
|
scope_deny,
|
||||||
|
segment_id,
|
||||||
|
)?;
|
||||||
|
guard.save()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Admit and stage a registration without publishing it. Owned installation
|
||||||
|
/// constructs its cleanup authority after admission and before committing.
|
||||||
|
pub(super) fn prepare_worker_with_deny(
|
||||||
|
guard: &mut LockFileGuard,
|
||||||
|
worker_name: String,
|
||||||
|
pid: u32,
|
||||||
|
socket: PathBuf,
|
||||||
|
scope_allow: Vec<ScopeRule>,
|
||||||
|
scope_deny: Vec<ScopeRule>,
|
||||||
|
segment_id: SegmentId,
|
||||||
) -> Result<(), ScopeLockError> {
|
) -> Result<(), ScopeLockError> {
|
||||||
reclaim_stale(guard);
|
reclaim_stale(guard);
|
||||||
if guard.data().find(&worker_name).is_some() {
|
if guard.data().find(&worker_name).is_some() {
|
||||||
@@ -101,7 +125,6 @@ pub fn register_worker_with_deny(
|
|||||||
delegated_from: None,
|
delegated_from: None,
|
||||||
segment_id: Some(segment_id),
|
segment_id: Some(segment_id),
|
||||||
});
|
});
|
||||||
guard.save()?;
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -245,6 +245,14 @@ impl LockFileGuard {
|
|||||||
/// Persist with atomic replacement while the separate allocation lock stays
|
/// Persist with atomic replacement while the separate allocation lock stays
|
||||||
/// held. A crash exposes either the old complete table or the new one.
|
/// held. A crash exposes either the old complete table or the new one.
|
||||||
pub fn save(&mut self) -> io::Result<()> {
|
pub fn save(&mut self) -> io::Result<()> {
|
||||||
|
let mut published = false;
|
||||||
|
self.save_with_publication(&mut published)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Report this save's rename while the caller still holds the table lock.
|
||||||
|
/// In-memory mutation alone is not evidence that an acquisition took place.
|
||||||
|
pub(super) fn save_with_publication(&mut self, published: &mut bool) -> io::Result<()> {
|
||||||
|
*published = false;
|
||||||
let json = serde_json::to_vec_pretty(&self.data).map_err(io::Error::other)?;
|
let json = serde_json::to_vec_pretty(&self.data).map_err(io::Error::other)?;
|
||||||
let mut temp_os = self.data_path.as_os_str().to_owned();
|
let mut temp_os = self.data_path.as_os_str().to_owned();
|
||||||
temp_os.push(".tmp");
|
temp_os.push(".tmp");
|
||||||
@@ -264,6 +272,7 @@ impl LockFileGuard {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
fs::rename(&temp_path, &self.data_path)?;
|
fs::rename(&temp_path, &self.data_path)?;
|
||||||
|
*published = true;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
if take_save_fault(&self.data_path, SaveFaultPoint::AfterRename) {
|
if take_save_fault(&self.data_path, SaveFaultPoint::AfterRename) {
|
||||||
return Err(io::Error::other(
|
return Err(io::Error::other(
|
||||||
|
|||||||
+760
-28
@@ -533,7 +533,7 @@ use crate::prompt::source::PromptCatalogSource;
|
|||||||
use crate::prompt::system::{SystemPromptContext, SystemPromptError, SystemPromptTemplate};
|
use crate::prompt::system::{SystemPromptContext, SystemPromptError, SystemPromptTemplate};
|
||||||
use crate::runtime::dir;
|
use crate::runtime::dir;
|
||||||
use crate::runtime::worker_allocation::{
|
use crate::runtime::worker_allocation::{
|
||||||
self, ScopeAllocationGuard, ScopeLockError, SegmentActivationGuard,
|
self, ScopeAllocationCleanup, ScopeAllocationGuard, ScopeLockError, SegmentActivationGuard,
|
||||||
};
|
};
|
||||||
use crate::skill::{SkillActivationResponse, SkillClientError};
|
use crate::skill::{SkillActivationResponse, SkillClientError};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -2951,6 +2951,69 @@ struct PendingCompactionCleanup {
|
|||||||
compaction_id: String,
|
compaction_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Retains startup task scopes after a consuming controller startup fails.
|
||||||
|
///
|
||||||
|
/// Call `shutdown` only after startup has failed with no live Controller. Keep
|
||||||
|
/// this handle on errors or timeout and retry; it owns any in-flight settlement
|
||||||
|
/// future, including join handles drained from the registries. On successful
|
||||||
|
/// controller transfer, drop the factory's handle without shutting it down.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct WorkerStartupFeatureCleanup {
|
||||||
|
inner: Arc<StartupFeatureCleanupInner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct StartupFeatureCleanupInner {
|
||||||
|
registries: Mutex<Vec<FeatureBackgroundTaskRegistry>>,
|
||||||
|
settlement: tokio::sync::Mutex<
|
||||||
|
Option<futures_util::future::BoxFuture<'static, Result<(), crate::hook::HookError>>>,
|
||||||
|
>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkerStartupFeatureCleanup {
|
||||||
|
fn new(registry: FeatureBackgroundTaskRegistry) -> Self {
|
||||||
|
Self {
|
||||||
|
inner: Arc::new(StartupFeatureCleanupInner {
|
||||||
|
registries: Mutex::new(vec![registry]),
|
||||||
|
settlement: tokio::sync::Mutex::new(None),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn retain(&self, registry: FeatureBackgroundTaskRegistry) {
|
||||||
|
self.inner
|
||||||
|
.registries
|
||||||
|
.lock()
|
||||||
|
.expect("startup Feature cleanup poisoned")
|
||||||
|
.push(registry);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stops and joins every captured registry. Cancelling this call preserves
|
||||||
|
/// its settlement future for the next call, rather than dropping task joins.
|
||||||
|
pub async fn shutdown(&self) -> Result<(), crate::hook::HookError> {
|
||||||
|
let mut settlement = self.inner.settlement.lock().await;
|
||||||
|
let pending = settlement.get_or_insert_with(|| {
|
||||||
|
let registries = self
|
||||||
|
.inner
|
||||||
|
.registries
|
||||||
|
.lock()
|
||||||
|
.expect("startup Feature cleanup poisoned")
|
||||||
|
.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
let mut failure = None;
|
||||||
|
for registry in registries {
|
||||||
|
if let Err(error) = registry.shutdown().await {
|
||||||
|
failure.get_or_insert(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
failure.map_or(Ok(()), Err)
|
||||||
|
})
|
||||||
|
});
|
||||||
|
let result = pending.as_mut().await;
|
||||||
|
*settlement = None;
|
||||||
|
result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// An independent agent execution unit.
|
/// An independent agent execution unit.
|
||||||
///
|
///
|
||||||
/// Holds a [`Engine`] directly and persists session state via
|
/// Holds a [`Engine`] directly and persists session state via
|
||||||
@@ -2997,6 +3060,8 @@ pub struct Worker<C: LlmClient, St: Store> {
|
|||||||
hook_registry: Option<Arc<HookRegistry>>,
|
hook_registry: Option<Arc<HookRegistry>>,
|
||||||
/// Executable background tasks registered by successfully installed features.
|
/// Executable background tasks registered by successfully installed features.
|
||||||
feature_background_tasks: FeatureBackgroundTaskRegistry,
|
feature_background_tasks: FeatureBackgroundTaskRegistry,
|
||||||
|
/// Present only while an opted-in factory is capturing startup task scopes.
|
||||||
|
startup_feature_cleanup: Option<WorkerStartupFeatureCleanup>,
|
||||||
/// Enabled chat invocation metadata and executable handlers.
|
/// Enabled chat invocation metadata and executable handlers.
|
||||||
feature_invocations: FeatureInvocationRegistry,
|
feature_invocations: FeatureInvocationRegistry,
|
||||||
/// Internal Workers install an explicit Feature composition and disable
|
/// Internal Workers install an explicit Feature composition and disable
|
||||||
@@ -3087,9 +3152,8 @@ pub struct Worker<C: LlmClient, St: Store> {
|
|||||||
/// Workers built via `from_manifest` / `from_manifest_spawned` /
|
/// Workers built via `from_manifest` / `from_manifest_spawned` /
|
||||||
/// `restore_from_manifest` (production paths); `None` for the
|
/// `restore_from_manifest` (production paths); `None` for the
|
||||||
/// low-level `Worker::new` constructor used in tests, which bypasses
|
/// low-level `Worker::new` constructor used in tests, which bypasses
|
||||||
/// the registry. Kept purely for its `Drop` impl, which releases
|
/// the registry. Cleanup handles can retain the allocation beyond Worker
|
||||||
/// the allocation when the Worker is dropped.
|
/// drop; only the final shared owner attempts best-effort release.
|
||||||
#[allow(dead_code)]
|
|
||||||
scope_allocation: Option<ScopeAllocationGuard>,
|
scope_allocation: Option<ScopeAllocationGuard>,
|
||||||
/// Socket path of the spawning Worker. `Some` only for Workers built via
|
/// Socket path of the spawning Worker. `Some` only for Workers built via
|
||||||
/// `from_manifest_spawned`. Consumed by the controller to fire
|
/// `from_manifest_spawned`. Consumed by the controller to fire
|
||||||
@@ -3155,7 +3219,7 @@ pub struct Worker<C: LlmClient, St: Store> {
|
|||||||
impl<C: LlmClient, St: Store> Drop for Worker<C, St> {
|
impl<C: LlmClient, St: Store> Drop for Worker<C, St> {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
// A failed activation authority nests the same machine-wide lock used by
|
// A failed activation authority nests the same machine-wide lock used by
|
||||||
// `ScopeAllocationGuard::drop`. Release the nested guard first so Worker
|
// allocation owner's final Drop. Release the nested guard first so Worker
|
||||||
// teardown cannot self-deadlock while removing its outer allocation.
|
// teardown cannot self-deadlock while removing its outer allocation.
|
||||||
let authority = match self.failed_activation_authority.get_mut() {
|
let authority = match self.failed_activation_authority.get_mut() {
|
||||||
Ok(authority) => authority.take(),
|
Ok(authority) => authority.take(),
|
||||||
@@ -3165,6 +3229,47 @@ impl<C: LlmClient, St: Store> Drop for Worker<C, St> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl<C: LlmClient, St: Store> Worker<C, St> {
|
||||||
|
/// Opt in before a consuming startup call. The handle captures the current
|
||||||
|
/// task registry and each subsequently installed registry until successful
|
||||||
|
/// controller transfer ends capture. Repeated calls share the same capture.
|
||||||
|
pub fn retain_startup_feature_cleanup(&mut self) -> WorkerStartupFeatureCleanup {
|
||||||
|
self.startup_feature_cleanup
|
||||||
|
.get_or_insert_with(|| {
|
||||||
|
WorkerStartupFeatureCleanup::new(self.feature_background_tasks.clone())
|
||||||
|
})
|
||||||
|
.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Observes explicit factory cleanup ownership without enabling capture.
|
||||||
|
pub fn startup_feature_cleanup_handle(&self) -> Option<WorkerStartupFeatureCleanup> {
|
||||||
|
self.startup_feature_cleanup.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// End factory capture immediately before transferring to a live Controller.
|
||||||
|
/// The factory must also drop its handle after startup returns success.
|
||||||
|
pub fn finish_startup_feature_cleanup_capture(&mut self) {
|
||||||
|
self.startup_feature_cleanup = None;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retain cleanup authority before passing the reconstructed Worker into a
|
||||||
|
/// consuming factory. Explicit release is only valid after factory failure
|
||||||
|
/// with no live Controller; drop the handle on successful controller transfer.
|
||||||
|
pub fn restore_allocation_cleanup_handle(&self) -> Option<ScopeAllocationCleanup> {
|
||||||
|
self.scope_allocation
|
||||||
|
.as_ref()
|
||||||
|
.map(ScopeAllocationGuard::cleanup_handle)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn construction_failure(&self, error: WorkerError) -> WorkerError {
|
||||||
|
WorkerError::ConstructionCleanup {
|
||||||
|
source: Box::new(error),
|
||||||
|
allocation: self.restore_allocation_cleanup_handle(),
|
||||||
|
sessions: self.workdir_sessions.clone(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl<C: LlmClient + 'static, St: Store + 'static> Worker<C, St> {
|
impl<C: LlmClient + 'static, St: Store + 'static> Worker<C, St> {
|
||||||
pub async fn stop_feature_runtime(&mut self, reason: impl Into<String>) {
|
pub async fn stop_feature_runtime(&mut self, reason: impl Into<String>) {
|
||||||
if let Some(hooks) = self.hook_registry.clone() {
|
if let Some(hooks) = self.hook_registry.clone() {
|
||||||
@@ -3352,6 +3457,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
hook_builder: HookRegistryBuilder::new(),
|
hook_builder: HookRegistryBuilder::new(),
|
||||||
hook_registry: None,
|
hook_registry: None,
|
||||||
feature_background_tasks: FeatureBackgroundTaskRegistry::default(),
|
feature_background_tasks: FeatureBackgroundTaskRegistry::default(),
|
||||||
|
startup_feature_cleanup: None,
|
||||||
feature_invocations: FeatureInvocationRegistry::default(),
|
feature_invocations: FeatureInvocationRegistry::default(),
|
||||||
manifest_lifecycle_features_enabled: true,
|
manifest_lifecycle_features_enabled: true,
|
||||||
interceptor_installed: false,
|
interceptor_installed: false,
|
||||||
@@ -3850,6 +3956,9 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
|||||||
return report;
|
return report;
|
||||||
}
|
}
|
||||||
self.feature_background_tasks = report.background_tasks.clone();
|
self.feature_background_tasks = report.background_tasks.clone();
|
||||||
|
if let Some(cleanup) = &self.startup_feature_cleanup {
|
||||||
|
cleanup.retain(self.feature_background_tasks.clone());
|
||||||
|
}
|
||||||
self.feature_invocations = report.chat_invocations.clone();
|
self.feature_invocations = report.chat_invocations.clone();
|
||||||
for instruction in report.installed_instruction_contributions() {
|
for instruction in report.installed_instruction_contributions() {
|
||||||
self.register_feature_instruction(instruction);
|
self.register_feature_instruction(instruction);
|
||||||
@@ -7906,7 +8015,7 @@ where
|
|||||||
let workdir_sessions =
|
let workdir_sessions =
|
||||||
workdir_sessions_from_authority(&common.filesystem_authority, &scope);
|
workdir_sessions_from_authority(&common.filesystem_authority, &scope);
|
||||||
|
|
||||||
let mut worker = Self {
|
let worker = Self {
|
||||||
backend_job: std::sync::OnceLock::new(),
|
backend_job: std::sync::OnceLock::new(),
|
||||||
manifest,
|
manifest,
|
||||||
engine: Some(worker),
|
engine: Some(worker),
|
||||||
@@ -7925,6 +8034,7 @@ where
|
|||||||
hook_builder: HookRegistryBuilder::new(),
|
hook_builder: HookRegistryBuilder::new(),
|
||||||
hook_registry: None,
|
hook_registry: None,
|
||||||
feature_background_tasks: FeatureBackgroundTaskRegistry::default(),
|
feature_background_tasks: FeatureBackgroundTaskRegistry::default(),
|
||||||
|
startup_feature_cleanup: None,
|
||||||
feature_invocations: FeatureInvocationRegistry::default(),
|
feature_invocations: FeatureInvocationRegistry::default(),
|
||||||
manifest_lifecycle_features_enabled: true,
|
manifest_lifecycle_features_enabled: true,
|
||||||
interceptor_installed: false,
|
interceptor_installed: false,
|
||||||
@@ -7966,10 +8076,15 @@ where
|
|||||||
log_writer: None,
|
log_writer: None,
|
||||||
notification_committer: None,
|
notification_committer: None,
|
||||||
};
|
};
|
||||||
worker.apply_permissions_from_manifest();
|
worker.finish_pending_construction()
|
||||||
worker.apply_prune_from_manifest();
|
}
|
||||||
worker.write_worker_metadata_pending()?;
|
|
||||||
Ok(worker)
|
fn finish_pending_construction(mut self) -> Result<Self, WorkerError> {
|
||||||
|
self.apply_permissions_from_manifest();
|
||||||
|
self.apply_prune_from_manifest();
|
||||||
|
self.write_worker_metadata_pending()
|
||||||
|
.map_err(|error| self.construction_failure(error))?;
|
||||||
|
Ok(self)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Complete the Host-owned Session-to-Subject attribution boundary before a
|
/// Complete the Host-owned Session-to-Subject attribution boundary before a
|
||||||
@@ -8306,6 +8421,7 @@ where
|
|||||||
hook_builder: HookRegistryBuilder::new(),
|
hook_builder: HookRegistryBuilder::new(),
|
||||||
hook_registry: None,
|
hook_registry: None,
|
||||||
feature_background_tasks: FeatureBackgroundTaskRegistry::default(),
|
feature_background_tasks: FeatureBackgroundTaskRegistry::default(),
|
||||||
|
startup_feature_cleanup: None,
|
||||||
feature_invocations: FeatureInvocationRegistry::default(),
|
feature_invocations: FeatureInvocationRegistry::default(),
|
||||||
manifest_lifecycle_features_enabled: false,
|
manifest_lifecycle_features_enabled: false,
|
||||||
interceptor_installed: false,
|
interceptor_installed: false,
|
||||||
@@ -8438,6 +8554,7 @@ where
|
|||||||
hook_builder: HookRegistryBuilder::new(),
|
hook_builder: HookRegistryBuilder::new(),
|
||||||
hook_registry: None,
|
hook_registry: None,
|
||||||
feature_background_tasks: FeatureBackgroundTaskRegistry::default(),
|
feature_background_tasks: FeatureBackgroundTaskRegistry::default(),
|
||||||
|
startup_feature_cleanup: None,
|
||||||
feature_invocations: FeatureInvocationRegistry::default(),
|
feature_invocations: FeatureInvocationRegistry::default(),
|
||||||
manifest_lifecycle_features_enabled: true,
|
manifest_lifecycle_features_enabled: true,
|
||||||
interceptor_installed: false,
|
interceptor_installed: false,
|
||||||
@@ -8751,6 +8868,11 @@ where
|
|||||||
if state.system_prompt.is_none() {
|
if state.system_prompt.is_none() {
|
||||||
return Err(WorkerError::SegmentSystemPromptMissing { segment_id });
|
return Err(WorkerError::SegmentSystemPromptMissing { segment_id });
|
||||||
}
|
}
|
||||||
|
let restored_history_entries =
|
||||||
|
restore_history_entries(session_id, segment_id, &raw_entries).map_err(|error| {
|
||||||
|
WorkerError::InvalidState(format!("restore typed Worker session history: {error}"))
|
||||||
|
})?;
|
||||||
|
let flow_runtime_state = restored_flow_runtime_state(&state.extensions)?;
|
||||||
let mirror_entries: Vec<LogEntry> = raw_entries.clone();
|
let mirror_entries: Vec<LogEntry> = raw_entries.clone();
|
||||||
let scope_config = effective_restore_scope_config(&store, &manifest)?;
|
let scope_config = effective_restore_scope_config(&store, &manifest)?;
|
||||||
|
|
||||||
@@ -8797,10 +8919,6 @@ where
|
|||||||
// (the Worker's one and only write path that prepends a summary at
|
// (the Worker's one and only write path that prepends a summary at
|
||||||
// history[0]). Restoring the anchor lets Anthropic re-use a
|
// history[0]). Restoring the anchor lets Anthropic re-use a
|
||||||
// stable cache prefix for long-lived restored sessions.
|
// stable cache prefix for long-lived restored sessions.
|
||||||
let restored_history_entries =
|
|
||||||
restore_history_entries(session_id, segment_id, &raw_entries).map_err(|error| {
|
|
||||||
WorkerError::InvalidState(format!("restore typed Worker session history: {error}"))
|
|
||||||
})?;
|
|
||||||
let restored_history = restored_history_entries
|
let restored_history = restored_history_entries
|
||||||
.iter()
|
.iter()
|
||||||
.map(|entry| entry.item.clone())
|
.map(|entry| entry.item.clone())
|
||||||
@@ -8839,14 +8957,13 @@ where
|
|||||||
filesystem_authority: common.filesystem_authority,
|
filesystem_authority: common.filesystem_authority,
|
||||||
workdir_sessions,
|
workdir_sessions,
|
||||||
workspace_context: common.workspace_context,
|
workspace_context: common.workspace_context,
|
||||||
flow_runtime_state: Arc::new(Mutex::new(restored_flow_runtime_state(
|
flow_runtime_state: Arc::new(Mutex::new(flow_runtime_state)),
|
||||||
&state.extensions,
|
|
||||||
)?)),
|
|
||||||
flow_feature_enabled: false,
|
flow_feature_enabled: false,
|
||||||
scope,
|
scope,
|
||||||
hook_builder: HookRegistryBuilder::new(),
|
hook_builder: HookRegistryBuilder::new(),
|
||||||
hook_registry: None,
|
hook_registry: None,
|
||||||
feature_background_tasks: FeatureBackgroundTaskRegistry::default(),
|
feature_background_tasks: FeatureBackgroundTaskRegistry::default(),
|
||||||
|
startup_feature_cleanup: None,
|
||||||
feature_invocations: FeatureInvocationRegistry::default(),
|
feature_invocations: FeatureInvocationRegistry::default(),
|
||||||
manifest_lifecycle_features_enabled: true,
|
manifest_lifecycle_features_enabled: true,
|
||||||
interceptor_installed: false,
|
interceptor_installed: false,
|
||||||
@@ -8896,17 +9013,24 @@ where
|
|||||||
worker
|
worker
|
||||||
.session
|
.session
|
||||||
.restore_pending_activations(&state.extensions);
|
.restore_pending_activations(&state.extensions);
|
||||||
worker
|
worker.finish_restored_construction().await
|
||||||
.pending_submission_handle()
|
}
|
||||||
.reconcile_uploaded_file_pins()?;
|
|
||||||
worker.apply_permissions_from_manifest();
|
async fn finish_restored_construction(mut self) -> Result<Self, WorkerError> {
|
||||||
worker.apply_prune_from_manifest();
|
self.pending_submission_handle()
|
||||||
worker.write_worker_metadata_active(SegmentLocation {
|
.reconcile_uploaded_file_pins()
|
||||||
session_id,
|
.map_err(|error| self.construction_failure(error.into()))?;
|
||||||
segment_id,
|
self.apply_permissions_from_manifest();
|
||||||
})?;
|
self.apply_prune_from_manifest();
|
||||||
worker.reconcile_restored_delegations().await?;
|
self.write_worker_metadata_active(SegmentLocation {
|
||||||
Ok(worker)
|
session_id: self.session_id(),
|
||||||
|
segment_id: self.segment_id(),
|
||||||
|
})
|
||||||
|
.map_err(|error| self.construction_failure(error))?;
|
||||||
|
self.reconcile_restored_delegations()
|
||||||
|
.await
|
||||||
|
.map_err(|error| self.construction_failure(error))?;
|
||||||
|
Ok(self)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn reconcile_restored_delegations(&mut self) -> Result<(), WorkerError> {
|
async fn reconcile_restored_delegations(&mut self) -> Result<(), WorkerError> {
|
||||||
@@ -9745,6 +9869,16 @@ fn compact_failure_category(error: &WorkerError) -> CompactFailureCategory {
|
|||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub enum WorkerError {
|
pub enum WorkerError {
|
||||||
|
/// A constructor failed after acquiring resources. Retain these handles until
|
||||||
|
/// explicit cleanup succeeds; the Worker has dropped before this is returned.
|
||||||
|
#[error("{source}")]
|
||||||
|
ConstructionCleanup {
|
||||||
|
#[source]
|
||||||
|
source: Box<WorkerError>,
|
||||||
|
allocation: Option<ScopeAllocationCleanup>,
|
||||||
|
sessions: Arc<WorkdirSessionRouter>,
|
||||||
|
},
|
||||||
|
|
||||||
#[error("invalid durable Worker state: {0}")]
|
#[error("invalid durable Worker state: {0}")]
|
||||||
InvalidState(String),
|
InvalidState(String),
|
||||||
|
|
||||||
@@ -12224,6 +12358,604 @@ mod build_summary_prompt_tests {
|
|||||||
assert_eq!(restored.active_run_turn_count, None);
|
assert_eq!(restored.active_run_turn_count, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn restore_cleanup_handle_retains_allocation_after_worker_drop() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let store = session_store::FsStore::new(dir.path().join("sessions")).unwrap();
|
||||||
|
let mut worker = Worker::new(
|
||||||
|
minimal_manifest(),
|
||||||
|
Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient),
|
||||||
|
store,
|
||||||
|
WorkerWorkspaceContext::no_workspace(),
|
||||||
|
WorkerFilesystemAuthority::None,
|
||||||
|
Scope::empty(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(worker.restore_allocation_cleanup_handle().is_none());
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
worker.scope_allocation = Some(
|
||||||
|
worker_allocation::install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"test-worker".into(),
|
||||||
|
std::process::id(),
|
||||||
|
dir.path().join("worker.sock"),
|
||||||
|
Vec::new(),
|
||||||
|
worker.segment_id(),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let cleanup = worker.restore_allocation_cleanup_handle().unwrap();
|
||||||
|
|
||||||
|
drop(worker);
|
||||||
|
{
|
||||||
|
let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("test-worker").is_some());
|
||||||
|
}
|
||||||
|
cleanup.release().unwrap();
|
||||||
|
let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("test-worker").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Constructor finalization uses real Worker, allocation, Workdir, and storage
|
||||||
|
// implementations. Only the model and metadata-write failure boundary are substituted.
|
||||||
|
mod construction_cleanup_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
type ConstructionStore =
|
||||||
|
session_store::CombinedStore<session_store::FsStore, session_store::FsWorkerStore>;
|
||||||
|
|
||||||
|
async fn constructed_worker(
|
||||||
|
temp: &tempfile::TempDir,
|
||||||
|
) -> Worker<Box<dyn LlmClient>, ConstructionStore> {
|
||||||
|
let store = session_store::CombinedStore::new(
|
||||||
|
session_store::FsStore::new(temp.path().join("sessions")).unwrap(),
|
||||||
|
session_store::FsWorkerStore::new(temp.path().join("workers")).unwrap(),
|
||||||
|
);
|
||||||
|
let mut worker = Worker::new(
|
||||||
|
minimal_manifest(),
|
||||||
|
Engine::<Box<dyn LlmClient>, Mutable, SessionHistoryMetadata>::new_annotated(
|
||||||
|
Box::new(NoopClient),
|
||||||
|
),
|
||||||
|
store,
|
||||||
|
WorkerWorkspaceContext::no_workspace(),
|
||||||
|
WorkerFilesystemAuthority::local(
|
||||||
|
temp.path().to_path_buf(),
|
||||||
|
temp.path().to_path_buf(),
|
||||||
|
),
|
||||||
|
Scope::writable(temp.path()).unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
worker.scope_allocation = Some(
|
||||||
|
worker_allocation::install_top_level_at_for_test(
|
||||||
|
temp.path().join("workers.json"),
|
||||||
|
worker.manifest.worker.name.clone(),
|
||||||
|
std::process::id(),
|
||||||
|
temp.path().join("worker.sock"),
|
||||||
|
Vec::new(),
|
||||||
|
worker.segment_id(),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
worker
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn post_construction_failures_retain_resources_through_failed_release_retry() {
|
||||||
|
for stage in [
|
||||||
|
"pending_metadata",
|
||||||
|
"restore_pins",
|
||||||
|
"restore_metadata",
|
||||||
|
"restore_delegations",
|
||||||
|
] {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let mut worker = constructed_worker(&temp).await;
|
||||||
|
if stage != "pending_metadata" {
|
||||||
|
worker.ensure_segment_head().await.unwrap();
|
||||||
|
}
|
||||||
|
let name = worker.manifest.worker.name.clone();
|
||||||
|
let lock_path = temp.path().join("workers.json");
|
||||||
|
let sessions = Arc::downgrade(&worker.workdir_sessions());
|
||||||
|
let session = worker.workdir_sessions().resolve(None).unwrap().session;
|
||||||
|
let writes = Arc::new(AtomicUsize::new(0));
|
||||||
|
let writer_calls = writes.clone();
|
||||||
|
let metadata_path = temp
|
||||||
|
.path()
|
||||||
|
.join("workers")
|
||||||
|
.join(&name)
|
||||||
|
.join("metadata.json");
|
||||||
|
worker.worker_metadata_writer = Some(Arc::new(move |_| {
|
||||||
|
writer_calls.fetch_add(1, Ordering::SeqCst);
|
||||||
|
if stage == "restore_delegations" {
|
||||||
|
// A successful publication followed by an unreadable retained
|
||||||
|
// record forces the subsequent delegation reconciliation to fail.
|
||||||
|
std::fs::create_dir_all(&metadata_path)?;
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(WorkerStoreError::Io(std::io::Error::other(
|
||||||
|
"metadata write refused",
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
if stage == "restore_pins" {
|
||||||
|
let pins_dir = temp
|
||||||
|
.path()
|
||||||
|
.join("sessions")
|
||||||
|
.join(worker.session_id().to_string())
|
||||||
|
.join("artifacts")
|
||||||
|
.join("paste");
|
||||||
|
std::fs::create_dir_all(pins_dir.join(".aggregate.lock")).unwrap();
|
||||||
|
}
|
||||||
|
let result = if stage == "pending_metadata" {
|
||||||
|
worker.finish_pending_construction()
|
||||||
|
} else {
|
||||||
|
worker.finish_restored_construction().await
|
||||||
|
};
|
||||||
|
let error = match result {
|
||||||
|
Err(error) => error,
|
||||||
|
Ok(_) => panic!("{stage} must fail after Worker construction"),
|
||||||
|
};
|
||||||
|
let WorkerError::ConstructionCleanup {
|
||||||
|
source,
|
||||||
|
allocation,
|
||||||
|
sessions: retained,
|
||||||
|
} = &error
|
||||||
|
else {
|
||||||
|
panic!("{stage} lost construction cleanup authority: {error:?}");
|
||||||
|
};
|
||||||
|
if stage == "restore_pins" {
|
||||||
|
assert!(matches!(
|
||||||
|
source.as_ref(),
|
||||||
|
WorkerError::Store(StoreError::Io(_))
|
||||||
|
));
|
||||||
|
assert_eq!(writes.load(Ordering::SeqCst), 0);
|
||||||
|
} else {
|
||||||
|
assert!(matches!(
|
||||||
|
source.as_ref(),
|
||||||
|
WorkerError::WorkerStore(WorkerStoreError::Io(_))
|
||||||
|
));
|
||||||
|
assert_eq!(writes.load(Ordering::SeqCst), 1);
|
||||||
|
}
|
||||||
|
assert_eq!(error.to_string(), source.to_string());
|
||||||
|
assert_eq!(
|
||||||
|
std::error::Error::source(&error).unwrap().to_string(),
|
||||||
|
source.to_string()
|
||||||
|
);
|
||||||
|
assert!(Arc::ptr_eq(&sessions.upgrade().unwrap(), retained));
|
||||||
|
let allocation = allocation.clone().expect("constructed Worker's allocation");
|
||||||
|
let retained = retained.clone();
|
||||||
|
drop(error);
|
||||||
|
{
|
||||||
|
let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(
|
||||||
|
table.data().find(&name).is_some(),
|
||||||
|
"{stage} released during Worker Drop"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
session
|
||||||
|
.stat(workdir::StatRequest {
|
||||||
|
path: workdir::WorkdirPath::root()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
retained.close_all().await.unwrap();
|
||||||
|
assert!(retained.is_empty());
|
||||||
|
assert!(
|
||||||
|
session
|
||||||
|
.stat(workdir::StatRequest {
|
||||||
|
path: workdir::WorkdirPath::root()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
|
||||||
|
worker_allocation::fail_next_save_for_test(&lock_path);
|
||||||
|
assert!(matches!(allocation.release(), Err(ScopeLockError::Io(_))));
|
||||||
|
{
|
||||||
|
let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(
|
||||||
|
table.data().find(&name).is_some(),
|
||||||
|
"{stage} lost failed release"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
allocation.release().unwrap();
|
||||||
|
allocation.clone().release().unwrap();
|
||||||
|
let table = worker_allocation::LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find(&name).is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn malformed_restore_state_is_rejected_before_filesystem_preparation() {
|
||||||
|
for corrupt_flow in [false, true] {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let store = session_store::CombinedStore::new(
|
||||||
|
session_store::FsStore::new(temp.path().join("sessions")).unwrap(),
|
||||||
|
session_store::FsWorkerStore::new(temp.path().join("workers")).unwrap(),
|
||||||
|
);
|
||||||
|
let session_id = session_store::new_session_id();
|
||||||
|
let segment_id = session_store::new_segment_id();
|
||||||
|
let mut entries = vec![LogEntry::AnnotatedSegmentStart {
|
||||||
|
ts: 0,
|
||||||
|
session_id,
|
||||||
|
system_prompt: Some("saved prompt".into()),
|
||||||
|
config: RequestConfig::default(),
|
||||||
|
history: Vec::new(),
|
||||||
|
forked_from: None,
|
||||||
|
compacted_from: None,
|
||||||
|
}];
|
||||||
|
if corrupt_flow {
|
||||||
|
entries.push(LogEntry::Extension {
|
||||||
|
ts: 1,
|
||||||
|
domain: FLOW_RUNTIME_EXTENSION_DOMAIN.into(),
|
||||||
|
payload: serde_json::Value::Null,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
entries.push(LogEntry::ToolResultCorrected {
|
||||||
|
ts: 1,
|
||||||
|
entry: to_logged_history_entry(&history_entry(
|
||||||
|
Item::user_message("correction without a target"),
|
||||||
|
WorkerHistoryProvenance::LegacyUnknown,
|
||||||
|
)),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
store
|
||||||
|
.create_segment(session_id, segment_id, &entries)
|
||||||
|
.unwrap();
|
||||||
|
let result = Worker::<Box<dyn LlmClient>, _>::restore_from_manifest_with_context_and_model_client(
|
||||||
|
session_id,
|
||||||
|
segment_id,
|
||||||
|
minimal_manifest(),
|
||||||
|
store,
|
||||||
|
PromptCatalogSource::builtins_only(),
|
||||||
|
WorkerWorkspaceContext::no_workspace(),
|
||||||
|
// If preparation runs first, it must reject this missing root,
|
||||||
|
// without touching the process-global allocation table.
|
||||||
|
WorkerFilesystemAuthority::local(temp.path().join("missing"), temp.path().join("missing")),
|
||||||
|
Some(Box::new(NoopClient)),
|
||||||
|
).await;
|
||||||
|
let error = match result {
|
||||||
|
Err(error) => error,
|
||||||
|
Ok(_) => panic!("corrupt restore state must be rejected"),
|
||||||
|
};
|
||||||
|
let WorkerError::InvalidState(message) = error else {
|
||||||
|
panic!("restore parsed state too late: {error:?}");
|
||||||
|
};
|
||||||
|
assert!(message.contains(if corrupt_flow {
|
||||||
|
"Flow runtime state"
|
||||||
|
} else {
|
||||||
|
"typed Worker session history"
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Real Feature installation, task registries, Workdir provider/router, and
|
||||||
|
// PreparedWorker startup; only the model and task completion barrier are test implementations.
|
||||||
|
mod startup_feature_cleanup_tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::feature::background::{
|
||||||
|
BackgroundTaskCancellation, BackgroundTaskContext, BackgroundTaskSpec,
|
||||||
|
FeatureBackgroundTask,
|
||||||
|
};
|
||||||
|
use crate::feature::{
|
||||||
|
BackgroundTaskDeclaration, FeatureDescriptor, FeatureId, FeatureInstallContext,
|
||||||
|
FeatureInstallError, FeatureModule,
|
||||||
|
};
|
||||||
|
use crate::hook::{HookError, HookErrorCategory};
|
||||||
|
use tokio::sync::Notify;
|
||||||
|
|
||||||
|
type StartupStore =
|
||||||
|
session_store::CombinedStore<session_store::FsStore, session_store::FsWorkerStore>;
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct TaskSignals {
|
||||||
|
entered: Notify,
|
||||||
|
cancelled: AtomicBool,
|
||||||
|
release: Notify,
|
||||||
|
exited: AtomicBool,
|
||||||
|
session: Mutex<Option<workdir::WorkdirSessionHandle>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct BarrierTask {
|
||||||
|
signals: Arc<TaskSignals>,
|
||||||
|
_lifetime: Arc<()>,
|
||||||
|
panic_on_release: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl FeatureBackgroundTask for BarrierTask {
|
||||||
|
async fn run(
|
||||||
|
&self,
|
||||||
|
_context: BackgroundTaskContext,
|
||||||
|
cancellation: BackgroundTaskCancellation,
|
||||||
|
) -> Result<(), HookError> {
|
||||||
|
self.signals.entered.notify_one();
|
||||||
|
cancellation.cancelled().await;
|
||||||
|
self.signals.cancelled.store(true, Ordering::Release);
|
||||||
|
self.signals.release.notified().await;
|
||||||
|
assert!(!self.panic_on_release, "injected startup task join failure");
|
||||||
|
let session = self.signals.session.lock().unwrap().clone();
|
||||||
|
if let Some(session) = session {
|
||||||
|
assert!(
|
||||||
|
session_open(&session).await,
|
||||||
|
"Feature task must retain Workdir access until it is quiescent"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
self.signals.exited.store(true, Ordering::Release);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct BarrierFeature(BarrierTask);
|
||||||
|
|
||||||
|
impl FeatureModule for BarrierFeature {
|
||||||
|
fn descriptor(&self) -> FeatureDescriptor {
|
||||||
|
FeatureDescriptor::builtin("startup-barrier", "Startup barrier")
|
||||||
|
.with_background_task(BackgroundTaskDeclaration::worker_managed(
|
||||||
|
"barrier",
|
||||||
|
"Startup task completion barrier",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn install(
|
||||||
|
&self,
|
||||||
|
context: &mut FeatureInstallContext<'_>,
|
||||||
|
) -> Result<(), FeatureInstallError> {
|
||||||
|
context.background_tasks().register(
|
||||||
|
BackgroundTaskSpec::single_flight(
|
||||||
|
BackgroundTaskDeclaration::worker_managed(
|
||||||
|
"barrier",
|
||||||
|
"Startup task completion barrier",
|
||||||
|
),
|
||||||
|
Duration::from_secs(60),
|
||||||
|
),
|
||||||
|
self.0.clone(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn worker(temp: &tempfile::TempDir) -> Worker<NoopClient, StartupStore> {
|
||||||
|
let store = session_store::CombinedStore::new(
|
||||||
|
session_store::FsStore::new(temp.path().join("sessions")).unwrap(),
|
||||||
|
session_store::FsWorkerStore::new(temp.path().join("workers")).unwrap(),
|
||||||
|
);
|
||||||
|
Worker::new(
|
||||||
|
minimal_manifest(),
|
||||||
|
Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient),
|
||||||
|
store,
|
||||||
|
WorkerWorkspaceContext::no_workspace(),
|
||||||
|
WorkerFilesystemAuthority::None,
|
||||||
|
Scope::empty(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn install(
|
||||||
|
worker: &mut Worker<NoopClient, StartupStore>,
|
||||||
|
panic_on_release: bool,
|
||||||
|
) -> (Arc<TaskSignals>, std::sync::Weak<()>) {
|
||||||
|
let signals = Arc::new(TaskSignals::default());
|
||||||
|
let lifetime = Arc::new(());
|
||||||
|
let weak = Arc::downgrade(&lifetime);
|
||||||
|
let report = worker.install_features(FeatureRegistryBuilder::new().with_module(
|
||||||
|
BarrierFeature(BarrierTask {
|
||||||
|
signals: signals.clone(),
|
||||||
|
_lifetime: lifetime,
|
||||||
|
panic_on_release,
|
||||||
|
}),
|
||||||
|
));
|
||||||
|
assert!(!report.has_errors());
|
||||||
|
(signals, weak)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn start(worker: &Worker<NoopClient, StartupStore>, signals: &TaskSignals) {
|
||||||
|
worker
|
||||||
|
.feature_background_tasks
|
||||||
|
.start(
|
||||||
|
&FeatureId::builtin("startup-barrier"),
|
||||||
|
"barrier",
|
||||||
|
HookInvocationContext {
|
||||||
|
workspace_id: None,
|
||||||
|
worker_id: "startup-owner".into(),
|
||||||
|
session_id: worker.session_id().to_string(),
|
||||||
|
run_id: None,
|
||||||
|
turn_index: None,
|
||||||
|
call_id: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), signals.entered.notified())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bind_session(
|
||||||
|
worker: &mut Worker<NoopClient, StartupStore>,
|
||||||
|
temp: &tempfile::TempDir,
|
||||||
|
) -> workdir::WorkdirSessionHandle {
|
||||||
|
let session: workdir::WorkdirSessionHandle =
|
||||||
|
Arc::new(workdir::LocalWorkdirSession::materialized_bound(
|
||||||
|
workdir::Workdir::new("startup-task-cleanup"),
|
||||||
|
temp.path().to_path_buf(),
|
||||||
|
temp.path().to_path_buf(),
|
||||||
|
SharedScope::new(Scope::writable(temp.path()).unwrap()),
|
||||||
|
workdir::WorkdirSessionCapabilities::ALL,
|
||||||
|
));
|
||||||
|
worker.bind_single_workdir_session(Some(session.clone()));
|
||||||
|
session
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn session_open(session: &workdir::WorkdirSessionHandle) -> bool {
|
||||||
|
session
|
||||||
|
.stat(workdir::StatRequest {
|
||||||
|
path: workdir::WorkdirPath::root(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.is_ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fail_consuming_start(
|
||||||
|
worker: Worker<NoopClient, StartupStore>,
|
||||||
|
temp: &tempfile::TempDir,
|
||||||
|
) {
|
||||||
|
let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some();
|
||||||
|
let blocked_parent = temp.path().join("blocked-runtime");
|
||||||
|
std::fs::write(&blocked_parent, "not a directory").unwrap();
|
||||||
|
let result = crate::PreparedWorker::new(
|
||||||
|
worker,
|
||||||
|
crate::WorkerBootstrapLayout::RuntimeManagedRun {
|
||||||
|
run_dir: blocked_parent.join("1"),
|
||||||
|
bash_output_dir: temp.path().join("spill"),
|
||||||
|
},
|
||||||
|
crate::WorkerControllerTransport::InProcess,
|
||||||
|
)
|
||||||
|
.start()
|
||||||
|
.await;
|
||||||
|
let Err(crate::WorkerBootstrapError::Controller {
|
||||||
|
source,
|
||||||
|
cleanup_failed,
|
||||||
|
}) = result
|
||||||
|
else {
|
||||||
|
panic!("runtime parent file must reject controller startup");
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
source.kind(),
|
||||||
|
std::io::ErrorKind::NotADirectory | std::io::ErrorKind::AlreadyExists
|
||||||
|
));
|
||||||
|
assert_eq!(cleanup_failed, factory_owned_cleanup);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn consumed_worker_failure_retains_all_task_joins_across_cleanup_timeout() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let mut worker = worker(&temp).await;
|
||||||
|
let session = bind_session(&mut worker, &temp);
|
||||||
|
let sessions = worker.workdir_sessions();
|
||||||
|
let (first, first_lifetime) = install(&mut worker, false);
|
||||||
|
*first.session.lock().unwrap() = Some(session.clone());
|
||||||
|
start(&worker, &first).await;
|
||||||
|
let cleanup = worker.retain_startup_feature_cleanup();
|
||||||
|
let retry_owner = cleanup.clone();
|
||||||
|
let (second, second_lifetime) = install(&mut worker, false);
|
||||||
|
*second.session.lock().unwrap() = Some(session.clone());
|
||||||
|
start(&worker, &second).await;
|
||||||
|
fail_consuming_start(worker, &temp).await;
|
||||||
|
|
||||||
|
assert!(!sessions.aliases().is_empty());
|
||||||
|
assert!(session_open(&session).await);
|
||||||
|
assert!(first_lifetime.upgrade().is_some());
|
||||||
|
assert!(second_lifetime.upgrade().is_some());
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(50), cleanup.shutdown())
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(first.cancelled.load(Ordering::Acquire));
|
||||||
|
drop(cleanup);
|
||||||
|
|
||||||
|
// A fresh shutdown of the drained registry would falsely succeed
|
||||||
|
// once the second task finishes, abandoning the first task's join.
|
||||||
|
second.release.notify_one();
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(50), retry_owner.shutdown())
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(!first.exited.load(Ordering::Acquire));
|
||||||
|
assert!(session_open(&session).await);
|
||||||
|
first.release.notify_one();
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), retry_owner.shutdown())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(first.exited.load(Ordering::Acquire));
|
||||||
|
assert!(second.exited.load(Ordering::Acquire));
|
||||||
|
assert!(session_open(&session).await);
|
||||||
|
sessions.close_all().await.unwrap();
|
||||||
|
assert!(sessions.aliases().is_empty());
|
||||||
|
assert!(!session_open(&session).await);
|
||||||
|
drop(retry_owner);
|
||||||
|
assert!(first_lifetime.upgrade().is_none());
|
||||||
|
assert!(second_lifetime.upgrade().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn startup_without_factory_cleanup_owner_still_closes_sessions_on_error() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let mut worker = worker(&temp).await;
|
||||||
|
let session = bind_session(&mut worker, &temp);
|
||||||
|
let sessions = worker.workdir_sessions();
|
||||||
|
assert!(session_open(&session).await);
|
||||||
|
fail_consuming_start(worker, &temp).await;
|
||||||
|
assert!(sessions.aliases().is_empty());
|
||||||
|
assert!(!session_open(&session).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cleanup_error_settles_other_registries_and_keeps_owner_for_retry() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let mut worker = worker(&temp).await;
|
||||||
|
let (first, first_lifetime) = install(&mut worker, true);
|
||||||
|
start(&worker, &first).await;
|
||||||
|
let cleanup = worker.retain_startup_feature_cleanup();
|
||||||
|
let (second, second_lifetime) = install(&mut worker, false);
|
||||||
|
start(&worker, &second).await;
|
||||||
|
fail_consuming_start(worker, &temp).await;
|
||||||
|
first.release.notify_one();
|
||||||
|
second.release.notify_one();
|
||||||
|
|
||||||
|
let error = tokio::time::timeout(Duration::from_secs(5), cleanup.shutdown())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap_err();
|
||||||
|
assert_eq!(error.category, HookErrorCategory::Internal);
|
||||||
|
assert!(second.exited.load(Ordering::Acquire));
|
||||||
|
assert!(first_lifetime.upgrade().is_some());
|
||||||
|
assert!(second_lifetime.upgrade().is_some());
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), cleanup.shutdown())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
drop(cleanup);
|
||||||
|
assert!(first_lifetime.upgrade().is_none());
|
||||||
|
assert!(second_lifetime.upgrade().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn successful_controller_transfer_releases_old_startup_task_scopes() {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let mut worker = worker(&temp).await;
|
||||||
|
let (_signals, old_scope) = install(&mut worker, false);
|
||||||
|
let cleanup = worker.retain_startup_feature_cleanup();
|
||||||
|
let started = crate::PreparedWorker::new(
|
||||||
|
worker,
|
||||||
|
crate::WorkerBootstrapLayout::RuntimeManagedRun {
|
||||||
|
run_dir: temp.path().join("runs/1"),
|
||||||
|
bash_output_dir: temp.path().join("spill"),
|
||||||
|
},
|
||||||
|
crate::WorkerControllerTransport::InProcess,
|
||||||
|
)
|
||||||
|
.start()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(old_scope.upgrade().is_some());
|
||||||
|
drop(cleanup);
|
||||||
|
let released = old_scope.upgrade().is_none();
|
||||||
|
stop_body_publication_controller(started).await;
|
||||||
|
assert!(
|
||||||
|
released,
|
||||||
|
"live Controller must not retain factory startup capture"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn post_metadata_allocation_failure_fences_appends_and_restart_converges() {
|
async fn post_metadata_allocation_failure_fences_appends_and_restart_converges() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
@@ -759,6 +759,211 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
mod socket_startup_failure {
|
||||||
|
//! Connect the real controller startup, command observer, socket bind and session rollback.
|
||||||
|
//! Only the Workdir telemetry boundary and LLM are substituted. This ordinary controller
|
||||||
|
//! integration target uses an isolated directory at the socket path to reject the real bind.
|
||||||
|
use super::*;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use tokio::sync::broadcast;
|
||||||
|
use workdir::WorkdirSession;
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct ObservedSession {
|
||||||
|
local: LocalWorkdirSession,
|
||||||
|
socket_path: PathBuf,
|
||||||
|
events: broadcast::Sender<workdir::CommandEvent>,
|
||||||
|
subscriptions: AtomicUsize,
|
||||||
|
receivers_at_close: Mutex<Vec<usize>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl WorkdirSession for ObservedSession {
|
||||||
|
fn workdir(&self) -> &Workdir {
|
||||||
|
self.local.workdir()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
||||||
|
self.local.capabilities()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn stat(
|
||||||
|
&self,
|
||||||
|
request: workdir::StatRequest,
|
||||||
|
) -> Result<workdir::StatResult, WorkdirError> {
|
||||||
|
self.local.stat(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read(
|
||||||
|
&self,
|
||||||
|
request: workdir::ReadRequest,
|
||||||
|
) -> Result<workdir::ReadResult, WorkdirError> {
|
||||||
|
self.local.read(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write(
|
||||||
|
&self,
|
||||||
|
request: workdir::WriteRequest,
|
||||||
|
) -> Result<workdir::WriteResult, WorkdirError> {
|
||||||
|
self.local.write(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn edit(
|
||||||
|
&self,
|
||||||
|
request: workdir::EditRequest,
|
||||||
|
) -> Result<workdir::EditResult, WorkdirError> {
|
||||||
|
self.local.edit(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list(
|
||||||
|
&self,
|
||||||
|
request: workdir::ListRequest,
|
||||||
|
) -> Result<workdir::ListResult, WorkdirError> {
|
||||||
|
self.local.list(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn glob(
|
||||||
|
&self,
|
||||||
|
request: workdir::GlobRequest,
|
||||||
|
) -> Result<workdir::GlobResult, WorkdirError> {
|
||||||
|
self.local.glob(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn grep(
|
||||||
|
&self,
|
||||||
|
request: workdir::GrepRequest,
|
||||||
|
) -> Result<workdir::GrepResult, WorkdirError> {
|
||||||
|
self.local.grep(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn start_command(
|
||||||
|
&self,
|
||||||
|
request: CommandRequest,
|
||||||
|
) -> Result<workdir::CommandHandle, WorkdirError> {
|
||||||
|
self.local.start_command(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn command_status(
|
||||||
|
&self,
|
||||||
|
handle: workdir::CommandHandle,
|
||||||
|
) -> Result<workdir::CommandStatus, WorkdirError> {
|
||||||
|
self.local.command_status(handle).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn command_output(
|
||||||
|
&self,
|
||||||
|
request: CommandOutputRequest,
|
||||||
|
) -> Result<workdir::CommandOutput, WorkdirError> {
|
||||||
|
self.local.command_output(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cancel_command(&self, handle: workdir::CommandHandle) -> Result<(), WorkdirError> {
|
||||||
|
self.local.cancel_command(handle).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<workdir::CommandEvent>> {
|
||||||
|
// Runtime creation and initial file writes must already have succeeded. A directory
|
||||||
|
// survives SocketServer's stale-file removal and makes the subsequent Unix bind fail.
|
||||||
|
std::fs::create_dir(&self.socket_path).unwrap();
|
||||||
|
self.subscriptions.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Some(self.events.subscribe())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn close(&self) -> Result<(), WorkdirError> {
|
||||||
|
// Record before any await: aborting without joining must not pass by getting a
|
||||||
|
// scheduler turn during the provider's asynchronous close. The sender stays alive
|
||||||
|
// even after close, so channel closure cannot accidentally dispose of the observer.
|
||||||
|
self.receivers_at_close
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push(self.events.receiver_count());
|
||||||
|
self.local.close().await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn socket_bind_failure_joins_command_observer_before_closing_session() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let run_dir = root.path().join("runs/1");
|
||||||
|
let socket_path = run_dir.join("worker.sock");
|
||||||
|
let client = MockClient::new(simple_text_events());
|
||||||
|
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
||||||
|
let store = CombinedStore::new(
|
||||||
|
FsStore::new(root.path().join("sessions")).unwrap(),
|
||||||
|
FsWorkerStore::new(root.path().join("workers")).unwrap(),
|
||||||
|
);
|
||||||
|
let engine =
|
||||||
|
Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
||||||
|
client.clone(),
|
||||||
|
);
|
||||||
|
let mut worker = Worker::new(
|
||||||
|
manifest,
|
||||||
|
engine,
|
||||||
|
store,
|
||||||
|
WorkerWorkspaceContext::local_filesystem(None),
|
||||||
|
WorkerFilesystemAuthority::local(root.path().into(), root.path().into()),
|
||||||
|
manifest::Scope::writable(root.path()).unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let (events, unused_receiver) = broadcast::channel(8);
|
||||||
|
drop(unused_receiver);
|
||||||
|
let session = Arc::new(ObservedSession {
|
||||||
|
local: LocalWorkdirSession::materialized_bound(
|
||||||
|
Workdir::new("socket-startup-failure"),
|
||||||
|
root.path().into(),
|
||||||
|
root.path().into(),
|
||||||
|
worker.scope().clone(),
|
||||||
|
WorkdirSessionCapabilities::ALL,
|
||||||
|
),
|
||||||
|
socket_path: socket_path.clone(),
|
||||||
|
events,
|
||||||
|
subscriptions: AtomicUsize::new(0),
|
||||||
|
receivers_at_close: Mutex::new(Vec::new()),
|
||||||
|
});
|
||||||
|
worker.bind_single_workdir_session(Some(session.clone()));
|
||||||
|
let sessions = worker.workdir_sessions();
|
||||||
|
|
||||||
|
let result = tokio::time::timeout(
|
||||||
|
std::time::Duration::from_secs(5),
|
||||||
|
WorkerController::spawn_runtime_managed_run(
|
||||||
|
worker,
|
||||||
|
&run_dir,
|
||||||
|
&root.path().join("bash-output"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("socket startup and rollback must settle");
|
||||||
|
let error = match result {
|
||||||
|
Err(error) => error,
|
||||||
|
Ok(_) => panic!("a directory at the Unix socket path must reject controller startup"),
|
||||||
|
};
|
||||||
|
assert_eq!(error.kind(), std::io::ErrorKind::AddrInUse);
|
||||||
|
assert_eq!(session.subscriptions.load(Ordering::SeqCst), 1);
|
||||||
|
assert_eq!(*session.receivers_at_close.lock().unwrap(), vec![0]);
|
||||||
|
assert_eq!(session.events.receiver_count(), 0);
|
||||||
|
assert!(sessions.aliases().is_empty());
|
||||||
|
// No controller retains the Worker/Engine after the failed startup.
|
||||||
|
assert_eq!(Arc::strong_count(&client.call_count), 1);
|
||||||
|
assert_eq!(client.call_count.load(Ordering::SeqCst), 0);
|
||||||
|
assert!(socket_path.is_dir());
|
||||||
|
assert!(tokio::net::UnixStream::connect(&socket_path).await.is_err());
|
||||||
|
assert!(matches!(
|
||||||
|
session
|
||||||
|
.start_command(CommandRequest {
|
||||||
|
command: "printf unreachable".to_owned(),
|
||||||
|
timeout_secs: 5,
|
||||||
|
output_limit: 1024,
|
||||||
|
cwd: workdir::WorkdirPath::root(),
|
||||||
|
spill_dir: None,
|
||||||
|
tool_call_id: None,
|
||||||
|
})
|
||||||
|
.await,
|
||||||
|
Err(WorkdirError::Unavailable(_))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
|
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
|
||||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||||
loop {
|
loop {
|
||||||
|
|||||||
@@ -85,7 +85,10 @@ impl RuntimeWorkerFactory for CapturingFactory {
|
|||||||
async fn restore_controller(
|
async fn restore_controller(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
) -> std::result::Result<RuntimeWorkerController, String> {
|
) -> std::result::Result<
|
||||||
|
RuntimeWorkerController,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
let worker_ref = request.worker_ref.clone();
|
let worker_ref = request.worker_ref.clone();
|
||||||
let controller = self.inner.restore_controller(request).await?;
|
let controller = self.inner.restore_controller(request).await?;
|
||||||
self.captures
|
self.captures
|
||||||
|
|||||||
@@ -494,9 +494,18 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFac
|
|||||||
async fn restore_controller(
|
async fn restore_controller(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
|
) -> std::result::Result<
|
||||||
|
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
self.controller(request.worker_ref.worker_id.to_string(), true)
|
self.controller(request.worker_ref.worker_id.to_string(), true)
|
||||||
.await
|
.await
|
||||||
|
.map_err(|_| {
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
||||||
|
"attention fixture reconstruction failed",
|
||||||
|
)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -832,9 +832,18 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFact
|
|||||||
async fn restore_controller(
|
async fn restore_controller(
|
||||||
&self,
|
&self,
|
||||||
request: delivery_execution::WorkerExecutionRestoreRequest,
|
request: delivery_execution::WorkerExecutionRestoreRequest,
|
||||||
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
|
) -> std::result::Result<
|
||||||
|
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
self.controller(request.worker_ref.worker_id.to_string(), true)
|
self.controller(request.worker_ref.worker_id.to_string(), true)
|
||||||
.await
|
.await
|
||||||
|
.map_err(|_| {
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
||||||
|
"delivery fixture reconstruction failed",
|
||||||
|
)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -584,7 +584,10 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto
|
|||||||
async fn restore_controller(
|
async fn restore_controller(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
|
) -> std::result::Result<
|
||||||
|
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
self.inner.restore_controller(request).await
|
self.inner.restore_controller(request).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user