fix: preserve restore effects and fence prepared attachment replay

This commit is contained in:
2026-10-23 00:21:02 +09:00
parent 73a3e674c9
commit 907392cce8
4 changed files with 952 additions and 30 deletions
+104 -8
View File
@@ -2150,6 +2150,16 @@ impl Runtime {
{
return Err(conflict());
}
if let Err(error) = owner.validate_workdir_preparation_authority(&candidate) {
return Ok((
Some(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired,
"worker_restore_authority_changed",
sanitize_worker_create_failure_message(&error.to_string()),
)),
None,
));
}
} else {
// Startup/older persisted pending operations have a lifecycle
// owner but no public request receipt. Bind recovery to that
@@ -2215,12 +2225,18 @@ impl Runtime {
candidate.pending_restore = Some(pending);
candidate.restore_guard.rotate();
candidate.restore_guard.active_request_id = Some(request.request_id.clone());
let workdir_attachment_preparation_base = request
.preparation
.as_ref()
.and_then(|preparation| preparation.workdir_attachments.as_ref())
.map(|_| candidate.logical_workdir_attachments.clone());
candidate.restore_guard.owners.insert(
request.request_id.clone(),
RestoreRequestOwner {
fingerprint,
operation_id,
request,
workdir_attachment_preparation_base,
preparation_applied: recovering_pending,
execution_started: recovering_pending,
receipt: None,
@@ -2433,14 +2449,15 @@ impl Runtime {
{
return Ok(RuntimeWorkerRestoreResult::accepted(worker.detail()));
}
let preparation = worker
let owner = worker
.restore_guard
.active_request_id
.as_ref()
.and_then(|id| worker.restore_guard.owners.get(id))
.filter(|owner| !owner.preparation_applied)
.and_then(|owner| owner.request.preparation.as_ref());
let prepared_worker = match worker.prepared_restore_record(preparation) {
.and_then(|id| worker.restore_guard.owners.get(id));
let prepared_worker = match owner.map_or_else(
|| Ok(worker.clone()),
|owner| owner.prepared_worker_record(worker),
) {
Ok(prepared) => prepared,
Err(error) => {
return Ok(RuntimeWorkerRestoreResult::failed(
@@ -4398,8 +4415,7 @@ impl Runtime {
worker_id: worker_ref.worker_id,
});
}
let mut candidate =
previous.prepared_restore_record(owner.request.preparation.as_ref())?;
let mut candidate = owner.prepared_worker_record(previous)?;
candidate
.restore_guard
.owners
@@ -6277,12 +6293,48 @@ pub(crate) struct RestoreRequestOwner {
fingerprint: String,
operation_id: WorkerLifecycleOperationId,
request: runtime_api::WorkerRestoreRequest,
// This is the admission comparison input, not a second attachment authority.
// Without it an old preparation could hide a later canonical revocation.
// Older owners lacking this fact cannot safely reapply attachment overrides.
#[serde(default, skip_serializing_if = "Option::is_none")]
workdir_attachment_preparation_base: Option<Vec<LogicalWorkdirAttachment>>,
preparation_applied: bool,
#[serde(default)]
execution_started: bool,
receipt: Option<RuntimeWorkerRestoreResult>,
}
impl RestoreRequestOwner {
fn validate_workdir_preparation_authority(
&self,
worker: &WorkerRecord,
) -> Result<(), RuntimeError> {
if !self.preparation_applied
&& self
.request
.preparation
.as_ref()
.is_some_and(|preparation| preparation.workdir_attachments.is_some())
&& self.workdir_attachment_preparation_base.as_ref()
!= Some(&worker.logical_workdir_attachments)
{
return Err(RuntimeError::InvalidRequest(
"current Worker authority differs from admitted Restore attachment preparation"
.into(),
));
}
Ok(())
}
fn prepared_worker_record(&self, worker: &WorkerRecord) -> Result<WorkerRecord, RuntimeError> {
if self.preparation_applied {
return Ok(worker.clone());
}
self.validate_workdir_preparation_authority(worker)?;
worker.prepared_restore_record(self.request.preparation.as_ref())
}
}
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct RestoreGuard {
@@ -11995,7 +12047,7 @@ mod tests {
let mut request = runtime.test_restore_request(&worker.worker_ref);
request.preparation = Some(serde_json::from_value(serde_json::json!({
"workspace_api": { "workspace_id": "workspace-a", "base_url": "https://prepared.example" },
"workdir_attachments": [], "repository_access": []
"workdir_attachments": [{ "alias": "checkout", "working_directory_id": "prepared-logical-workdir" }], "repository_access": []
})).unwrap());
let gate = Arc::new(RestoreGate::default());
*backend.restore_gate.lock().unwrap() = Some(gate.clone());
@@ -12014,6 +12066,26 @@ mod tests {
let state = runtime.lock().unwrap();
let record = state.worker(&worker.worker_ref).unwrap();
assert!(record.request.as_ref().unwrap().workspace_api.is_none());
assert!(record.logical_workdir_attachments.is_empty());
let owner = record.restore_guard.owners.values().next().unwrap();
let mut saved_owner = serde_json::to_value(owner).unwrap();
let decoded: RestoreRequestOwner = serde_json::from_value(saved_owner.clone()).unwrap();
assert!(
decoded
.validate_workdir_preparation_authority(record)
.is_ok()
);
saved_owner
.as_object_mut()
.unwrap()
.remove("workdir_attachment_preparation_base");
let legacy: RestoreRequestOwner = serde_json::from_value(saved_owner).unwrap();
assert!(
legacy
.validate_workdir_preparation_authority(record)
.is_err(),
"a missing historical comparison input must not be invented from current authority"
);
assert!(
!record
.restore_guard
@@ -12033,6 +12105,11 @@ mod tests {
.base_url,
"https://prepared.example"
);
assert_eq!(
backend.restore_requests.lock().unwrap()[0].logical_workdir_attachments[0]
.working_directory_id,
"prepared-logical-workdir"
);
assert!(backend.activation_operation_ids.lock().unwrap().is_empty());
gate.release();
assert_eq!(
@@ -12053,6 +12130,25 @@ mod tests {
.base_url,
"https://prepared.example"
);
let record = state.worker(&worker.worker_ref).unwrap();
assert_eq!(
record.logical_workdir_attachments[0].working_directory_id,
"prepared-logical-workdir"
);
let owner = record.restore_guard.owners.values().next().unwrap();
assert!(owner.preparation_applied);
let mut revoked = record.clone();
revoked
.apply_logical_workdir_attachments(Vec::new())
.unwrap();
assert!(
owner
.prepared_worker_record(&revoked)
.unwrap()
.logical_workdir_attachments
.is_empty(),
"a committed preparation is not a perpetual grant"
);
}
#[cfg(feature = "ws-server")]
+52 -17
View File
@@ -360,20 +360,21 @@ async fn prepare_restore_operation<F: RuntimeWorkerFactory + ?Sized>(
failure
})?;
if repaired {
Ok(Box::new(RepairedRestoreCandidate { candidate }))
Ok(Box::new(EffectRetainingRestoreCandidate { candidate }))
} else {
Ok(candidate)
}
}
// A completed repair is a canonical effect even when later activation fails
// before its own mutation boundary. Carry that evidence with the resource owner.
struct RepairedRestoreCandidate {
// Completed repair or an earlier activation may have changed canonical state.
// Keep that operation-owned evidence through reconstruction and even an
// activation failure before the replacement candidate's own mutation boundary.
struct EffectRetainingRestoreCandidate {
candidate: Box<dyn RuntimeRestoreCandidate>,
}
#[async_trait]
impl RuntimeRestoreCandidate for RepairedRestoreCandidate {
impl RuntimeRestoreCandidate for EffectRetainingRestoreCandidate {
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
self.candidate.initial_state()
}
@@ -2953,6 +2954,9 @@ struct PendingRuntimeRestore {
task: tokio::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
result: Mutex<Option<Result<Box<dyn RuntimeRestoreCandidate>, RuntimeRestoreFailure>>>,
activated: Mutex<Option<Result<RuntimeWorkerController, RuntimeRestoreFailure>>>,
// Monotonic for this operation, not a claim that its current candidate has
// started. A reconstructed candidate has a new result and no completion in
// `activated`; prior activation/effect evidence must survive that handoff.
activation_started: AtomicBool,
discard_started: AtomicBool,
discard_result: Mutex<Option<Result<(), RuntimeRestoreCleanupFailure>>>,
@@ -2960,6 +2964,16 @@ struct PendingRuntimeRestore {
}
impl PendingRuntimeRestore {
fn owns_prepared_candidate(&self) -> bool {
matches!(
self.result
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_ref(),
Some(Ok(_))
)
}
async fn wait(&self, timeout: Duration) -> Result<(), String> {
let mut task = self.task.lock().await;
if let Some(message) = self
@@ -3279,23 +3293,40 @@ where
&& (failure.durable_effects_possible
|| owner.activation_started.load(Ordering::Acquire))
{
let prior_effects = failure.durable_effects_possible
|| owner.activation_started.load(Ordering::Acquire);
let result =
match prepare_restore_operation(factory.as_ref(), owner.request.clone())
.await
{
Ok(candidate) => Ok(candidate),
Err(failure) => Err(finish_restore_failure(
factory.as_ref(),
&owner.request,
failure,
)
.await),
Ok(candidate) => {
Ok(Box::new(EffectRetainingRestoreCandidate { candidate })
as Box<dyn RuntimeRestoreCandidate>)
}
Err(mut failure) => {
failure.durable_effects_possible |= prior_effects;
Err(finish_restore_failure(
factory.as_ref(),
&owner.request,
failure,
)
.await)
}
};
owner.activation_started.store(false, Ordering::Release);
// A completed reprepare replaces attempt resources, never
// the operation's effect history. Only a ready candidate
// may clear the old activation completion for a new attempt.
let activated = match &result {
Ok(_) => None,
Err(failure) if owner.activation_started.load(Ordering::Acquire) => {
Some(Err(failure.clone()))
}
Err(_) => None,
};
*owner
.activated
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
.unwrap_or_else(std::sync::PoisonError::into_inner) = activated;
*owner
.result
.lock()
@@ -4019,7 +4050,9 @@ where
failure.to_string(),
);
}
if pending.activation_started.load(Ordering::Acquire) {
if pending.activation_started.load(Ordering::Acquire)
&& !pending.owns_prepared_candidate()
{
let mut activated = match pending.activated.lock() {
Ok(result) => result,
Err(_) => {
@@ -5205,7 +5238,8 @@ where
};
}
}
if pending.activation_started.load(Ordering::Acquire) {
if pending.activation_started.load(Ordering::Acquire) && !pending.owns_prepared_candidate()
{
let activated = pending
.activated
.lock()
@@ -5330,7 +5364,8 @@ where
return Err("restored Worker operation identity does not match".into());
}
self.wait_pending_restore(&pending, self.spawn_restore_timeout)?;
if !pending.activation_started.swap(true, Ordering::AcqRel) {
if pending.owns_prepared_candidate() {
pending.activation_started.store(true, Ordering::Release);
let owner = pending.clone();
let factory = self.factory.clone();
let task = self.spawn_on_adapter_runtime(async move {
@@ -895,4 +895,210 @@ mod restore_candidate {
assert_eq!(model.call_count.load(Ordering::SeqCst), 0);
}
}
#[test]
#[serial_test::serial(worker_allocation)]
fn attachment_preparation_replay_cannot_hide_logical_authority_revocation() {
use crate::fs_store::{
FsRuntimeStoreOptions, PersistedWorkerExecution, PersistedWorkerExecutionState,
PersistedWorkerRecord,
};
let mut fixture = Fixture::new();
Arc::get_mut(&mut fixture.backend)
.unwrap()
.spawn_restore_timeout = Duration::from_secs(2);
let worker_ref = fixture.request.worker_ref.clone();
let scope = fixture.request.workspace_scope.clone().unwrap();
let options = FsRuntimeStoreOptions {
root: fixture._root.path().join("catalog"),
runtime_id: "candidate-runtime".into(),
display_name: None,
};
let attachment = crate::catalog::LogicalWorkdirAttachment {
alias: WorkdirAttachmentAlias::new("checkout").unwrap(),
working_directory_id: "workspace-only-workdir".into(),
capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY,
};
// Only the initial Runtime catalog is seeded. The saved Profile, queue,
// restore admission, revocation, replay and Stop use their real owners.
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
options.clone(),
fixture.backend.clone(),
)
.unwrap();
let create = fixture.request.request.clone();
runtime
.test_runtime_store()
.write_worker_record(&PersistedWorkerRecord {
restore_guard: crate::runtime::RestoreGuard::default(),
worker_ref: worker_ref.clone(),
worker_id: worker_ref.worker_id,
profile: create.profile.clone(),
display_name: None,
profile_source: create.profile_source.reference(),
config_bundle: create.config_bundle.clone(),
created_at_ms: None,
status: crate::catalog::WorkerStatus::Stopped,
last_finished_submission_request_id: None,
execution_state: PersistedWorkerExecutionState::Available(
PersistedWorkerExecution {
request: create,
binding: None,
restore_intent: crate::catalog::WorkerRestoreIntent::Explicit,
},
),
workspace_id: Some(scope.workspace_id.clone()),
workdir_attachments: Vec::new(),
logical_workdir_attachments: vec![attachment.clone()],
})
.unwrap();
drop(runtime);
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
options.clone(),
fixture.backend.clone(),
)
.unwrap();
assert!(
runtime
.worker_detail(&worker_ref)
.unwrap()
.workdir_attachments
.is_empty()
);
let before = fixture.canonical_files();
let authority = || -> serde_json::Value {
serde_json::from_slice(
&fs::read(
options
.root
.join("workers")
.join(worker_ref.worker_id.to_string())
.join("worker.json"),
)
.unwrap(),
)
.unwrap()
};
let release = Arc::new(tokio::sync::Notify::new());
let (entered_tx, entered) = mpsc::channel();
*fixture
.backend
.factory
.restore_faults
.factory_gate
.lock()
.unwrap() = Some(release.clone());
*fixture
.backend
.factory
.restore_faults
.factory_entered
.lock()
.unwrap() = Some(entered_tx);
let mut request = runtime.test_restore_request(&worker_ref);
request.preparation.as_mut().unwrap().workdir_attachments =
Some(serde_json::from_value(serde_json::to_value(vec![attachment]).unwrap()).unwrap());
let first = runtime
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
.unwrap();
entered.recv_timeout(WAIT).unwrap();
assert_eq!(
first.state,
server_api::WorkerRestoreState::ReconciliationRequired,
"{first:?}"
);
let pending = fixture
.backend
.pending_restore(&worker_ref)
.unwrap()
.unwrap();
assert!(!pending.request.logical_workdir_attachments.is_empty());
assert!(pending.request.previous_workdir_attachments.is_empty());
assert!(pending.request.deferred_workdir_sessions.is_empty());
runtime
.replace_worker_workdir_attachments_scoped(&scope, &worker_ref, Vec::new())
.unwrap();
assert!(authority().get("logical_workdir_attachments").is_none());
release.notify_one();
let replay = runtime
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
.unwrap();
assert_eq!(
replay.state,
server_api::WorkerRestoreState::ReconciliationRequired,
"revocation hidden by saved preparation: {replay:?}"
);
assert!(
replay
.message
.as_deref()
.unwrap_or_default()
.contains("current Worker authority differs"),
"{replay:?}"
);
let record = authority();
assert!(record.get("logical_workdir_attachments").is_none());
let owner = &record["restore_guard"]["owners"][&request.request_id];
assert_eq!(
owner["operation_id"],
serde_json::to_value(pending.request.operation_id).unwrap()
);
assert!(owner["receipt"].is_null());
assert_eq!(
owner["workdir_attachment_preparation_base"]
.as_array()
.unwrap()
.len(),
1
);
let loaded = runtime.test_runtime_store().load_runtime_state().unwrap();
let durable_guard =
serde_json::to_value(&loaded.workers[&worker_ref.worker_id].restore_guard).unwrap();
assert_eq!(durable_guard["owners"][&request.request_id], *owner);
let (coordinated, preparation) = runtime
.coordinate_worker_restore_operation(
&worker_ref,
runtime_api::WorkerRestoreCoordinationRequest {
expected_observation_token: request.expected_observation_token.clone(),
request_id: request.request_id.clone(),
preparation: None,
},
Some(&scope),
)
.unwrap();
assert_eq!(
coordinated.unwrap().state,
server_api::WorkerRestoreState::ReconciliationRequired
);
assert!(
preparation.is_none(),
"coordinator must not export the revoked preparation again"
);
assert!(fixture.backend.workers.lock().unwrap().is_empty());
assert!(Arc::ptr_eq(
&pending,
&fixture
.backend
.pending_restore(&worker_ref)
.unwrap()
.unwrap()
));
assert!(!pending.activation_started.load(Ordering::SeqCst));
fixture.assert_quiet();
assert_eq!(fixture.canonical_files(), before);
runtime
.stop_worker_scoped(&scope, &worker_ref, None)
.unwrap();
fixture.assert_released();
fixture.assert_quiet();
assert_eq!(fixture.canonical_files(), before);
assert!(authority().get("logical_workdir_attachments").is_none());
assert_ne!(
runtime
.restore_worker_operation_scoped(&scope, &worker_ref, request)
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
}
}
@@ -100,6 +100,148 @@ impl WorkingDirectoryMaterializer for RestoreOpenCountingMaterializer {
}
}
// Substitute only initial creation: a new aggregate/Session is left pending
// before its first head, without ever materializing or rewinding a saved head.
// All Restore preflight, repair, retry, activation and Stop use the real backend.
struct PendingProfileSeedBackend {
inner: Arc<WorkerRuntimeExecutionBackend>,
}
impl WorkerExecutionBackend for PendingProfileSeedBackend {
fn backend_id(&self) -> &str {
self.inner.backend_id()
}
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
let aggregate = self
.inner
.factory
.worker_aggregate_dir(&request.worker_ref)
.unwrap();
let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&request.worker_ref);
self.inner
.run_on_adapter_runtime(async move {
assert!(
!aggregate.join("session").exists(),
"pending fixture requires a new Session aggregate"
);
let manifest = manifest::WorkerManifest::from_toml(&format!(
r#"
[worker]
name = "{name}"
pwd = "./"
[scope]
allow = []
[model]
scheme = "anthropic"
model_id = "test-model"
auth = {{ kind = "none" }}
[engine]
max_tokens = 100
"#
))
.unwrap();
let sessions = WorkerSessionStore::new(aggregate.join("session")).unwrap();
let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap();
assert!(metadata.read_by_name(&name).unwrap().is_none());
let engine =
Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
MockClient::new(Vec::new()),
);
let mut head = Worker::new(
manifest.clone(),
engine,
CombinedStore::new(sessions.clone(), metadata.clone()),
WorkerWorkspaceContext::unavailable(
None,
"initial creation interrupted before first head",
),
WorkerFilesystemAuthority::None,
Scope::empty(),
)
.await
.unwrap();
head.enable_worker_metadata_write_through().unwrap();
let mut saved = metadata.read_by_name(&name).unwrap().unwrap();
let active = saved.active.as_ref().unwrap();
assert!(active.segment_id.is_none());
assert!(sessions.list_segments(active.session_id).unwrap().is_empty());
saved.resolved_manifest_snapshot =
Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap());
metadata.write(&saved).unwrap();
drop(head);
Ok(())
})
.unwrap();
WorkerExecutionSpawnResult::Connected {
worker_state: protocol::WorkerStateSnapshot::initial(),
workdir_attachments: Vec::new(),
}
}
fn fetch_workspace_config(
&self,
request: WorkspaceConfigFetchRequest,
) -> Result<WorkspaceConfigFetchResult, String> {
self.inner.fetch_workspace_config(request)
}
fn observe_workspace_prompt_projection(
&self,
projection: worker::WorkspacePromptProjection,
) -> Result<(), String> {
self.inner.observe_workspace_prompt_projection(projection)
}
fn preflight_restore(
&self,
request: &WorkerExecutionRestoreRequest,
) -> Result<(), WorkerExecutionResult> {
self.inner.preflight_restore(request)
}
fn completed_restore_failure(
&self,
operation_id: crate::execution::WorkerLifecycleOperationId,
worker_ref: &WorkerRef,
) -> Option<WorkerExecutionSpawnResult> {
self.inner
.completed_restore_failure(operation_id, worker_ref)
}
fn restore_worker(&self, request: WorkerExecutionRestoreRequest) -> WorkerExecutionSpawnResult {
self.inner.restore_worker(request)
}
fn reconcile_restore(
&self,
request: WorkerExecutionRestoreRequest,
) -> WorkerExecutionSpawnResult {
self.inner.reconcile_restore(request)
}
fn activate_restored_worker(
&self,
operation_id: crate::execution::WorkerLifecycleOperationId,
worker_ref: &WorkerRef,
) -> Result<(), String> {
self.inner
.activate_restored_worker(operation_id, worker_ref)
}
fn dispatch_input(&self, worker_ref: &WorkerRef, input: WorkerInput) -> WorkerExecutionResult {
self.inner.dispatch_input(worker_ref, input)
}
fn dispatch_method(&self, worker_ref: &WorkerRef, method: Method) -> WorkerExecutionResult {
self.inner.dispatch_method(worker_ref, method)
}
fn stop_worker_operation(&self, request: WorkerExecutionStopRequest) -> WorkerExecutionResult {
self.inner.stop_worker_operation(request)
}
}
struct ProfileFailureFixture {
runtime: EmbeddedRuntime,
backend: Arc<WorkerRuntimeExecutionBackend>,
@@ -115,6 +257,14 @@ impl ProfileFailureFixture {
}
fn with_saved_inputs(timeout: Duration, model: Option<MockClient>) -> Self {
Self::with_seed(timeout, model, false)
}
fn with_pending_head(timeout: Duration) -> Self {
Self::with_seed(timeout, None, true)
}
fn with_seed(timeout: Duration, model: Option<MockClient>, pending_head: bool) -> Self {
let root = restore_failure_tempdir();
let repo = root.path().join("g");
fs::create_dir(&repo).unwrap();
@@ -141,22 +291,31 @@ impl ProfileFailureFixture {
opens: provider_opens.clone(),
}),
);
let execution: Arc<dyn WorkerExecutionBackend> = if pending_head {
Arc::new(PendingProfileSeedBackend {
inner: backend.clone(),
})
} else {
backend.clone()
};
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
crate::fs_store::FsRuntimeStoreOptions {
root: store,
runtime_id: "r".into(),
display_name: None,
},
backend.clone(),
execution,
)
.unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let mut request = create_request("restore failure recovery");
request.profile = ProfileSelector::Builtin("default".into());
request.workdir_attachment_requests = vec![WorkingDirectoryAttachmentRequest {
alias: WorkdirAttachmentAlias::new("checkout").unwrap(),
working_directory: working_directory_request(&repo),
}];
if !pending_head {
request.workdir_attachment_requests = vec![WorkingDirectoryAttachmentRequest {
alias: WorkdirAttachmentAlias::new("checkout").unwrap(),
working_directory: working_directory_request(&repo),
}];
}
let worker = runtime.create_worker(request).unwrap();
let seed_handle = queue_inputs.then(|| {
backend.workers.lock().unwrap()[&worker.worker_ref]
@@ -203,6 +362,127 @@ impl ProfileFailureFixture {
*self.backend.factory.restore_faults.fail_at.lock().unwrap() = Some(stage);
}
fn pin_unclaimed_restore_upload(&self) -> (PathBuf, BTreeMap<PathBuf, Vec<u8>>) {
let aggregate = self
.backend
.factory
.worker_aggregate_dir(&self.worker.worker_ref)
.unwrap();
let name =
ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&self.worker.worker_ref);
let metadata = WorkerAggregateStore::open_read_only(&aggregate, name.clone()).unwrap();
let session_id = metadata
.read_by_name(&name)
.unwrap()
.unwrap()
.active
.unwrap()
.session_id;
let session = WorkerSessionStore::new(aggregate.join("session")).unwrap();
let orphan = session
.write_uploaded_file(
session_id,
"orphan.txt",
"text/plain",
b"restore effect evidence",
session_store::UploadedFileLimits::default(),
)
.unwrap();
session
.pin_uploaded_file(session_id, &orphan, "uncommitted-restore-input")
.unwrap();
let artifacts = aggregate.join("session/artifacts/paste");
let pinned = persisted_files(&artifacts);
assert!(!pinned.is_empty());
(artifacts, pinned)
}
fn assert_retained_restore_effects(
&self,
request: &runtime_api::WorkerRestoreRequest,
pending: &Arc<PendingRuntimeRestore>,
stage: RuntimeRestoreStage,
) {
let retained = self
.backend
.pending_restore(&self.worker.worker_ref)
.unwrap()
.expect("resource cleanup must not remove the operation's effect owner");
assert!(Arc::ptr_eq(pending, &retained));
{
let result = retained.result.lock().unwrap();
let failure = result.as_ref().unwrap().as_ref().err().unwrap();
assert_eq!(failure.stage, stage);
assert_eq!(failure.cleanup, RuntimeRestoreCleanup::Confirmed);
assert!(
failure.durable_effects_possible,
"resource cleanup cannot erase prior canonical effects: {failure:?}"
);
}
let record: serde_json::Value = serde_json::from_slice(
&fs::read(
self.root
.path()
.join("s/workers")
.join(self.worker.worker_ref.worker_id.to_string())
.join("worker.json"),
)
.unwrap(),
)
.unwrap();
assert_eq!(
record["restore_guard"]["active_request_id"],
request.request_id
);
let operation = serde_json::to_value(pending.request.operation_id).unwrap();
let owner = &record["restore_guard"]["owners"][&request.request_id];
assert_eq!(owner["operation_id"], operation);
assert!(
owner["receipt"].is_null(),
"false terminal receipt: {owner:?}"
);
assert_eq!(
record["execution_state"]["state"],
"reconciliation_required"
);
assert_eq!(
record["execution_state"]["execution"]["operation"],
"restore"
);
assert_eq!(
record["execution_state"]["execution"]["intent"]["operation_id"],
operation
);
assert!(
self.backend
.completed_restore_failures
.lock()
.unwrap()
.is_empty()
);
assert!(self.backend.workers.lock().unwrap().is_empty());
assert!(
self.backend
.factory
.failed_restore_resources
.lock()
.unwrap()
.is_empty(),
"cleanup confirmed resources, not rollback of canonical effects"
);
assert!(
self.backend
.factory
.observation_hub
.get(&self.worker.worker_ref)
.is_none()
);
if !pending.request.deferred_workdir_sessions.is_empty() {
self.assert_closed_deferred_sessions(pending);
}
assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0);
}
fn assert_released(&self) {
assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0);
assert!(self.backend.pending_restores.lock().unwrap().is_empty());
@@ -422,6 +702,311 @@ impl ProfileFailureFixture {
}
}
#[test]
#[serial_test::serial(worker_allocation)]
fn profile_restore_prior_effects_survive_read_only_reprepare_failure() {
for (activation_stage, read_stage) in [
(RuntimeRestoreStage::Feature, RuntimeRestoreStage::Metadata),
(
RuntimeRestoreStage::Controller,
RuntimeRestoreStage::Reconstruction,
),
] {
let fixture = ProfileFailureFixture::new(Duration::from_secs(5));
let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload();
fixture.fail(activation_stage);
let request = fixture
.runtime
.test_restore_request(&fixture.worker.worker_ref);
let first = fixture
.runtime
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
.unwrap();
assert_eq!(
first.state,
server_api::WorkerRestoreState::ReconciliationRequired,
"{activation_stage:?}: {first:?}"
);
let pending = fixture
.backend
.pending_restore(&fixture.worker.worker_ref)
.unwrap()
.unwrap();
assert!(pending.activation_started.load(Ordering::SeqCst));
fixture.assert_retained_restore_effects(&request, &pending, activation_stage);
let effected = persisted_files(&artifacts);
assert_ne!(
effected, pinned,
"activation must actually reconcile the orphan pin"
);
// The next attempt needs no repair, and fails before any new mutation.
// Resource cleanup is confirmed, but the first attempt's pin change
// was not rolled back. Repeated sends must retain the same owner.
for _ in 0..2 {
fixture.fail(read_stage);
let replay = fixture
.runtime
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
.unwrap();
assert_eq!(
replay.state,
server_api::WorkerRestoreState::ReconciliationRequired,
"{activation_stage:?} then {read_stage:?}: {replay:?}"
);
fixture.assert_retained_restore_effects(&request, &pending, read_stage);
assert!(pending.activation_started.load(Ordering::SeqCst));
assert_eq!(persisted_files(&artifacts), effected);
}
fixture.resume_same_operation(request, pending.request.operation_id);
}
}
#[test]
#[serial_test::serial(worker_allocation)]
fn profile_restore_prior_effects_from_pending_head_repair_survive_read_only_reprepare_failure() {
let fixture = ProfileFailureFixture::with_pending_head(Duration::from_secs(5));
let aggregate = fixture
.backend
.factory
.worker_aggregate_dir(&fixture.worker.worker_ref)
.unwrap();
let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&fixture.worker.worker_ref);
let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap();
let initial = metadata.read_by_name(&name).unwrap().unwrap();
let pending_session = initial.active.as_ref().unwrap().session_id;
assert!(initial.active.as_ref().unwrap().segment_id.is_none());
{
let before = WorkerSessionStore::new(aggregate.join("session")).unwrap();
assert!(before.list_sessions().unwrap().is_empty());
assert!(before.list_segments(pending_session).unwrap().is_empty());
}
// The independent new Worker has never published a head. Its first segment
// and active pointer must be created by production repair, not fixture
// rewinding/deletion of a materialized Session.
fixture.fail(RuntimeRestoreStage::Metadata);
let request = fixture
.runtime
.test_restore_request(&fixture.worker.worker_ref);
let first = fixture
.runtime
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
.unwrap();
assert_eq!(
first.state,
server_api::WorkerRestoreState::ReconciliationRequired,
"{first:?}"
);
let pending = fixture
.backend
.pending_restore(&fixture.worker.worker_ref)
.unwrap()
.unwrap();
assert!(!pending.activation_started.load(Ordering::SeqCst));
assert!(pending.request.deferred_workdir_sessions.is_empty());
fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Metadata);
let failure = pending
.result
.lock()
.unwrap()
.as_ref()
.unwrap()
.as_ref()
.err()
.unwrap()
.clone();
let repaired = WorkerAggregateStore::open_read_only(&aggregate, &name)
.unwrap()
.read_by_name(&name)
.unwrap()
.unwrap();
eprintln!(
"pending-head repair first={first:?} failure={failure:?} active={:?}",
repaired.active
);
let active = repaired
.active
.as_ref()
.expect("real repair must retain the pending Session's active pointer");
assert_eq!(active.session_id, pending_session);
// Store handles opened before the first Manifest retain session_id=None;
// they cannot observe that another owner has since published this Session.
// Reopen read-only to require the canonical Manifest as well as its logs.
let session = WorkerSessionStore::open_read_only(aggregate.join("session"))
.unwrap_or_else(|error| {
panic!(
"repair Manifest reread failed: {error}; first={first:?}; failure={failure:?}; active={active:?}"
)
});
assert_eq!(session.list_sessions().unwrap(), vec![pending_session]);
let segments = session.list_segments(pending_session).unwrap();
assert_eq!(
segments.len(),
1,
"real repair must commit exactly one seed; failure={failure:?}; active={active:?}"
);
assert_eq!(active.segment_id, Some(segments[0]));
let entries = session
.read_all_read_only(pending_session, segments[0])
.unwrap();
assert!(matches!(
entries.first(),
Some(LogEntry::AnnotatedSegmentStart { .. })
));
assert!(
session_store::collect_state(&entries)
.system_prompt
.is_some()
);
assert_ne!(
repaired, initial,
"repair must publish the first active head"
);
let repaired_session = persisted_files(&aggregate.join("session"));
for _ in 0..2 {
fixture.fail(RuntimeRestoreStage::Reconstruction);
let replay = fixture
.runtime
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
.unwrap();
assert_eq!(
replay.state,
server_api::WorkerRestoreState::ReconciliationRequired,
"repair effect cannot become a rollback on the next read failure: {replay:?}"
);
fixture.assert_retained_restore_effects(
&request,
&pending,
RuntimeRestoreStage::Reconstruction,
);
let current_metadata = WorkerAggregateStore::open_read_only(&aggregate, &name).unwrap();
assert_eq!(
current_metadata.read_by_name(&name).unwrap().unwrap(),
repaired
);
let current_session =
WorkerSessionStore::open_read_only(aggregate.join("session")).unwrap();
assert_eq!(
current_session.list_sessions().unwrap(),
vec![pending_session]
);
assert_eq!(
current_session.list_segments(pending_session).unwrap(),
segments
);
assert_eq!(
persisted_files(&aggregate.join("session")),
repaired_session
);
}
fixture.resume_same_operation(request, pending.request.operation_id);
}
#[test]
#[serial_test::serial(worker_allocation)]
fn profile_restore_prior_effects_survive_rebuilt_candidate_pre_effect_activation_failure() {
let fixture = ProfileFailureFixture::new(Duration::from_secs(5));
let (artifacts, pinned) = fixture.pin_unclaimed_restore_upload();
fixture.fail(RuntimeRestoreStage::Feature);
let request = fixture
.runtime
.test_restore_request(&fixture.worker.worker_ref);
let first = fixture
.runtime
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
.unwrap();
assert_eq!(
first.state,
server_api::WorkerRestoreState::ReconciliationRequired,
"{first:?}"
);
let pending = fixture
.backend
.pending_restore(&fixture.worker.worker_ref)
.unwrap()
.unwrap();
fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Feature);
let effected = persisted_files(&artifacts);
assert_ne!(effected, pinned);
let release = Arc::new(tokio::sync::Notify::new());
let (entered_tx, entered) = mpsc::channel();
*fixture
.backend
.factory
.restore_faults
.factory_gate
.lock()
.unwrap() = Some(release.clone());
*fixture
.backend
.factory
.restore_faults
.factory_entered
.lock()
.unwrap() = Some(entered_tx);
let runtime = fixture.runtime.clone();
let worker_ref = fixture.worker.worker_ref.clone();
let resend = request.clone();
let retry = std::thread::spawn(move || runtime.restore_worker_operation(&worker_ref, resend));
let reached = entered.recv_timeout(Duration::from_secs(5));
// This gate is after real read-only reconstruction. Change only saved
// metadata, so the rebuilt candidate's first activation revalidation fails
// before allocation/pin/metadata/delegation mutation. No factory or repair
// implementation is replaced by a fake.
let aggregate = fixture
.backend
.factory
.worker_aggregate_dir(&fixture.worker.worker_ref)
.unwrap();
let name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&fixture.worker.worker_ref);
let metadata = WorkerAggregateStore::new(&aggregate, &name).unwrap();
let mut changed = metadata.read_by_name(&name).unwrap().unwrap();
changed.workspace_root = Some(fixture.root.path().join("changed-after-reconstruction"));
metadata.write(&changed).unwrap();
release.notify_one();
let rebuilt = retry.join().unwrap().unwrap();
reached.expect("same-operation retry must reach the real reconstructed candidate");
assert_eq!(
rebuilt.state,
server_api::WorkerRestoreState::ReconciliationRequired,
"{rebuilt:?}"
);
assert_eq!(
fixture
.backend
.factory
.restore_faults
.executions
.load(Ordering::SeqCst),
2
);
assert!(pending.activation_started.load(Ordering::SeqCst));
fixture.assert_retained_restore_effects(
&request,
&pending,
RuntimeRestoreStage::Reconstruction,
);
assert_eq!(metadata.read_by_name(&name).unwrap().unwrap(), changed);
assert_eq!(persisted_files(&artifacts), effected);
fixture.fail(RuntimeRestoreStage::Metadata);
let replay = fixture
.runtime
.restore_worker_operation(&fixture.worker.worker_ref, request.clone())
.unwrap();
assert_eq!(
replay.state,
server_api::WorkerRestoreState::ReconciliationRequired,
"successful reconstruction must not erase the older effect: {replay:?}"
);
fixture.assert_retained_restore_effects(&request, &pending, RuntimeRestoreStage::Metadata);
assert_eq!(metadata.read_by_name(&name).unwrap().unwrap(), changed);
assert_eq!(persisted_files(&artifacts), effected);
fixture.resume_same_operation(request, pending.request.operation_id);
}
#[test]
#[serial_test::serial(worker_allocation)]
fn profile_restore_timeout_attachment_revocation_cannot_reactivate_old_candidate() {