test: synchronize owned runtime crashes at restore decision checkpoints

tests/e2e/Cargo.toml

@@ -10,6 +10,7 @@ autotests = false
 default = []
 e2e = []
 fixture-support = ["yoi-workspace-server/test-support", "worker-runtime/test-support"]
+restore-checkpoints = ["fixture-support", "worker-runtime/restore-checkpoints"]
 
 [dependencies]
 libc.workspace = true

tests/e2e/README.md

@@ -39,6 +39,24 @@ process終了ケースはそのAccepted receipt確認後に環境が所有する
 
 これは**commit済みかつterminal receipt保存後**のprocess終了であり、未commit候補やcommit直後・receipt前の終了証明ではない。固定Codex資産へSubmit/Notifyを送らないため、通常入力・Paused admissionの製品E2Eの証明でもない。検証jobはこの明示targetを選択する必要があり、通常suiteだけでは代替しない。
 
+## Opt-in Runtime checkpoint process試験
+
+上のfeatureless製品4case（target内の既存seed projection試験1件とは別）に加え、`restore-checkpoints`を明示した実Runtimeで2つのlive windowを試験する。このfeatureはoffline fixture-supportと独立しており、通常RuntimeではCLI入口も環境変数による入口も無い。
+
+```sh
+# 同期を有効にしたRuntimeだけを専用targetへ。通常製品binaryを上書きしない。
+cargo build --locked -p worker-runtime --bin yoi-runtime \
+  --features restore-checkpoints --target-dir target/e2e-checkpoint-products
+cargo test --locked -p yoi-e2e --features e2e,restore-checkpoints \
+  --test worker_restore checkpoint:: -- --test-threads=1
+```
+
+`WorkerE2eEnvironment::use_checkpoint_runtime`はその環境のRuntime binary/hashと、隔離root内の明示configだけを選択する。global provider・環境変数・通常起動への暗黙設定は変更しない。configは一度だけCLIへ渡し、再起動時は渡さない。test側のlink featureをfeatureless製品binaryへ混入させない。
+
+共有harnessはloopback ackのstage/Worker/operation/PID/nonceを確認し、直ちに所有childをkill/reapする。**offline正本を確認してから**同roots/identity/trustを再起動するため、ack後のdeadlineとの競合を成功扱いしない。NotCommittedの実私有候補は同operationのcleanup-only→RolledBack/Stopped、Committed/preterminalは同じdecision→実protocol/full Session snapshot→Acceptedへ回収する。元のrequest/token、owner数1、同operation、stale fresh intent拒否、保存source hash、restore run path数、最終cleanupを検証する。JSONでpending状態を作らず、seedは初回のみ。
+
+明示解除・peer EOF・observer scopeの意図的panicも試験する。gateの8秒deadlineとscoped HTTP callの15秒deadlineが待機を制限し、通常operationの継続後に環境のRAIIがprocess/rootを回収する。これはtest scope失敗の回収証明であり、OSによるharness強制終了や電源断の一般的な保証ではない。原fixtureへのSubmit/Notifyや実モデル呼出しは行わない。checkpoint ack/at-death/recovery、kill/reap、binary/hashとcleanup artifactはfeaturelessログと分けて報告する。
+
 通常のharness回帰試験（製品processは起動しない）:
 
 ```sh

tests/e2e/src/worker_environment.rs

@@ -266,6 +266,8 @@ pub struct WorkerE2eEnvironment {
     access_token: Option<String>,
     client: Client,
     binaries: WorkerBinaryProviderInfo,
+    #[cfg(feature = "restore-checkpoints")]
+    restore_checkpoint_config: Option<PathBuf>,
 }
 
 impl WorkerE2eEnvironment {
@@ -331,11 +333,49 @@ impl WorkerE2eEnvironment {
             access_token: None,
             client,
             binaries,
+            #[cfg(feature = "restore-checkpoints")]
+            restore_checkpoint_config: None,
         };
         environment.write_metadata("created")?;
         Ok(environment)
     }
 
+    /// Select the separately built test-enabled Runtime for this environment
+    /// only. The checkpoint file must already belong to its isolated root.
+    #[cfg(feature = "restore-checkpoints")]
+    pub fn use_checkpoint_runtime(&mut self, binary: PathBuf, config: &Path) -> Result<()> {
+        if self.runtime.is_some() || self.server.is_some() {
+            return Err(HarnessError::Protocol(
+                "checkpoint selection requires an unstarted environment".into(),
+            ));
+        }
+        let config = config.canonicalize()?;
+        if !config.starts_with(self.root.canonicalize()?) || !config.is_file() {
+            return Err(HarnessError::Protocol(
+                "checkpoint config is outside the owned environment".into(),
+            ));
+        }
+        let record = binary_record(
+            "yoi-runtime",
+            binary.canonicalize()?,
+            "restore-checkpoints-opt-in".into(),
+            None,
+        )?;
+        let runtime = self
+            .binaries
+            .binaries
+            .iter_mut()
+            .find(|record| record.name == "yoi-runtime")
+            .ok_or_else(|| HarnessError::Protocol("missing Runtime binary record".into()))?;
+        *runtime = record;
+        self.restore_checkpoint_config = Some(config);
+        fs::write(
+            self.artifacts_dir.join("binary-provider.json"),
+            serde_json::to_vec_pretty(&self.binaries)?,
+        )?;
+        Ok(())
+    }
+
     pub fn start(&mut self) -> Result<()> {
         self.start_backend()?;
         self.start_runtime()
@@ -639,6 +679,10 @@ impl WorkerE2eEnvironment {
             "--fs-root",
             self.runtime_data_dir.to_string_lossy().as_ref(),
         ]);
+        #[cfg(feature = "restore-checkpoints")]
+        if let Some(config) = self.restore_checkpoint_config.take() {
+            command.arg("--test-restore-checkpoint").arg(config);
+        }
         let runtime = ManagedProcess::spawn(
             "runtime",
             command,
@@ -684,6 +728,13 @@ impl WorkerE2eEnvironment {
     /// Kill/reap only the Runtime child owned by this environment, retaining roots
     /// and identities. Restart neither seeds storage nor rotates restore requests.
     pub fn crash_and_restart_runtime(&mut self) -> Result<()> {
+        self.crash_runtime()?;
+        self.start_runtime_process()
+    }
+
+    /// Leave the owned Runtime reaped and offline for read-only canonical crash
+    /// inspection before restart. No root, identity or request is recreated.
+    pub fn crash_runtime(&mut self) -> Result<()> {
         let runtime = self
             .runtime
             .as_mut()
@@ -708,6 +759,17 @@ impl WorkerE2eEnvironment {
         }
         self.runtime_url = None;
         self.runtime_shutdown = WorkerProcessShutdownOutcome::NotStarted;
+        Ok(())
+    }
+
+    /// Restart the previously initialized identity without seed or checkpoint flags.
+    #[cfg(feature = "restore-checkpoints")]
+    pub fn restart_runtime_after_crash(&mut self) -> Result<()> {
+        if self.runtime.is_some() || self.runtime_public_identity.is_none() {
+            return Err(HarnessError::Protocol(
+                "Runtime is not an offline initialized child".into(),
+            ));
+        }
         self.start_runtime_process()
     }
 
@@ -794,6 +856,37 @@ impl WorkerE2eEnvironment {
         )
     }
 
+    /// Build one authenticated call that may run in a scoped test thread while
+    /// the owner observes a checkpoint. The HTTP request deadline bounds joining
+    /// that thread even on test failure; credentials never enter artifacts.
+    #[cfg(feature = "restore-checkpoints")]
+    pub fn pending_restore_call(
+        &self,
+        worker_id: &str,
+        request_id: &str,
+        expected_observation_token: &str,
+    ) -> Result<impl FnOnce() -> Result<Value> + Send + 'static + use<>> {
+        let token = self.access_token.as_deref().ok_or_else(|| {
+            HarnessError::Protocol("Backend access token is not initialized".into())
+        })?;
+        let request = self
+            .client
+            .post(self.worker_url(worker_id, "restore")?)
+            .bearer_auth(token)
+            .timeout(STARTUP_WAIT)
+            .json(&serde_json::json!({"request_id": request_id,
+                "expected_observation_token": expected_observation_token}))
+            .build()
+            .map_err(|_| HarnessError::Protocol("build pending restore request".into()))?;
+        let client = self.client.clone();
+        Ok(move || {
+            let response = client
+                .execute(request)
+                .map_err(|_| HarnessError::Protocol("pending restore transport failed".into()))?;
+            decode_api_response(response, "pending restore")
+        })
+    }
+
     /// Forward one unchanged public restore request through an owned loopback
     /// relay, consume its complete upstream response, and close without delivering
     /// any response bytes. The returned JSON is relay evidence, not caller success.

tests/e2e/tests/worker_restore.rs

@@ -1,4 +1,7 @@
 //! Fixed, externally supplied Worker restore. Never generates a fixture or sends model input.
+#[cfg(feature = "restore-checkpoints")]
+#[path = "worker_restore/checkpoint.rs"]
+mod checkpoint;
 #[path = "worker_restore/fixture.rs"]
 mod fixture;
 #[path = "worker_restore/seed.rs"]

tests/e2e/tests/worker_restore/checkpoint.rs

@@ -0,0 +1,496 @@
+//! Opt-in real-process checkpoint cases; never substitute the featureless lane.
+use super::*;
+use std::collections::BTreeMap;
+use std::io::{Read, Write};
+use std::net::{TcpListener, TcpStream};
+use std::path::{Path, PathBuf};
+use worker_runtime::restore_checkpoints::{
+    MAX_RESTORE_CHECKPOINT_ACK_BYTES, RestoreCheckpointAck, RestoreCheckpointCommit,
+    RestoreCheckpointConfig, RestoreCheckpointStage,
+};
+
+struct Peer {
+    listener: TcpListener,
+    config: RestoreCheckpointConfig,
+    connection: Option<TcpStream>,
+}
+
+impl Peer {
+    fn configure(
+        environment: &mut WorkerE2eEnvironment,
+        fixture: &Fixture,
+        stage: RestoreCheckpointStage,
+    ) -> Result<Self> {
+        let listener = TcpListener::bind("127.0.0.1:0")?;
+        listener.set_nonblocking(true)?;
+        let config = RestoreCheckpointConfig {
+            endpoint: listener.local_addr()?,
+            worker_id: fixture::support(fixture.manifest.worker_id.parse())?,
+            stage,
+            nonce: format!(
+                "owned-checkpoint-{}-{}",
+                std::process::id(),
+                listener.local_addr()?.port()
+            ),
+        };
+        let path = environment.root.join("restore-checkpoint.json");
+        fs::write(&path, serde_json::to_vec(&config)?)?;
+        let provider = yoi_e2e::worker_binary_provider_info()?;
+        let binary = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+            .join("../../target/e2e-checkpoint-products")
+            .join(provider.profile)
+            .join("yoi-runtime");
+        environment.use_checkpoint_runtime(binary, &path)?;
+        Ok(Self {
+            listener,
+            config,
+            connection: None,
+        })
+    }
+
+    fn acknowledge(&mut self, environment: &WorkerE2eEnvironment) -> Result<RestoreCheckpointAck> {
+        let deadline = Instant::now() + Duration::from_secs(15);
+        let (mut stream, address) = loop {
+            match self.listener.accept() {
+                Ok(value) => break value,
+                Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
+                    if Instant::now() >= deadline {
+                        return Err(failure("checkpoint ack deadline exceeded"));
+                    }
+                    std::thread::sleep(Duration::from_millis(10));
+                }
+                Err(error) => return Err(error.into()),
+            }
+        };
+        if !address.ip().is_loopback() {
+            return Err(failure("non-loopback checkpoint peer"));
+        }
+        let mut bytes = Vec::new();
+        loop {
+            let remaining = deadline.saturating_duration_since(Instant::now());
+            if remaining.is_zero() {
+                return Err(failure("checkpoint ack read deadline exceeded"));
+            }
+            stream.set_read_timeout(Some(remaining))?;
+            let mut byte = [0];
+            stream.read_exact(&mut byte)?;
+            if byte == [b'\n'] {
+                break;
+            }
+            bytes.push(byte[0]);
+            if bytes.len() > MAX_RESTORE_CHECKPOINT_ACK_BYTES {
+                return Err(failure("checkpoint ack oversized"));
+            }
+        }
+        let ack: RestoreCheckpointAck = serde_json::from_slice(&bytes)?;
+        if ack.nonce != self.config.nonce
+            || ack.stage != self.config.stage
+            || ack.worker_id != self.config.worker_id
+            || ack.runtime_id != environment.runtime_id
+            || Some(ack.pid) != environment.runtime_pid()
+            || !ack.receipt_absent
+        {
+            return Err(failure(
+                "checkpoint ack does not match owned process and stage",
+            ));
+        }
+        fs::write(
+            environment.artifacts_dir.join("checkpoint-ack.json"),
+            serde_json::to_vec_pretty(&ack)?,
+        )?;
+        self.connection = Some(stream);
+        Ok(ack)
+    }
+
+    fn release(&mut self) -> Result<()> {
+        let mut stream = self
+            .connection
+            .take()
+            .ok_or_else(|| failure("checkpoint not acknowledged"))?;
+        stream.set_write_timeout(Some(Duration::from_secs(1)))?;
+        stream.write_all(b"R")?;
+        Ok(())
+    }
+}
+
+fn aggregate_path(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> PathBuf {
+    environment
+        .runtime_data_dir
+        .join("runtime/workers")
+        .join(&fixture.manifest.worker_id)
+}
+
+fn canonical(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> Result<serde_json::Value> {
+    let mut bytes = Vec::new();
+    fs::File::open(aggregate_path(environment, fixture).join("worker.json"))?
+        .take(8 * 1024 * 1024 + 1)
+        .read_to_end(&mut bytes)?;
+    if bytes.len() > 8 * 1024 * 1024 {
+        return Err(failure("canonical aggregate oversized"));
+    }
+    Ok(serde_json::from_slice(&bytes)?)
+}
+
+fn saved_source_hashes(
+    environment: &WorkerE2eEnvironment,
+    fixture: &Fixture,
+) -> Result<BTreeMap<String, String>> {
+    fn visit(root: &Path, path: &Path, files: &mut BTreeMap<String, String>) -> Result<()> {
+        if files.len() > 128 {
+            return Err(failure("saved source file budget exceeded"));
+        }
+        let metadata = fs::symlink_metadata(path)?;
+        if metadata.file_type().is_symlink() {
+            return Err(failure("unexpected saved source symlink"));
+        }
+        if metadata.is_dir() {
+            for entry in fs::read_dir(path)? {
+                visit(root, &entry?.path(), files)?;
+            }
+        } else if metadata.is_file() {
+            if metadata.len() > 8 * 1024 * 1024 {
+                return Err(failure("saved source byte budget exceeded"));
+            }
+            use sha2::{Digest, Sha256};
+            let digest: String = Sha256::digest(fs::read(path)?)
+                .iter()
+                .map(|byte| format!("{byte:02x}"))
+                .collect();
+            files.insert(
+                path.strip_prefix(root)
+                    .unwrap()
+                    .to_string_lossy()
+                    .into_owned(),
+                digest,
+            );
+        } else {
+            return Err(failure("unexpected saved source entry type"));
+        }
+        Ok(())
+    }
+    let root = aggregate_path(environment, fixture);
+    let mut files = BTreeMap::new();
+    for name in ["metadata.json", "session"] {
+        visit(&root, &root.join(name), &mut files)?;
+    }
+    if files.len() < 2 {
+        return Err(failure("saved source inventory is incomplete"));
+    }
+    Ok(files)
+}
+
+fn restore_runs(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> Result<Vec<String>> {
+    let runs = aggregate_path(environment, fixture).join("runs");
+    if !runs.exists() {
+        return Ok(Vec::new());
+    }
+    let mut names = Vec::new();
+    for entry in fs::read_dir(runs)? {
+        let entry = entry?;
+        if entry.file_name().to_string_lossy().starts_with("restore-") {
+            if !entry.file_type()?.is_dir() {
+                return Err(failure("restore run is not a directory"));
+            }
+            names.push(entry.file_name().to_string_lossy().into_owned());
+        }
+    }
+    names.sort();
+    Ok(names)
+}
+
+#[test]
+fn acknowledged_not_committed_candidate_process_death_recovers_same_request_as_rollback()
+-> Result<()> {
+    crash_window(RestoreCheckpointStage::PreparedNotCommitted)
+}
+
+#[test]
+fn acknowledged_committed_preterminal_process_death_reconstructs_same_decision() -> Result<()> {
+    crash_window(RestoreCheckpointStage::CommittedPreterminal)
+}
+
+fn crash_window(stage: RestoreCheckpointStage) -> Result<()> {
+    let fixture = Fixture::load()?;
+    let result = (|| {
+        let mut environment = WorkerE2eEnvironment::new()?;
+        let mut peer = Peer::configure(&mut environment, &fixture, stage)?;
+        let mut installed = seed::start_in(&fixture, environment)?;
+        let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?;
+        let token = stopped
+            .restore_observation_token
+            .ok_or_else(|| failure("missing original observation"))?;
+        let original_sources = saved_source_hashes(&installed.environment, &fixture)?;
+        let request_id = "owned-process-checkpoint-request";
+        let call = installed.environment.pending_restore_call(
+            &fixture.manifest.worker_id,
+            request_id,
+            &token,
+        )?;
+        let ack = std::thread::scope(|scope| -> Result<RestoreCheckpointAck> {
+            let request = scope.spawn(call);
+            let ack = peer.acknowledge(&installed.environment)?;
+            installed.environment.crash_runtime()?;
+            // With the actual child reaped there can be no deadline race while
+            // inspecting the exact decision and nonterminal owner at process death.
+            let saved = canonical(&installed.environment, &fixture)?;
+            let op = saved
+                .pointer("/execution_state/execution/intent")
+                .ok_or_else(|| failure("checkpoint lacks canonical pending operation"))?;
+            let owner = &saved["restore_guard"]["owners"][request_id];
+            assert_eq!(op["operation_id"], serde_json::to_value(ack.operation_id)?);
+            assert_eq!(owner["operation_id"], op["operation_id"]);
+            assert!(owner["receipt"].is_null());
+            assert_eq!(owner["request"]["expected_observation_token"], token);
+            assert_eq!(op["source_repair_possible"], false);
+            let committed = stage == RestoreCheckpointStage::CommittedPreterminal;
+            assert_eq!(
+                op["commit"],
+                if committed {
+                    "committed"
+                } else {
+                    "not_committed"
+                }
+            );
+            assert_eq!(
+                ack.commit,
+                if committed {
+                    RestoreCheckpointCommit::Committed
+                } else {
+                    RestoreCheckpointCommit::NotCommitted
+                }
+            );
+            assert_eq!(
+                saved_source_hashes(&installed.environment, &fixture)?,
+                original_sources
+            );
+            assert_eq!(
+                restore_runs(&installed.environment, &fixture)?,
+                vec![format!("restore-{}", ack.operation_id)]
+            );
+            fs::write(
+                installed
+                    .environment
+                    .artifacts_dir
+                    .join("checkpoint-at-death.json"),
+                serde_json::to_vec_pretty(&serde_json::json!({
+                    "runtime_reaped_before_inspection": installed.environment.runtime_pid().is_none(),
+                    "operation_id": op["operation_id"], "commit": op["commit"],
+                    "receipt_absent": owner["receipt"].is_null(),
+                    "source_repair_possible": op["source_repair_possible"],
+                    "same_original_request_and_token": true,
+                    "restore_runs": restore_runs(&installed.environment, &fixture)?,
+                    "saved_source_hashes": saved_source_hashes(&installed.environment, &fixture)?,
+                }))?,
+            )?;
+            let outcome = request
+                .join()
+                .map_err(|_| failure("public restore caller panicked"))?;
+            if let Ok(value) = outcome {
+                let receipt: WorkerRestoreResponse = serde_json::from_value(value)?;
+                assert_ne!(
+                    receipt.result.state,
+                    WorkerRestoreState::Accepted,
+                    "no caller Accepted may precede either checkpoint kill"
+                );
+            }
+            installed.environment.restart_runtime_after_crash()?;
+            assert_ne!(installed.environment.runtime_pid(), Some(ack.pid));
+            Ok(ack)
+        })?;
+        drop(peer);
+        let recovered = installed.environment.restore_worker(
+            &fixture.manifest.worker_id,
+            request_id,
+            &token,
+        )?;
+        let receipt: WorkerRestoreResponse = serde_json::from_value(recovered.clone())?;
+        let committed = stage == RestoreCheckpointStage::CommittedPreterminal;
+        assert_eq!(
+            receipt.result.state,
+            if committed {
+                WorkerRestoreState::Accepted
+            } else {
+                WorkerRestoreState::RolledBack
+            }
+        );
+        assert_eq!(
+            installed.environment.restore_worker(
+                &fixture.manifest.worker_id,
+                request_id,
+                &token
+            )?,
+            recovered
+        );
+        let final_saved = canonical(&installed.environment, &fixture)?;
+        let owners = final_saved["restore_guard"]["owners"]
+            .as_object()
+            .ok_or_else(|| failure("missing canonical owners"))?;
+        assert_eq!(owners.len(), 1);
+        assert_eq!(
+            owners[request_id]["operation_id"],
+            serde_json::to_value(ack.operation_id)?
+        );
+        assert_eq!(
+            owners[request_id]["request"]["expected_observation_token"],
+            token
+        );
+        assert_eq!(
+            owners[request_id]["receipt"]["state"],
+            if committed { "accepted" } else { "rolled_back" }
+        );
+        let stale = installed.environment.restore_worker(
+            &fixture.manifest.worker_id,
+            "new-stale-checkpoint-intent",
+            &token,
+        );
+        if !matches!(stale, Err(error) if error.to_string().contains("HTTP 409")) {
+            return Err(failure("fresh stale intent was not rejected"));
+        }
+        fs::write(
+            installed
+                .environment
+                .artifacts_dir
+                .join("checkpoint-recovery.json"),
+            serde_json::to_vec_pretty(&serde_json::json!({
+                "ack": ack, "recovered_state": receipt.result.state,
+                "canonical_operation_id": owners[request_id]["operation_id"],
+                "owner_count": owners.len(), "same_original_request_and_token": true,
+                "saved_source_hashes_before": original_sources,
+                "saved_source_hashes_after": saved_source_hashes(&installed.environment, &fixture)?,
+                "restore_runs_after": restore_runs(&installed.environment, &fixture)?,
+            }))?,
+        )?;
+        if committed {
+            assert_single_state(&installed.environment, &fixture, "idle")?;
+            assert_eq!(
+                restore_runs(&installed.environment, &fixture)?,
+                vec![format!("restore-{}", ack.operation_id)]
+            );
+            let mut subscription = WorkerSessionSubscription::connect(
+                &installed.environment,
+                &fixture.manifest.worker_id,
+                &installed.snapshot,
+            )?;
+            subscription.assert_alive()?;
+            stop_and_cleanup(installed, vec![subscription], &fixture)
+        } else {
+            assert_single_state(&installed.environment, &fixture, "stopped")?;
+            assert!(restore_runs(&installed.environment, &fixture)?.is_empty());
+            assert_eq!(
+                saved_source_hashes(&installed.environment, &fixture)?,
+                original_sources
+            );
+            let report = installed.environment.cleanup()?;
+            assert!(report.cleanup_success && !report.fixture_root_exists_after);
+            Ok(())
+        }
+    })();
+    fixture.source.assert_unchanged()?;
+    result
+}
+
+#[test]
+fn acknowledged_checkpoint_peer_disconnect_resumes_original_operation_and_tears_down() -> Result<()>
+{
+    observer_completion(false)
+}
+
+#[test]
+fn acknowledged_checkpoint_explicit_release_resumes_original_operation_and_tears_down() -> Result<()>
+{
+    observer_completion(true)
+}
+
+fn observer_completion(release: bool) -> Result<()> {
+    let fixture = Fixture::load()?;
+    let result = (|| {
+        let mut environment = WorkerE2eEnvironment::new()?;
+        let mut peer = Peer::configure(
+            &mut environment,
+            &fixture,
+            RestoreCheckpointStage::PreparedNotCommitted,
+        )?;
+        let installed = seed::start_in(&fixture, environment)?;
+        let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?;
+        let token = stopped
+            .restore_observation_token
+            .ok_or_else(|| failure("missing observation"))?;
+        let call = installed.environment.pending_restore_call(
+            &fixture.manifest.worker_id,
+            "peer-disconnect-request",
+            &token,
+        )?;
+        std::thread::scope(|scope| -> Result<()> {
+            let request = scope.spawn(call);
+            peer.acknowledge(&installed.environment)?;
+            // EOF models an abandoned observer, not a successful restore result.
+            if release {
+                peer.release()?;
+            } else {
+                drop(peer);
+            }
+            let value = request
+                .join()
+                .map_err(|_| failure("public caller panicked"))??;
+            let response: WorkerRestoreResponse = serde_json::from_value(value)?;
+            assert_eq!(response.result.state, WorkerRestoreState::Accepted);
+            Ok(())
+        })?;
+        let mut subscription = WorkerSessionSubscription::connect(
+            &installed.environment,
+            &fixture.manifest.worker_id,
+            &installed.snapshot,
+        )?;
+        subscription.assert_alive()?;
+        stop_and_cleanup(installed, vec![subscription], &fixture)
+    })();
+    fixture.source.assert_unchanged()?;
+    result
+}
+
+#[test]
+fn acknowledged_checkpoint_failed_observer_scope_has_bounded_owned_cleanup() -> Result<()> {
+    let fixture = Fixture::load()?;
+    let environment = WorkerE2eEnvironment::new()?;
+    let root = environment.root.clone();
+    let artifacts = environment.artifacts_dir.clone();
+    let started = Instant::now();
+    let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<()> {
+        let mut environment = environment;
+        let mut peer = Peer::configure(
+            &mut environment,
+            &fixture,
+            RestoreCheckpointStage::PreparedNotCommitted,
+        )?;
+        let installed = seed::start_in(&fixture, environment)?;
+        let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?;
+        let token = stopped
+            .restore_observation_token
+            .ok_or_else(|| failure("missing observation"))?;
+        let call = installed.environment.pending_restore_call(
+            &fixture.manifest.worker_id,
+            "failed-observer-request",
+            &token,
+        )?;
+        std::thread::scope(|scope| -> Result<()> {
+            scope.spawn(call);
+            peer.acknowledge(&installed.environment)?;
+            // Leave the peer connected while unwinding the scoped HTTP caller.
+            // The Runtime's own deadline must unblock it, then RAII must reap
+            // both owned processes and remove the environment, not wait forever.
+            panic!("intentional checkpoint observer failure");
+        })
+    }));
+    assert!(outcome.is_err(), "the observer failure was not exercised");
+    assert!(
+        started.elapsed() < Duration::from_secs(45),
+        "owned cleanup exceeded scenario budget"
+    );
+    assert!(!root.exists(), "failed observer left its isolated root");
+    let cleanup: serde_json::Value =
+        serde_json::from_slice(&fs::read(artifacts.join("cleanup.json"))?)?;
+    assert_eq!(cleanup["cleanup_success"], true);
+    assert_eq!(cleanup["fixture_root_exists_after"], false);
+    fixture.source.assert_unchanged()?;
+    Ok(())
+}

tests/e2e/tests/worker_restore/seed.rs

@@ -42,7 +42,10 @@ pub struct Installed {
 }
 
 pub fn start(fixture: &Fixture) -> Result<Installed> {
-    let mut environment = WorkerE2eEnvironment::new()?;
+    start_in(fixture, WorkerE2eEnvironment::new()?)
+}
+
+pub fn start_in(fixture: &Fixture, mut environment: WorkerE2eEnvironment) -> Result<Installed> {
     super::phase(&environment, "fixture-copy-verify")?;
     let copy = environment.root.join("provided-fixture");
     fixture.source.copy_to(&copy)?;