fix: merge develop and reconcile schema histories
Preserve both published schema-103 histories and converge their Passkey results and Job input retention features transactionally at schema 104. Retain local Reviewer, integration-event, auth and sidebar fixes while adopting develop restore and E2E updates; regenerate API contracts. Validation: schema convergence regressions, full Server/Worker/Runtime suites, server-api and merge-request tests, Web unit/component tests, root cargo check, E2E harness tests and restore/auth target compilation, generated artifacts, fmt and naming checks. Product-process E2E not rerun; live services and databases untouched.
This commit is contained in:
@@ -10,6 +10,7 @@ autotests = false
|
||||
default = []
|
||||
e2e = []
|
||||
fixture-support = ["yoi-workspace-server/test-support", "worker-runtime/test-support"]
|
||||
restore-checkpoints = ["fixture-support", "worker-runtime/restore-checkpoints"]
|
||||
|
||||
[dependencies]
|
||||
libc.workspace = true
|
||||
|
||||
+28
-2
@@ -23,7 +23,7 @@
|
||||
|
||||
所有crateの`test-support`依存はテスト側の非default `fixture-support` featureだけで有効にする。製品binary providerは別Cargo invocationと専用`target/e2e-products`へ`yoi`・`yoi-server`・`yoi-runtime`を追加featureなしでビルドする。support test自身をこのtargetへビルドしてはならない。`YOI_E2E_BIN`等の明示binary overrideを使用する場合も、呼出し側が同じ分離を保証する。既存TUI用`e2e-test` featureは別の`target/e2e-tui-products`へビルドし、restore製品binaryへ混入しない。
|
||||
|
||||
固定資産の正常restore・並行隔離E2E:
|
||||
固定資産の正常restore・並行隔離・応答喪失回復E2E:
|
||||
|
||||
```sh
|
||||
cargo test -p yoi-e2e --features e2e,fixture-support --test worker_restore
|
||||
@@ -32,7 +32,33 @@ YOI_E2E_RESTORE_FIXTURE=target/fixture-candidate-v1 \
|
||||
cargo test -p yoi-e2e --features e2e,fixture-support --test worker_restore
|
||||
```
|
||||
|
||||
標準資産は`tests/e2e/fixtures/worker-codex-v1`の`manifest.json`・`worker-template.json`・`expected.json`・`workspace-config.dcdl`。manifestの各asset hash/size、schemaと固定ID、source revision、modelを検証する。Workspace configのimport参照を黙って最新へ置換せず、Gitでsource revisionに対する`resources`と`crates/builtin-source`の一致を確認する(Git履歴がない場合も失敗)。移植したSessionの正規公開projectionを供給された会話行と照合し、その全snapshotを実executionのWebSocketと比較する。欠落fixture・hash不一致・未実行は成功扱いしない。
|
||||
標準資産は`tests/e2e/fixtures/worker-codex-v1`の`manifest.json`・`worker-template.json`・`expected.json`・`workspace-config.dcdl`。manifestの各asset hash/size、schemaと固定ID、生成元commit、modelを検証する。Workspace configのimport参照を黙って最新へ置換せず、Gitで生成元commitに対する`resources`と`crates/builtin-source`の一致を確認する(Git履歴がない場合も失敗)。移植したSessionの正規公開projectionを供給された会話行と照合し、その全snapshotを実executionのWebSocketと比較する。欠落fixture・hash不一致・未実行は成功扱いしない。
|
||||
|
||||
応答喪失ケースは同じ`WorkerE2eEnvironment`内の単発loopback relayで公開restore要求をそのまま転送し、完全なupstream応答を受け取った後に呼出元へ0byteで切断する。呼出元のtransport失敗とrelay側のAccepted観測は別の事実として記録する。回収は元のrequest ID・観測tokenをそのまま使い、fresh intentへの古いtokenはHTTP409を要求する。
|
||||
|
||||
process終了ケースはそのAccepted receipt確認後に環境が所有するRuntime childだけをkill/reapし、同じroot・Runtime identity・trustから再起動する。seedの再実行・tokenの差替えはしない。再起動前後のcanonical owner全体のhash・operation ID・owner数が一致し、公開APIの保存結果とfull Session snapshotを回収できることを確認する。公開Worker投影の観測tokenは再起動で変わるため、そのfieldだけはcurrent detailとの一致を別に確認する。canonical診断はboundedな読取りのみで、要求・receiptの内容や認証情報はartifactへ保存しない。
|
||||
|
||||
これは**commit済みかつterminal receipt保存後**のprocess終了であり、未commit候補やcommit直後・receipt前の終了証明ではない。固定Codex資産へSubmit/Notifyを送らないため、通常入力・Paused admissionの製品E2Eの証明でもない。検証jobはこの明示targetを選択する必要があり、通常suiteだけでは代替しない。
|
||||
|
||||
## Opt-in Runtime checkpoint process試験
|
||||
|
||||
上のfeatureless製品4case(target内の既存seed projection試験1件とは別)に加え、`restore-checkpoints`を明示した実Runtimeでdecision前後とactivation後・terminal前のlive windowを試験する。このfeatureはoffline fixture-supportと独立しており、通常RuntimeではCLI入口も環境変数による入口も無い。
|
||||
|
||||
```sh
|
||||
# 同期を有効にしたRuntimeだけを専用targetへ。通常製品binaryを上書きしない。
|
||||
cargo build --locked -p worker-runtime --bin yoi-runtime \
|
||||
--features restore-checkpoints --target-dir target/e2e-checkpoint-products
|
||||
cargo test --locked -p yoi-e2e --features e2e,restore-checkpoints \
|
||||
--test worker_restore checkpoint:: -- --test-threads=1
|
||||
```
|
||||
|
||||
`WorkerE2eEnvironment::use_checkpoint_runtime`はその環境のRuntime binary/hashと、隔離root内の明示configだけを選択する。global provider・環境変数・通常起動への暗黙設定は変更しない。configは一度だけCLIへ渡し、再起動時は渡さない。test側のlink featureをfeatureless製品binaryへ混入させない。
|
||||
|
||||
共有harnessはloopback ackのstage/Worker/operation/PID/nonceを確認し、直ちに所有childをkill/reapする。**offline正本を確認してから**同roots/identity/trustを再起動するため、ack後のdeadlineとの競合を成功扱いしない。NotCommittedの実私有候補は同operationのcleanup-only→RolledBack/Stopped、Committed/preterminalは同じdecision→実protocol/full Session snapshot→Acceptedへ回収する。元のrequest/token、owner数1、同operation、stale fresh intent拒否、保存source hash、restore run path数、最終cleanupを検証する。JSONでpending状態を作らず、seedは初回のみ。
|
||||
|
||||
orphan upload pinの代表repairは、隔離copyへ正規Session storeのupload/pin APIで作った未結付けpinを、実Profileのcommit後activationが解放する経路を使う。Prepared窓は未修復のsource保持、Activated窓は実修復bytes/hashの保持を区別し、kill/reap後に正本を読んでから再起動する。materialized headの固定資産を巻き戻さないため、この試験は先行repair済みNotCommittedのprocess証明ではない。Activated後のstale allocationは通常登録と同じallocation authorityで回収し、live ownerが残る場合はrun artifactsを回収しない。
|
||||
|
||||
明示解除・peer EOF・observer scopeの意図的panicも試験する。gateの8秒deadlineとscoped HTTP callの15秒deadlineが待機を制限し、通常operationの継続後に環境のRAIIがprocess/rootを回収する。これはtest scope失敗の回収証明であり、OSによるharness強制終了や電源断の一般的な保証ではない。原fixtureへのSubmit/Notifyや実モデル呼出しは行わない。checkpoint ack/at-death/recovery、kill/reap、binary/hashとcleanup artifactはfeaturelessログと分けて報告する。
|
||||
|
||||
通常のharness回帰試験(製品processは起動しない):
|
||||
|
||||
|
||||
+208
-45
@@ -1,41 +1,88 @@
|
||||
//! Byte-preserving fixture transport, independent of the owning store's schema validation.
|
||||
use std::collections::BTreeMap;
|
||||
//! Bounded byte-preserving transport, independent of the owning store's schema validation.
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs;
|
||||
use std::io::{Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::{HarnessError, Result};
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct Limits {
|
||||
file_bytes: u64,
|
||||
total_bytes: u64,
|
||||
entries: usize,
|
||||
depth: usize,
|
||||
}
|
||||
|
||||
const LIMITS: Limits = Limits {
|
||||
file_bytes: 8 * 1024 * 1024,
|
||||
total_bytes: 32 * 1024 * 1024,
|
||||
entries: 256,
|
||||
depth: 8,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FixtureSource {
|
||||
root: PathBuf,
|
||||
hashes: BTreeMap<PathBuf, String>,
|
||||
flat_files: Option<BTreeSet<PathBuf>>,
|
||||
}
|
||||
|
||||
fn reject(message: &'static str) -> HarnessError {
|
||||
HarnessError::Protocol(message.into())
|
||||
}
|
||||
|
||||
impl FixtureSource {
|
||||
/// Record every supplied file, including the manifest. No fixture is generated
|
||||
/// or repaired here; the typed store importer must additionally validate it.
|
||||
/// Capture a bounded tree without generating or repairing store data.
|
||||
pub fn capture(root: &Path) -> Result<Self> {
|
||||
let hashes = inventory(root)?;
|
||||
Self::capture_with(root, None)
|
||||
}
|
||||
|
||||
/// Validate the complete flat file set before reading any content. The same
|
||||
/// allowlist is retained for copy and later source-integrity checks.
|
||||
pub fn capture_flat(root: &Path, names: &[&str]) -> Result<Self> {
|
||||
let files: BTreeSet<_> = names.iter().map(PathBuf::from).collect();
|
||||
if files.len() != names.len()
|
||||
|| files.iter().any(|name| {
|
||||
name.components().count() != 1
|
||||
|| !matches!(
|
||||
name.components().next(),
|
||||
Some(std::path::Component::Normal(_))
|
||||
)
|
||||
})
|
||||
{
|
||||
return Err(reject("invalid fixture file allowlist"));
|
||||
}
|
||||
Self::capture_with(root, Some(files))
|
||||
}
|
||||
|
||||
fn capture_with(root: &Path, flat_files: Option<BTreeSet<PathBuf>>) -> Result<Self> {
|
||||
let hashes = inventory(root, flat_files.as_ref(), LIMITS)?;
|
||||
if hashes.is_empty() {
|
||||
return Err(HarnessError::Protocol("provided fixture is empty".into()));
|
||||
return Err(reject("provided fixture is empty"));
|
||||
}
|
||||
Ok(Self {
|
||||
root: root.to_owned(),
|
||||
hashes,
|
||||
flat_files,
|
||||
})
|
||||
}
|
||||
|
||||
/// Read a known asset through a descriptor with a real byte limit, including
|
||||
/// when its size grows after metadata inspection. Errors never include paths.
|
||||
pub fn read_bounded(path: &Path) -> Result<Vec<u8>> {
|
||||
read_regular(path, LIMITS.file_bytes)
|
||||
}
|
||||
|
||||
pub fn hashes(&self) -> &BTreeMap<PathBuf, String> {
|
||||
&self.hashes
|
||||
}
|
||||
|
||||
pub fn assert_unchanged(&self) -> Result<()> {
|
||||
if inventory(&self.root)? != self.hashes {
|
||||
return Err(HarnessError::Protocol(
|
||||
"provided fixture changed during execution".into(),
|
||||
));
|
||||
if inventory(&self.root, self.flat_files.as_ref(), LIMITS)? != self.hashes {
|
||||
return Err(reject("provided fixture changed during execution"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -51,31 +98,26 @@ impl FixtureSource {
|
||||
.canonicalize()?
|
||||
.starts_with(self.root.canonicalize()?)
|
||||
{
|
||||
return Err(HarnessError::Protocol(
|
||||
"fixture copy destination is inside the original".into(),
|
||||
));
|
||||
return Err(reject("fixture copy destination is inside the original"));
|
||||
}
|
||||
fs::create_dir(destination)?;
|
||||
let mut remaining = LIMITS.total_bytes;
|
||||
for (relative, expected) in &self.hashes {
|
||||
let bytes = fs::read(self.root.join(relative))?;
|
||||
let bytes = read_regular(&self.root.join(relative), LIMITS.file_bytes.min(remaining))?;
|
||||
remaining -= bytes.len() as u64;
|
||||
if hash(&bytes) != *expected {
|
||||
return Err(HarnessError::Protocol(
|
||||
"fixture changed while copying".into(),
|
||||
));
|
||||
return Err(reject("fixture changed while copying"));
|
||||
}
|
||||
let target = destination.join(relative);
|
||||
fs::create_dir_all(target.parent().unwrap())?;
|
||||
use std::io::Write;
|
||||
let mut file = fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(target)?;
|
||||
file.write_all(&bytes)?;
|
||||
}
|
||||
if inventory(destination)? != self.hashes {
|
||||
return Err(HarnessError::Protocol(
|
||||
"fixture copy failed hash verification".into(),
|
||||
));
|
||||
if inventory(destination, self.flat_files.as_ref(), LIMITS)? != self.hashes {
|
||||
return Err(reject("fixture copy failed hash verification"));
|
||||
}
|
||||
self.assert_unchanged()
|
||||
}
|
||||
@@ -91,47 +133,121 @@ fn hash(bytes: &[u8]) -> String {
|
||||
)
|
||||
}
|
||||
|
||||
fn inventory(root: &Path) -> Result<BTreeMap<PathBuf, String>> {
|
||||
fn visit(root: &Path, path: &Path, files: &mut BTreeMap<PathBuf, String>) -> Result<()> {
|
||||
let metadata = fs::symlink_metadata(path)?;
|
||||
if metadata.is_dir() {
|
||||
for entry in fs::read_dir(path)? {
|
||||
visit(root, &entry?.path(), files)?;
|
||||
fn read_limited(reader: impl Read, limit: u64) -> Result<Vec<u8>> {
|
||||
let mut bytes = Vec::new();
|
||||
reader
|
||||
.take(limit + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|_| reject("fixture content read failed"))?;
|
||||
if bytes.len() as u64 > limit {
|
||||
return Err(reject("fixture byte budget exceeded"));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn read_regular(path: &Path, limit: u64) -> Result<Vec<u8>> {
|
||||
let mut options = fs::OpenOptions::new();
|
||||
options.read(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
// Do not follow a replaced final symlink or block on a replaced FIFO.
|
||||
options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK);
|
||||
}
|
||||
let file = options
|
||||
.open(path)
|
||||
.map_err(|_| reject("fixture asset open failed"))?;
|
||||
let metadata = file
|
||||
.metadata()
|
||||
.map_err(|_| reject("fixture asset metadata failed"))?;
|
||||
if !metadata.is_file() || metadata.len() > limit {
|
||||
return Err(reject("fixture asset is not a bounded regular file"));
|
||||
}
|
||||
read_limited(file, limit)
|
||||
}
|
||||
|
||||
fn inventory(
|
||||
root: &Path,
|
||||
flat: Option<&BTreeSet<PathBuf>>,
|
||||
limits: Limits,
|
||||
) -> Result<BTreeMap<PathBuf, String>> {
|
||||
struct Budget {
|
||||
entries: usize,
|
||||
bytes: u64,
|
||||
}
|
||||
fn visit(
|
||||
root: &Path,
|
||||
path: &Path,
|
||||
depth: usize,
|
||||
limits: Limits,
|
||||
flat: Option<&BTreeSet<PathBuf>>,
|
||||
budget: &mut Budget,
|
||||
files: &mut BTreeSet<PathBuf>,
|
||||
) -> Result<()> {
|
||||
if depth > limits.depth || budget.entries == 0 {
|
||||
return Err(reject("fixture tree budget exceeded"));
|
||||
}
|
||||
budget.entries -= 1;
|
||||
let metadata = fs::symlink_metadata(path).map_err(|_| reject("fixture metadata failed"))?;
|
||||
let relative = path.strip_prefix(root).unwrap();
|
||||
if depth != 0 && flat.is_some_and(|names| !names.contains(relative)) {
|
||||
return Err(reject("unexpected fixture file set"));
|
||||
}
|
||||
if metadata.is_dir() && (depth == 0 || flat.is_none()) {
|
||||
for entry in fs::read_dir(path).map_err(|_| reject("fixture directory read failed"))? {
|
||||
let entry = entry.map_err(|_| reject("fixture directory entry failed"))?;
|
||||
visit(root, &entry.path(), depth + 1, limits, flat, budget, files)?;
|
||||
}
|
||||
} else if metadata.is_file() {
|
||||
files.insert(
|
||||
path.strip_prefix(root).unwrap().to_owned(),
|
||||
hash(&fs::read(path)?),
|
||||
);
|
||||
} else if metadata.is_file() && depth != 0 {
|
||||
if metadata.len() > limits.file_bytes || metadata.len() > budget.bytes {
|
||||
return Err(reject("fixture byte budget exceeded"));
|
||||
}
|
||||
budget.bytes -= metadata.len();
|
||||
files.insert(relative.to_owned());
|
||||
} else {
|
||||
return Err(HarnessError::Protocol(
|
||||
"fixture contains a symlink or special file".into(),
|
||||
));
|
||||
return Err(reject("fixture contains a symlink or special file"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
if !fs::symlink_metadata(root)?.is_dir() {
|
||||
return Err(HarnessError::Protocol(
|
||||
"fixture source must be a real directory".into(),
|
||||
));
|
||||
// Preflight the entire shape and metadata budget before reading any body.
|
||||
let mut files = BTreeSet::new();
|
||||
visit(
|
||||
root,
|
||||
root,
|
||||
0,
|
||||
limits,
|
||||
flat,
|
||||
&mut Budget {
|
||||
entries: limits.entries,
|
||||
bytes: limits.total_bytes,
|
||||
},
|
||||
&mut files,
|
||||
)?;
|
||||
if flat.is_some_and(|expected| expected != &files) {
|
||||
return Err(reject("unexpected fixture file set"));
|
||||
}
|
||||
let mut files = BTreeMap::new();
|
||||
visit(root, root, &mut files)?;
|
||||
Ok(files)
|
||||
let mut remaining = limits.total_bytes;
|
||||
let mut hashes = BTreeMap::new();
|
||||
for relative in files {
|
||||
let bytes = read_regular(&root.join(&relative), limits.file_bytes.min(remaining))?;
|
||||
remaining -= bytes.len() as u64;
|
||||
hashes.insert(relative, hash(&bytes));
|
||||
}
|
||||
Ok(hashes)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// These are transport-only byte samples, never saved Worker substitutes.
|
||||
// Transport-only synthetic bytes, never a substitute saved Worker fixture.
|
||||
#[test]
|
||||
fn copies_are_independent_and_source_mutation_is_detected() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
let source = temp.path().join("source");
|
||||
fs::create_dir(&source)?;
|
||||
fs::write(source.join("bytes"), b"original")?;
|
||||
let fixture = FixtureSource::capture(&source)?;
|
||||
let fixture = FixtureSource::capture_flat(&source, &["bytes"])?;
|
||||
assert!(fixture.copy_to(&source.join("nested-copy")).is_err());
|
||||
assert!(!source.join("nested-copy").exists());
|
||||
fixture.assert_unchanged()?;
|
||||
@@ -161,6 +277,52 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flat_allowlist_rejects_extra_entries_before_reading_content() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
fs::write(temp.path().join("asset"), b"ok")?;
|
||||
// Sparse oversized input is never read. Even an allowed file that cannot
|
||||
// be decoded does not take precedence over an unexpected file set.
|
||||
fs::File::create(temp.path().join("extra"))?.set_len(LIMITS.file_bytes + 1)?;
|
||||
let error = FixtureSource::capture_flat(temp.path(), &["asset"]).unwrap_err();
|
||||
assert!(error.to_string().contains("unexpected fixture file set"));
|
||||
fs::remove_file(temp.path().join("extra"))?;
|
||||
assert!(FixtureSource::capture_flat(temp.path(), &["missing"]).is_err());
|
||||
fs::create_dir(temp.path().join("dir"))?;
|
||||
assert!(FixtureSource::capture_flat(temp.path(), &["asset", "dir"]).is_err());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tree_count_depth_total_and_actual_read_budgets_are_enforced() -> Result<()> {
|
||||
let temp = tempfile::tempdir()?;
|
||||
fs::write(temp.path().join("a"), b"1234")?;
|
||||
fs::write(temp.path().join("b"), b"1234")?;
|
||||
for limits in [
|
||||
Limits {
|
||||
entries: 2,
|
||||
..LIMITS
|
||||
},
|
||||
Limits {
|
||||
total_bytes: 7,
|
||||
..LIMITS
|
||||
},
|
||||
Limits {
|
||||
file_bytes: 3,
|
||||
..LIMITS
|
||||
},
|
||||
] {
|
||||
assert!(inventory(temp.path(), None, limits).is_err());
|
||||
}
|
||||
fs::create_dir_all(temp.path().join("nested/deep"))?;
|
||||
assert!(inventory(temp.path(), None, Limits { depth: 1, ..LIMITS }).is_err());
|
||||
// A reader can return more bytes than earlier metadata reported. Bound
|
||||
// actual IO rather than trusting that observation or an EOF at stat size.
|
||||
assert!(read_limited(std::io::Cursor::new(b"12345"), 4).is_err());
|
||||
assert_eq!(read_limited(std::io::Cursor::new(b"1234"), 4)?, b"1234");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn rejects_symbolic_links() -> Result<()> {
|
||||
@@ -168,6 +330,7 @@ mod tests {
|
||||
fs::write(temp.path().join("bytes"), b"original")?;
|
||||
std::os::unix::fs::symlink("bytes", temp.path().join("link"))?;
|
||||
assert!(FixtureSource::capture(temp.path()).is_err());
|
||||
assert!(FixtureSource::read_bounded(&temp.path().join("link")).is_err());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,7 +137,9 @@ fn tested_yoi_env_policy_overview() -> TestedYoiEnvPolicy {
|
||||
#[derive(Debug)]
|
||||
pub enum HarnessError {
|
||||
Io(io::Error),
|
||||
Json(serde_json::Error),
|
||||
// Never retain decode errors: serde messages can contain rejected field names
|
||||
// or values from private fixture/protocol payloads, including through Debug.
|
||||
Json,
|
||||
CommandFailed {
|
||||
program: PathBuf,
|
||||
args: Vec<String>,
|
||||
@@ -163,7 +165,7 @@ impl std::fmt::Display for HarnessError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Io(err) => write!(f, "io error: {err}"),
|
||||
Self::Json(err) => write!(f, "json error: {err}"),
|
||||
Self::Json => write!(f, "invalid JSON document"),
|
||||
Self::CommandFailed {
|
||||
program,
|
||||
args,
|
||||
@@ -209,8 +211,8 @@ impl From<io::Error> for HarnessError {
|
||||
}
|
||||
|
||||
impl From<serde_json::Error> for HarnessError {
|
||||
fn from(value: serde_json::Error) -> Self {
|
||||
Self::Json(value)
|
||||
fn from(_: serde_json::Error) -> Self {
|
||||
Self::Json
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+926
-117
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,198 @@
|
||||
//! A single bounded response-loss injection in the shared product harness.
|
||||
use std::io::{Read, Write};
|
||||
use std::net::{IpAddr, TcpListener, TcpStream};
|
||||
use std::time::Instant;
|
||||
|
||||
use reqwest::blocking::{Client, Request};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{EXIT_WAIT, HTTP_WAIT_STEP, STARTUP_WAIT};
|
||||
use crate::{HarnessError, Result};
|
||||
|
||||
const MAX_HEADER: usize = 16 * 1024;
|
||||
const MAX_MESSAGE: usize = 2 * 1024 * 1024;
|
||||
|
||||
fn failure(message: &str) -> HarnessError {
|
||||
HarnessError::Protocol(message.into())
|
||||
}
|
||||
|
||||
pub(super) fn drop_response(client: &Client, mut request: Request) -> Result<Value> {
|
||||
let host: IpAddr = request
|
||||
.url()
|
||||
.host_str()
|
||||
.ok_or_else(|| failure("missing relay host"))?
|
||||
.parse()
|
||||
.map_err(|_| failure("relay target must be a literal loopback address"))?;
|
||||
if !host.is_loopback() || request.url().scheme() != "http" {
|
||||
return Err(failure(
|
||||
"response-loss relay only permits owned loopback HTTP",
|
||||
));
|
||||
}
|
||||
let port = request
|
||||
.url()
|
||||
.port()
|
||||
.ok_or_else(|| failure("missing relay port"))?;
|
||||
let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))?;
|
||||
listener.set_nonblocking(true)?;
|
||||
let relay_address = listener.local_addr()?;
|
||||
request
|
||||
.url_mut()
|
||||
.set_port(Some(relay_address.port()))
|
||||
.map_err(|_| failure("set relay port"))?;
|
||||
request
|
||||
.url_mut()
|
||||
.set_host(Some("127.0.0.1"))
|
||||
.map_err(|_| failure("set relay host"))?;
|
||||
std::thread::scope(|scope| {
|
||||
let relay = scope.spawn(move || -> Result<Value> {
|
||||
let deadline = Instant::now() + STARTUP_WAIT;
|
||||
let mut downstream = loop {
|
||||
match listener.accept() {
|
||||
Ok((stream, peer)) if peer.ip().is_loopback() => break stream,
|
||||
Ok(_) => return Err(failure("non-loopback relay peer")),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
|
||||
if Instant::now() >= deadline {
|
||||
return Err(failure("response-loss relay accept deadline exceeded"));
|
||||
}
|
||||
std::thread::sleep(HTTP_WAIT_STEP);
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
};
|
||||
let request_bytes = read_message(&mut DeadlineStream {
|
||||
stream: &mut downstream,
|
||||
deadline,
|
||||
})?;
|
||||
let mut upstream = TcpStream::connect_timeout(&(host, port).into(), EXIT_WAIT)?;
|
||||
upstream.set_write_timeout(Some(EXIT_WAIT))?;
|
||||
// The body, request identity and authorization bytes are forwarded
|
||||
// unchanged. Neither request nor response headers enter diagnostics.
|
||||
upstream.write_all(&request_bytes)?;
|
||||
let response = read_message(&mut DeadlineStream {
|
||||
stream: &mut upstream,
|
||||
deadline,
|
||||
})?;
|
||||
let boundary =
|
||||
header_end(&response).ok_or_else(|| failure("missing upstream header"))?;
|
||||
let status = std::str::from_utf8(&response[..boundary])
|
||||
.ok()
|
||||
.and_then(|header| header.lines().next())
|
||||
.and_then(|line| line.split_whitespace().nth(1));
|
||||
if status != Some("200") {
|
||||
return Err(failure("restore relay upstream did not return HTTP 200"));
|
||||
}
|
||||
let value = serde_json::from_slice(&response[boundary..])?;
|
||||
// Intentionally send zero bytes back; the caller must observe loss.
|
||||
drop(downstream);
|
||||
Ok(value)
|
||||
});
|
||||
let caller_failed = client.execute(request).is_err();
|
||||
let evidence = relay
|
||||
.join()
|
||||
.map_err(|_| failure("response-loss relay panicked"))??;
|
||||
if !caller_failed {
|
||||
return Err(failure("caller unexpectedly received a response"));
|
||||
}
|
||||
Ok(evidence)
|
||||
})
|
||||
}
|
||||
|
||||
struct DeadlineStream<'a> {
|
||||
stream: &'a mut TcpStream,
|
||||
deadline: Instant,
|
||||
}
|
||||
|
||||
impl Read for DeadlineStream<'_> {
|
||||
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
|
||||
let remaining = self.deadline.saturating_duration_since(Instant::now());
|
||||
if remaining.is_zero() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"relay deadline exceeded",
|
||||
));
|
||||
}
|
||||
self.stream.set_read_timeout(Some(remaining))?;
|
||||
self.stream.read(buffer)
|
||||
}
|
||||
}
|
||||
|
||||
fn header_end(bytes: &[u8]) -> Option<usize> {
|
||||
bytes
|
||||
.windows(4)
|
||||
.position(|window| window == b"\r\n\r\n")
|
||||
.map(|i| i + 4)
|
||||
}
|
||||
|
||||
fn read_message(stream: &mut impl Read) -> Result<Vec<u8>> {
|
||||
let mut bytes = Vec::new();
|
||||
let mut buffer = [0; 4096];
|
||||
let mut expected = None;
|
||||
loop {
|
||||
let count = stream.read(&mut buffer)?;
|
||||
if count == 0 {
|
||||
return Err(failure("relay peer closed before complete HTTP message"));
|
||||
}
|
||||
bytes.extend_from_slice(&buffer[..count]);
|
||||
if bytes.len() > MAX_MESSAGE {
|
||||
return Err(failure("relay HTTP byte budget exceeded"));
|
||||
}
|
||||
if expected.is_none() {
|
||||
if let Some(boundary) = header_end(&bytes) {
|
||||
if boundary > MAX_HEADER {
|
||||
return Err(failure("relay HTTP header budget exceeded"));
|
||||
}
|
||||
let header = std::str::from_utf8(&bytes[..boundary])
|
||||
.map_err(|_| failure("invalid relay HTTP header"))?;
|
||||
let mut lengths = header
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case("content-length"))
|
||||
.map(|(_, value)| value.trim().parse::<usize>());
|
||||
let length = lengths
|
||||
.next()
|
||||
.ok_or_else(|| failure("relay requires Content-Length"))?
|
||||
.map_err(|_| failure("invalid relay Content-Length"))?;
|
||||
if lengths.next().is_some()
|
||||
|| length > MAX_MESSAGE - boundary
|
||||
|| header
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.any(|(name, _)| name.eq_ignore_ascii_case("transfer-encoding"))
|
||||
{
|
||||
return Err(failure("unsupported relay HTTP framing"));
|
||||
}
|
||||
expected = Some(boundary + length);
|
||||
} else if bytes.len() > MAX_HEADER {
|
||||
return Err(failure("relay HTTP header budget exceeded"));
|
||||
}
|
||||
}
|
||||
if let Some(expected) = expected {
|
||||
if bytes.len() == expected {
|
||||
return Ok(bytes);
|
||||
}
|
||||
if bytes.len() > expected {
|
||||
return Err(failure("unexpected relay trailing bytes"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn framing_rejects_incomplete_ambiguous_or_unbounded_messages() {
|
||||
for bytes in [
|
||||
b"HTTP/1.1 200 OK\r\n\r\n".as_slice(),
|
||||
b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nshort",
|
||||
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nContent-Length: 0\r\n\r\n",
|
||||
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nTransfer-Encoding: chunked\r\n\r\n",
|
||||
b"HTTP/1.1 200 OK\r\nContent-Length: 999999999\r\n\r\n",
|
||||
] {
|
||||
assert!(read_message(&mut &bytes[..]).is_err());
|
||||
}
|
||||
let valid = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\n{}";
|
||||
assert_eq!(read_message(&mut &valid[..]).unwrap(), valid);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,7 @@
|
||||
//! Fixed, externally supplied Worker restore. Never generates a fixture or sends model input.
|
||||
#[cfg(feature = "restore-checkpoints")]
|
||||
#[path = "worker_restore/checkpoint.rs"]
|
||||
mod checkpoint;
|
||||
#[path = "worker_restore/fixture.rs"]
|
||||
mod fixture;
|
||||
#[path = "worker_restore/seed.rs"]
|
||||
@@ -143,6 +146,186 @@ fn concurrent_fixed_id_restores_keep_sessions_storage_and_cleanup_isolated() ->
|
||||
result
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lost_http_response_recovers_exact_request_and_rejects_fresh_stale_intent() -> Result<()> {
|
||||
response_loss_recovery(false)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn committed_response_loss_survives_owned_runtime_process_kill_and_restart() -> Result<()> {
|
||||
response_loss_recovery(true)
|
||||
}
|
||||
|
||||
fn response_loss_recovery(crash_runtime: bool) -> Result<()> {
|
||||
let fixture = Fixture::load()?;
|
||||
let result = (|| {
|
||||
let mut installed = seed::start(&fixture)?;
|
||||
let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?;
|
||||
let token = stopped
|
||||
.restore_observation_token
|
||||
.as_deref()
|
||||
.filter(|token| !token.is_empty())
|
||||
.ok_or_else(|| failure("missing stopped observation token"))?;
|
||||
let request_id = "fixed-fixture-response-loss";
|
||||
phase(
|
||||
&installed.environment,
|
||||
"drop-http-response-after-complete-upstream",
|
||||
)?;
|
||||
let observed = installed.environment.restore_worker_losing_response(
|
||||
&fixture.manifest.worker_id,
|
||||
request_id,
|
||||
token,
|
||||
)?;
|
||||
let receipt: WorkerRestoreResponse = serde_json::from_value(observed.clone())?;
|
||||
if receipt.result.state != WorkerRestoreState::Accepted {
|
||||
return Err(failure(
|
||||
"upstream did not reach Accepted before response loss",
|
||||
));
|
||||
}
|
||||
fs::write(
|
||||
installed
|
||||
.environment
|
||||
.artifacts_dir
|
||||
.join("response-loss.json"),
|
||||
serde_json::to_vec_pretty(&serde_json::json!({
|
||||
"caller_received_response": false,
|
||||
"relay_upstream_state": receipt.result.state,
|
||||
"request_id": request_id,
|
||||
"crash_after_committed_terminal_receipt": crash_runtime,
|
||||
}))?,
|
||||
)?;
|
||||
let owner_before = restore_owner_evidence(&installed.environment, &fixture, request_id)?;
|
||||
if crash_runtime {
|
||||
phase(
|
||||
&installed.environment,
|
||||
"kill-owned-runtime-after-committed-receipt",
|
||||
)?;
|
||||
installed.environment.crash_and_restart_runtime()?;
|
||||
}
|
||||
phase(
|
||||
&installed.environment,
|
||||
"same-request-original-token-recovery",
|
||||
)?;
|
||||
let recovered =
|
||||
installed
|
||||
.environment
|
||||
.restore_worker(&fixture.manifest.worker_id, request_id, token)?;
|
||||
if crash_runtime {
|
||||
// The public response projects the current observation fence. Runtime
|
||||
// startup invalidates prior active observations; this is not a change
|
||||
// to the original request token or its saved outcome.
|
||||
let current = installed
|
||||
.environment
|
||||
.worker_detail(&fixture.manifest.worker_id)?;
|
||||
assert_eq!(
|
||||
recovered.pointer("/result/worker/restore_observation_token"),
|
||||
current.get("restore_observation_token")
|
||||
);
|
||||
let mut expected = observed.clone();
|
||||
expected["result"]["worker"]["restore_observation_token"] =
|
||||
recovered["result"]["worker"]["restore_observation_token"].clone();
|
||||
assert_eq!(
|
||||
recovered, expected,
|
||||
"restart may change only the public observation fence, not the saved outcome"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(
|
||||
recovered, observed,
|
||||
"exact request must recover the saved receipt"
|
||||
);
|
||||
}
|
||||
assert_single_state(&installed.environment, &fixture, "idle")?;
|
||||
let mut subscription = WorkerSessionSubscription::connect(
|
||||
&installed.environment,
|
||||
&fixture.manifest.worker_id,
|
||||
&installed.snapshot,
|
||||
)?;
|
||||
subscription.assert_alive()?;
|
||||
let stale = installed.environment.restore_worker(
|
||||
&fixture.manifest.worker_id,
|
||||
"fixed-fixture-fresh-stale-intent",
|
||||
token,
|
||||
);
|
||||
match stale {
|
||||
Err(error) if error.to_string().contains("HTTP 409") => {}
|
||||
_ => {
|
||||
return Err(failure(
|
||||
"fresh intent with old observation was not HTTP 409",
|
||||
));
|
||||
}
|
||||
}
|
||||
let owner_after = restore_owner_evidence(&installed.environment, &fixture, request_id)?;
|
||||
assert_eq!(
|
||||
owner_after, owner_before,
|
||||
"saved owner/operation/receipt must not change"
|
||||
);
|
||||
fs::write(
|
||||
installed
|
||||
.environment
|
||||
.artifacts_dir
|
||||
.join("restore-owner-evidence.json"),
|
||||
serde_json::to_vec_pretty(&serde_json::json!({
|
||||
"before": owner_before, "after": owner_after,
|
||||
"request_token_replaced": false,
|
||||
}))?,
|
||||
)?;
|
||||
installed.environment.assert_connected()?;
|
||||
subscription.assert_alive()?;
|
||||
stop_and_cleanup(installed, vec![subscription], &fixture)
|
||||
})();
|
||||
fixture.source.assert_unchanged()?;
|
||||
result
|
||||
}
|
||||
|
||||
// Read-only diagnostic of the product-written aggregate; never seeds or edits it.
|
||||
// Persist only a digest, operation ID and owner count, not prompts or credentials.
|
||||
fn restore_owner_evidence(
|
||||
environment: &WorkerE2eEnvironment,
|
||||
fixture: &Fixture,
|
||||
request_id: &str,
|
||||
) -> Result<serde_json::Value> {
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::io::Read;
|
||||
let path = environment
|
||||
.runtime_data_dir
|
||||
.join("runtime/workers")
|
||||
.join(&fixture.manifest.worker_id)
|
||||
.join("worker.json");
|
||||
let mut bytes = Vec::new();
|
||||
fs::File::open(path)?
|
||||
.take(8 * 1024 * 1024 + 1)
|
||||
.read_to_end(&mut bytes)?;
|
||||
if bytes.len() > 8 * 1024 * 1024 {
|
||||
return Err(failure("Worker aggregate byte budget exceeded"));
|
||||
}
|
||||
let aggregate: serde_json::Value = serde_json::from_slice(&bytes)?;
|
||||
let owners = aggregate
|
||||
.pointer("/restore_guard/owners")
|
||||
.and_then(serde_json::Value::as_object)
|
||||
.ok_or_else(|| failure("canonical Worker omitted restore owners"))?;
|
||||
let owner = owners
|
||||
.get(request_id)
|
||||
.ok_or_else(|| failure("canonical owner missing"))?;
|
||||
if owner
|
||||
.pointer("/receipt/state")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
!= Some("accepted")
|
||||
{
|
||||
return Err(failure("canonical owner has no Accepted receipt"));
|
||||
}
|
||||
let operation = owner
|
||||
.get("operation_id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.ok_or_else(|| failure("canonical owner omitted operation ID"))?;
|
||||
let digest: String = Sha256::digest(serde_json::to_vec(owner)?)
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect();
|
||||
Ok(
|
||||
serde_json::json!({"operation_id": operation, "owner_sha256": digest, "owner_count": owners.len()}),
|
||||
)
|
||||
}
|
||||
|
||||
fn restore(
|
||||
installed: &Installed,
|
||||
fixture: &Fixture,
|
||||
|
||||
@@ -0,0 +1,498 @@
|
||||
//! Opt-in real-process checkpoint cases; never substitute the featureless lane.
|
||||
#[path = "repair.rs"]
|
||||
mod repair;
|
||||
use super::*;
|
||||
use std::collections::BTreeMap;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::path::{Path, PathBuf};
|
||||
use worker_runtime::restore_checkpoints::{
|
||||
MAX_RESTORE_CHECKPOINT_ACK_BYTES, RestoreCheckpointAck, RestoreCheckpointCommit,
|
||||
RestoreCheckpointConfig, RestoreCheckpointStage,
|
||||
};
|
||||
|
||||
struct Peer {
|
||||
listener: TcpListener,
|
||||
config: RestoreCheckpointConfig,
|
||||
connection: Option<TcpStream>,
|
||||
}
|
||||
|
||||
impl Peer {
|
||||
fn configure(
|
||||
environment: &mut WorkerE2eEnvironment,
|
||||
fixture: &Fixture,
|
||||
stage: RestoreCheckpointStage,
|
||||
) -> Result<Self> {
|
||||
let listener = TcpListener::bind("127.0.0.1:0")?;
|
||||
listener.set_nonblocking(true)?;
|
||||
let config = RestoreCheckpointConfig {
|
||||
endpoint: listener.local_addr()?,
|
||||
worker_id: fixture::support(fixture.manifest.worker_id.parse())?,
|
||||
stage,
|
||||
nonce: format!(
|
||||
"owned-checkpoint-{}-{}",
|
||||
std::process::id(),
|
||||
listener.local_addr()?.port()
|
||||
),
|
||||
};
|
||||
let path = environment.root.join("restore-checkpoint.json");
|
||||
fs::write(&path, serde_json::to_vec(&config)?)?;
|
||||
let provider = yoi_e2e::worker_binary_provider_info()?;
|
||||
let binary = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../target/e2e-checkpoint-products")
|
||||
.join(provider.profile)
|
||||
.join("yoi-runtime");
|
||||
environment.use_checkpoint_runtime(binary, &path)?;
|
||||
Ok(Self {
|
||||
listener,
|
||||
config,
|
||||
connection: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn acknowledge(&mut self, environment: &WorkerE2eEnvironment) -> Result<RestoreCheckpointAck> {
|
||||
let deadline = Instant::now() + Duration::from_secs(15);
|
||||
let (mut stream, address) = loop {
|
||||
match self.listener.accept() {
|
||||
Ok(value) => break value,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
|
||||
if Instant::now() >= deadline {
|
||||
return Err(failure("checkpoint ack deadline exceeded"));
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
};
|
||||
if !address.ip().is_loopback() {
|
||||
return Err(failure("non-loopback checkpoint peer"));
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
loop {
|
||||
let remaining = deadline.saturating_duration_since(Instant::now());
|
||||
if remaining.is_zero() {
|
||||
return Err(failure("checkpoint ack read deadline exceeded"));
|
||||
}
|
||||
stream.set_read_timeout(Some(remaining))?;
|
||||
let mut byte = [0];
|
||||
stream.read_exact(&mut byte)?;
|
||||
if byte == [b'\n'] {
|
||||
break;
|
||||
}
|
||||
bytes.push(byte[0]);
|
||||
if bytes.len() > MAX_RESTORE_CHECKPOINT_ACK_BYTES {
|
||||
return Err(failure("checkpoint ack oversized"));
|
||||
}
|
||||
}
|
||||
let ack: RestoreCheckpointAck = serde_json::from_slice(&bytes)?;
|
||||
if ack.nonce != self.config.nonce
|
||||
|| ack.stage != self.config.stage
|
||||
|| ack.worker_id != self.config.worker_id
|
||||
|| ack.runtime_id != environment.runtime_id
|
||||
|| Some(ack.pid) != environment.runtime_pid()
|
||||
|| !ack.receipt_absent
|
||||
{
|
||||
return Err(failure(
|
||||
"checkpoint ack does not match owned process and stage",
|
||||
));
|
||||
}
|
||||
fs::write(
|
||||
environment.artifacts_dir.join("checkpoint-ack.json"),
|
||||
serde_json::to_vec_pretty(&ack)?,
|
||||
)?;
|
||||
self.connection = Some(stream);
|
||||
Ok(ack)
|
||||
}
|
||||
|
||||
fn release(&mut self) -> Result<()> {
|
||||
let mut stream = self
|
||||
.connection
|
||||
.take()
|
||||
.ok_or_else(|| failure("checkpoint not acknowledged"))?;
|
||||
stream.set_write_timeout(Some(Duration::from_secs(1)))?;
|
||||
stream.write_all(b"R")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn aggregate_path(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> PathBuf {
|
||||
environment
|
||||
.runtime_data_dir
|
||||
.join("runtime/workers")
|
||||
.join(&fixture.manifest.worker_id)
|
||||
}
|
||||
|
||||
fn canonical(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> Result<serde_json::Value> {
|
||||
let mut bytes = Vec::new();
|
||||
fs::File::open(aggregate_path(environment, fixture).join("worker.json"))?
|
||||
.take(8 * 1024 * 1024 + 1)
|
||||
.read_to_end(&mut bytes)?;
|
||||
if bytes.len() > 8 * 1024 * 1024 {
|
||||
return Err(failure("canonical aggregate oversized"));
|
||||
}
|
||||
Ok(serde_json::from_slice(&bytes)?)
|
||||
}
|
||||
|
||||
fn saved_source_hashes(
|
||||
environment: &WorkerE2eEnvironment,
|
||||
fixture: &Fixture,
|
||||
) -> Result<BTreeMap<String, String>> {
|
||||
fn visit(root: &Path, path: &Path, files: &mut BTreeMap<String, String>) -> Result<()> {
|
||||
if files.len() > 128 {
|
||||
return Err(failure("saved source file budget exceeded"));
|
||||
}
|
||||
let metadata = fs::symlink_metadata(path)?;
|
||||
if metadata.file_type().is_symlink() {
|
||||
return Err(failure("unexpected saved source symlink"));
|
||||
}
|
||||
if metadata.is_dir() {
|
||||
for entry in fs::read_dir(path)? {
|
||||
visit(root, &entry?.path(), files)?;
|
||||
}
|
||||
} else if metadata.is_file() {
|
||||
if metadata.len() > 8 * 1024 * 1024 {
|
||||
return Err(failure("saved source byte budget exceeded"));
|
||||
}
|
||||
use sha2::{Digest, Sha256};
|
||||
let digest: String = Sha256::digest(fs::read(path)?)
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect();
|
||||
files.insert(
|
||||
path.strip_prefix(root)
|
||||
.unwrap()
|
||||
.to_string_lossy()
|
||||
.into_owned(),
|
||||
digest,
|
||||
);
|
||||
} else {
|
||||
return Err(failure("unexpected saved source entry type"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
let root = aggregate_path(environment, fixture);
|
||||
let mut files = BTreeMap::new();
|
||||
for name in ["metadata.json", "session"] {
|
||||
visit(&root, &root.join(name), &mut files)?;
|
||||
}
|
||||
if files.len() < 2 {
|
||||
return Err(failure("saved source inventory is incomplete"));
|
||||
}
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn restore_runs(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> Result<Vec<String>> {
|
||||
let runs = aggregate_path(environment, fixture).join("runs");
|
||||
if !runs.exists() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut names = Vec::new();
|
||||
for entry in fs::read_dir(runs)? {
|
||||
let entry = entry?;
|
||||
if entry.file_name().to_string_lossy().starts_with("restore-") {
|
||||
if !entry.file_type()?.is_dir() {
|
||||
return Err(failure("restore run is not a directory"));
|
||||
}
|
||||
names.push(entry.file_name().to_string_lossy().into_owned());
|
||||
}
|
||||
}
|
||||
names.sort();
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acknowledged_not_committed_candidate_process_death_recovers_same_request_as_rollback()
|
||||
-> Result<()> {
|
||||
crash_window(RestoreCheckpointStage::PreparedNotCommitted)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acknowledged_committed_preterminal_process_death_reconstructs_same_decision() -> Result<()> {
|
||||
crash_window(RestoreCheckpointStage::CommittedPreterminal)
|
||||
}
|
||||
|
||||
fn crash_window(stage: RestoreCheckpointStage) -> Result<()> {
|
||||
let fixture = Fixture::load()?;
|
||||
let result = (|| {
|
||||
let mut environment = WorkerE2eEnvironment::new()?;
|
||||
let mut peer = Peer::configure(&mut environment, &fixture, stage)?;
|
||||
let mut installed = seed::start_in(&fixture, environment)?;
|
||||
let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?;
|
||||
let token = stopped
|
||||
.restore_observation_token
|
||||
.ok_or_else(|| failure("missing original observation"))?;
|
||||
let original_sources = saved_source_hashes(&installed.environment, &fixture)?;
|
||||
let request_id = "owned-process-checkpoint-request";
|
||||
let call = installed.environment.pending_restore_call(
|
||||
&fixture.manifest.worker_id,
|
||||
request_id,
|
||||
&token,
|
||||
)?;
|
||||
let ack = std::thread::scope(|scope| -> Result<RestoreCheckpointAck> {
|
||||
let request = scope.spawn(call);
|
||||
let ack = peer.acknowledge(&installed.environment)?;
|
||||
installed.environment.crash_runtime()?;
|
||||
// With the actual child reaped there can be no deadline race while
|
||||
// inspecting the exact decision and nonterminal owner at process death.
|
||||
let saved = canonical(&installed.environment, &fixture)?;
|
||||
let op = saved
|
||||
.pointer("/execution_state/execution/intent")
|
||||
.ok_or_else(|| failure("checkpoint lacks canonical pending operation"))?;
|
||||
let owner = &saved["restore_guard"]["owners"][request_id];
|
||||
assert_eq!(op["operation_id"], serde_json::to_value(ack.operation_id)?);
|
||||
assert_eq!(owner["operation_id"], op["operation_id"]);
|
||||
assert!(owner["receipt"].is_null());
|
||||
assert_eq!(owner["request"]["expected_observation_token"], token);
|
||||
assert_eq!(op["source_repair_possible"], false);
|
||||
let committed = stage == RestoreCheckpointStage::CommittedPreterminal;
|
||||
assert_eq!(
|
||||
op["commit"],
|
||||
if committed {
|
||||
"committed"
|
||||
} else {
|
||||
"not_committed"
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
ack.commit,
|
||||
if committed {
|
||||
RestoreCheckpointCommit::Committed
|
||||
} else {
|
||||
RestoreCheckpointCommit::NotCommitted
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
saved_source_hashes(&installed.environment, &fixture)?,
|
||||
original_sources
|
||||
);
|
||||
assert_eq!(
|
||||
restore_runs(&installed.environment, &fixture)?,
|
||||
vec![format!("restore-{}", ack.operation_id)]
|
||||
);
|
||||
fs::write(
|
||||
installed
|
||||
.environment
|
||||
.artifacts_dir
|
||||
.join("checkpoint-at-death.json"),
|
||||
serde_json::to_vec_pretty(&serde_json::json!({
|
||||
"runtime_reaped_before_inspection": installed.environment.runtime_pid().is_none(),
|
||||
"operation_id": op["operation_id"], "commit": op["commit"],
|
||||
"receipt_absent": owner["receipt"].is_null(),
|
||||
"source_repair_possible": op["source_repair_possible"],
|
||||
"same_original_request_and_token": true,
|
||||
"restore_runs": restore_runs(&installed.environment, &fixture)?,
|
||||
"saved_source_hashes": saved_source_hashes(&installed.environment, &fixture)?,
|
||||
}))?,
|
||||
)?;
|
||||
let outcome = request
|
||||
.join()
|
||||
.map_err(|_| failure("public restore caller panicked"))?;
|
||||
if let Ok(value) = outcome {
|
||||
let receipt: WorkerRestoreResponse = serde_json::from_value(value)?;
|
||||
assert_ne!(
|
||||
receipt.result.state,
|
||||
WorkerRestoreState::Accepted,
|
||||
"no caller Accepted may precede either checkpoint kill"
|
||||
);
|
||||
}
|
||||
installed.environment.restart_runtime_after_crash()?;
|
||||
assert_ne!(installed.environment.runtime_pid(), Some(ack.pid));
|
||||
Ok(ack)
|
||||
})?;
|
||||
drop(peer);
|
||||
let recovered = installed.environment.restore_worker(
|
||||
&fixture.manifest.worker_id,
|
||||
request_id,
|
||||
&token,
|
||||
)?;
|
||||
let receipt: WorkerRestoreResponse = serde_json::from_value(recovered.clone())?;
|
||||
let committed = stage == RestoreCheckpointStage::CommittedPreterminal;
|
||||
assert_eq!(
|
||||
receipt.result.state,
|
||||
if committed {
|
||||
WorkerRestoreState::Accepted
|
||||
} else {
|
||||
WorkerRestoreState::RolledBack
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
installed.environment.restore_worker(
|
||||
&fixture.manifest.worker_id,
|
||||
request_id,
|
||||
&token
|
||||
)?,
|
||||
recovered
|
||||
);
|
||||
let final_saved = canonical(&installed.environment, &fixture)?;
|
||||
let owners = final_saved["restore_guard"]["owners"]
|
||||
.as_object()
|
||||
.ok_or_else(|| failure("missing canonical owners"))?;
|
||||
assert_eq!(owners.len(), 1);
|
||||
assert_eq!(
|
||||
owners[request_id]["operation_id"],
|
||||
serde_json::to_value(ack.operation_id)?
|
||||
);
|
||||
assert_eq!(
|
||||
owners[request_id]["request"]["expected_observation_token"],
|
||||
token
|
||||
);
|
||||
assert_eq!(
|
||||
owners[request_id]["receipt"]["state"],
|
||||
if committed { "accepted" } else { "rolled_back" }
|
||||
);
|
||||
let stale = installed.environment.restore_worker(
|
||||
&fixture.manifest.worker_id,
|
||||
"new-stale-checkpoint-intent",
|
||||
&token,
|
||||
);
|
||||
if !matches!(stale, Err(error) if error.to_string().contains("HTTP 409")) {
|
||||
return Err(failure("fresh stale intent was not rejected"));
|
||||
}
|
||||
fs::write(
|
||||
installed
|
||||
.environment
|
||||
.artifacts_dir
|
||||
.join("checkpoint-recovery.json"),
|
||||
serde_json::to_vec_pretty(&serde_json::json!({
|
||||
"ack": ack, "recovered_state": receipt.result.state,
|
||||
"canonical_operation_id": owners[request_id]["operation_id"],
|
||||
"owner_count": owners.len(), "same_original_request_and_token": true,
|
||||
"saved_source_hashes_before": original_sources,
|
||||
"saved_source_hashes_after": saved_source_hashes(&installed.environment, &fixture)?,
|
||||
"restore_runs_after": restore_runs(&installed.environment, &fixture)?,
|
||||
}))?,
|
||||
)?;
|
||||
if committed {
|
||||
assert_single_state(&installed.environment, &fixture, "idle")?;
|
||||
assert_eq!(
|
||||
restore_runs(&installed.environment, &fixture)?,
|
||||
vec![format!("restore-{}", ack.operation_id)]
|
||||
);
|
||||
let mut subscription = WorkerSessionSubscription::connect(
|
||||
&installed.environment,
|
||||
&fixture.manifest.worker_id,
|
||||
&installed.snapshot,
|
||||
)?;
|
||||
subscription.assert_alive()?;
|
||||
stop_and_cleanup(installed, vec![subscription], &fixture)
|
||||
} else {
|
||||
assert_single_state(&installed.environment, &fixture, "stopped")?;
|
||||
assert!(restore_runs(&installed.environment, &fixture)?.is_empty());
|
||||
assert_eq!(
|
||||
saved_source_hashes(&installed.environment, &fixture)?,
|
||||
original_sources
|
||||
);
|
||||
let report = installed.environment.cleanup()?;
|
||||
assert!(report.cleanup_success && !report.fixture_root_exists_after);
|
||||
Ok(())
|
||||
}
|
||||
})();
|
||||
fixture.source.assert_unchanged()?;
|
||||
result
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acknowledged_checkpoint_peer_disconnect_resumes_original_operation_and_tears_down() -> Result<()>
|
||||
{
|
||||
observer_completion(false)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acknowledged_checkpoint_explicit_release_resumes_original_operation_and_tears_down() -> Result<()>
|
||||
{
|
||||
observer_completion(true)
|
||||
}
|
||||
|
||||
fn observer_completion(release: bool) -> Result<()> {
|
||||
let fixture = Fixture::load()?;
|
||||
let result = (|| {
|
||||
let mut environment = WorkerE2eEnvironment::new()?;
|
||||
let mut peer = Peer::configure(
|
||||
&mut environment,
|
||||
&fixture,
|
||||
RestoreCheckpointStage::PreparedNotCommitted,
|
||||
)?;
|
||||
let installed = seed::start_in(&fixture, environment)?;
|
||||
let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?;
|
||||
let token = stopped
|
||||
.restore_observation_token
|
||||
.ok_or_else(|| failure("missing observation"))?;
|
||||
let call = installed.environment.pending_restore_call(
|
||||
&fixture.manifest.worker_id,
|
||||
"peer-disconnect-request",
|
||||
&token,
|
||||
)?;
|
||||
std::thread::scope(|scope| -> Result<()> {
|
||||
let request = scope.spawn(call);
|
||||
peer.acknowledge(&installed.environment)?;
|
||||
// EOF models an abandoned observer, not a successful restore result.
|
||||
if release {
|
||||
peer.release()?;
|
||||
} else {
|
||||
drop(peer);
|
||||
}
|
||||
let value = request
|
||||
.join()
|
||||
.map_err(|_| failure("public caller panicked"))??;
|
||||
let response: WorkerRestoreResponse = serde_json::from_value(value)?;
|
||||
assert_eq!(response.result.state, WorkerRestoreState::Accepted);
|
||||
Ok(())
|
||||
})?;
|
||||
let mut subscription = WorkerSessionSubscription::connect(
|
||||
&installed.environment,
|
||||
&fixture.manifest.worker_id,
|
||||
&installed.snapshot,
|
||||
)?;
|
||||
subscription.assert_alive()?;
|
||||
stop_and_cleanup(installed, vec![subscription], &fixture)
|
||||
})();
|
||||
fixture.source.assert_unchanged()?;
|
||||
result
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acknowledged_checkpoint_failed_observer_scope_has_bounded_owned_cleanup() -> Result<()> {
|
||||
let fixture = Fixture::load()?;
|
||||
let environment = WorkerE2eEnvironment::new()?;
|
||||
let root = environment.root.clone();
|
||||
let artifacts = environment.artifacts_dir.clone();
|
||||
let started = Instant::now();
|
||||
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<()> {
|
||||
let mut environment = environment;
|
||||
let mut peer = Peer::configure(
|
||||
&mut environment,
|
||||
&fixture,
|
||||
RestoreCheckpointStage::PreparedNotCommitted,
|
||||
)?;
|
||||
let installed = seed::start_in(&fixture, environment)?;
|
||||
let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?;
|
||||
let token = stopped
|
||||
.restore_observation_token
|
||||
.ok_or_else(|| failure("missing observation"))?;
|
||||
let call = installed.environment.pending_restore_call(
|
||||
&fixture.manifest.worker_id,
|
||||
"failed-observer-request",
|
||||
&token,
|
||||
)?;
|
||||
std::thread::scope(|scope| -> Result<()> {
|
||||
scope.spawn(call);
|
||||
peer.acknowledge(&installed.environment)?;
|
||||
// Leave the peer connected while unwinding the scoped HTTP caller.
|
||||
// The Runtime's own deadline must unblock it, then RAII must reap
|
||||
// both owned processes and remove the environment, not wait forever.
|
||||
panic!("intentional checkpoint observer failure");
|
||||
})
|
||||
}));
|
||||
assert!(outcome.is_err(), "the observer failure was not exercised");
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(45),
|
||||
"owned cleanup exceeded scenario budget"
|
||||
);
|
||||
assert!(!root.exists(), "failed observer left its isolated root");
|
||||
let cleanup: serde_json::Value =
|
||||
serde_json::from_slice(&fs::read(artifacts.join("cleanup.json"))?)?;
|
||||
assert_eq!(cleanup["cleanup_success"], true);
|
||||
assert_eq!(cleanup["fixture_root_exists_after"], false);
|
||||
fixture.source.assert_unchanged()?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -15,13 +15,13 @@ const ASSETS: [&str; 3] = [
|
||||
"worker-template.json",
|
||||
"workspace-config.dcdl",
|
||||
];
|
||||
const MAX_ASSET_BYTES: u64 = 8 * 1024 * 1024;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Manifest {
|
||||
pub format_version: u32,
|
||||
pub source_revision: String,
|
||||
#[serde(rename = "source_revision")]
|
||||
pub source_commit: String,
|
||||
pub runtime_schema_version: u32,
|
||||
pub session_schema_version: u32,
|
||||
pub workspace_id: String,
|
||||
@@ -46,7 +46,8 @@ pub struct Asset {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Expected {
|
||||
source_revision: String,
|
||||
#[serde(rename = "source_revision")]
|
||||
source_commit: String,
|
||||
model: String,
|
||||
worker_id: String,
|
||||
session_id: String,
|
||||
@@ -88,6 +89,8 @@ impl Fixture {
|
||||
.unwrap_or_else(|| {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/worker-codex-v1")
|
||||
});
|
||||
let file_names: Vec<_> = std::iter::once("manifest.json").chain(ASSETS).collect();
|
||||
let source = FixtureSource::capture_flat(&root, &file_names)?;
|
||||
let manifest: Manifest =
|
||||
serde_json::from_slice(&bounded_read(&root.join("manifest.json"))?)?;
|
||||
if manifest.format_version != 1
|
||||
@@ -96,9 +99,9 @@ impl Fixture {
|
||||
|| manifest.session_schema_version == 0
|
||||
|| manifest.acquisition.is_empty()
|
||||
|| manifest.source_file_sha256.is_empty()
|
||||
|| manifest.source_revision.len() != 40
|
||||
|| manifest.source_commit.len() != 40
|
||||
|| !manifest
|
||||
.source_revision
|
||||
.source_commit
|
||||
.bytes()
|
||||
.all(|c| c.is_ascii_hexdigit())
|
||||
{
|
||||
@@ -108,13 +111,6 @@ impl Fixture {
|
||||
if names != BTreeSet::from(ASSETS) {
|
||||
return Err(failure("unexpected fixture asset set"));
|
||||
}
|
||||
for name in ASSETS {
|
||||
bounded_read(&root.join(name))?;
|
||||
}
|
||||
let source = FixtureSource::capture(&root)?;
|
||||
if source.hashes().len() != 4 {
|
||||
return Err(failure("fixture has unmanifested files"));
|
||||
}
|
||||
for (name, asset) in &manifest.files {
|
||||
if asset.bytes != fs::metadata(root.join(name))?.len()
|
||||
|| source.hashes().get(Path::new(name)) != Some(&format!("sha256:{}", asset.sha256))
|
||||
@@ -124,7 +120,7 @@ impl Fixture {
|
||||
}
|
||||
let expected: Expected =
|
||||
serde_json::from_slice(&bounded_read(&root.join("expected.json"))?)?;
|
||||
if expected.source_revision != manifest.source_revision
|
||||
if expected.source_commit != manifest.source_commit
|
||||
|| expected.model != manifest.model
|
||||
|| expected.worker_id != manifest.worker_id
|
||||
|| expected.session_id != manifest.session_id
|
||||
@@ -134,7 +130,7 @@ impl Fixture {
|
||||
"expected conversation identity differs from manifest",
|
||||
));
|
||||
}
|
||||
verify_builtin_revision(&manifest.source_revision)?;
|
||||
verify_builtin_commit(&manifest.source_commit)?;
|
||||
let fixture = Self {
|
||||
root,
|
||||
source,
|
||||
@@ -142,6 +138,7 @@ impl Fixture {
|
||||
expected,
|
||||
};
|
||||
fixture.template(&fixture.root)?;
|
||||
fixture.source.assert_unchanged()?;
|
||||
Ok(fixture)
|
||||
}
|
||||
|
||||
@@ -220,40 +217,31 @@ impl Fixture {
|
||||
}
|
||||
|
||||
fn bounded_read(path: &Path) -> Result<Vec<u8>> {
|
||||
let metadata = fs::symlink_metadata(path).map_err(|error| {
|
||||
failure(format!(
|
||||
"required fixture asset {}: {error}",
|
||||
path.display()
|
||||
))
|
||||
})?;
|
||||
if !metadata.is_file() || metadata.len() > MAX_ASSET_BYTES {
|
||||
return Err(failure("fixture asset is not a bounded regular file"));
|
||||
}
|
||||
Ok(fs::read(path)?)
|
||||
FixtureSource::read_bounded(path)
|
||||
}
|
||||
|
||||
fn verify_builtin_revision(revision: &str) -> Result<()> {
|
||||
fn verify_builtin_commit(source_commit: &str) -> Result<()> {
|
||||
let workspace = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.parent()
|
||||
.unwrap()
|
||||
.parent()
|
||||
.unwrap();
|
||||
// Resolve the immutable commit before diff; bad revisions must not be treated
|
||||
// Resolve the immutable commit before diff; unavailable commits must not be treated
|
||||
// as an empty diff. All resources are checked, including transitive imports.
|
||||
let commit = Command::new("git")
|
||||
.args(["cat-file", "-t", revision])
|
||||
.args(["cat-file", "-t", source_commit])
|
||||
.current_dir(workspace)
|
||||
.output()?;
|
||||
if !commit.status.success() || commit.stdout != b"commit\n" {
|
||||
return Err(failure(
|
||||
"fixture source revision is unavailable for builtin verification",
|
||||
"fixture source commit is unavailable for builtin verification",
|
||||
));
|
||||
}
|
||||
let status = Command::new("git")
|
||||
.args([
|
||||
"diff",
|
||||
"--quiet",
|
||||
revision,
|
||||
source_commit,
|
||||
"--",
|
||||
"resources",
|
||||
"crates/builtin-source",
|
||||
@@ -262,7 +250,7 @@ fn verify_builtin_revision(revision: &str) -> Result<()> {
|
||||
.status()?;
|
||||
if !status.success() {
|
||||
return Err(failure(
|
||||
"builtin/config resources differ from fixture source revision; provide a new version or frozen references",
|
||||
"builtin/config resources differ from fixture source commit; provide a new version or frozen references",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
//! Real orphan-pin repair on an installed fixed-asset copy, not a synthetic
|
||||
//! pending head. Only the postcommit activation path can reconcile these pins.
|
||||
use super::*;
|
||||
use session_store::{Store, UploadedFileLimits, WorkerSessionStore};
|
||||
|
||||
const PIN_OWNER: &str = "interrupted-pin-before-queue-checkpoint";
|
||||
const UPLOAD: &[u8] = b"isolated orphan upload; never model input";
|
||||
|
||||
#[test]
|
||||
fn orphan_pin_not_committed_process_death_preserves_unrepaired_source() -> Result<()> {
|
||||
repair_crash(RestoreCheckpointStage::PreparedNotCommitted)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphan_pin_activated_process_death_preserves_real_repair_and_committed_decision() -> Result<()> {
|
||||
repair_crash(RestoreCheckpointStage::ActivatedPreterminal)
|
||||
}
|
||||
|
||||
fn repair_crash(stage: RestoreCheckpointStage) -> Result<()> {
|
||||
let fixture = Fixture::load()?;
|
||||
let result = (|| {
|
||||
let mut environment = WorkerE2eEnvironment::new()?;
|
||||
let mut peer = Peer::configure(&mut environment, &fixture, stage)?;
|
||||
let session_id = fixture::support(fixture.manifest.session_id.parse())?;
|
||||
let mut upload = None;
|
||||
let mut installed = seed::start_in_with_source_setup(&fixture, environment, |aggregate| {
|
||||
let store = fixture::support(WorkerSessionStore::new(aggregate.join("session")))?;
|
||||
let reference = fixture::support(store.write_uploaded_file(
|
||||
session_id,
|
||||
"orphan.txt",
|
||||
"text/plain",
|
||||
UPLOAD,
|
||||
UploadedFileLimits::default(),
|
||||
))?;
|
||||
fixture::support(store.pin_uploaded_file(session_id, &reference, PIN_OWNER))?;
|
||||
upload = Some(reference);
|
||||
Ok(())
|
||||
})?;
|
||||
let upload = upload.ok_or_else(|| failure("offline upload setup not called"))?;
|
||||
let relative = format!("session/artifacts/paste/{}.file.json", upload.artifact_id);
|
||||
let pin_path = aggregate_path(&installed.environment, &fixture).join(&relative);
|
||||
let pin_before: serde_json::Value = serde_json::from_slice(&fs::read(&pin_path)?)?;
|
||||
assert_eq!(pin_before["pending_owner_id"], PIN_OWNER);
|
||||
let before = saved_source_hashes(&installed.environment, &fixture)?;
|
||||
let initial = canonical(&installed.environment, &fixture)?;
|
||||
let token = assert_single_state(&installed.environment, &fixture, "stopped")?
|
||||
.restore_observation_token
|
||||
.ok_or_else(|| failure("missing original token"))?;
|
||||
let request_id = "real-orphan-repair-crash-request";
|
||||
let committed = stage == RestoreCheckpointStage::ActivatedPreterminal;
|
||||
let call = installed.environment.pending_restore_call(
|
||||
&fixture.manifest.worker_id,
|
||||
request_id,
|
||||
&token,
|
||||
)?;
|
||||
let (ack, at_death) = std::thread::scope(|scope| -> Result<_> {
|
||||
let request = scope.spawn(call);
|
||||
let ack = peer.acknowledge(&installed.environment)?;
|
||||
installed.environment.crash_runtime()?;
|
||||
let saved = canonical(&installed.environment, &fixture)?;
|
||||
let op = &saved["execution_state"]["execution"]["intent"];
|
||||
let owner = &saved["restore_guard"]["owners"][request_id];
|
||||
assert_eq!(saved["restore_guard"]["active_request_id"], request_id);
|
||||
assert_eq!(op["operation_id"], serde_json::to_value(ack.operation_id)?);
|
||||
assert_eq!(owner["operation_id"], op["operation_id"]);
|
||||
assert_eq!(owner["request"]["expected_observation_token"], token);
|
||||
assert!(owner["receipt"].is_null());
|
||||
assert_eq!(
|
||||
op["commit"],
|
||||
if committed {
|
||||
"committed"
|
||||
} else {
|
||||
"not_committed"
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
ack.commit,
|
||||
if committed {
|
||||
RestoreCheckpointCommit::Committed
|
||||
} else {
|
||||
RestoreCheckpointCommit::NotCommitted
|
||||
}
|
||||
);
|
||||
// The durable Committed decision authorizes activation effects. The
|
||||
// precommit repair callback is not used for a materialized head.
|
||||
assert_eq!(op["source_repair_possible"], false);
|
||||
let at_death = saved_source_hashes(&installed.environment, &fixture)?;
|
||||
let pin_after: serde_json::Value = serde_json::from_slice(&fs::read(&pin_path)?)?;
|
||||
let mut expected_pin = pin_before.clone();
|
||||
if committed {
|
||||
expected_pin
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("pending_owner_id");
|
||||
}
|
||||
assert_eq!(
|
||||
pin_after, expected_pin,
|
||||
"real repair changes only orphan ownership"
|
||||
);
|
||||
if committed {
|
||||
assert_ne!(
|
||||
at_death[&relative], before[&relative],
|
||||
"actual pin bytes must change"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(
|
||||
at_death, before,
|
||||
"preparation cannot repair pin or metadata"
|
||||
);
|
||||
}
|
||||
// In both windows the original conversation is immutable.
|
||||
for (path, hash) in &before {
|
||||
if path == "session/session.json" || path.starts_with("session/segments/") {
|
||||
assert_eq!(at_death.get(path), Some(hash), "{path}");
|
||||
}
|
||||
}
|
||||
let store = fixture::support(WorkerSessionStore::open_read_only(
|
||||
aggregate_path(&installed.environment, &fixture).join("session"),
|
||||
))?;
|
||||
assert_eq!(
|
||||
fixture::support(store.read_uploaded_file(session_id, &upload))?,
|
||||
UPLOAD
|
||||
);
|
||||
assert_eq!(
|
||||
restore_runs(&installed.environment, &fixture)?,
|
||||
vec![format!("restore-{}", ack.operation_id)]
|
||||
);
|
||||
fs::write(
|
||||
installed
|
||||
.environment
|
||||
.artifacts_dir
|
||||
.join("repair-at-death.json"),
|
||||
serde_json::to_vec_pretty(&serde_json::json!({
|
||||
"ack":ack, "initial_execution":initial["execution_state"],
|
||||
"canonical_commit":op["commit"], "source_repair_possible":op["source_repair_possible"],
|
||||
"operation_id":op["operation_id"], "owner_operation_id":owner["operation_id"],
|
||||
"receipt_absent":owner["receipt"].is_null(), "original_request_and_token":true,
|
||||
"pin_pending_owner_before":pin_before["pending_owner_id"],
|
||||
"pin_pending_owner_after":pin_after["pending_owner_id"],
|
||||
"hashes_before":before, "hashes_at_death":at_death,
|
||||
"restore_runs":restore_runs(&installed.environment, &fixture)?,
|
||||
}))?,
|
||||
)?;
|
||||
let outcome = request
|
||||
.join()
|
||||
.map_err(|_| failure("restore caller panicked"))?;
|
||||
if let Ok(value) = outcome {
|
||||
let receipt: WorkerRestoreResponse = serde_json::from_value(value)?;
|
||||
assert_ne!(receipt.result.state, WorkerRestoreState::Accepted);
|
||||
}
|
||||
installed.environment.restart_runtime_after_crash()?;
|
||||
assert_ne!(installed.environment.runtime_pid(), Some(ack.pid));
|
||||
Ok((ack, at_death))
|
||||
})?;
|
||||
drop(peer);
|
||||
let recovered = installed.environment.restore_worker(
|
||||
&fixture.manifest.worker_id,
|
||||
request_id,
|
||||
&token,
|
||||
)?;
|
||||
let receipt: WorkerRestoreResponse = serde_json::from_value(recovered.clone())?;
|
||||
fs::write(
|
||||
installed
|
||||
.environment
|
||||
.artifacts_dir
|
||||
.join("repair-first-recovery.json"),
|
||||
serde_json::to_vec_pretty(&recovered)?,
|
||||
)?;
|
||||
assert_eq!(
|
||||
receipt.result.state,
|
||||
if committed {
|
||||
WorkerRestoreState::Accepted
|
||||
} else {
|
||||
WorkerRestoreState::RolledBack
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
installed.environment.restore_worker(
|
||||
&fixture.manifest.worker_id,
|
||||
request_id,
|
||||
&token
|
||||
)?,
|
||||
recovered
|
||||
);
|
||||
let final_saved = canonical(&installed.environment, &fixture)?;
|
||||
let owners = final_saved["restore_guard"]["owners"]
|
||||
.as_object()
|
||||
.ok_or_else(|| failure("missing owners"))?;
|
||||
assert_eq!(owners.len(), 1);
|
||||
let owner = &owners[request_id];
|
||||
assert_eq!(
|
||||
owner["operation_id"],
|
||||
serde_json::to_value(ack.operation_id)?
|
||||
);
|
||||
assert_eq!(owner["request"]["expected_observation_token"], token);
|
||||
assert_eq!(
|
||||
owner["receipt"]["state"],
|
||||
if committed { "accepted" } else { "rolled_back" }
|
||||
);
|
||||
let after = saved_source_hashes(&installed.environment, &fixture)?;
|
||||
assert_eq!(
|
||||
after, at_death,
|
||||
"restart/replay cannot restore the orphan pin or change saved source"
|
||||
);
|
||||
let stale = installed.environment.restore_worker(
|
||||
&fixture.manifest.worker_id,
|
||||
"fresh-stale-repair-intent",
|
||||
&token,
|
||||
);
|
||||
assert!(matches!(stale, Err(error) if error.to_string().contains("HTTP 409")));
|
||||
fs::write(
|
||||
installed
|
||||
.environment
|
||||
.artifacts_dir
|
||||
.join("repair-recovery.json"),
|
||||
serde_json::to_vec_pretty(&serde_json::json!({
|
||||
"ack":ack, "state":receipt.result.state, "diagnostics":receipt.result.diagnostics,
|
||||
"operation_id":owner["operation_id"], "owner_count":owners.len(),
|
||||
"hashes_after_recovery":after, "restore_runs":restore_runs(&installed.environment, &fixture)?,
|
||||
"same_original_request_and_token":true, "fresh_stale_rejected":true,
|
||||
}))?,
|
||||
)?;
|
||||
if committed {
|
||||
assert_single_state(&installed.environment, &fixture, "idle")?;
|
||||
assert_eq!(
|
||||
restore_runs(&installed.environment, &fixture)?,
|
||||
vec![format!("restore-{}", ack.operation_id)]
|
||||
);
|
||||
let mut subscription = WorkerSessionSubscription::connect(
|
||||
&installed.environment,
|
||||
&fixture.manifest.worker_id,
|
||||
&installed.snapshot,
|
||||
)?;
|
||||
subscription.assert_alive()?;
|
||||
stop_and_cleanup(installed, vec![subscription], &fixture)
|
||||
} else {
|
||||
assert_single_state(&installed.environment, &fixture, "stopped")?;
|
||||
assert!(restore_runs(&installed.environment, &fixture)?.is_empty());
|
||||
let report = installed.environment.cleanup()?;
|
||||
assert!(report.cleanup_success && !report.fixture_root_exists_after);
|
||||
Ok(())
|
||||
}
|
||||
})();
|
||||
fixture.source.assert_unchanged()?;
|
||||
result
|
||||
}
|
||||
@@ -42,7 +42,20 @@ pub struct Installed {
|
||||
}
|
||||
|
||||
pub fn start(fixture: &Fixture) -> Result<Installed> {
|
||||
let mut environment = WorkerE2eEnvironment::new()?;
|
||||
start_in(fixture, WorkerE2eEnvironment::new()?)
|
||||
}
|
||||
|
||||
pub fn start_in(fixture: &Fixture, environment: WorkerE2eEnvironment) -> Result<Installed> {
|
||||
start_in_with_source_setup(fixture, environment, |_| Ok(()))
|
||||
}
|
||||
|
||||
// The callback runs only on the installed isolation copy, while Runtime is offline.
|
||||
// Restart never calls the seed or this setup again.
|
||||
pub fn start_in_with_source_setup(
|
||||
fixture: &Fixture,
|
||||
mut environment: WorkerE2eEnvironment,
|
||||
setup: impl FnOnce(&Path) -> Result<()>,
|
||||
) -> Result<Installed> {
|
||||
super::phase(&environment, "fixture-copy-verify")?;
|
||||
let copy = environment.root.join("provided-fixture");
|
||||
fixture.source.copy_to(©)?;
|
||||
@@ -111,6 +124,12 @@ pub fn start(fixture: &Fixture) -> Result<Installed> {
|
||||
backend_job: None,
|
||||
};
|
||||
support(template.install(&options, request, &bootstrap.manifest))?;
|
||||
setup(
|
||||
&options
|
||||
.root
|
||||
.join("workers")
|
||||
.join(template.worker_id().to_string()),
|
||||
)?;
|
||||
let sessions = support(session_store::WorkerSessionStore::open_read_only(
|
||||
options
|
||||
.root
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
//! Fail-closed preflight shared by restore fixtures across Worker, Runtime and Server.
|
||||
//!
|
||||
//! The normal runner supplies a separate namespace per process; a manually
|
||||
//! configured invocation may use the checkout-owned base itself. This module
|
||||
//! never sets process-wide environment or creates directories/tables.
|
||||
|
||||
use std::fs;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
fn validate_restore_test_paths(
|
||||
base: &Path,
|
||||
resolved: [Option<PathBuf>; 5],
|
||||
) -> Result<PathBuf, &'static str> {
|
||||
let root = resolved[0]
|
||||
.as_deref()
|
||||
.filter(|config| config.file_name().is_some_and(|name| name == "config"))
|
||||
.and_then(Path::parent)
|
||||
.ok_or("config")?;
|
||||
let namespace = root.strip_prefix(base).map_err(|_| "config")?;
|
||||
if namespace
|
||||
.components()
|
||||
.any(|part| !matches!(part, Component::Normal(_)))
|
||||
{
|
||||
return Err("config");
|
||||
}
|
||||
for ((label, suffix), actual) in [
|
||||
("config", "config"),
|
||||
("data", "data"),
|
||||
("secrets", "data"),
|
||||
("runtime", "run"),
|
||||
("allocation", "run/workers.json"),
|
||||
]
|
||||
.into_iter()
|
||||
.zip(resolved.iter())
|
||||
{
|
||||
if actual.as_deref() != Some(root.join(suffix).as_path()) {
|
||||
return Err(label);
|
||||
}
|
||||
}
|
||||
Ok(root.to_path_buf())
|
||||
}
|
||||
|
||||
fn require_owned_directory(checkout: &Path, directory: &Path) {
|
||||
let mut checked = checkout.to_path_buf();
|
||||
for part in directory.strip_prefix(checkout).unwrap().components() {
|
||||
assert!(matches!(part, Component::Normal(_)));
|
||||
checked.push(part);
|
||||
let metadata = fs::symlink_metadata(&checked)
|
||||
.expect("create isolated restore test directories before launching the test process");
|
||||
assert!(
|
||||
metadata.is_dir() && !metadata.file_type().is_symlink(),
|
||||
"restore test directory must not traverse symlinks"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn require_restore_test_environment() {
|
||||
let checkout = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.parent()
|
||||
.unwrap()
|
||||
.parent()
|
||||
.unwrap();
|
||||
let base = checkout.join("target/restore-test-environment");
|
||||
// Compare all resolver results before inspecting even an absent allocation
|
||||
// table (its lookup can create a lock). Never inspect a live fallback path.
|
||||
let root = validate_restore_test_paths(
|
||||
&base,
|
||||
[
|
||||
manifest::paths::config_dir(),
|
||||
manifest::paths::data_dir(),
|
||||
manifest::paths::secret_data_dir(),
|
||||
manifest::paths::runtime_dir(),
|
||||
manifest::paths::worker_allocation_path(),
|
||||
],
|
||||
)
|
||||
.unwrap_or_else(|label| {
|
||||
panic!(
|
||||
"restore fixture requires isolated {label}: run deno task test, or configure YOI_CONFIG_DIR=<checkout>/target/restore-test-environment/config, YOI_DATA_DIR=<checkout>/target/restore-test-environment/data, YOI_RUNTIME_DIR=<checkout>/target/restore-test-environment/run before launch"
|
||||
)
|
||||
});
|
||||
assert_eq!(fs::canonicalize(checkout).unwrap(), checkout);
|
||||
for directory in [
|
||||
root.clone(),
|
||||
root.join("config"),
|
||||
root.join("data"),
|
||||
root.join("run"),
|
||||
] {
|
||||
require_owned_directory(checkout, &directory);
|
||||
}
|
||||
for name in ["workers.json", "workers.json.lock"] {
|
||||
match fs::symlink_metadata(root.join("run").join(name)) {
|
||||
Ok(metadata) => assert!(
|
||||
metadata.is_file() && !metadata.file_type().is_symlink(),
|
||||
"owned allocation paths must be regular files, not symlinks"
|
||||
),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => panic!("inspect owned restore allocation path: {error}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restore_fixture_path_validation_rejects_fallback_before_resource_access() {
|
||||
let base = Path::new("/owned/test-environment");
|
||||
for root in [base.to_path_buf(), base.join("run-1/worker-shard-2")] {
|
||||
let safe = ["config", "data", "data", "run", "run/workers.json"]
|
||||
.map(|suffix| Some(root.join(suffix)));
|
||||
assert_eq!(
|
||||
validate_restore_test_paths(base, safe.clone()),
|
||||
Ok(root.clone())
|
||||
);
|
||||
for index in 0..safe.len() {
|
||||
for fallback in [None, Some(PathBuf::from("/not-a-fixture"))] {
|
||||
let mut unsafe_paths = safe.clone();
|
||||
unsafe_paths[index] = fallback;
|
||||
assert!(validate_restore_test_paths(base, unsafe_paths).is_err());
|
||||
}
|
||||
}
|
||||
}
|
||||
for root in [
|
||||
base.join("../escape"),
|
||||
base.join("safe/../../escape"),
|
||||
PathBuf::from("/outside"),
|
||||
] {
|
||||
let paths = ["config", "data", "data", "run", "run/workers.json"]
|
||||
.map(|suffix| Some(root.join(suffix)));
|
||||
assert!(validate_restore_test_paths(base, paths).is_err());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user