fix: adapt T-769 recovery onto current restore lane checkpoint

UNCOMPILED TRANSPORT / PRESERVATION CHECKPOINT, NOT A VALIDATED CANDIDATE. One-time recovery authorization permits publication before validation so the parent can validate this exact source in a separate Workdir. Prior compilation attempts were interrupted by Workdir-session unavailability; cause and test outcome remain unknown. No Cargo build/check/test was started in this bounded checkpoint turn.

Merge current lane 4ef7b03651 into existing source b7a22146a0. Selectively adapt typed factory failure and same-operation cleanup/replay while preserving current saved Feature restore and operation-time authorization, logical/deferred Workdirs, T765 Host/schema and T768 replay-safe Subject grant. No T770/T771 implementation.

All 19 target-relative paths and added test bodies were source-audited. Workspace fmt check, scoped rustfmt, whitespace/diff checks passed; conflict markers zero. Excluded unrelated formatter changes and untracked compiler artifacts. Local audit: target/t769-checkpoint/AUDIT.md.

Handoff to parent: compile and run focused and package/API regressions on this exact source, fix as necessary, then obtain fresh independent review. Existing MR 01a15606-a6ba-7d23-8d60-0874607399e7 only; no new review requested here. No acceptance, approval, adoption, target/develop push, integration, Ticket closure or service update is asserted.
This commit is contained in:
2026-10-21 00:20:54 +09:00
238 changed files with 29879 additions and 5011 deletions
+1
View File
@@ -56,6 +56,7 @@ server-api = { path = "../server-api" }
workdir.workspace = true
[dev-dependencies]
workdir = { workspace = true, features = ["http-client"] }
futures.workspace = true
agen.workspace = true
serial_test = "3.4.0"
+3
View File
@@ -432,6 +432,9 @@ pub struct WorkerExecutionRestoreRequest {
pub previous_workdir_attachments: Vec<crate::catalog::WorkingDirectoryAttachmentStatus>,
pub logical_workdir_attachments: Vec<crate::catalog::LogicalWorkdirAttachment>,
pub workdir_attachments: BTreeMap<workdir::WorkdirAttachmentAlias, WorkingDirectoryBinding>,
/// Process-owned sessions that open providers only on a Workdir operation.
pub deferred_workdir_sessions:
BTreeMap<workdir::WorkdirAttachmentAlias, workdir::WorkdirSessionHandle>,
pub config_bundle: Option<ConfigBundle>,
}
+45 -12
View File
@@ -657,18 +657,32 @@ fn plan_runtime_store_migration(
));
}
if current_schema_version == SCHEMA_VERSION {
// Runtime and Session schemas evolve independently. Probe only through
// Session Store's non-mutating current-schema admission boundary; any
// conversion still runs on the existing owner-locked staging tree.
let migration_required = crate::retention::sessions_require_migration(root)?;
let excluded_ephemeral_paths = if migration_required {
runtime_tree_exclusions(root)?
} else {
Vec::new()
};
let archived_orphan_worker_count = if migration_required {
orphaned_worker_directories(root)?.len()
} else {
0
};
let plan = FsRuntimeStoreMigrationPlan {
current_schema_version,
target_schema_version: SCHEMA_VERSION,
migration_required: false,
migration_required,
worker_count: 0,
archived_orphan_worker_count: 0,
archived_orphan_worker_count,
migrated_worker_aggregate_count: 0,
migrated_diagnostic_worker_ref_count: 0,
cleared_diagnostic_worker_ref_count: 0,
mapping_digest: legacy_worker_identity_mapping_digest(&[]),
mappings: Vec::new(),
excluded_ephemeral_paths: Vec::new(),
excluded_ephemeral_paths,
};
return Ok((plan, Vec::new()));
}
@@ -1808,7 +1822,9 @@ fn migrate_runtime_store(
Ok(plan)
}
fn archive_orphaned_worker_directories(root: &Path) -> Result<usize, RuntimeError> {
fn orphaned_worker_directories(
root: &Path,
) -> Result<Vec<(std::ffi::OsString, PathBuf)>, RuntimeError> {
let workers_dir = root.join(WORKERS_DIR);
let mut orphaned = fs::read_dir(&workers_dir)
.map_err(|source| runtime_io_error("read Worker records", &workers_dir, source))?
@@ -1821,6 +1837,11 @@ fn archive_orphaned_worker_directories(root: &Path) -> Result<usize, RuntimeErro
})
.collect::<Vec<_>>();
orphaned.sort_by(|(left, _), (right, _)| left.cmp(right));
Ok(orphaned)
}
fn archive_orphaned_worker_directories(root: &Path) -> Result<usize, RuntimeError> {
let orphaned = orphaned_worker_directories(root)?;
if orphaned.is_empty() {
return Ok(0);
}
@@ -1855,6 +1876,21 @@ fn migrate_runtime_store_in_place(
if !plan.migration_required {
return Ok(plan);
}
if plan.current_schema_version == SCHEMA_VERSION {
// Session-only migration: do not reserialize Runtime/Worker identity,
// Restore guards, recovery fences, or retained execution receipts.
// As in a Runtime schema migration, retain orphan trees outside the
// Worker catalog before validating the completed staging tree.
let archived = archive_orphaned_worker_directories(root)?;
if archived != plan.archived_orphan_worker_count {
return Err(runtime_store_corrupt(
root,
"orphan count changed during Session migration".into(),
));
}
crate::retention::migrate_staged_sessions(root)?;
return Ok(plan);
}
let runtime_path = root.join(RUNTIME_FILE);
let bytes =
fs::read(&runtime_path).map_err(|error| runtime_io_error("read", &runtime_path, error))?;
@@ -1948,11 +1984,9 @@ fn migrate_runtime_store_in_place(
),
));
}
// Schema 11 already migrated Sessions; the recovery-fence bump must not
// rewrite retained history or archive attestations.
if plan.current_schema_version != SOURCE_SCHEMA_VERSION {
crate::retention::migrate_staged_sessions(root)?;
}
// A Runtime schema bump says nothing about the retained Session schema.
// Session Store owns versioned conversion, including v4 -> generic v5.
crate::retention::migrate_staged_sessions(root)?;
let (document, _) = migrate_runtime_document(
document,
plan.current_schema_version,
@@ -3011,9 +3045,8 @@ mod tests {
} else {
drop(result.unwrap());
assert_eq!(stored_schema_version(&root), u64::from(SCHEMA_VERSION));
let manifest: serde_json::Value =
read_json(&session.join("session.json"), "test").unwrap();
assert_eq!(manifest["schema_version"], 4);
session_store::WorkerSessionStore::open_read_only(&session)
.expect("startup migration must admit the current Session schema");
let once = fs::read(session.join("session.json")).unwrap();
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-test").unwrap());
assert_eq!(fs::read(session.join("session.json")).unwrap(), once);
+5 -8
View File
@@ -67,7 +67,7 @@ use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use tokio::net::TcpListener;
use workdir::{
CommandOutput, CommandStatus, WorkdirSessionHandle, dispatch_workdir_session_operation,
CommandOutput, WorkdirSessionHandle, dispatch_workdir_session_operation,
http::{
OpenWorkdirSessionRequest, OpenWorkdirSessionResponse, WorkdirSessionId,
WorkdirSessionOperation, WorkdirSessionOperationRequest, WorkdirSessionOperationResult,
@@ -914,13 +914,9 @@ async fn run_workdir_session_operation(
{
Ok(result) => result?,
Err(_) => CommandOutput {
exit_code: None,
timed_out: false,
status: CommandStatus::Running,
content: String::new(),
next_cursor: Some(cursor),
truncated: false,
output_path: None,
chunks: Vec::new(),
next_cursor: cursor,
terminal: None,
},
};
WorkdirSessionOperationResult::CommandOutput(output)
@@ -2426,6 +2422,7 @@ pub enum RuntimeHttpServerError {
#[cfg(test)]
mod tests {
mod command_transport_tests;
#[cfg(feature = "fs-store")]
mod session_material_tests;
@@ -0,0 +1,408 @@
//! Loopback integration: production RemoteWorkdirSession, signed HTTP middleware,
//! command operation handler, local process pipes, and JSON DTOs. Only opening the
//! pre-seeded session is a fixture; no Runtime Worker/materializer is claimed here.
use super::*;
use std::time::Duration;
use workdir::http::{
RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture,
};
use workdir::{CommandOutputRequest, CommandRequest, CommandStream, WorkdirSession};
struct SignedAuthorization {
identity: RuntimeIdentityMaterial,
trust: WorkspaceIssuerTrustRecord,
}
impl std::fmt::Debug for SignedAuthorization {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("SignedAuthorization")
}
}
impl WorkdirHttpAuthorization for SignedAuthorization {
fn bearer_token(
&self,
method: String,
path: String,
body: Vec<u8>,
) -> WorkdirHttpAuthorizationFuture {
let claims = signed_claims(&self.trust, &method, &path, &body);
let token =
issue_workspace_capability_token(&self.identity.signing_key().unwrap(), &claims)
.unwrap();
Box::pin(async move { Ok(token) })
}
}
struct Harness {
server: tokio::task::JoinHandle<()>,
session: WorkdirSessionHandle,
_root: tempfile::TempDir,
remote: RemoteWorkdirSession,
client: reqwest::Client,
operation_url: url::Url,
authorization: Arc<SignedAuthorization>,
}
impl Drop for Harness {
fn drop(&mut self) {
self.server.abort();
}
}
impl Harness {
async fn raw_operation(&self, operation: serde_json::Value) -> reqwest::Response {
let body = serde_json::to_vec(&serde_json::json!({"operation": operation})).unwrap();
let token = self
.authorization
.bearer_token(
"POST".into(),
self.operation_url.path().into(),
body.clone(),
)
.await
.unwrap();
self.client
.post(self.operation_url.clone())
.bearer_auth(token)
.header("content-type", "application/json")
.body(body)
.send()
.await
.unwrap()
}
async fn new() -> Self {
let root = tempfile::tempdir().unwrap();
let session: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::materialized_bound(
Workdir::new("wd-transport"),
root.path().to_path_buf(),
root.path().to_path_buf(),
SharedScope::new(Scope::writable(root.path()).unwrap()),
WorkdirSessionCapabilities::ALL,
));
let identity = RuntimeIdentityMaterial::generate("transport-key").unwrap();
let trust = workspace_trust(&identity);
let state = RuntimeHttpState {
runtime: Runtime::new_memory(),
local_token: None,
workspace_auth: Some(Arc::new(workspace_test_auth(vec![trust.clone()]))),
workdir_sessions: Arc::new(Mutex::new(HashMap::from([(
"session-transport".into(),
RuntimeHttpWorkdirSession {
owner: RuntimeWorkspaceScope::new("workspace-a", "https://backend.test"),
session: session.clone(),
},
)]))),
ssh_keyscan_program: Arc::new(PathBuf::from("ssh-keyscan")),
};
let app = Router::new()
.route(
"/v1/working-directories/{id}/sessions",
post(|| async {
Json(OpenWorkdirSessionResponse {
session_id: WorkdirSessionId::new("session-transport").unwrap(),
workdir_id: Workdir::new("wd-transport").id().clone(),
capabilities: WorkdirSessionCapabilities::ALL,
})
}),
)
.route(
"/v1/workdir-sessions/{session_id}/operations",
post(run_workdir_session_operation),
)
.route(
"/v1/workdir-sessions/{session_id}",
delete(close_workdir_session),
)
.with_state(state.clone())
.layer(middleware::from_fn_with_state(state, require_runtime_auth));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base: url::Url = format!("http://{}", listener.local_addr().unwrap())
.parse()
.unwrap();
let server = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
RUNTIME_WORKSPACE_SCOPE_HEADER,
"workspace-a".parse().unwrap(),
);
let client = reqwest::Client::builder()
.no_proxy()
.default_headers(headers)
.build()
.unwrap();
let authorization = Arc::new(SignedAuthorization { identity, trust });
let operation_url = base
.join("/v1/workdir-sessions/session-transport/operations")
.unwrap();
let remote = RemoteWorkdirSession::open_with_authorization(
client.clone(),
base,
authorization.clone(),
Workdir::new("wd-transport").id().clone(),
OpenWorkdirSessionRequest::default(),
)
.await
.unwrap();
Self {
server,
session,
_root: root,
remote,
client,
operation_url,
authorization,
}
}
}
#[tokio::test]
async fn real_http_command_pages_preserve_binary_streams_replay_and_nonzero_exit() {
tokio::time::timeout(Duration::from_secs(20), async {
let harness = Harness::new().await;
let handle = harness.remote.start_command(CommandRequest {
transport_byte_budget: workdir::MAX_COMMAND_TRANSPORT_BYTES,
command: "head -c 2097152 /dev/zero | tr '\\0' x; printf '\\342\\202\\254\\377\\000'; printf 'stderr\\346\\227\\245' >&2; exit 9".into(),
timeout_secs: 15, cwd: WorkdirPath::root(), tool_call_id: Some("http-tool-call".into()),
}).await.unwrap();
let mut cursor = 0;
let mut stdout = Vec::new();
let mut stderr = Vec::new();
let terminal = loop {
let request = CommandOutputRequest { handle: handle.clone(), cursor, limit: 16384, wait: true };
let page = harness.remote.command_output(request.clone()).await.unwrap();
assert_eq!(harness.remote.command_output(request).await.unwrap(), page);
assert!(page.chunks.iter().map(|chunk| chunk.bytes.len()).sum::<usize>() <= 16384);
for chunk in page.chunks {
assert_eq!(chunk.sequence, cursor);
cursor += chunk.bytes.len() as u64;
let target = match chunk.stream { CommandStream::Stdout => &mut stdout, CommandStream::Stderr => &mut stderr };
assert_eq!(chunk.offset, target.len() as u64);
target.extend(chunk.bytes);
}
assert_eq!(cursor, page.next_cursor);
if let Some(terminal) = page.terminal { break terminal; }
};
let mut expected = vec![b'x'; 2097152];
expected.extend_from_slice(b"\xe2\x82\xac\xff\0");
assert_eq!(stdout, expected);
assert_eq!(stderr, b"stderr\xe6\x97\xa5");
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
assert_eq!(terminal.exit_code, Some(9));
assert!(harness.remote.command_output(CommandOutputRequest { handle, cursor: 0, limit: 1, wait: false }).await.is_err());
harness.remote.close().await.unwrap();
assert!(harness.session.command_snapshot().is_empty());
}).await.expect("bounded HTTP command transport");
}
#[tokio::test]
async fn real_http_cancel_drains_tail_before_terminal_and_timeout_is_distinct() {
tokio::time::timeout(Duration::from_secs(15), async {
let harness = Harness::new().await;
for (command, timeout, cancel, expected) in [
(
"printf ready; exec yes x",
10,
true,
workdir::CommandStatus::Cancelled,
),
(
"printf ready; exec tail -f /dev/null",
1,
false,
workdir::CommandStatus::TimedOut,
),
] {
let handle = harness
.remote
.start_command(CommandRequest {
transport_byte_budget: workdir::MAX_COMMAND_TRANSPORT_BYTES,
command: command.into(),
timeout_secs: timeout,
cwd: WorkdirPath::root(),
tool_call_id: None,
})
.await
.unwrap();
let mut cursor = 0;
let first = harness
.remote
.command_output(CommandOutputRequest {
handle: handle.clone(),
cursor,
limit: 5,
wait: true,
})
.await
.unwrap();
assert_eq!(
first
.chunks
.iter()
.flat_map(|chunk| chunk.bytes.iter().copied())
.collect::<Vec<_>>(),
b"ready"
);
cursor = first.next_cursor;
if cancel {
harness.remote.cancel_command(handle.clone()).await.unwrap();
}
loop {
let page = harness
.remote
.command_output(CommandOutputRequest {
handle: handle.clone(),
cursor,
limit: workdir::MAX_COMMAND_READ_BYTES,
wait: true,
})
.await
.unwrap();
cursor = page.next_cursor;
if let Some(terminal) = page.terminal {
assert_eq!(terminal.status, expected);
break;
}
}
}
harness.remote.close().await.unwrap();
})
.await
.expect("bounded cancellation and timeout over HTTP");
}
#[tokio::test]
async fn real_http_command_budget_bounds_unique_payload_and_reports_larger_seen_totals() {
tokio::time::timeout(Duration::from_secs(20), async {
let harness = Harness::new().await;
for budget in [0, 12345] {
let handle = harness
.remote
.start_command(CommandRequest {
command: "head -c 1048576 /dev/zero; head -c 65536 /dev/zero >&2; exit 9"
.into(),
timeout_secs: 10,
transport_byte_budget: budget,
cwd: WorkdirPath::root(),
tool_call_id: None,
})
.await
.unwrap();
let mut cursor = 0;
let mut delivered = [0_u64; 2];
let terminal = loop {
let request = CommandOutputRequest {
handle: handle.clone(),
cursor,
limit: 997,
wait: true,
};
let page = harness
.remote
.command_output(request.clone())
.await
.unwrap();
assert_eq!(harness.remote.command_output(request).await.unwrap(), page);
for chunk in page.chunks {
assert_eq!(chunk.sequence, cursor);
cursor += chunk.bytes.len() as u64;
let index = match chunk.stream {
CommandStream::Stdout => 0,
CommandStream::Stderr => 1,
};
assert_eq!(chunk.offset, delivered[index]);
delivered[index] += chunk.bytes.len() as u64;
}
assert_eq!(cursor, page.next_cursor);
assert!(
cursor <= budget,
"RemoteWorkdirSession must not receive bytes beyond budget"
);
if let Some(terminal) = page.terminal {
break terminal;
}
};
assert_eq!(cursor, budget);
assert_eq!(terminal.stdout_bytes, 1048576);
assert_eq!(terminal.stderr_bytes, 65536);
assert_eq!(terminal.stdout_delivered_bytes, delivered[0]);
assert_eq!(terminal.stderr_delivered_bytes, delivered[1]);
assert_eq!(
terminal.truncation_reason,
Some(workdir::CommandOutputTruncation::ByteBudget)
);
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
assert_eq!(terminal.exit_code, Some(9));
assert!(
harness
.remote
.command_output(CommandOutputRequest {
handle: handle.clone(),
cursor: budget + 1,
limit: 1,
wait: false,
})
.await
.is_err()
);
if budget > 0 {
assert!(
harness
.remote
.command_output(CommandOutputRequest {
handle,
cursor: 0,
limit: 1,
wait: false,
})
.await
.is_err()
);
}
}
harness.remote.close().await.unwrap();
})
.await
.expect("bounded HTTP byte-budget transport");
}
#[tokio::test]
async fn real_http_rejects_invalid_or_legacy_budget_requests_before_process_start() {
tokio::time::timeout(Duration::from_secs(10), async {
let harness = Harness::new().await;
let request = CommandRequest {
command: "touch must-not-exist".into(),
timeout_secs: 1,
transport_byte_budget: workdir::MAX_COMMAND_TRANSPORT_BYTES + 1,
cwd: WorkdirPath::root(),
tool_call_id: None,
};
let oversize =
serde_json::to_value(WorkdirSessionOperation::CommandStart(request)).unwrap();
assert_eq!(
harness.raw_operation(oversize.clone()).await.status(),
reqwest::StatusCode::BAD_REQUEST
);
let mut missing = oversize.clone();
// CommandStart is externally tagged by operation / request.
let object = missing.as_object_mut().unwrap();
let request = object.get_mut("request").unwrap().as_object_mut().unwrap();
request.remove("transport_byte_budget");
assert_eq!(
harness.raw_operation(missing).await.status(),
reqwest::StatusCode::BAD_REQUEST
);
let mut legacy = oversize;
let request = legacy["request"].as_object_mut().unwrap();
request.insert("transport_byte_budget".into(), serde_json::json!(0));
request.insert("spill_dir".into(), serde_json::json!("/private/host"));
assert_eq!(
harness.raw_operation(legacy).await.status(),
reqwest::StatusCode::BAD_REQUEST
);
assert!(!harness._root.path().join("must-not-exist").exists());
harness.remote.close().await.unwrap();
})
.await
.expect("bounded HTTP budget admission");
}
@@ -35,6 +35,7 @@ fn message(id: &str, role: LoggedRole, text: &str) -> LoggedHistoryEntry {
content: vec![LoggedContentPart::Text { text: text.into() }],
},
metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(id.into()),
origin: LoggedSessionHistoryOrigin::LocalHumanInput,
derivation: None,
+395 -65
View File
@@ -641,6 +641,13 @@ impl FsWorkerRetentionProvider {
let (manifest, session_root) = self.session_archive_identity(archive_id)?;
let archive_dir = self.archive_dir(archive_id)?;
let manifest = validate_existing_archive(&archive_dir, &manifest, false)?;
session_store::WorkerSessionStore::open_read_only(&session_root).map_err(|_| {
RuntimeError::StoreCorrupt {
operation: "read Worker Session archive",
path: session_root.clone(),
message: "archive Session is unavailable in the current schema".into(),
}
})?;
Ok((manifest, session_root))
}
@@ -653,13 +660,7 @@ impl FsWorkerRetentionProvider {
&archive_dir.join("manifest.json"),
"read Worker Session archive",
)?;
if manifest.schema_version != ARCHIVE_SCHEMA_VERSION || manifest.archive_id != archive_id {
return Err(RuntimeError::StoreCorrupt {
operation: "read Worker Session archive",
path: archive_dir.join("manifest.json"),
message: "archive manifest identity mismatch".to_string(),
});
}
validate_archive_identity(&archive_dir, &manifest)?;
Ok((manifest, archive_dir.join("session")))
}
@@ -1204,9 +1205,14 @@ fn archive_storage(
}
}
/// Called only on the private Runtime migration staging tree under its owner
/// lock. Failure discards the whole tree, including manifests and attestations.
pub(crate) fn migrate_staged_sessions(root: &Path) -> Result<(), RuntimeError> {
// Share discovery and erasure fencing between Runtime preflight and staging.
// Unreadable archives are retained but excluded before any conversion writes;
// one historical archive must not block the live Runtime's startup. Session
// Store owns conversion admission, including the current resource limits.
fn for_each_live_session(
root: &Path,
mut visit: impl FnMut(&Path, bool) -> Result<(), RuntimeError>,
) -> Result<(), RuntimeError> {
for directory in ["workers", "orphaned-workers", "archives/workers"] {
let directory = root.join(directory);
let entries = match fs::read_dir(&directory) {
@@ -1228,69 +1234,232 @@ pub(crate) fn migrate_staged_sessions(root: &Path) -> Result<(), RuntimeError> {
})?;
let aggregate = entry.path();
let session = aggregate.join("session");
if !session
.join("session.json")
.try_exists()
.map_err(|source| RuntimeError::StoreIo {
operation: "inspect Session migration",
path: session.clone(),
source,
})?
{
continue;
}
let identity: CanonicalSessionManifest = read_json(
&session.join("session.json"),
"read Session migration identity",
)?;
let session_id = identity.session_id.parse().map_err(|_| {
RuntimeError::InvalidRequest("Session migration identity is invalid".into())
})?;
let is_archive = directory == root.join("archives/workers");
let identity = (|| {
if !session
.join("session.json")
.try_exists()
.map_err(|source| RuntimeError::StoreIo {
operation: "inspect Session migration",
path: session.clone(),
source,
})?
{
return Ok(None);
}
let identity: CanonicalSessionManifest = read_json(
&session.join("session.json"),
"read Session migration identity",
)?;
identity.session_id.parse().map(Some).map_err(|_| {
RuntimeError::InvalidRequest("Session migration identity is invalid".into())
})
})();
let session_id = match identity {
Ok(Some(session_id)) => session_id,
Ok(None) => continue,
Err(error) if is_archive => {
warn_unavailable_archive(
&aggregate,
ArchiveMigrationFailure::runtime("session_identity", &error),
);
continue;
}
Err(error) => return Err(error),
};
match runtime_session_retention(root).ensure_live(session_id) {
Ok(()) => {}
// Erasure fences remain authoritative even if physical removal
// was interrupted. Do not inspect or migrate fenced originals.
// Retention-ledger failures are not archive-local. Never bypass
// erasure authority while isolating an unreadable archive.
Err(session_store::StoreError::SessionErased(_)) => continue,
Err(error) => return Err(retention_error(error)),
}
let archive = if directory == root.join("archives/workers") {
let manifest: WorkerSessionArchiveManifest = read_json(
&aggregate.join("manifest.json"),
"read archive before Session migration",
)?;
Some(validate_existing_archive(&aggregate, &manifest, false)?)
} else {
None
};
session_store::WorkerSessionStore::new(&session).map_err(|error| {
RuntimeError::StoreCorrupt {
operation: "migrate Session schema",
path: session.clone(),
message: error.to_string(),
if is_archive {
if let Err(failure) = validate_archive_migration(&aggregate) {
warn_unavailable_archive(&aggregate, failure);
continue;
}
})?;
if let Some(ingestion) = archive {
let (checksum_sha256, bytes, file_count) = checksum_files(
&collect_files(&session, "migrate archive storage")?,
"migrate archive storage",
)?;
atomic_write_json(
&aggregate.join(ARCHIVE_STORAGE_FILE),
&MigratedArchiveStorage {
schema_version: 1,
ingestion,
checksum_sha256,
bytes,
file_count,
},
"commit migrated archive storage",
)?;
}
// Publication failures still abort the outer staging transaction.
visit(&aggregate, is_archive)?;
}
}
Ok(())
}
struct ArchiveMigrationFailure {
stage: &'static str,
reason: &'static str,
}
impl ArchiveMigrationFailure {
fn runtime(stage: &'static str, error: &RuntimeError) -> Self {
let reason = match error {
RuntimeError::StoreIo { source, .. } => safe_io_reason(source),
RuntimeError::StoreCorrupt { message, .. } => match message.as_str() {
"archive manifest identity mismatch" => "archive_identity_mismatch",
"archive id collision or manifest mismatch" => "archive_manifest_mismatch",
"archive checksum or content summary mismatch" => "archive_checksum_mismatch",
"archive storage attestation does not match ingestion receipt" => {
"archive_storage_attestation_mismatch"
}
_ => "invalid_stored_data",
},
RuntimeError::InvalidRequest(_) => "invalid_identity",
_ => "archive_validation_failed",
};
Self { stage, reason }
}
fn conversion(error: session_store::StoreError) -> Self {
use session_store::StoreError;
let reason = match error {
StoreError::Io(error) => safe_io_reason(&error),
StoreError::Serde(_) => "invalid_session_manifest",
StoreError::Corrupt { message, .. }
if message.starts_with("unsupported Worker Session schema version ") =>
{
"unsupported_session_schema"
}
// The legacy decoder wraps InputMetadataError in a path-bearing Corrupt
// message. Recognize only its fixed cause; never publish that wrapper.
StoreError::Corrupt { message, .. }
if message.ends_with(&format!(
": {}",
session_store::InputMetadataError::ResourceLimit
)) =>
{
"input_resource_limit_exceeded"
}
StoreError::Corrupt { .. } => "invalid_session_log",
StoreError::InvalidInputMetadata(session_store::InputMetadataError::ResourceLimit) => {
"input_resource_limit_exceeded"
}
StoreError::InvalidInputMetadata(_) => "invalid_input_metadata",
StoreError::ReadLimitExceeded => "read_limit_exceeded",
_ => "session_conversion_failed",
};
Self {
stage: "session_conversion",
reason,
}
}
}
fn safe_io_reason(error: &std::io::Error) -> &'static str {
match error.kind() {
std::io::ErrorKind::NotFound => "not_found",
std::io::ErrorKind::PermissionDenied => "permission_denied",
std::io::ErrorKind::InvalidData => "invalid_data",
_ => "io_error",
}
}
fn validate_archive_migration(aggregate: &Path) -> Result<(), ArchiveMigrationFailure> {
let session = aggregate.join("session");
// Current archives need no conversion. Their payload integrity is checked
// on access, not by scanning every retained byte during Runtime startup.
if session_store::WorkerSessionStore::open_read_only(&session).is_ok() {
return Ok(());
}
let manifest: WorkerSessionArchiveManifest = read_json(
&aggregate.join("manifest.json"),
"read archive before Session migration",
)
.map_err(|error| ArchiveMigrationFailure::runtime("archive_manifest", &error))?;
validate_existing_archive(aggregate, &manifest, false)
.map_err(|error| ArchiveMigrationFailure::runtime("archive_integrity", &error))?;
session_store::WorkerSessionStore::validate_migration(&session)
.map_err(ArchiveMigrationFailure::conversion)
}
fn warn_unavailable_archive(aggregate: &Path, failure: ArchiveMigrationFailure) {
let archive_id = aggregate.file_name().unwrap_or_default().to_string_lossy();
tracing::warn!(
archive_id = %bounded_diagnostic_id(&archive_id),
code = "session_archive_unavailable",
stage = failure.stage,
reason = failure.reason,
"Retained Session archive could not be admitted for migration; original bytes retained"
);
}
/// Read-only admission probe. Only convertible archives request a staged pass;
/// unreadable originals remain available for archive-local diagnostics/recovery.
/// Never create a writable Session store or rewrite archive attestations here.
pub(crate) fn sessions_require_migration(root: &Path) -> Result<bool, RuntimeError> {
let mut required = false;
for_each_live_session(root, |aggregate, _| {
required |=
session_store::WorkerSessionStore::open_read_only(aggregate.join("session")).is_err();
Ok(())
})?;
Ok(required)
}
/// Called only on the private Runtime migration staging tree under its owner
/// lock. Failure discards the whole tree, including manifests and attestations.
pub(crate) fn migrate_staged_sessions(root: &Path) -> Result<(), RuntimeError> {
for_each_live_session(root, |aggregate, is_archive| {
let session = aggregate.join("session");
// Current Sessions need no writable admission, and their storage
// attestations must remain byte-for-byte unchanged.
if session_store::WorkerSessionStore::open_read_only(&session).is_ok() {
return Ok(());
}
let archive = if is_archive {
let manifest: WorkerSessionArchiveManifest = read_json(
&aggregate.join("manifest.json"),
"read archive before Session migration",
)?;
Some(validate_existing_archive(aggregate, &manifest, false)?)
} else {
None
};
session_store::WorkerSessionStore::new(&session).map_err(|error| {
RuntimeError::StoreCorrupt {
operation: "migrate Session schema",
path: session.clone(),
message: error.to_string(),
}
})?;
if let Some(ingestion) = archive {
let (checksum_sha256, bytes, file_count) = checksum_files(
&collect_files(&session, "migrate archive storage")?,
"migrate archive storage",
)?;
atomic_write_json(
&aggregate.join(ARCHIVE_STORAGE_FILE),
&MigratedArchiveStorage {
schema_version: 1,
ingestion,
checksum_sha256,
bytes,
file_count,
},
"commit migrated archive storage",
)?;
}
Ok(())
})
}
fn validate_archive_identity(
archive_dir: &Path,
manifest: &WorkerSessionArchiveManifest,
) -> Result<(), RuntimeError> {
if manifest.schema_version != ARCHIVE_SCHEMA_VERSION
|| archive_dir.file_name().and_then(|name| name.to_str()) != Some(&manifest.archive_id)
{
return Err(RuntimeError::StoreCorrupt {
operation: "read Worker Session archive",
path: archive_dir.join("manifest.json"),
message: "archive manifest identity mismatch".into(),
});
}
Ok(())
}
fn validate_existing_archive(
archive_dir: &Path,
expected: &WorkerSessionArchiveManifest,
@@ -1300,6 +1469,7 @@ fn validate_existing_archive(
&archive_dir.join("manifest.json"),
"verify Worker Session archive",
)?;
validate_archive_identity(archive_dir, &existing)?;
let storage = archive_storage(archive_dir, &existing)?;
let mut comparable_expected = expected.clone();
comparable_expected.archived_at_unix_seconds = existing.archived_at_unix_seconds;
@@ -1753,12 +1923,16 @@ fn temporary_path(path: &Path) -> PathBuf {
path.with_file_name(format!(".{name}.tmp-{}-{sequence}", std::process::id()))
}
#[cfg(all(test, feature = "fs-store"))]
#[path = "retention/runtime_v5_tests.rs"]
mod runtime_v5_tests;
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{Arc, Barrier};
fn write_json(path: &Path, value: &impl Serialize) {
pub(super) fn write_json(path: &Path, value: &impl Serialize) {
fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(path, serde_json::to_vec_pretty(value).unwrap()).unwrap();
}
@@ -1786,7 +1960,7 @@ mod tests {
fs::write(worker.join("runs/attempt-a/worker.sock"), b"not retained").unwrap();
}
fn request(
pub(super) fn request(
worker_id: WorkerId,
disposition: SessionDisposition,
) -> WorkerRetentionExecutionRequest {
@@ -1831,6 +2005,7 @@ mod tests {
}],
},
metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId("original".into()),
origin: LoggedSessionHistoryOrigin::LocalHumanInput,
derivation: None,
@@ -1844,6 +2019,7 @@ mod tests {
}],
},
metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId("summary".into()),
origin: LoggedSessionHistoryOrigin::DerivedSummary,
derivation: Some(session_store::LoggedHistoryDerivation {
@@ -2002,7 +2178,7 @@ mod tests {
}
#[test]
fn session_migration_rejects_corrupt_archive_before_replacing_its_bytes() {
fn session_migration_preserves_corrupt_archive_without_blocking_other_sessions() {
let temp = tempfile::tempdir().unwrap();
let worker = WorkerId::now_v7();
let compact = old_compacted_source(temp.path(), worker);
@@ -2014,11 +2190,165 @@ mod tests {
.join("session/segments")
.join(format!("{compact}.jsonl"));
fs::write(&path, b"corrupt\n").unwrap();
assert!(migrate_staged_sessions(temp.path()).is_err());
migrate_staged_sessions(temp.path()).unwrap();
assert_eq!(fs::read(&path).unwrap(), b"corrupt\n");
assert!(!archive.join(ARCHIVE_STORAGE_FILE).exists());
}
#[cfg(feature = "fs-store")]
#[test]
fn archive_admission_logs_safe_causes_and_startup_preserves_original_bytes() {
// Exercise FsRuntimeStore startup, real archive admission and Session conversion;
// only the tracing sink is substituted, and all storage is temporary.
#[derive(Clone)]
struct Capture(Arc<std::sync::Mutex<Vec<u8>>>);
impl std::io::Write for Capture {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
self.0.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
use crate::fs_store::FsRuntimeStore;
const PRIVATE: &str = "private-session-content-token=/private/host/session";
for (case, stage, reason) in [
("identity_json", "session_identity", "invalid_stored_data"),
("identity_value", "session_identity", "invalid_identity"),
("manifest_json", "archive_manifest", "invalid_stored_data"),
(
"manifest_identity",
"archive_integrity",
"archive_identity_mismatch",
),
("storage_json", "archive_integrity", "invalid_stored_data"),
("checksum", "archive_integrity", "archive_checksum_mismatch"),
("schema", "session_conversion", "unsupported_session_schema"),
("log", "session_conversion", "invalid_session_log"),
(
"resource_limit",
"session_conversion",
"input_resource_limit_exceeded",
),
] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("private-runtime-root");
drop(
crate::Runtime::with_fs_store(
crate::fs_store::FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a"),
)
.unwrap(),
);
let worker = WorkerId::now_v7();
let segment = old_compacted_source(&root, worker);
let provider = FsWorkerRetentionProvider::new(&root);
let worker_dir = provider.worker_dir(worker);
let log = worker_dir
.join("session/segments")
.join(format!("{segment}.jsonl"));
if case == "log" {
fs::write(&log, format!("{{\"type\":\"{PRIVATE}\"}}\n")).unwrap();
} else if case == "schema" {
let path = worker_dir.join("session/session.json");
let mut manifest: serde_json::Value = read_json(&path, "test").unwrap();
manifest["schema_version"] = serde_json::json!(999);
write_json(&path, &manifest);
} else if case == "resource_limit" {
let mut bytes = fs::read_to_string(&log).unwrap();
for line in include_str!("retention/fixtures/system-input-v4.jsonl").lines() {
let mut entry: serde_json::Value = serde_json::from_str(line).unwrap();
if let Some(item) = entry.pointer_mut("/entry/item")
&& item.get("snapshot").is_some()
{
item["snapshot"] = serde_json::json!({
"private": PRIVATE.repeat(128 * 1024 / PRIVATE.len() + 1)
});
}
bytes.push_str(&serde_json::to_string(&entry).unwrap());
bytes.push('\n');
}
fs::write(&log, bytes).unwrap();
}
commit_session_archive(
&provider,
&request(worker, SessionDisposition::Archive),
&worker_dir,
)
.unwrap();
fs::remove_dir_all(worker_dir).unwrap();
let archive = provider.archive_dir("archive-a").unwrap();
match case {
"identity_json" => write_json(
&archive.join("session/session.json"),
&serde_json::json!({"session_id": [PRIVATE]}),
),
"identity_value" => write_json(
&archive.join("session/session.json"),
&serde_json::json!({"session_id": PRIVATE}),
),
"manifest_json" => write_json(
&archive.join("manifest.json"),
&serde_json::json!({"schema_version": PRIVATE}),
),
"manifest_identity" => {
let path = archive.join("manifest.json");
let mut manifest: WorkerSessionArchiveManifest =
read_json(&path, "test").unwrap();
manifest.archive_id = PRIVATE.into();
write_json(&path, &manifest);
}
"storage_json" => write_json(
&archive.join(ARCHIVE_STORAGE_FILE),
&serde_json::json!({"schema_version": PRIVATE}),
),
"checksum" => fs::write(
archive
.join("session/segments")
.join(format!("{segment}.jsonl")),
PRIVATE,
)
.unwrap(),
_ => {}
}
let snapshot = || {
collect_files(&archive, "test")
.unwrap()
.into_iter()
.map(|(relative, path)| (relative, fs::read(path).unwrap()))
.collect::<Vec<_>>()
};
let before = snapshot();
for _ in 0..2 {
let output = Arc::new(std::sync::Mutex::new(Vec::new()));
let writer = Capture(output.clone());
let subscriber = tracing_subscriber::fmt()
.with_ansi(false)
.without_time()
.with_writer(move || writer.clone())
.finish();
tracing::subscriber::with_default(subscriber, || {
tracing::callsite::rebuild_interest_cache();
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
});
let log = String::from_utf8(output.lock().unwrap().clone()).unwrap();
assert!(log.contains("session_archive_unavailable"), "{case}: {log}");
assert!(log.contains("archive_id=archive-a"), "{case}: {log}");
assert!(log.contains(&format!("stage=\"{stage}\"")), "{case}: {log}");
assert!(
log.contains(&format!("reason=\"{reason}\"")),
"{case}: {log}"
);
assert!(log.contains("original bytes retained"), "{case}: {log}");
assert!(!log.contains("private-session-content"), "{case}: {log}");
assert!(!log.contains("/private"), "{case}: {log}");
assert!(!log.contains("private-runtime-root"), "{case}: {log}");
assert_eq!(snapshot(), before, "{case}");
assert!(!archive.join(ARCHIVE_STORAGE_FILE).exists() || case == "storage_json");
}
}
}
#[test]
fn session_migration_does_not_read_or_upgrade_erased_originals() {
let temp = tempfile::tempdir().unwrap();
@@ -0,0 +1,2 @@
{"kind":"annotated_system_item","ts":101,"entry":{"item":{"kind":"journal_startup","body":"journal body\nunchanged","snapshot":{"entries":[{"text":"private snapshot"}]},"snapshot_sha256":"snapshot-digest","prompt_provenance":{"workspace_id":"workspace","source_digest":"source","projection_digest":"projection","logical_name":"journal_startup"}},"metadata":{"entry_id":"journal-id","origin":{"kind":"legacy_unknown"},"derivation":{"sources":["source-id"]}}},"extensions":[{"domain":"migration-fixture","payload":{"kept":true}}]}
{"kind":"annotated_system_item","ts":103,"entry":{"item":{"kind":"subject_behavior_refresh","body":"rendered subject body","behavior_md":"exact behavior","prompt_provenance":{"source_digest":"subject-source","projection_digest":"subject-projection","logical_name":"subject_behavior"}},"metadata":{"entry_id":"subject-id","origin":{"kind":"backend_instruction","operation_id":"op"}}}}
@@ -0,0 +1,774 @@
//! Runtime integration, not a second implementation of Session migration.
//! The Feature records are frozen v4 wire copied from the Session Store fixture;
//! only the empty segment envelope is built with current APIs.
use super::tests::{request, write_json};
use super::*;
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
use serde_json::{Value, json};
use session_store::{LogEntry, SegmentId, SessionId, Store, WorkerSessionStore};
use std::collections::BTreeMap;
const V4_SYSTEM_INPUTS: &str = include_str!("fixtures/system-input-v4.jsonl");
fn write_v4_session(aggregate: &Path) -> (SessionId, SegmentId) {
let root = aggregate.join("session");
let session = session_store::new_session_id();
let segment = session_store::new_segment_id();
let store = WorkerSessionStore::new(&root).unwrap();
store
.create_segment(
session,
segment,
&[LogEntry::AnnotatedSegmentStart {
ts: 1,
session_id: session,
system_prompt: None,
config: Default::default(),
history: vec![],
forked_from: None,
compacted_from: None,
}],
)
.unwrap();
let path = root.join("segments").join(format!("{segment}.jsonl"));
let mut bytes = fs::read(&path).unwrap();
bytes.extend_from_slice(V4_SYSTEM_INPUTS.as_bytes());
fs::write(path, bytes).unwrap();
write_json(
&root.join("session.json"),
&json!({
"schema_version": 4, "session_id": session,
}),
);
(session, segment)
}
fn stopped_worker(root: &Path, schema: u32) -> (WorkerId, SessionId, SegmentId) {
let worker = WorkerId::now_v7();
let aggregate = root.join("workers").join(worker.to_string());
// Serialize request defaults just as a production aggregate writer does.
let request: crate::catalog::CreateWorkerRequest = serde_json::from_value(json!({
"worker_id": worker,
"workspace_api": {"workspace_id": "workspace-a", "base_url": "https://workspace.invalid"},
"create_fingerprint": "runtime-v5-fixture",
"profile": {"kind": "builtin", "value": "builtin:coder"},
"profile_source": {"kind": "workspace_config", "archive": {
"id": "archive-1", "digest": "sha256:archive", "size_bytes": 0,
"source_graph": {"source_count": 0, "total_source_bytes": 0,
"entrypoints": {}, "import_count": 0}
}}
}))
.unwrap();
write_json(
&aggregate.join("worker.json"),
&json!({
"schema_version": schema,
"worker_ref": {"worker_id": worker}, "worker_id": worker,
"profile": request.profile,
"profile_source": request.profile_source.reference(),
"display_name": null, "config_bundle": null,
"created_at_ms": 1234, "status": "stopped", "workspace_id": "workspace-a",
"last_finished_submission_request_id": "retained-submission",
"restore_guard": {"observation_token": "retained-token",
"active_request_id": null, "owners": {}},
"execution_state": {"state": "available", "execution": {
"request": request, "binding": {}, "restore_intent": "explicit"
}}
}),
);
let (session, segment) = write_v4_session(&aggregate);
let mut metadata = session_store::WorkerMetadata::new(
format!("worker-runtime-{worker}"),
Some(session_store::WorkerActiveSegmentRef::active_segment(
session, segment,
)),
);
metadata.workspace_id = Some("workspace-a".into());
write_json(&aggregate.join("metadata.json"), &metadata);
(worker, session, segment)
}
fn runtime_fixture(root: &Path, schema: u32) -> (WorkerId, SessionId, SegmentId) {
fs::create_dir_all(root.join("workers")).unwrap();
let fenced_worker = WorkerId::now_v7();
write_json(
&root.join("runtime.json"),
&json!({
"schema_version": schema, "display_name": "retained Runtime",
"backend": "fs_store", "status": "running", "next_diagnostic_id": 1,
"config_bundles": {}, "workspace_owners": {"workspace-a": "owner-a"},
"diagnostics": [], "subject_releases": [],
"subject_create_recovery_fences": {fenced_worker.to_string(): {
"request": {"workspace_id": "workspace-a", "worker_id": fenced_worker,
"create_fingerprint": "retained-create", "operation_id": "retained-recovery"},
"response": null
}}
}),
);
stopped_worker(root, schema)
}
// Include every file, not just schema markers: failures must preserve original
// history, metadata, archive receipts, checksums and retention evidence together.
fn tree_bytes(root: &Path) -> BTreeMap<PathBuf, Vec<u8>> {
fn walk(root: &Path, at: &Path, result: &mut BTreeMap<PathBuf, Vec<u8>>) {
for entry in fs::read_dir(at).unwrap() {
let entry = entry.unwrap();
if entry.file_type().unwrap().is_dir() {
walk(root, &entry.path(), result);
} else {
result.insert(
entry.path().strip_prefix(root).unwrap().to_path_buf(),
fs::read(entry.path()).unwrap(),
);
}
}
}
let mut result = BTreeMap::new();
walk(root, root, &mut result);
result
}
fn assert_generic(root: &Path, session: SessionId, segment: SegmentId) {
use session_store::{InputCompactionUse, InputEvidenceUse, InputObservationUse};
let store = WorkerSessionStore::open_read_only(root).unwrap();
let entries = store.read_all_read_only(session, segment).unwrap();
assert_eq!(entries.len(), 3);
let originals: Vec<Value> = V4_SYSTEM_INPUTS
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
for (log, old) in entries[1..].iter().zip(originals) {
let LogEntry::AnnotatedSystemItem { entry, .. } = log else {
panic!("lost System record: {log:?}");
};
let current = serde_json::to_value(log).unwrap();
assert_eq!(current["ts"], old["ts"]);
for field in ["entry_id", "origin", "derivation"] {
assert_eq!(
current["entry"]["metadata"][field],
old["entry"]["metadata"][field]
);
}
assert_eq!(current["entry"]["item"]["kind"], "input");
assert_eq!(
entry.item.history_text(),
old["entry"]["item"]["body"].as_str().unwrap()
);
let input = entry.metadata.input_metadata.as_ref().unwrap();
let usage = input.usage();
assert_eq!(usage.recording_material, InputEvidenceUse::Exclude);
assert_eq!(usage.memory_evidence, InputEvidenceUse::Exclude);
assert_eq!(usage.observation, InputObservationUse::Hidden);
let kind = old["entry"]["item"]["kind"].as_str().unwrap();
let schema = if kind == "journal_startup" {
assert_eq!(usage.compaction, InputCompactionUse::ReferenceInput);
assert_eq!(current["extensions"], old["extensions"]);
"journal:startup"
} else {
assert_eq!(usage.compaction, InputCompactionUse::Retain);
"subject:behavior"
};
let payload = input.feature_payload().unwrap();
assert_eq!(payload.schema(), schema);
assert_eq!(payload.version(), 1);
let mut expected_payload = old["entry"]["item"].clone();
expected_payload.as_object_mut().unwrap().remove("kind");
expected_payload.as_object_mut().unwrap().remove("body");
assert_eq!(
payload.decode::<Value>(schema, 1).unwrap(),
expected_payload
);
let public = serde_json::to_value(input.public_item().unwrap()).unwrap();
assert_eq!(public["kind"], kind);
assert_eq!(public["body"], old["entry"]["item"]["body"]);
assert_eq!(public["presentation"], json!({"kind": "context"}));
}
}
fn completed_v4_archive(
root: &Path,
schema: u32,
) -> (
SessionId,
SegmentId,
WorkerRetentionExecutionRequest,
WorkerRetentionExecutionResult,
) {
let (worker, session, segment) = stopped_worker(root, schema);
let provider = FsWorkerRetentionProvider::new(root);
let request = request(worker, SessionDisposition::Archive);
// Frozen old-runtime receipt: ingestion and removal completed before the
// current strict decoder existed. Do not ask the v5 reader to mint it.
let manifest =
commit_session_archive(&provider, &request, &provider.worker_dir(worker)).unwrap();
let completed = WorkerRetentionExecutionResult {
operation_id: request.operation_id.clone(),
input_fingerprint: request.input_fingerprint.clone(),
worker_id: worker,
session_disposition: SessionDisposition::Archive,
diagnostics_disposition: DiagnosticsDisposition::Purge,
archive: Some(manifest),
source_removed: true,
diagnostics_retained: false,
};
write_json(
&provider.operation_path(&request.operation_id).unwrap(),
&RetentionOperationReceipt {
schema_version: OPERATION_SCHEMA_VERSION,
request: request.clone(),
result: completed.clone(),
},
);
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
(session, segment, request, completed)
}
#[test]
fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
for runtime_schema in [11, 12] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
let orphan = root
.join("orphaned-workers")
.join(WorkerId::now_v7().to_string());
let (orphan_session, orphan_segment) = write_v4_session(&orphan);
// Include an unregistered tree still under workers, not just an
// already-archived orphan. Both Runtime source schemas must fence it
// out of the live catalog without discarding its Session.
let loose_id = WorkerId::now_v7();
let loose = root.join("workers").join(loose_id.to_string());
let (loose_session, loose_segment) = write_v4_session(&loose);
let retained_loose = root.join("orphaned-workers").join(loose_id.to_string());
let (archived_session, archived_segment, request, completed) =
completed_v4_archive(&root, runtime_schema);
let provider = FsWorkerRetentionProvider::new(&root);
let archive = provider.archive_dir("archive-a").unwrap();
// Also cover an archive which already has a local storage attestation
// from an earlier schema migration. Keep ingestion bytes immutable;
// harmless marker whitespace makes current storage differ from them.
if runtime_schema == 12 {
let marker = archive.join("session/session.json");
let mut bytes = fs::read(&marker).unwrap();
bytes.push(b'\n');
fs::write(marker, bytes).unwrap();
let ingestion = completed.archive.clone().unwrap();
let (checksum_sha256, bytes, file_count) = checksum_files(
&collect_files(&archive.join("session"), "test").unwrap(),
"test",
)
.unwrap();
write_json(
&archive.join(ARCHIVE_STORAGE_FILE),
&MigratedArchiveStorage {
schema_version: 1,
ingestion,
checksum_sha256,
bytes,
file_count,
},
);
}
let storage_before = fs::read(archive.join(ARCHIVE_STORAGE_FILE)).ok();
let manifest_before = fs::read(archive.join("manifest.json")).unwrap();
let receipt_path = provider.operation_path(&request.operation_id).unwrap();
let receipt_before = fs::read(&receipt_path).unwrap();
let stopped = root.join("workers").join(worker.to_string());
assert!(WorkerSessionStore::open_read_only(stopped.join("session")).is_err());
// Current decode itself must not be made permissive to fix routing.
for line in V4_SYSTEM_INPUTS.lines() {
assert!(serde_json::from_str::<LogEntry>(line).is_err());
}
let before = tree_bytes(&root);
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
let plan = FsRuntimeStore::migration_plan(&options).unwrap();
assert!(plan.migration_required);
assert_eq!(plan.archived_orphan_worker_count, 1);
assert_eq!(plan.current_schema_version, runtime_schema);
assert_eq!(tree_bytes(&root), before, "preflight mutated source");
// Exercise normal startup, not just the private migration helper.
let opened = FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap();
assert!(
matches!(
FsRuntimeStore::migrate(&options),
Err(RuntimeError::RuntimeStoreAlreadyOpen { .. })
),
"owner lock not fenced"
);
drop(opened);
assert_generic(&stopped.join("session"), session, segment);
assert_generic(&orphan.join("session"), orphan_session, orphan_segment);
assert!(!loose.exists());
assert_generic(
&retained_loose.join("session"),
loose_session,
loose_segment,
);
assert_generic(&archive.join("session"), archived_session, archived_segment);
assert_eq!(
fs::read(archive.join("manifest.json")).unwrap(),
manifest_before
);
assert_eq!(fs::read(&receipt_path).unwrap(), receipt_before);
let (manifest, archive_session) = provider.session_archive("archive-a").unwrap();
assert_eq!(completed.archive.as_ref(), Some(&manifest));
assert_eq!(provider.execute(&request).unwrap(), completed);
if let Some(storage_before) = storage_before {
assert_ne!(
fs::read(archive.join(ARCHIVE_STORAGE_FILE)).unwrap(),
storage_before
);
}
let storage: MigratedArchiveStorage =
read_json(&archive.join(ARCHIVE_STORAGE_FILE), "test").unwrap();
assert_eq!(storage.ingestion, manifest);
let (checksum, bytes, count) =
checksum_files(&collect_files(&archive_session, "test").unwrap(), "test").unwrap();
assert_eq!(
(storage.checksum_sha256, storage.bytes, storage.file_count),
(checksum, bytes, count)
);
if runtime_schema == 12 {
for name in [
PathBuf::from("runtime.json"),
PathBuf::from(format!("workers/{worker}/worker.json")),
PathBuf::from(format!("workers/{worker}/metadata.json")),
] {
assert_eq!(fs::read(root.join(&name)).unwrap(), before[&name]);
}
}
let once = tree_bytes(&root);
assert!(
!FsRuntimeStore::migrate(&options)
.unwrap()
.migration_required
);
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
assert_eq!(tree_bytes(&root), once, "retry rewrote migrated evidence");
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
// session-storage, not the immutable receipt's old checksum, attests
// current bytes. Tampering with those bytes must remain detectable.
let path = archive_session
.join("segments")
.join(format!("{archived_segment}.jsonl"));
fs::write(path, b"tampered\n").unwrap();
assert!(provider.session_archive("archive-a").is_err());
}
}
#[test]
fn corrupt_archive_does_not_block_startup_or_healthy_session_migration_and_repair_converges() {
for runtime_schema in [11, 12] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
let (archived_session, archived_segment, request, completed) =
completed_v4_archive(&root, runtime_schema);
let provider = FsWorkerRetentionProvider::new(&root);
let archive = provider.archive_dir("archive-a").unwrap();
let path = archive
.join("session/segments")
.join(format!("{archived_segment}.jsonl"));
let original = fs::read(&path).unwrap();
fs::write(&path, b"corrupt archive\n").unwrap();
let (healthy_worker, healthy_session, healthy_segment) =
stopped_worker(&root, runtime_schema);
let mut healthy_request =
super::tests::request(healthy_worker, SessionDisposition::Archive);
healthy_request.operation_id = "healthy-operation".into();
healthy_request.archive_id = Some("healthy-archive".into());
let healthy_manifest = commit_session_archive(
&provider,
&healthy_request,
&provider.worker_dir(healthy_worker),
)
.unwrap();
fs::remove_dir_all(provider.worker_dir(healthy_worker)).unwrap();
let before = tree_bytes(&archive);
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
assert!(
FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
assert_eq!(tree_bytes(&archive), before);
assert!(provider.session_archive("archive-a").is_err());
let (manifest, healthy_root) = provider.session_archive("healthy-archive").unwrap();
assert_eq!(manifest, healthy_manifest);
assert_generic(&healthy_root, healthy_session, healthy_segment);
assert!(!archive.join(ARCHIVE_STORAGE_FILE).exists());
assert_generic(
&root
.join("workers")
.join(worker.to_string())
.join("session"),
session,
segment,
);
assert!(
!FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
assert_eq!(tree_bytes(&archive), before);
fs::write(&path, original).unwrap();
FsRuntimeStore::migrate(&options).unwrap();
assert_generic(
&root
.join("workers")
.join(worker.to_string())
.join("session"),
session,
segment,
);
assert_generic(&archive.join("session"), archived_session, archived_segment);
assert_eq!(provider.execute(&request).unwrap(), completed);
}
}
#[test]
fn unmigratable_archive_preserves_evidence_across_startup_and_reopen() {
for (field, value) in [
("snapshot_sha256", json!(42)),
("behavior_md", json!([])),
("snapshot", json!({"large": "x".repeat(128 * 1024 + 1)})),
("body", json!("x".repeat(256 * 1024 + 1))),
] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (live_worker, live_session, live_segment) = runtime_fixture(&root, 12);
let (worker, _, segment) = stopped_worker(&root, 12);
let provider = FsWorkerRetentionProvider::new(&root);
let path = provider
.worker_dir(worker)
.join("session/segments")
.join(format!("{segment}.jsonl"));
let source = fs::read_to_string(&path).unwrap();
let mut replaced = false;
let mut altered = String::new();
for line in source.lines() {
let mut entry: Value = serde_json::from_str(line).unwrap();
if let Some(item) = entry.pointer_mut("/entry/item")
&& item.get(field).is_some()
{
item[field] = value.clone();
replaced = true;
}
altered.push_str(&serde_json::to_string(&entry).unwrap());
altered.push('\n');
}
assert!(replaced);
fs::write(&path, altered).unwrap();
let request = request(worker, SessionDisposition::Archive);
let manifest =
commit_session_archive(&provider, &request, &provider.worker_dir(worker)).unwrap();
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
let archive = provider.archive_dir("archive-a").unwrap();
validate_existing_archive(&archive, &manifest, false).unwrap();
let before = tree_bytes(&archive);
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
for _ in 0..2 {
let runtime = crate::Runtime::with_fs_store(options.clone()).unwrap();
assert_eq!(runtime.summary().unwrap().worker_count, 1);
runtime.bind_runtime_identity("runtime-a").unwrap();
let search = runtime_api::SessionPublicSearchRequest {
workspace_id: "workspace-a".into(),
source: runtime_api::SessionPublicSource::Archived {
archive_id: "archive-a".into(),
},
expected_session_id: manifest.source_session_id.clone(),
expected_generation: None,
query: None,
kind: None,
tool_name: None,
tool_part: runtime_api::SessionPublicToolPart::Both,
scan_cursor: None,
limit: 10,
max_scan_bytes: 1024 * 1024,
max_segments: 10,
max_entries: 100,
};
let availability = runtime
.session_public_search_scoped(
&crate::runtime::RuntimeWorkspaceScope::new("workspace-a", "owner-a"),
&search,
)
.unwrap();
assert!(matches!(
availability,
runtime_api::SessionPublicSearchAvailability::Unavailable {
reason: runtime_api::SessionPublicUnavailableReason::ArchiveIncomplete,
..
}
));
drop(runtime);
assert_eq!(tree_bytes(&archive), before);
validate_existing_archive(&archive, &manifest, false).unwrap();
assert!(matches!(
provider.session_archive("archive-a"),
Err(RuntimeError::StoreCorrupt { .. })
));
assert!(
!FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
assert_generic(
&provider.worker_dir(live_worker).join("session"),
live_session,
live_segment,
);
}
}
}
#[test]
fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() {
for (path, field, value) in [
("session/session.json", None, Value::Null),
("manifest.json", None, Value::Null),
(ARCHIVE_STORAGE_FILE, None, Value::Null),
("session/session.json", Some("schema_version"), json!(999)),
("manifest.json", Some("schema_version"), json!(999)),
(
"manifest.json",
Some("archive_id"),
json!("another-archive"),
),
] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, _, _) = runtime_fixture(&root, 12);
completed_v4_archive(&root, 12);
let provider = FsWorkerRetentionProvider::new(&root);
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
let archive = provider.archive_dir("archive-a").unwrap();
if let Some(field) = field {
let mut record: Value = read_json(&archive.join(path), "test").unwrap();
record[field] = value;
write_json(&archive.join(path), &record);
} else {
fs::write(archive.join(path), b"{broken\n").unwrap();
}
let before = tree_bytes(&archive);
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
for _ in 0..2 {
assert!(
!FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
let runtime = crate::Runtime::with_fs_store(options.clone()).unwrap();
assert_eq!(runtime.summary().unwrap().worker_count, 0);
drop(runtime);
assert!(provider.session_archive("archive-a").is_err());
assert_eq!(tree_bytes(&archive), before);
}
}
}
#[test]
fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
runtime_fixture(&root, 12);
let (session, _, _, _) = completed_v4_archive(&root, 12);
let ledger = root.join("retention/session-sources");
fs::create_dir_all(&ledger).unwrap();
fs::write(ledger.join(format!("{session}.json")), b"{broken\n").unwrap();
let before = tree_bytes(&root);
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
assert!(FsRuntimeStore::migration_plan(&options).is_err());
assert!(crate::Runtime::with_fs_store(options).is_err());
assert_eq!(tree_bytes(&root), before);
}
#[test]
fn runtime_v5_probe_and_staging_skip_erased_originals() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, live_session, live_segment) = runtime_fixture(&root, 12);
let erased = root
.join("orphaned-workers")
.join(WorkerId::now_v7().to_string());
let (session, segment) = write_v4_session(&erased);
runtime_session_retention(&root)
.delete(
session,
session_store::retention::SessionDeletionIntent::ErasePermanently,
|| Ok(()),
)
.unwrap();
fs::write(
erased
.join("session/segments")
.join(format!("{segment}.jsonl")),
b"fenced malformed JSON\n",
)
.unwrap();
let before = tree_bytes(&erased);
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
assert!(
FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
FsRuntimeStore::migrate(&options).unwrap();
assert_generic(
&root
.join("workers")
.join(worker.to_string())
.join("session"),
live_session,
live_segment,
);
assert_eq!(tree_bytes(&erased), before);
assert!(
!FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
assert!(matches!(
runtime_session_retention(&root).ensure_live(session),
Err(session_store::StoreError::SessionErased(_))
));
}
#[test]
fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
for corruption in ["broken-json", "future-schema"] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, 12);
let session_root = root
.join("workers")
.join(worker.to_string())
.join("session");
if corruption == "broken-json" {
fs::write(
session_root
.join("segments")
.join(format!("{segment}.jsonl")),
b"{not valid JSON\n",
)
.unwrap();
} else {
write_json(
&session_root.join("session.json"),
&json!({
"schema_version": 999, "session_id": session,
}),
);
}
let before = tree_bytes(&root);
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
assert!(
FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
assert_eq!(tree_bytes(&root), before);
assert!(FsRuntimeStore::migrate(&options).is_err());
assert_eq!(tree_bytes(&root), before);
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
}
}
#[cfg(unix)]
#[test]
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, _, _) = runtime_fixture(&root, 12);
let before = tree_bytes(&root);
let runs = root.join("workers").join(worker.to_string()).join("runs");
fs::create_dir_all(runs.join("1")).unwrap();
let listener = std::os::unix::net::UnixListener::bind(runs.join("1/worker.sock")).unwrap();
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
let error = FsRuntimeStore::migrate(&options).unwrap_err();
assert!(error.to_string().contains("active Worker socket"));
drop(listener);
fs::remove_dir_all(runs).unwrap();
assert_eq!(tree_bytes(&root), before);
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
}
#[test]
fn current_runtime_upgrades_v5_role_metadata_through_staging_without_reserializing_owner_evidence()
{
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, 12);
let aggregate = root.join("workers").join(worker.to_string());
let path = aggregate
.join("session/segments")
.join(format!("{segment}.jsonl"));
let old = fs::read_to_string(&path).unwrap();
let header = old.lines().next().unwrap();
let source = include_str!(
"../../../session-store/src/legacy_session_log/tests/fixtures/v5-role-metadata.jsonl"
);
fs::write(&path, format!("{header}\n{source}")).unwrap();
write_json(
&aggregate.join("session/session.json"),
&json!({"schema_version":5, "session_id": session}),
);
let owner_before = fs::read(root.join("runtime.json")).unwrap();
let worker_before = fs::read(aggregate.join("worker.json")).unwrap();
let metadata_before = fs::read(aggregate.join("metadata.json")).unwrap();
assert!(WorkerSessionStore::open_read_only(aggregate.join("session")).is_err());
let options = FsRuntimeStoreOptions::new(&root).with_runtime_id("runtime-a");
assert!(
FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
FsRuntimeStore::migrate(&options).unwrap();
let store = WorkerSessionStore::open_read_only(aggregate.join("session")).unwrap();
let entries = store.read_all_read_only(session, segment).unwrap();
assert_eq!(entries.len(), 6);
for (current, old) in entries[1..].iter().zip(source.lines()) {
let current = serde_json::to_value(current).unwrap();
let old: Value = serde_json::from_str(old).unwrap();
assert_eq!(current["ts"], old["ts"]);
assert_eq!(current["entry"]["item"], old["entry"]["item"]);
for field in ["entry_id", "origin", "derivation"] {
assert_eq!(
current["entry"]["metadata"][field],
old["entry"]["metadata"][field]
);
}
if current["entry"]["metadata"]["origin"]["kind"] == "derived_summary" {
assert_eq!(
current["entry"]["metadata"]["input_metadata"]["usage"]["recording_material"],
"exclude"
);
assert_eq!(
current["entry"]["metadata"]["input_metadata"]["usage"]["memory_evidence"],
"exclude"
);
}
}
assert_eq!(fs::read(root.join("runtime.json")).unwrap(), owner_before);
assert_eq!(
fs::read(aggregate.join("worker.json")).unwrap(),
worker_before
);
assert_eq!(
fs::read(aggregate.join("metadata.json")).unwrap(),
metadata_before
);
let once = tree_bytes(&aggregate.join("session"));
assert!(
!FsRuntimeStore::migration_plan(&options)
.unwrap()
.migration_required
);
FsRuntimeStore::migrate(&options).unwrap();
assert_eq!(tree_bytes(&aggregate.join("session")), once);
}
+236 -366
View File
@@ -1865,20 +1865,23 @@ impl Runtime {
if let Some(scope) = scope {
self.ensure_worker_in_workspace(scope, worker_ref)?;
}
self.replace_worker_workdir_attachments_under_lock(worker_ref, attachments)
self.replace_worker_workdir_attachments_under_lock(worker_ref, attachments, false)
}
fn replace_worker_workdir_attachments_under_lock(
&self,
worker_ref: &WorkerRef,
mut attachments: Vec<LogicalWorkdirAttachment>,
restoring: bool,
) -> Result<WorkerDetail, RuntimeError> {
validate_logical_workdir_attachments(&attachments)?;
attachments.sort_by(|left, right| left.alias.cmp(&right.alias));
let mut state = self.lock()?;
for attachment in &attachments {
state.ensure_workdir_not_removing(&attachment.working_directory_id)?;
if !restoring {
for attachment in &attachments {
state.ensure_workdir_not_removing(&attachment.working_directory_id)?;
}
}
let previous = state.worker(worker_ref)?.clone();
{
@@ -2146,7 +2149,6 @@ impl Runtime {
),
RuntimeError,
> {
let mut preparation_authorized = !coordinate && request.preparation.is_some();
let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?;
let _operation_guard = operation_lock
.lock()
@@ -2191,7 +2193,7 @@ impl Runtime {
let conflict = || RuntimeError::RestoreObservationConflict {
worker_id: worker_ref.worker_id,
};
if let Some(mut owner) = candidate
if let Some(owner) = candidate
.restore_guard
.owners
.get(&request.request_id)
@@ -2204,13 +2206,6 @@ impl Runtime {
if let Some(receipt) = &owner.receipt {
return Ok((Some(receipt.clone()), None));
}
if let Some(preparation) = &owner.request.preparation {
for id in
preparation_workdir_ids(preparation, state.runtime_identity.as_deref())
{
state.ensure_workdir_not_removing(id)?;
}
}
if candidate.restore_guard.active_request_id.as_deref() != Some(&request.request_id)
|| candidate
.pending_restore
@@ -2218,151 +2213,6 @@ impl Runtime {
{
return Err(conflict());
}
if coordinate && !owner.preparation_applied {
if let Some(preparation) = owner.request.preparation.as_mut() {
if preparation.repository_access_workdirs.is_empty()
&& !preparation.repository_access.is_empty()
{
preparation.repository_access_workdirs = preparation
.repository_access
.iter()
.map(|access| runtime_api::RepositoryAccessWorkdirReference {
runtime_id: access.materialization.runtime_id.clone(),
working_directory_id: access.working_directory_id.clone(),
})
.collect();
candidate
.restore_guard
.owners
.insert(request.request_id.clone(), owner.clone());
match state.persist_worker_record(&candidate) {
Ok(()) => {
state
.workers
.insert(worker_ref.worker_id, candidate.clone());
}
Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => {
state.workers.insert(worker_ref.worker_id, candidate);
return Ok((
Some(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired,
"worker_restore_preparation_commit_unknown",
sanitize_worker_create_failure_message(
&error.to_string(),
),
)),
None,
));
}
Err(error) => {
return Ok((
Some(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired,
"worker_restore_preparation_commit_failed",
sanitize_worker_create_failure_message(
&error.to_string(),
),
)),
None,
));
}
}
}
}
}
if !owner.preparation_applied {
if let (Some(original), Some(incoming)) =
(&owner.request.preparation, &request.preparation)
{
// Server-regenerated bindings are not part of the public fingerprint.
// Reuse admitted bindings and accept only resource completion for those references.
let references_match = incoming.repository_access.len()
== original.repository_access_workdirs.len()
&& original.repository_access_workdirs.iter().all(|reference| {
incoming
.repository_access
.iter()
.filter(|access| {
reference.working_directory_id
== access.working_directory_id
&& reference.runtime_id
== access.materialization.runtime_id
})
.count()
== 1
});
let snapshot_matches = original.repository_access.is_empty()
|| (original.repository_access.len()
== incoming.repository_access.len()
&& original
.repository_access
.iter()
.zip(&incoming.repository_access)
.all(|(original, incoming)| {
let mut normalized = incoming.clone();
if let (Some(old), Some(new)) = (
&original.materialization.ssh,
&mut normalized.materialization.ssh,
) {
new.secret_resource = old.secret_resource.clone();
new.expires_at_epoch_seconds =
old.expires_at_epoch_seconds;
}
&normalized == original
}));
if !original.repository_access_workdirs.is_empty() {
preparation_authorized =
!coordinate && references_match && snapshot_matches;
}
if references_match
&& snapshot_matches
&& original.repository_access != incoming.repository_access
{
candidate
.restore_guard
.owners
.get_mut(&request.request_id)
.unwrap()
.request
.preparation
.as_mut()
.unwrap()
.repository_access = incoming.repository_access.clone();
match state.persist_worker_record(&candidate) {
Ok(()) => {
state
.workers
.insert(worker_ref.worker_id, candidate.clone());
}
Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => {
state.workers.insert(worker_ref.worker_id, candidate);
return Ok((
Some(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired,
"worker_restore_preparation_commit_unknown",
sanitize_worker_create_failure_message(
&error.to_string(),
),
)),
None,
));
}
Err(error) => {
return Ok((
Some(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired,
"worker_restore_preparation_commit_failed",
sanitize_worker_create_failure_message(
&error.to_string(),
),
)),
None,
));
}
}
}
}
}
} else {
// Startup/older persisted pending operations have a lifecycle
// owner but no public request receipt. Bind recovery to that
@@ -2417,13 +2267,6 @@ impl Runtime {
if recovering_pending {
request.preparation = None;
}
if let Some(preparation) = &request.preparation {
for id in
preparation_workdir_ids(preparation, state.runtime_identity.as_deref())
{
state.ensure_workdir_not_removing(id)?;
}
}
let pending = candidate
.pending_restore
.unwrap_or_else(|| PendingWorkerRestore {
@@ -2481,12 +2324,8 @@ impl Runtime {
return Ok((None, owner.request.preparation.clone()));
}
}
self.restore_admitted_worker_under_lock_with_preparation(
worker_ref,
WorkerRestoreMode::Explicit,
preparation_authorized,
)
.map(|result| (Some(result), None))
self.restore_admitted_worker_under_lock(worker_ref, WorkerRestoreMode::Explicit)
.map(|result| (Some(result), None))
}
#[cfg(all(test, feature = "fs-store"))]
@@ -2522,15 +2361,6 @@ impl Runtime {
&self,
worker_ref: &WorkerRef,
mode: WorkerRestoreMode,
) -> Result<RuntimeWorkerRestoreResult, RuntimeError> {
self.restore_admitted_worker_under_lock_with_preparation(worker_ref, mode, false)
}
fn restore_admitted_worker_under_lock_with_preparation(
&self,
worker_ref: &WorkerRef,
mode: WorkerRestoreMode,
preparation_authorized: bool,
) -> Result<RuntimeWorkerRestoreResult, RuntimeError> {
self.lock()?
.ensure_subject_not_reserved(worker_ref, false)?;
@@ -2595,15 +2425,7 @@ impl Runtime {
message,
)
};
if owner.request.preparation.as_ref().is_some_and(|prep| {
prep.repository_access.len() < prep.repository_access_workdirs.len()
|| (!prep.repository_access_workdirs.is_empty() && !preparation_authorized)
}) {
return Ok(uncertain(
"Admitted Restore is awaiting Workspace resource completion".into(),
));
}
// Complete idempotent binding/access effects under the original pending_restore ID.
// Apply reference updates under the original pending_restore ID; no provider is opened.
let mut effects_started = false;
if let Err(error) = self.apply_restore_preparation_under_lock(
worker_ref,
@@ -2744,42 +2566,10 @@ impl Runtime {
.map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?;
validate_logical_workdir_attachments(&attachments)?;
*effects_started = true;
self.replace_worker_workdir_attachments_under_lock(worker_ref, attachments)?;
}
for access in preparation.repository_access {
if preparation
.repository_access_workdirs
.iter()
.any(|reference| {
reference.working_directory_id == access.working_directory_id
&& reference.runtime_id == access.materialization.runtime_id
})
{
// Workspace completed these at the Workdir owner Runtime after snapshot persistence.
// Never send cross-Runtime credentials to this Worker's Runtime.
continue;
}
let access: WorkingDirectoryRepositoryAccessRequest = serde_json::from_value(
serde_json::to_value(access)
.map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?,
)
.map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?;
*effects_started = true;
let runtime = self.clone();
// The public embedded interface is synchronous. A separate IO executor
// avoids nested Tokio block_on while retaining the lifecycle lock.
std::thread::spawn(move || {
let executor = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?;
executor.block_on(
runtime.authorize_working_directory_repository_access_from_resource(access),
)
})
.join()
.map_err(|_| RuntimeError::StatePoisoned)??;
self.replace_worker_workdir_attachments_under_lock(worker_ref, attachments, true)?;
}
// Retained legacy SSH preparation is not a prerequisite or an effect
// of restoring a Worker. Sessions resolve live access when used.
Ok(())
}
@@ -2864,6 +2654,7 @@ impl Runtime {
previous_workdir_attachments: worker.workdir_attachments.clone(),
logical_workdir_attachments: worker.logical_workdir_attachments.clone(),
workdir_attachments: BTreeMap::new(),
deferred_workdir_sessions: BTreeMap::new(),
config_bundle: None,
};
(
@@ -2876,28 +2667,6 @@ impl Runtime {
)
};
let _workdir_operation = self.reserve_workdir_use(
requested_workdir_ids(&request.request)
.into_iter()
.chain(
request
.previous_workdir_attachments
.iter()
.map(|attachment| {
attachment
.working_directory
.summary
.working_directory_id
.as_str()
}),
)
.chain(
request
.logical_workdir_attachments
.iter()
.map(|attachment| attachment.working_directory_id.as_str()),
),
)?;
if !reconcile && let Err(result) = backend.preflight_restore(&request) {
// A preflight may attest an already-connected Controller snapshot.
// Repeated restore is read-only; do not replace its operation owner.
@@ -3136,7 +2905,7 @@ impl Runtime {
Ok(receipt)
}
WorkerExecutionSpawnResult::ReconciliationRequired {
result: _result,
result,
worker_state,
workdir_attachments,
} => {
@@ -3165,7 +2934,11 @@ impl Runtime {
Ok(RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::ReconciliationRequired,
"worker_restore_reconciliation_required",
"Worker restore result is uncertain; reread or retry the Worker restore",
sanitize_worker_create_failure_message(&format!(
"Worker restore requires reconciliation ({:?}): {}",
result.outcome,
result.message_or_default(),
)),
))
}
WorkerExecutionSpawnResult::Errored(_result) => {
@@ -8944,7 +8717,7 @@ mod tests {
}
#[test]
fn active_workdir_cleanup_rejects_same_id_admissions_without_blocking_other_workers() {
fn active_workdir_cleanup_blocks_access_but_allows_reference_only_restore() {
let (runtime, backend) = runtime_and_backend();
let owner = scope("workspace-a", "server-a");
let worker = runtime
@@ -9004,36 +8777,43 @@ mod tests {
.stop_worker_scoped(&owner, &worker.worker_ref, None)
.unwrap();
let restore = runtime.test_restore_request(&worker.worker_ref);
assert_workdir_busy(
runtime
.coordinate_worker_restore_operation(
&worker.worker_ref,
runtime_api::WorkerRestoreCoordinationRequest {
expected_observation_token: restore.expected_observation_token,
request_id: restore.request_id,
preparation: Some(runtime_api::WorkerRestorePreparation {
workdir_attachments: Some(vec![
runtime_api::LogicalWorkdirAttachment {
alias: attachment.alias,
working_directory_id: attachment.working_directory_id,
capabilities: attachment.capabilities,
},
]),
..Default::default()
}),
},
Some(&owner),
)
.unwrap_err(),
);
assert!(
let (result, preparation) = runtime
.coordinate_worker_restore_operation(
&worker.worker_ref,
runtime_api::WorkerRestoreCoordinationRequest {
expected_observation_token: restore.expected_observation_token.clone(),
request_id: restore.request_id.clone(),
preparation: Some(runtime_api::WorkerRestorePreparation {
workdir_attachments: Some(vec![runtime_api::LogicalWorkdirAttachment {
alias: attachment.alias.clone(),
working_directory_id: attachment.working_directory_id.clone(),
capabilities: attachment.capabilities,
}]),
..Default::default()
}),
},
Some(&owner),
)
.unwrap();
assert!(result.is_none());
assert!(preparation.is_some());
let restored = runtime
.restore_worker_operation_scoped(&owner, &worker.worker_ref, restore)
.unwrap();
assert_eq!(restored.state, WorkerRestoreState::Accepted);
assert_eq!(
runtime
.lock()
.unwrap()
.worker(&worker.worker_ref)
.unwrap()
.pending_restore
.is_none()
.logical_workdir_attachments,
vec![attachment]
);
assert_workdir_busy(
runtime
.open_workdir_session_scoped(&owner, id, None)
.unwrap_err(),
);
assert_eq!(runtime.list_workers_scoped(&owner).unwrap().len(), 1);
// Different IDs and Worker creation remain available during cleanup.
@@ -10795,102 +10575,68 @@ mod tests {
}
#[test]
fn restore_guard_resource_snapshot_requires_workspace_completion_before_execution() {
let (runtime, backend) = runtime_and_backend();
let scope = scope("workspace-a", "server-a");
let worker = runtime
.create_worker_scoped(
&scope,
scoped_task_request("resource coordination", "workspace-a"),
)
.unwrap();
runtime.stop_worker(&worker.worker_ref, None).unwrap();
let mut request = runtime.test_restore_request(&worker.worker_ref);
let mut preparation = runtime_api::WorkerRestorePreparation::default();
preparation.repository_access_workdirs.push(
runtime_api::RepositoryAccessWorkdirReference {
runtime_id: "peer-runtime".into(),
working_directory_id: "peer-workdir".into(),
},
);
let mut coordination = runtime_api::WorkerRestoreCoordinationRequest {
expected_observation_token: request.expected_observation_token.clone(),
request_id: request.request_id.clone(),
preparation: Some(preparation.clone()),
};
runtime
.coordinate_worker_restore_operation(
&worker.worker_ref,
coordination.clone(),
Some(&scope),
)
.unwrap();
let operation_id = runtime
.lock()
.unwrap()
.worker(&worker.worker_ref)
.unwrap()
.pending_restore
.unwrap()
.operation_id;
// Model a retained resource snapshot before the peer authorization callback completes.
preparation
.repository_access
.push(runtime_api::WorkingDirectoryRepositoryAccessRequest {
working_directory_id: "peer-workdir".into(),
materialization: runtime_api::RepositoryMaterializationContext {
workspace_id: "workspace-a".into(),
runtime_id: "peer-runtime".into(),
operation_id: format!("worker-restore:{}", request.request_id),
config_projection_digest: "snapshot".into(),
ssh: None,
fn retained_restore_resource_snapshot_does_not_require_workspace_completion() {
for completed in [false, true] {
let (runtime, backend) = runtime_and_backend();
let scope = scope("workspace-a", "server-a");
let worker = runtime
.create_worker_scoped(
&scope,
scoped_task_request("resource coordination", "workspace-a"),
)
.unwrap();
runtime.stop_worker(&worker.worker_ref, None).unwrap();
let mut request = runtime.test_restore_request(&worker.worker_ref);
let mut preparation = runtime_api::WorkerRestorePreparation::default();
preparation.repository_access_workdirs.push(
runtime_api::RepositoryAccessWorkdirReference {
runtime_id: "offline-peer".into(),
working_directory_id: "peer-workdir".into(),
},
});
coordination.preparation = Some(preparation.clone());
assert!(
runtime
.coordinate_worker_restore_operation(&worker.worker_ref, coordination, Some(&scope))
.unwrap()
.1
.is_some()
);
request.preparation = None;
assert_eq!(
runtime
);
if completed {
preparation.repository_access.push(
runtime_api::WorkingDirectoryRepositoryAccessRequest {
working_directory_id: "peer-workdir".into(),
materialization: runtime_api::RepositoryMaterializationContext {
workspace_id: "workspace-a".into(),
runtime_id: "offline-peer".into(),
operation_id: format!("worker-restore:{}", request.request_id),
config_projection_digest: "snapshot".into(),
ssh: None,
},
},
);
}
for include_resources in [false, completed] {
let mut admitted = preparation.clone();
if !include_resources {
admitted.repository_access.clear();
}
runtime
.coordinate_worker_restore_operation(
&worker.worker_ref,
runtime_api::WorkerRestoreCoordinationRequest {
expected_observation_token: request.expected_observation_token.clone(),
request_id: request.request_id.clone(),
preparation: Some(admitted),
},
Some(&scope),
)
.unwrap();
}
request.preparation = None;
let restored = runtime
.restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone())
.unwrap()
.state,
WorkerRestoreState::ReconciliationRequired
);
assert_eq!(
runtime
.restore_admitted_worker_under_lock(&worker.worker_ref, WorkerRestoreMode::Explicit)
.unwrap()
.state,
WorkerRestoreState::ReconciliationRequired
);
assert_eq!(*backend.restore_count.lock().unwrap(), 0);
request.preparation = Some(runtime_api::WorkerRestorePreparation::default());
assert_eq!(
runtime
.restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone())
.unwrap()
.state,
WorkerRestoreState::ReconciliationRequired
);
assert_eq!(*backend.restore_count.lock().unwrap(), 0);
request.preparation = Some(preparation);
assert_eq!(
runtime
.unwrap();
assert_eq!(restored.state, WorkerRestoreState::Accepted);
let replay = runtime
.restore_worker_operation_scoped(&scope, &worker.worker_ref, request)
.unwrap()
.state,
WorkerRestoreState::Accepted
);
assert_eq!(
backend.restore_operation_ids.lock().unwrap().as_slice(),
&[operation_id]
);
.unwrap();
assert_eq!(replay, restored);
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
assert!(backend.repository_accesses.lock().unwrap().is_empty());
}
}
#[test]
@@ -12434,6 +12180,7 @@ mod tests {
session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: vec![protocol::SessionSnapshotEntry {
use_policy: None,
entry_id: "restored-log-entry".to_owned(),
timestamp: 1,
provenance: protocol::SessionEntryProvenance::LegacyUnknown,
@@ -14007,6 +13754,129 @@ mod tests {
let _ = std::fs::remove_dir_all(root);
}
#[test]
fn restore_reconciliation_exposes_bounded_redacted_backend_reason() {
for has_state in [false, true] {
let (runtime, backend) = runtime_and_backend();
let worker = runtime
.create_worker(task_request("restore diagnosis"))
.unwrap();
let worker_state = if has_state {
runtime
.worker_detail(&worker.worker_ref)
.unwrap()
.worker_state
} else {
None
};
runtime.stop_worker(&worker.worker_ref, None).unwrap();
*backend.restore_result.lock().unwrap() = Some(
WorkerExecutionSpawnResult::ReconciliationRequired {
result: WorkerExecutionResult::busy(
WorkerExecutionOperation::Restore,
format!(
"controller acknowledgement timed out at /private/runtime/worker.sock token=restore-token password hidden-value {}",
"x".repeat(512),
),
),
worker_state,
workdir_attachments: Vec::new(),
},
);
let result = runtime
.restore_worker_operation(
&worker.worker_ref,
runtime.test_restore_request(&worker.worker_ref),
)
.unwrap();
assert_eq!(result.state, WorkerRestoreState::ReconciliationRequired);
assert_eq!(
result.reason_code.as_deref(),
Some("worker_restore_reconciliation_required")
);
let message = result.message.unwrap();
assert!(
message.contains("controller acknowledgement timed out"),
"{message}"
);
assert!(message.contains("Busy"), "{message}");
assert!(!message.contains("/private"), "{message}");
assert!(!message.contains("restore-token"), "{message}");
assert!(!message.contains("hidden-value"), "{message}");
assert!(message.chars().count() <= WORKER_CREATE_FAILURE_MESSAGE_LIMIT + 1);
assert!(
runtime
.lock()
.unwrap()
.worker(&worker.worker_ref)
.unwrap()
.pending_restore
.is_some()
);
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
}
}
#[cfg(feature = "fs-store")]
#[test]
fn startup_restore_persists_redacted_reconciliation_reason() {
// Real Runtime startup and durable diagnostics; only execution is substituted.
let dir = tempfile::tempdir().unwrap();
let options = FsRuntimeStoreOptions::new(dir.path().join("runtime"))
.with_runtime_id("restore-diagnostics");
let initial = Runtime::with_fs_store_and_execution_backend(
options.clone(),
Arc::new(TestExecutionBackend::default()),
)
.unwrap();
initial.store_config_bundle(test_bundle()).unwrap();
let worker = initial
.create_worker(task_request("startup diagnosis"))
.unwrap();
drop(initial);
let backend = Arc::new(TestExecutionBackend::default());
*backend.restore_result.lock().unwrap() =
Some(WorkerExecutionSpawnResult::ReconciliationRequired {
result: WorkerExecutionResult::errored(
WorkerExecutionOperation::Restore,
"controller connection timed out at /private/worker.sock token=hidden-token",
),
worker_state: None,
workdir_attachments: Vec::new(),
});
let restored =
Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap();
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
assert!(
restored
.lock()
.unwrap()
.worker(&worker.worker_ref)
.unwrap()
.pending_restore
.is_some()
);
drop(restored);
let reopened = Runtime::with_fs_store(options).unwrap();
let state = reopened.lock().unwrap();
let diagnostic = state
.diagnostics
.iter()
.find(|diagnostic| {
diagnostic.code == "worker_execution_restore_failed"
&& diagnostic.worker_ref.as_ref() == Some(&worker.worker_ref)
})
.unwrap();
assert!(
diagnostic
.message
.contains("controller connection timed out")
);
assert!(diagnostic.message.contains("Errored"));
assert!(!diagnostic.message.contains("/private"));
assert!(!diagnostic.message.contains("hidden-token"));
}
#[test]
fn rolled_back_restore_settles_the_pending_operation() {
let (runtime, backend) = runtime_and_backend();
@@ -398,6 +398,7 @@ mod tests {
content: vec![LoggedContentPart::Text { text: text.into() }],
},
metadata: LoggedSessionHistoryMetadata {
input_metadata: None,
entry_id: LoggedSessionHistoryEntryId(id.to_string()),
origin: LoggedSessionHistoryOrigin::HumanInput {
account_id: "account-1".to_string(),
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,322 @@
//! Restore retains attachment identity without opening a provider. Failed opens
//! are retryable at operation time; closing an unopened session never opens it.
use std::sync::Arc;
use async_trait::async_trait;
use tokio::sync::{Mutex, broadcast};
use workdir::*;
type OpenSession = dyn Fn() -> Result<WorkdirSessionHandle, String> + Send + Sync;
pub(super) struct DeferredWorkdirSession {
workdir: Workdir,
capabilities: WorkdirSessionCapabilities,
open: Arc<OpenSession>,
state: Mutex<State>,
events: broadcast::Sender<CommandEvent>,
}
#[derive(Default)]
struct State {
session: Option<WorkdirSessionHandle>,
event_bridge: Option<tokio::task::JoinHandle<()>>,
closed: bool,
}
impl Drop for State {
fn drop(&mut self) {
if let Some(task) = &self.event_bridge {
task.abort();
}
}
}
impl std::fmt::Debug for DeferredWorkdirSession {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DeferredWorkdirSession")
.field("workdir", &self.workdir)
.finish_non_exhaustive()
}
}
impl DeferredWorkdirSession {
pub(super) fn handle(
workdir_id: &str,
capabilities: WorkdirSessionCapabilities,
open: impl Fn() -> Result<WorkdirSessionHandle, String> + Send + Sync + 'static,
) -> WorkdirSessionHandle {
Arc::new(Self {
workdir: Workdir::new(workdir_id),
capabilities,
open: Arc::new(open),
state: Mutex::new(State::default()),
events: broadcast::channel(1024).0,
})
}
async fn session(&self) -> Result<WorkdirSessionHandle, WorkdirError> {
let mut state = self.state.lock().await;
if state.closed {
return Err(WorkdirError::SessionClosed);
}
if let Some(session) = &state.session {
return Ok(session.clone());
}
let open = self.open.clone();
let session = tokio::task::spawn_blocking(move || open())
.await
.map_err(|_| WorkdirError::Unavailable("Workdir session opener failed".into()))?
.map_err(WorkdirError::Unavailable)?;
if session.workdir().id() != self.workdir.id() {
session.close().await?;
return Err(WorkdirError::Unavailable(
"Workdir session identity mismatch".into(),
));
}
if let Some(mut source) = session.subscribe_command_events() {
let events = self.events.clone();
state.event_bridge = Some(tokio::spawn(async move {
loop {
match source.recv().await {
Ok(event) => {
let _ = events.send(event);
}
Err(broadcast::error::RecvError::Lagged(_)) => continue,
Err(broadcast::error::RecvError::Closed) => break,
}
}
}));
}
state.session = Some(session.clone());
Ok(session)
}
}
macro_rules! impl_deferred_session {
($( $method:ident($request:ident: $request_type:ty) -> $result:ty; )*) => {
#[async_trait]
impl WorkdirSession for DeferredWorkdirSession {
fn workdir(&self) -> &Workdir {
&self.workdir
}
fn capabilities(&self) -> WorkdirSessionCapabilities {
self.capabilities
}
$(
async fn $method(&self, $request: $request_type) -> Result<$result, WorkdirError> {
self.session().await?.$method($request).await
}
)*
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<CommandEvent>> {
Some(self.events.subscribe())
}
fn command_snapshot(&self) -> Vec<CommandSnapshot> {
self.state.try_lock().ok().and_then(|state| {
state.session.as_ref().map(|session| session.command_snapshot())
}).unwrap_or_default()
}
async fn close(&self) -> Result<(), WorkdirError> {
let mut state = self.state.lock().await;
state.closed = true;
if let Some(session) = &state.session {
session.close().await?;
}
if let Some(task) = state.event_bridge.take() {
task.abort();
let _ = task.await;
}
Ok(())
}
}
};
}
impl_deferred_session! {
authorize_scope_path(request: WorkdirScopeAuthorizationRequest) -> ();
scope_rules_overlap(request: WorkdirScopeOverlapRequest) -> bool;
checkout_search(request: CheckoutSearchRequest) -> CheckoutSearchResult;
checkout_observe(path: WorkdirPath) -> CheckoutObservation;
checkout_execute(request: CheckoutRequest) -> CheckoutResult;
stat(request: StatRequest) -> StatResult;
read(request: ReadRequest) -> ReadResult;
read_bytes(request: ReadBytesRequest) -> ReadBytesResult;
write(request: WriteRequest) -> WriteResult;
edit(request: EditRequest) -> EditResult;
list(request: ListRequest) -> ListResult;
glob(request: GlobRequest) -> GlobResult;
grep(request: GrepRequest) -> GrepResult;
start_command(request: CommandRequest) -> CommandHandle;
command_status(handle: CommandHandle) -> CommandStatus;
command_output(request: CommandOutputRequest) -> CommandOutput;
cancel_command(handle: CommandHandle) -> ();
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
fn short_tempdir() -> tempfile::TempDir {
let target = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.parent()
.unwrap()
.join("target");
std::fs::create_dir_all(&target).unwrap();
tempfile::Builder::new()
.prefix("d")
.rand_bytes(3)
.tempdir_in(target)
.unwrap()
}
fn read_request() -> ReadRequest {
ReadRequest {
path: WorkdirPath::new("README.md").unwrap(),
offset: 0,
limit: 10,
max_bytes: 1024,
}
}
#[tokio::test]
async fn unopened_session_close_does_not_connect_and_is_terminal() {
let attempts = Arc::new(AtomicUsize::new(0));
let observed = attempts.clone();
let session = DeferredWorkdirSession::handle(
"offline",
WorkdirSessionCapabilities::READ_ONLY,
move || {
observed.fetch_add(1, Ordering::SeqCst);
Err("offline".into())
},
);
assert_eq!(session.workdir().id().as_str(), "offline");
assert!(session.subscribe_command_events().is_some());
session.close().await.unwrap();
session.close().await.unwrap();
assert!(matches!(
session.read(read_request()).await,
Err(WorkdirError::SessionClosed)
));
assert_eq!(attempts.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn router_cleanup_of_unopened_session_never_connects_or_follows_reused_alias() {
let attempts = Arc::new(AtomicUsize::new(0));
let observed = attempts.clone();
let session = DeferredWorkdirSession::handle(
"old",
WorkdirSessionCapabilities::READ_ONLY,
move || {
observed.fetch_add(1, Ordering::SeqCst);
Err("offline".into())
},
);
let router = WorkdirSessionRouter::new();
let alias = WorkdirAttachmentAlias::new("w").unwrap();
let routed = router.attach(alias.clone(), session.clone()).unwrap();
assert!(routed.subscribe_command_events().is_some());
assert!(routed.command_snapshot().is_empty());
router.close_all().await.unwrap();
router.close_all().await.unwrap();
assert!(router.aliases().is_empty());
assert!(matches!(
session.read(read_request()).await,
Err(WorkdirError::SessionClosed)
));
assert_eq!(attempts.load(Ordering::SeqCst), 0);
let new_attempts = Arc::new(AtomicUsize::new(0));
let new_observed = new_attempts.clone();
let replacement = DeferredWorkdirSession::handle(
"new",
WorkdirSessionCapabilities::READ_ONLY,
move || {
new_observed.fetch_add(1, Ordering::SeqCst);
Err("offline".into())
},
);
let new_routed = router.attach(alias, replacement).unwrap();
// A retained old connection cannot open the replacement provider.
assert!(routed.read(read_request()).await.is_err());
assert_eq!(new_attempts.load(Ordering::SeqCst), 0);
assert!(matches!(
new_routed.read(read_request()).await,
Err(WorkdirError::Unavailable(_))
));
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
// Even after a failed open, cleanup must not try connecting again.
router.close_all().await.unwrap();
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
assert_eq!(attempts.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn failed_open_retries_on_use_and_keeps_read_only_capabilities() {
let temp = short_tempdir();
std::fs::write(temp.path().join("README.md"), "restored").unwrap();
let root = temp.path().to_path_buf();
let attempts = Arc::new(AtomicUsize::new(0));
let observed = attempts.clone();
let session = DeferredWorkdirSession::handle(
"saved",
WorkdirSessionCapabilities::READ_ONLY,
move || {
if observed.fetch_add(1, Ordering::SeqCst) == 0 {
return Err("offline".into());
}
let scope = super::super::runtime_attachment_scope(
&root,
WorkdirSessionCapabilities::READ_ONLY,
)?;
Ok(super::super::runtime_local_workdir_session(
"saved",
&root,
&root,
scope,
WorkdirSessionCapabilities::READ_ONLY,
Default::default(),
Vec::new(),
))
},
);
assert_eq!(attempts.load(Ordering::SeqCst), 0);
assert!(matches!(
session.read(read_request()).await,
Err(WorkdirError::Unavailable(_))
));
assert_eq!(
session.read(read_request()).await.unwrap().bytes,
b"restored"
);
assert_eq!(
session.read(read_request()).await.unwrap().bytes,
b"restored"
);
assert_eq!(attempts.load(Ordering::SeqCst), 2);
assert!(
session
.write(WriteRequest {
path: WorkdirPath::new("README.md").unwrap(),
content: b"forbidden".to_vec(),
expected_hash: None
})
.await
.is_err()
);
assert_eq!(
std::fs::read(temp.path().join("README.md")).unwrap(),
b"restored"
);
session.close().await.unwrap();
}
}
@@ -2,33 +2,137 @@
// Worker/bootstrap, local Git Workdir provider, and allocation table. Model I/O
// is not invoked; this is a deterministic ordinary suite, not product E2E.
fn restore_failure_tempdir() -> tempfile::TempDir {
// Explicitly stay inside this checkout, even when TMPDIR points elsewhere.
// Keep fixture components short; no symlink/alias to evade socket limits.
let target = Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.parent()
.unwrap()
.join("target");
fs::create_dir_all(&target).unwrap();
tempfile::Builder::new()
.prefix("r")
.rand_bytes(3)
.tempdir_in(target)
.unwrap()
}
struct RestoreOpenCountingMaterializer {
inner: RuntimeGitMaterializer,
opens: Arc<AtomicUsize>,
}
impl WorkingDirectoryMaterializer for RestoreOpenCountingMaterializer {
fn materialize(
&self,
worker_ref: &WorkerRef,
request: &WorkingDirectoryRequest,
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
self.inner.materialize(worker_ref, request)
}
fn create(
&self,
request: &WorkingDirectoryRequest,
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
self.inner.create(request)
}
fn authorize_repository_access(
&self,
request: &WorkingDirectoryRepositoryAccessRequest,
) -> Result<(), WorkingDirectoryDiagnostic> {
self.inner.authorize_repository_access(request)
}
fn observe_repository_ref(
&self,
request: &RepositoryRefObservationRequest,
) -> Result<RepositoryRefObservation, WorkingDirectoryDiagnostic> {
self.inner.observe_repository_ref(request)
}
fn bind_working_directory(
&self,
id: &str,
relative_cwd: Option<&str>,
) -> Result<WorkingDirectoryBinding, WorkingDirectoryDiagnostic> {
self.opens.fetch_add(1, Ordering::SeqCst);
self.inner.bind_working_directory(id, relative_cwd)
}
fn preflight_bind_working_directory(
&self,
id: &str,
relative_cwd: Option<&str>,
) -> Result<(), WorkingDirectoryDiagnostic> {
self.inner
.preflight_bind_working_directory(id, relative_cwd)
}
fn list_working_directories(
&self,
) -> Result<Vec<WorkingDirectoryStatus>, WorkingDirectoryDiagnostic> {
self.inner.list_working_directories()
}
fn working_directory_status(
&self,
id: &str,
) -> Result<WorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
self.inner.working_directory_status(id)
}
fn cleanup_working_directory(
&self,
id: &str,
) -> Result<WorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
self.inner.cleanup_working_directory(id)
}
fn cleanup(&self, binding: &WorkingDirectoryBinding) -> Result<(), WorkingDirectoryDiagnostic> {
self.inner.cleanup(binding)
}
}
struct ProfileFailureFixture {
runtime: EmbeddedRuntime,
backend: Arc<WorkerRuntimeExecutionBackend>,
worker: crate::catalog::WorkerDetail,
_repo: tempfile::TempDir,
provider_opens: Arc<AtomicUsize>,
root: tempfile::TempDir,
}
impl ProfileFailureFixture {
fn new(timeout: Duration) -> Self {
let root = tempfile::tempdir().unwrap();
let repo = create_clean_repo();
let store = root.path().join("private-restore-test/runtime");
let root = restore_failure_tempdir();
let repo = root.path().join("g");
fs::create_dir(&repo).unwrap();
git(&repo, &["init"]);
git(&repo, &["config", "user.email", "test@example.invalid"]);
git(&repo, &["config", "user.name", "Yoi Test"]);
fs::write(repo.join("README.md"), "clean\n").unwrap();
git(&repo, &["add", "README.md"]);
git(&repo, &["commit", "-m", "init"]);
let store = root.path().join("s");
let provider_opens = Arc::new(AtomicUsize::new(0));
let backend = Arc::new(
WorkerRuntimeExecutionBackend::new(
ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(&store),
)
.unwrap()
.with_spawn_restore_timeout(timeout)
.with_working_directory_materializer(RuntimeGitMaterializer::new(
root.path().join("workdirs"),
)),
.with_working_directory_materializer(RestoreOpenCountingMaterializer {
inner: RuntimeGitMaterializer::new(root.path().join("w")),
opens: provider_opens.clone(),
}),
);
let runtime = EmbeddedRuntime::with_fs_store_and_execution_backend(
crate::fs_store::FsRuntimeStoreOptions {
root: store,
runtime_id: "restore-failure-test".into(),
runtime_id: "r".into(),
display_name: None,
},
backend.clone(),
@@ -39,13 +143,15 @@ impl ProfileFailureFixture {
request.profile = ProfileSelector::Builtin("default".into());
request.workdir_attachment_requests = vec![WorkingDirectoryAttachmentRequest {
alias: WorkdirAttachmentAlias::new("checkout").unwrap(),
working_directory: working_directory_request(repo.path()),
working_directory: working_directory_request(&repo),
}];
let worker = runtime.create_worker(request).unwrap();
runtime.stop_worker(&worker.worker_ref, None).unwrap();
// Count restore provider access, not the initial create/stop setup.
provider_opens.store(0, Ordering::SeqCst);
Self {
root,
_repo: repo,
provider_opens,
runtime,
backend,
worker,
@@ -57,6 +163,7 @@ impl ProfileFailureFixture {
}
fn assert_released(&self) {
assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0);
assert!(self.backend.pending_restores.lock().unwrap().is_empty());
assert!(self.backend.workers.lock().unwrap().is_empty());
assert!(
@@ -176,16 +283,22 @@ impl ProfileFailureFixture {
.load(Ordering::SeqCst),
calls + 1
);
// Restore/cleanup and old-request replay must not provider-open the
// retained attachment, including after a new execution was accepted.
assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0);
// Only tear down after the fresh explicit Restore has already succeeded.
self.runtime
.stop_worker(&self.worker.worker_ref, None)
.unwrap();
assert_eq!(self.provider_opens.load(Ordering::SeqCst), 0);
}
}
#[test]
#[serial_test::serial(worker_allocation)]
fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() {
// Binding means runtime Worker/attachment-router binding, not eager provider
// materialization. Provider access stays deferred through every boundary.
for stage in [
RuntimeRestoreStage::Reconstruction,
RuntimeRestoreStage::Binding,
@@ -224,8 +337,7 @@ fn profile_restore_multistage_failure_cleans_then_fresh_restore_without_stop() {
#[test]
#[serial_test::serial(worker_allocation)]
fn profile_restore_cleanup_failure_retains_owner_and_original_cause_until_same_operation_recovers()
{
fn profile_restore_cleanup_failure_retains_owner_and_cause_without_opening_deferred_provider() {
let fixture = ProfileFailureFixture::new(Duration::from_secs(5));
fixture.fail(RuntimeRestoreStage::Feature);
fixture
@@ -265,6 +377,20 @@ fn profile_restore_cleanup_failure_retains_owner_and_original_cause_until_same_o
"{message}"
);
assert!(message.contains("Workdir"), "{message}");
{
let retained = pending.result.lock().unwrap();
let failure = retained.as_ref().unwrap().as_ref().err().unwrap();
assert_eq!(failure.stage, RuntimeRestoreStage::Feature);
assert_eq!(failure.resources, RuntimeRestoreResources::WorkerStartup);
assert_eq!(
failure.cleanup,
RuntimeRestoreCleanup::Failed(RuntimeRestoreCleanupFailure::Workdir)
);
assert_eq!(
failure.cleanup_failure,
Some(RuntimeRestoreCleanupFailure::Workdir)
);
}
let owned = fixture
.backend
.factory
@@ -274,6 +400,9 @@ fn profile_restore_cleanup_failure_retains_owner_and_original_cause_until_same_o
.get(&fixture.worker.worker_ref)
.cloned()
.unwrap();
assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0);
assert!(pending.request.workdir_attachments.is_empty());
assert!(!pending.request.deferred_workdir_sessions.is_empty());
assert_eq!(owned.operation_id, pending.request.operation_id);
assert!(owned.allocation.is_some());
assert!(
@@ -307,6 +436,7 @@ fn profile_restore_cleanup_failure_retains_owner_and_original_cause_until_same_o
.load(Ordering::SeqCst),
1
);
assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0);
fixture
.backend
.factory
@@ -334,6 +464,27 @@ fn profile_restore_cleanup_failure_retains_owner_and_original_cause_until_same_o
.all(|sessions| sessions.aliases().is_empty())
);
fixture.assert_released();
let unopened = pending.request.deferred_workdir_sessions.clone();
fixture
.backend
.run_on_adapter_runtime(async move {
for session in unopened.values() {
assert!(matches!(
session
.read(workdir::ReadRequest {
path: workdir::WorkdirPath::new("README.md").unwrap(),
offset: 0,
limit: 10,
max_bytes: 1024,
})
.await,
Err(workdir::WorkdirError::SessionClosed)
));
}
Ok(())
})
.unwrap();
assert_eq!(fixture.provider_opens.load(Ordering::SeqCst), 0);
eprintln!(
"restore-fault cleanup=Workdir owner-retained=true retry=same-operation outcome=RolledBack"
);
@@ -466,7 +617,19 @@ fn profile_restore_real_socket_start_failure_removes_partial_run_and_allocation(
"{diagnostic}"
);
assert!(diagnostic.contains("InvalidInput"), "{diagnostic}");
assert!(!diagnostic.contains("private-restore-test"));
// This case deliberately exercises the real Unix socket path-limit error;
// positive restore cases retain the factory's normal InProcess transport.
// Short fixture roots must not accidentally turn it into a successful bind.
let operation_id =
fixture.backend.completed_restore_failures.lock().unwrap()[&fixture.worker.worker_ref].0;
let socket = fixture
.backend
.factory
.worker_restore_run_dir(&fixture.worker.worker_ref, operation_id)
.unwrap()
.join("worker.sock");
assert!(socket.as_os_str().as_encoded_bytes().len() >= 108);
assert!(!diagnostic.contains(&fixture.root.path().to_string_lossy().to_string()));
fixture.assert_released();
fixture
.backend
@@ -844,102 +1007,150 @@ fn profile_restore_failure_settlement_write_retry_preserves_completed_cause_with
}
}
struct RacedBindingFactory {
changed_checkout: PathBuf,
preflights: AtomicUsize,
}
#[async_trait]
impl RuntimeWorkerFactory for RacedBindingFactory {
async fn spawn_controller(
&self,
_: WorkerExecutionSpawnRequest,
) -> Result<RuntimeWorkerController, String> {
panic!("binding-only failure must not create a controller")
}
async fn preflight_restore(&self, _: &WorkerExecutionRestoreRequest) -> Result<(), String> {
// Inject another owner's checkout change AFTER the real materializer's
// read-only checks, but before its mutable bind. Recovery stays real.
if self.preflights.fetch_add(1, Ordering::SeqCst) == 0 {
fs::remove_dir_all(&self.changed_checkout).unwrap();
}
Ok(())
}
async fn restore_controller(
&self,
_: WorkerExecutionRestoreRequest,
) -> Result<RuntimeWorkerController, RuntimeRestoreFailure> {
panic!("factory execution must not run after partial binding failed")
}
}
// Preflight accepts the real retained Profile source, but reconstruction
// discovers that its committed segment has no system prompt. No fault checkpoint
// supplies this Err.
#[test]
fn partial_restore_binding_failure_keeps_exact_operation_without_factory_cleanup_claim() {
let root = tempfile::tempdir().unwrap();
let repo = create_clean_repo();
let materializer = RuntimeGitMaterializer::new(root.path());
let first = materializer
.create(&working_directory_request(repo.path()))
.unwrap();
let second = materializer
.create(&working_directory_request(repo.path()))
.unwrap();
let mut request = direct_restore_request(WorkerRef::new(WorkerId::now_v7()));
request.previous_workdir_attachments = vec![
WorkingDirectoryAttachmentStatus {
alias: WorkdirAttachmentAlias::new("first").unwrap(),
working_directory: first.status(),
},
WorkingDirectoryAttachmentStatus {
alias: WorkdirAttachmentAlias::new("second").unwrap(),
working_directory: second.status(),
},
];
let backend = WorkerRuntimeExecutionBackend::new(RacedBindingFactory {
changed_checkout: second.root().to_path_buf(),
preflights: AtomicUsize::new(0),
})
.unwrap()
.with_working_directory_materializer(materializer);
assert!(matches!(
backend.restore_worker(request.clone()),
WorkerExecutionSpawnResult::ReconciliationRequired { .. }
));
let pending = backend
.pending_restore(&request.worker_ref)
.unwrap()
.unwrap();
assert_eq!(pending.request.operation_id, request.operation_id);
assert_eq!(pending.request.workdir_attachments.len(), 1);
assert_eq!(
pending
.request
.workdir_attachments
.values()
.next()
.unwrap()
.root(),
first.root()
#[serial_test::serial(worker_allocation)]
fn profile_restore_missing_committed_system_prompt_rolls_back_and_replays_classified_cause() {
let root = restore_failure_tempdir();
let runtime_store = root.path().join("s");
let worker_ref = WorkerRef::new(WorkerId::now_v7());
let (session_id, segment_id, _) = write_retained_profile_worker(&runtime_store, &worker_ref);
let worker_name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref);
let aggregate_dir = runtime_store
.join("workers")
.join(worker_ref.worker_id.to_string());
let metadata_store = WorkerAggregateStore::new(&aggregate_dir, &worker_name).unwrap();
let mut metadata = metadata_store.read_by_name(&worker_name).unwrap().unwrap();
let manifest = WorkerManifest::from_toml(
r#"
[worker]
name = "r"
[scope]
allow = []
[model]
scheme = "anthropic"
model_id = "test"
auth = { kind = "none" }
[engine]
instruction = "default"
max_tokens = 100
[feature.memory.profile]
enabled = false
"#,
)
.unwrap();
metadata.resolved_manifest_snapshot =
Some(manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap());
metadata_store.write(&metadata).unwrap();
let session_store = WorkerSessionStore::new(aggregate_dir.join("session")).unwrap();
let entries_before = session_store.read_all(session_id, segment_id).unwrap();
assert!(
session_store::collect_state(&entries_before)
.system_prompt
.is_none()
);
for _ in 0..2 {
assert!(matches!(
backend.reconcile_restore(request.clone()),
WorkerExecutionSpawnResult::ReconciliationRequired { .. }
));
let backend = WorkerRuntimeExecutionBackend::new(
ProfileRuntimeWorkerFactory::new(root.path().join("p"))
.with_runtime_store_dir(&runtime_store),
)
.unwrap();
let request = direct_restore_request(worker_ref.clone());
backend.preflight_restore(&request).unwrap();
let first = match backend.restore_worker(request.clone()) {
WorkerExecutionSpawnResult::RolledBack(result) => result,
other => panic!("missing committed prompt must finish rollback: {other:?}"),
};
let diagnostic = first.message.as_deref().unwrap();
assert!(
diagnostic.contains("saved segment system prompt missing"),
"{diagnostic}"
);
assert!(diagnostic.contains("Reconstruction"), "{diagnostic}");
assert!(diagnostic.contains("Confirmed"), "{diagnostic}");
assert!(diagnostic.len() < 512, "{diagnostic}");
for private in [
root.path().to_string_lossy().into_owned(),
session_id.to_string(),
segment_id.to_string(),
"retained response".into(),
] {
assert!(!diagnostic.contains(&private), "{diagnostic}");
}
assert_eq!(backend.factory.preflights.load(Ordering::SeqCst), 1);
assert!(!backend.stop_worker(&request.worker_ref).is_accepted());
assert!(Arc::ptr_eq(
&pending,
&backend
.pending_restore(&request.worker_ref)
.unwrap()
.unwrap()
));
// A completed Err is not evidence of zero resources. Require the owner to
// attest cleanup, and independently check its allocation/artifact authority.
{
let completed = backend.completed_restore_failures.lock().unwrap();
let (operation_id, failure) = completed.get(&worker_ref).unwrap();
assert_eq!(*operation_id, request.operation_id);
assert_eq!(failure.stage, RuntimeRestoreStage::Reconstruction);
assert_eq!(failure.resources, RuntimeRestoreResources::WorkerStartup);
assert_eq!(failure.cleanup, RuntimeRestoreCleanup::Confirmed);
assert_eq!(failure.cleanup_failure, None);
}
assert!(backend.pending_restore(&worker_ref).unwrap().is_none());
assert!(backend.workers.lock().unwrap().is_empty());
eprintln!(
"restore-fault stage=mutable-binding outcome=ReconciliationRequired partial-bindings=retained factory-executions=0"
assert!(
backend
.factory
.failed_restore_resources
.lock()
.unwrap()
.is_empty()
);
assert!(
backend
.factory
.observation_hub
.workers
.lock()
.unwrap()
.is_empty()
);
assert!(
!backend
.factory
.worker_restore_run_dir(&worker_ref, request.operation_id)
.unwrap()
.exists()
);
{
let table = worker::runtime::worker_allocation::LockFileGuard::open(
&worker::runtime::worker_allocation::default_allocation_path().unwrap(),
)
.unwrap();
assert!(table.data().find(&worker_name).is_none());
assert!(table.data().find("r").is_none());
}
for replay in [
backend.reconcile_restore(request.clone()),
backend.restore_worker(request),
] {
let WorkerExecutionSpawnResult::RolledBack(result) = replay else {
panic!("same request must replay confirmed failure: {replay:?}");
};
assert_eq!(
serde_json::to_value(result).unwrap(),
serde_json::to_value(&first).unwrap()
);
}
assert_eq!(
backend
.factory
.restore_faults
.executions
.load(Ordering::SeqCst),
1
);
assert!(backend.pending_restore(&worker_ref).unwrap().is_none());
assert_eq!(
metadata_store.read_by_name(&worker_name).unwrap().unwrap(),
metadata
);
assert_eq!(
serde_json::to_value(session_store.read_all(session_id, segment_id).unwrap()).unwrap(),
serde_json::to_value(entries_before).unwrap()
);
// Fixture teardown drops the retained passive local binding. This is not
// evidence that credential/policy materialization has been rolled back.
}