fix: merge worker restore into develop host lifecycle

Integrate integration/worker-restore without reviving removed Worker socket/discovery paths. Keep restore candidate allocation and canonical effects deferred until activation, with registry reclamation and observer publication fences on the in-process controller.

Preserve develop BackendJob/provenance, immutable Journal body capture and erasure, and Workdir diagnostics. Adapt obsolete socket fault tests to in-process publication and registry drift boundaries.

Validation: root cargo check; ordinary worker, worker-runtime and standalone package tests; Server restore (43) and Journal (49) filters; mcp/server-api/runtime-api/session-store tests; optional Runtime feature check; OpenAPI and six generated TypeScript checks; Web restore unit tests (15); runner tests (21); fmt, naming and diff checks. Full workspace/Server suites, E2E, live/model and fixture generation not run.
This commit is contained in:
2026-10-28 00:06:17 +09:00
89 changed files with 33540 additions and 1812 deletions
Generated
+6
View File
@@ -6712,15 +6712,21 @@ name = "yoi-e2e"
version = "0.0.0"
dependencies = [
"chrono",
"config-source",
"libc",
"manifest",
"protocol",
"reqwest",
"serde",
"serde_json",
"server-api",
"session-store",
"sha2 0.11.0",
"tempfile",
"tungstenite 0.29.0",
"webauthn-authenticator-rs",
"webauthn-rs-proto",
"worker-runtime",
"yoi-workspace-server",
]
+171 -70
View File
@@ -617,7 +617,8 @@ impl fmt::Display for McpRequestStage {
}
}
/// A running initialized stdio MCP client.
/// An owned stdio MCP process, initialized by `connect` or explicitly after
/// `spawn_uninitialized` when startup cleanup must retain it before any await.
pub struct McpStdioClient {
server_name: String,
limits: McpStdioLimits,
@@ -636,6 +637,9 @@ pub struct McpStdioClient {
impl McpStdioClient {
/// Spawn, initialize, negotiate capabilities, and send notifications/initialized.
/// Initialization errors attempt shutdown, but this convenience API cannot
/// return an unreaped client on failure/cancellation. Its error is not proof
/// of cleanup; use `spawn_uninitialized` with retained ownership when needed.
pub async fn connect(
spec: McpStdioServerSpec,
limits: McpStdioLimits,
@@ -667,6 +671,21 @@ impl McpStdioClient {
async fn spawn(
spec: McpStdioServerSpec,
limits: McpStdioLimits,
) -> Result<Self, McpClientError> {
Self::spawn_uninitialized(spec, limits)
}
/// Spawn without awaiting the handshake, so a cleanup owner can retain the
/// client before initialization can fail or be cancelled. Call
/// `initialize_with_timeout` before discovery; on error/cancellation, shut
/// down this same client rather than spawning a replacement.
///
/// Once the process spawns, this returns its owner even if an expected pipe
/// is missing. Initialization then fails, but cleanup can still reap the
/// actual child rather than relying on kill-on-drop for a post-spawn error.
pub fn spawn_uninitialized(
spec: McpStdioServerSpec,
limits: McpStdioLimits,
) -> Result<Self, McpClientError> {
let redactor = spec.redactor();
let mut command = Command::new(&spec.command);
@@ -688,29 +707,23 @@ impl McpStdioClient {
McpErrorKind::Io(redactor.redact(&err.to_string())),
)
})?;
let stdin = child.stdin.take().ok_or_else(|| {
McpClientError::new(
&spec.name,
McpPhase::Spawn,
McpErrorKind::Protocol("child stdin was not piped".into()),
)
})?;
let stdout = child.stdout.take().ok_or_else(|| {
McpClientError::new(
&spec.name,
McpPhase::Spawn,
McpErrorKind::Protocol("child stdout was not piped".into()),
)
})?;
let stderr = child.stderr.take().ok_or_else(|| {
McpClientError::new(
&spec.name,
McpPhase::Spawn,
McpErrorKind::Protocol("child stderr was not piped".into()),
)
})?;
let stdin = child.stdin.take();
let stdout = child.stdout.take();
let stderr = child.stderr.take();
let missing_pipe = if stdin.is_none() {
Some("stdin")
} else if stdout.is_none() {
Some("stdout")
} else if stderr.is_none() {
Some("stderr")
} else {
None
};
let stdin: SharedStdin = Arc::new(Mutex::new(Some(Box::new(stdin))));
let stdin: SharedStdin =
Arc::new(Mutex::new(stdin.map(|stdin| {
Box::new(stdin) as Box<dyn AsyncWrite + Send + Unpin>
})));
let diagnostics = Arc::new(Mutex::new(BoundedDiagnostics::new(
spec.name.clone(),
limits.max_diagnostic_lines,
@@ -718,7 +731,18 @@ impl McpStdioClient {
)));
let list_changes = Arc::new(Mutex::new(BoundedListChanged::new(spec.name.clone())));
let (tx, rx) = mpsc::channel(16);
let reader_task = spawn_stdout_reader(
// Do not return an error that discards an already spawned child. Queue
// setup failure before reader events so initialization cannot succeed.
if let Some(pipe) = missing_pipe {
tx.try_send(ReaderEvent::Error(McpClientError::new(
&spec.name,
McpPhase::Spawn,
McpErrorKind::Protocol(format!("child {pipe} was not piped")),
)))
.expect("new response channel has capacity");
}
let reader_task = match stdout {
Some(stdout) => spawn_stdout_reader(
spec.name.clone(),
stdout,
stdin.clone(),
@@ -726,8 +750,13 @@ impl McpStdioClient {
limits.clone(),
redactor.clone(),
list_changes.clone(),
);
let stderr_task = spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone());
),
None => tokio::spawn(async {}),
};
let stderr_task = match stderr {
Some(stderr) => spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone()),
None => tokio::spawn(async {}),
};
Ok(Self {
server_name: spec.name,
@@ -746,6 +775,25 @@ impl McpStdioClient {
})
}
/// Perform the bounded handshake on an already retained client. Failure or
/// cancellation does not drop or shut down the client: its owner must do so.
/// Do not retry initialization on a partially initialized connection.
pub async fn initialize_with_timeout(&mut self) -> Result<(), McpClientError> {
let started = Instant::now();
match timeout(self.limits.startup_timeout, self.initialize()).await {
Ok(result) => result,
Err(_) => Err(McpClientError::new(
&self.server_name,
McpPhase::Initialize,
McpErrorKind::Timeout {
operation: "startup".to_string(),
elapsed: started.elapsed(),
},
)
.with_diagnostics(self.snapshot_diagnostics().await)),
}
}
async fn initialize(&mut self) -> Result<(), McpClientError> {
let result: InitializeResult = self
.request(
@@ -1080,6 +1128,8 @@ impl McpStdioClient {
}
/// Close stdin and wait for process exit, falling back to terminate and kill.
/// Errors and cancellation retain the child for another attempt on this
/// client. Only confirmed reaping releases the process handle.
pub async fn shutdown(&mut self) -> Result<ShutdownReport, McpClientError> {
self.shutdown_started = true;
let deadline = Instant::now() + self.limits.shutdown_timeout;
@@ -1091,64 +1141,55 @@ impl McpStdioClient {
.await
.is_ok();
let mut child = match self.child.take() {
Some(child) => child,
None => return Ok(ShutdownReport::already_finished()),
};
// Keep the actual process handle in the client across every await and
// error. Taking it before wait loses retry/reaping authority on cancel.
if self.child.is_none() {
return Ok(ShutdownReport::already_finished());
}
if !closed {
self.reader_task.abort();
return self.terminate_then_kill(child).await;
return self.terminate_then_kill().await;
}
let child = self.child.as_mut().expect("checked child");
if let Ok(Some(status)) = child.try_wait() {
self.reader_task.abort();
self.stderr_task.abort();
return Ok(ShutdownReport {
exit_status: Some(status),
terminated: false,
killed: false,
});
return Ok(self.finish_shutdown(status, false, false));
}
match timeout_at(deadline, child.wait()).await {
Ok(Ok(status)) => {
self.reader_task.abort();
self.stderr_task.abort();
Ok(ShutdownReport {
exit_status: Some(status),
terminated: false,
killed: false,
})
}
Ok(Ok(status)) => Ok(self.finish_shutdown(status, false, false)),
Ok(Err(err)) => Err(McpClientError::new(
&self.server_name,
McpPhase::Shutdown,
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
)
.with_diagnostics(self.snapshot_diagnostics().await)),
Err(_) => self.terminate_then_kill(child).await,
Err(_) => self.terminate_then_kill().await,
}
}
async fn terminate_then_kill(
fn finish_shutdown(
&mut self,
mut child: Child,
) -> Result<ShutdownReport, McpClientError> {
let mut terminated = false;
let mut killed = false;
if send_terminate(&mut child).is_ok() {
terminated = true;
}
match timeout(self.limits.kill_timeout, child.wait()).await {
Ok(Ok(status)) => {
status: ExitStatus,
terminated: bool,
killed: bool,
) -> ShutdownReport {
// No await separates confirmed reaping from releasing the handle.
drop(self.child.take());
self.reader_task.abort();
self.stderr_task.abort();
Ok(ShutdownReport {
ShutdownReport {
exit_status: Some(status),
terminated,
killed,
})
}
}
async fn terminate_then_kill(&mut self) -> Result<ShutdownReport, McpClientError> {
let child = self.child.as_mut().expect("shutdown retains child");
let terminated = send_terminate(child).is_ok();
match timeout(self.limits.kill_timeout, child.wait()).await {
Ok(Ok(status)) => Ok(self.finish_shutdown(status, terminated, false)),
Ok(Err(err)) => Err(McpClientError::new(
&self.server_name,
McpPhase::Shutdown,
@@ -1163,7 +1204,6 @@ impl McpStdioClient {
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
)
})?;
killed = true;
let status = timeout(self.limits.kill_timeout, child.wait())
.await
.map_err(|_| {
@@ -1183,13 +1223,7 @@ impl McpStdioClient {
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
)
})?;
self.reader_task.abort();
self.stderr_task.abort();
Ok(ShutdownReport {
exit_status: Some(status),
terminated,
killed,
})
Ok(self.finish_shutdown(status, terminated, true))
}
}
}
@@ -1197,11 +1231,11 @@ impl McpStdioClient {
impl Drop for McpStdioClient {
fn drop(&mut self) {
if !self.shutdown_started {
// An attempted shutdown is not proof of exit. Errors and cancellation
// leave the child here so Drop must still make a best-effort kill.
if let Some(child) = &mut self.child {
let _ = child.start_kill();
}
}
self.reader_task.abort();
self.stderr_task.abort();
}
@@ -1784,3 +1818,70 @@ fn send_terminate(child: &mut Child) -> Result<(), ()> {
#[cfg(test)]
mod deadline_tests;
#[cfg(all(test, unix))]
mod shutdown_retention_tests {
//! The real client and OS process are used; only the external server is a
//! local shell that acknowledges initialization and ignores TERM/EOF.
use super::*;
use std::future::{Future, poll_fn};
use std::task::Poll;
#[tokio::test]
async fn cancelled_graceful_or_terminate_wait_retries_reaping_the_same_child() {
for cancel_in_fallback in [false, true] {
let script = r#"trap '' TERM
read init || exit 1
printf '%s\n' '{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-11-25","capabilities":{},"serverInfo":{"name":"retained","version":"1"}}}'
while :; do :; done
"#;
let mut client = McpStdioClient::connect(
McpStdioServerSpec::new("retained", "/bin/sh").args(["-c", script]),
McpStdioLimits {
shutdown_timeout: if cancel_in_fallback {
Duration::ZERO
} else {
Duration::from_secs(30)
},
kill_timeout: Duration::from_secs(30),
..Default::default()
},
)
.await
.expect("initialize after installing TERM handler");
let pid = client.child.as_ref().unwrap().id();
let mut shutdown = Box::pin(client.shutdown());
poll_fn(|cx| {
assert!(shutdown.as_mut().poll(cx).is_pending());
Poll::Ready(())
})
.await;
drop(shutdown);
assert_eq!(client.child.as_ref().and_then(Child::id), pid);
assert!(client.shutdown_started);
client.limits.shutdown_timeout = Duration::from_millis(20);
client.limits.kill_timeout = Duration::from_millis(50);
let report = timeout(Duration::from_secs(2), client.shutdown())
.await
.expect("retry bounded")
.expect("retry reaps original child");
assert!(
report.exit_status.is_some(),
"not an already-finished receipt"
);
assert!(report.killed, "TERM was ignored");
assert!(client.child.is_none());
let mut status = 0;
assert_eq!(
unsafe { libc::waitpid(pid.unwrap() as libc::pid_t, &mut status, libc::WNOHANG) },
-1
);
assert_eq!(
std::io::Error::last_os_error().raw_os_error(),
Some(libc::ECHILD),
"shutdown, not a later observer, reaped the original process"
);
}
}
}
+18 -5
View File
@@ -86,6 +86,8 @@ pub enum WorkerRestoreState {
Accepted,
Rejected,
RolledBack,
/// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.
Superseded,
ReconciliationRequired,
}
@@ -1578,13 +1580,24 @@ mod tests {
)
.unwrap();
assert!(request.preparation.is_none());
for state in [
"accepted",
"rejected",
"rolled_back",
for (state, wire) in [
(WorkerRestoreState::Accepted, "accepted"),
(WorkerRestoreState::Rejected, "rejected"),
(WorkerRestoreState::RolledBack, "rolled_back"),
(WorkerRestoreState::Superseded, "superseded"),
(
WorkerRestoreState::ReconciliationRequired,
"reconciliation_required",
),
] {
assert!(serde_json::from_value::<WorkerRestoreState>(serde_json::json!(state)).is_ok());
assert_eq!(
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(wire)).unwrap(),
state
);
assert_eq!(
serde_json::to_value(state).unwrap(),
serde_json::json!(wire)
);
}
assert!(
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(
+8 -4
View File
@@ -552,8 +552,9 @@ pub struct WorkspaceWorkerSessionHistoryResponse {
/// comparisons do not replace admission checks under the Runtime operation lock; conflicts do
/// not trigger automatic token replacement.
/// [`WorkerRestoreState`] Accepted means Restore committed, not that the subsequent run succeeded.
/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects, and
/// ReconciliationRequired needing settled-state checks. Response loss does not imply rollback.
/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects,
/// Superseded by a later Stop, and ReconciliationRequired needing settled-state checks.
/// Response loss does not imply rollback.
#[api(reqwest, axum, openapi)]
pub trait ServerApi {
#[get("/health", status = 200, error_status = 400)]
@@ -10579,8 +10580,8 @@ pub enum WorkerOperationState {
/// Cross-layer outcome of an explicit Worker restore operation.
///
/// `Rejected` is reserved for read-only preflight failures. Once live restore
/// work starts, failure is either a confirmed `RolledBack` operation or a
/// `ReconciliationRequired` result whose commit state must be reread/retried.
/// work starts, distinguish confirmed `RolledBack`, unresolved
/// `ReconciliationRequired`, and a later Stop's `Superseded` outcome.
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")]
@@ -10588,6 +10589,8 @@ pub enum WorkerRestoreState {
Accepted,
Rejected,
RolledBack,
/// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.
Superseded,
ReconciliationRequired,
}
@@ -12681,6 +12684,7 @@ mod tests {
(WorkerRestoreState::Accepted, "accepted"),
(WorkerRestoreState::Rejected, "rejected"),
(WorkerRestoreState::RolledBack, "rolled_back"),
(WorkerRestoreState::Superseded, "superseded"),
(
WorkerRestoreState::ReconciliationRequired,
"reconciliation_required",
@@ -194,6 +194,21 @@ impl WorkerSessionStore {
Ok(store)
}
/// Open a current retained Session without creating or migrating anything.
/// The live constraint is checked here and again by every later publication;
/// attaching the control store itself does not acquire publication authority.
pub fn open_existing_with_retention(
root: impl Into<PathBuf>,
retention: crate::retention::SessionRetentionStore,
) -> Result<Self, StoreError> {
let mut store = Self::open_read_only(root)?;
if let Some(session) = *store.session_id.lock().unwrap() {
retention.ensure_live(session)?;
}
store.retention = Some(retention);
Ok(store)
}
fn lock_publication(
&self,
session: SessionId,
+36 -4
View File
@@ -785,6 +785,12 @@ mod tests {
#[derive(Default)]
struct PendingRuntimeBackend {
operations: std::sync::Mutex<Vec<worker_runtime::execution::WorkerLifecycleOperationId>>,
owners: std::sync::Mutex<
std::collections::HashMap<
worker_runtime::execution::WorkerLifecycleOperationId,
worker_runtime::identity::WorkerRef,
>,
>,
}
impl worker_runtime::execution::WorkerExecutionBackend for PendingRuntimeBackend {
@@ -806,6 +812,21 @@ mod tests {
)
}
fn restore_recovery_disposition(
&self,
operation_id: worker_runtime::execution::WorkerLifecycleOperationId,
worker_ref: &worker_runtime::identity::WorkerRef,
) -> Option<worker_runtime::execution::RestoreRecoveryDisposition> {
use worker_runtime::execution::RestoreRecoveryDisposition as D;
Some(
if self.owners.lock().unwrap().get(&operation_id) == Some(worker_ref) {
D::Continue
} else {
D::Absent
},
)
}
fn restore_worker(
&self,
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
@@ -813,6 +834,10 @@ mod tests {
use worker_runtime::execution::*;
let mut operations = self.operations.lock().unwrap();
operations.push(request.operation_id);
self.owners
.lock()
.unwrap()
.insert(request.operation_id, request.worker_ref.clone());
if operations.len() == 1 {
WorkerExecutionSpawnResult::ReconciliationRequired {
result: WorkerExecutionResult::errored(
@@ -1119,9 +1144,14 @@ mod tests {
}
#[tokio::test]
async fn rejected_resume_does_not_observe_or_open_retained_state() {
async fn settled_nonaccepted_resume_does_not_retry_observe_or_open_retained_state() {
for state in [
BackendWorkerRestoreState::Rejected,
BackendWorkerRestoreState::RolledBack,
BackendWorkerRestoreState::Superseded,
] {
let lifecycle = FakeBackendWorkerLifecycle {
restore_response: restore_response(BackendWorkerRestoreState::Rejected),
restore_response: restore_response(state),
observation: session_observation(serde_json::json!({
"availability": "unavailable",
"reason": "retention_missing",
@@ -1144,15 +1174,16 @@ mod tests {
expected_observation_token: "observed-worker-state".into(),
request_id: "resume-operation".into(),
}),
|_| Ok(false),
|_| panic!("settled nonaccepted restore must not offer recovery retry"),
)
.await
.unwrap_err()
.to_string();
assert!(error.contains("Rejected"));
assert!(error.contains(&format!("{state:?}")));
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
}
}
#[test]
fn resume_candidates_have_no_current_worker_state() {
@@ -1176,6 +1207,7 @@ mod tests {
for state in [
BackendWorkerRestoreState::Rejected,
BackendWorkerRestoreState::RolledBack,
BackendWorkerRestoreState::Superseded,
BackendWorkerRestoreState::ReconciliationRequired,
] {
let mut response = restore_response(state);
+6 -2
View File
@@ -19,15 +19,19 @@ required-features = ["ws-server", "fs-store"]
[features]
default = ["ws-server", "fs-store"]
fs-store = ["dep:fs4"]
# Offline fixture tooling only; never enabled by the product defaults.
test-support = ["fs-store", "ws-server"]
# Explicit process-test synchronization; independent of offline fixture tooling.
restore-checkpoints = ["fs-store"]
http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"]
ws-server = ["http-server", "axum/ws", "tokio/sync"]
[dependencies]
agen.workspace = true
async-trait.workspace = true
base64.workspace = true
axum = { workspace = true, optional = true }
futures = { workspace = true, optional = true }
futures.workspace = true
decodal.workspace = true
flow = { path = "../flow" }
fs4 = { workspace = true, optional = true }
+58
View File
@@ -8,3 +8,61 @@ Manages Worker creation, stopping, and Restore, execution connections within the
- [`execution`] — execution backends and operation-specific results.
- [`working_directory`] — repository materialization, session resources, and safe deletion.
- [`workspace_issuer`] — Workspace signature verification and replay prevention.
## Offline stopped-Worker fixtures
The nondefault `test-support` feature exposes `test_support::StoppedWorkerTemplate`.
It is intentionally limited to current-schema, stopped, attachment-free artificial
conversations with one Segment, text history, and no pending execution/activation.
It does not add a product endpoint or change create/restore authorization.
1. With the source Runtime offline, call `StoppedWorkerTemplate::extract(worker_dir)`.
The source must have its existing Runtime owner lock and identity. Extraction is
read-only; the returned serde template excludes source auth, saved Manifest,
request/config references, run directories, and Runtime operation history.
2. Initialize **new** Runtime identity and Workspace trust in a fresh destination
using normal setup. Preserve the template's Runtime and Workspace IDs.
3. Call `template.install(&options, request, &resolved_manifest)` before starting
the Runtime. Supply a fresh typed `CreateWorkerRequest` (including config/profile
references and trusted memory settings) and `manifest::WorkerManifest` resolved
for that environment. Its Worker name must equal `template.worker_name()`.
4. Start normally and use the ordinary explicit restore operation. Installation
itself creates no execution and leaves the Worker stopped with a fresh guard.
The template preserves the original Session manifest/JSONL bytes, including stable
entry IDs, timestamps, and settled submission receipts. It rejects resumable queues
and Feature invocation history. A saved system prompt is required for restore;
checkpoint receipts must match the Worker's typed receipt schema. Preserved records
must use canonical writer field shapes: unknown/duplicate fields (including nested
fields), discarded optional fields, and opaque Feature history payloads are rejected,
not stripped or rewritten. Portability/secret-marker checks do not replace
review of the supplied artificial conversation or its independently verified hashes.
The installer refuses any destination content other than initialized `auth.toml`;
interrupted publication is fail-closed, not an overwrite/recovery facility.
## Owned-process Restore checkpoints
The independent, nondefault `restore-checkpoints` feature exists only for explicit
process tests in the shared `tests/e2e` harness. Offline `test-support` does not enable
it. Normal binaries reject `--test-restore-checkpoint`; no environment variable or
HTTP endpoint activates this facility. Do not deploy a checkpoint-enabled binary
as a replacement for the default product.
A test-enabled Runtime accepts `--test-restore-checkpoint <config.json>` with a
bounded regular JSON file selecting a loopback socket, Worker ID, one stage and a
diagnostic nonce. The nonce is only a handshake label, never execution identity,
request authority or routing state. The one-shot stages are real prepared candidate
with `NotCommitted`, and durable `Committed` before activation/terminal receipt.
Before acknowledging, the existing bounded canonical reader validates the exact
Worker/operation/decision, syncs its file/directories and requires no saved receipt.
No pending JSON is fabricated and no lifecycle work is skipped.
The Runtime sends bounded newline-delimited JSON evidence, then waits at most eight
seconds for byte `R`. EOF, invalid reply, unavailable peer or deadline releases the
wait into the original operation. The Runtime-state mutex is not held across that
wait; existing per-Worker ownership still covers the actual candidate and cleanup.
The harness verifies the ack, kills/reaps only its owned child and inspects the
canonical aggregate offline before restarting the same roots/identity without the
checkpoint flag or another fixture install. These tests are a separate cfg/binary
lane from featureless product regression; they do not certify arbitrary OS failure,
external power loss or a real model call.
+253 -4
View File
@@ -215,6 +215,38 @@ impl WorkerExecutionResult {
}
}
// Private one-shot synchronization at real publication boundaries. Disconnect
// releases the waiter during unwinding; timeout bounds a forgotten test owner.
#[cfg(test)]
#[derive(Debug)]
pub(crate) struct RestoreTestGate<T> {
pub(crate) entered: std::sync::mpsc::Sender<T>,
pub(crate) release: std::sync::mpsc::Receiver<()>,
}
#[cfg(test)]
impl<T> RestoreTestGate<T> {
pub(crate) fn wait(self, evidence: T) {
let _ = self.entered.send(evidence);
assert!(
!matches!(
self.release
.recv_timeout(std::time::Duration::from_secs(30)),
Err(std::sync::mpsc::RecvTimeoutError::Timeout)
),
"publication test gate timed out"
);
}
}
#[cfg(all(test, feature = "ws-server"))]
#[derive(Debug)]
pub(crate) struct RestoreFlushFault {
pub(crate) remaining_successes: usize,
pub(crate) gate: RestoreTestGate<protocol::Event>,
pub(crate) restored: std::sync::mpsc::Sender<Vec<protocol::Event>>,
}
/// Runtime hooks available to an execution backend for one Worker.
#[cfg(feature = "ws-server")]
type WorkerObservationPublisher = Arc<
@@ -242,10 +274,15 @@ enum WorkerExecutionPublicationState {
#[derive(Clone)]
pub struct WorkerExecutionContext {
worker_ref: WorkerRef,
restore_progress: Option<Arc<dyn Fn(WorkerLifecycleOperationId) + Send + Sync>>,
restore_repair_authorizer:
Option<Arc<dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync>>,
#[cfg(feature = "ws-server")]
observation_publisher: WorkerObservationPublisher,
#[cfg(feature = "ws-server")]
publication_state: Arc<Mutex<WorkerExecutionPublicationState>>,
#[cfg(all(test, feature = "ws-server"))]
flush_fault: Arc<Mutex<Option<RestoreFlushFault>>>,
}
impl WorkerExecutionContext {
@@ -256,7 +293,11 @@ impl WorkerExecutionContext {
) -> Self {
Self {
worker_ref,
restore_progress: None,
restore_repair_authorizer: None,
observation_publisher,
#[cfg(test)]
flush_fault: Arc::default(),
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Active)),
}
}
@@ -268,7 +309,11 @@ impl WorkerExecutionContext {
) -> Self {
Self {
worker_ref,
restore_progress: None,
restore_repair_authorizer: None,
observation_publisher,
#[cfg(test)]
flush_fault: Arc::default(),
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Pending(
VecDeque::new(),
))),
@@ -277,12 +322,56 @@ impl WorkerExecutionContext {
#[cfg(not(feature = "ws-server"))]
pub(crate) fn new(worker_ref: WorkerRef) -> Self {
Self { worker_ref }
Self {
worker_ref,
restore_progress: None,
restore_repair_authorizer: None,
}
}
#[cfg(not(feature = "ws-server"))]
pub(crate) fn candidate(worker_ref: WorkerRef) -> Self {
Self { worker_ref }
Self {
worker_ref,
restore_progress: None,
restore_repair_authorizer: None,
}
}
pub(crate) fn with_restore_repair_authorizer(
mut self,
authorizer: Arc<
dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync,
>,
) -> Self {
self.restore_repair_authorizer = Some(authorizer);
self
}
pub(crate) fn authorize_restore_source_repair(
&self,
operation: WorkerLifecycleOperationId,
) -> Result<(), RuntimeError> {
match &self.restore_repair_authorizer {
Some(authorizer) => authorizer(operation),
// Direct factory/component users own their source repair separately;
// only Runtime contexts carry a Runtime aggregate authority callback.
None => Ok(()),
}
}
pub(crate) fn with_restore_progress(
mut self,
progress: Arc<dyn Fn(WorkerLifecycleOperationId) + Send + Sync>,
) -> Self {
self.restore_progress = Some(progress);
self
}
pub(crate) fn restore_progressed(&self, operation_id: WorkerLifecycleOperationId) {
if let Some(progress) = &self.restore_progress {
progress(operation_id);
}
}
pub fn worker_ref(&self) -> &WorkerRef {
@@ -335,20 +424,65 @@ impl WorkerExecutionContext {
let Some(event) = event else {
return Ok(());
};
if let Err(error) = (self.observation_publisher)(self.worker_ref.clone(), event.clone())
{
#[cfg(test)]
let fault = {
let mut fault = self.flush_fault.lock().unwrap();
if fault.as_ref().is_some_and(|f| f.remaining_successes == 0) {
fault.take()
} else {
if let Some(fault) = fault.as_mut() {
fault.remaining_successes -= 1;
}
None
}
};
#[cfg(test)]
let mut restored = None;
#[cfg(test)]
let publication = if let Some(fault) = fault {
restored = Some(fault.restored);
fault.gate.wait(event.clone());
Err(RuntimeError::InvalidRequest(
"injected buffered publication failure".into(),
))
} else {
(self.observation_publisher)(self.worker_ref.clone(), event.clone())
};
#[cfg(not(test))]
let publication = (self.observation_publisher)(self.worker_ref.clone(), event.clone());
if let Err(error) = publication {
let mut state = self
.publication_state
.lock()
.map_err(|_| RuntimeError::StatePoisoned)?;
if let WorkerExecutionPublicationState::Activating(events) = &mut *state {
events.push_front(event);
#[cfg(test)]
if let Some(restored) = restored {
let _ = restored.send(events.iter().cloned().collect());
}
}
return Err(error);
}
}
}
#[cfg(all(test, feature = "ws-server"))]
pub(crate) fn test_fail_flush(&self, fault: RestoreFlushFault) {
assert!(self.flush_fault.lock().unwrap().replace(fault).is_none());
}
#[cfg(all(test, feature = "ws-server"))]
pub(crate) fn test_buffered_events(&self) -> Vec<protocol::Event> {
match &*self.publication_state.lock().unwrap() {
WorkerExecutionPublicationState::Pending(events)
| WorkerExecutionPublicationState::Activating(events) => {
events.iter().cloned().collect()
}
_ => Vec::new(),
}
}
#[cfg(not(feature = "ws-server"))]
pub(crate) fn activate_candidate(&self) -> Result<(), RuntimeError> {
Ok(())
@@ -424,6 +558,7 @@ pub struct WorkerExecutionSpawnRequest {
/// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker.
#[derive(Clone, Debug)]
pub struct WorkerExecutionRestoreRequest {
pub source_repair_possible: bool,
pub operation_id: WorkerLifecycleOperationId,
pub worker_ref: WorkerRef,
pub request: crate::catalog::CreateWorkerRequest,
@@ -565,6 +700,19 @@ impl fmt::Debug for WorkerProtocolTransport {
}
}
/// Derived resource disposition for an existing restore owner. Not durable
/// authority: Runtime must reread its decision before choosing any action.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RestoreRecoveryDisposition {
/// No live or pending owner for this exact operation. Not cleanup evidence.
Absent,
InFlight,
Continue,
Discard,
/// A retained cleanup-only owner; never reconstruct or activate it.
Reclaim,
}
/// The connection selects this backend; WorkerRef identifies an execution,
/// never a capability or authorization token.
pub trait WorkerExecutionBackend: Send + Sync + 'static {
@@ -633,6 +781,33 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
Ok(())
}
/// Completed resource-owner evidence retained while Runtime settlement may
/// still need retry. This is not a new request or execution authorization.
fn completed_restore_failure(
&self,
_operation_id: WorkerLifecycleOperationId,
_worker_ref: &WorkerRef,
) -> Option<WorkerExecutionSpawnResult> {
None
}
/// Opt in to Runtime-owned autonomous restore continuation. Pending tasks
/// signal completion through the original execution context; callbacks must
/// not wait for Runtime locks. Independent of resource disposition support.
fn supports_restore_continuation(&self) -> bool {
false
}
/// Describe exact operation resource ownership for warm/cold recovery
/// dispatch. This does not opt in to callbacks or autonomous continuation.
fn restore_recovery_disposition(
&self,
_operation_id: WorkerLifecycleOperationId,
_worker_ref: &WorkerRef,
) -> Option<RestoreRecoveryDisposition> {
None
}
fn reconcile_restore(
&self,
request: WorkerExecutionRestoreRequest,
@@ -640,6 +815,42 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
self.restore_worker(request)
}
/// Reclaim an uncommitted operation after restart without preparing,
/// repairing, or activating a replacement candidate. Runtime must establish
/// the durable uncommitted decision before calling this seam. Unsupported
/// cleanup is uncertainty, never a successful rollback.
fn reclaim_uncommitted_restore(
&self,
_request: WorkerExecutionRestoreRequest,
) -> WorkerExecutionSpawnResult {
WorkerExecutionSpawnResult::ReconciliationRequired {
result: WorkerExecutionResult::unsupported(
WorkerExecutionOperation::Restore,
"backend cannot reclaim an uncommitted restore after restart",
),
worker_state: None,
workdir_attachments: Vec::new(),
}
}
/// Discard only the exact unpublished operation after Runtime has proved
/// its durable decision uncommitted. This is not Stop and must not affect a
/// connected execution. RolledBack requires confirmed cleanup and no repair.
fn discard_uncommitted_restore(
&self,
_operation_id: WorkerLifecycleOperationId,
_worker_ref: &WorkerRef,
) -> WorkerExecutionSpawnResult {
WorkerExecutionSpawnResult::ReconciliationRequired {
result: WorkerExecutionResult::unsupported(
WorkerExecutionOperation::Restore,
"backend cannot confirm unpublished restore cleanup",
),
worker_state: None,
workdir_attachments: Vec::new(),
}
}
fn activate_restored_worker(
&self,
_operation_id: WorkerLifecycleOperationId,
@@ -901,6 +1112,28 @@ impl WorkerExecutionBackendRef {
self.backend.preflight_restore(request)
}
pub(crate) fn completed_restore_failure(
&self,
operation_id: WorkerLifecycleOperationId,
worker_ref: &WorkerRef,
) -> Option<WorkerExecutionSpawnResult> {
self.backend
.completed_restore_failure(operation_id, worker_ref)
}
pub(crate) fn supports_restore_continuation(&self) -> bool {
self.backend.supports_restore_continuation()
}
pub(crate) fn restore_recovery_disposition(
&self,
operation_id: WorkerLifecycleOperationId,
worker_ref: &WorkerRef,
) -> Option<RestoreRecoveryDisposition> {
self.backend
.restore_recovery_disposition(operation_id, worker_ref)
}
pub(crate) fn reconcile_restore(
&self,
request: WorkerExecutionRestoreRequest,
@@ -908,6 +1141,22 @@ impl WorkerExecutionBackendRef {
self.backend.reconcile_restore(request)
}
pub(crate) fn reclaim_uncommitted_restore(
&self,
request: WorkerExecutionRestoreRequest,
) -> WorkerExecutionSpawnResult {
self.backend.reclaim_uncommitted_restore(request)
}
pub(crate) fn discard_uncommitted_restore(
&self,
operation_id: WorkerLifecycleOperationId,
worker_ref: &WorkerRef,
) -> WorkerExecutionSpawnResult {
self.backend
.discard_uncommitted_restore(operation_id, worker_ref)
}
pub(crate) fn activate_restored_worker(
&self,
operation_id: WorkerLifecycleOperationId,
File diff suppressed because it is too large Load Diff
+84
View File
@@ -25,6 +25,7 @@ use crate::interaction::{WorkerInput, WorkerInteractionAck};
use crate::management::{RuntimeSummary, WorkerDeleteResult};
#[cfg(feature = "ws-server")]
use crate::observation::WorkerObservationCursor;
#[cfg(feature = "fs-store")]
use crate::retention::{
WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory,
};
@@ -1568,6 +1569,7 @@ fn protocol_error_event(message: impl Into<String>) -> protocol::Event {
}
}
#[cfg(feature = "fs-store")]
#[allow(dead_code)]
async fn worker_retention_inventory(
State(state): State<RuntimeHttpState>,
@@ -1589,6 +1591,7 @@ async fn worker_retention_inventory(
.map_err(RuntimeHttpRestError::runtime)
}
#[cfg(feature = "fs-store")]
#[allow(dead_code)]
async fn execute_worker_retention(
State(state): State<RuntimeHttpState>,
@@ -2654,6 +2657,87 @@ mod tests {
.unwrap()
}
// Generated management routes and real Workspace capability middleware stay
// connected even when this process has no filesystem retention provider.
#[cfg(not(feature = "fs-store"))]
#[tokio::test]
async fn memory_retention_routes_preserve_auth_and_return_typed_unavailable() {
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
let trust = workspace_trust(&identity);
let runtime = Runtime::new_memory();
let app = runtime_http_router_with_workspace_auth(
runtime,
None,
workspace_test_auth(vec![trust.clone()]),
);
let worker_id = WorkerId::now_v7();
let body = runtime_api::WorkerRetentionExecutionRequest {
operation_id: "memory-retention".into(),
input_fingerprint: "input".into(),
archive_id: None,
workspace_id: "workspace-a".into(),
source_runtime_id: "runtime-test".into(),
worker_id,
source_created_at: "2026-01-01T00:00:00Z".into(),
removed_at: "2026-01-02T00:00:00Z".into(),
effective_profile: None,
retention_class: None,
policy_id: "policy".into(),
session_disposition: runtime_api::SessionDisposition::Purge,
diagnostics_disposition: runtime_api::DiagnosticsDisposition::Purge,
};
for (method, path, body) in [
(
"GET",
format!("/v1/workers/{worker_id}/retention/inventory"),
vec![],
),
(
"POST",
format!("/v1/workers/{worker_id}/retention/execute"),
serde_json::to_vec(&body).unwrap(),
),
] {
let unsigned = app
.clone()
.oneshot(
Request::builder()
.method(method)
.uri(&path)
.header(header::CONTENT_TYPE, "application/json")
.body(Body::from(body.clone()))
.unwrap(),
)
.await
.unwrap();
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
let claims = signed_claims(&trust, method, &path, &body);
let response =
signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
let error: RuntimeHttpErrorResponse = read_json(response).await;
assert_eq!(error.error.code, "worker_retention_unavailable");
}
for (mut value, expected_status, expected_code) in [
(body.clone(), StatusCode::NOT_FOUND, "worker_not_found"),
(body.clone(), StatusCode::BAD_REQUEST, "worker_id_mismatch"),
] {
if expected_status == StatusCode::NOT_FOUND {
value.workspace_id = "workspace-b".into();
} else {
value.worker_id = WorkerId::now_v7();
}
let path = format!("/v1/workers/{worker_id}/retention/execute");
let bytes = serde_json::to_vec(&value).unwrap();
let claims = signed_claims(&trust, "POST", &path, &bytes);
let response =
signed_request(app.clone(), &identity, &claims, bytes, "workspace-a").await;
assert_eq!(response.status(), expected_status);
let error: RuntimeHttpErrorResponse = read_json(response).await;
assert_eq!(error.error.code, expected_code);
}
}
#[tokio::test]
async fn ordinary_signed_ping_succeeds_without_handshake_and_rejects_replay() {
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
@@ -719,6 +719,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
server_api::WorkerRestoreState::RolledBack => {
runtime_api::WorkerRestoreState::RolledBack
}
server_api::WorkerRestoreState::Superseded => {
runtime_api::WorkerRestoreState::Superseded
}
server_api::WorkerRestoreState::ReconciliationRequired => {
runtime_api::WorkerRestoreState::ReconciliationRequired
}
@@ -769,6 +772,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
server_api::WorkerRestoreState::RolledBack => {
runtime_api::WorkerRestoreState::RolledBack
}
server_api::WorkerRestoreState::Superseded => {
runtime_api::WorkerRestoreState::Superseded
}
server_api::WorkerRestoreState::ReconciliationRequired => {
runtime_api::WorkerRestoreState::ReconciliationRequired
}
@@ -881,6 +887,8 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
)
})?;
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
#[cfg(feature = "fs-store")]
{
let inventory = self
.state
.runtime
@@ -889,6 +897,16 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
.map_err(api_error)?;
response(inventory)
}
#[cfg(not(feature = "fs-store"))]
{
let _ = (scope, worker_ref);
Err(runtime_api::RuntimeApiError::new(
503,
"worker_retention_unavailable",
"Worker retention requires filesystem storage",
))
}
}
async fn execute_retention(
&self,
@@ -905,21 +923,23 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
)
})?;
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
let request: WorkerRetentionExecutionRequest = request(value)?;
if request.workspace_id != scope.workspace_id {
if value.workspace_id != scope.workspace_id {
return Err(runtime_api::RuntimeApiError::new(
StatusCode::NOT_FOUND.as_u16(),
"worker_not_found",
"worker does not belong to the authenticated Workspace",
));
}
if request.worker_id != worker_ref.worker_id {
if value.worker_id != worker_ref.worker_id {
return Err(runtime_api::RuntimeApiError::new(
StatusCode::BAD_REQUEST.as_u16(),
"worker_id_mismatch",
"Worker retention request worker_id does not match route worker_id",
));
}
#[cfg(feature = "fs-store")]
{
let request: WorkerRetentionExecutionRequest = request(value)?;
let result = self
.state
.runtime
@@ -928,4 +948,11 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
.map_err(api_error)?;
response(result)
}
#[cfg(not(feature = "fs-store"))]
Err(runtime_api::RuntimeApiError::new(
503,
"worker_retention_unavailable",
"Worker retention requires filesystem storage",
))
}
}
+7
View File
@@ -16,6 +16,11 @@ pub mod management;
pub mod observation;
pub mod profile_archive;
pub mod resource;
#[cfg(feature = "restore-checkpoints")]
pub mod restore_checkpoints;
#[cfg(test)]
#[path = "../../../tests/support/restore_environment.rs"]
mod restore_test_environment;
#[cfg(feature = "fs-store")]
pub mod retention;
mod runtime;
@@ -32,6 +37,8 @@ pub mod workspace_issuer;
#[cfg(feature = "http-server")]
pub mod workspace_request;
#[cfg(feature = "test-support")]
pub use fs_store::test_support;
#[cfg(feature = "fs-store")]
pub use fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
pub use management::RuntimeOptions;
+72
View File
@@ -117,6 +117,12 @@ fn run() -> Result<(), ProcessError> {
let listener = tokio::net::TcpListener::bind(config.http.bind_addr).await?;
let local_addr = listener.local_addr()?;
let worker_runtime = build_runtime(&config)?;
#[cfg(feature = "restore-checkpoints")]
if let Some(checkpoint) = &config.test_restore_checkpoint {
worker_runtime
.install_test_restore_checkpoint(checkpoint.clone())
.map_err(ProcessError::Runtime)?;
}
eprintln!(
"yoi-runtime listening on {local_addr}; intended client is a trusted backend/proxy, not a browser"
);
@@ -391,6 +397,21 @@ where
let (flag, inline_value) = split_flag_value(arg)?;
match flag.as_str() {
"--help" | "-h" => return Ok(None),
#[cfg(feature = "restore-checkpoints")]
"--test-restore-checkpoint" => {
let path = take_value(&flag, inline_value, &mut args)?;
if config.test_restore_checkpoint.is_some() {
return Err(ProcessError::usage(
"--test-restore-checkpoint may only be specified once".into(),
));
}
config.test_restore_checkpoint = Some(
worker_runtime::restore_checkpoints::RestoreCheckpointConfig::from_path(
Path::new(&path),
)
.map_err(ProcessError::Runtime)?,
);
}
"--bind" => {
let value = take_value(&flag, inline_value, &mut args)?;
config.http.bind_addr = value.parse::<SocketAddr>().map_err(|error| {
@@ -465,6 +486,12 @@ where
}
apply_store_selection(&mut config);
#[cfg(feature = "restore-checkpoints")]
if config.test_restore_checkpoint.is_some() && config.no_store {
return Err(ProcessError::usage(
"--test-restore-checkpoint requires an fs Runtime store".into(),
));
}
Ok(Some(config))
}
@@ -521,6 +548,8 @@ struct ProcessConfig {
no_store: bool,
backend_resource_endpoint: Option<String>,
backend_resource_token: Option<String>,
#[cfg(feature = "restore-checkpoints")]
test_restore_checkpoint: Option<worker_runtime::restore_checkpoints::RestoreCheckpointConfig>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -542,6 +571,8 @@ impl ProcessConfig {
no_store: false,
backend_resource_endpoint: None,
backend_resource_token: None,
#[cfg(feature = "restore-checkpoints")]
test_restore_checkpoint: None,
})
}
@@ -1227,6 +1258,47 @@ Auth commands:
mod tests {
use super::*;
#[test]
fn checkpoint_cli_is_absent_by_default() {
let config = parse_args([] as [&str; 0]).unwrap().unwrap();
#[cfg(feature = "restore-checkpoints")]
assert!(config.test_restore_checkpoint.is_none());
#[cfg(not(feature = "restore-checkpoints"))]
{
let _ = config;
let error = parse_args(["--test-restore-checkpoint", "never-opened.json"]).unwrap_err();
assert!(error.to_string().contains("unknown argument"));
}
}
#[cfg(feature = "restore-checkpoints")]
#[test]
fn checkpoint_cli_requires_valid_explicit_file_and_fs_store() {
use worker_runtime::restore_checkpoints::{
RestoreCheckpointConfig, RestoreCheckpointStage,
};
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("checkpoint.json");
let checkpoint = RestoreCheckpointConfig {
endpoint: "127.0.0.1:1234".parse().unwrap(),
worker_id: protocol::WorkerId::now_v7(),
stage: RestoreCheckpointStage::CommittedPreterminal,
nonce: "cli-test".into(),
};
std::fs::write(&path, serde_json::to_vec(&checkpoint).unwrap()).unwrap();
let args = ["--test-restore-checkpoint", path.to_str().unwrap()];
let config = parse_args(args).unwrap().unwrap();
assert_eq!(config.test_restore_checkpoint, Some(checkpoint));
assert!(parse_args([args[0], args[1], "--no-store"]).is_err());
assert!(parse_args([args[0], args[1], args[0], args[1]]).is_err());
assert!(parse_args([args[0]]).is_err());
std::fs::write(&path, b"private-invalid-body").unwrap();
let error = parse_args(args).unwrap_err().to_string();
assert!(!error.contains("private-invalid-body"));
std::fs::remove_file(&path).unwrap();
assert!(parse_args(args).is_err());
}
#[test]
fn help_does_not_start_server() {
assert!(parse_args(["--help"]).unwrap().is_none());
+2
View File
@@ -1,6 +1,8 @@
#[cfg(feature = "http-server")]
use crate::auth::BACKEND_RESOURCE_FETCH_PERMISSION;
use crate::identity::WorkerId;
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef, sha256_hex};
#[cfg(feature = "http-server")]
use crate::workspace_request::RuntimeWorkspaceRequestClient;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
@@ -0,0 +1,144 @@
//! Explicit, ephemeral synchronization for owned Runtime process tests only.
//!
//! The nonce authenticates no Runtime action and is never a routing identity.
//! Socket failure, EOF and the fixed deadline only release the gate; the lifecycle
//! owner continues its original operation and remains responsible for cleanup.
use std::fs::OpenOptions;
use std::io::{Read, Write};
use std::net::{SocketAddr, TcpStream};
use std::path::Path;
use std::time::{Duration, Instant};
use protocol::WorkerId;
use serde::{Deserialize, Serialize};
use crate::error::RuntimeError;
use crate::execution::WorkerLifecycleOperationId;
pub const RESTORE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(8);
pub const MAX_RESTORE_CHECKPOINT_CONFIG_BYTES: usize = 4096;
pub const MAX_RESTORE_CHECKPOINT_ACK_BYTES: usize = 4096;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RestoreCheckpointStage {
PreparedNotCommitted,
CommittedPreterminal,
ActivatedPreterminal,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RestoreCheckpointCommit {
NotCommitted,
Committed,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RestoreCheckpointConfig {
pub endpoint: SocketAddr,
pub worker_id: WorkerId,
pub stage: RestoreCheckpointStage,
pub nonce: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RestoreCheckpointAck {
pub nonce: String,
pub stage: RestoreCheckpointStage,
pub runtime_id: String,
pub worker_id: WorkerId,
pub operation_id: WorkerLifecycleOperationId,
pub commit: RestoreCheckpointCommit,
pub receipt_absent: bool,
pub pid: u32,
}
fn invalid_config() -> RuntimeError {
// Do not include serde errors, paths or input contents in process diagnostics.
RuntimeError::InvalidRequest("invalid test restore checkpoint configuration".into())
}
impl RestoreCheckpointConfig {
pub fn validate(&self) -> Result<(), RuntimeError> {
if !self.endpoint.ip().is_loopback()
|| self.endpoint.port() == 0
|| self.nonce.is_empty()
|| self.nonce.len() > 256
|| self.nonce.chars().any(char::is_control)
{
return Err(invalid_config());
}
Ok(())
}
pub fn from_path(path: &Path) -> Result<Self, RuntimeError> {
let mut options = OpenOptions::new();
options.read(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
// A FIFO must not stall startup before its regular-file check.
options.custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW);
}
let file = options.open(path).map_err(|_| invalid_config())?;
let metadata = file.metadata().map_err(|_| invalid_config())?;
if !metadata.is_file() || metadata.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 {
return Err(invalid_config());
}
let mut bytes = Vec::new();
file.take(MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 + 1)
.read_to_end(&mut bytes)
.map_err(|_| invalid_config())?;
if bytes.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES {
return Err(invalid_config());
}
let config: Self = serde_json::from_slice(&bytes).map_err(|_| invalid_config())?;
config.validate()?;
Ok(config)
}
}
pub(crate) fn wait_for_release(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) {
let _ = exchange(config, ack);
}
fn remaining(deadline: Instant) -> std::io::Result<Duration> {
deadline
.checked_duration_since(Instant::now())
.filter(|remaining| !remaining.is_zero())
.ok_or_else(|| std::io::Error::from(std::io::ErrorKind::TimedOut))
}
fn exchange(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) -> std::io::Result<()> {
let deadline = Instant::now() + RESTORE_CHECKPOINT_TIMEOUT;
let mut bytes = serde_json::to_vec(ack)?;
bytes.push(b'\n');
if bytes.len() > MAX_RESTORE_CHECKPOINT_ACK_BYTES {
return Err(std::io::Error::from(std::io::ErrorKind::InvalidData));
}
let mut stream = TcpStream::connect_timeout(&config.endpoint, remaining(deadline)?)?;
// Each syscall shares one budget; partial writes cannot reset the deadline.
let mut pending = bytes.as_slice();
while !pending.is_empty() {
stream.set_write_timeout(Some(remaining(deadline)?))?;
let written = stream.write(pending)?;
if written == 0 {
return Err(std::io::Error::from(std::io::ErrorKind::WriteZero));
}
pending = &pending[written..];
}
stream.set_read_timeout(Some(remaining(deadline)?))?;
let mut release = [0];
stream.read_exact(&mut release)?;
if release != [b'R'] {
return Err(std::io::Error::from(std::io::ErrorKind::InvalidData));
}
Ok(())
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,139 @@
use super::*;
use std::io::{BufRead, BufReader};
use std::net::TcpListener;
use std::thread;
fn config(endpoint: SocketAddr) -> RestoreCheckpointConfig {
RestoreCheckpointConfig {
endpoint,
worker_id: WorkerId::now_v7(),
stage: RestoreCheckpointStage::PreparedNotCommitted,
nonce: "checkpoint-test-handshake".into(),
}
}
fn ack(config: &RestoreCheckpointConfig) -> RestoreCheckpointAck {
RestoreCheckpointAck {
nonce: config.nonce.clone(),
stage: config.stage,
runtime_id: "checkpoint-runtime".into(),
worker_id: config.worker_id,
operation_id: WorkerLifecycleOperationId::new(),
commit: RestoreCheckpointCommit::NotCommitted,
receipt_absent: true,
pid: std::process::id(),
}
}
#[test]
fn config_rejects_nonliteral_nonloopback_bad_stage_unknown_fields_and_invalid_worker() {
let valid = serde_json::to_value(config("127.0.0.1:1234".parse().unwrap())).unwrap();
for (field, value) in [
("endpoint", "localhost:1234"),
("endpoint", "192.0.2.1:1234"),
("endpoint", "0.0.0.0:1234"),
("endpoint", "127.0.0.1:0"),
("endpoint", "[::ffff:127.0.0.1]:1234"),
("worker_id", "not-a-worker-id"),
("stage", "after_terminal"),
("nonce", ""),
("nonce", "secret\nbody"),
("timeout", "unbounded"),
] {
let mut invalid = valid.clone();
invalid[field] = value.into();
let parsed = serde_json::from_value::<RestoreCheckpointConfig>(invalid);
assert!(
parsed.is_err() || parsed.unwrap().validate().is_err(),
"{field}"
);
}
for endpoint in ["127.0.0.1:1234", "[::1]:1234"] {
config(endpoint.parse().unwrap()).validate().unwrap();
}
}
#[test]
fn config_file_bounds_and_errors_do_not_disclose_contents() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("config.json");
assert!(RestoreCheckpointConfig::from_path(&path).is_err());
std::fs::write(&path, vec![b'x'; MAX_RESTORE_CHECKPOINT_CONFIG_BYTES + 1]).unwrap();
assert!(RestoreCheckpointConfig::from_path(&path).is_err());
std::fs::write(
&path,
br#"{"nonce":"private-config-body","stage":"secret"}"#,
)
.unwrap();
let error = RestoreCheckpointConfig::from_path(&path)
.unwrap_err()
.to_string();
assert!(!error.contains("private-config-body"));
assert!(!error.contains("secret"));
let mut valid = config("127.0.0.1:1234".parse().unwrap());
valid.nonce = "x".repeat(257);
assert!(valid.validate().is_err());
valid.nonce = "valid".into();
std::fs::write(&path, serde_json::to_vec(&valid).unwrap()).unwrap();
assert_eq!(RestoreCheckpointConfig::from_path(&path).unwrap(), valid);
assert!(RestoreCheckpointConfig::from_path(dir.path()).is_err());
}
#[test]
fn explicit_release_eof_and_invalid_release_close_the_owned_stream() {
for release in [Some(b'R'), None, Some(b'X')] {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let config = config(listener.local_addr().unwrap());
let expected = ack(&config);
let sent = expected.clone();
let task = thread::spawn(move || exchange(&config, &sent));
let (mut stream, _) = listener.accept().unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(2)))
.unwrap();
let mut line = String::new();
BufReader::new(&mut stream).read_line(&mut line).unwrap();
assert!(line.len() <= MAX_RESTORE_CHECKPOINT_ACK_BYTES);
assert_eq!(
serde_json::from_str::<RestoreCheckpointAck>(&line).unwrap(),
expected
);
if let Some(byte) = release {
stream.write_all(&[byte]).unwrap();
} else {
stream.shutdown(std::net::Shutdown::Write).unwrap();
}
assert_eq!(task.join().unwrap().is_ok(), release == Some(b'R'));
let mut byte = [0];
assert_eq!(stream.read(&mut byte).unwrap(), 0);
}
}
#[test]
fn absent_listener_resumes_without_waiting_or_retrying() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let config = config(listener.local_addr().unwrap());
drop(listener);
let start = Instant::now();
wait_for_release(&config, &ack(&config));
assert!(start.elapsed() < Duration::from_secs(2));
}
#[test]
fn silent_peer_hits_fixed_deadline_and_stream_is_dropped() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let config = config(listener.local_addr().unwrap());
let start = Instant::now();
let task = thread::spawn(move || exchange(&config, &ack(&config)));
let (mut stream, _) = listener.accept().unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(12)))
.unwrap();
let mut reader = BufReader::new(&mut stream);
let mut line = String::new();
reader.read_line(&mut line).unwrap();
assert!(task.join().unwrap().is_err());
assert!(start.elapsed() >= RESTORE_CHECKPOINT_TIMEOUT - Duration::from_millis(100));
assert!(start.elapsed() < RESTORE_CHECKPOINT_TIMEOUT + Duration::from_secs(3));
assert_eq!(reader.read(&mut [0]).unwrap(), 0);
}
@@ -3,7 +3,7 @@
//! only the empty segment envelope is built with current APIs.
use super::tests::{request, write_json};
use super::*;
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions, SCHEMA_VERSION};
use serde_json::{Value, json};
use session_store::{LogEntry, SegmentId, SessionId, Store, WorkerSessionStore};
use std::collections::BTreeMap;
@@ -226,7 +226,7 @@ fn completed_v4_archive(
#[test]
fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
for runtime_schema in [11, 12] {
for runtime_schema in [11, 12, SCHEMA_VERSION] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
@@ -329,7 +329,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
(storage.checksum_sha256, storage.bytes, storage.file_count),
(checksum, bytes, count)
);
if runtime_schema == 12 {
if runtime_schema == SCHEMA_VERSION {
for name in [
PathBuf::from("runtime.json"),
PathBuf::from(format!("workers/{worker}/worker.json")),
@@ -346,8 +346,8 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
);
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
assert_eq!(tree_bytes(&root), once, "retry rewrote migrated evidence");
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists());
// session-storage, not the immutable receipt's old checksum, attests
// current bytes. Tampering with those bytes must remain detectable.
let path = archive_session
@@ -360,7 +360,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
#[test]
fn corrupt_archive_does_not_block_startup_or_healthy_session_migration_and_repair_converges() {
for runtime_schema in [11, 12] {
for runtime_schema in [11, 12, SCHEMA_VERSION] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
@@ -440,8 +440,8 @@ fn unmigratable_archive_preserves_evidence_across_startup_and_reopen() {
] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (live_worker, live_session, live_segment) = runtime_fixture(&root, 12);
let (worker, _, segment) = stopped_worker(&root, 12);
let (live_worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION);
let (worker, _, segment) = stopped_worker(&root, SCHEMA_VERSION);
let provider = FsWorkerRetentionProvider::new(&root);
let path = provider
.worker_dir(worker)
@@ -542,8 +542,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() {
] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, _, _) = runtime_fixture(&root, 12);
completed_v4_archive(&root, 12);
let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION);
completed_v4_archive(&root, SCHEMA_VERSION);
let provider = FsWorkerRetentionProvider::new(&root);
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
let archive = provider.archive_dir("archive-a").unwrap();
@@ -575,8 +575,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() {
fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
runtime_fixture(&root, 12);
let (session, _, _, _) = completed_v4_archive(&root, 12);
runtime_fixture(&root, SCHEMA_VERSION);
let (session, _, _, _) = completed_v4_archive(&root, SCHEMA_VERSION);
let ledger = root.join("retention/session-sources");
fs::create_dir_all(&ledger).unwrap();
fs::write(ledger.join(format!("{session}.json")), b"{broken\n").unwrap();
@@ -591,7 +591,7 @@ fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
fn runtime_v5_probe_and_staging_skip_erased_originals() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, live_session, live_segment) = runtime_fixture(&root, 12);
let (worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION);
let erased = root
.join("orphaned-workers")
.join(WorkerId::now_v7().to_string());
@@ -643,7 +643,7 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
for corruption in ["broken-json", "future-schema"] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, 12);
let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION);
let session_root = root
.join("workers")
.join(worker.to_string())
@@ -674,19 +674,19 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
assert_eq!(tree_bytes(&root), before);
assert!(FsRuntimeStore::migrate(&options).is_err());
assert_eq!(tree_bytes(&root), before);
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists());
}
}
#[cfg(unix)]
#[test]
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
// Relative socket names keep this fixture independent of the checkout/TMPDIR
// length. The TempDir still owns the same tree; no process cwd is changed.
// The same TempDir owns the relative bind/connect reference. No cwd change
// or alias is needed, and the real live-socket migration refusal stays tested.
let temp = tempfile::tempdir_in(".").unwrap();
let root = Path::new(temp.path().file_name().unwrap()).join("runtime");
let (worker, _, _) = runtime_fixture(&root, 12);
let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION);
let before = tree_bytes(&root);
let runs = root.join("workers").join(worker.to_string()).join("runs");
fs::create_dir_all(runs.join("1")).unwrap();
@@ -697,7 +697,7 @@ fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
drop(listener);
fs::remove_dir_all(runs).unwrap();
assert_eq!(tree_bytes(&root), before);
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
}
#[test]
@@ -705,7 +705,7 @@ fn current_runtime_upgrades_v5_role_metadata_through_staging_without_reserializi
{
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, 12);
let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION);
let aggregate = root.join("workers").join(worker.to_string());
let path = aggregate
.join("session/segments")
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,112 @@
use super::*;
use crate::restore_checkpoints::{
RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage,
wait_for_release,
};
impl Runtime {
/// Install an explicitly selected, one-shot process-test gate after normal startup.
/// It is not persisted or consulted by admission, routing or recovery authority.
pub fn install_test_restore_checkpoint(
&self,
config: RestoreCheckpointConfig,
) -> Result<(), RuntimeError> {
config.validate()?;
let mut state = self.lock()?;
if !matches!(state.persistence, RuntimePersistence::Fs(_)) {
return Err(RuntimeError::InvalidRequest(
"test restore checkpoint requires an fs Runtime store".into(),
));
}
state.restore_checkpoint = Some(config);
Ok(())
}
// The lifecycle caller holds the per-Worker operation lock. No Runtime-state
// mutex is retained while waiting; the real backend candidate remains owned by
// the same operation throughout this synchronous call.
pub(super) fn test_restore_checkpoint(
&self,
worker_ref: &WorkerRef,
operation_id: WorkerLifecycleOperationId,
stage: RestoreCheckpointStage,
) {
let config = {
let Ok(mut state) = self.lock() else {
return;
};
let slot = &mut state.restore_checkpoint;
if !slot.as_ref().is_some_and(|config| {
config.worker_id == worker_ref.worker_id && config.stage == stage
}) {
return;
}
// Consume even if canonical verification or the handshake fails.
slot.take().expect("matched checkpoint")
};
let Ok(ack) = self.restore_checkpoint_ack(&config, worker_ref, operation_id) else {
return;
};
wait_for_release(&config, &ack);
}
pub(super) fn restore_checkpoint_ack(
&self,
config: &RestoreCheckpointConfig,
worker_ref: &WorkerRef,
operation_id: WorkerLifecycleOperationId,
) -> Result<RestoreCheckpointAck, RuntimeError> {
let invalid = || {
RuntimeError::InvalidRequest("test restore checkpoint authority did not match".into())
};
let state = self.lock()?;
let RuntimePersistence::Fs(store) = &state.persistence else {
return Err(invalid());
};
// The existing helper validates the exact bounded canonical aggregate and
// syncs that same file plus its owning directories. Never manufacture a
// journal marker or infer commit from an in-memory projection.
let record = store.read_durable_worker_record(worker_ref)?;
let PersistedWorkerExecutionState::ReconciliationRequired(
PersistedWorkerLifecycleOperation::Restore(pending),
) = &record.execution_state
else {
return Err(invalid());
};
let expected = match config.stage {
RestoreCheckpointStage::PreparedNotCommitted => RestoreCommitFact::NotCommitted,
RestoreCheckpointStage::CommittedPreterminal
| RestoreCheckpointStage::ActivatedPreterminal => RestoreCommitFact::Committed,
};
if config.worker_id != worker_ref.worker_id
|| pending.operation_id != operation_id
|| pending.commit != expected
{
return Err(invalid());
}
let guard = &record.restore_guard;
let id = guard.active_request_id.as_ref().ok_or_else(invalid)?;
let owner = guard.owners.get(id).ok_or_else(invalid)?;
if owner.operation_id != operation_id
|| owner.request.request_id != *id
|| owner.receipt.is_some()
{
return Err(invalid());
}
let runtime_id = state.runtime_identity.clone().ok_or_else(invalid)?;
Ok(RestoreCheckpointAck {
nonce: config.nonce.clone(),
stage: config.stage,
runtime_id,
worker_id: record.worker_id,
operation_id,
commit: match expected {
RestoreCommitFact::NotCommitted => RestoreCheckpointCommit::NotCommitted,
RestoreCommitFact::Committed => RestoreCheckpointCommit::Committed,
RestoreCommitFact::LegacyUnknown => unreachable!(),
},
receipt_absent: true,
pid: std::process::id(),
})
}
}
@@ -0,0 +1,304 @@
use super::*;
use crate::restore_checkpoints::{
RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage,
};
use std::io::{BufRead, BufReader, Write};
use std::net::TcpListener;
use std::time::Duration;
#[test]
fn checkpoint_stages_ack_canonical_decision_and_eof_resumes_same_operation_once() {
// Component scope: durable Runtime with a substituted execution backend.
// Real prepared Controllers and process death are owned by the E2E suite.
for (stage, release) in [
(RestoreCheckpointStage::PreparedNotCommitted, "release"),
(RestoreCheckpointStage::CommittedPreterminal, "eof"),
(RestoreCheckpointStage::CommittedPreterminal, "timeout"),
(RestoreCheckpointStage::ActivatedPreterminal, "release"),
] {
let dir = tempfile::tempdir().unwrap();
let backend = Arc::new(TestExecutionBackend::default());
let runtime = Runtime::with_fs_store_and_execution_backend(
FsRuntimeStoreOptions::new(dir.path().join("runtime"))
.with_runtime_id("checkpoint-runtime"),
backend.clone(),
)
.unwrap();
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let worker = runtime
.create_worker(task_request("checkpoint original operation"))
.unwrap();
runtime.stop_worker(&worker.worker_ref, None).unwrap();
let request = runtime.test_restore_request(&worker.worker_ref);
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
runtime
.install_test_restore_checkpoint(RestoreCheckpointConfig {
endpoint: listener.local_addr().unwrap(),
worker_id: worker.worker_id,
stage,
nonce: "runtime-stage-test".into(),
})
.unwrap();
let cloned = runtime.clone();
let worker_ref = worker.worker_ref.clone();
let sent_request = request.clone();
let (sent, received) = std::sync::mpsc::channel();
let task = std::thread::spawn(move || {
sent.send(cloned.restore_worker_operation(&worker_ref, sent_request))
.unwrap();
});
listener.set_nonblocking(true).unwrap();
let stream = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(async {
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
let (stream, _) = tokio::time::timeout(Duration::from_secs(5), listener.accept())
.await
.expect("checkpoint did not connect")
.unwrap();
stream.into_std().unwrap()
});
stream.set_nonblocking(false).unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(3)))
.unwrap();
let mut reader = BufReader::new(stream);
let mut line = String::new();
reader.read_line(&mut line).unwrap();
let ack: RestoreCheckpointAck = serde_json::from_str(&line).unwrap();
assert_eq!(ack.stage, stage);
assert_eq!(ack.nonce, "runtime-stage-test");
assert_eq!(ack.runtime_id, "checkpoint-runtime");
assert_eq!(ack.worker_id, worker.worker_id);
assert_eq!(ack.pid, std::process::id());
assert!(ack.receipt_absent);
assert_eq!(
ack.commit,
match stage {
RestoreCheckpointStage::PreparedNotCommitted =>
RestoreCheckpointCommit::NotCommitted,
RestoreCheckpointStage::CommittedPreterminal
| RestoreCheckpointStage::ActivatedPreterminal => RestoreCheckpointCommit::Committed,
}
);
assert_eq!(
backend.restore_operation_ids.lock().unwrap().as_slice(),
&[ack.operation_id]
);
if stage == RestoreCheckpointStage::ActivatedPreterminal {
assert_eq!(backend.activation_operation_ids.lock().unwrap().as_slice(), &[ack.operation_id]);
} else {
assert!(backend.activation_operation_ids.lock().unwrap().is_empty());
}
assert!(received.try_recv().is_err());
let canonical = runtime_store(&runtime)
.read_durable_worker_record(&worker.worker_ref)
.unwrap();
let id = canonical.restore_guard.active_request_id.as_ref().unwrap();
assert_eq!(id, &request.request_id);
assert!(canonical.restore_guard.owners[id].receipt.is_none());
// Release, EOF and timeout all resume the original lifecycle owner.
let start = std::time::Instant::now();
if release == "release" {
reader.get_mut().write_all(b"R").unwrap();
} else if release == "eof" {
reader
.get_mut()
.shutdown(std::net::Shutdown::Write)
.unwrap();
}
let result = received
.recv_timeout(Duration::from_secs(12))
.unwrap()
.unwrap();
assert!(start.elapsed() < Duration::from_secs(11));
task.join().unwrap();
assert_eq!(result.state, WorkerRestoreState::Accepted);
assert_eq!(
backend.activation_operation_ids.lock().unwrap().as_slice(),
&[ack.operation_id]
);
assert_eq!(
runtime
.restore_worker_operation(&worker.worker_ref, request)
.unwrap(),
result
);
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
assert!(runtime.lock().unwrap().restore_checkpoint.is_none());
runtime.stop_worker(&worker.worker_ref, None).unwrap();
let next = runtime.test_restore_request(&worker.worker_ref);
assert_eq!(
runtime
.restore_worker_operation(&worker.worker_ref, next)
.unwrap()
.state,
WorkerRestoreState::Accepted
);
assert_eq!(*backend.restore_count.lock().unwrap(), 2);
}
}
#[test]
fn checkpoint_disabled_mismatched_or_unreachable_does_not_change_restore_result() {
for mode in ["disabled", "mismatch", "unreachable"] {
let dir = tempfile::tempdir().unwrap();
let backend = Arc::new(TestExecutionBackend::default());
let runtime = Runtime::with_fs_store_and_execution_backend(
FsRuntimeStoreOptions::new(dir.path().join("runtime"))
.with_runtime_id("checkpoint-runtime"),
backend.clone(),
)
.unwrap();
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let worker = runtime
.create_worker(task_request("checkpoint absent"))
.unwrap();
runtime.stop_worker(&worker.worker_ref, None).unwrap();
let request = runtime.test_restore_request(&worker.worker_ref);
if mode != "disabled" {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let endpoint = listener.local_addr().unwrap();
drop(listener);
runtime
.install_test_restore_checkpoint(RestoreCheckpointConfig {
endpoint,
worker_id: if mode == "mismatch" {
WorkerId::now_v7()
} else {
worker.worker_id
},
stage: RestoreCheckpointStage::PreparedNotCommitted,
nonce: "absent-test".into(),
})
.unwrap();
}
let result = runtime
.restore_worker_operation(&worker.worker_ref, request.clone())
.unwrap();
assert_eq!(result.state, WorkerRestoreState::Accepted, "{mode}");
assert_eq!(
runtime
.restore_worker_operation(&worker.worker_ref, request)
.unwrap(),
result
);
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
assert_eq!(
runtime.lock().unwrap().restore_checkpoint.is_some(),
mode == "mismatch"
);
}
}
#[test]
fn checkpoint_verification_requires_canonical_worker_operation_commit_and_absent_receipt() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("runtime");
let backend = Arc::new(TestExecutionBackend::default());
let runtime = Runtime::with_fs_store_and_execution_backend(
FsRuntimeStoreOptions::new(&root).with_runtime_id("checkpoint-runtime"),
backend.clone(),
)
.unwrap();
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let worker = runtime
.create_worker(task_request("checkpoint validation"))
.unwrap();
runtime.stop_worker(&worker.worker_ref, None).unwrap();
let request = runtime.test_restore_request(&worker.worker_ref);
*backend.restore_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::Errored(
WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, "not yet connected"),
));
assert_eq!(
runtime
.restore_worker_operation(&worker.worker_ref, request.clone())
.unwrap()
.state,
WorkerRestoreState::ReconciliationRequired
);
let operation = backend.restore_operation_ids.lock().unwrap()[0];
let mut config = RestoreCheckpointConfig {
endpoint: "127.0.0.1:1234".parse().unwrap(),
worker_id: worker.worker_id,
stage: RestoreCheckpointStage::PreparedNotCommitted,
nonce: "verification-test".into(),
};
let path = root
.join("workers")
.join(worker.worker_id.to_string())
.join("worker.json");
let before = std::fs::read(&path).unwrap();
assert!(
runtime
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
.is_ok()
);
assert!(
runtime
.restore_checkpoint_ack(
&config,
&worker.worker_ref,
WorkerLifecycleOperationId::new()
)
.is_err()
);
config.worker_id = WorkerId::now_v7();
assert!(
runtime
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
.is_err()
);
config.worker_id = worker.worker_id;
config.stage = RestoreCheckpointStage::CommittedPreterminal;
assert!(
runtime
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
.is_err()
);
config.stage = RestoreCheckpointStage::ActivatedPreterminal;
assert!(runtime.restore_checkpoint_ack(&config, &worker.worker_ref, operation).is_err());
config.stage = RestoreCheckpointStage::PreparedNotCommitted;
runtime_store(&runtime).fail_next_worker_reconcile_sync();
assert!(
runtime
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
.is_err()
);
assert_eq!(
std::fs::read(&path).unwrap(),
before,
"verification must not write authority"
);
// In-memory equality must not hide an unreadable canonical aggregate.
std::fs::write(&path, b"unreadable canonical aggregate").unwrap();
assert!(
runtime
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
.is_err()
);
assert_eq!(
std::fs::read(&path).unwrap(),
b"unreadable canonical aggregate"
);
std::fs::write(&path, before).unwrap();
backend.restore_result.lock().unwrap().take();
assert_eq!(
runtime
.restore_worker_operation(&worker.worker_ref, request)
.unwrap()
.state,
WorkerRestoreState::Accepted
);
assert!(
runtime
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
.is_err(),
"terminal operation cannot ack a preterminal checkpoint"
);
}
@@ -0,0 +1,522 @@
//! Runtime-owned continuation of an admitted restore, independent of its caller.
//!
//! Completion callbacks only signal a condition variable. They never acquire a
//! Runtime/lifecycle lock and are never joined by this driver while holding one.
//! Each step revalidates the canonical owner under the existing Worker lock;
//! waiting/backoff and driver teardown happen outside it. A repeated obstruction
//! parks this owner with its reason, rather than spinning or inventing rollback.
use super::*;
use crate::execution::RestoreRecoveryDisposition;
use std::collections::HashMap;
use std::sync::{Condvar, Weak};
use std::thread::JoinHandle;
use std::time::Duration;
type Key = (WorkerRef, WorkerLifecycleOperationId);
#[derive(Debug, Default)]
pub(super) struct RestoreContinuations {
owners: Mutex<HashMap<Key, Arc<Continuation>>>,
#[cfg(test)]
paused: std::sync::atomic::AtomicBool,
#[cfg(test)]
pub(super) terminal_receipt_gate:
Mutex<Option<crate::execution::RestoreTestGate<WorkerLifecycleOperationId>>>,
}
#[derive(Debug, Default)]
struct Schedule {
signaled: bool,
cancelled: bool,
finished: bool,
blocked_reason: Option<String>,
parked: bool,
}
#[derive(Debug, Default)]
struct Continuation {
schedule: Mutex<Schedule>,
wake: Condvar,
task: Mutex<Option<JoinHandle<()>>>,
}
impl RestoreContinuations {
pub(super) fn signal(&self, worker_ref: &WorkerRef, operation: WorkerLifecycleOperationId) {
if let Some(owner) = self
.owners
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&(worker_ref.clone(), operation))
{
let mut schedule = owner
.schedule
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
schedule.signaled = true;
owner.wake.notify_all();
}
}
}
impl Drop for RestoreContinuations {
fn drop(&mut self) {
let owners = self
.owners
.get_mut()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for owner in owners.values() {
let mut schedule = owner
.schedule
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
schedule.cancelled = true;
owner.wake.notify_all();
}
for owner in owners.values() {
if let Some(task) = owner
.task
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
{
if task.thread().id() != std::thread::current().id() {
let _ = task.join();
}
}
}
}
}
#[cfg(test)]
pub(crate) struct ContinuationPause(Arc<RestoreContinuations>);
#[cfg(test)]
impl Drop for ContinuationPause {
fn drop(&mut self) {
self.0
.paused
.store(false, std::sync::atomic::Ordering::Release);
for owner in self.0.owners.lock().unwrap().values() {
owner.schedule.lock().unwrap().signaled = true;
owner.wake.notify_all();
}
}
}
enum Step {
Finished,
Waiting,
Contended,
Retry(String),
}
impl Runtime {
pub(super) fn restore_repair_authorizer(
&self,
worker_ref: WorkerRef,
) -> Arc<dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync> {
let inner = Arc::downgrade(&self.inner);
Arc::new(move |operation| {
let inner = inner.upgrade().ok_or(RuntimeError::RuntimeStopped)?;
// The factory caller owns the lifecycle lock. Never reacquire it
// here: it may be awaiting this exact factory task.
let mut state = inner.lock().map_err(|_| RuntimeError::StatePoisoned)?;
state.ensure_running()?;
state.refresh_uncertain_worker_write(&worker_ref)?;
let worker = state.worker(&worker_ref)?;
state.ensure_subject_not_reserved(&worker_ref, false)?;
if worker.pending_stop.is_some()
|| worker.pending_restore.is_none_or(|pending| {
pending.operation_id != operation
|| pending.commit == RestoreCommitFact::LegacyUnknown
})
{
return Err(RuntimeError::RestoreObservationConflict {
worker_id: worker_ref.worker_id,
});
}
if let Some(owner) = worker
.restore_guard
.active_request_id
.as_ref()
.and_then(|id| worker.restore_guard.owners.get(id))
{
owner.validate_workdir_preparation_authority(worker)?;
}
if worker
.pending_restore
.is_some_and(|pending| pending.source_repair_possible)
{
return Ok(());
}
let mut candidate = worker.clone();
candidate
.pending_restore
.as_mut()
.expect("validated restore owner")
.source_repair_possible = true;
// Grant before touching another canonical store. Unknown durability
// denies repair until authoritative reconciliation; no old-memory write.
state.persist_restore_worker_record(&candidate)?;
state.workers.insert(worker_ref.worker_id, candidate);
Ok(())
})
}
#[cfg(test)]
pub(crate) fn test_wait_restore_continuation_parked(
&self,
worker_ref: &WorkerRef,
operation: WorkerLifecycleOperationId,
timeout: Duration,
) -> String {
let owner = self
.restore_continuations
.owners
.lock()
.unwrap()
.get(&(worker_ref.clone(), operation))
.cloned()
.unwrap();
let (schedule, waited) = owner
.wake
.wait_timeout_while(owner.schedule.lock().unwrap(), timeout, |s| {
!s.parked && !s.finished
})
.unwrap();
assert!(
!waited.timed_out() && schedule.parked,
"continuation did not park: {:?}",
schedule.blocked_reason
);
schedule.blocked_reason.clone().expect("parked reason")
}
/// Gate autonomous steps when a component test observes a single fault
/// boundary. Dedicated continuation tests run without this gate.
#[cfg(test)]
pub(crate) fn test_pause_restore_continuation(&self) -> ContinuationPause {
assert!(
!self
.restore_continuations
.paused
.swap(true, std::sync::atomic::Ordering::AcqRel)
);
ContinuationPause(self.restore_continuations.clone())
}
#[cfg(test)]
pub(crate) fn test_wait_restore_continuation(
&self,
worker_ref: &WorkerRef,
operation: WorkerLifecycleOperationId,
timeout: Duration,
) {
let owner = self
.restore_continuations
.owners
.lock()
.unwrap()
.get(&(worker_ref.clone(), operation))
.cloned()
.expect("continuation owner");
let schedule = owner.schedule.lock().unwrap();
let (schedule, waited) = owner
.wake
.wait_timeout_while(schedule, timeout, |s| !s.finished && !s.cancelled)
.unwrap();
assert!(
!waited.timed_out() && schedule.finished,
"restore continuation did not settle: {:?}",
schedule.blocked_reason
);
}
pub(super) fn ensure_restore_continuation(
&self,
worker_ref: &WorkerRef,
) -> Result<(), RuntimeError> {
let operation = {
let state = self.lock()?;
let worker = state.worker(worker_ref)?;
worker
.pending_restore
.map(|pending| pending.operation_id)
.or_else(|| {
worker
.restore_guard
.active_request_id
.as_ref()
.and_then(|id| worker.restore_guard.owners.get(id))
.filter(|owner| owner.receipt.is_none())
.map(|owner| owner.operation_id)
})
};
if let Some(operation) = operation {
self.start_restore_continuation(worker_ref.clone(), operation)?;
}
Ok(())
}
pub(super) fn start_restore_continuation(
&self,
worker_ref: WorkerRef,
operation: WorkerLifecycleOperationId,
) -> Result<(), RuntimeError> {
{
let state = self.lock()?;
let Some(backend) = &state.execution_backend else {
return Ok(());
};
if !backend.supports_restore_continuation()
|| backend
.restore_recovery_disposition(operation, &worker_ref)
.is_none()
{
return Ok(());
}
}
let key = (worker_ref.clone(), operation);
let mut owners = self
.restore_continuations
.owners
.lock()
.map_err(|_| RuntimeError::StatePoisoned)?;
if owners.contains_key(&key) {
return Ok(());
}
let owner = Arc::new(Continuation::default());
let retained = owner.clone();
let state = Arc::downgrade(&self.inner);
let operations = Arc::downgrade(&self.worker_operations);
let continuations = Arc::downgrade(&self.restore_continuations);
let task = std::thread::Builder::new()
.name("restore-continuation".into())
.spawn(move || {
run(
state,
operations,
continuations,
retained,
worker_ref,
operation,
);
})
.map_err(|error| {
RuntimeError::InvalidRequest(format!(
"Restore continuation could not start: {error}"
))
})?;
*owner.task.lock().map_err(|_| RuntimeError::StatePoisoned)? = Some(task);
owners.insert(key, owner);
Ok(())
}
fn continue_restore_operation(
&self,
worker_ref: &WorkerRef,
operation: WorkerLifecycleOperationId,
) -> Result<Step, RuntimeError> {
#[cfg(test)]
if self
.restore_continuations
.paused
.load(std::sync::atomic::Ordering::Acquire)
{
return Ok(Step::Contended);
}
let lease = self.worker_operation_lock(worker_ref.worker_id)?;
// Another caller may be waiting for a backend task. Do not queue a join
// of that caller/driver while retaining the lock it needs to complete.
let _guard = match lease.try_lock() {
Ok(guard) => guard,
Err(std::sync::TryLockError::WouldBlock) => return Ok(Step::Contended),
Err(_) => return Err(RuntimeError::StatePoisoned),
};
let stop_is_owned = {
let mut state = self.lock()?;
state.ensure_running()?;
state.refresh_uncertain_worker_write(worker_ref)?;
let worker = state.worker(worker_ref)?;
worker.pending_stop.is_some()
&& worker
.pending_restore
.is_some_and(|pending| pending.operation_id == operation)
};
if stop_is_owned {
// Stop supersedes publication, not ownership of late resources.
// Retain this same continuation until its cleanup is confirmed.
self.reconcile_pending_stop_under_lock(worker_ref)?;
return Ok(Step::Finished);
}
let (backend, pending) = {
let mut state = self.lock()?;
state.ensure_running()?;
state.refresh_uncertain_worker_write(worker_ref)?;
let worker = state.worker(worker_ref)?;
if worker.pending_stop.is_some() {
return Ok(Step::Finished);
}
let owner = worker
.restore_guard
.active_request_id
.as_ref()
.and_then(|id| worker.restore_guard.owners.get(id));
if owner.is_some_and(|owner| owner.operation_id != operation || owner.receipt.is_some())
{
return Ok(Step::Finished);
}
let Some(pending) = worker.pending_restore else {
return Ok(Step::Finished);
};
if pending.operation_id != operation {
return Ok(Step::Finished);
}
if pending.commit == RestoreCommitFact::LegacyUnknown {
return Ok(Step::Retry(
"Saved Restore has no authoritative commit evidence".into(),
));
}
state.ensure_subject_not_reserved(worker_ref, false)?;
if let Some(owner) = owner {
owner.validate_workdir_preparation_authority(worker)?;
}
let Some(backend) = state.execution_backend.clone() else {
return Ok(Step::Finished);
};
(backend, pending)
};
match backend.restore_recovery_disposition(operation, worker_ref) {
None => return Ok(Step::Finished),
Some(RestoreRecoveryDisposition::InFlight) => return Ok(Step::Waiting),
Some(RestoreRecoveryDisposition::Discard)
if pending.commit == RestoreCommitFact::NotCommitted =>
{
match backend.discard_uncommitted_restore(operation, worker_ref) {
WorkerExecutionSpawnResult::RolledBack(result) => {
let receipt = RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::RolledBack,
"worker_restore_rolled_back",
result.message_or_default(),
);
#[cfg(feature = "fs-store")]
self.lock()?
.record_restore_failure(worker_ref, &result, &receipt)?;
#[cfg(not(feature = "fs-store"))]
self.settle_restore_rollback(worker_ref, &receipt)?;
return Ok(Step::Finished);
}
result => {
return Ok(Step::Retry(format!(
"Restore cleanup remains unresolved: {result:?}"
)));
}
}
}
Some(_) => {}
}
let result = self.restore_admitted_worker_under_lock(worker_ref, pending.mode)?;
if backend.restore_recovery_disposition(operation, worker_ref)
== Some(RestoreRecoveryDisposition::InFlight)
{
return Ok(Step::Waiting);
}
if result.state == WorkerRestoreState::ReconciliationRequired {
Ok(Step::Retry(result.message.unwrap_or_else(|| {
"Restore reconciliation remains pending".into()
})))
} else {
Ok(Step::Finished)
}
}
}
fn run(
inner: Weak<Mutex<RuntimeState>>,
operations: Weak<Mutex<BTreeMap<WorkerId, Arc<Mutex<()>>>>>,
continuations: Weak<RestoreContinuations>,
owner: Arc<Continuation>,
worker_ref: WorkerRef,
operation: WorkerLifecycleOperationId,
) {
let mut repeated = 0u32;
let mut last_reason = None;
loop {
let (Some(inner), Some(worker_operations), Some(restore_continuations)) = (
inner.upgrade(),
operations.upgrade(),
continuations.upgrade(),
) else {
break;
};
owner
.schedule
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.parked = false;
let runtime = Runtime {
inner,
worker_operations,
restore_continuations,
};
let step = runtime.continue_restore_operation(&worker_ref, operation);
// No lifecycle lock or strong Runtime/registry reference survives waiting.
drop(runtime);
let mut schedule = owner
.schedule
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if schedule.cancelled {
break;
}
let contended = matches!(&step, Ok(Step::Contended));
let reason = match step {
Ok(Step::Finished) => {
schedule.finished = true;
owner.wake.notify_all();
break;
}
Ok(Step::Waiting) | Ok(Step::Contended) => None,
Ok(Step::Retry(reason)) => Some(reason),
Err(error) => Some(sanitize_worker_create_failure_message(&error.to_string())),
};
if let Some(reason) = &reason {
repeated = repeated.saturating_add(1);
last_reason = Some(reason.clone());
schedule.blocked_reason = Some(reason.clone());
}
// Completed failed attempts can have signaled while this step joined
// them. Do not let our own unchanged failure reset the retry budget.
if reason.is_some() {
schedule.signaled = false;
}
if (!contended && reason.is_none()) || repeated >= 4 {
if repeated >= 4 {
schedule.parked = true;
owner.wake.notify_all();
tracing::warn!(worker_id = %worker_ref.worker_id, operation_id = %operation, reason = ?last_reason,
"Restore continuation parked after bounded attempts; operation remains unresolved");
}
schedule = owner
.wake
.wait_while(schedule, |s| !s.signaled && !s.cancelled)
.unwrap_or_else(std::sync::PoisonError::into_inner);
} else {
let delay = Duration::from_millis(100 * (1u64 << repeated.min(4)));
schedule = owner
.wake
.wait_timeout_while(schedule, delay, |s| !s.signaled && !s.cancelled)
.unwrap_or_else(std::sync::PoisonError::into_inner)
.0;
}
if schedule.cancelled {
break;
}
if schedule.signaled {
schedule.signaled = false;
// A completion wakes this same owner, not a new request. Its next
// observed result, not the signal itself, determines progress.
}
}
}
+872
View File
@@ -0,0 +1,872 @@
//! Offline support for the approved, attachment-free stopped Worker fixture.
//!
//! Enabled only by `test-support`. This is not an import API: extraction accepts a
//! current-schema, settled, single-Segment aggregate; installation accepts only a
//! fresh Runtime directory containing its newly initialized `auth.toml`. No
//! Runtime or execution backend is constructed. Original Session bytes are never
//! rewritten. The caller must separately review/attest the artificial conversation;
//! structural checks and secret-marker checks cannot prove arbitrary prose safe.
use super::*;
use crate::catalog::ProfileSourceArchiveSource;
use crate::workspace_issuer::{
WorkspaceIssuerTrustRecord, WorkspaceIssuerTrustState, validate_workspace_issuer_trust_records,
};
use session_store::{
SegmentId, SessionId, WorkerActiveSegmentRef, WorkerMetadata, WorkerSessionStore,
};
#[cfg(test)]
#[path = "test_support_tests.rs"]
mod tests;
const TEMPLATE_VERSION: u32 = 1;
const MAX_SESSION_BYTES: usize = 8 * 1024 * 1024;
/// Portable fixture, intentionally without Debug (Session material is private).
/// Fields are private and every deserialized template is validated on install.
/// Serialize this value to retain the fixture; do not serialize its source aggregate.
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StoppedWorkerTemplate {
template_version: u32,
runtime_schema_version: u32,
runtime_id: String,
workspace_id: String,
worker_id: WorkerId,
created_at_ms: Option<u64>,
session_id: SessionId,
segment_id: SegmentId,
session_manifest: Vec<u8>,
segment_jsonl: Vec<u8>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct SessionManifest {
schema_version: u32,
session_id: SessionId,
}
// Read-only projection: never retain source key material in a template.
#[derive(Deserialize)]
struct AuthIdentity {
identity_id: String,
}
#[derive(Deserialize)]
struct SourceAuth {
identity: AuthIdentity,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct InitializedAuth {
identity: crate::auth::RuntimeIdentityMaterial,
workspace_issuers: Vec<WorkspaceIssuerTrustRecord>,
#[serde(default)]
archived_legacy_issuers: Vec<String>,
}
#[derive(Deserialize)]
struct GuardState {
active_request_id: Option<String>,
owners: BTreeMap<String, GuardOwner>,
}
#[derive(Deserialize)]
struct GuardOwner {
receipt: Option<serde::de::IgnoredAny>,
}
fn reject(message: &'static str) -> RuntimeError {
RuntimeError::InvalidRequest(format!("stopped Worker fixture: {message}"))
}
fn io<T>(result: std::io::Result<T>) -> Result<T, RuntimeError> {
result.map_err(|_| reject("filesystem operation failed"))
}
fn decode<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> Result<T, RuntimeError> {
serde_json::from_slice(bytes).map_err(|_| reject("invalid typed document"))
}
fn no_symlink_components(path: &Path) -> Result<(), RuntimeError> {
let mut current = PathBuf::new();
for part in path.components() {
if matches!(part, std::path::Component::ParentDir) {
return Err(reject("parent traversal is unsupported"));
}
current.push(part);
let metadata = io(fs::symlink_metadata(&current))?;
if metadata.file_type().is_symlink() {
return Err(reject("symlinks are unsupported"));
}
}
Ok(())
}
fn check_tree(path: &Path, budget: &mut usize) -> Result<(), RuntimeError> {
if *budget == 0 {
return Err(reject("file count exceeds fixture limit"));
}
*budget -= 1;
let metadata = io(fs::symlink_metadata(path))?;
if metadata.is_dir() {
for entry in io(fs::read_dir(path))? {
check_tree(&io(entry)?.path(), budget)?;
}
} else if !metadata.is_file() {
return Err(reject("symlinks and special files are unsupported"));
}
Ok(())
}
fn regular_file(path: &Path) -> Result<(), RuntimeError> {
no_symlink_components(path)?;
if !io(fs::symlink_metadata(path))?.is_file() {
return Err(reject("expected a regular file"));
}
Ok(())
}
fn bytes(path: &Path) -> Result<Vec<u8>, RuntimeError> {
regular_file(path)?;
let mut options = OpenOptions::new();
options.read(true);
#[cfg(target_os = "linux")]
{
use std::os::unix::fs::OpenOptionsExt;
options.custom_flags(libc::O_NOATIME | libc::O_NOFOLLOW);
}
let file = io(options.open(path))?;
let mut result = Vec::new();
io(file
.take(MAX_SESSION_BYTES as u64 + 1)
.read_to_end(&mut result))?;
if result.len() > MAX_SESSION_BYTES {
return Err(reject("file exceeds fixture limit"));
}
Ok(result)
}
fn text(path: &Path) -> Result<String, RuntimeError> {
String::from_utf8(bytes(path)?).map_err(|_| reject("invalid UTF-8 document"))
}
fn portable_id(id: &str) -> bool {
!id.is_empty()
&& id.len() <= 256
&& id
.bytes()
.all(|c| c.is_ascii_alphanumeric() || b"-_".contains(&c))
}
impl StoppedWorkerTemplate {
pub fn runtime_id(&self) -> &str {
&self.runtime_id
}
pub fn workspace_id(&self) -> &str {
&self.workspace_id
}
pub fn worker_id(&self) -> WorkerId {
self.worker_id
}
pub fn worker_name(&self) -> String {
runtime_worker_name(self.worker_id)
}
pub fn session_id(&self) -> SessionId {
self.session_id
}
pub fn segment_id(&self) -> SegmentId {
self.segment_id
}
/// Extract without migration, rewriting metadata, or copying credentials,
/// resolved manifests, execution-operation receipts, or run directories.
/// The source Runtime must be offline; its existing owner lock is held for
/// the entire read. The path must be `<runtime>/workers/<WorkerId>`.
pub fn extract(worker_dir: &Path) -> Result<Self, RuntimeError> {
no_symlink_components(worker_dir)?;
let workers = worker_dir
.parent()
.ok_or_else(|| reject("invalid aggregate path"))?;
if workers.file_name().and_then(|s| s.to_str()) != Some(WORKERS_DIR) {
return Err(reject("invalid aggregate path"));
}
let root = workers
.parent()
.ok_or_else(|| reject("invalid Runtime path"))?;
// Require the existing lock, rather than creating a file in the source.
let lock_path = root.with_file_name(format!(
".{}.runtime-owner.lock",
root.file_name()
.and_then(|s| s.to_str())
.ok_or_else(|| reject("invalid Runtime path"))?
));
regular_file(&lock_path)?;
let lock = io(OpenOptions::new().read(true).open(lock_path))?;
if !io(FileExt::try_lock_shared(&lock))? {
return Err(reject("source Runtime is not offline"));
}
let _lock = RuntimeStoreOwnerLock { file: lock };
check_tree(worker_dir, &mut 256)?;
let snapshot: RuntimeSnapshot = decode(&bytes(&root.join(RUNTIME_FILE))?)?;
snapshot
.validate(&root.join(RUNTIME_FILE))
.map_err(|_| reject("unsupported Runtime schema"))?;
if !snapshot.subject_create_recovery_fences.is_empty()
|| !snapshot.subject_releases.is_empty()
{
return Err(reject("source has lifecycle operations"));
}
let record: WorkerAggregateRecord = decode(&bytes(&worker_dir.join(WORKER_FILE))?)?;
let record = record
.validate(&worker_dir.join(WORKER_FILE))
.map_err(|_| reject("invalid Worker schema or identity"))?;
let guard: GuardState = decode(
&serde_json::to_vec(&record.restore_guard)
.map_err(|_| reject("invalid restore guard"))?,
)?;
if record.status != WorkerStatus::Stopped
|| guard.active_request_id.is_some()
|| guard.owners.values().any(|owner| owner.receipt.is_none())
{
return Err(reject("Worker is not settled and stopped"));
}
let PersistedWorkerExecutionState::Available(execution) = &record.execution_state else {
return Err(reject("pending or unavailable execution metadata"));
};
// A stopped aggregate may retain the empty historical binding marker.
// It is deliberately not copied to the installed Worker.
if execution.restore_intent != WorkerRestoreIntent::Explicit {
return Err(reject("Worker requires explicit restore"));
}
if !record.workdir_attachments.is_empty() || !record.logical_workdir_attachments.is_empty()
{
return Err(reject("attachments are unsupported"));
}
check_request(&execution.request, false)?;
let workspace_id = record
.workspace_id
.ok_or_else(|| reject("missing Workspace identity"))?;
let metadata: WorkerMetadata = decode(&bytes(&worker_dir.join(WORKER_METADATA_FILE))?)?;
crate::worker_backend::validate_fixture_restore_metadata(&metadata, &execution.request)
.map_err(|_| reject("invalid source restore Manifest or settings"))?;
if metadata.worker_name != runtime_worker_name(record.worker_id)
|| metadata.workspace_id.as_deref() != Some(&workspace_id)
|| !metadata.spawned_children.is_empty()
|| !metadata.reclaimed_children.is_empty()
|| !metadata.peers.is_empty()
|| metadata.subjektiv_session_attribution.is_some()
{
return Err(reject("unsupported or inconsistent Worker metadata"));
}
let active = metadata
.active
.ok_or_else(|| reject("missing active Session"))?;
let segment_id = active
.segment_id
.ok_or_else(|| reject("unmaterialized Session"))?;
let source_auth: SourceAuth = toml::from_str(&text(&root.join("auth.toml"))?)
.map_err(|_| reject("missing Runtime identity"))?;
let session = worker_dir.join("session");
let fixture = Self {
template_version: TEMPLATE_VERSION,
runtime_schema_version: SCHEMA_VERSION,
runtime_id: source_auth.identity.identity_id,
workspace_id,
worker_id: record.worker_id,
created_at_ms: record.created_at_ms,
session_id: active.session_id,
segment_id,
session_manifest: bytes(&session.join("session.json"))?,
segment_jsonl: bytes(&session.join("segments").join(format!("{segment_id}.jsonl")))?,
};
fixture.validate_shape()?;
fixture.validate_session(&session)?;
Ok(fixture)
}
/// Install after normal identity/trust initialization, before Runtime start.
/// `request` and `manifest` supply NEW backend/config/profile references and
/// freshly resolved policy, not a launch. The saved source Manifest is never
/// reused. `manifest.worker.name` must equal `worker_name()`.
/// Only `auth.toml` may already exist. The owner lock fences live Runtimes;
/// staged files are synced and published without overwriting any Worker.
/// An interrupted publication fails closed as an incomplete Runtime layout.
pub fn install(
&self,
options: &FsRuntimeStoreOptions,
request: CreateWorkerRequest,
manifest: &manifest::WorkerManifest,
) -> Result<(), RuntimeError> {
self.validate_shape()?;
check_request(&request, true)?;
if manifest.worker.name != self.worker_name() {
return Err(reject("resolved Manifest Worker name mismatch"));
}
let mut metadata = WorkerMetadata::new(
self.worker_name(),
Some(WorkerActiveSegmentRef {
session_id: self.session_id,
segment_id: Some(self.segment_id),
}),
)
.with_workspace_id(&self.workspace_id);
metadata.resolved_manifest_snapshot = Some(
manifest::write_persisted_worker_manifest_snapshot(manifest)
.map_err(|_| reject("invalid resolved Manifest"))?,
);
crate::worker_backend::validate_fixture_restore_metadata(&metadata, &request)
.map_err(|_| reject("resolved Manifest does not match fresh request settings"))?;
if options.runtime_id != self.runtime_id
|| request.worker_id != self.worker_id
|| request
.workspace_api
.as_ref()
.map(|api| api.workspace_id.as_str())
!= Some(&self.workspace_id)
{
return Err(reject("installation identity mismatch"));
}
no_symlink_components(&options.root)?;
check_tree(&options.root, &mut 256)?;
// Validate the lock location before the ordinary owner-lock opener.
let lock_path = options.root.with_file_name(format!(
".{}.runtime-owner.lock",
options
.root
.file_name()
.and_then(|s| s.to_str())
.ok_or_else(|| reject("invalid Runtime path"))?
));
if fs::symlink_metadata(&lock_path).is_ok() {
regular_file(&lock_path)?;
}
let lock = acquire_runtime_store_owner_lock(&options.root)?;
validate_fresh_runtime_store_layout(&options.root)
.map_err(|_| reject("destination is not fresh; refusing overwrite"))?;
let auth: InitializedAuth = toml::from_str(&text(&options.root.join("auth.toml"))?)
.map_err(|_| reject("initialize Runtime identity and trust before install"))?;
if auth.identity.identity_id != self.runtime_id
|| !auth.archived_legacy_issuers.is_empty()
|| auth.identity.signing_key().is_err()
{
return Err(reject("invalid destination Runtime identity"));
}
validate_workspace_issuer_trust_records(&auth.workspace_issuers)
.map_err(|_| reject("invalid destination Workspace trust"))?;
let api = request
.workspace_api
.as_ref()
.expect("validated Workspace binding");
if !auth.workspace_issuers.iter().any(|trust| {
trust.workspace_id == self.workspace_id
&& trust.backend_url == api.base_url
&& trust.state == WorkspaceIssuerTrustState::Active
}) {
return Err(reject("request does not match initialized Workspace trust"));
}
let stage = options
.root
.join(format!(".fixture-{}", uuid::Uuid::now_v7()));
io(fs::create_dir(&stage))?;
let store = FsRuntimeStore {
root: stage.clone(),
_owner_lock: Some(lock),
worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)),
#[cfg(test)]
worker_reconcile_sync_fault: Default::default(),
#[cfg(test)]
worker_reconcile_read_fault: Default::default(),
#[cfg(test)]
worker_reconcile_counts: Default::default(),
};
let result = (|| {
let worker = PersistedWorkerRecord {
restore_guard: crate::runtime::RestoreGuard::default(),
worker_ref: WorkerRef::new(self.worker_id),
worker_id: self.worker_id,
profile: request.profile.clone(),
display_name: request.display_name.clone(),
profile_source: request.profile_source.reference(),
config_bundle: request.config_bundle.clone(),
created_at_ms: self.created_at_ms,
status: WorkerStatus::Stopped,
last_finished_submission_request_id: None,
workspace_id: Some(self.workspace_id.clone()),
workdir_attachments: Vec::new(),
logical_workdir_attachments: Vec::new(),
execution_state: PersistedWorkerExecutionState::Available(
PersistedWorkerExecution {
request,
binding: None,
restore_intent: WorkerRestoreIntent::Explicit,
},
),
};
WorkerAggregateRecord::from_persisted(&worker)
.validate(&store.worker_dir(&self.worker_id).join(WORKER_FILE))
.map_err(|_| reject("invalid installation request"))?;
store.write_worker_record(&worker)?;
let worker_dir = store.worker_dir(&self.worker_id);
atomic_write_json(
&worker_dir.join(WORKER_METADATA_FILE),
&metadata,
"write fixture metadata",
)?;
let session = worker_dir.join("session");
io(fs::create_dir_all(session.join("segments")))?;
write_new(&session.join("session.json"), &self.session_manifest)?;
write_new(
&session
.join("segments")
.join(format!("{}.jsonl", self.segment_id)),
&self.segment_jsonl,
)?;
self.validate_session(&session)?;
store.write_runtime_snapshot(&PersistedRuntimeState {
subject_create_recovery_fences: BTreeMap::new(),
subject_releases: Vec::new(),
// Match normal store initialization: this is catalog availability,
// not a process or Worker execution start.
display_name: options.display_name.clone(),
status: RuntimeStatus::Running,
next_diagnostic_id: 1,
workers: BTreeMap::new(),
workspace_owners: BTreeMap::new(),
diagnostics: Vec::new(),
})?;
let loaded = store.load_runtime_state()?;
if loaded.workers.len() != 1 || !loaded.diagnostics.is_empty() {
return Err(reject("staged aggregate failed ordinary store roundtrip"));
}
sync_runtime_tree(&stage)?;
// No destination existed when checked under the owner lock. Hard-link
// the snapshot last with create-new semantics; never replace a file.
// Reserve the destination catalog with create-new semantics. Even
// an empty preexisting workers directory must never be replaced.
let destination_workers = options.root.join(WORKERS_DIR);
io(fs::create_dir(&destination_workers))?;
io(fs::rename(
store.worker_dir(&self.worker_id),
destination_workers.join(self.worker_id.to_string()),
))?;
sync_directory(&destination_workers, "publish stopped Worker aggregate")?;
io(fs::hard_link(
stage.join(RUNTIME_FILE),
options.root.join(RUNTIME_FILE),
))?;
sync_directory(&options.root, "publish stopped fixture")?;
Ok(())
})();
// Only remove our private staging directory, never existing destination data.
let cleanup = fs::remove_dir_all(&stage);
result?;
io(cleanup)?;
sync_directory(&options.root, "finish stopped fixture")
}
fn validate_shape(&self) -> Result<(), RuntimeError> {
// Version 1 templates carry only settled Session material, never a
// lifecycle operation. Schema 12 -> 13 changes pending decision evidence,
// not that payload. Accept the fixed schema-12 artifact read-only and
// install through the current aggregate owner; never rewrite its bytes.
if self.template_version != TEMPLATE_VERSION
|| !matches!(
self.runtime_schema_version,
RESTORE_ADMISSION_SCHEMA_VERSION | SCHEMA_VERSION
)
{
return Err(reject("unsupported template schema"));
}
if !portable_id(&self.runtime_id)
|| !portable_id(&self.workspace_id)
|| self.session_id.get_version_num() != 7
|| self.segment_id.get_version_num() != 7
|| self.session_manifest.len() > MAX_SESSION_BYTES
|| self.segment_jsonl.len() > MAX_SESSION_BYTES
{
return Err(reject("invalid portable identity or size"));
}
let manifest: SessionManifest = decode(&self.session_manifest)?;
if manifest.session_id != self.session_id
|| manifest.schema_version == 0
|| !self.segment_jsonl.ends_with(b"\n")
{
return Err(reject("Session identity or committed boundary mismatch"));
}
// Read-only JSON traversal, never mutation/reconstruction of Session bytes.
// Reject known secret/host markers even in otherwise valid typed records.
for line in self
.segment_jsonl
.split(|b| *b == b'\n')
.filter(|line| !line.is_empty())
{
let _: PortableJson = decode(line)?;
validate_preserved_record(line)?;
}
Ok(())
}
fn validate_history_entry(
&self,
entry: &session_store::LoggedHistoryEntry,
ids: &mut std::collections::BTreeSet<String>,
) -> Result<(), RuntimeError> {
if !portable_id(&entry.metadata.entry_id.0)
|| !ids.insert(entry.metadata.entry_id.0.clone())
{
return Err(reject("duplicate or invalid entry identity"));
}
if entry
.metadata
.input_metadata
.as_ref()
.is_some_and(|metadata| {
metadata.feature_payload().is_some() || metadata.public_item().is_some()
})
{
// These are opaque owner-specific payloads, not part of the narrow
// artificial conversation contract. Never preserve unchecked JSON.
return Err(reject("Feature history payloads are unsupported"));
}
if !matches!(
entry.item,
session_store::LoggedItem::Message { .. }
| session_store::LoggedItem::AssistantResponseBoundary { .. }
) {
return Err(reject("only text conversation records are supported"));
}
if let session_store::LoggedSessionHistoryOrigin::ModelOutput { worker } =
&entry.metadata.origin
{
// Session producer IDs identify the original execution, not the
// Workspace catalog Worker. Preserve them; do not rebind/rewrite them.
if !portable_id(&worker.worker_id)
|| worker
.workspace_id
.as_ref()
.is_some_and(|id| id != &self.workspace_id)
|| worker
.runtime_id
.as_ref()
.is_some_and(|id| id != &self.runtime_id)
{
return Err(reject("Session origin identity mismatch"));
}
}
Ok(())
}
fn validate_session(&self, root: &Path) -> Result<(), RuntimeError> {
for entry in io(fs::read_dir(root))? {
let entry = io(entry)?;
match entry.file_name().to_str() {
Some("session.json" | "segments") => {}
Some("artifacts") => check_empty_artifacts(&entry.path())?,
_ => return Err(reject("unsupported Session files")),
}
}
let entries = io(fs::read_dir(root.join("segments")))?.collect::<Result<Vec<_>, _>>();
let entries = io(entries)?;
if entries.len() != 1
|| entries[0].file_name() != format!("{}.jsonl", self.segment_id).as_str()
{
return Err(reject("only the fixed single-Segment fixture is supported"));
}
let store = WorkerSessionStore::open_read_only(root)
.map_err(|_| reject("unsupported Session schema"))?;
let logs = store
.read_all_read_only_bounded(self.session_id, self.segment_id, MAX_SESSION_BYTES as u64)
.map_err(|_| reject("invalid canonical Session records"))?
.0;
if !matches!(logs.first(), Some(session_store::LogEntry::AnnotatedSegmentStart {
session_id, system_prompt: Some(_), forked_from: None, compacted_from: None, ..
}) if *session_id == self.session_id)
{
return Err(reject(
"missing system prompt, invalid Segment seed, or unsupported lineage",
));
}
let mut ids = std::collections::BTreeSet::new();
for (index, log) in logs.iter().enumerate() {
use session_store::LogEntry;
match log {
LogEntry::AnnotatedSegmentStart { history, .. }
if index == 0 && history.is_empty() => {}
LogEntry::AnnotatedUserInput {
history,
extensions,
..
} => {
for extension in extensions {
validate_settled_activation(extension)?;
}
for entry in history {
self.validate_history_entry(entry, &mut ids)?;
}
}
LogEntry::AnnotatedAssistantItem { entry, .. } => {
self.validate_history_entry(entry, &mut ids)?
}
LogEntry::RunCompleted {
entry_id: Some(id), ..
} => {
if !portable_id(&id.0) || !ids.insert(id.0.clone()) {
return Err(reject("duplicate or invalid entry identity"));
}
}
LogEntry::Invoke { .. } | LogEntry::LlmUsage { .. } | LogEntry::TurnEnd { .. } => {}
_ => {
return Err(reject(
"unsupported conversation record or pending operation",
));
}
}
}
let state = session_store::collect_state(&logs);
if state.last_run_interrupted || state.active_run_turn_count.is_some() {
return Err(reject("Session has pending execution"));
}
for segments in &state.user_segments {
if segments
.iter()
.any(|segment| !matches!(segment, protocol::Segment::Text { .. }))
{
return Err(reject("Session attachments are unsupported"));
}
}
Ok(())
}
}
// Compare *fields*, not re-encoded bytes or scalar values (e.g. f32 JSON
// normalization). Serde's permissive/untagged/flattened compatibility readers
// can drop unknown data without surfacing it to an ignored-field callback. A
// retained field must survive the owner's typed projection at every depth.
// This intentionally accepts only canonical writer shapes: even an explicit
// empty optional field omitted by that writer is rejected, not repaired.
// The projection is validation-only; installation always writes original bytes.
fn validate_preserved_record(bytes: &[u8]) -> Result<(), RuntimeError> {
let original: serde_json::Value = decode(bytes)?;
let record: session_store::LogEntry = decode(bytes)?;
let projection =
serde_json::to_value(&record).map_err(|_| reject("invalid typed Session record"))?;
require_preserved_fields(&original, &projection)
}
fn require_preserved_fields(
original: &serde_json::Value,
projection: &serde_json::Value,
) -> Result<(), RuntimeError> {
use serde_json::Value;
match (original, projection) {
(Value::Object(original), Value::Object(projection)) => {
for (key, value) in original {
let projected = projection
.get(key)
.ok_or_else(|| reject("unknown or noncanonical Session field"))?;
require_preserved_fields(value, projected)?;
}
}
(Value::Array(original), Value::Array(projection))
if original.len() == projection.len() =>
{
for (value, projected) in original.iter().zip(projection) {
require_preserved_fields(value, projected)?;
}
}
(Value::Null, Value::Null)
| (Value::Bool(_), Value::Bool(_))
| (Value::Number(_), Value::Number(_))
| (Value::String(_), Value::String(_)) => {}
_ => return Err(reject("noncanonical Session field shape")),
}
Ok(())
}
// Exact receipt wire fields owned by worker::PendingActivationState. Keep these
// fixture-only mirrors strict; do not change the production compatibility reader.
#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct SubmissionReceipt {
submission_request_id: String,
source_namespace: String,
submission_id: String,
payload_digest: String,
disposition: protocol::SubmissionDisposition,
}
#[derive(Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct NotificationReceipt {
notification_request_id: String,
source_namespace: String,
payload_digest: String,
}
// The approved conversation has committed submission checkpoints in its original
// Session records. Keep their receipt bytes, but reject any resumable queue or
// Feature invocation history. This projection confers no execution authority.
fn validate_settled_activation(
extension: &session_store::SessionExtension,
) -> Result<(), RuntimeError> {
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Checkpoint {
#[serde(rename = "next_activation_sequence")]
_next_activation_sequence: u64,
activating: Option<serde::de::IgnoredAny>,
activating_notification: Vec<serde::de::IgnoredAny>,
pending: Vec<serde::de::IgnoredAny>,
pending_notifications: Vec<serde::de::IgnoredAny>,
#[serde(rename = "receipts")]
_receipts: Vec<SubmissionReceipt>,
#[serde(rename = "notification_receipts")]
_notification_receipts: Vec<NotificationReceipt>,
invocation_receipts: Vec<serde::de::IgnoredAny>,
invocation_recovery_blocked: bool,
}
if extension.domain != "worker.pending_activations.v1" {
return Err(reject("unsupported Session extension"));
}
let checkpoint: Checkpoint = serde_json::from_value(extension.payload.clone())
.map_err(|_| reject("invalid Session activation checkpoint"))?;
if checkpoint.activating.is_some()
|| !checkpoint.activating_notification.is_empty()
|| !checkpoint.pending.is_empty()
|| !checkpoint.pending_notifications.is_empty()
|| !checkpoint.invocation_receipts.is_empty()
|| checkpoint.invocation_recovery_blocked
{
return Err(reject(
"Session has pending activation or invocation history",
));
}
Ok(())
}
fn check_request(request: &CreateWorkerRequest, install: bool) -> Result<(), RuntimeError> {
if !request.workdir_attachment_requests.is_empty()
|| !request.workdir_attachments.is_empty()
|| request.backend_job.is_some()
|| request.subjektiv_attached
|| request.worker_observation_enabled
|| !request.worker_observation_grants.is_empty()
|| (install && request.initial_input.is_some())
{
return Err(reject(
"unsupported capabilities, attachments, or initial input",
));
}
if install
&& (request.create_fingerprint.trim().is_empty()
|| request.config_bundle.is_none()
|| !matches!(
request.profile_source,
ProfileSourceArchiveSource::WorkspaceConfig { .. }
))
{
return Err(reject(
"fresh config/profile references and create fingerprint are required",
));
}
Ok(())
}
fn check_empty_artifacts(root: &Path) -> Result<(), RuntimeError> {
for entry in io(fs::read_dir(root))? {
let entry = io(entry)?;
let metadata = io(entry.metadata())?;
if metadata.is_dir() {
check_empty_artifacts(&entry.path())?;
} else if entry.file_name() != ".aggregate.lock" || metadata.len() != 0 {
return Err(reject("Session artifacts are unsupported"));
}
}
Ok(())
}
fn write_new(path: &Path, bytes: &[u8]) -> Result<(), RuntimeError> {
let mut file = io(OpenOptions::new().write(true).create_new(true).open(path))?;
io(file.write_all(bytes))?;
io(file.sync_all())
}
// A streaming validation visitor, not a generic document available to callers.
struct PortableJson;
impl<'de> Deserialize<'de> for PortableJson {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct Visitor;
impl<'de> serde::de::Visitor<'de> for Visitor {
type Value = PortableJson;
fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("portable fixture JSON")
}
fn visit_bool<E: serde::de::Error>(self, _: bool) -> Result<PortableJson, E> {
Ok(PortableJson)
}
fn visit_i64<E: serde::de::Error>(self, _: i64) -> Result<PortableJson, E> {
Ok(PortableJson)
}
fn visit_u64<E: serde::de::Error>(self, _: u64) -> Result<PortableJson, E> {
Ok(PortableJson)
}
fn visit_f64<E: serde::de::Error>(self, _: f64) -> Result<PortableJson, E> {
Ok(PortableJson)
}
fn visit_unit<E: serde::de::Error>(self) -> Result<PortableJson, E> {
Ok(PortableJson)
}
fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<PortableJson, E> {
let lower = value.to_ascii_lowercase();
if [
"://",
"private key",
"bearer ",
"sk-",
"api_key",
"access_token",
"refresh_token",
"private_key",
"authorization",
]
.iter()
.any(|marker| lower.contains(marker))
|| value
.split_whitespace()
.any(|word| word.starts_with('/') || word.contains(":\\"))
{
return Err(E::custom("nonportable or sensitive fixture material"));
}
Ok(PortableJson)
}
fn visit_seq<A: serde::de::SeqAccess<'de>>(
self,
mut seq: A,
) -> Result<PortableJson, A::Error> {
while seq.next_element::<PortableJson>()?.is_some() {}
Ok(PortableJson)
}
fn visit_map<A: serde::de::MapAccess<'de>>(
self,
mut map: A,
) -> Result<PortableJson, A::Error> {
let mut keys = std::collections::BTreeSet::new();
while let Some(key) = map.next_key::<String>()? {
// A duplicate key can hide discarded data even inside an
// otherwise opaque extension payload. Never retain it.
if !keys.insert(key.clone()) {
return Err(serde::de::Error::custom("duplicate fixture JSON field"));
}
serde::de::Visitor::visit_str::<A::Error>(Visitor, &key)?;
map.next_value::<PortableJson>()?;
}
Ok(PortableJson)
}
}
deserializer.deserialize_any(Visitor)
}
}
@@ -0,0 +1,919 @@
use super::*;
use crate::auth::RuntimeIdentityMaterial;
use crate::catalog::{ProfileSelector, WorkspaceApiRef};
use crate::profile_archive::ProfileSourceGraphSummary;
use session_store::{LogEntry, Store};
use sha2::{Digest, Sha256};
const RUNTIME: &str = "fixture-runtime";
const WORKSPACE: &str = "fixture-workspace";
const NEW_URL: &str = "http://127.0.0.1:34567";
#[derive(Serialize)]
#[serde(untagged)]
enum ReceiptFixture<T> {
Valid(T),
Malformed(u8),
}
#[derive(Default, Serialize)]
struct CheckpointFixture {
next_activation_sequence: u64,
activating: Option<u8>,
activating_notification: Vec<u8>,
pending: Vec<u8>,
pending_notifications: Vec<u8>,
receipts: Vec<ReceiptFixture<SubmissionReceipt>>,
notification_receipts: Vec<ReceiptFixture<NotificationReceipt>>,
invocation_receipts: Vec<u8>,
invocation_recovery_blocked: bool,
}
fn settled_checkpoint() -> CheckpointFixture {
CheckpointFixture {
next_activation_sequence: 2,
receipts: vec![ReceiptFixture::Valid(SubmissionReceipt {
submission_request_id: "completed-submission-request".into(),
source_namespace: "fixture-source".into(),
submission_id: "completed-submission".into(),
payload_digest: "fixture-payload-digest".into(),
disposition: protocol::SubmissionDisposition::Started,
})],
notification_receipts: vec![ReceiptFixture::Valid(NotificationReceipt {
notification_request_id: "completed-notification-request".into(),
source_namespace: "fixture-source".into(),
payload_digest: "fixture-notification-digest".into(),
})],
..Default::default()
}
}
fn checkpoint_extension(checkpoint: &CheckpointFixture) -> session_store::SessionExtension {
session_store::SessionExtension::new(
"worker.pending_activations.v1",
serde_json::to_value(checkpoint).unwrap(),
)
}
fn worker_id() -> WorkerId {
"01a1657d-b762-7791-87ae-a49f28cadbce".parse().unwrap()
}
fn resolved() -> manifest::WorkerManifest {
let mut config = manifest::WorkerManifestConfig::resolution_defaults();
config.worker.name = Some(runtime_worker_name(worker_id()));
manifest::WorkerManifest::try_from(config).unwrap()
}
fn request() -> CreateWorkerRequest {
CreateWorkerRequest {
worker_id: worker_id(),
create_fingerprint: "new-create-fingerprint".into(),
profile: ProfileSelector::Builtin("default".into()),
display_name: Some("fixture".into()),
profile_source: ProfileSourceArchiveSource::WorkspaceConfig {
archive: ProfileSourceArchiveRef {
id: "new-profile".into(),
digest: "new-profile-digest".into(),
size_bytes: 1,
source_graph: ProfileSourceGraphSummary {
source_count: 1,
total_source_bytes: 1,
entrypoints: BTreeMap::from([("builtin:default".into(), "default.dcl".into())]),
import_count: 0,
},
},
},
config_bundle: Some(ConfigBundleRef {
id: "new-config".into(),
digest: "new-config-digest".into(),
}),
workspace_api: Some(WorkspaceApiRef {
workspace_id: WORKSPACE.into(),
base_url: NEW_URL.into(),
}),
initial_input: None,
workdir_attachment_requests: Vec::new(),
workdir_attachments: Vec::new(),
worker_observation_enabled: false,
worker_observation_grants: Vec::new(),
memory_settings: Some(manifest::WorkspaceMemorySettingsSnapshot {
workspace_id: WORKSPACE.into(),
language: "en".into(),
}),
subjektiv_attached: false,
backend_job: None,
}
}
fn initialize(root: &Path, runtime_id: &str, workspace_id: &str, url: &str) {
#[derive(Serialize)]
struct Auth {
identity: RuntimeIdentityMaterial,
workspace_issuers: Vec<WorkspaceIssuerTrustRecord>,
}
let issuer = RuntimeIdentityMaterial::generate("fixture-issuer").unwrap();
let digest = Sha256::digest(crate::auth::decode_public_key(&issuer.public_key).unwrap());
let fingerprint = format!(
"sha256:{}",
digest
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>()
);
let trust = WorkspaceIssuerTrustRecord::from_bundle(
server_api::WorkspacePublicIdentityBundle {
workspace_id: workspace_id.into(),
backend_url: url.into(),
key_id: "fixture-issuer".into(),
algorithm: "ed25519".into(),
public_key: issuer.public_key,
public_key_fingerprint: fingerprint,
},
"fixture-trust".into(),
100,
)
.unwrap();
fs::create_dir_all(root).unwrap();
fs::write(
root.join("auth.toml"),
toml::to_string(&Auth {
identity: RuntimeIdentityMaterial::generate(runtime_id).unwrap(),
workspace_issuers: vec![trust],
})
.unwrap(),
)
.unwrap();
}
fn record(request: CreateWorkerRequest) -> PersistedWorkerRecord {
PersistedWorkerRecord {
restore_guard: crate::runtime::RestoreGuard::default(),
worker_ref: WorkerRef::new(request.worker_id),
worker_id: request.worker_id,
profile: request.profile.clone(),
display_name: request.display_name.clone(),
profile_source: request.profile_source.reference(),
config_bundle: request.config_bundle.clone(),
created_at_ms: Some(123456),
status: WorkerStatus::Stopped,
last_finished_submission_request_id: Some("old-submission-operation".into()),
workspace_id: Some(WORKSPACE.into()),
workdir_attachments: vec![],
logical_workdir_attachments: vec![],
execution_state: PersistedWorkerExecutionState::Available(PersistedWorkerExecution {
request,
binding: Some(PersistedWorkerExecutionBinding {}),
restore_intent: WorkerRestoreIntent::Explicit,
}),
}
}
fn source() -> (tempfile::TempDir, PathBuf, StoppedWorkerTemplate) {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
initialize(&root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456");
drop(
crate::Runtime::with_fs_store(FsRuntimeStoreOptions::new(&root).with_runtime_id(RUNTIME))
.unwrap(),
);
let opened = FsRuntimeStore::open_or_create(root.clone(), RUNTIME).unwrap();
let mut old = request();
old.workspace_api.as_mut().unwrap().base_url = "http://127.0.0.1:23456".into();
old.config_bundle.as_mut().unwrap().id = "old-config".into();
opened.store.write_worker_record(&record(old)).unwrap();
let worker = opened.store.worker_dir(&worker_id());
let session_id = "01a1657d-b800-7777-8888-111111111111".parse().unwrap();
let segment_id = "01a1657d-b818-77c1-bd73-176172a4bd82".parse().unwrap();
let mut metadata = WorkerMetadata::new(
runtime_worker_name(worker_id()),
Some(WorkerActiveSegmentRef {
session_id,
segment_id: Some(segment_id),
}),
)
.with_workspace_id(WORKSPACE)
.with_workspace_root(PathBuf::from("/old/host/path"));
metadata.resolved_manifest_snapshot =
Some(manifest::write_persisted_worker_manifest_snapshot(&resolved()).unwrap());
atomic_write_json(
&worker.join(WORKER_METADATA_FILE),
&metadata,
"test metadata",
)
.unwrap();
let sessions = WorkerSessionStore::new(worker.join("session")).unwrap();
let history = session_store::LoggedHistoryEntry {
item: session_store::LoggedItem::Message {
role: session_store::LoggedRole::Assistant,
content: vec![session_store::LoggedContentPart::Text {
text: "artificial fixture reply".into(),
}],
},
metadata: session_store::LoggedSessionHistoryMetadata {
entry_id: session_store::LoggedSessionHistoryEntryId("fixed-entry-id".into()),
origin: session_store::LoggedSessionHistoryOrigin::ModelOutput {
worker: session_store::LoggedWorkerSubject {
workspace_id: Some(WORKSPACE.into()),
runtime_id: Some(RUNTIME.into()),
worker_id: worker_id().to_string(),
},
},
derivation: None,
input_metadata: None,
},
};
sessions
.create_segment(
session_id,
segment_id,
&[
LogEntry::AnnotatedSegmentStart {
ts: 123,
session_id,
system_prompt: Some("Harmless artificial fixture instructions.".into()),
config: Default::default(),
history: vec![],
forked_from: None,
compacted_from: None,
},
LogEntry::Invoke {
ts: 124,
trigger: protocol::InvokeKind::UserSend,
},
LogEntry::AnnotatedUserInput {
ts: 124,
segments: vec![protocol::Segment::text(
"Harmless artificial fixture input.",
)],
extensions: vec![checkpoint_extension(&settled_checkpoint())],
history: vec![session_store::LoggedHistoryEntry {
item: session_store::LoggedItem::Message {
role: session_store::LoggedRole::User,
content: vec![session_store::LoggedContentPart::Text {
text: "Harmless artificial fixture input.".into(),
}],
},
metadata: session_store::LoggedSessionHistoryMetadata {
entry_id: session_store::LoggedSessionHistoryEntryId(
"fixed-user-id".into(),
),
origin: session_store::LoggedSessionHistoryOrigin::BackendInstruction {
operation_id: None,
},
derivation: None,
input_metadata: None,
},
}],
},
LogEntry::AnnotatedAssistantItem {
ts: 125,
entry: history,
},
LogEntry::LlmUsage {
ts: 125,
history_len: 2,
input_total_tokens: 1,
cache_read_tokens: 0,
cache_write_tokens: 0,
output_tokens: 1,
},
LogEntry::TurnEnd {
ts: 126,
turn_count: 1,
},
LogEntry::RunCompleted {
entry_id: Some(session_store::LoggedSessionHistoryEntryId(
"fixed-terminal-id".into(),
)),
ts: 126,
interrupted: false,
result: agen::EngineResult::Finished,
active_run_turn_count: None,
},
],
)
.unwrap();
fs::create_dir_all(worker.join("runs/old-run")).unwrap();
fs::write(worker.join("runs/old-run/private.log"), "not exported").unwrap();
drop(opened);
let template = StoppedWorkerTemplate::extract(&worker).unwrap();
(temp, worker, template)
}
fn destination(template: &StoppedWorkerTemplate) -> (tempfile::TempDir, FsRuntimeStoreOptions) {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
initialize(
&root,
template.runtime_id(),
template.workspace_id(),
NEW_URL,
);
(
temp,
FsRuntimeStoreOptions::new(root).with_runtime_id(template.runtime_id()),
)
}
fn assert_fresh(options: &FsRuntimeStoreOptions) {
assert_eq!(fs::read_dir(&options.root).unwrap().count(), 1);
assert!(options.root.join("auth.toml").is_file());
}
#[test]
fn extraction_install_roundtrip_preserves_bytes_and_is_inactive() {
let (_source, worker, template) = source();
let original_worker = fs::read(worker.join(WORKER_FILE)).unwrap();
let original_metadata = fs::read(worker.join(WORKER_METADATA_FILE)).unwrap();
let exported = serde_json::to_vec(&template).unwrap();
let exported_text = String::from_utf8(exported.clone()).unwrap();
for forbidden in [
"old-config",
"old-submission-operation",
"restore_guard",
"private_key",
"http://",
"/old/host",
"private.log",
"resolved_manifest_snapshot",
] {
assert!(!exported_text.contains(forbidden));
}
let template: StoppedWorkerTemplate = serde_json::from_slice(&exported).unwrap();
let (_target, options) = destination(&template);
let auth_before = fs::read(options.root.join("auth.toml")).unwrap();
template.install(&options, request(), &resolved()).unwrap();
assert_eq!(
fs::read(options.root.join("auth.toml")).unwrap(),
auth_before
);
assert_eq!(fs::read(worker.join(WORKER_FILE)).unwrap(), original_worker);
assert_eq!(
fs::read(worker.join(WORKER_METADATA_FILE)).unwrap(),
original_metadata
);
let installed = options
.root
.join(WORKERS_DIR)
.join(template.worker_id().to_string());
assert_eq!(
fs::read(installed.join("session/session.json")).unwrap(),
template.session_manifest
);
assert_eq!(
fs::read(installed.join(format!("session/segments/{}.jsonl", template.segment_id())))
.unwrap(),
template.segment_jsonl
);
assert!(!installed.join("runs").exists());
let metadata: WorkerMetadata =
decode(&fs::read(installed.join(WORKER_METADATA_FILE)).unwrap()).unwrap();
assert_eq!(metadata.active.unwrap().session_id, template.session_id());
assert!(metadata.workspace_root.is_none() && metadata.resolved_manifest_snapshot.is_some());
let opened = FsRuntimeStore::open_or_create(options.root.clone(), RUNTIME).unwrap();
let state = opened.state.unwrap();
assert!(state.diagnostics.is_empty());
let saved = state.workers.get(&worker_id()).unwrap();
assert_eq!(saved.status, WorkerStatus::Stopped);
assert!(saved.last_finished_submission_request_id.is_none());
let PersistedWorkerExecutionState::Available(execution) = &saved.execution_state else {
panic!("not available")
};
assert_eq!(execution.restore_intent, WorkerRestoreIntent::Explicit);
assert!(execution.binding.is_none());
assert_eq!(execution.request, request());
// Fresh restore guard, no operation owner/receipt history.
assert_ne!(
serde_json::to_vec(&saved.restore_guard).unwrap(),
serde_json::to_vec(
&decode::<WorkerAggregateRecord>(&original_worker)
.unwrap()
.restore_guard
)
.unwrap()
);
let guard: GuardState = decode(&serde_json::to_vec(&saved.restore_guard).unwrap()).unwrap();
assert!(guard.active_request_id.is_none() && guard.owners.is_empty());
drop(opened.store);
struct NoExecution;
impl crate::execution::WorkerExecutionBackend for NoExecution {
fn backend_id(&self) -> &str {
"fixture-no-execution"
}
fn spawn_worker(
&self,
_: crate::execution::WorkerExecutionSpawnRequest,
) -> crate::execution::WorkerExecutionSpawnResult {
panic!("fixture executed")
}
fn dispatch_input(
&self,
_: &WorkerRef,
_: crate::interaction::WorkerInput,
) -> crate::execution::WorkerExecutionResult {
panic!("fixture submitted")
}
fn restore_worker(
&self,
_: crate::execution::WorkerExecutionRestoreRequest,
) -> crate::execution::WorkerExecutionSpawnResult {
panic!("fixture auto-restored")
}
}
let runtime =
crate::Runtime::with_fs_store_and_execution_backend(options.clone(), Arc::new(NoExecution))
.unwrap();
assert_eq!(runtime.list_workers().unwrap().len(), 1);
assert_eq!(runtime.summary().unwrap().active_worker_count, 0);
assert_eq!(
runtime
.worker_detail(&WorkerRef::new(worker_id()))
.unwrap()
.status,
WorkerStatus::Stopped
);
drop(runtime);
let runtime = crate::Runtime::with_fs_store(options).unwrap();
assert_eq!(
runtime.list_workers().unwrap()[0].status,
WorkerStatus::Stopped
);
}
#[test]
fn install_rejects_schema_identity_input_and_overwrite() {
let (_source, _, template) = source();
for change in [
"template",
"runtime-schema",
"session-schema",
"session-id",
"segment-id",
"runtime-id",
"workspace-id",
"worker-id",
"origin-id",
"duplicate-id",
"truncated",
] {
let (_target, options) = destination(&template);
let mut invalid = template.clone();
match change {
"template" => invalid.template_version += 1,
"runtime-schema" => invalid.runtime_schema_version += 1,
"session-schema" => {
invalid.session_manifest = format!(
"{{\"schema_version\":999,\"session_id\":\"{}\"}}",
invalid.session_id
)
.into_bytes()
}
"session-id" => invalid.session_id = uuid::Uuid::now_v7(),
"segment-id" => invalid.segment_id = uuid::Uuid::nil(), // no external attestation: valid new Segment ID is otherwise portable
"runtime-id" => invalid.runtime_id = "other-runtime".into(),
"workspace-id" => invalid.workspace_id = "other-workspace".into(),
"worker-id" => invalid.worker_id = WorkerId::now_v7(),
"origin-id" => {
invalid.segment_jsonl = String::from_utf8(invalid.segment_jsonl)
.unwrap()
.replace(RUNTIME, "other-runtime")
.into_bytes()
}
"duplicate-id" => {
let last = invalid
.segment_jsonl
.split_inclusive(|b| *b == b'\n')
.last()
.unwrap()
.to_vec();
invalid.segment_jsonl.extend(last);
}
"truncated" => {
invalid.segment_jsonl.pop();
}
_ => unreachable!(),
}
assert!(
invalid.install(&options, request(), &resolved()).is_err(),
"accepted {change}"
);
assert_fresh(&options);
}
let (_target, options) = destination(&template);
let mut launch = request();
launch.initial_input = Some(crate::interaction::WorkerInput::user("do not execute"));
assert!(template.install(&options, launch, &resolved()).is_err());
assert_fresh(&options);
template.install(&options, request(), &resolved()).unwrap();
let worker_path = options
.root
.join(WORKERS_DIR)
.join(worker_id().to_string())
.join(WORKER_FILE);
let before = fs::read(&worker_path).unwrap();
assert!(template.install(&options, request(), &resolved()).is_err());
assert_eq!(fs::read(worker_path).unwrap(), before);
}
#[test]
fn rejects_uninitialized_untrusted_live_and_partial_destination() {
let (_source, _, template) = source();
let (_target, options) = destination(&template);
let held = acquire_runtime_store_owner_lock(&options.root).unwrap();
assert!(template.install(&options, request(), &resolved()).is_err());
assert_fresh(&options);
drop(held);
initialize(&options.root, "wrong-runtime", WORKSPACE, NEW_URL);
assert!(template.install(&options, request(), &resolved()).is_err());
initialize(&options.root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456");
assert!(template.install(&options, request(), &resolved()).is_err());
fs::remove_file(options.root.join("auth.toml")).unwrap();
assert!(template.install(&options, request(), &resolved()).is_err());
initialize(&options.root, RUNTIME, WORKSPACE, NEW_URL);
fs::create_dir(options.root.join(WORKERS_DIR)).unwrap();
assert!(template.install(&options, request(), &resolved()).is_err());
}
#[test]
fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() {
for change in [
"running",
"pending",
"guard-pending",
"attachment",
"schema",
"worker-id",
"metadata-workspace",
"metadata-session",
"artifact",
"secret",
"url",
"path",
] {
let (_source, worker, _) = source();
let mut aggregate: WorkerAggregateRecord =
decode(&fs::read(worker.join(WORKER_FILE)).unwrap()).unwrap();
match change {
"running" => aggregate.status = WorkerStatus::Running,
"pending" => {
aggregate.execution_state = WorkerExecutionStateRecord::ReconciliationRequired(
PersistedWorkerLifecycleOperation::Restore(PersistedWorkerRestoreOperation {
operation_id: WorkerLifecycleOperationId::new(),
mode: PersistedWorkerRestoreMode::Explicit,
commit: crate::runtime::RestoreCommitFact::NotCommitted,
source_repair_possible: false,
request: request(),
binding: None,
last_settled_status: WorkerStatus::Stopped,
}),
)
}
"guard-pending" => aggregate.restore_guard = serde_json::from_str(r#"{"observation_token":"old-token","active_request_id":"pending-restore","owners":{}}"#).unwrap(),
"attachment" => aggregate.logical_workdir_attachments.push(LogicalWorkdirAttachment {
alias: workdir::WorkdirAttachmentAlias::new("source").unwrap(),
working_directory_id: "old-workdir".into(),
capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY,
}),
"schema" => aggregate.schema_version += 1,
"worker-id" => aggregate.worker_id = WorkerId::now_v7(),
"metadata-workspace" | "metadata-session" => {
let mut metadata: WorkerMetadata =
decode(&fs::read(worker.join(WORKER_METADATA_FILE)).unwrap()).unwrap();
if change == "metadata-workspace" {
metadata.workspace_id = Some("wrong-workspace".into());
} else {
metadata.active.as_mut().unwrap().session_id = uuid::Uuid::now_v7();
}
atomic_write_json(
&worker.join(WORKER_METADATA_FILE),
&metadata,
"test mismatch",
)
.unwrap();
}
"artifact" => {
fs::create_dir_all(worker.join("session/artifacts/paste")).unwrap();
fs::write(
worker.join("session/artifacts/paste/payload"),
"unsupported",
)
.unwrap();
}
"secret" | "url" | "path" => {
let segment = fs::read_dir(worker.join("session/segments"))
.unwrap()
.next()
.unwrap()
.unwrap()
.path();
let replacement = match change {
"secret" => "Bearer secret",
"url" => "https://old-host.invalid",
_ => "/old/host/path",
};
let text = fs::read_to_string(&segment)
.unwrap()
.replace("artificial fixture reply", replacement);
fs::write(segment, text).unwrap();
}
_ => unreachable!(),
}
atomic_write_json(&worker.join(WORKER_FILE), &aggregate, "test aggregate").unwrap();
assert!(
StoppedWorkerTemplate::extract(&worker).is_err(),
"accepted {change}"
);
}
let (_source, worker, _) = source();
let held =
acquire_runtime_store_owner_lock(worker.parent().unwrap().parent().unwrap()).unwrap();
assert!(StoppedWorkerTemplate::extract(&worker).is_err());
drop(held);
}
#[test]
fn schema_v12_settled_template_installs_current_aggregate_without_rewriting_session() {
let (_source, _, mut template) = source();
template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION;
let bytes = serde_json::to_vec(&template).unwrap();
let (_target, options) = destination(&template);
template.install(&options, request(), &resolved()).unwrap();
assert_eq!(serde_json::to_vec(&template).unwrap(), bytes);
let worker = options.root.join(WORKERS_DIR).join(worker_id().to_string());
assert_eq!(
fs::read(worker.join("session/session.json")).unwrap(),
template.session_manifest
);
assert_eq!(
fs::read(worker.join(format!("session/segments/{}.jsonl", template.segment_id))).unwrap(),
template.segment_jsonl
);
let aggregate: WorkerAggregateRecord =
read_bounded_json(&worker.join(WORKER_FILE), "read installed fixture").unwrap();
let saved = aggregate.validate(&worker.join(WORKER_FILE)).unwrap();
assert_eq!(saved.status, WorkerStatus::Stopped);
assert!(matches!(
saved.execution_state,
PersistedWorkerExecutionState::Available(_)
));
// Unsupported template schemas do not enter the versioned reader.
template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION - 1;
assert!(template.validate_shape().is_err());
template.runtime_schema_version = SCHEMA_VERSION + 1;
assert!(template.validate_shape().is_err());
}
#[test]
fn fresh_manifest_and_memory_settings_are_required() {
let (_source, _, template) = source();
let (_target, options) = destination(&template);
let mut manifest = resolved();
manifest.worker.name = "wrong-worker".into();
assert!(template.install(&options, request(), &manifest).is_err());
let mut binding = request();
binding.memory_settings = None;
assert!(template.install(&options, binding, &resolved()).is_err());
let mut binding = request();
binding.memory_settings.as_mut().unwrap().workspace_id = "other-workspace".into();
assert!(template.install(&options, binding, &resolved()).is_err());
assert_fresh(&options);
}
#[test]
fn settled_checkpoints_preserve_receipts_but_refuse_pending_work() {
let mut checkpoint = settled_checkpoint();
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_ok());
checkpoint.pending.push(1);
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
checkpoint.pending.clear();
checkpoint.activating = Some(1);
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
checkpoint.activating = None;
checkpoint.invocation_receipts.push(1);
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
checkpoint.invocation_receipts.clear();
checkpoint.invocation_recovery_blocked = true;
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
}
// Deliberately corrupt only test-owned synthetic source files, never the lab.
fn assert_rejected_session_bytes(worker: &Path, template: &StoppedWorkerTemplate, bytes: Vec<u8>) {
let path = worker.join(format!("session/segments/{}.jsonl", template.segment_id()));
fs::write(path, &bytes).unwrap();
let extraction_error = StoppedWorkerTemplate::extract(worker)
.err()
.expect("invalid source accepted");
assert!(
!extraction_error
.to_string()
.contains("synthetic-discarded-material")
);
let (_target, options) = destination(template);
let mut invalid = template.clone();
invalid.segment_jsonl = bytes;
let installation_error = invalid
.install(&options, request(), &resolved())
.unwrap_err();
assert!(
!installation_error
.to_string()
.contains("synthetic-discarded-material")
);
assert_fresh(&options);
}
fn synthetic_logs(template: &StoppedWorkerTemplate) -> Vec<LogEntry> {
template
.segment_jsonl
.split(|byte| *byte == b'\n')
.filter(|line| !line.is_empty())
.map(|line| serde_json::from_slice(line).unwrap())
.collect()
}
fn encode_synthetic_logs(logs: &[LogEntry]) -> Vec<u8> {
let mut bytes = Vec::new();
for log in logs {
serde_json::to_writer(&mut bytes, log).unwrap();
bytes.push(b'\n');
}
bytes
}
#[tokio::test]
async fn missing_system_prompt_matches_worker_restore_rejection() {
let (_source, worker_dir, template) = source();
let mut logs = synthetic_logs(&template);
assert!(session_store::collect_state(&logs).system_prompt.is_some());
let LogEntry::AnnotatedSegmentStart { system_prompt, .. } = &mut logs[0] else {
panic!("missing seed")
};
*system_prompt = None;
assert_rejected_session_bytes(&worker_dir, &template, encode_synthetic_logs(&logs));
// The owning Worker constructor rejects this before model-client creation,
// allocation, or execution. Do not start a controller or submit any input.
let store = session_store::CombinedStore::new(
WorkerSessionStore::open_read_only(worker_dir.join("session")).unwrap(),
session_store::WorkerAggregateStore::open_read_only(&worker_dir, template.worker_name())
.unwrap(),
);
let restored = worker::Worker::restore_from_manifest(
template.session_id(),
template.segment_id(),
resolved(),
store,
worker::PromptCatalogSource::builtins_only(),
)
.await;
assert!(matches!(
restored,
Err(worker::WorkerError::SegmentSystemPromptMissing { .. })
));
}
#[test]
fn malformed_submission_and_notification_receipts_fail_extraction_and_install() {
let (_source, worker, template) = source();
for notification in [false, true] {
let mut checkpoint = settled_checkpoint();
if notification {
checkpoint.notification_receipts =
vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)];
} else {
checkpoint.receipts = vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)];
}
let mut logs = synthetic_logs(&template);
for log in &mut logs {
if let LogEntry::AnnotatedUserInput { extensions, .. } = log {
*extensions = vec![checkpoint_extension(&checkpoint)];
}
}
assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs));
}
}
#[test]
fn unknown_fields_at_every_retained_record_depth_are_rejected_without_secret_diagnostics() {
let (_source, worker, template) = source();
let original = String::from_utf8(template.segment_jsonl.clone()).unwrap();
// These values are harmless synthetic markers, not lab/private material.
// The compatibility decoder normally discards many of these fields.
let locations = [
r#""kind":"annotated_segment_start""#,
r#""kind":"invoke""#,
r#""kind":"annotated_user_input""#,
r#""kind":"annotated_assistant_item""#,
r#""kind":"llm_usage""#,
r#""kind":"turn_end""#,
r#""kind":"run_completed""#,
r#""max_tokens":null"#,
r#""kind":"message","role":"assistant""#,
r#""kind":"text","text":"artificial fixture reply""#,
r#""kind":"text","content":"Harmless artificial fixture input.""#,
r#""entry_id":"fixed-entry-id""#,
r#""kind":"model_output""#,
r#""worker_id":"01a1657d-b762-7791-87ae-a49f28cadbce""#,
r#""domain":"worker.pending_activations.v1""#,
r#""next_activation_sequence":2"#,
r#""submission_request_id":"completed-submission-request""#,
r#""notification_request_id":"completed-notification-request""#,
];
for location in locations {
assert!(original.contains(location));
for extra in [
r#", "reasoning":{"text":"synthetic-discarded-material"}"#,
r#", "encrypted_content":"synthetic-discarded-material""#,
] {
let altered = original.replacen(location, &format!("{location}{extra}"), 1);
assert_rejected_session_bytes(&worker, &template, altered.into_bytes());
}
}
let duplicate = original.replacen(
r#""role":"assistant""#,
r#""role":"assistant","role":"assistant""#,
1,
);
assert_rejected_session_bytes(&worker, &template, duplicate.into_bytes());
}
#[test]
fn opaque_feature_payloads_cannot_bypass_strict_session_validation() {
let (_source, worker, template) = source();
let mut logs = synthetic_logs(&template);
for log in &mut logs {
if let LogEntry::AnnotatedUserInput { history, .. } = log {
if let session_store::LoggedItem::Message { role, .. } = &mut history[0].item {
*role = session_store::LoggedRole::System;
}
history[0].metadata.input_metadata = Some(
session_store::HistoryInputMetadata::new(
session_store::HistoryInputUsePolicy {
recording_material: session_store::InputEvidenceUse::Exclude,
memory_evidence: session_store::InputEvidenceUse::Exclude,
observation: session_store::InputObservationUse::Hidden,
compaction: session_store::InputCompactionUse::Retain,
},
Some(
session_store::FeatureInputPayload::encode(
"fixture.test",
1,
&BTreeMap::from([("reasoning", "harmless synthetic payload")]),
)
.unwrap(),
),
None,
)
.unwrap(),
);
}
}
assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs));
}
#[cfg(unix)]
#[test]
fn refuses_source_and_destination_symlinks() {
let (_source, worker, template) = source();
let (_target, options) = destination(&template);
std::os::unix::fs::symlink(worker.join(WORKER_FILE), options.root.join("linked")).unwrap();
assert!(template.install(&options, request(), &resolved()).is_err());
fs::remove_file(options.root.join("linked")).unwrap();
std::os::unix::fs::symlink(worker.join(WORKER_FILE), worker.join("runs/link")).unwrap();
assert!(StoppedWorkerTemplate::extract(&worker).is_err());
}
/// Opt-in only: source path supplied explicitly; assertions never display content.
#[test]
#[ignore = "requires the approved artificial lab aggregate via YOI_STOPPED_FIXTURE_SOURCE"]
fn approved_lab_extract_install() {
let path = PathBuf::from(
std::env::var_os("YOI_STOPPED_FIXTURE_SOURCE").expect("explicit source required"),
);
let template = StoppedWorkerTemplate::extract(&path).unwrap();
let (_target, options) = destination(&template);
let mut bind = request();
bind.worker_id = template.worker_id();
bind.workspace_api.as_mut().unwrap().workspace_id = template.workspace_id().into();
bind.memory_settings.as_mut().unwrap().workspace_id = template.workspace_id().into();
let mut manifest = resolved();
manifest.worker.name = template.worker_name();
template.install(&options, bind, &manifest).unwrap();
let installed = options
.root
.join(WORKERS_DIR)
.join(template.worker_id().to_string());
for relative in [
"session/session.json".to_string(),
format!("session/segments/{}.jsonl", template.segment_id()),
] {
assert!(
fs::read(path.join(&relative)).unwrap() == fs::read(installed.join(relative)).unwrap()
);
}
let runtime = crate::Runtime::with_fs_store(options).unwrap();
assert_eq!(runtime.list_workers().unwrap().len(), 1);
assert_eq!(runtime.summary().unwrap().active_worker_count, 0);
}
File diff suppressed because it is too large Load Diff
@@ -193,6 +193,21 @@ mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
fn short_tempdir() -> tempfile::TempDir {
let target = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.parent()
.unwrap()
.join("target");
std::fs::create_dir_all(&target).unwrap();
tempfile::Builder::new()
.prefix("d")
.rand_bytes(3)
.tempdir_in(target)
.unwrap()
}
fn read_request() -> ReadRequest {
ReadRequest {
path: WorkdirPath::new("README.md").unwrap(),
@@ -225,9 +240,60 @@ mod tests {
assert_eq!(attempts.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn router_cleanup_of_unopened_session_never_connects_or_follows_reused_alias() {
let attempts = Arc::new(AtomicUsize::new(0));
let observed = attempts.clone();
let session = DeferredWorkdirSession::handle(
"old",
WorkdirSessionCapabilities::READ_ONLY,
move || {
observed.fetch_add(1, Ordering::SeqCst);
Err("offline".into())
},
);
let router = WorkdirSessionRouter::new();
let alias = WorkdirAttachmentAlias::new("w").unwrap();
let routed = router.attach(alias.clone(), session.clone()).unwrap();
assert!(routed.subscribe_command_events().is_some());
assert!(routed.command_snapshot().is_empty());
router.close_all().await.unwrap();
router.close_all().await.unwrap();
assert!(router.aliases().is_empty());
assert!(matches!(
session.read(read_request()).await,
Err(WorkdirError::SessionClosed)
));
assert_eq!(attempts.load(Ordering::SeqCst), 0);
let new_attempts = Arc::new(AtomicUsize::new(0));
let new_observed = new_attempts.clone();
let replacement = DeferredWorkdirSession::handle(
"new",
WorkdirSessionCapabilities::READ_ONLY,
move || {
new_observed.fetch_add(1, Ordering::SeqCst);
Err("offline".into())
},
);
let new_routed = router.attach(alias, replacement).unwrap();
// A retained old connection cannot open the replacement provider.
assert!(routed.read(read_request()).await.is_err());
assert_eq!(new_attempts.load(Ordering::SeqCst), 0);
assert!(matches!(
new_routed.read(read_request()).await,
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
));
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
// Even after a failed open, cleanup must not try connecting again.
router.close_all().await.unwrap();
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
assert_eq!(attempts.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn failed_open_retries_on_use_and_keeps_read_only_capabilities() {
let temp = tempfile::tempdir().unwrap();
let temp = short_tempdir();
std::fs::write(temp.path().join("README.md"), "restored").unwrap();
let root = temp.path().to_path_buf();
let attempts = Arc::new(AtomicUsize::new(0));
@@ -0,0 +1,185 @@
//! Explicit, operation-owned repair of saved Session material. This is not
//! read-only candidate preparation: the canonical Session/attribution is the
//! receipt for replay after a lost response, and effects are reported as such.
use super::*;
impl ProfileRuntimeWorkerFactory {
pub(super) async fn recover_saved_restore_source(
&self,
request: &WorkerExecutionRestoreRequest,
) -> Result<bool, RuntimeRestoreFailure> {
let cause = |cause| RuntimeRestoreFailure::started(RuntimeRestoreStage::Recovery, cause);
let name = Self::runtime_worker_name_for_ref(&request.worker_ref);
let root = self
.worker_aggregate_dir(&request.worker_ref)
.map_err(|_| cause("restore recovery aggregate unavailable"))?;
let metadata_store = WorkerAggregateStore::open_read_only(&root, &name)
.map_err(|_| cause("restore recovery metadata open failed"))?;
let metadata = metadata_store
.read_by_name(&name)
.map_err(|_| cause("restore recovery metadata read failed"))?
.ok_or_else(|| cause("restore recovery metadata missing"))?;
let active = metadata
.active
.as_ref()
.ok_or_else(|| cause("restore recovery Session missing"))?;
let pending = active.segment_id.is_none();
let incomplete = request.request.subjektiv_attached
&& !matches!(
metadata.subjektiv_session_attribution,
Some(session_store::SubjektivSessionAttributionState::Confirmed { .. })
);
if !pending && !incomplete {
return Ok(false);
}
request
.context
.authorize_restore_source_repair(request.operation_id)
.map_err(|_| cause("canonical source repair authority is not durably confirmed"))?;
// The accepted Runtime operation already owns this Session. Its stable
// Session identity and sole committed seed, not a new restore request,
// are used to reconcile an interrupted first-head publication.
let session_store = WorkerSessionStore::new_with_retention(
root.join("session"),
self.session_retention()
.map_err(|_| cause("restore recovery retention unavailable"))?,
)
.map_err(|_| cause("restore recovery Session open failed"))?;
let (manifest, mut loader) = Self::manifest_for_restore(&metadata, &request.request)
.map_err(|_| cause("restore recovery saved manifest invalid"))?;
let workspace = RuntimeWorkspaceBackendRef::from_worker_request(
&request.request,
self.runtime_id.as_deref(),
)
.map_err(cause)?;
#[cfg(feature = "http-server")]
let request_client = request
.request
.workspace_api
.as_ref()
.and_then(|api| self.workspace_request_clients.get(&api.workspace_id));
let context = workspace.worker_context(
#[cfg(feature = "http-server")]
&request.worker_ref,
#[cfg(feature = "http-server")]
request.workspace_scope.as_ref(),
#[cfg(feature = "http-server")]
self.worker_mutation_identity.as_ref(),
#[cfg(feature = "http-server")]
request_client,
#[cfg(feature = "http-server")]
self.embedded_worker_mutation_dispatcher.as_ref(),
#[cfg(feature = "http-server")]
Some(self.prompt_projection_cache.clone()),
);
if pending && context.workspace_id().is_some() {
let bundle = request
.config_bundle
.as_ref()
.ok_or_else(|| cause("pending recovery launch material missing"))?;
let projection = self
.observe_bundle_prompt_projection(
bundle,
request
.request
.workspace_api
.as_ref()
.map(|api| api.workspace_id.as_str()),
)
.map_err(|_| cause("pending recovery Prompt projection invalid"))?
.ok_or_else(|| cause("pending recovery Prompt projection missing"))?;
loader = loader.with_effective_catalog(projection.projection.catalog.clone());
}
self.failed_restore_resources
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(
request.worker_ref.clone(),
FailedRestoreResources {
operation_id: request.operation_id,
sessions: Vec::new(),
candidate_workdir_cleanup: None,
allocation: None,
feature_tasks: None,
},
);
let store = CombinedStore::new(session_store, metadata_store);
let worker_result = Worker::restore_pending_from_worker_metadata_with_context(
&name,
manifest,
store,
loader,
context,
WorkerFilesystemAuthority::None,
)
.await;
let mut worker = worker_result.map_err(|error| {
self.retain_construction_failure(request, &error);
cause(restore_worker_cause(&error))
})?;
let tasks = worker.retain_startup_feature_cleanup();
let sessions = worker.workdir_sessions();
let allocation = worker.restore_allocation_cleanup_handle();
self.failed_restore_resources
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(
request.worker_ref.clone(),
FailedRestoreResources {
operation_id: request.operation_id,
sessions: vec![sessions.clone()],
candidate_workdir_cleanup: None,
allocation: allocation.clone(),
feature_tasks: Some(tasks.clone()),
},
);
let result = async {
if let Some(binding) = request.request.backend_job.as_ref() {
worker
.bind_backend_job(binding.clone())
.map_err(|_| cause("recovery Backend Job binding failed"))?;
}
worker
.finalize_subjektiv_session_attribution(
request.request.subjektiv_attached,
SubjektivSessionAttributionLifecycle::RestoredSession,
)
.map_err(|error| cause(restore_worker_cause(&error)))?;
if pending {
worker::WorkerController::recover_pending_head(&mut worker, &root)
.await
.map_err(|_| cause("pending Session head recovery failed"))?;
}
Ok::<_, RuntimeRestoreFailure>(())
}
.await;
// Never release allocation while the Worker guard is attached. Retained
// handles in the same operation remain available on every cleanup error.
drop(worker);
let cleanup = async {
tasks
.shutdown()
.await
.map_err(|_| cause("recovery Feature settlement failed"))?;
sessions
.close_all()
.await
.map_err(|_| cause("recovery Workdir close failed"))?;
if let Some(allocation) = allocation {
allocation
.release()
.map_err(|_| cause("recovery allocation release failed"))?;
}
Ok::<_, RuntimeRestoreFailure>(())
}
.await;
result?;
cleanup?;
self.failed_restore_resources
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&request.worker_ref);
Ok(true)
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,927 @@
// Included by the production Profile publication suite. Each fault is at a
// distinct partial-publication boundary; no factory result is substituted.
fn publication_gate<T>() -> (
mpsc::Sender<()>,
mpsc::Receiver<T>,
crate::execution::RestoreTestGate<T>,
) {
let (release, receiver) = mpsc::channel();
let (entered, evidence) = mpsc::channel();
(
release,
evidence,
crate::execution::RestoreTestGate {
entered,
release: receiver,
},
)
}
fn assert_single_publication(fixture: &Fixture, expected: usize) {
let faults = &fixture.backend.factory.restore_faults;
assert_eq!(
faults.executions.load(Ordering::SeqCst),
expected,
"construction reran"
);
assert_eq!(
faults.effects_calls.load(Ordering::SeqCst),
expected,
"restore effects reran"
);
assert_eq!(
faults.publication_attempts.load(Ordering::SeqCst),
expected,
"Controller start reran"
);
fixture.assert_quiet();
}
fn assert_real_attach(
fixture: &Fixture,
handle: &WorkerHandle,
) -> crate::execution::WorkerProtocolTransport {
let worker_ref = &fixture.request.worker_ref;
assert!(matches!(
fixture
.backend
.worker_session(crate::execution::WorkerSessionObservationRequest {
worker_ref: worker_ref.clone(),
}),
runtime_api::WorkerSessionAvailability::LiveProtocol
));
let transport = fixture
.backend
.clone()
.attach_worker_protocol(worker_ref)
.unwrap();
transport.validate().unwrap();
let paired = subscribe_worker_protocol_session(handle);
assert_eq!(
serde_json::to_value(&transport.snapshot).unwrap(),
serde_json::to_value(paired.snapshot_event).unwrap()
);
assert!(
matches!(&transport.snapshot, Event::Snapshot { session, greeting, state, in_flight, .. }
if session == &session_store::public_snapshot::project_current_session_snapshot(&fixture.entries())
&& !greeting.tools.is_empty() && state.state == protocol::WorkerState::Idle && in_flight.is_empty())
);
transport
}
fn observe_publication_markers(
fixture: &Fixture,
observations: tokio::sync::broadcast::Receiver<crate::observation::WorkerObservationEvent>,
markers: &'static [&'static str],
) -> tokio::sync::broadcast::Receiver<crate::observation::WorkerObservationEvent> {
fixture
.backend
.run_on_adapter_runtime(async move {
let mut observations = observations;
let mut counts = vec![0; markers.len()];
let mut snapshots = 0;
tokio::time::timeout(WAIT, async {
loop {
let observed = observations.recv().await.unwrap();
if matches!(observed.payload, Event::Snapshot { .. }) {
snapshots += 1;
}
for (index, marker) in markers.iter().enumerate() {
if is_marker(&observed.payload, marker) {
counts[index] += 1;
}
}
if counts.last() == Some(&1) {
break;
}
}
})
.await
.map_err(|_| format!("lost original subscription markers: {counts:?}"))?;
assert_eq!(
counts,
vec![1; markers.len()],
"lost or duplicated prefix on original receiver"
);
assert_eq!(snapshots, 1, "initial full snapshot was lost or replayed");
while let Ok(event) = observations.try_recv() {
assert!(
!markers
.iter()
.any(|marker| is_marker(&event.payload, marker)),
"duplicate after tail fence"
);
}
Ok(observations)
})
.unwrap()
}
fn partial_registration_recovers_same_controller(hub: bool) {
let mut fixture = Fixture::no_model();
Arc::get_mut(&mut fixture.backend)
.unwrap()
.spawn_restore_timeout = Duration::from_millis(250);
let runtime = stopped_runtime(&fixture);
let pause = runtime.test_pause_restore_continuation();
let worker_ref = fixture.request.worker_ref.clone();
let scope = fixture.request.workspace_scope.as_ref().unwrap();
let request = runtime.test_restore_request(&worker_ref);
let observations = runtime.subscribe_worker_observation().unwrap();
let (started_release, started_entered) = activation_gate(&fixture, true);
let first = runtime
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
.unwrap();
assert_eq!(
first.state,
server_api::WorkerRestoreState::ReconciliationRequired
);
let started = started_entered.recv_timeout(WAIT).unwrap();
let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started);
let old = started.handle.as_ref().unwrap().clone();
let canonical = fixture.canonical_files();
alert_marker(&old, "registration-original-paired-prefix");
let (release, entered, fault) = publication_gate();
if hub {
*fixture
.backend
.factory
.observation_hub
.registration_gate
.lock()
.unwrap() = Some(RestoreHubRegistrationGate {
gate: fault,
fail: true,
});
} else {
*fixture.backend.bridge_fault.lock().unwrap() = Some(fault);
}
started_release.release();
drop(pause);
let boundary_handle = entered.recv_timeout(WAIT).unwrap();
assert!(boundary_handle.same_controller(&old));
let durable = durable_operation(&runtime, &worker_ref).unwrap();
assert_eq!(durable.operation_id, started.operation_id);
assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed);
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
if hub {
let (_, shared, _) = fixture
.backend
.factory
.observation_hub
.get(&worker_ref)
.unwrap();
assert!(
Arc::ptr_eq(&shared, &old.shared_state),
"real hub insert preceded fault"
);
} else {
assert!(
fixture
.backend
.factory
.observation_hub
.get(&worker_ref)
.is_none()
);
}
let (operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref);
assert_eq!(operation, started.operation_id);
assert!(!uncertain);
assert!(
context
.test_buffered_events()
.iter()
.any(|event| matches!(event, Event::Snapshot { .. }))
);
alert_marker(&old, "registration-during-fault");
let pause = runtime.test_pause_restore_continuation();
release.send(()).unwrap();
// This lock also observes the end of the actual registration attempt, not
// merely the test's entered notification. Its retained scope owns cleanup.
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
assert!(old.same_controller(&execution.handle));
assert_eq!(execution.restore_operation_id, Some(operation));
assert_eq!(
execution.tasks.tasks.lock().unwrap().len(),
if hub { 2 } else { 1 }
);
let bridge = execution.tasks.protocol_bridge.lock().unwrap();
if hub {
assert!(bridge.is_none());
} else {
let bridge = bridge.as_ref().unwrap();
assert!(
!bridge.snapshot_pending,
"snapshot already registered, do not replay it"
);
let streams = bridge.streams.lock().unwrap();
assert!(
streams.is_some(),
"original paired streams survived failed task admission"
);
assert_eq!(
serde_json::to_value(&streams.as_ref().unwrap().snapshot_event).unwrap(),
serde_json::to_value(&started.snapshot).unwrap()
);
}
drop(bridge);
assert!(
!old.protocol_is_closed(),
"publication failure must not clean a committed Controller"
);
assert_single_publication(&fixture, 1);
drop(pause);
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
assert!(durable_operation(&runtime, &worker_ref).is_none());
let owner = restore_owner(&runtime, &worker_ref, &request.request_id);
assert_eq!(
owner["operation_id"],
serde_json::to_value(operation).unwrap()
);
assert_eq!(owner["receipt"]["state"], "accepted");
assert!(
fixture
.backend
.pending_restore(&worker_ref)
.unwrap()
.is_none()
);
assert!(!pending.task_in_flight.load(Ordering::Acquire));
assert!(pending.activated.lock().unwrap().is_none());
assert!(pending.result.lock().unwrap().is_none());
assert!(pending.task.try_lock().unwrap().is_none());
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
assert!(old.same_controller(&fixture.backend.workers.lock().unwrap()[&worker_ref].handle));
let transport = assert_real_attach(&fixture, &old);
assert_eq!(fixture.canonical_files(), canonical);
assert_single_publication(&fixture, 1);
alert_marker(&old, "registration-tail-fence");
let observations = observe_publication_markers(
&fixture,
observations,
&[
"registration-original-paired-prefix",
"registration-during-fault",
"registration-tail-fence",
],
);
assert_eq!(
runtime
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_single_publication(&fixture, 1);
// Retain the original transport/handle and scope through Stop/replacement.
runtime
.stop_worker_scoped(scope, &worker_ref, None)
.unwrap();
assert!(old.protocol_is_closed());
assert!(transport.validate().is_err());
let replacement_request = runtime.test_restore_request(&worker_ref);
assert_eq!(
runtime
.restore_worker_operation_scoped(scope, &worker_ref, replacement_request.clone())
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
let replacement = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
assert!(!old.same_controller(&replacement.handle));
assert_ne!(replacement.restore_operation_id, Some(operation));
assert!(
transport
.dispatch(Method::Pause {
command: protocol::WorkerCommandEnvelope::new(801)
})
.is_err()
);
assert!(
transport
.dispatch(Method::Shutdown {
command: protocol::WorkerCommandEnvelope::new(802)
})
.is_err()
);
for method in [
Method::Pause {
command: protocol::WorkerCommandEnvelope::new(803),
},
Method::Shutdown {
command: protocol::WorkerCommandEnvelope::new(804),
},
] {
let old_command = old.clone();
assert!(
fixture
.backend
.run_on_adapter_runtime(async move {
old_command
.send(method)
.await
.map_err(|error| error.to_string())
})
.is_err(),
"closed original Controller command must not route to replacement"
);
}
fixture
.backend
.factory
.deactivate_stopped_controller(&worker_ref, &old)
.unwrap();
let (_, shared, _) = fixture
.backend
.factory
.observation_hub
.get(&worker_ref)
.unwrap();
assert!(Arc::ptr_eq(&shared, &replacement.handle.shared_state));
alert_marker(&old, "registration-stopped-event-must-not-leak");
alert_marker(&replacement.handle, "registration-replacement-fence");
fixture
.backend
.run_on_adapter_runtime(async move {
let mut observations = observations;
tokio::time::timeout(WAIT, async {
loop {
let event = observations.recv().await.unwrap();
assert!(!is_marker(
&event.payload,
"registration-stopped-event-must-not-leak"
));
if is_marker(&event.payload, "registration-replacement-fence") {
break;
}
}
})
.await
.map_err(|_| "replacement observation fence missing".to_string())?;
drop(transport);
Ok(())
})
.unwrap();
assert_single_publication(&fixture, 2);
assert_eq!(
runtime
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_eq!(restore_owner_count(&runtime, &worker_ref), 2);
let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
assert!(current.handle.same_controller(&replacement.handle));
assert_eq!(
current.restore_operation_id,
replacement.restore_operation_id
);
assert_eq!(
replacement.handle.shared_state.snapshot().state,
protocol::WorkerState::Idle
);
drop(current);
drop(execution);
drop(replacement);
drop(pending);
runtime
.stop_worker_scoped(scope, &worker_ref, None)
.unwrap();
fixture.assert_released();
}
fn restore_owner_count(runtime: &EmbeddedRuntime, worker_ref: &WorkerRef) -> usize {
let saved = runtime
.test_runtime_store()
.read_worker_record(worker_ref)
.unwrap();
serde_json::to_value(saved.restore_guard).unwrap()["owners"]
.as_object()
.unwrap()
.len()
}
#[test]
#[serial_test::serial(worker_allocation)]
fn bridge_partial_snapshot_registration_retains_original_streams_and_autonomously_recovers() {
partial_registration_recovers_same_controller(false);
}
#[test]
#[serial_test::serial(worker_allocation)]
fn hub_partial_insert_registration_retains_original_controller_and_autonomously_recovers() {
partial_registration_recovers_same_controller(true);
}
#[test]
#[serial_test::serial(worker_allocation)]
fn context_partial_flush_retains_only_undelivered_queue_and_recovers_without_repost() {
let mut fixture = Fixture::no_model();
Arc::get_mut(&mut fixture.backend)
.unwrap()
.spawn_restore_timeout = Duration::from_millis(250);
let runtime = stopped_runtime(&fixture);
let pause = runtime.test_pause_restore_continuation();
let worker_ref = fixture.request.worker_ref.clone();
let scope = fixture.request.workspace_scope.as_ref().unwrap();
let request = runtime.test_restore_request(&worker_ref);
let observations = runtime.subscribe_worker_observation().unwrap();
let (started_release, started_entered) = activation_gate(&fixture, true);
assert_eq!(
runtime
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
.unwrap()
.state,
server_api::WorkerRestoreState::ReconciliationRequired
);
let started = started_entered.recv_timeout(WAIT).unwrap();
let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started);
let old = started.handle.unwrap();
let (operation, context, _) = runtime.test_pending_restore_context(&worker_ref);
// Stop after the actual bridge and hub registration, without failing that
// boundary, so the flush begins with its real original full snapshot.
let (hub_release, hub_entered, hub_gate) = publication_gate();
*fixture
.backend
.factory
.observation_hub
.registration_gate
.lock()
.unwrap() = Some(RestoreHubRegistrationGate {
gate: hub_gate,
fail: false,
});
started_release.release();
drop(pause);
assert!(
hub_entered
.recv_timeout(WAIT)
.unwrap()
.same_controller(&old)
);
let buffered_prefix = context.test_buffered_events();
assert!(matches!(
buffered_prefix.first(),
Some(Event::Snapshot { .. })
));
let prefix_length = buffered_prefix.len();
let marker = |text: &str| Event::Error {
code: protocol::ErrorCode::Internal,
message: text.into(),
};
context
.publish_protocol_event(marker("flush-delivered"))
.unwrap();
context
.publish_protocol_event(marker("flush-failed"))
.unwrap();
context
.publish_protocol_event(marker("flush-pending"))
.unwrap();
let (release, entered, gate) = publication_gate();
let (restored, restored_queue) = mpsc::channel();
context.test_fail_flush(crate::execution::RestoreFlushFault {
remaining_successes: prefix_length + 1,
gate,
restored,
});
drop(hub_release);
assert!(
matches!(entered.recv_timeout(WAIT).unwrap(), Event::Error { message, .. } if message == "flush-failed")
);
let partial_execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
assert!(old.same_controller(&partial_execution.handle));
assert_eq!(partial_execution.restore_operation_id, Some(operation));
assert_eq!(partial_execution.tasks.tasks.lock().unwrap().len(), 2);
let (_, shared, _) = fixture
.backend
.factory
.observation_hub
.get(&worker_ref)
.unwrap();
assert!(Arc::ptr_eq(&shared, &old.shared_state));
let pause = runtime.test_pause_restore_continuation();
context
.publish_protocol_event(marker("flush-arrived-during-error"))
.unwrap();
release.send(()).unwrap();
let restored = restored_queue.recv_timeout(WAIT).unwrap();
let undelivered: Vec<_> = restored
.iter()
.filter_map(|event| match event {
Event::Error { message, .. } if message.starts_with("flush-") => Some(message.as_str()),
_ => None,
})
.collect();
assert_eq!(
undelivered,
[
"flush-failed",
"flush-pending",
"flush-arrived-during-error"
]
);
assert!(
!restored
.iter()
.any(|event| matches!(event, Event::Snapshot { .. })),
"delivered snapshot must not rejoin the failed suffix"
);
assert_eq!(
serde_json::to_value(context.test_buffered_events()).unwrap(),
serde_json::to_value(&restored).unwrap()
);
assert_eq!(
durable_operation(&runtime, &worker_ref)
.unwrap()
.operation_id,
operation
);
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
// The next terminal gate observes successful retry only after the same
// queued owner has repaired its failed callback and remaining suffix.
let (terminal_release, terminal_entered, gate) = publication_gate();
runtime.test_terminal_receipt_gate(gate);
drop(pause);
assert_eq!(terminal_entered.recv_timeout(WAIT).unwrap(), operation);
assert!(context.test_buffered_events().is_empty());
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
assert_eq!(
durable_operation(&runtime, &worker_ref).unwrap().commit,
crate::runtime::RestoreCommitFact::Committed
);
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
assert!(old.same_controller(&execution.handle));
assert_eq!(execution.restore_operation_id, Some(operation));
drop(terminal_release); // RAII disconnect, not a second request.
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
assert_eq!(
restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"],
"accepted"
);
assert_real_attach(&fixture, &old);
fixture
.backend
.run_on_adapter_runtime(async move {
let mut observations = observations;
let mut messages = Vec::new();
let mut snapshots = 0;
while let Ok(event) = observations.try_recv() {
match event.payload {
Event::Error { message, .. } if message.starts_with("flush-") => {
messages.push(message)
}
Event::Snapshot { .. } => snapshots += 1,
_ => {}
}
}
assert_eq!(
messages,
[
"flush-delivered",
"flush-failed",
"flush-pending",
"flush-arrived-during-error"
]
);
assert_eq!(snapshots, 1);
Ok(())
})
.unwrap();
assert_single_publication(&fixture, 1);
drop(partial_execution);
drop(execution);
drop(pending);
runtime
.stop_worker_scoped(scope, &worker_ref, None)
.unwrap();
fixture.assert_released();
}
#[derive(Clone, Copy, Debug)]
enum TerminalFault {
BeforeRename,
AfterRename,
SyncUnknown,
ReadUnknown,
}
fn published_terminal_receipt_fault(fault: TerminalFault) {
let mut fixture = Fixture::no_model();
// Reuse the existing ordinary non-network model boundary, not the product
// Codex profile. All fault/convergence assertions precede the sole input.
let script = MockClient::sequential(vec![MockResponse::Complete(simple_text_events())]);
let injected = script.clone();
let factory = fixture
.backend
.factory
.as_ref()
.clone()
.with_spawn_model_client_factory(move || Box::new(injected.clone()));
fixture.backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap());
let runtime = Arc::new(stopped_runtime(&fixture));
let pause = runtime.test_pause_restore_continuation();
let worker_ref = fixture.request.worker_ref.clone();
let scope = fixture.request.workspace_scope.clone().unwrap();
let request = runtime.test_restore_request(&worker_ref);
let observations = runtime.subscribe_worker_observation().unwrap();
let mut catalog = runtime
.subscribe_event_selector_scoped(
&scope,
protocol::subscription::EventSubscriptionSelector::RuntimeWorkers,
)
.unwrap();
let (release, entered, gate) = publication_gate();
runtime.test_terminal_receipt_gate(gate);
let restoring = {
let runtime = runtime.clone();
let worker_ref = worker_ref.clone();
let scope = scope.clone();
let request = request.clone();
std::thread::spawn(move || {
runtime.restore_worker_operation_scoped(&scope, &worker_ref, request)
})
};
let operation = entered.recv_timeout(WAIT).unwrap();
// Restore legitimately repairs derived metadata before publication. Receipt
// failures must not rerun those effects; compare from this actual boundary.
let canonical = fixture.canonical_files();
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
let old = execution.handle.clone();
assert_eq!(execution.restore_operation_id, Some(operation));
assert!(
fixture
.backend
.pending_restore(&worker_ref)
.unwrap()
.is_none()
);
assert_eq!(execution.tasks.tasks.lock().unwrap().len(), 2);
assert!(execution.tasks.protocol_bridge.lock().unwrap().is_none());
assert_eq!(
durable_operation(&runtime, &worker_ref)
.unwrap()
.operation_id,
operation
);
assert_eq!(
durable_operation(&runtime, &worker_ref).unwrap().commit,
crate::runtime::RestoreCommitFact::Committed
);
let owner = restore_owner(&runtime, &worker_ref, &request.request_id);
assert!(owner["receipt"].is_null());
let (same_operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref);
assert_eq!(same_operation, operation);
assert!(!uncertain);
assert!(context.test_buffered_events().is_empty());
let transport = assert_real_attach(&fixture, &old);
assert_single_publication(&fixture, 1);
alert_marker(&old, "terminal-before-receipt");
let store = runtime.test_runtime_store();
let reconcile_before = store.test_worker_reconcile_counts();
store.fail_next_worker_write(if matches!(fault, TerminalFault::BeforeRename) {
AtomicWriteFault::BeforeRename
} else {
AtomicWriteFault::AfterRename
});
match fault {
TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(),
TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(),
_ => {}
}
release.send(()).unwrap();
let first = restoring.join().unwrap().unwrap();
let reconcile_after = store.test_worker_reconcile_counts();
let reconciliation = (
reconcile_after.0 - reconcile_before.0,
reconcile_after.1 - reconcile_before.1,
);
assert_eq!(
reconciliation,
match fault {
TerminalFault::BeforeRename => (0, 0),
TerminalFault::AfterRename => (1, 4), // canonical file plus Worker/workers/root directories
TerminalFault::SyncUnknown => (1, 1),
TerminalFault::ReadUnknown => (1, 0),
},
"{fault:?}: actual canonical read/sync attempts"
);
assert_eq!(
first.state,
if matches!(fault, TerminalFault::AfterRename) {
server_api::WorkerRestoreState::Accepted
} else {
server_api::WorkerRestoreState::ReconciliationRequired
},
"{fault:?}: {first:?}"
);
assert_eq!(fixture.canonical_files(), canonical);
assert!(!old.protocol_is_closed());
transport.validate().unwrap();
assert_single_publication(&fixture, 1);
let disk_owner = restore_owner(&runtime, &worker_ref, &request.request_id);
assert_eq!(disk_owner["operation_id"], owner["operation_id"]);
match fault {
TerminalFault::BeforeRename => {
assert!(disk_owner["receipt"].is_null());
assert!(!runtime.test_pending_restore_context(&worker_ref).2);
}
TerminalFault::AfterRename => {
assert_eq!(disk_owner["receipt"]["state"], "accepted");
assert!(durable_operation(&runtime, &worker_ref).is_none());
}
TerminalFault::SyncUnknown | TerminalFault::ReadUnknown => {
assert_eq!(
disk_owner["receipt"]["state"], "accepted",
"visible rename alone is not durable proof"
);
let (same_operation, _, fenced) = runtime.test_pending_restore_context(&worker_ref);
assert_eq!(same_operation, operation);
assert!(
fenced,
"unknown canonical outcome must retain the memory owner/write fence"
);
// Keep durability unproved for one competing Stop as well: it must
// fail before any Shutdown, never manufacture a Stopped judgment.
match fault {
TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(),
TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(),
_ => unreachable!(),
}
assert!(matches!(
runtime.stop_worker_scoped(&scope, &worker_ref, None),
Err(crate::error::RuntimeError::StoreCommitOutcomeUnknown { .. })
));
assert!(!old.protocol_is_closed());
assert!(!execution.shutdown_requested.load(Ordering::Acquire));
assert!(runtime.test_pending_restore_context(&worker_ref).2);
assert_single_publication(&fixture, 1);
}
}
// Recovery is autonomous after the failed write / confirmed unknown fence.
// No POST, fresh request, or input is needed to make it converge.
drop(pause);
if !matches!(fault, TerminalFault::AfterRename) {
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
}
assert!(durable_operation(&runtime, &worker_ref).is_none());
assert_eq!(
restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"],
"accepted"
);
let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
assert!(old.same_controller(&current.handle));
assert_eq!(current.restore_operation_id, Some(operation));
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
assert_real_attach(&fixture, &old);
assert_single_publication(&fixture, 1);
// A canonical terminal reread may bypass the final redundant upsert. The
// original subscriber must still have the current state and observation token.
let fresh_catalog = runtime
.subscribe_event_selector_scoped(
&scope,
protocol::subscription::EventSubscriptionSelector::RuntimeWorkers,
)
.unwrap();
let protocol::subscription::SubscriptionSnapshot::Workers { workers } =
fresh_catalog.snapshot()
else {
panic!("Runtime catalog must return Workers");
};
assert_eq!(workers.len(), 1);
let mut last_applied = None;
use futures::FutureExt;
while let Some(update) = catalog.recv().now_or_never() {
match update.unwrap().payload {
protocol::subscription::SubscriptionEventPayload::WorkerUpserted { worker } => {
last_applied = Some(worker);
}
payload => panic!("unexpected catalog update: {payload:?}"),
}
}
assert!(
serde_json::to_value(last_applied.expect("restore must publish its catalog state"))
.unwrap()
== serde_json::to_value(&workers[0]).unwrap(),
"original catalog subscriber diverged after {fault:?}"
);
drop(fresh_catalog);
drop(catalog);
alert_marker(&old, "terminal-after-recovery");
let _observations = observe_publication_markers(
&fixture,
observations,
&["terminal-before-receipt", "terminal-after-recovery"],
);
let accepted = runtime
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
.unwrap();
assert_eq!(accepted.state, server_api::WorkerRestoreState::Accepted);
assert_eq!(
runtime
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
.unwrap(),
accepted
);
assert_eq!(restore_owner_count(&runtime, &worker_ref), 1);
let mut stale = request;
stale.request_id = "terminal-fresh-stale-intent".into();
assert!(matches!(
runtime.restore_worker_operation_scoped(&scope, &worker_ref, stale),
Err(crate::error::RuntimeError::RestoreObservationConflict { .. })
));
assert_single_publication(&fixture, 1);
assert_eq!(
script.call_count.load(Ordering::SeqCst),
0,
"receipt recovery cannot trigger input or a model call"
);
// This transport was paired before the failed receipt write. Exercise its
// normal log-entry receiver after autonomous settlement, not a new attach.
transport
.dispatch(Method::submit_text(
"terminal-ordinary-input",
"terminal-ordinary-input",
))
.unwrap();
fixture
.backend
.run_on_adapter_runtime(async move {
let mut transport = transport;
let (mut admitted, mut committed, mut text, mut ended) = (false, false, false, false);
tokio::time::timeout(WAIT, async {
while !(admitted && committed && text && ended) {
match transport.events.recv().await.unwrap() {
Event::SubmissionAccepted {
submission_request_id,
..
} if submission_request_id == "terminal-ordinary-input" => admitted = true,
Event::UserMessage { segments, .. } => {
assert_eq!(
segments,
vec![protocol::Segment::text("terminal-ordinary-input")]
);
assert!(!committed, "paired log receiver duplicated input");
committed = true;
}
Event::TextDone { text: output } => {
assert_eq!(output, "hello from worker");
text = true;
}
Event::RunEnd { .. } => ended = true,
Event::SubmissionRejected { .. } | Event::Error { .. } => {
panic!("ordinary input failed after receipt recovery")
}
_ => {}
}
}
})
.await
.map_err(|_| {
"ordinary input failed to settle on original paired transport".to_string()
})?;
drop(transport);
Ok(())
})
.unwrap();
assert_eq!(script.call_count.load(Ordering::SeqCst), 1);
assert_eq!(fixture.entries().iter().filter(|entry| matches!(entry,
LogEntry::AnnotatedUserInput { segments, .. } if segments == &vec![protocol::Segment::text("terminal-ordinary-input")]
)).count(), 1);
assert_single_publication(&fixture, 1);
drop(execution);
drop(current);
runtime
.stop_worker_scoped(&scope, &worker_ref, None)
.unwrap();
assert!(old.protocol_is_closed());
fixture.assert_released();
}
#[test]
#[serial_test::serial(worker_allocation)]
fn published_controller_terminal_receipt_before_rename_autonomously_settles_without_reconstruction()
{
published_terminal_receipt_fault(TerminalFault::BeforeRename);
}
#[test]
#[serial_test::serial(worker_allocation)]
fn published_controller_terminal_receipt_after_rename_is_canonically_reread_and_synced() {
published_terminal_receipt_fault(TerminalFault::AfterRename);
}
#[test]
#[serial_test::serial(worker_allocation)]
fn published_controller_terminal_receipt_sync_unknown_retains_owner_and_write_fence() {
published_terminal_receipt_fault(TerminalFault::SyncUnknown);
}
#[test]
#[serial_test::serial(worker_allocation)]
fn published_controller_terminal_receipt_read_unknown_retains_owner_and_write_fence() {
published_terminal_receipt_fault(TerminalFault::ReadUnknown);
}
+48 -7
View File
@@ -1,23 +1,32 @@
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
#[cfg(feature = "http-server")]
use std::time::Duration;
use std::time::{SystemTime, UNIX_EPOCH};
#[cfg(feature = "http-server")]
use crate::workspace_request::{
RuntimeWorkspaceRequest, RuntimeWorkspaceRequestClient, RuntimeWorkspaceRequestError,
};
use worker::WorkspaceResponse;
#[cfg(feature = "http-server")]
use worker::{
WorkspaceBinaryRequest, WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError,
WorkspacePromptCatalogResolution, WorkspacePromptProjection, WorkspaceRequest,
WorkspaceRequestMethod, WorkspaceResponse, WorkspaceServerOperation,
WorkspaceRequestMethod, WorkspaceServerOperation,
};
use crate::auth::{
RuntimeAuthError, RuntimeIdentityMaterial, RuntimeWorkerMutationSourceSigner,
WORKER_REMOVE_PERMISSION, WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION,
WorkerMutationActorKind, WorkerMutationOperation, WorkerMutationSourceClaims, new_token_id,
WORKER_REMOVE_PERMISSION, WorkerMutationActorKind, WorkerMutationOperation,
WorkerMutationSourceClaims, new_token_id,
};
#[cfg(feature = "http-server")]
use crate::auth::{WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION};
use crate::runtime::RuntimeWorkspaceScope;
#[cfg(feature = "http-server")]
use crate::worker_backend::WorkspacePromptProjectionCache;
#[cfg(feature = "http-server")]
fn ticket_assignment_role_segment(role: server_api::TicketAssignmentRole) -> &'static str {
match role {
server_api::TicketAssignmentRole::Orchestrator => "orchestrator",
@@ -144,6 +153,7 @@ pub trait EmbeddedWorkerMutationDispatcher: Send + Sync {
#[derive(Clone)]
enum RuntimeWorkerMutationTransport {
#[cfg(feature = "http-server")]
Remote {
request_client: RuntimeWorkspaceRequestClient,
},
@@ -161,6 +171,7 @@ pub struct RuntimeWorkerMutationForwarder {
}
impl RuntimeWorkerMutationForwarder {
#[cfg(feature = "http-server")]
pub fn remote(
identity: &RuntimeIdentityMaterial,
scope: RuntimeWorkspaceScope,
@@ -206,6 +217,7 @@ impl RuntimeWorkerMutationForwarder {
target_worker_id,
)?;
match (&self.transport, proof) {
#[cfg(feature = "http-server")]
(
RuntimeWorkerMutationTransport::Remote { request_client },
RuntimeOwnedWorkerMutationProof::Remote(token),
@@ -231,6 +243,7 @@ impl RuntimeWorkerMutationForwarder {
}
}
#[cfg(feature = "http-server")]
struct RemoteWorkerRemoveHttpRequest {
request_client: RuntimeWorkspaceRequestClient,
source_worker_id: String,
@@ -240,6 +253,7 @@ struct RemoteWorkerRemoveHttpRequest {
reason: String,
}
#[cfg(feature = "http-server")]
fn execute_remote_worker_remove_http(
request: RemoteWorkerRemoveHttpRequest,
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
@@ -263,6 +277,7 @@ fn execute_remote_worker_remove_http(
execute_remote_worker_remove_http_blocking(request)
}
#[cfg(feature = "http-server")]
fn execute_remote_worker_remove_http_blocking(
request: RemoteWorkerRemoveHttpRequest,
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
@@ -315,6 +330,7 @@ fn execute_remote_worker_remove_http_blocking(
}
}
#[cfg(feature = "http-server")]
#[derive(Clone)]
pub struct RuntimeOwnedWorkspaceClient {
workspace_id: String,
@@ -325,6 +341,7 @@ pub struct RuntimeOwnedWorkspaceClient {
prompt_projection_cache: Option<Arc<WorkspacePromptProjectionCache>>,
}
#[cfg(feature = "http-server")]
impl RuntimeOwnedWorkspaceClient {
pub fn new(
workspace_id: impl Into<String>,
@@ -495,6 +512,7 @@ impl RuntimeOwnedWorkspaceClient {
}
}
#[cfg(feature = "http-server")]
impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
@@ -510,6 +528,7 @@ impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
}
}
#[cfg(feature = "http-server")]
fn generated_workspace_response<T: serde::Serialize>(
response: Result<T, server_api::client_support::ClientError<server_api::RepositoryApiError>>,
) -> Result<WorkspaceResponse, WorkspaceClientError> {
@@ -530,6 +549,7 @@ fn generated_workspace_response<T: serde::Serialize>(
}
}
#[cfg(feature = "http-server")]
impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
fn workspace_id(&self) -> Option<&str> {
Some(self.request_client.workspace_id())
@@ -929,12 +949,17 @@ fn unix_now_seconds() -> u64 {
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "http-server")]
use crate::auth::{
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WorkerMutationSourceExpectation,
decode_runtime_request_source_claims, decode_worker_mutation_source_claims,
request_body_digest, verify_worker_mutation_source_proof,
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, decode_runtime_request_source_claims,
request_body_digest,
};
use crate::auth::{
WorkerMutationSourceExpectation, decode_worker_mutation_source_claims,
verify_worker_mutation_source_proof,
};
#[cfg(feature = "http-server")]
fn binary_loopback(response: Vec<u8>) -> (String, std::thread::JoinHandle<(String, Vec<u8>)>) {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
@@ -972,6 +997,7 @@ mod tests {
(base_url, server)
}
#[cfg(feature = "http-server")]
#[test]
fn generated_ticket_assignment_transport_signs_exact_authoring_and_bounded_read_subjects() {
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
@@ -1104,6 +1130,7 @@ mod tests {
}
}
#[cfg(feature = "http-server")]
#[test]
fn binary_roundtrip_signs_exact_query_method_worker_and_raw_body() {
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
@@ -1194,6 +1221,7 @@ mod tests {
}
}
#[cfg(feature = "http-server")]
#[test]
fn binary_responses_enforce_declared_and_streamed_limits_with_fixed_errors() {
for response in [
@@ -1213,6 +1241,7 @@ mod tests {
}
}
#[cfg(feature = "http-server")]
#[test]
fn binary_redirect_returns_status_without_forwarding_authentication() {
let (base_url, server) = binary_loopback(b"HTTP/1.1 307 Temporary Redirect\r\nLocation: http://127.0.0.1:1/private\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec());
@@ -1231,6 +1260,7 @@ mod tests {
assert!(result.body.is_empty());
}
#[cfg(feature = "http-server")]
#[test]
fn binary_invalid_or_normalized_targets_fail_with_fixed_errors_before_http() {
let client = RuntimeOwnedWorkspaceClient::new(
@@ -1277,6 +1307,7 @@ mod tests {
);
}
#[cfg(feature = "http-server")]
#[test]
fn current_prompt_projection_uses_the_shared_runtime_cache_without_http() {
let cache = Arc::new(WorkspacePromptProjectionCache::default());
@@ -1312,6 +1343,7 @@ mod tests {
assert!(Arc::ptr_eq(&projection.catalog, &second.catalog));
}
#[cfg(feature = "http-server")]
#[test]
fn prompt_projection_cache_miss_fetches_content_identity() {
use std::io::{Read, Write};
@@ -1356,6 +1388,7 @@ mod tests {
assert_eq!(fetched.projection.source_digest, "source-3");
}
#[cfg(feature = "http-server")]
#[test]
fn concurrent_prompt_projection_miss_fetches_once_and_shares_catalog() {
use std::io::{Read, Write};
@@ -1427,6 +1460,7 @@ mod tests {
);
}
#[cfg(feature = "http-server")]
#[test]
fn current_prompt_projection_rejects_cross_workspace_response() {
use std::io::{Read, Write};
@@ -1473,6 +1507,7 @@ mod tests {
assert!(error.to_string().contains("scope mismatch"));
}
#[cfg(feature = "http-server")]
#[test]
fn ordinary_workspace_forwarding_stamps_runtime_identity_and_signs_path_and_query() {
use std::io::{Read, Write};
@@ -1531,6 +1566,7 @@ mod tests {
);
}
#[cfg(feature = "http-server")]
#[test]
fn workspace_worker_discovery_signs_dedicated_permission_and_encoded_query() {
use std::io::{Read, Write};
@@ -1652,6 +1688,7 @@ mod tests {
);
}
#[cfg(feature = "http-server")]
#[tokio::test(flavor = "multi_thread")]
async fn remote_forwarder_is_safe_in_async_runtime_and_stamps_signed_proof() {
use std::io::{Read, Write};
@@ -1815,6 +1852,7 @@ mod tests {
assert_eq!(reason, "retire obsolete Worker");
}
#[cfg(feature = "http-server")]
#[test]
fn runtime_owned_workspace_client_returns_successful_response_without_request_timeout() {
// Effective deadline clearing is exercised with a short inherited deadline in
@@ -1838,6 +1876,7 @@ mod tests {
server.join().unwrap();
}
#[cfg(feature = "http-server")]
#[test]
fn runtime_owned_workspace_client_honors_per_request_timeouts() {
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
@@ -1857,6 +1896,7 @@ mod tests {
server.join().unwrap();
}
#[cfg(feature = "http-server")]
#[test]
fn runtime_owned_workspace_client_reports_request_timeouts() {
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
@@ -1878,6 +1918,7 @@ mod tests {
server.join().unwrap();
}
#[cfg(feature = "http-server")]
fn delayed_workspace_response(delay: Duration) -> (String, std::thread::JoinHandle<()>) {
use std::io::{Read, Write};
use std::net::TcpListener;
@@ -3676,6 +3676,7 @@ mod tests {
#[test]
fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() {
crate::restore_test_environment::require_restore_test_environment();
let repo = create_clean_repo();
// ssh-agent and the broker receive relative socket paths in this owned
// fixture tree, so the test does not depend on checkout/TMPDIR length.
+264 -1
View File
@@ -7,6 +7,7 @@ use workdir::WorkdirSessionHandle;
use crate::PromptCatalogSource;
use crate::controller::{ShutdownReceiver, WorkerController, WorkerHandle};
pub use crate::worker::{PreparedRestoredWorker, RestoreCandidateWorkdirCleanup};
use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext};
use manifest::WorkerManifest;
@@ -161,6 +162,38 @@ where
Ok(PreparedWorker::new(worker, self.layout))
}
/// Reconstruct a saved materialized Worker without allocation, attribution
/// repair, Feature installation, or controller exposure. The caller owns the
/// later controller layout and must activate before starting it.
pub async fn prepare_restored_candidate(
self,
worker_name: &str,
) -> Result<PreparedRestoredWorker<St>, WorkerBootstrapError> {
let mut candidate =
Worker::prepare_restored_from_worker_metadata_with_context_and_model_client(
worker_name,
self.manifest,
self.store,
self.prompt_catalog,
self.workspace_context,
self.filesystem_authority,
self.model_client,
)
.await
.map_err(WorkerBootstrapError::Worker)?;
if let Some(workdir_session) = self.workdir_session {
candidate
.worker_mut()
.map_err(WorkerBootstrapError::Worker)?
.bind_single_workdir_session(Some(workdir_session));
}
candidate
.worker_mut()
.map_err(WorkerBootstrapError::Worker)?
.bind_subjektiv_host(self.subjektiv_host);
Ok(candidate)
}
pub async fn prepare_restored(
self,
worker_name: &str,
@@ -243,6 +276,7 @@ where
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
let cleanup_sessions = worker.workdir_sessions();
let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some();
let controller = match layout {
WorkerBootstrapLayout::Direct { runtime_base } => {
WorkerController::spawn_owned(worker, &runtime_base).await
@@ -259,7 +293,10 @@ where
controller_task,
}),
Err(source) => {
let cleanup_failed = cleanup_sessions.close_all().await.is_err();
// Factory-owned task scopes must settle before sessions close.
// Cleanup remains unconfirmed here; preserve the startup source.
let cleanup_failed =
factory_owned_cleanup || cleanup_sessions.close_all().await.is_err();
Err(WorkerBootstrapError::Controller {
source,
cleanup_failed,
@@ -267,3 +304,229 @@ where
}
}
}
/// Host bindings for one unpublished restored Controller. These do not expose
/// Worker execution or permit arbitrary mutation during preparation.
pub struct RestoredControllerBindings {
pub backend_job: Option<crate::BackendJobExecutionBinding>,
pub workdir_sessions: std::sync::Arc<workdir::WorkdirSessionRouter>,
pub observation_provider:
Option<std::sync::Arc<dyn crate::feature::builtin::WorkerObservationProvider>>,
pub subjektiv_attached: bool,
}
/// The same real receiver, snapshot sources and paired subscription retained
/// privately across the Runtime's durable restore decision.
pub struct UnpublishedRestoredController<St: Store> {
initialized: Option<crate::controller::InitializedWorkerController<Box<dyn LlmClient>, St>>,
activation: Option<crate::worker::RestoredWorkerActivation>,
streams: Option<crate::ipc::protocol_session::WorkerProtocolSessionStreams>,
subjektiv_attached: bool,
effects_applied: bool,
activation_failed: bool,
closing: bool,
}
pub struct ActivatedRestoredController {
pub controller: BootstrappedWorker,
pub streams: crate::ipc::protocol_session::WorkerProtocolSessionStreams,
}
impl<St> PreparedRestoredWorker<St>
where
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
pub fn workspace_client_handle(
&mut self,
) -> Result<std::sync::Arc<dyn crate::WorkspaceClient>, WorkerError> {
Ok(self.worker_mut()?.workspace_client_handle())
}
pub fn retain_startup_feature_cleanup(
&mut self,
) -> Result<crate::WorkerStartupFeatureCleanup, WorkerError> {
Ok(self.worker_mut()?.retain_startup_feature_cleanup())
}
/// Initialize the real Controller without allocation, canonical restore
/// writes, attribution changes, registry reclamation or input consumption.
/// MCP discovery and Feature tasks belong to the retained startup cleanup.
pub async fn prepare_runtime_controller(
mut self,
run_dir: &std::path::Path,
bindings: RestoredControllerBindings,
) -> Result<UnpublishedRestoredController<St>, WorkerBootstrapError> {
let worker = self.worker_mut().map_err(WorkerBootstrapError::Worker)?;
if let Some(binding) = bindings.backend_job {
worker.bind_backend_job(binding).map_err(|error| {
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.into()))
})?;
}
worker
.workdir_sessions()
.close_all()
.await
.map_err(|error| {
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.to_string()))
})?;
worker.bind_workdir_sessions(bindings.workdir_sessions);
worker.bind_worker_observation_provider(bindings.observation_provider);
let flow_enabled = worker.manifest().feature.flow.enabled;
if flow_enabled {
let report = worker
.install_runtime_flow_transition_feature()
.map_err(|error| {
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error))
})?;
if report.reports.iter().any(|report| !report.installed) {
return Err(WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(
"Flow transition installation refused".into(),
)));
}
}
let (worker, activation) = self
.into_controller_parts()
.map_err(WorkerBootstrapError::Worker)?;
let initialized = WorkerController::initialize_restored(worker, run_dir)
.await
.map_err(|source| WorkerBootstrapError::Controller {
source,
cleanup_failed: true,
})?;
if flow_enabled {
initialized.handle.shared_state.enable_flow_transition();
}
let streams = {
let _guard = initialized.handle.protocol_snapshot_publish_guard();
crate::ipc::protocol_session::subscribe_worker_protocol_session(&initialized.handle)
};
Ok(UnpublishedRestoredController {
initialized: Some(initialized),
activation: Some(activation),
streams: Some(streams),
subjektiv_attached: bindings.subjektiv_attached,
effects_applied: false,
activation_failed: false,
closing: false,
})
}
}
impl<St> UnpublishedRestoredController<St>
where
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
pub fn snapshot(&self) -> &protocol::Event {
&self
.streams
.as_ref()
.expect("unpublished restore streams")
.snapshot_event
}
/// Only publication failures after confirmed effects can retry on this
/// exact owner. Partial activation must be reconciled, never blindly rerun.
pub fn can_retry_publication(&self) -> bool {
self.initialized.is_some()
&& self.effects_applied
&& !self.activation_failed
&& !self.closing
}
/// Apply canonical restore effects after the host's durable decision, while
/// retaining the same private Controller for subsequent publication.
pub async fn apply_effects(&mut self) -> Result<(), WorkerBootstrapError> {
let initialized = self
.initialized
.as_mut()
.ok_or_else(|| WorkerBootstrapError::Worker(WorkerError::RestoreCandidateClosing))?;
if self.activation_failed || self.closing {
return Err(WorkerBootstrapError::Worker(
WorkerError::RestoreCandidateClosing,
));
}
// Preparation's read-only authorization is not a publication grant.
// Revalidate current Host authority after commit, before effects/input,
// including a publication retry on this same private Controller.
initialized
.worker
.validate_subjektiv_body_publication()
.map_err(WorkerBootstrapError::Worker)?;
if !self.effects_applied {
self.activation_failed = true;
self.activation
.as_ref()
.expect("restore activation")
.activate(&mut initialized.worker)
.await
.map_err(WorkerBootstrapError::Worker)?;
initialized
.worker
.finalize_subjektiv_session_attribution(
self.subjektiv_attached,
crate::SubjektivSessionAttributionLifecycle::RestoredSession,
)
.map_err(WorkerBootstrapError::Worker)?;
self.effects_applied = true;
self.activation_failed = false;
self.activation = None;
}
Ok(())
}
pub fn allocation_cleanup(
&self,
) -> Option<crate::runtime::worker_allocation::ScopeAllocationCleanup> {
self.initialized
.as_ref()
.and_then(|initialized| initialized.worker.restore_allocation_cleanup_handle())
}
pub async fn activate(&mut self) -> Result<ActivatedRestoredController, WorkerBootstrapError> {
self.apply_effects().await?;
let initialized = self
.initialized
.as_mut()
.expect("same initialized Controller");
let publication = initialized.prepare_publication().await.map_err(|source| {
WorkerBootstrapError::Controller {
source,
cleanup_failed: true,
}
})?;
let initialized = self
.initialized
.take()
.expect("same initialized Controller");
let (handle, shutdown, controller_task) = initialized.start(publication);
Ok(ActivatedRestoredController {
controller: BootstrappedWorker {
handle,
shutdown,
controller_task,
},
streams: self.streams.take().expect("same paired subscription"),
})
}
pub async fn discard(&mut self) -> Result<(), WorkerError> {
self.closing = true;
if let Some(initialized) = self.initialized.as_mut() {
initialized
.worker
.retain_startup_feature_cleanup()
.shutdown()
.await
.map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?;
initialized
.worker
.workdir_sessions()
.close_all()
.await
.map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?;
}
self.streams = None;
self.initialized = None;
Ok(())
}
}
+373 -30
View File
@@ -21,7 +21,7 @@ use crate::shutdown_after_idle::{
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
take_shutdown_request_after_status,
};
use crate::spawn::registry::SpawnedWorkerRegistry;
use crate::spawn::registry::{PreparedSpawnedWorkerRegistryLoad, SpawnedWorkerRegistry};
use crate::spawn::tool::sub_worker_spawn_tool;
use crate::worker::{
PendingSubmissionHandle, SystemItemCommitter, Worker, WorkerError, WorkerRunResult,
@@ -654,7 +654,56 @@ pub type ShutdownReceiver = oneshot::Receiver<()>;
pub struct WorkerController;
// Own the real command receiver and all snapshot sources before starting any
// protocol listener or controller loop. Fresh startup may materialize the head;
// Runtime restore requires an already materialized source and defers canonical
// restore effects until its durable decision. Tool discovery can own external
// resources, so private initialization is not synonymous with side-effect-free.
pub(crate) struct InitializedWorkerController<C: LlmClient, St: Store> {
pub(crate) worker: Worker<C, St>,
pub(crate) handle: WorkerHandle,
method_rx: mpsc::Receiver<Method>,
shutdown_tx: oneshot::Sender<()>,
shutdown_rx: ShutdownReceiver,
registry_reclaim: PreparedSpawnedWorkerRegistryLoad<St>,
shutdown_after_idle: ShutdownAfterIdleRequest,
command_events: Option<PreparedWorkdirCommandEvents>,
}
pub(crate) struct PreparedControllerPublication {
command_observer: Option<tokio::task::JoinHandle<()>>,
}
impl WorkerController {
/// Explicit recovery of an unmaterialized Session head. This installs the
/// saved Feature composition needed to produce the seed, but creates no
/// Controller, input channel, socket, observer, or queue consumer. The caller
/// owns the same operation's Feature/session/allocation cleanup on every exit.
pub async fn recover_pending_head<C, St>(
worker: &mut Worker<C, St>,
runtime_base: &Path,
) -> std::io::Result<()>
where
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
if worker.needs_initial_session_head_materialization() {
worker
.validate_subjektiv_body_publication()
.map_err(std::io::Error::other)?;
let registry = SpawnedWorkerRegistry::new_internal(
worker.manifest().worker.name.clone(),
worker.scope().clone(),
);
register_worker_tools(worker, runtime_base.to_path_buf(), registry, None, None).await?;
worker
.materialize_durable_session_head()
.await
.map_err(std::io::Error::other)?;
}
Ok(())
}
pub async fn spawn<C, St>(
worker: Worker<C, St>,
runtime_base: &Path,
@@ -734,8 +783,12 @@ impl WorkerController {
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
let sessions = worker.workdir_sessions();
let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some();
let result = Self::spawn_initialized(worker, runtime_base, runtime_run).await;
// An opted-in factory must quiesce its captured Feature tasks before
// closing their Workdir sessions; this wrapper cannot confirm that order.
if result.is_err()
&& !factory_owned_cleanup
&& let Err(error) = sessions.close_all().await
{
tracing::warn!(%error, "Workdir sessions close after controller startup failure failed");
@@ -744,7 +797,7 @@ impl WorkerController {
}
async fn spawn_initialized<C, St>(
mut worker: Worker<C, St>,
worker: Worker<C, St>,
runtime_base: &Path,
runtime_run: Option<&Path>,
) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error>
@@ -752,6 +805,52 @@ impl WorkerController {
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
Self::initialize(worker, runtime_base, runtime_run)
.await?
.publish()
.await
}
async fn initialize<C, St>(
worker: Worker<C, St>,
runtime_base: &Path,
runtime_run: Option<&Path>,
) -> Result<InitializedWorkerController<C, St>, std::io::Error>
where
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
Self::initialize_inner(worker, runtime_base, runtime_run, true).await
}
pub(crate) async fn initialize_restored<St>(
worker: Worker<Box<dyn LlmClient>, St>,
run_dir: &Path,
) -> std::io::Result<InitializedWorkerController<Box<dyn LlmClient>, St>>
where
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
let parent = run_dir
.parent()
.ok_or_else(|| std::io::Error::other("run path has no parent"))?;
Self::initialize_inner(worker, parent, Some(run_dir), false).await
}
async fn initialize_inner<C, St>(
mut worker: Worker<C, St>,
runtime_base: &Path,
runtime_run: Option<&Path>,
canonical_startup: bool,
) -> Result<InitializedWorkerController<C, St>, std::io::Error>
where
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
if !canonical_startup && worker.needs_initial_session_head_materialization() {
return Err(std::io::Error::other(
"restore source requires operation-owned head recovery",
));
}
worker
.validate_subjektiv_body_publication()
.map_err(std::io::Error::other)?;
@@ -773,21 +872,13 @@ impl WorkerController {
});
let spawner_name = worker.manifest().worker.name.clone();
let loaded_registry = SpawnedWorkerRegistry::load_from_worker_state_with_reclaim(
let registry_reclaim = SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim(
runtime_dir.clone(),
worker.store().clone(),
spawner_name.clone(),
Some(worker.scope().clone()),
)
.await?;
let reclaimed_unreachable = loaded_registry.reclaimed_unreachable;
let spawned_registry = loaded_registry.registry;
if reclaimed_unreachable {
worker.push_notify(
"Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume."
.to_string(),
);
}
)?;
let spawned_registry = registry_reclaim.registry.clone();
// Hand the alerter to the Worker so internal operations (compaction,
// AGENTS.md ingestion during the first turn) can emit user-facing
@@ -878,7 +969,6 @@ impl WorkerController {
if let Some(fs_for_view) = fs_for_view.as_ref() {
shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view.clone()));
}
let artifact_store: Arc<dyn Store> = Arc::new(worker.store().clone());
let session_id = worker.session_id();
let pending_activations = worker.pending_activation_state();
@@ -903,10 +993,71 @@ impl WorkerController {
pending_activations,
};
// Start the observer only after fallible head/recovery initialization.
let command_observer = fs_for_view
let command_events = fs_for_view
.as_ref()
.and_then(|session| wire_workdir_command_events(session, &in_flight));
.map(|session| PreparedWorkdirCommandEvents::new(session, &handle.in_flight));
Ok(InitializedWorkerController {
worker,
handle,
method_rx,
shutdown_tx,
shutdown_rx,
registry_reclaim,
shutdown_after_idle,
command_events,
})
}
}
impl<C, St> InitializedWorkerController<C, St>
where
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
async fn publish(
mut self,
) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error> {
let publication = self.prepare_publication().await?;
Ok(self.start(publication))
}
pub(crate) async fn prepare_publication(
&mut self,
) -> std::io::Result<PreparedControllerPublication> {
let loaded_registry = self.registry_reclaim.apply().await?;
debug_assert!(Arc::ptr_eq(
&loaded_registry.registry,
&self.handle.spawned_registry
));
if loaded_registry.reclaimed_unreachable {
self.worker.push_notify("Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume.".to_string());
}
let command_observer = self
.command_events
.take()
.and_then(PreparedWorkdirCommandEvents::start);
Ok(PreparedControllerPublication { command_observer })
}
pub(crate) fn start(
self,
publication: PreparedControllerPublication,
) -> (WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>) {
let Self {
mut worker,
handle,
method_rx,
shutdown_tx,
shutdown_rx,
shutdown_after_idle,
..
} = self;
let PreparedControllerPublication { command_observer } = publication;
let runtime_dir = handle.runtime_dir.clone();
let shared_state = handle.shared_state.clone();
let working_event_tx = handle.working_event_tx.clone();
let spawned_registry = handle.spawned_registry.clone();
// === 5. controller_loop ===
// Clone cancel sender and notification buffer before moving worker
// into the controller task so the in-flight turn can be reached
@@ -915,6 +1066,7 @@ impl WorkerController {
let pause_tx = worker.engine_mut().pause_sender();
let notify_buffer = worker.notify_buffer_handle();
worker.finish_startup_feature_cleanup_capture();
let controller_task = tokio::spawn(controller_loop(
worker,
method_rx,
@@ -930,7 +1082,7 @@ impl WorkerController {
command_observer,
));
Ok((handle, shutdown_rx, controller_task))
(handle, shutdown_rx, controller_task)
}
}
@@ -938,15 +1090,35 @@ pub(crate) fn wire_workdir_command_events(
session: &Arc<dyn WorkdirSession>,
in_flight: &InFlightEvents,
) -> Option<tokio::task::JoinHandle<()>> {
PreparedWorkdirCommandEvents::new(session, in_flight).start()
}
struct PreparedWorkdirCommandEvents {
session: std::sync::Weak<dyn WorkdirSession>,
in_flight: InFlightEvents,
events: Option<broadcast::Receiver<WorkdirCommandEvent>>,
}
impl PreparedWorkdirCommandEvents {
fn new(session: &Arc<dyn WorkdirSession>, in_flight: &InFlightEvents) -> Self {
// Subscribe before reading command state; hold the receiver privately
// across the durable decision instead of creating a publication gap.
let events = session.subscribe_command_events();
in_flight.replace_command_snapshot(protocol_command_snapshots(session.as_ref()));
let Some(mut events) = session.subscribe_command_events() else {
return None;
};
// Keep only a weak reference in the observer task. Holding the session
// strongly here would keep its broadcast sender alive forever and prevent
// the receiver from observing closure during Worker teardown.
let session = Arc::downgrade(session);
let in_flight = in_flight.clone();
Self {
session: Arc::downgrade(session),
in_flight: in_flight.clone(),
events,
}
}
fn start(self) -> Option<tokio::task::JoinHandle<()>> {
let Self {
session,
in_flight,
events,
} = self;
let mut events = events?;
Some(tokio::spawn(async move {
loop {
match events.recv().await {
@@ -963,6 +1135,7 @@ pub(crate) fn wire_workdir_command_events(
}
}))
}
}
fn protocol_command_snapshots(session: &dyn WorkdirSession) -> Vec<ProtocolCommandSnapshot> {
session
@@ -1687,9 +1860,17 @@ where
.add_module(crate::feature::builtin::orchestration::orchestration_feature());
}
if let Some(workspace_root) = local_workspace_root.as_ref() {
if let Some(module) =
let module = if let Some(cleanup) = worker.startup_feature_cleanup_handle() {
crate::feature::mcp::discover_stdio_tool_feature_with_cleanup(
&mcp_config,
workspace_root,
&cleanup.mcp_discovery(),
)
.await?
} else {
crate::feature::mcp::discover_stdio_tool_feature(&mcp_config, workspace_root).await
{
};
if let Some(module) = module {
feature_registry = feature_registry.with_module(module);
}
}
@@ -3750,6 +3931,159 @@ mod tests {
}
}
// Component connection: real Worker/Store, initialized endpoint, paired
// protocol subscription and Controller loop. The model boundary is an
// empty non-network script; this test must not request any generation.
#[tokio::test]
async fn initialized_controller_publishes_the_same_endpoint_and_subscription() {
let root = TempDir::new().unwrap();
let manifest = manifest::WorkerManifest::from_toml(
"[worker]\nname = 'p'\n[model]\nscheme = 'anthropic'\nmodel_id = 'test'\n[engine]\n[scope]\n",
)
.unwrap();
let client = ScriptedClient::new(vec![]);
let requests = client.requests.clone();
let engine =
Engine::<_, agen::state::Mutable, crate::SessionHistoryMetadata>::new_annotated(
client,
);
let store = session_store::CombinedStore::new(
session_store::FsStore::new(root.path().join("sessions")).unwrap(),
session_store::FsWorkerStore::new(root.path().join("workers")).unwrap(),
);
let mut worker = Worker::new(
manifest,
engine,
store,
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
crate::worker::WorkerFilesystemAuthority::None,
manifest::Scope::empty(),
)
.await
.unwrap();
worker.set_system_prompt_template(
crate::SystemPromptTemplate::parse(
"default",
crate::PromptCatalogSource::builtins_only(),
)
.unwrap(),
);
let prepared = WorkerController::initialize(worker, root.path(), None)
.await
.unwrap();
let endpoint = prepared.handle.clone();
let mut streams =
crate::ipc::protocol_session::subscribe_worker_protocol_session(&endpoint);
let entries = endpoint.committed_entries();
assert!(matches!(
&streams.snapshot_event,
Event::Snapshot { greeting, state, session, .. }
if !greeting.tools.is_empty()
&& state.state == WorkerState::Idle
&& session == &session_store::public_snapshot::project_current_session_snapshot(&entries)
));
assert!(!endpoint.protocol_is_closed());
assert!(!endpoint.runtime_dir.path().join("sock").exists());
assert!(!endpoint.runtime_dir.path().join("status.json").exists());
assert!(!endpoint.runtime_dir.path().join("manifest.toml").exists());
assert!(matches!(
streams.events.try_recv(),
Err(broadcast::error::TryRecvError::Empty)
));
assert!(requests.lock().unwrap().is_empty());
let (handle, shutdown, task) = prepared.publish().await.unwrap();
assert!(handle.same_controller(&endpoint));
assert!(!handle.runtime_dir.path().join("sock").exists());
assert!(!handle.runtime_dir.path().join("status.json").exists());
assert!(!handle.runtime_dir.path().join("manifest.toml").exists());
handle.send(Method::ListPendingSubmissions).await.unwrap();
let pending = tokio::time::timeout(Duration::from_secs(5), async {
loop {
if let Event::PendingSubmissionsChanged { pending } =
streams.events.recv().await.unwrap()
{
break pending;
}
}
})
.await
.unwrap();
assert_eq!(pending, protocol::PendingSubmissionsSnapshot::default());
assert_eq!(
serde_json::to_value(handle.committed_entries()).unwrap(),
serde_json::to_value(entries).unwrap(),
);
handle
.send(Method::Shutdown {
command: WorkerCommandEnvelope { command_id: 1 },
})
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(5), shutdown)
.await
.unwrap()
.unwrap();
tokio::time::timeout(Duration::from_secs(5), task)
.await
.unwrap()
.unwrap();
assert!(endpoint.protocol_is_closed());
assert!(!endpoint.runtime_dir.path().join("sock").exists());
assert!(requests.lock().unwrap().is_empty());
}
#[tokio::test]
async fn tool_initialization_failure_does_not_reclaim_saved_children() {
let root = TempDir::new().unwrap();
let mut manifest = manifest::WorkerManifest::from_toml(
"[worker]\nname = 'p'\n[model]\nscheme = 'anthropic'\nmodel_id = 'test'\n[engine]\n[scope]\n",
).unwrap();
manifest.feature.ticket.enabled = true;
let client = ScriptedClient::new(vec![]);
let requests = client.requests.clone();
let store = session_store::CombinedStore::new(
session_store::FsStore::new(root.path().join("sessions")).unwrap(),
session_store::FsWorkerStore::new(root.path().join("workers")).unwrap(),
);
// New feature activation is refused by the real unavailable host;
// legacy child metadata must not be consumed before that refusal.
let worker = Worker::new(
manifest,
Engine::<_, agen::state::Mutable, crate::SessionHistoryMetadata>::new_annotated(
client,
),
store.clone(),
crate::worker::WorkerWorkspaceContext::no_workspace(),
crate::worker::WorkerFilesystemAuthority::None,
manifest::Scope::empty(),
)
.await
.unwrap();
let mut metadata = session_store::WorkerMetadata::new(
"p",
Some(session_store::WorkerActiveSegmentRef {
session_id: worker.session_id(),
segment_id: Some(worker.segment_id()),
}),
);
metadata
.spawned_children
.push(session_store::WorkerSpawnedChild {
worker_name: "legacy".into(),
socket_path: root.path().join("legacy.sock"),
callback_address: root.path().join("p/sock"),
scope_delegated: Vec::new(),
});
store.write(&metadata).unwrap();
let before = store.read_by_name("p").unwrap();
let result = WorkerController::initialize(worker, root.path(), None).await;
assert!(result.is_err());
assert_eq!(store.read_by_name("p").unwrap(), before);
assert!(!root.path().join("p/spawned_workers.json").exists());
assert!(requests.lock().unwrap().is_empty());
}
#[derive(Debug)]
struct DriveWorkspaceHost {
available: bool,
@@ -4077,10 +4411,19 @@ mod tests {
assert!(feature_install < materialize_gate);
assert!(materialize_gate < materialize);
assert!(materialize < exposure);
let observer = startup.find("let command_observer = fs_for_view").unwrap();
let observer_preparation = startup
.find("PreparedWorkdirCommandEvents::new(session")
.unwrap();
let recover = startup.find(".recover_unfinished_compaction()").unwrap();
let reclaim = startup
.find("self.registry_reclaim.apply().await?")
.unwrap();
let observer_start = startup
.find(".and_then(PreparedWorkdirCommandEvents::start)")
.unwrap();
let loop_start = startup.find("tokio::spawn(controller_loop(").unwrap();
assert!(recover < observer && observer < loop_start);
assert!(recover < observer_preparation && observer_preparation < reclaim);
assert!(reclaim < observer_start && observer_start < loop_start);
assert!(
startup[materialize_gate..materialize]
.contains("needs_initial_session_head_materialization")
+273 -35
View File
@@ -8,6 +8,7 @@
//! the regular registry decides Tool-name collisions.
use std::collections::BTreeMap;
use std::io;
use std::path::Path;
use std::sync::Arc;
@@ -25,6 +26,7 @@ use mcp::stdio::{
};
use serde::Serialize;
use serde_json::{Map, Value};
use tokio::sync::Mutex;
use super::{
FeatureDescriptor, FeatureDiagnostic, FeatureInstallContext, FeatureInstallError,
@@ -52,8 +54,88 @@ const MAX_LIST_ITEMS: usize = 128;
const MAX_RESOURCE_CONTENTS: usize = 16;
const MAX_PROMPT_MESSAGES: usize = 32;
/// Owns discovery clients until process reaping is confirmed. Retain a clone in
/// startup cleanup *before* awaiting discovery. Stop/cancel discovery before
/// calling `shutdown`; the shared lock serializes discovery and cleanup, not
/// cancellation of an in-flight discovery task.
///
/// Error or cancellation leaves the actual clients here for another shutdown
/// attempt. Dropping the final owner only provides best-effort process killing.
#[derive(Clone, Default)]
pub struct McpDiscoveryCleanup {
clients: Arc<Mutex<Vec<McpStdioClient>>>,
}
impl McpDiscoveryCleanup {
pub fn new() -> Self {
Self::default()
}
/// Try every retained client once, removing only confirmed shutdowns. An
/// error is not a cleanup receipt; retry this same owner, not discovery.
pub async fn shutdown(&self) -> io::Result<()> {
let mut clients = self.clients.lock().await;
let mut first_error = None;
let mut index = 0;
while index < clients.len() {
match clients[index].shutdown().await {
Ok(_) => {
clients.remove(index);
}
Err(err) => {
first_error.get_or_insert_with(|| discovery_shutdown_error(&err));
index += 1;
}
}
}
match first_error {
Some(err) => Err(err),
None => Ok(()),
}
}
}
fn discovery_shutdown_error(err: &McpClientError) -> io::Error {
io::Error::other(bounded_diagnostic(format!(
"MCP discovery shutdown unconfirmed; client retained for cleanup retry: {err}"
)))
}
/// Restore preparation must fail if discovery cannot confirm process shutdown.
/// The caller retains `cleanup` before this future is polled, including during
/// initialization. Ordinary discovery failures remain isolated contributions
/// only after their client has been reaped.
pub async fn discover_stdio_tool_feature_with_cleanup(
config: &McpConfig,
workspace_root: &Path,
cleanup: &McpDiscoveryCleanup,
) -> Result<Option<McpStdioToolFeature>, io::Error> {
if config.stdio_servers.is_empty() {
return Ok(None);
}
let mut feature = McpStdioToolFeature::new();
for server in &config.stdio_servers {
match resolve_stdio_server(server, workspace_root, None) {
Ok(spec) => {
feature.add_contribution(
discover_server_tools_with_cleanup(spec, cleanup, McpStdioLimits::default())
.await?,
);
}
Err(err) => {
feature.add_diagnostic(FeatureDiagnostic::error(bounded_diagnostic(format!(
"failed to resolve MCP stdio server `{}`: {err}",
server.name
))))
}
}
}
Ok(Some(feature))
}
/// Discover enabled MCP stdio server tools and return a single feature module
/// containing startup contributions for normal ToolRegistry installation.
/// Callers needing retryable startup cleanup must use the cleanup-aware API.
pub async fn discover_stdio_tool_feature(
config: &McpConfig,
workspace_root: &Path,
@@ -81,21 +163,66 @@ pub async fn discover_stdio_tool_feature(
}
async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderContribution {
let declaration = provider_declaration(&spec.name, None);
let cleanup = McpDiscoveryCleanup::new();
match discover_server_tools_with_cleanup(spec, &cleanup, McpStdioLimits::default()).await {
Ok(contribution) => contribution,
Err(err) => {
ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string()))
}
}
}
async fn discover_server_tools_with_cleanup(
spec: McpStdioServerSpec,
cleanup: &McpDiscoveryCleanup,
limits: McpStdioLimits,
) -> io::Result<ProtocolProviderContribution> {
let declaration = provider_declaration(&spec.name, None);
let mut clients = cleanup.clients.lock().await;
if !clients.is_empty() {
return Err(io::Error::other(
"MCP discovery cleanup is pending; shut down the retained owner before retrying discovery",
));
}
let client = match McpStdioClient::spawn_uninitialized(spec.clone(), limits) {
Ok(client) => client,
Err(err) => {
// The synchronous spawn API only errors before a process exists;
// pipe/handshake failures retain a client and use the shutdown path.
return Ok(ProtocolProviderContribution::failed(
declaration,
bounded_diagnostic(err.to_string()),
));
}
};
// Spawn is synchronous: the owner holds the client before the first await
// that can leave a running process on initialization failure/cancellation.
clients.push(client);
let client = clients.last_mut().expect("retained discovery client");
let contribution = match client.initialize_with_timeout().await {
Ok(()) => discover_connected_server_tools(spec, client).await,
Err(err) => {
ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string()))
}
};
client
.shutdown()
.await
.map_err(|err| discovery_shutdown_error(&err))?;
drop(clients.pop());
Ok(contribution)
}
async fn discover_connected_server_tools(
spec: McpStdioServerSpec,
client: &mut McpStdioClient,
) -> ProtocolProviderContribution {
let declaration = provider_declaration(&spec.name, None);
let mut contribution = ProtocolProviderContribution::ready(declaration.clone());
let server_namespace = sanitize_segment(&spec.name);
let execution_spec = spec.clone();
let mut client = match McpStdioClient::connect(spec, McpStdioLimits::default()).await {
Ok(client) => client,
Err(err) => {
return ProtocolProviderContribution::failed(
declaration,
bounded_diagnostic(err.to_string()),
);
}
};
let (server_version, has_tools, has_resources, has_prompts) = client
.initialize_result()
.map(|result| {
@@ -133,18 +260,10 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
{
Ok(list) => list,
Err(err) => {
let mut failed = ProtocolProviderContribution::failed(
return ProtocolProviderContribution::failed(
declaration,
bounded_diagnostic(err.to_string()),
);
if let Err(shutdown_err) = client.shutdown().await {
failed = failed.with_diagnostic(FeatureDiagnostic::warning(
bounded_diagnostic(format!(
"MCP server shutdown after discovery failure failed: {shutdown_err}"
)),
));
}
return failed;
}
};
let list_changes = client.snapshot_list_changes().await;
@@ -166,20 +285,12 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
{
Ok(list) => list,
Err(err) => {
let mut failed = ProtocolProviderContribution::failed(
return ProtocolProviderContribution::failed(
declaration,
bounded_diagnostic(format!(
"MCP server sent notifications/tools/list_changed during tool discovery and refresh failed: {err}"
)),
);
if let Err(shutdown_err) = client.shutdown().await {
failed = failed.with_diagnostic(FeatureDiagnostic::warning(
bounded_diagnostic(format!(
"MCP server shutdown after discovery refresh failure failed: {shutdown_err}"
)),
));
}
return failed;
}
};
let refresh_changes = client.snapshot_list_changes().await;
@@ -197,13 +308,6 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
} else {
None
};
let shutdown_result = client.shutdown().await;
if let Err(err) = shutdown_result {
contribution =
contribution.with_diagnostic(FeatureDiagnostic::warning(bounded_diagnostic(format!(
"MCP server shutdown after capability discovery failed: {err}"
))));
}
if let Some(list) = list {
contribution = normalize_listed_tools(
@@ -1819,6 +1923,140 @@ read exit_notification || true
McpStdioServerSpec::new("shell-mock", "/bin/sh").args(["-c".to_string(), script])
}
#[cfg(unix)]
fn discovery_config(spec: McpStdioServerSpec) -> McpConfig {
McpConfig {
stdio_servers: vec![manifest::McpStdioServerConfig {
name: spec.name,
command: spec.command,
args: spec.args,
cwd: None,
env: Default::default(),
}],
}
}
#[cfg(unix)]
#[tokio::test]
async fn cancelled_discovery_retains_initializing_client_in_cleanup_clone() {
use std::future::{Future, poll_fn};
use std::task::Poll;
use std::time::Duration;
let config = discovery_config(
McpStdioServerSpec::new("initializing", "/bin/sh")
.args(["-c", "while read line; do :; done"]),
);
let cleanup = McpDiscoveryCleanup::new();
let retained = cleanup.clone();
let mut discovery = Box::pin(discover_stdio_tool_feature_with_cleanup(
&config,
Path::new("."),
&cleanup,
));
poll_fn(|cx| {
assert!(discovery.as_mut().poll(cx).is_pending());
Poll::Ready(())
})
.await;
drop(discovery);
drop(cleanup);
assert_eq!(retained.clients.lock().await.len(), 1);
let retry_error =
discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &retained)
.await
.err()
.expect("discovery must not replace an unreaped client");
assert!(retry_error.to_string().contains("cleanup is pending"));
assert_eq!(retained.clients.lock().await.len(), 1);
tokio::time::timeout(Duration::from_secs(3), retained.shutdown())
.await
.expect("cleanup bounded")
.expect("initializing process reaped");
assert!(retained.clients.lock().await.is_empty());
retained
.shutdown()
.await
.expect("confirmed cleanup repeatable");
}
#[cfg(unix)]
#[tokio::test]
async fn cancelled_owner_shutdown_keeps_client_for_retry() {
use std::future::{Future, poll_fn};
use std::task::Poll;
use std::time::Duration;
let cleanup = McpDiscoveryCleanup::new();
cleanup.clients.lock().await.push(
McpStdioClient::spawn_uninitialized(
McpStdioServerSpec::new("shutdown-pending", "/bin/sh")
.args(["-c", "while :; do :; done"]),
McpStdioLimits {
shutdown_timeout: Duration::from_millis(50),
kill_timeout: Duration::from_millis(100),
..Default::default()
},
)
.expect("local process spawned"),
);
let mut shutdown = Box::pin(cleanup.shutdown());
poll_fn(|cx| {
assert!(shutdown.as_mut().poll(cx).is_pending());
Poll::Ready(())
})
.await;
drop(shutdown);
assert_eq!(cleanup.clients.lock().await.len(), 1);
tokio::time::timeout(Duration::from_secs(2), cleanup.shutdown())
.await
.expect("retry bounded")
.expect("retained client shutdown");
assert!(cleanup.clients.lock().await.is_empty());
}
#[cfg(unix)]
#[tokio::test]
async fn cleanup_aware_discovery_reaps_success_and_initialize_failure() {
let cleanup = McpDiscoveryCleanup::new();
let config = discovery_config(shell_capability_server(r#"{"resources":{}}"#));
let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup)
.await
.expect("discovery cleanup confirmed")
.expect("configured feature");
assert!(
feature.contributions[0]
.tools
.iter()
.any(|tool| { tool.name() == "Mcp_shell_mock_resources_read" })
);
assert!(cleanup.clients.lock().await.is_empty());
let spec = McpStdioServerSpec::new("init-rejected", "/bin/sh").args([
"-c",
r#"read init || exit 1
printf '%s\n' '{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"initialization rejected"}}'
while read line; do :; done
"#,
]);
let config = discovery_config(spec);
let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup)
.await
.expect("failure isolated only after cleanup")
.expect("failed contribution preserved");
assert!(feature.contributions[0].tools.is_empty());
assert!(
feature.contributions[0]
.diagnostics
.iter()
.any(|diagnostic| {
diagnostic.severity == FeatureDiagnosticSeverity::Error
&& diagnostic.message.contains("initialization rejected")
})
);
assert!(cleanup.clients.lock().await.is_empty());
}
#[test]
fn mcp_resource_prompt_operation_tools_are_namespaced_and_untrusted() {
let declaration = provider_declaration("demo", Some("1.0.0"));
+9 -5
View File
@@ -19,6 +19,9 @@ pub mod model_client;
mod paste_artifact_tool;
pub mod profile_manifest;
pub mod prompt;
#[cfg(test)]
#[path = "../../../tests/support/restore_environment.rs"]
mod restore_test_environment;
pub mod runtime;
pub mod segment_log_sink;
mod session_capture;
@@ -70,9 +73,10 @@ pub use session_history::{
pub use shared_state::WorkerSharedState;
pub use worker::{
LocalWorkingDirectory, SubjektivSessionAttributionLifecycle, Worker, WorkerError,
WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, WorkspaceBinaryRequest,
WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse,
WorkspaceServerOperation, WorkspaceWorkerDiscoveryRequest, apply_worker_manifest,
marker_workspace_client, unavailable_workspace_client,
WorkerFilesystemAuthority, WorkerRunResult, WorkerStartupFeatureCleanup,
WorkerWorkspaceContext, WorkspaceBinaryRequest, WorkspaceBinaryResponse, WorkspaceClient,
WorkspaceClientError, WorkspaceId, WorkspaceIdError, WorkspacePromptCatalogResolution,
WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse, WorkspaceServerOperation,
WorkspaceWorkerDiscoveryRequest, apply_worker_manifest, marker_workspace_client,
unavailable_workspace_client,
};
@@ -21,8 +21,9 @@ pub use error::ScopeLockError;
#[cfg(test)]
pub(crate) use lifecycle::install_top_level_at_for_test;
pub use lifecycle::{
ScopeAllocationGuard, SegmentActivationCommitError, SegmentActivationGuard, SegmentLockInfo,
adopt_allocation, install_top_level, install_top_level_with_deny, lookup_segment,
ScopeAllocationCleanup, ScopeAllocationGuard, SegmentActivationCommitError,
SegmentActivationGuard, SegmentLockInfo, adopt_allocation, install_top_level,
install_top_level_with_deny, lookup_segment,
};
pub use mutate::{
delegate_scope, reclaim_delegated_scope, reclaim_stale, reclaim_stale_with, register_worker,
@@ -11,6 +11,14 @@ use session_store::SegmentId;
pub enum ScopeLockError {
#[error("I/O error on workers.json: {0}")]
Io(#[from] io::Error),
#[error("worker `{worker_name}` allocation still has a live guard")]
AllocationGuardAttached { worker_name: String },
#[error("allocation acquisition was published but its save failed: {source}")]
AcquisitionSaveFailed {
#[source]
source: io::Error,
cleanup: super::lifecycle::ScopeAllocationCleanup,
},
#[error("worker `{0}` is already allocated")]
DuplicateWorkerName(String),
#[error("requested scope `{}` conflicts with worker allocation `{competitor}` rule `{}`", .rule.target.display(), .competitor_rule.target.display())]
@@ -46,3 +54,15 @@ pub enum ScopeLockError {
actual: Option<SegmentId>,
},
}
impl ScopeLockError {
/// Retain the release authority for an owned acquisition whose table rename
/// succeeded before save returned an error. Refusals and unpublished saves
/// do not confer ownership of any preexisting allocation.
pub fn allocation_cleanup_handle(&self) -> Option<super::lifecycle::ScopeAllocationCleanup> {
match self {
Self::AcquisitionSaveFailed { cleanup, .. } => Some(cleanup.clone()),
_ => None,
}
}
}
@@ -3,6 +3,7 @@
//! up on drop.
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use manifest::ScopeRule;
use session_store::SegmentId;
@@ -11,13 +12,82 @@ use super::error::ScopeLockError;
use super::mutate::release_worker;
use super::table::{LockFileGuard, default_allocation_path};
/// Owned allocation: on drop, opens the lock file and releases this
/// Worker's entry. The guard keeps only the name + lock-file path; it
/// does not hold the `flock` for the Worker's lifetime.
/// Owned allocation shared with any retained cleanup handles. Only the final
/// owner's drop attempts best-effort release; no `flock` is held for its lifetime.
#[derive(Debug)]
pub struct ScopeAllocationGuard {
owner: Arc<ScopeAllocationOwner>,
}
/// Retains an allocation's release authority even after its Worker is consumed
/// or dropped. A successful controller transfer should drop this handle without
/// explicitly releasing the live Worker's allocation.
#[derive(Debug, Clone)]
pub struct ScopeAllocationCleanup {
owner: Arc<ScopeAllocationOwner>,
}
#[derive(Debug)]
struct ScopeAllocationOwner {
worker_name: String,
lock_path: PathBuf,
state: Mutex<ScopeAllocationOwnerState>,
}
#[derive(Debug)]
struct ScopeAllocationOwnerState {
guard_attached: bool,
released: bool,
}
impl ScopeAllocationOwner {
fn release(&self) -> Result<(), ScopeLockError> {
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
if state.guard_attached {
return Err(ScopeLockError::AllocationGuardAttached {
worker_name: self.worker_name.clone(),
});
}
if state.released {
return Ok(());
}
let mut guard = LockFileGuard::open(&self.lock_path)?;
if guard.data().find(&self.worker_name).is_some() {
release_worker(&mut guard, &self.worker_name)?;
} else {
// A previous save may have removed the entry but failed after rename.
// Complete the durable save before reporting release as confirmed.
guard.save()?;
}
state.released = true;
Ok(())
}
}
impl Drop for ScopeAllocationOwner {
fn drop(&mut self) {
let _ = self.release();
}
}
impl Drop for ScopeAllocationGuard {
fn drop(&mut self) {
self.owner
.state
.lock()
.unwrap_or_else(|error| error.into_inner())
.guard_attached = false;
}
}
impl ScopeAllocationCleanup {
/// Release only after the factory has returned an error and no live
/// Controller can still use this allocation. An attached guard rejects
/// release. Success is idempotent across all clones; failure retains the
/// authority for an explicit retry.
pub fn release(&self) -> Result<(), ScopeLockError> {
self.owner.release()
}
}
/// Exclusive authority to activate a replacement Segment for one allocated Worker.
@@ -56,12 +126,58 @@ impl SegmentActivationCommitError {
}
impl ScopeAllocationGuard {
fn new(worker_name: String, lock_path: PathBuf) -> Self {
Self {
owner: Arc::new(ScopeAllocationOwner {
worker_name,
lock_path,
state: Mutex::new(ScopeAllocationOwnerState {
guard_attached: true,
released: false,
}),
}),
}
}
fn commit_acquisition(self, table: &mut LockFileGuard) -> Result<Self, ScopeLockError> {
let mut published = false;
match table.save_with_publication(&mut published) {
Ok(()) => Ok(self),
Err(source) => {
let cleanup = if published {
Some(self.cleanup_handle())
} else {
// A staged entry is not ours until rename. Disarm before
// detaching so Drop cannot remove a preexisting allocation
// (or reopen the table lock still held by this caller).
self.owner
.state
.lock()
.unwrap_or_else(|error| error.into_inner())
.released = true;
None
};
drop(self);
match cleanup {
Some(cleanup) => Err(ScopeLockError::AcquisitionSaveFailed { source, cleanup }),
None => Err(ScopeLockError::Io(source)),
}
}
}
}
pub fn worker_name(&self) -> &str {
&self.worker_name
&self.owner.worker_name
}
pub fn lock_path(&self) -> &Path {
&self.lock_path
&self.owner.lock_path
}
pub fn cleanup_handle(&self) -> ScopeAllocationCleanup {
ScopeAllocationCleanup {
owner: Arc::clone(&self.owner),
}
}
/// Lock the machine-wide table and validate a Segment replacement before
@@ -71,24 +187,25 @@ impl ScopeAllocationGuard {
expected: SegmentId,
replacement: SegmentId,
) -> Result<SegmentActivationGuard, ScopeLockError> {
let guard = LockFileGuard::open(&self.lock_path)?;
let guard = LockFileGuard::open(self.lock_path())?;
let actual = guard
.data()
.allocations
.iter()
.find(|allocation| allocation.worker_name == self.worker_name)
.find(|allocation| allocation.worker_name == self.worker_name())
.and_then(|allocation| allocation.segment_id);
if actual != Some(expected) {
return Err(ScopeLockError::SegmentChanged {
worker_name: self.worker_name.clone(),
worker_name: self.worker_name().to_owned(),
expected,
actual,
});
}
if let Some(existing) = guard.data().allocations.iter().find(|allocation| {
allocation.segment_id == Some(replacement) && allocation.worker_name != self.worker_name
allocation.segment_id == Some(replacement)
&& allocation.worker_name != self.worker_name()
}) {
return Err(ScopeLockError::SegmentConflict {
segment_id: replacement,
@@ -99,7 +216,7 @@ impl ScopeAllocationGuard {
Ok(SegmentActivationGuard {
guard,
worker_name: self.worker_name.clone(),
worker_name: self.worker_name().to_owned(),
expected,
replacement,
})
@@ -145,14 +262,6 @@ impl SegmentActivationGuard {
}
}
impl Drop for ScopeAllocationGuard {
fn drop(&mut self) {
if let Ok(mut guard) = LockFileGuard::open(&self.lock_path) {
let _ = release_worker(&mut guard, &self.worker_name);
}
}
}
/// Open the default lock file, register a top-level Worker, and return a
/// guard that will release the allocation on drop.
pub fn install_top_level(
@@ -185,7 +294,7 @@ pub fn install_top_level_with_deny(
) -> Result<ScopeAllocationGuard, ScopeLockError> {
let lock_path = default_allocation_path()?;
let mut guard = LockFileGuard::open(&lock_path)?;
super::mutate::register_worker_with_deny(
super::mutate::prepare_worker_with_deny(
&mut guard,
worker_name.clone(),
pid,
@@ -194,10 +303,7 @@ pub fn install_top_level_with_deny(
scope_deny,
segment_id,
)?;
Ok(ScopeAllocationGuard {
worker_name,
lock_path,
})
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
}
#[cfg(test)]
@@ -210,7 +316,7 @@ pub(crate) fn install_top_level_at_for_test(
segment_id: SegmentId,
) -> Result<ScopeAllocationGuard, ScopeLockError> {
let mut guard = LockFileGuard::open(&lock_path)?;
super::mutate::register_worker_with_deny(
super::mutate::prepare_worker_with_deny(
&mut guard,
worker_name.clone(),
pid,
@@ -219,10 +325,7 @@ pub(crate) fn install_top_level_at_for_test(
Vec::new(),
segment_id,
)?;
Ok(ScopeAllocationGuard {
worker_name,
lock_path,
})
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
}
/// Take ownership of an existing allocation that was pre-registered by
@@ -240,6 +343,15 @@ pub fn adopt_allocation(
segment_id: SegmentId,
) -> Result<ScopeAllocationGuard, ScopeLockError> {
let lock_path = default_allocation_path()?;
adopt_allocation_at(lock_path, worker_name, new_pid, segment_id)
}
fn adopt_allocation_at(
lock_path: PathBuf,
worker_name: String,
new_pid: u32,
segment_id: SegmentId,
) -> Result<ScopeAllocationGuard, ScopeLockError> {
let mut guard = LockFileGuard::open(&lock_path)?;
let alloc = guard
.data_mut()
@@ -247,11 +359,7 @@ pub fn adopt_allocation(
.ok_or_else(|| ScopeLockError::UnknownWorker(worker_name.clone()))?;
alloc.pid = new_pid;
alloc.segment_id = Some(segment_id);
guard.save()?;
Ok(ScopeAllocationGuard {
worker_name,
lock_path,
})
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
}
/// Information about a Worker that currently holds an allocation for a
@@ -329,6 +437,388 @@ mod tests {
}
}
#[test]
fn retained_cleanup_handle_defers_release_until_final_owner_drop() {
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let guard = install_top_level_at_for_test(
lock_path.clone(),
"retained".into(),
std::process::id(),
sock("retained"),
Vec::new(),
sid(),
)
.unwrap();
let cleanup = guard.cleanup_handle();
let retained = cleanup.clone();
drop(guard);
drop(cleanup);
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("retained").is_some());
}
drop(retained);
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("retained").is_none());
}
#[test]
fn dropping_cleanup_handle_after_transfer_preserves_live_guard() {
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let segment = sid();
let guard = install_top_level_at_for_test(
lock_path.clone(),
"transferred".into(),
std::process::id(),
sock("transferred"),
Vec::new(),
segment,
)
.unwrap();
drop(guard.cleanup_handle());
// A successful controller transfer leaves the guard usable, including
// the existing activation protocol, without retaining a cleanup handle.
let replacement = sid();
guard
.begin_segment_activation(segment, replacement)
.unwrap()
.commit()
.unwrap();
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert_eq!(
table.data().find("transferred").unwrap().segment_id,
Some(replacement)
);
}
drop(guard);
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("transferred").is_none());
}
#[test]
fn cleanup_rejects_live_guard_then_releases_only_detached_allocation() {
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let guard = install_top_level_at_for_test(
lock_path.clone(),
"live".into(),
std::process::id(),
sock("live"),
Vec::new(),
sid(),
)
.unwrap();
let _competitor = install_top_level_at_for_test(
lock_path.clone(),
"competitor".into(),
std::process::id(),
sock("competitor"),
Vec::new(),
sid(),
)
.unwrap();
let cleanup = guard.cleanup_handle();
assert!(matches!(
cleanup.release(),
Err(ScopeLockError::AllocationGuardAttached { worker_name }) if worker_name == "live"
));
drop(cleanup.clone());
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("live").is_some());
assert!(table.data().find("competitor").is_some());
}
drop(guard);
cleanup.release().unwrap();
cleanup.clone().release().unwrap();
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("live").is_none());
assert!(table.data().find("competitor").is_some());
}
#[test]
fn published_install_failure_retains_cleanup_across_error_drop_and_retry() {
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let _competitor = install_top_level_at_for_test(
lock_path.clone(),
"competitor".into(),
std::process::id(),
sock("competitor"),
Vec::new(),
sid(),
)
.unwrap();
let segment = sid();
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
let error = install_top_level_at_for_test(
lock_path.clone(),
"failed".into(),
std::process::id(),
sock("failed"),
Vec::new(),
segment,
)
.unwrap_err();
assert!(matches!(
&error,
ScopeLockError::AcquisitionSaveFailed { .. }
));
let cleanup = error.allocation_cleanup_handle().unwrap();
let retry = error.allocation_cleanup_handle().unwrap();
drop(error);
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert_eq!(
table.data().find("failed").unwrap().segment_id,
Some(segment)
);
assert!(table.data().find("competitor").is_some());
}
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
drop(cleanup);
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("failed").is_some());
}
retry.release().unwrap();
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("failed").is_none());
assert!(table.data().find("competitor").is_some());
}
#[test]
fn refused_install_has_no_cleanup_authority_over_existing_allocation() {
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let segment = sid();
let _competitor = install_top_level_at_for_test(
lock_path.clone(),
"competitor".into(),
std::process::id(),
sock("competitor"),
vec![write_rule("/shared", true)],
segment,
)
.unwrap();
let cases = [
("competitor", sid(), Vec::new()),
("other", segment, Vec::new()),
("other", sid(), vec![write_rule("/shared", true)]),
];
for (name, requested_segment, scope) in cases {
let error = install_top_level_at_for_test(
lock_path.clone(),
name.into(),
std::process::id(),
sock("refused"),
scope,
requested_segment,
)
.unwrap_err();
assert!(matches!(
&error,
ScopeLockError::DuplicateWorkerName(_)
| ScopeLockError::SegmentConflict { .. }
| ScopeLockError::WriteConflict { .. }
));
assert!(error.allocation_cleanup_handle().is_none());
drop(error);
let table = LockFileGuard::open(&lock_path).unwrap();
assert_eq!(table.data().allocations.len(), 1);
assert_eq!(
table.data().find("competitor").unwrap().socket,
sock("competitor")
);
assert_eq!(
table.data().find("competitor").unwrap().segment_id,
Some(segment)
);
}
}
#[test]
fn unpublished_install_failure_has_no_cleanup_authority() {
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let _competitor = install_top_level_at_for_test(
lock_path.clone(),
"competitor".into(),
std::process::id(),
sock("competitor"),
Vec::new(),
sid(),
)
.unwrap();
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
let error = install_top_level_at_for_test(
lock_path.clone(),
"failed".into(),
std::process::id(),
sock("failed"),
Vec::new(),
sid(),
)
.unwrap_err();
assert!(matches!(&error, ScopeLockError::Io(_)));
assert!(error.allocation_cleanup_handle().is_none());
drop(error);
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("failed").is_none());
assert!(table.data().find("competitor").is_some());
}
#[test]
fn published_adoption_failure_retains_cleanup_for_retry() {
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
{
let mut table = LockFileGuard::open(&lock_path).unwrap();
delegate_placeholder(&mut table, "child", std::process::id());
delegate_placeholder(&mut table, "competitor", std::process::id());
}
let segment = sid();
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
let error = adopt_allocation_at(
lock_path.clone(),
"child".into(),
std::process::id(),
segment,
)
.unwrap_err();
let cleanup = error.allocation_cleanup_handle().unwrap();
drop(error);
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert_eq!(
table.data().find("child").unwrap().segment_id,
Some(segment)
);
}
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("child").is_none());
assert!(table.data().find("competitor").is_some());
}
cleanup.release().unwrap();
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("child").is_none());
assert!(table.data().find("competitor").is_some());
}
#[test]
fn unpublished_adoption_failure_preserves_placeholder_without_cleanup_authority() {
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let placeholder_pid = std::process::id();
{
let mut table = LockFileGuard::open(&lock_path).unwrap();
delegate_placeholder(&mut table, "child", placeholder_pid);
}
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
let error = adopt_allocation_at(
lock_path.clone(),
"child".into(),
placeholder_pid.wrapping_add(1),
sid(),
)
.unwrap_err();
assert!(matches!(&error, ScopeLockError::Io(_)));
assert!(error.allocation_cleanup_handle().is_none());
drop(error);
let table = LockFileGuard::open(&lock_path).unwrap();
let allocation = table.data().find("child").unwrap();
assert_eq!(allocation.pid, placeholder_pid);
assert_eq!(allocation.segment_id, None);
}
#[test]
fn failed_cleanup_release_retains_authority_for_retry() {
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
for point in [SaveFaultPoint::BeforeRename, SaveFaultPoint::AfterRename] {
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let guard = install_top_level_at_for_test(
lock_path.clone(),
"failed".into(),
std::process::id(),
sock("failed"),
vec![read_rule("/shared", true)],
sid(),
)
.unwrap();
let _other = install_top_level_at_for_test(
lock_path.clone(),
"other".into(),
std::process::id(),
sock("other"),
vec![read_rule("/shared", true)],
sid(),
)
.unwrap();
let cleanup = guard.cleanup_handle();
let retry = cleanup.clone();
drop(guard);
fail_next_save_at_for_test(&lock_path, point);
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
drop(cleanup);
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert_eq!(
table.data().find("failed").is_some(),
point == SaveFaultPoint::BeforeRename
);
assert!(table.data().find("other").is_some());
}
retry.release().unwrap();
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("failed").is_none());
assert!(table.data().find("other").is_some());
}
// Success disarms every clone and final Drop, even if the name is
// subsequently allocated to a different Worker.
let _replacement = install_top_level_at_for_test(
lock_path.clone(),
"failed".into(),
std::process::id(),
sock("replacement"),
vec![read_rule("/shared", true)],
sid(),
)
.unwrap();
retry.clone().release().unwrap();
drop(retry);
let table = LockFileGuard::open(&lock_path).unwrap();
assert_eq!(
table.data().find("failed").unwrap().socket,
sock("replacement")
);
assert!(table.data().find("other").is_some());
}
}
#[test]
fn adopt_allocation_rewrites_pid_and_releases_on_drop() {
let dir = TempDir::new().unwrap();
@@ -358,6 +848,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let _sandbox = RuntimeDirSandbox::new(dir.path());
let err = adopt_allocation("ghost".into(), 42, sid()).unwrap_err();
assert!(err.allocation_cleanup_handle().is_none());
assert!(matches!(err, ScopeLockError::UnknownWorker(ref n) if n == "ghost"));
}
@@ -57,6 +57,30 @@ pub fn register_worker_with_deny(
scope_allow: Vec<ScopeRule>,
scope_deny: Vec<ScopeRule>,
segment_id: SegmentId,
) -> Result<(), ScopeLockError> {
prepare_worker_with_deny(
guard,
worker_name,
pid,
socket,
scope_allow,
scope_deny,
segment_id,
)?;
guard.save()?;
Ok(())
}
/// Admit and stage a registration without publishing it. Owned installation
/// constructs its cleanup authority after admission and before committing.
pub(super) fn prepare_worker_with_deny(
guard: &mut LockFileGuard,
worker_name: String,
pid: u32,
socket: PathBuf,
scope_allow: Vec<ScopeRule>,
scope_deny: Vec<ScopeRule>,
segment_id: SegmentId,
) -> Result<(), ScopeLockError> {
reclaim_stale(guard);
if guard.data().find(&worker_name).is_some() {
@@ -101,7 +125,6 @@ pub fn register_worker_with_deny(
delegated_from: None,
segment_id: Some(segment_id),
});
guard.save()?;
Ok(())
}
@@ -246,6 +246,14 @@ impl LockFileGuard {
/// Persist with atomic replacement while the separate allocation lock stays
/// held. A crash exposes either the old complete table or the new one.
pub fn save(&mut self) -> io::Result<()> {
let mut published = false;
self.save_with_publication(&mut published)
}
/// Report this save's rename while the caller still holds the table lock.
/// In-memory mutation alone is not evidence that an acquisition took place.
pub(super) fn save_with_publication(&mut self, published: &mut bool) -> io::Result<()> {
*published = false;
let json = serde_json::to_vec_pretty(&self.data).map_err(io::Error::other)?;
let mut temp_os = self.data_path.as_os_str().to_owned();
temp_os.push(".tmp");
@@ -265,6 +273,7 @@ impl LockFileGuard {
));
}
fs::rename(&temp_path, &self.data_path)?;
*published = true;
#[cfg(test)]
if take_save_fault(&self.data_path, SaveFaultPoint::AfterRename) {
return Err(io::Error::other(
+369 -41
View File
@@ -340,6 +340,84 @@ pub struct SpawnedWorkerRegistryLoad {
pub reclaimed_unreachable: bool,
}
/// Read-only restore preparation using the registry that will be published by the Host.
/// The Host must retain its lifecycle exclusion from preparation through application;
/// the metadata reread detects source drift, but is not an atomic store transaction.
/// Dropping an unapplied plan does not reclaim legacy child state.
pub(crate) struct PreparedSpawnedWorkerRegistryLoad<St> {
pub(crate) registry: Arc<SpawnedWorkerRegistry>,
/// Projected outcome, not evidence that reclamation has already happened.
pub(crate) reclaimed_unreachable: bool,
runtime_dir: Arc<RuntimeDir>,
store: St,
worker_name: String,
parent_scope: Option<SharedScope>,
persisted_children: Vec<WorkerSpawnedChild>,
valid_records: Vec<SpawnedWorkerRecord>,
legacy_projection_exists: bool,
}
impl<St> PreparedSpawnedWorkerRegistryLoad<St>
where
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
{
/// Apply only after all other fallible tool initialization, before socket publication.
/// Effects retain their original order: projection, canonical reclaim history, then
/// allocation/scope. An error may follow committed effects; no rollback is attempted.
pub(crate) async fn apply(&mut self) -> io::Result<SpawnedWorkerRegistryLoad> {
let metadata = self
.store
.read_by_name(&self.worker_name)
.map_err(store_error_to_io)?;
let reclaimed: Vec<_> = self
.persisted_children
.iter()
.map(reclaimed_child_from_metadata)
.collect();
// A prior apply may have committed the canonical reclaim and then failed
// in allocation/scope cleanup. Under the host's same lifecycle lock,
// confirm its exact retained facts rather than appending them again.
let already_reclaimed = !reclaimed.is_empty()
&& metadata.as_ref().is_some_and(|metadata| {
metadata.spawned_children.is_empty()
&& metadata.reclaimed_children.ends_with(&reclaimed)
});
if !already_reclaimed
&& metadata
.as_ref()
.map(|metadata| metadata.spawned_children.as_slice())
.unwrap_or_default()
!= self.persisted_children.as_slice()
{
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"spawned-child metadata changed after registry preparation",
));
}
// Runtime projection is migration input only; the normal Internal registry is never
// materialized into spawned_workers.json. Only existence was captured, not its contents.
if !self.persisted_children.is_empty() || self.legacy_projection_exists {
self.runtime_dir.write_spawned_workers(&[]).await?;
}
if !reclaimed.is_empty() && !already_reclaimed {
self.store
.reclaim_spawned_children(&self.worker_name, reclaimed)
.map_err(store_error_to_io)?;
}
for record in &self.valid_records {
reclaim_record(&self.worker_name, self.parent_scope.as_ref(), record)?;
}
let reclaimed_unreachable = self.reclaimed_unreachable;
self.reclaimed_unreachable = false;
Ok(SpawnedWorkerRegistryLoad {
registry: self.registry.clone(),
reclaimed_unreachable,
})
}
}
impl SpawnedWorkerRegistry {
pub(crate) fn bash_output(&self) -> tools::bash_output::BashOutputView {
self.bash_output.lock().unwrap().clone()
@@ -426,6 +504,23 @@ impl SpawnedWorkerRegistry {
worker_name: String,
parent_scope: Option<SharedScope>,
) -> io::Result<SpawnedWorkerRegistryLoad>
where
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
{
Self::prepare_from_worker_state_with_reclaim(runtime_dir, store, worker_name, parent_scope)?
.apply()
.await
}
/// Capture legacy child facts without starting children or changing metadata,
/// projection, allocation, or scope. Tool initialization must clone this registry,
/// not construct a replacement, and defer reclamation to the owned plan's `apply`.
pub(crate) fn prepare_from_worker_state_with_reclaim<St>(
runtime_dir: Arc<RuntimeDir>,
store: St,
worker_name: String,
parent_scope: Option<SharedScope>,
) -> io::Result<PreparedSpawnedWorkerRegistryLoad<St>>
where
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
{
@@ -442,54 +537,33 @@ impl SpawnedWorkerRegistry {
Ok(record) => {
warn!(
worker = %record.worker_name,
"reclaiming legacy persisted process SubWorker during Internal session restore"
"captured legacy persisted process SubWorker for reclamation during Internal session restore"
);
valid_records.push(record);
}
Err(error) => warn!(
error = %error,
worker = %child.worker_name,
"clearing corrupt legacy persisted process SubWorker record"
"captured corrupt legacy persisted process SubWorker record for clearing"
),
}
}
// Runtime projection is migration input only; the normal Internal registry is never
// materialized into spawned_workers.json.
let legacy_projection_exists = runtime_dir.path().join("spawned_workers.json").exists();
if !persisted_children.is_empty() || legacy_projection_exists {
runtime_dir.write_spawned_workers(&[]).await?;
}
if !persisted_children.is_empty() {
let reclaimed = persisted_children
.iter()
.map(reclaimed_child_from_metadata)
.collect();
store
.reclaim_spawned_children(&worker_name, reclaimed)
.map_err(store_error_to_io)?;
}
for record in &valid_records {
reclaim_record(&worker_name, parent_scope.as_ref(), record)?;
}
Ok(SpawnedWorkerRegistryLoad {
registry: Arc::new(Self {
internal_records: std::sync::Mutex::new(Vec::new()),
service_records: std::sync::Mutex::new(Vec::new()),
#[cfg(test)]
fail_service_stops: std::sync::Mutex::new(HashMap::new()),
internal_names: std::sync::Mutex::new(HashSet::new()),
internal_shutting_down: AtomicBool::new(false),
pending_internal_spawns: AtomicUsize::new(0),
pending_internal_notify: Notify::new(),
internal_spawn_cleanup_failed: AtomicBool::new(false),
parent_scope,
bash_output: Mutex::new(tools::bash_output::BashOutputView::new()),
parent_protocol: Mutex::new(None),
protocol_publish_lock: Arc::new(Mutex::new(())),
}),
let registry = match &parent_scope {
Some(scope) => Self::new_internal(worker_name.clone(), scope.clone()),
None => Self::new(runtime_dir.clone()),
};
Ok(PreparedSpawnedWorkerRegistryLoad {
registry,
reclaimed_unreachable: !persisted_children.is_empty(),
runtime_dir,
store,
worker_name,
parent_scope,
persisted_children,
valid_records,
legacy_projection_exists,
})
}
@@ -1140,6 +1214,10 @@ impl Drop for SpawnedWorkerRegistry {
.filter(|record| !record.scope_reclaimed.load(Ordering::Acquire))
.flat_map(delegated_write_rules)
.collect::<Vec<_>>();
// Empty prepared registries own no delegations, including when abandoned or rejected.
if write_rules.is_empty() {
return;
}
let _ = parent_scope.update(|current| current.with_removed_deny_rules(write_rules));
}
}
@@ -1165,8 +1243,8 @@ fn reclaim_record(
parent_scope: Option<&SharedScope>,
record: &SpawnedWorkerRecord,
) -> io::Result<()> {
if let Ok(path) = worker_allocation::default_allocation_path() {
if let Ok(mut guard) = worker_allocation::LockFileGuard::open(&path) {
let path = worker_allocation::default_allocation_path().map_err(io::Error::other)?;
let mut guard = worker_allocation::LockFileGuard::open(&path).map_err(io::Error::other)?;
match worker_allocation::reclaim_delegated_scope(
&mut guard,
parent_name,
@@ -1176,8 +1254,7 @@ fn reclaim_record(
Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {}
Err(error) => return Err(io::Error::other(error)),
}
}
}
drop(guard);
if let Some(parent_scope) = parent_scope {
let write_rules = record
.scope_delegated
@@ -1240,10 +1317,18 @@ fn store_error_to_io(error: WorkerStoreError) -> io::Error {
#[cfg(test)]
mod tests {
//! Legacy restore cases connect the registry to canonical aggregate storage,
//! runtime projections, and temporary SharedScope rules. Successful apply uses
//! a corrupt legacy record to exercise durable clearing without consulting the
//! process-global legacy allocation file; valid-record scope reclaim is also
//! covered by the existing external restore test.
use std::path::PathBuf;
use std::time::Duration;
use manifest::{Scope, ScopeConfig};
use session_store::LogEntry;
use session_store::{LogEntry, WorkerAggregateStore, WorkerMetadata, WorkerSpawnedScopeRule};
use tempfile::TempDir;
use super::*;
use crate::internal_worker::{
@@ -1251,6 +1336,249 @@ mod tests {
test_internal_worker_session_with_greeting,
};
struct LegacyRestoreFixture {
runtime_dir: Arc<RuntimeDir>,
store: WorkerAggregateStore,
parent_scope: SharedScope,
scope_root: PathBuf,
metadata_path: PathBuf,
// Keep all storage and scope paths alive until the other fixture fields are dropped.
_root: TempDir,
}
impl LegacyRestoreFixture {
async fn new(permission: &str) -> Self {
let root = TempDir::new().unwrap();
let scope_root = root.path().join("scope");
std::fs::create_dir(&scope_root).unwrap();
let aggregate = root.path().join("aggregate");
let store = WorkerAggregateStore::new(&aggregate, "parent").unwrap();
let runtime_dir = Arc::new(
RuntimeDir::create(&root.path().join("runtime"), "parent")
.await
.unwrap(),
);
let mut metadata =
WorkerMetadata::new("parent", None).with_workspace_id("restore-test-workspace");
metadata.spawned_children.push(WorkerSpawnedChild {
worker_name: "legacy-child".into(),
socket_path: root.path().join("legacy.sock"),
callback_address: runtime_dir.path().join("sock"),
scope_delegated: vec![WorkerSpawnedScopeRule {
target: scope_root.clone(),
permission: permission.into(),
recursive: true,
symlink_policy: Default::default(),
}],
});
store.write(&metadata).unwrap();
let write_rule = ScopeRule {
target: scope_root.clone(),
permission: Permission::Write,
recursive: true,
symlink_policy: Default::default(),
};
let parent_scope = SharedScope::new(
Scope::from_config(&ScopeConfig {
allow: vec![write_rule.clone()],
deny: vec![write_rule],
})
.unwrap(),
);
Self {
runtime_dir,
store,
parent_scope,
scope_root,
metadata_path: aggregate.join("metadata.json"),
_root: root,
}
}
fn prepare(&self) -> PreparedSpawnedWorkerRegistryLoad<WorkerAggregateStore> {
SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim(
self.runtime_dir.clone(),
self.store.clone(),
"parent".into(),
Some(self.parent_scope.clone()),
)
.unwrap()
}
fn projection_path(&self) -> PathBuf {
self.runtime_dir.path().join("spawned_workers.json")
}
}
#[tokio::test]
async fn registry_preparation_and_abandonment_leave_legacy_state_unchanged() {
for projection_exists in [false, true] {
let fixture = LegacyRestoreFixture::new("write").await;
if projection_exists {
// Projection contents are deliberately not decoded during preparation.
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
}
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
let projection_before = std::fs::read(fixture.projection_path()).ok();
let scope_before = fixture.parent_scope.snapshot();
assert!(!scope_before.is_writable(&fixture.scope_root));
let prepared = fixture.prepare();
assert!(prepared.reclaimed_unreachable);
assert!(prepared.registry.list_internal().is_empty());
assert!(prepared.registry.internal_worker_snapshots().is_empty());
assert_eq!(
std::fs::read(&fixture.metadata_path).unwrap(),
metadata_before
);
assert_eq!(
std::fs::read(fixture.projection_path()).ok(),
projection_before
);
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
assert!(!fixture.runtime_dir.path().join("sock").exists());
drop(prepared);
assert_eq!(
std::fs::read(&fixture.metadata_path).unwrap(),
metadata_before
);
assert_eq!(
std::fs::read(fixture.projection_path()).ok(),
projection_before
);
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
}
}
#[tokio::test]
async fn prepared_apply_clears_canonical_children_and_projection_on_same_registry() {
let fixture = LegacyRestoreFixture::new("unsupported-legacy-permission").await;
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
let scope_before = fixture.parent_scope.snapshot();
let mut prepared = fixture.prepare();
let shared_registry = prepared.registry.clone();
// Unrelated metadata may advance while tools are initialized.
fixture.store.add_peer("parent", "new-peer").unwrap();
let mut expected = fixture.store.read_by_name("parent").unwrap().unwrap();
expected.reclaimed_children.extend(
expected
.spawned_children
.iter()
.map(reclaimed_child_from_metadata),
);
expected.spawned_children.clear();
let loaded = prepared.apply().await.unwrap();
assert!(loaded.reclaimed_unreachable);
assert!(Arc::ptr_eq(&shared_registry, &loaded.registry));
assert!(shared_registry.list_internal().is_empty());
assert_eq!(
fixture.store.read_by_name("parent").unwrap(),
Some(expected)
);
let projection: serde_json::Value =
serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap();
assert_eq!(projection, serde_json::json!([]));
// Corrupt records are audited and cleared, not interpreted as scope grants.
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
assert!(
!fixture
.parent_scope
.snapshot()
.is_writable(&fixture.scope_root)
);
assert!(!fixture.runtime_dir.path().join("sock").exists());
}
#[tokio::test]
async fn prepared_apply_rejects_child_source_drift_before_any_effects() {
for drift in ["scope", "socket", "removal", "metadata-deletion"] {
let fixture = LegacyRestoreFixture::new("write").await;
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
let scope_before = fixture.parent_scope.snapshot();
let mut prepared = fixture.prepare();
if drift == "metadata-deletion" {
fixture.store.delete_by_name("parent").unwrap();
} else {
fixture
.store
.update_by_name("parent", |metadata| match drift {
"scope" => {
metadata.spawned_children[0].scope_delegated[0].recursive = false
}
"socket" => {
metadata.spawned_children[0].socket_path =
fixture.runtime_dir.path().join("replacement.sock");
}
"removal" => metadata.spawned_children.clear(),
_ => unreachable!(),
})
.unwrap();
}
let metadata_before = std::fs::read(&fixture.metadata_path).ok();
let projection_before = std::fs::read(fixture.projection_path()).unwrap();
let error = prepared
.apply()
.await
.err()
.expect("source drift must fail");
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert_eq!(std::fs::read(&fixture.metadata_path).ok(), metadata_before);
assert_eq!(
std::fs::read(fixture.projection_path()).unwrap(),
projection_before
);
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
assert!(!fixture.runtime_dir.path().join("sock").exists());
}
}
#[tokio::test]
async fn prepared_apply_clears_projection_only_without_reporting_child_reclaim() {
let fixture = LegacyRestoreFixture::new("write").await;
fixture
.store
.set_spawned_children("parent", Vec::new())
.unwrap();
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
let mut prepared = fixture.prepare();
assert!(!prepared.reclaimed_unreachable);
let loaded = prepared.apply().await.unwrap();
assert!(!loaded.reclaimed_unreachable);
assert_eq!(
std::fs::read(&fixture.metadata_path).unwrap(),
metadata_before
);
let projection: serde_json::Value =
serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap();
assert_eq!(projection, serde_json::json!([]));
}
#[tokio::test]
async fn prepared_apply_projection_failure_keeps_canonical_children_and_scope() {
let fixture = LegacyRestoreFixture::new("write").await;
std::fs::create_dir(fixture.projection_path()).unwrap();
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
let scope_before = fixture.parent_scope.snapshot();
let mut prepared = fixture.prepare();
assert!(prepared.apply().await.is_err());
assert_eq!(
std::fs::read(&fixture.metadata_path).unwrap(),
metadata_before
);
assert!(fixture.projection_path().is_dir());
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
}
fn registry() -> Arc<SpawnedWorkerRegistry> {
let scope = Scope::from_config(&ScopeConfig {
allow: vec![ScopeRule {
File diff suppressed because it is too large Load Diff
+249 -9
View File
@@ -1,3 +1,4 @@
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
@@ -150,6 +151,19 @@ impl LlmClient for MockClient {
// Helpers
// ---------------------------------------------------------------------------
#[path = "../../../tests/support/restore_environment.rs"]
mod restore_test_environment;
use restore_test_environment::require_restore_test_environment;
fn owned_relative_runtime_base() -> (tempfile::TempDir, PathBuf) {
require_restore_test_environment();
let temp = tempfile::tempdir_in(".").unwrap();
// The relative reference and the TempDir name the same owned tree, without
// putting the checkout prefix into sockaddr_un or changing process cwd.
let runtime_base = Path::new(temp.path().file_name().unwrap()).join("runtime");
(temp, runtime_base)
}
fn simple_text_events() -> Vec<LlmEvent> {
vec![
LlmEvent::text_block_start(0),
@@ -295,6 +309,7 @@ async fn make_worker_with_manifest_and_workspace_context(
manifest: WorkerManifest,
workspace_context: WorkerWorkspaceContext,
) -> (Worker<MockClient, TestStore>, std::path::PathBuf) {
require_restore_test_environment();
let store_tmp = tempfile::tempdir().unwrap();
let store = CombinedStore::new(
FsStore::new(store_tmp.path()).unwrap(),
@@ -320,18 +335,21 @@ async fn make_worker_with_manifest_and_workspace_context(
}
async fn spawn_controller(worker: Worker<MockClient, TestStore>) -> WorkerHandle {
let tmp = tempfile::tempdir().unwrap();
let runtime_base = tmp.path().to_owned();
std::mem::forget(tmp);
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
let (tmp, runtime_base) = owned_relative_runtime_base();
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
.await
.unwrap();
// Retain ownership until controller shutdown (or test-runtime teardown).
tokio::spawn(async move {
let _ = shutdown_rx.await;
drop(tmp);
});
handle
}
#[tokio::test]
async fn canonical_worker_handle_uploads_and_submits_attachment() {
let root = tempfile::tempdir().unwrap();
let (root, runtime_base) = owned_relative_runtime_base();
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap();
let store = CombinedStore::new(
@@ -353,7 +371,7 @@ async fn canonical_worker_handle_uploads_and_submits_attachment() {
)
.await
.unwrap();
let (handle, shutdown_rx) = WorkerController::spawn(worker, &root.path().join("runtime"))
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
.await
.unwrap();
let mut rx = handle.subscribe();
@@ -762,6 +780,227 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
));
}
mod registry_startup_failure {
//! Connect real controller preparation, registry source fences and session cleanup.
//! The Workdir telemetry boundary substitutes a concurrent metadata change at subscription;
//! the real registry must reject drift before starting its prepared observer.
use super::*;
use session_store::WorkerMetadataStore;
use tokio::sync::broadcast;
use workdir::WorkdirSession;
struct ObservedSession {
local: LocalWorkdirSession,
metadata_store: TestStore,
worker_name: String,
events: broadcast::Sender<workdir::CommandEvent>,
subscriptions: AtomicUsize,
receivers_at_close: Mutex<Vec<usize>>,
}
impl std::fmt::Debug for ObservedSession {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ObservedSession")
.field("worker_name", &self.worker_name)
.finish_non_exhaustive()
}
}
#[async_trait]
impl WorkdirSession for ObservedSession {
fn workdir(&self) -> &Workdir {
self.local.workdir()
}
fn capabilities(&self) -> WorkdirSessionCapabilities {
self.local.capabilities()
}
async fn stat(
&self,
request: workdir::StatRequest,
) -> Result<workdir::StatResult, WorkdirError> {
self.local.stat(request).await
}
async fn read(
&self,
request: workdir::ReadRequest,
) -> Result<workdir::ReadResult, WorkdirError> {
self.local.read(request).await
}
async fn write(
&self,
request: workdir::WriteRequest,
) -> Result<workdir::WriteResult, WorkdirError> {
self.local.write(request).await
}
async fn edit(
&self,
request: workdir::EditRequest,
) -> Result<workdir::EditResult, WorkdirError> {
self.local.edit(request).await
}
async fn list(
&self,
request: workdir::ListRequest,
) -> Result<workdir::ListResult, WorkdirError> {
self.local.list(request).await
}
async fn glob(
&self,
request: workdir::GlobRequest,
) -> Result<workdir::GlobResult, WorkdirError> {
self.local.glob(request).await
}
async fn grep(
&self,
request: workdir::GrepRequest,
) -> Result<workdir::GrepResult, WorkdirError> {
self.local.grep(request).await
}
async fn start_command(
&self,
request: CommandRequest,
) -> Result<workdir::CommandHandle, WorkdirError> {
self.local.start_command(request).await
}
async fn command_status(
&self,
handle: workdir::CommandHandle,
) -> Result<workdir::CommandStatus, WorkdirError> {
self.local.command_status(handle).await
}
async fn command_output(
&self,
request: CommandOutputRequest,
) -> Result<workdir::CommandOutput, WorkdirError> {
self.local.command_output(request).await
}
async fn cancel_command(&self, handle: workdir::CommandHandle) -> Result<(), WorkdirError> {
self.local.cancel_command(handle).await
}
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<workdir::CommandEvent>> {
// Interleave a canonical change after read-only registry preparation.
let mut metadata = self
.metadata_store
.read_by_name(&self.worker_name)
.unwrap()
.unwrap_or_else(|| session_store::WorkerMetadata::new(&self.worker_name, None));
metadata
.spawned_children
.push(session_store::WorkerSpawnedChild {
worker_name: "legacy-drift".into(),
socket_path: PathBuf::new(),
callback_address: PathBuf::new(),
scope_delegated: vec![],
});
self.metadata_store.write(&metadata).unwrap();
self.subscriptions.fetch_add(1, Ordering::SeqCst);
Some(self.events.subscribe())
}
async fn close(&self) -> Result<(), WorkdirError> {
// Record before any await: aborting without joining must not pass by getting a
// scheduler turn during the provider's asynchronous close. The sender stays alive
// even after close, so channel closure cannot accidentally dispose of the observer.
self.receivers_at_close
.lock()
.unwrap()
.push(self.events.receiver_count());
self.local.close().await
}
}
#[tokio::test(flavor = "current_thread")]
async fn registry_drift_drops_prepared_observer_before_closing_session() {
let root = tempfile::tempdir().unwrap();
let run_dir = root.path().join("runs/1");
let client = MockClient::new(simple_text_events());
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
let store = CombinedStore::new(
FsStore::new(root.path().join("sessions")).unwrap(),
FsWorkerStore::new(root.path().join("workers")).unwrap(),
);
let engine =
Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
client.clone(),
);
let mut worker = Worker::new(
manifest,
engine,
store.clone(),
WorkerWorkspaceContext::local_filesystem(None),
WorkerFilesystemAuthority::local(root.path().into(), root.path().into()),
manifest::Scope::writable(root.path()).unwrap(),
)
.await
.unwrap();
let (events, unused_receiver) = broadcast::channel(8);
drop(unused_receiver);
let session = Arc::new(ObservedSession {
local: LocalWorkdirSession::materialized_bound(
Workdir::new("registry-startup-failure"),
root.path().into(),
root.path().into(),
worker.scope().clone(),
WorkdirSessionCapabilities::ALL,
),
metadata_store: store,
worker_name: worker.manifest().worker.name.clone(),
events,
subscriptions: AtomicUsize::new(0),
receivers_at_close: Mutex::new(Vec::new()),
});
worker.bind_single_workdir_session(Some(session.clone()));
let sessions = worker.workdir_sessions();
let result = tokio::time::timeout(
std::time::Duration::from_secs(5),
WorkerController::spawn_runtime_managed_run(worker, &run_dir),
)
.await
.expect("registry rejection and cleanup must settle");
let error = match result {
Err(error) => error,
Ok(_) => panic!("registry source drift must reject controller startup"),
};
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
assert_eq!(session.subscriptions.load(Ordering::SeqCst), 1);
assert_eq!(*session.receivers_at_close.lock().unwrap(), vec![0]);
assert_eq!(session.events.receiver_count(), 0);
assert!(sessions.aliases().is_empty());
// No controller retains the Worker/Engine after the failed startup.
assert_eq!(Arc::strong_count(&client.call_count), 1);
assert_eq!(client.call_count.load(Ordering::SeqCst), 0);
assert!(!run_dir.join("worker.sock").exists());
assert!(!run_dir.join("status.json").exists());
assert!(!run_dir.join("manifest.toml").exists());
assert!(matches!(
session
.start_command(CommandRequest {
command: "printf unreachable".to_owned(),
timeout_secs: 5,
transport_byte_budget: workdir::MAX_COMMAND_TRANSPORT_BYTES,
cwd: workdir::WorkdirPath::root(),
tool_call_id: None,
})
.await,
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
));
}
}
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
loop {
@@ -1561,11 +1800,12 @@ permission = "write"
"#;
let client = MockClient::new(simple_text_events());
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
let tmp = tempfile::tempdir().unwrap();
let result = WorkerController::spawn(worker, tmp.path()).await;
let (_tmp, runtime_base) = owned_relative_runtime_base();
let result = WorkerController::spawn(worker, &runtime_base).await;
assert!(
result.is_ok(),
"feature exposure must not imply delegation authority"
"feature exposure must not imply delegation authority: {:?}",
result.as_ref().err()
);
}
+4
View File
@@ -10,6 +10,10 @@ autobins = false
name = "yoi-server"
path = "src/main.rs"
[features]
# Offline fixture preparation only; build product binaries without this feature.
test-support = []
[dependencies]
api-macros = { workspace = true, features = ["axum"] }
async-trait.workspace = true
+3
View File
@@ -5113,6 +5113,9 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
runtime_api::WorkerRestoreState::RolledBack => {
server_api::WorkerRestoreState::RolledBack
}
runtime_api::WorkerRestoreState::Superseded => {
server_api::WorkerRestoreState::Superseded
}
runtime_api::WorkerRestoreState::ReconciliationRequired => {
server_api::WorkerRestoreState::ReconciliationRequired
}
+5
View File
@@ -19,12 +19,17 @@ pub mod repositories;
pub mod repository_access;
pub mod repository_source;
pub mod resource_broker;
#[cfg(test)]
#[path = "../../../tests/support/restore_environment.rs"]
mod restore_test_environment;
pub mod retention;
pub mod runtime_settings;
pub mod runtime_subscription;
pub mod server;
pub mod skills;
pub mod store;
#[cfg(feature = "test-support")]
pub use store::fixture_support as test_support;
pub mod subjektiv;
#[cfg(test)]
mod ticket_browse_performance_tests;
+26
View File
@@ -48742,6 +48742,12 @@ mod tests {
std::sync::Mutex<Vec<worker_runtime::execution::WorkerLifecycleOperationId>>,
reconcile_operations:
std::sync::Mutex<Vec<worker_runtime::execution::WorkerLifecycleOperationId>>,
restore_owners: std::sync::Mutex<
std::collections::HashMap<
worker_runtime::execution::WorkerLifecycleOperationId,
worker_runtime::identity::WorkerRef,
>,
>,
inputs: std::sync::Mutex<Vec<(worker_runtime::identity::WorkerRef, String)>>,
protocol_methods:
Arc<std::sync::Mutex<Vec<(worker_runtime::identity::WorkerRef, protocol::Method)>>>,
@@ -48775,6 +48781,7 @@ mod tests {
restore_failure: std::sync::Mutex::new(None),
restore_operations: std::sync::Mutex::new(Vec::new()),
reconcile_operations: std::sync::Mutex::new(Vec::new()),
restore_owners: Default::default(),
inputs: std::sync::Mutex::new(Vec::new()),
protocol_methods: Arc::new(std::sync::Mutex::new(Vec::new())),
stops: std::sync::Mutex::new(Vec::new()),
@@ -48979,6 +48986,10 @@ mod tests {
.lock()
.unwrap()
.push(request.operation_id);
self.restore_owners
.lock()
.unwrap()
.insert(request.operation_id, request.worker_ref.clone());
if let Some(result) = self.restore_failure.lock().unwrap().take() {
return result;
}
@@ -49014,6 +49025,21 @@ mod tests {
}
}
fn restore_recovery_disposition(
&self,
operation_id: worker_runtime::execution::WorkerLifecycleOperationId,
worker_ref: &worker_runtime::identity::WorkerRef,
) -> Option<worker_runtime::execution::RestoreRecoveryDisposition> {
use worker_runtime::execution::RestoreRecoveryDisposition as D;
Some(
if self.restore_owners.lock().unwrap().get(&operation_id) == Some(worker_ref) {
D::Continue
} else {
D::Absent
},
)
}
fn reconcile_restore(
&self,
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
@@ -11,6 +11,20 @@ use worker_runtime::worker_backend::{
WorkerRuntimeExecutionBackend,
};
pub(super) fn connected_restore_tempdir() -> tempfile::TempDir {
crate::restore_test_environment::require_restore_test_environment();
let checkout = Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.parent()
.unwrap();
tempfile::Builder::new()
.prefix("c")
.rand_bytes(3)
.tempdir_in(checkout.join("target"))
.unwrap()
}
const WATCHDOG: Duration = Duration::from_secs(45);
#[derive(Clone)]
@@ -82,24 +96,91 @@ impl RuntimeWorkerFactory for CapturingFactory {
) -> std::result::Result<(), String> {
self.inner.preflight_restore(request).await
}
async fn restore_controller(
async fn recover_restore_source(
&self,
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
self.inner.recover_restore_source(request).await
}
async fn prepare_restore(
&self,
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<RuntimeWorkerController, String> {
) -> std::result::Result<
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
let worker_ref = request.worker_ref.clone();
let controller = self.inner.restore_controller(request).await?;
let inner = self.inner.prepare_restore(request).await?;
Ok(Box::new(CapturingCandidate {
inner,
worker_ref,
captures: self.captures.clone(),
}))
}
async fn cleanup_failed_restore(
&self,
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
self.inner.cleanup_failed_restore(request).await
}
async fn reconcile_stopped_worker(
&self,
worker_ref: &worker_runtime::identity::WorkerRef,
) -> std::result::Result<(), String> {
self.inner.reconcile_stopped_worker(worker_ref).await
}
fn activate_restored_controller(
&self,
worker_ref: &worker_runtime::identity::WorkerRef,
workspace_id: Option<&str>,
handle: &worker::WorkerHandle,
) -> std::result::Result<(), String> {
self.inner
.activate_restored_controller(worker_ref, workspace_id, handle)
}
}
struct CapturingCandidate {
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
worker_ref: worker_runtime::identity::WorkerRef,
captures: Arc<Captures>,
}
#[async_trait::async_trait]
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for CapturingCandidate {
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
self.inner.initial_state()
}
async fn activate(
&mut self,
) -> std::result::Result<
RuntimeWorkerController,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
let controller = self.inner.activate().await?;
self.captures
.handles
.lock()
.unwrap()
.push((worker_ref.clone(), controller.handle.clone()));
.push((self.worker_ref.clone(), controller.handle.clone()));
self.captures
.restored_clients
.lock()
.unwrap()
.push((worker_ref, controller.workspace_client.clone()));
.push((self.worker_ref.clone(), controller.workspace_client.clone()));
Ok(controller)
}
async fn discard(
&mut self,
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
self.inner.discard().await
}
}
struct Resources {
@@ -259,7 +340,7 @@ async fn call(tools: &BTreeMap<String, Arc<dyn Tool>>, name: &str, args: Value)
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn restored_runtime_parent_lists_and_inputs_existing_child() {
let root = tempfile::tempdir().unwrap();
let root = connected_restore_tempdir();
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
let mut config = test_server_config(root.path());
File diff suppressed because it is too large Load Diff
@@ -476,6 +476,7 @@ impl AttentionWorkerFactory {
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
controller_task: started.controller_task,
workspace_client,
restore_streams: None,
})
}
}
@@ -489,12 +490,61 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFac
self.controller(request.worker_ref.worker_id.to_string(), false)
.await
}
async fn restore_controller(
async fn prepare_restore(
&self,
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
self.controller(request.worker_ref.worker_id.to_string(), true)
.await
) -> std::result::Result<
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
Ok(Box::new(AttentionRestoreCandidate {
inputs: Some((
AttentionWorkerFactory {
root: self.root.clone(),
},
request.worker_ref.worker_id.to_string(),
)),
}))
}
}
// Preparation owns only fixture inputs; reconstruction and Controller startup
// remain lazy so discard never needs to shut down a started Worker.
struct AttentionRestoreCandidate {
inputs: Option<(AttentionWorkerFactory, String)>,
}
#[async_trait::async_trait]
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for AttentionRestoreCandidate {
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
protocol::WorkerStateSnapshot::initial()
}
async fn activate(
&mut self,
) -> std::result::Result<
worker_runtime::worker_backend::RuntimeWorkerController,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
"attention fixture candidate already consumed",
)
})?;
factory.controller(worker_id, true).await.map_err(|_| {
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
"attention fixture reconstruction failed",
)
})
}
async fn discard(
&mut self,
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
self.inputs.take();
Ok(())
}
}
@@ -601,7 +651,7 @@ async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() {
async fn real_attention_worker_start_restore(lose_response: bool) {
use session_store::{LogEntry, Store, WorkerSessionStore};
let dir = tempfile::tempdir().unwrap();
let dir = restored_parent_control_tests::connected_restore_tempdir();
init_clean_git_workspace(dir.path());
let config = test_server_config(dir.path());
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
@@ -14,17 +14,37 @@ async fn notification_diagnostics_http_auth_scope_and_readonly_evidence() {
seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID);
let ticket = delivery_ticket(&api, "Notification diagnostics auth fixture");
delivery_assign(&api, ticket.id.as_str(), "diagnostics-assignment").await;
let path = format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",ticket.id.as_str());
let path = format!(
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",
ticket.id.as_str()
);
let page = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
assert_eq!(page["items"].as_array().unwrap().len(), 1);
assert_eq!(page["items"][0]["model_processing_confirmed"], false);
assert!(page["items"][0].get("content").is_none());
let again = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
assert_eq!(again, page);
let response=build_router(api.clone()).oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap()).await.unwrap();
let response = build_router(api.clone())
.oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
delivery_http(&api,"GET","/api/w/other-workspace/notification-diagnostics",None,StatusCode::NOT_FOUND).await;
delivery_http(&api,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),None,StatusCode::BAD_REQUEST).await;
delivery_http(
&api,
"GET",
"/api/w/other-workspace/notification-diagnostics",
None,
StatusCode::NOT_FOUND,
)
.await;
delivery_http(
&api,
"GET",
&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),
None,
StatusCode::BAD_REQUEST,
)
.await;
}
fn delivery_assignment_path(ticket: &str) -> String {
@@ -814,6 +834,7 @@ impl DeliveryWorkerFactory {
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
controller_task: started.controller_task,
workspace_client,
restore_streams: None,
})
}
}
@@ -827,12 +848,62 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFact
self.controller(request.worker_ref.worker_id.to_string(), false)
.await
}
async fn restore_controller(
async fn prepare_restore(
&self,
request: delivery_execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
self.controller(request.worker_ref.worker_id.to_string(), true)
.await
) -> std::result::Result<
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
Ok(Box::new(DeliveryRestoreCandidate {
inputs: Some((
DeliveryWorkerFactory {
root: self.root.clone(),
client: self.client.clone(),
},
request.worker_ref.worker_id.to_string(),
)),
}))
}
}
// Retain the same scripted client/receipt store without opening either store or
// starting a Controller until activation. Discard drops only these owned inputs.
struct DeliveryRestoreCandidate {
inputs: Option<(DeliveryWorkerFactory, String)>,
}
#[async_trait::async_trait]
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for DeliveryRestoreCandidate {
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
protocol::WorkerStateSnapshot::initial()
}
async fn activate(
&mut self,
) -> std::result::Result<
worker_runtime::worker_backend::RuntimeWorkerController,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
"delivery fixture candidate already consumed",
)
})?;
factory.controller(worker_id, true).await.map_err(|_| {
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
"delivery fixture reconstruction failed",
)
})
}
async fn discard(
&mut self,
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
self.inputs.take();
Ok(())
}
}
@@ -931,6 +1002,7 @@ async fn delivery_real_fixture(
Arc<DeliveryResponseLossBackend>,
DeliveryScriptedClient,
) {
let _environment = restored_parent_control_tests::connected_restore_tempdir();
let config = test_server_config(root);
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
seed_test_registered_workspace(store.as_ref(), &config)
@@ -1136,9 +1208,22 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
"uncertain delivery changed request id or payload: {attempts:?}"
);
assert_eq!(delivery_receipts(&api, &worker).len(), 1);
let unknown = api.store.notification_diagnostics(TEST_WORKSPACE_ID, &server_api::NotificationDiagnosticsQuery {
assignment_id: Some(assignment.into()), ..Default::default()
}).unwrap().items.into_iter().find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown).expect("saved unknown acceptance must remain visible while Runtime already has its receipt");
let unknown = api
.store
.notification_diagnostics(
TEST_WORKSPACE_ID,
&server_api::NotificationDiagnosticsQuery {
assignment_id: Some(assignment.into()),
..Default::default()
},
)
.unwrap()
.items
.into_iter()
.find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown)
.expect(
"saved unknown acceptance must remain visible while Runtime already has its receipt",
);
assert!(!unknown.model_processing_confirmed);
// Change mutable rendering context while the acknowledgement is unknown.
// A new self-report is not a new input and cannot change the frozen retry.
@@ -1199,9 +1284,22 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
}
delivery_wait_idle(&reopened, &worker, 2).await;
assert_eq!(delivery_receipts(&reopened, &worker).len(), 2);
let evidence = delivery_http(&reopened,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",unknown.source_id),None,StatusCode::OK).await;
let evidence = delivery_http(
&reopened,
"GET",
&format!(
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",
unknown.source_id
),
None,
StatusCode::OK,
)
.await;
assert_eq!(evidence["items"][0]["stage"], "runtime_accepted");
assert_eq!(evidence["items"][0]["delivery_id"],json!(unknown.delivery_id));
assert_eq!(
evidence["items"][0]["delivery_id"],
json!(unknown.delivery_id)
);
assert_eq!(evidence["items"][0]["model_processing_confirmed"], false);
assert_eq!(evidence["items"][0]["retry_candidate"], false);
assert_eq!(
@@ -187,7 +187,14 @@ async fn config_call(
runtime
.invoke(
path.into(),
if matches!(operation,"edit"|"write") {wip_protocol::InterfaceReference {scope:"/".into(),name:format!("yoi.file/{operation}-text/v1")}} else {config_interface(path)},
if matches!(operation, "edit" | "write") {
wip_protocol::InterfaceReference {
scope: "/".into(),
name: format!("yoi.file/{operation}-text/v1"),
}
} else {
config_interface(path)
},
operation.into(),
args,
agen::tool::ToolExecutionContext::direct(),
@@ -581,11 +588,88 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
Ok(controller)
}
async fn restore_controller(
async fn preflight_restore(
&self,
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<(), String> {
self.inner.preflight_restore(request).await
}
async fn recover_restore_source(
&self,
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
self.inner.recover_restore_source(request).await
}
async fn prepare_restore(
&self,
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
self.inner.restore_controller(request).await
) -> std::result::Result<
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
let inner = self.inner.prepare_restore(request).await?;
Ok(Box::new(ConfigCaptureCandidate {
inner,
client: self.client.clone(),
controller_abort: self.controller_abort.clone(),
}))
}
async fn cleanup_failed_restore(
&self,
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
self.inner.cleanup_failed_restore(request).await
}
async fn reconcile_stopped_worker(
&self,
worker_ref: &worker_runtime::identity::WorkerRef,
) -> std::result::Result<(), String> {
self.inner.reconcile_stopped_worker(worker_ref).await
}
fn activate_restored_controller(
&self,
worker_ref: &worker_runtime::identity::WorkerRef,
workspace_id: Option<&str>,
handle: &worker::WorkerHandle,
) -> std::result::Result<(), String> {
self.inner
.activate_restored_controller(worker_ref, workspace_id, handle)
}
}
struct ConfigCaptureCandidate {
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
client: Arc<Mutex<Option<Arc<dyn worker::WorkspaceClient>>>>,
controller_abort: Arc<Mutex<Option<tokio::task::AbortHandle>>>,
}
#[async_trait]
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for ConfigCaptureCandidate {
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
self.inner.initial_state()
}
async fn activate(
&mut self,
) -> std::result::Result<
worker_runtime::worker_backend::RuntimeWorkerController,
worker_runtime::worker_backend::RuntimeRestoreFailure,
> {
let controller = self.inner.activate().await?;
*self.client.lock().unwrap() = Some(controller.workspace_client.clone());
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
Ok(controller)
}
async fn discard(
&mut self,
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
self.inner.discard().await
}
}
@@ -714,6 +798,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION,
decode_runtime_request_source_claims,
};
let root = restored_parent_control_tests::connected_restore_tempdir();
// Remote Runtime-owned HTTP adapter, including its real request signer.
let mut fixture = manual_worker_assignment_fixture().await;
let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap();
@@ -793,7 +878,6 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
// Actual embedded Runtime factory -> real Worker controller -> captured
// production WorkspaceClient -> signed HTTP -> production Backend middleware.
let root = tempfile::tempdir().unwrap();
init_clean_git_workspace(root.path());
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
+38 -4
View File
@@ -2603,6 +2603,10 @@ pub struct SqliteWorkspaceStore {
db_executor: crate::db_executor::DbExecutor,
}
#[cfg(feature = "test-support")]
#[path = "test_support.rs"]
pub mod fixture_support;
impl SqliteWorkspaceStore {
pub fn open(path: impl AsRef<Path>) -> Result<Self> {
let conn = Connection::open(path)?;
@@ -2914,6 +2918,33 @@ impl SqliteWorkspaceStore {
request_fingerprint: &str,
singleton_key: Option<&str>,
current_memory_settings: &WorkspaceMemorySettingsRecord,
) -> Result<WorkerCreateReservation> {
self.reserve_worker_create_with(
workspace_id,
runtime_id,
allocation_key,
request_fingerprint,
singleton_key,
current_memory_settings,
|_| Ok(None),
|_, _| Ok(()),
)
}
// Keep reservation validation, Memory binding and writes in one transaction. The
// nondefault test-support module supplies fixed identity / offline catalog hooks;
// ordinary creation always supplies a newly generated ID and no completion hook.
#[allow(clippy::too_many_arguments)]
fn reserve_worker_create_with(
&self,
workspace_id: &str,
runtime_id: &str,
allocation_key: &str,
request_fingerprint: &str,
singleton_key: Option<&str>,
current_memory_settings: &WorkspaceMemorySettingsRecord,
prepare: impl FnOnce(&rusqlite::Transaction<'_>) -> Result<Option<WorkerId>>,
complete: impl FnOnce(&rusqlite::Transaction<'_>, &WorkerCreateReservation) -> Result<()>,
) -> Result<WorkerCreateReservation> {
if allocation_key.trim().is_empty() || request_fingerprint.trim().is_empty() {
return Err(Error::InvalidInput(
@@ -2926,6 +2957,7 @@ impl SqliteWorkspaceStore {
validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?;
self.with_conn_mut(|conn| {
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
let fixed_worker_id = prepare(&tx)?;
let existing = tx
.query_row(
"SELECT worker_id, runtime_id, request_fingerprint, create_fingerprint, \
@@ -3108,7 +3140,7 @@ impl SqliteWorkspaceStore {
validate_workspace_memory_settings_snapshot(&snapshot, workspace_id)?;
let create_fingerprint =
bound_worker_create_fingerprint(request_fingerprint, &snapshot);
let worker_id = WorkerId::now_v7();
let worker_id = fixed_worker_id.unwrap_or_else(WorkerId::now_v7);
let worker_id_text = worker_id.to_string();
let now = chrono::Utc::now().to_rfc3339();
let singleton_generation = match singleton_owner.as_ref() {
@@ -3197,13 +3229,15 @@ impl SqliteWorkspaceStore {
));
}
}
tx.commit()?;
Ok(WorkerCreateReservation {
let reservation = WorkerCreateReservation {
worker_id,
create_fingerprint,
memory_settings: snapshot,
singleton,
})
};
complete(&tx, &reservation)?;
tx.commit()?;
Ok(reservation)
})
}
+688
View File
@@ -0,0 +1,688 @@
//! Offline Backend fixture preparation, available only with `test-support`.
//!
//! Use a new isolated database and the existing account, Workspace signing-identity /
//! `ControlPlaneStore::create_workspace_bootstrap`, and Runtime binding APIs first.
//! Generate fresh account credentials and signing keys for each environment; no fixture
//! credentials are accepted here. Install Profile configuration through the existing
//! config-source mechanisms before starting the product processes.
//!
//! This module seeds Backend ownership only. It neither writes Runtime storage nor
//! launches a Worker, fabricates Runtime observations, changes authorization, or sends
//! initial input. The caller must separately install the matching stopped Runtime
//! fixture. Build product binaries in a separate Cargo invocation **without** this
//! feature (Cargo features unify within an invocation).
use super::*;
use crate::hosts::{WorkerSpawnIntent, WorkerSpawnRequest, worker_spawn_create_fingerprint};
use worker_runtime::catalog::ProfileSelector;
/// One ordinary, attachment-free Worker from an externally validated fixture.
///
/// IDs must match the Runtime fixture. `request` is the unresolved canonical create
/// intent, not a Runtime launch request containing credentials or execution grants.
/// The Profile selector must already be installed by the harness. `initial_submit`
/// must be empty: saved Session contents belong to the Runtime fixture, not new input.
/// Singleton/service, Ticket-assigned and attachment-bearing Workers are deliberately
/// unsupported.
pub struct FixedWorkerSeed<'a> {
pub workspace_id: &'a str,
pub runtime_id: &'a str,
pub worker_id: WorkerId,
pub allocation_key: &'a str,
pub request: &'a WorkerSpawnRequest,
}
/// Insert a fixed Worker catalog entry and completed creation reservation atomically.
///
/// Uses the normal reservation validators, authoritative Memory snapshot, fingerprint
/// binding, resource-key allocator and immediate transaction. Even an exact repeat is
/// rejected: this is not an upsert/import/recovery API. Existing allocations, Worker
/// identities and residual lifecycle/attachment records are never adopted or repaired.
/// Returns the creation binding needed to check the corresponding Runtime fixture.
pub fn seed_fixed_worker(
store: &SqliteWorkspaceStore,
seed: FixedWorkerSeed<'_>,
) -> Result<WorkerCreateReservation> {
validate_identifier("workspace_id", seed.workspace_id)?;
validate_identifier("runtime_id", seed.runtime_id)?;
let request = seed.request;
if !request.initial_submit.is_empty() {
return Err(Error::InvalidInput(
"fixture Worker cannot contain initial_submit; offline seeding accepts no new input"
.into(),
));
}
if !matches!(request.intent, WorkerSpawnIntent::WorkspaceCoding)
|| request.singleton_key.is_some()
|| request.ticket_assignment.is_some()
|| request.resolved_subjektiv_attached
|| !request.workdir_attachment_requests.is_empty()
|| !request.resolved_workdir_attachment_requests.is_empty()
|| !request.resolved_workdir_attachments.is_empty()
|| request.resolved_config_bundle.is_some()
|| request.resolved_workspace_api.is_some()
|| request.resolved_memory_settings.is_some()
|| request.resolved_worker_observation_enabled
|| !request.resolved_worker_observation_grants.is_empty()
|| request.resolved_control_operation.is_some()
|| request.resolved_execution_grant.is_some()
{
return Err(Error::InvalidInput(
"fixture Worker must be ordinary unresolved create intent without singleton, assignments, attachments or grants".into(),
));
}
// Builtin selectors already carry their canonical `builtin:` prefix.
let profile = match &request.profile {
ProfileSelector::Builtin(name) | ProfileSelector::Named(name) => name,
};
validate_non_empty("fixture Profile selector", profile)?;
let worker_id = seed.worker_id.to_string();
let display_name = request
.requested_worker_name
.as_deref()
.unwrap_or(&worker_id);
validate_non_empty("fixture Worker display name", display_name)?;
let fingerprint = worker_spawn_create_fingerprint(request).map_err(Error::InvalidInput)?;
let memory = store.get_workspace_memory_settings(seed.workspace_id)?;
store.reserve_worker_create_with(
seed.workspace_id,
seed.runtime_id,
seed.allocation_key,
&fingerprint,
None,
&memory,
|tx| {
require_seed_target(tx, &seed)?;
Ok(Some(seed.worker_id))
},
|tx, reservation| {
let now = chrono::Utc::now().to_rfc3339();
tx.execute(
"INSERT INTO worker_registry (
workspace_id, runtime_id, worker_id, display_name, profile,
retention_state, created_at, updated_at
) VALUES (?1, ?2, ?3, ?4, ?5, 'normal', ?6, ?6)",
params![seed.workspace_id, seed.runtime_id, worker_id, display_name, profile, now],
)?;
let changed = tx.execute(
"UPDATE worker_create_reservations SET state = 'created', updated_at = ?4
WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 AND state = 'reserved'",
params![seed.workspace_id, seed.runtime_id, reservation.worker_id.to_string(), now],
)?;
if changed != 1 {
return Err(Error::Store("fixture creation reservation disappeared".into()));
}
Ok(())
},
)
}
fn require_seed_target(tx: &rusqlite::Transaction<'_>, seed: &FixedWorkerSeed<'_>) -> Result<()> {
let active: bool = tx.query_row(
"SELECT EXISTS(SELECT 1 FROM workspaces w
JOIN workspace_runtime_bindings r ON r.workspace_id = w.workspace_id
JOIN workspace_signing_identities s ON s.workspace_id = w.workspace_id
WHERE w.workspace_id = ?1 AND w.state = 'active'
AND r.runtime_id = ?2 AND r.revoked_at IS NULL
AND r.authentication_mode = 'workspace_identity' AND s.state = 'active')",
params![seed.workspace_id, seed.runtime_id],
|row| row.get(0),
)?;
if !active {
return Err(Error::InvalidInput(
"fixture requires an active bootstrapped Workspace and authenticated Runtime binding"
.into(),
));
}
let worker_id = seed.worker_id.to_string();
let allocation_exists: bool = tx.query_row(
"SELECT EXISTS(SELECT 1 FROM worker_create_reservations
WHERE workspace_id = ?1 AND allocation_key = ?2)",
params![seed.workspace_id, seed.allocation_key],
|row| row.get(0),
)?;
if allocation_exists {
return Err(Error::InvalidInput(
"fixture Worker allocation already exists".into(),
));
}
// Check by WorkerId, not just the proposed Runtime tuple: orphaned and terminal
// ownership is still a collision. These tables are owned by this store schema.
for table in [
"worker_create_reservations",
"worker_registry",
"worker_registry_observations",
"worker_registry_projection_removals",
"worker_removal_operations",
"worker_singleton_owners",
"worker_workdir_links",
"worker_restore_intents",
"worker_tombstones",
"worker_orphan_diagnostics",
"worker_registry_projection_diagnostics",
"worker_session_archives",
"worker_diagnostics_archives",
"ticket_assignment_worker_tombstones",
] {
let exists: bool = tx.query_row(
&format!("SELECT EXISTS(SELECT 1 FROM {table} WHERE worker_id = ?1)"),
[&worker_id],
|row| row.get(0),
)?;
if exists {
return Err(Error::InvalidInput(format!(
"fixture Worker identity already exists in {table}"
)));
}
}
// A resource key without a catalog/reservation must not be silently reused.
let mut statement = tx.prepare(
"SELECT resource_id FROM workspace_resource_keys WHERE resource_kind = 'worker'",
)?;
let resources = statement.query_map([], |row| row.get::<_, String>(0))?;
for resource in resources {
let resource = resource?;
let existing = resource
.is_ascii()
.then(|| parse_worker_resource_id(&resource))
.flatten()
.ok_or_else(|| {
Error::InvalidInput("unknown fixture Worker resource identity collision".into())
})?;
if existing.worker_id == worker_id {
return Err(Error::InvalidInput(
"fixture Worker resource identity already exists".into(),
));
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::hosts::{
WorkerSpawnAcceptanceRequirement, WorkerSpawnWorkingDirectoryAttachmentRequest,
WorkerSpawnWorkingDirectoryRequest,
};
use crate::workspace_signing_identity::{
InMemoryWorkspaceSigningMaterialStore, WorkspaceSigningIdentityService,
};
const WORKSPACE: &str = "0197a949-4b6b-7f2a-9d9a-1f87e3a4c501";
const RUNTIME: &str = "fixture-runtime";
const WORKER: &str = "0197a949-4b6b-7f2a-9d9a-1f87e3a4c502";
// Exercises the same public bootstrap APIs the offline harness uses. All account
// and key identities are newly generated, even across stores with fixed fixture IDs.
fn bootstrap() -> SqliteWorkspaceStore {
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
let account = AccountRecord {
account_id: format!("fixture-account-{}", Uuid::now_v7()),
kind: "user".into(),
handle: format!("fixture-{}", Uuid::now_v7()),
display_name: "Fixture owner".into(),
created_at: "2026-10-24T00:00:00Z".into(),
updated_at: "2026-10-24T00:00:00Z".into(),
};
store.upsert_account(&account).unwrap();
let identities = WorkspaceSigningIdentityService::new(
Arc::new(store.clone()),
Arc::new(InMemoryWorkspaceSigningMaterialStore::default()),
);
let (identity, operation) = identities
.prepare_workspace_creation(
"fixture-bootstrap",
"fixture-bootstrap",
WORKSPACE,
&account.account_id,
)
.unwrap();
store
.create_workspace_bootstrap(
&WorkspaceBootstrapRecord {
operation_key: "fixture-bootstrap".into(),
request_fingerprint: "fixture-bootstrap".into(),
workspace: WorkspaceRecord {
workspace_id: WORKSPACE.into(),
owner_account_id: account.account_id,
display_name: "Fixture".into(),
state: "active".into(),
created_at: account.created_at,
updated_at: account.updated_at,
},
repository: None,
},
&identity,
&operation,
)
.unwrap();
let runtime_key = worker_runtime::auth::RuntimeIdentityMaterial::generate(RUNTIME).unwrap();
store
.upsert_workspace_runtime_binding(
WorkspaceRuntimeBinding {
workspace_id: WORKSPACE.into(),
runtime_id: RUNTIME.into(),
display_name: "Fixture Runtime".into(),
base_url: "http://127.0.0.1:12345".into(),
public_key: runtime_key.public_key,
public_key_fingerprint: String::new(),
binding_id: String::new(),
authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity,
created_at: "2026-10-24T00:00:00Z".into(),
updated_at: "2026-10-24T00:00:00Z".into(),
revoked_at: None,
},
false,
)
.unwrap();
store
}
fn request() -> WorkerSpawnRequest {
WorkerSpawnRequest {
intent: WorkerSpawnIntent::WorkspaceCoding,
requested_worker_name: Some("Fixture Worker".into()),
singleton_key: None,
acceptance: WorkerSpawnAcceptanceRequirement::SocketReady,
profile: ProfileSelector::Builtin("builtin:coder".into()),
ticket_assignment: None,
initial_submit: Vec::new(),
workdir_attachment_requests: Vec::new(),
resolved_workdir_attachment_requests: Vec::new(),
resolved_workdir_attachments: Vec::new(),
resolved_config_bundle: None,
resolved_workspace_api: None,
resolved_memory_settings: None,
resolved_subjektiv_attached: false,
resolved_worker_observation_enabled: false,
resolved_worker_observation_grants: Vec::new(),
resolved_control_operation: None,
resolved_execution_grant: None,
}
}
fn seed(request: &WorkerSpawnRequest) -> FixedWorkerSeed<'_> {
FixedWorkerSeed {
workspace_id: WORKSPACE,
runtime_id: RUNTIME,
worker_id: WORKER.parse().unwrap(),
allocation_key: "fixture-allocation",
request,
}
}
fn worker() -> RuntimeWorkerRef {
RuntimeWorkerRef::new(RUNTIME, WORKER)
}
#[test]
fn fixed_ids_are_isolated_and_seed_has_no_runtime_observation() {
let a = bootstrap();
let b = bootstrap();
let request = request();
let first = seed_fixed_worker(&a, seed(&request)).unwrap();
let second = seed_fixed_worker(&b, seed(&request)).unwrap();
assert_eq!(first, second);
assert_eq!(first.worker_id.to_string(), WORKER);
assert_eq!(first.memory_settings.workspace_id, WORKSPACE);
assert_eq!(first.memory_settings.language, "English");
assert!(first.singleton.is_none());
assert_ne!(
a.get_workspace_signing_identity(WORKSPACE)
.unwrap()
.unwrap()
.public_key,
b.get_workspace_signing_identity(WORKSPACE)
.unwrap()
.unwrap()
.public_key
);
assert_ne!(
a.get_workspace_runtime_binding(WORKSPACE, RUNTIME)
.unwrap()
.unwrap()
.public_key,
b.get_workspace_runtime_binding(WORKSPACE, RUNTIME)
.unwrap()
.unwrap()
.public_key
);
for store in [&a, &b] {
assert_eq!(
store
.worker_resource_key(WORKSPACE, &worker())
.unwrap()
.as_deref(),
Some("W-1")
);
assert_eq!(
store
.resolve_worker_resource_reference(WORKSPACE, "W-1")
.unwrap(),
Some(worker())
);
assert!(
!store
.has_active_worker_create_reservation(WORKSPACE, &worker())
.unwrap()
);
let registry = store
.get_worker_registry(WORKSPACE, &worker())
.unwrap()
.unwrap();
assert_eq!(registry.profile.as_deref(), Some("builtin:coder"));
assert_eq!(registry.display_name, "Fixture Worker");
assert_eq!(registry.retention_state, "normal");
let projection = store
.with_conn(|conn| read_worker_registry_projection(conn, WORKSPACE, &worker()))
.unwrap()
.unwrap();
assert!(projection.observation.is_none());
assert!(projection.job.is_none());
}
}
#[test]
fn initial_submit_is_refused_without_registration_or_allocation() {
let store = bootstrap();
let mut request = request();
// Even an empty text segment is new input; the vector itself must be empty.
for text in ["new input must not be seeded", ""] {
request.initial_submit = vec![protocol::Segment::text(text)];
let error = seed_fixed_worker(&store, seed(&request)).unwrap_err();
assert!(
matches!(error, Error::InvalidInput(ref message) if message.contains("initial_submit"))
);
assert!(
store
.get_worker_registry(WORKSPACE, &worker())
.unwrap()
.is_none()
);
assert!(
store
.worker_resource_key(WORKSPACE, &worker())
.unwrap()
.is_none()
);
assert!(
!store
.has_active_worker_create_reservation(WORKSPACE, &worker())
.unwrap()
);
}
// The rejected attempts must not consume the allocation or resource counter.
request.initial_submit.clear();
seed_fixed_worker(&store, seed(&request)).unwrap();
assert_eq!(
store
.worker_resource_key(WORKSPACE, &worker())
.unwrap()
.as_deref(),
Some("W-1")
);
}
#[test]
fn exact_replay_changed_input_and_cross_runtime_identity_are_refused() {
let store = bootstrap();
let mut request = request();
seed_fixed_worker(&store, seed(&request)).unwrap();
let before = store.get_worker_registry(WORKSPACE, &worker()).unwrap();
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
request.requested_worker_name = Some("Must not overwrite".into());
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
let mut new_allocation = seed(&request);
new_allocation.allocation_key = "another-allocation";
assert!(seed_fixed_worker(&store, new_allocation).is_err());
let mut same_allocation = seed(&request);
same_allocation.worker_id = WorkerId::now_v7();
assert!(seed_fixed_worker(&store, same_allocation).is_err());
assert_eq!(
store.get_worker_registry(WORKSPACE, &worker()).unwrap(),
before
);
let other = bootstrap();
let mut record = before.unwrap();
record.worker.runtime_id = "different-runtime".into();
other.upsert_worker_registry(&record).unwrap();
assert!(seed_fixed_worker(&other, seed(&request)).is_err());
assert!(
other
.get_worker_registry(WORKSPACE, &worker())
.unwrap()
.is_none()
);
}
#[test]
fn reserved_and_orphaned_resource_identities_are_not_adopted() {
let store = bootstrap();
let request = request();
let memory = store.get_workspace_memory_settings(WORKSPACE).unwrap();
let ordinary = store
.reserve_worker_create(WORKSPACE, RUNTIME, "ordinary", "fingerprint", None, &memory)
.unwrap();
let mut collision = seed(&request);
collision.worker_id = ordinary.worker_id;
assert!(seed_fixed_worker(&store, collision).is_err());
// Store-level fault setup only: the external harness never writes SQL.
store
.with_conn(|conn| {
allocate_resource_key(
conn,
WORKSPACE,
WorkspaceResourceKind::Worker,
&worker_resource_id(&worker()),
"now",
)?;
Ok(())
})
.unwrap();
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
assert!(
store
.get_worker_registry(WORKSPACE, &worker())
.unwrap()
.is_none()
);
}
#[test]
fn unknown_resource_identity_and_retained_removal_fence_are_refused() {
let request = request();
let malformed = bootstrap();
malformed
.with_conn(|conn| {
allocate_resource_key(
conn,
WORKSPACE,
WorkspaceResourceKind::Worker,
"unknown-identity",
"now",
)?;
Ok(())
})
.unwrap();
assert!(matches!(
seed_fixed_worker(&malformed, seed(&request)),
Err(Error::InvalidInput(_))
));
assert!(
malformed
.get_worker_registry(WORKSPACE, &worker())
.unwrap()
.is_none()
);
let removed = bootstrap();
removed.with_conn(|conn| {
conn.execute(
"INSERT INTO worker_registry_projection_removals(workspace_id, runtime_id, worker_id) VALUES (?1, ?2, ?3)",
params![WORKSPACE, RUNTIME, WORKER],
)?;
Ok(())
}).unwrap();
assert!(matches!(
seed_fixed_worker(&removed, seed(&request)),
Err(Error::InvalidInput(_))
));
assert!(
removed
.worker_resource_key(WORKSPACE, &worker())
.unwrap()
.is_none()
);
}
#[test]
fn unknown_workspace_runtime_and_revoked_binding_are_refused() {
let store = bootstrap();
let request = request();
let mut unknown = seed(&request);
unknown.workspace_id = "unknown";
assert!(seed_fixed_worker(&store, unknown).is_err());
let mut unknown = seed(&request);
unknown.runtime_id = "unknown";
assert!(seed_fixed_worker(&store, unknown).is_err());
store
.revoke_workspace_runtime_binding(WORKSPACE, RUNTIME, "2026-10-24T01:00:00Z")
.unwrap();
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
assert!(
store
.worker_resource_key(WORKSPACE, &worker())
.unwrap()
.is_none()
);
}
#[test]
fn unsupported_intent_is_refused_without_partial_registration() {
let store = bootstrap();
let base = request();
let mut requests = Vec::new();
let mut request = base.clone();
request.singleton_key = Some("fixture-singleton".into());
requests.push(request);
let mut request = base.clone();
request.intent = WorkerSpawnIntent::WorkspaceOrchestrator;
requests.push(request);
let mut request = base.clone();
request.resolved_subjektiv_attached = true;
requests.push(request);
let mut request = base.clone();
request
.workdir_attachment_requests
.push(WorkerSpawnWorkingDirectoryAttachmentRequest {
alias: workdir::WorkdirAttachmentAlias::new("repo").unwrap(),
working_directory: WorkerSpawnWorkingDirectoryRequest {
repository_key: "repo".into(),
selector: None,
},
});
requests.push(request);
for request in requests {
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
}
assert!(
store
.get_worker_registry(WORKSPACE, &worker())
.unwrap()
.is_none()
);
assert!(
store
.worker_resource_key(WORKSPACE, &worker())
.unwrap()
.is_none()
);
assert_eq!(
seed_fixed_worker(&store, seed(&base))
.unwrap()
.worker_id
.to_string(),
WORKER
);
}
#[test]
fn late_catalog_failure_rolls_back_reservation_and_resource_allocation() {
let store = bootstrap();
let request = request();
store
.with_conn(|conn| {
conn.execute_batch(
"CREATE TEMP TRIGGER fail_fixture_catalog BEFORE INSERT ON worker_registry
BEGIN SELECT RAISE(ABORT, 'injected catalog failure'); END",
)?;
Ok(())
})
.unwrap();
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
assert!(
!store
.has_active_worker_create_reservation(WORKSPACE, &worker())
.unwrap()
);
assert!(
store
.worker_resource_key(WORKSPACE, &worker())
.unwrap()
.is_none()
);
store
.with_conn(|conn| {
conn.execute_batch("DROP TRIGGER fail_fixture_catalog")?;
Ok(())
})
.unwrap();
seed_fixed_worker(&store, seed(&request)).unwrap();
assert_eq!(
store
.worker_resource_key(WORKSPACE, &worker())
.unwrap()
.as_deref(),
Some("W-1")
);
}
#[test]
fn ordinary_creation_still_allocates_and_replays_without_catalog_insertion() {
let store = bootstrap();
let memory = store.get_workspace_memory_settings(WORKSPACE).unwrap();
let reserve = |key| {
store
.reserve_worker_create(WORKSPACE, RUNTIME, key, "fingerprint", None, &memory)
.unwrap()
};
let first = reserve("ordinary-one");
assert_eq!(first, reserve("ordinary-one"));
let second = reserve("ordinary-two");
assert_ne!(first.worker_id, second.worker_id);
assert_ne!(first.worker_id.to_string(), WORKER);
assert_eq!(
first.worker_id.as_uuid().get_version(),
Some(uuid::Version::SortRand)
);
let worker = RuntimeWorkerRef::new(RUNTIME, first.worker_id.to_string());
assert!(
store
.has_active_worker_create_reservation(WORKSPACE, &worker)
.unwrap()
);
assert!(
store
.get_worker_registry(WORKSPACE, &worker)
.unwrap()
.is_none()
);
assert!(
store
.reserve_worker_create(WORKSPACE, RUNTIME, "ordinary-one", "changed", None, &memory)
.is_err()
);
}
}
+2 -2
View File
@@ -1,8 +1,8 @@
{
"tasks": {
"test": "deno run --allow-read --allow-write=target --allow-run --allow-env tools/test.ts",
"test": "deno run --allow-read --allow-write=target --allow-run --allow-env --allow-sys=uid tools/test.ts",
"test:ticket:real-api": "cargo test -p yoi-workspace-server --lib ticket_web_roundtrip_tests -- --ignored",
"test:runner": "deno test --allow-read --allow-write --allow-run --allow-env tools/test.test.ts"
"test:runner": "deno test --allow-read --allow-write --allow-run --allow-env --allow-sys=uid tools/test.test.ts"
},
"fmt": {
"proseWrap": "never"
+11 -2
View File
@@ -15697,7 +15697,9 @@
"type": "object"
},
"WorkerRestoreState": {
"description": "Cross-layer outcome of an explicit Worker restore operation.\n\n`Rejected` is reserved for read-only preflight failures. Once live restore\nwork starts, failure is either a confirmed `RolledBack` operation or a\n`ReconciliationRequired` result whose commit state must be reread/retried.",
"description": "Cross-layer outcome of an explicit Worker restore operation.\n\n`Rejected` is reserved for read-only preflight failures. Once live restore\nwork starts, distinguish confirmed `RolledBack`, unresolved\n`ReconciliationRequired`, and a later Stop's `Superseded` outcome.",
"oneOf": [
{
"enum": [
"accepted",
"rejected",
@@ -15706,6 +15708,13 @@
],
"type": "string"
},
{
"const": "superseded",
"description": "A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.",
"type": "string"
}
]
},
"WorkerRetentionResponse": {
"additionalProperties": false,
"properties": {
@@ -18155,7 +18164,7 @@
"info": {
"title": "Yoi Server API",
"version": "0.1.0",
"x-yoi-source-digest": "sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7"
"x-yoi-source-digest": "sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e"
},
"openapi": "3.1.0",
"paths": {
+13
View File
@@ -9,6 +9,8 @@ autotests = false
[features]
default = []
e2e = []
fixture-support = ["yoi-workspace-server/test-support", "worker-runtime/test-support"]
restore-checkpoints = ["fixture-support", "worker-runtime/restore-checkpoints"]
[dependencies]
libc.workspace = true
@@ -17,11 +19,17 @@ serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
sha2.workspace = true
tempfile.workspace = true
protocol.workspace = true
tungstenite = { version = "0.29", default-features = false, features = ["handshake"] }
[dev-dependencies]
chrono = { version = "0.4", default-features = false, features = ["clock"] }
yoi-workspace-server.workspace = true
server-api.workspace = true
worker-runtime.workspace = true
session-store.workspace = true
config-source.workspace = true
manifest.workspace = true
webauthn-rs-proto = "0.5.5"
webauthn-authenticator-rs = { version = "0.5.5", default-features = false, features = ["softpasskey"] }
@@ -44,3 +52,8 @@ required-features = ["e2e"]
name = "worker_environment"
path = "tests/worker_environment.rs"
required-features = ["e2e"]
[[test]]
name = "worker_restore"
path = "tests/worker_restore.rs"
required-features = ["e2e", "fixture-support"]
+64
View File
@@ -7,3 +7,67 @@
- [`PanelHarness`]: PTY経由の入力、観測イベントによる同期、失敗時のartifact保存。
- [`FixtureWorkspace`]: TUI用WorkspaceとHOME/XDG領域の隔離。
- [`WorkerE2eEnvironment`]: Backend/Runtime用プロセスとfixtureの管理。
- [`FixtureSource`]: 提供資産の全ファイルSHA-256記録、独立copy、原本不変の検証。
- [`WorkerSessionSubscription`]: 公開Worker WebSocketの初期full Snapshot照合と停止確認。モデル呼出しは行わない。
## 固定Workerの段階起動
1. 提供fixtureを所有crateの型で検証し、`FixtureSource`で原本hashを記録する。各環境の一時領域へcopyする。fixtureが未提供・不適合なら失敗とし、代替Workerを生成しない。
2. `initialize_runtime_identity`で固定Runtime IDに新しい鍵を発行する。Backendの型付きstore APIで新しいAccount認証・Workspace署名identity・Runtime binding・固定Worker登録を準備する。
3. `start_seeded_backend`へ固定Workspace/Runtime IDとその環境の新しいbearerを渡す。Server起動後の公開APIからWorkspaceの公開identityを取得する。
4. `start_runtime_with_seed`はtrust設定後、Runtime processの起動前にseed closureを呼ぶ。ここで実Server URLを使いRuntime所有crateの型付きinstallを呼ぶ。Runtime store rootは`runtime_data_dir.join("runtime")`。
5. Worker一覧とdetailから停止状態・観測tokenを確認し、一つの固定request IDで`restore_worker`を一度だけ呼ぶ。結果不明・拒否は成功扱いせず、tokenを自動更新しない。
6. `WorkerSessionSubscription::connect`で期待`protocol::SessionSnapshot`全体(entry ID、timestamp、順序、本文を含む)を照合する。二環境を同時に保持し、一方のstop/cleanup後も他方の購読・一覧・内容を再確認する。
7. `stop_worker`、購読の`expect_stopped`、環境cleanup、原本`assert_unchanged`を確認する。
所有crateの`test-support`依存はテスト側の非default `fixture-support` featureだけで有効にする。製品binary providerは別Cargo invocationと専用`target/e2e-products`へ`yoi`・`yoi-server`・`yoi-runtime`を追加featureなしでビルドする。support test自身をこのtargetへビルドしてはならない。`YOI_E2E_BIN`等の明示binary overrideを使用する場合も、呼出し側が同じ分離を保証する。既存TUI用`e2e-test` featureは別の`target/e2e-tui-products`へビルドし、restore製品binaryへ混入しない。
固定資産の正常restore・並行隔離・応答喪失回復E2E:
```sh
cargo test -p yoi-e2e --features e2e,fixture-support --test worker_restore
# 親から提供された候補の検証(読取専用、原本の自動採用や生成はしない)
YOI_E2E_RESTORE_FIXTURE=target/fixture-candidate-v1 \
cargo test -p yoi-e2e --features e2e,fixture-support --test worker_restore
```
標準資産は`tests/e2e/fixtures/worker-codex-v1`の`manifest.json`・`worker-template.json`・`expected.json`・`workspace-config.dcdl`。manifestの各asset hash/size、schemaと固定ID、生成元commit、modelを検証する。Workspace configのimport参照を黙って最新へ置換せず、Gitで生成元commitに対する`resources`と`crates/builtin-source`の一致を確認する(Git履歴がない場合も失敗)。移植したSessionの正規公開projectionを供給された会話行と照合し、その全snapshotを実executionのWebSocketと比較する。欠落fixture・hash不一致・未実行は成功扱いしない。
応答喪失ケースは同じ`WorkerE2eEnvironment`内の単発loopback relayで公開restore要求をそのまま転送し、完全なupstream応答を受け取った後に呼出元へ0byteで切断する。呼出元のtransport失敗とrelay側のAccepted観測は別の事実として記録する。回収は元のrequest ID・観測tokenをそのまま使い、fresh intentへの古いtokenはHTTP409を要求する。
process終了ケースはそのAccepted receipt確認後に環境が所有するRuntime childだけをkill/reapし、同じroot・Runtime identity・trustから再起動する。seedの再実行・tokenの差替えはしない。再起動前後のcanonical owner全体のhash・operation ID・owner数が一致し、公開APIの保存結果とfull Session snapshotを回収できることを確認する。公開Worker投影の観測tokenは再起動で変わるため、そのfieldだけはcurrent detailとの一致を別に確認する。canonical診断はboundedな読取りのみで、要求・receiptの内容や認証情報はartifactへ保存しない。
これは**commit済みかつterminal receipt保存後**のprocess終了であり、未commit候補やcommit直後・receipt前の終了証明ではない。固定Codex資産へSubmit/Notifyを送らないため、通常入力・Paused admissionの製品E2Eの証明でもない。検証jobはこの明示targetを選択する必要があり、通常suiteだけでは代替しない。
## Opt-in Runtime checkpoint process試験
上のfeatureless製品4case(target内の既存seed projection試験1件とは別)に加え、`restore-checkpoints`を明示した実Runtimeでdecision前後とactivation後・terminal前のlive windowを試験する。このfeatureはoffline fixture-supportと独立しており、通常RuntimeではCLI入口も環境変数による入口も無い。
```sh
# 同期を有効にしたRuntimeだけを専用targetへ。通常製品binaryを上書きしない。
cargo build --locked -p worker-runtime --bin yoi-runtime \
--features restore-checkpoints --target-dir target/e2e-checkpoint-products
cargo test --locked -p yoi-e2e --features e2e,restore-checkpoints \
--test worker_restore checkpoint:: -- --test-threads=1
```
`WorkerE2eEnvironment::use_checkpoint_runtime`はその環境のRuntime binary/hashと、隔離root内の明示configだけを選択する。global provider・環境変数・通常起動への暗黙設定は変更しない。configは一度だけCLIへ渡し、再起動時は渡さない。test側のlink featureをfeatureless製品binaryへ混入させない。
共有harnessはloopback ackのstage/Worker/operation/PID/nonceを確認し、直ちに所有childをkill/reapする。**offline正本を確認してから**同roots/identity/trustを再起動するため、ack後のdeadlineとの競合を成功扱いしない。NotCommittedの実私有候補は同operationのcleanup-only→RolledBack/Stopped、Committed/preterminalは同じdecision→実protocol/full Session snapshot→Acceptedへ回収する。元のrequest/token、owner数1、同operation、stale fresh intent拒否、保存source hash、restore run path数、最終cleanupを検証する。JSONでpending状態を作らず、seedは初回のみ。
orphan upload pinの代表repairは、隔離copyへ正規Session storeのupload/pin APIで作った未結付けpinを、実Profileのcommit後activationが解放する経路を使う。Prepared窓は未修復のsource保持、Activated窓は実修復bytes/hashの保持を区別し、kill/reap後に正本を読んでから再起動する。materialized headの固定資産を巻き戻さないため、この試験は先行repair済みNotCommittedのprocess証明ではない。Activated後のstale allocationは通常登録と同じallocation authorityで回収し、live ownerが残る場合はrun artifactsを回収しない。
明示解除・peer EOF・observer scopeの意図的panicも試験する。gateの8秒deadlineとscoped HTTP callの15秒deadlineが待機を制限し、通常operationの継続後に環境のRAIIがprocess/rootを回収する。これはtest scope失敗の回収証明であり、OSによるharness強制終了や電源断の一般的な保証ではない。原fixtureへのSubmit/Notifyや実モデル呼出しは行わない。checkpoint ack/at-death/recovery、kill/reap、binary/hashとcleanup artifactはfeaturelessログと分けて報告する。
通常のharness回帰試験(製品processは起動しない):
```sh
cargo test -p yoi-e2e --lib --no-default-features
```
既存の製品process E2Eは別結果として実行・報告する:
```sh
cargo test -p yoi-e2e --features e2e --test worker_environment
cargo test -p yoi-e2e --features e2e --test panel --test rewind
```
@@ -0,0 +1,52 @@
{
"source_revision": "1639be98411921308238b3a6a6e75c784c045156",
"model": "codex-oauth/gpt-6.1-sol",
"worker_id": "01a1657d-b762-7791-87ae-a49f28cadbce",
"session_id": "01a1657d-b818-77c1-bd73-175a27562f63",
"rows": [
{
"entry_id": "01a1657f-18bf-7a03-b8b8-953d27041fe0",
"timestamp": 1792704190655,
"role": "user",
"parts": [
{
"kind": "text",
"content": "これは保存と復元を確認するための人工的な会話です。合言葉は「青い灯台」です。覚えたことを一文で返してください。Toolは使わないでください。"
}
]
},
{
"entry_id": "01a1657f-1f55-71d3-b418-8199b863f59a",
"timestamp": 1792704192341,
"role": "assistant",
"parts": [
{
"kind": "text",
"text": "合言葉は「青い灯台」と覚えました。"
}
]
},
{
"entry_id": "01a1657f-3053-71f3-bf4f-91290d82488b",
"timestamp": 1792704196691,
"role": "user",
"parts": [
{
"kind": "text",
"content": "先ほどの合言葉は何でしたか?合言葉だけを答えてください。"
}
]
},
{
"entry_id": "01a1657f-36ef-78b1-b0c8-b141f13b0c7b",
"timestamp": 1792704198383,
"role": "assistant",
"parts": [
{
"kind": "text",
"text": "青い灯台"
}
]
}
]
}
@@ -0,0 +1,34 @@
{
"format_version": 1,
"source_revision": "1639be98411921308238b3a6a6e75c784c045156",
"runtime_schema_version": 12,
"session_schema_version": 6,
"workspace_id": "01a16553-b901-7191-8ceb-8e2306fe3da9",
"runtime_id": "fixture-951707ee-93ca-4205-9c12-45f08a104a74",
"worker_id": "01a1657d-b762-7791-87ae-a49f28cadbce",
"session_id": "01a1657d-b818-77c1-bd73-175a27562f63",
"segment_id": "01a1657d-b818-77c1-bd73-176172a4bd82",
"model": "codex-oauth/gpt-6.1-sol",
"state": "stopped",
"acquisition": "Real isolated Codex OAuth two-turn artificial conversation; typed offline StoppedWorkerTemplate extraction. Session bytes unchanged; environment credentials and execution receipts excluded.",
"source_file_sha256": {
"worker.json": "912e620013a17f05c1aaf900da48c9ee0ecb1f54121efc3721e03f8755eb5eca",
"metadata.json": "657928a89ec621bd092b86924201ed7b562e9683261789ce18bdb1d248bdc450",
"session/session.json": "22b01daab68a28b9777a1f7abb9ef9f7f6cbae113089446602f5d15421e7c7f3",
"session/segments/01a1657d-b818-77c1-bd73-176172a4bd82.jsonl": "7c5663bb06d9406d5bd0a5086159b31c0d7799458fcb1c6f81e18b63c49f4df5"
},
"files": {
"worker-template.json": {
"bytes": 48343,
"sha256": "8bf6ce3c71c4f05c82314de93c1e8e51b7502b00915b6adb8b33046944e463b2"
},
"expected.json": {
"bytes": 1474,
"sha256": "b9846db47ecb8f3f53b2abd9d5f0c766122f86015a7a7df7addf8beab3e9c55a"
},
"workspace-config.dcdl": {
"bytes": 1141,
"sha256": "9b07507b71733081f077e2e9968a7f1f676c916a243bb355d1813f067a55c415"
}
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,22 @@
{
profile = {
default_profile = "project:fixture-chat";
entries = [{
selector = "project:fixture-chat";
label = "Isolated Codex fixture conversation";
description = "Artificial conversation only; no Workdir, Tools, Subject or background work.";
profile = (import "$builtin/profiles/job.dcdl") // {
slug = "fixture-chat";
worker = { mode = "tools"; };
model = { ref = "codex-oauth/gpt-6.1-sol"; };
scope = "workspace_read";
delegation_scope = "workspace_read";
permissions = { default_action = "deny"; rule = []; };
engine = { instruction = "default"; reasoning = "low"; max_tokens = 2048; max_turns = 4; language = "Japanese"; };
session = { record_event_trace = false; };
feature = { workdir_catalog = { enabled = false; }; };
};
}];
};
prompts = { default_prompt = "これは隔離環境の人工的な会話テストです。日本語で簡潔に応答してください。Toolや外部操作は行わず、ユーザーが会話中に提示した情報だけを使ってください。"; };
} as WorkspaceConfigSchema
+336
View File
@@ -0,0 +1,336 @@
//! Bounded byte-preserving transport, independent of the owning store's schema validation.
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use sha2::{Digest, Sha256};
use crate::{HarnessError, Result};
#[derive(Debug, Clone, Copy)]
struct Limits {
file_bytes: u64,
total_bytes: u64,
entries: usize,
depth: usize,
}
const LIMITS: Limits = Limits {
file_bytes: 8 * 1024 * 1024,
total_bytes: 32 * 1024 * 1024,
entries: 256,
depth: 8,
};
#[derive(Debug, Clone)]
pub struct FixtureSource {
root: PathBuf,
hashes: BTreeMap<PathBuf, String>,
flat_files: Option<BTreeSet<PathBuf>>,
}
fn reject(message: &'static str) -> HarnessError {
HarnessError::Protocol(message.into())
}
impl FixtureSource {
/// Capture a bounded tree without generating or repairing store data.
pub fn capture(root: &Path) -> Result<Self> {
Self::capture_with(root, None)
}
/// Validate the complete flat file set before reading any content. The same
/// allowlist is retained for copy and later source-integrity checks.
pub fn capture_flat(root: &Path, names: &[&str]) -> Result<Self> {
let files: BTreeSet<_> = names.iter().map(PathBuf::from).collect();
if files.len() != names.len()
|| files.iter().any(|name| {
name.components().count() != 1
|| !matches!(
name.components().next(),
Some(std::path::Component::Normal(_))
)
})
{
return Err(reject("invalid fixture file allowlist"));
}
Self::capture_with(root, Some(files))
}
fn capture_with(root: &Path, flat_files: Option<BTreeSet<PathBuf>>) -> Result<Self> {
let hashes = inventory(root, flat_files.as_ref(), LIMITS)?;
if hashes.is_empty() {
return Err(reject("provided fixture is empty"));
}
Ok(Self {
root: root.to_owned(),
hashes,
flat_files,
})
}
/// Read a known asset through a descriptor with a real byte limit, including
/// when its size grows after metadata inspection. Errors never include paths.
pub fn read_bounded(path: &Path) -> Result<Vec<u8>> {
read_regular(path, LIMITS.file_bytes)
}
pub fn hashes(&self) -> &BTreeMap<PathBuf, String> {
&self.hashes
}
pub fn assert_unchanged(&self) -> Result<()> {
if inventory(&self.root, self.flat_files.as_ref(), LIMITS)? != self.hashes {
return Err(reject("provided fixture changed during execution"));
}
Ok(())
}
/// The destination must not exist. A copy is never a link to the source.
pub fn copy_to(&self, destination: &Path) -> Result<()> {
self.assert_unchanged()?;
let parent = destination
.parent()
.filter(|path| !path.as_os_str().is_empty())
.unwrap_or_else(|| Path::new("."));
if parent
.canonicalize()?
.starts_with(self.root.canonicalize()?)
{
return Err(reject("fixture copy destination is inside the original"));
}
fs::create_dir(destination)?;
let mut remaining = LIMITS.total_bytes;
for (relative, expected) in &self.hashes {
let bytes = read_regular(&self.root.join(relative), LIMITS.file_bytes.min(remaining))?;
remaining -= bytes.len() as u64;
if hash(&bytes) != *expected {
return Err(reject("fixture changed while copying"));
}
let target = destination.join(relative);
fs::create_dir_all(target.parent().unwrap())?;
let mut file = fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(target)?;
file.write_all(&bytes)?;
}
if inventory(destination, self.flat_files.as_ref(), LIMITS)? != self.hashes {
return Err(reject("fixture copy failed hash verification"));
}
self.assert_unchanged()
}
}
fn hash(bytes: &[u8]) -> String {
format!(
"sha256:{}",
Sha256::digest(bytes)
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>()
)
}
fn read_limited(reader: impl Read, limit: u64) -> Result<Vec<u8>> {
let mut bytes = Vec::new();
reader
.take(limit + 1)
.read_to_end(&mut bytes)
.map_err(|_| reject("fixture content read failed"))?;
if bytes.len() as u64 > limit {
return Err(reject("fixture byte budget exceeded"));
}
Ok(bytes)
}
fn read_regular(path: &Path, limit: u64) -> Result<Vec<u8>> {
let mut options = fs::OpenOptions::new();
options.read(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
// Do not follow a replaced final symlink or block on a replaced FIFO.
options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK);
}
let file = options
.open(path)
.map_err(|_| reject("fixture asset open failed"))?;
let metadata = file
.metadata()
.map_err(|_| reject("fixture asset metadata failed"))?;
if !metadata.is_file() || metadata.len() > limit {
return Err(reject("fixture asset is not a bounded regular file"));
}
read_limited(file, limit)
}
fn inventory(
root: &Path,
flat: Option<&BTreeSet<PathBuf>>,
limits: Limits,
) -> Result<BTreeMap<PathBuf, String>> {
struct Budget {
entries: usize,
bytes: u64,
}
fn visit(
root: &Path,
path: &Path,
depth: usize,
limits: Limits,
flat: Option<&BTreeSet<PathBuf>>,
budget: &mut Budget,
files: &mut BTreeSet<PathBuf>,
) -> Result<()> {
if depth > limits.depth || budget.entries == 0 {
return Err(reject("fixture tree budget exceeded"));
}
budget.entries -= 1;
let metadata = fs::symlink_metadata(path).map_err(|_| reject("fixture metadata failed"))?;
let relative = path.strip_prefix(root).unwrap();
if depth != 0 && flat.is_some_and(|names| !names.contains(relative)) {
return Err(reject("unexpected fixture file set"));
}
if metadata.is_dir() && (depth == 0 || flat.is_none()) {
for entry in fs::read_dir(path).map_err(|_| reject("fixture directory read failed"))? {
let entry = entry.map_err(|_| reject("fixture directory entry failed"))?;
visit(root, &entry.path(), depth + 1, limits, flat, budget, files)?;
}
} else if metadata.is_file() && depth != 0 {
if metadata.len() > limits.file_bytes || metadata.len() > budget.bytes {
return Err(reject("fixture byte budget exceeded"));
}
budget.bytes -= metadata.len();
files.insert(relative.to_owned());
} else {
return Err(reject("fixture contains a symlink or special file"));
}
Ok(())
}
// Preflight the entire shape and metadata budget before reading any body.
let mut files = BTreeSet::new();
visit(
root,
root,
0,
limits,
flat,
&mut Budget {
entries: limits.entries,
bytes: limits.total_bytes,
},
&mut files,
)?;
if flat.is_some_and(|expected| expected != &files) {
return Err(reject("unexpected fixture file set"));
}
let mut remaining = limits.total_bytes;
let mut hashes = BTreeMap::new();
for relative in files {
let bytes = read_regular(&root.join(&relative), limits.file_bytes.min(remaining))?;
remaining -= bytes.len() as u64;
hashes.insert(relative, hash(&bytes));
}
Ok(hashes)
}
#[cfg(test)]
mod tests {
use super::*;
// Transport-only synthetic bytes, never a substitute saved Worker fixture.
#[test]
fn copies_are_independent_and_source_mutation_is_detected() -> Result<()> {
let temp = tempfile::tempdir()?;
let source = temp.path().join("source");
fs::create_dir(&source)?;
fs::write(source.join("bytes"), b"original")?;
let fixture = FixtureSource::capture_flat(&source, &["bytes"])?;
assert!(fixture.copy_to(&source.join("nested-copy")).is_err());
assert!(!source.join("nested-copy").exists());
fixture.assert_unchanged()?;
let first = temp.path().join("first");
let second = temp.path().join("second");
fixture.copy_to(&first)?;
fixture.copy_to(&second)?;
fs::write(first.join("bytes"), b"changed")?;
fs::remove_dir_all(first)?;
assert_eq!(fs::read(second.join("bytes"))?, b"original");
fixture.assert_unchanged()?;
fs::write(source.join("bytes"), b"changed")?;
assert!(fixture.assert_unchanged().is_err());
Ok(())
}
#[test]
fn rejects_missing_empty_existing_destination_and_extra_file() -> Result<()> {
let temp = tempfile::tempdir()?;
assert!(FixtureSource::capture(&temp.path().join("missing")).is_err());
assert!(FixtureSource::capture(temp.path()).is_err());
fs::write(temp.path().join("bytes"), b"original")?;
let source = FixtureSource::capture(temp.path())?;
assert!(source.copy_to(temp.path()).is_err());
fs::write(temp.path().join("extra"), b"new")?;
assert!(source.assert_unchanged().is_err());
Ok(())
}
#[test]
fn flat_allowlist_rejects_extra_entries_before_reading_content() -> Result<()> {
let temp = tempfile::tempdir()?;
fs::write(temp.path().join("asset"), b"ok")?;
// Sparse oversized input is never read. Even an allowed file that cannot
// be decoded does not take precedence over an unexpected file set.
fs::File::create(temp.path().join("extra"))?.set_len(LIMITS.file_bytes + 1)?;
let error = FixtureSource::capture_flat(temp.path(), &["asset"]).unwrap_err();
assert!(error.to_string().contains("unexpected fixture file set"));
fs::remove_file(temp.path().join("extra"))?;
assert!(FixtureSource::capture_flat(temp.path(), &["missing"]).is_err());
fs::create_dir(temp.path().join("dir"))?;
assert!(FixtureSource::capture_flat(temp.path(), &["asset", "dir"]).is_err());
Ok(())
}
#[test]
fn tree_count_depth_total_and_actual_read_budgets_are_enforced() -> Result<()> {
let temp = tempfile::tempdir()?;
fs::write(temp.path().join("a"), b"1234")?;
fs::write(temp.path().join("b"), b"1234")?;
for limits in [
Limits {
entries: 2,
..LIMITS
},
Limits {
total_bytes: 7,
..LIMITS
},
Limits {
file_bytes: 3,
..LIMITS
},
] {
assert!(inventory(temp.path(), None, limits).is_err());
}
fs::create_dir_all(temp.path().join("nested/deep"))?;
assert!(inventory(temp.path(), None, Limits { depth: 1, ..LIMITS }).is_err());
// A reader can return more bytes than earlier metadata reported. Bound
// actual IO rather than trusting that observation or an EOF at stat size.
assert!(read_limited(std::io::Cursor::new(b"12345"), 4).is_err());
assert_eq!(read_limited(std::io::Cursor::new(b"1234"), 4)?, b"1234");
Ok(())
}
#[cfg(unix)]
#[test]
fn rejects_symbolic_links() -> Result<()> {
let temp = tempfile::tempdir()?;
fs::write(temp.path().join("bytes"), b"original")?;
std::os::unix::fs::symlink("bytes", temp.path().join("link"))?;
assert!(FixtureSource::capture(temp.path()).is_err());
assert!(FixtureSource::read_bounded(&temp.path().join("link")).is_err());
Ok(())
}
}
+35 -5
View File
@@ -137,7 +137,9 @@ fn tested_yoi_env_policy_overview() -> TestedYoiEnvPolicy {
#[derive(Debug)]
pub enum HarnessError {
Io(io::Error),
Json(serde_json::Error),
// Never retain decode errors: serde messages can contain rejected field names
// or values from private fixture/protocol payloads, including through Debug.
Json,
CommandFailed {
program: PathBuf,
args: Vec<String>,
@@ -163,7 +165,7 @@ impl std::fmt::Display for HarnessError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Io(err) => write!(f, "io error: {err}"),
Self::Json(err) => write!(f, "json error: {err}"),
Self::Json => write!(f, "invalid JSON document"),
Self::CommandFailed {
program,
args,
@@ -209,8 +211,8 @@ impl From<io::Error> for HarnessError {
}
impl From<serde_json::Error> for HarnessError {
fn from(value: serde_json::Error) -> Self {
Self::Json(value)
fn from(_: serde_json::Error) -> Self {
Self::Json
}
}
@@ -1268,7 +1270,11 @@ impl Drop for PanelHarness {
}
}
mod fixture_source;
pub use fixture_source::FixtureSource;
mod worker_environment;
mod worker_session;
pub use worker_session::{WorkerSessionSubscription, validate_full_snapshot};
pub use worker_environment::{
WorkerBinaryProviderInfo, WorkerE2eEnvironment, WorkerEnvironmentCleanupReport,
@@ -1627,6 +1633,8 @@ fn resolve_yoi_binary() -> Result<BinaryProviderInfo> {
"e2e-test".to_owned(),
"--bin".to_owned(),
"yoi".to_owned(),
"--target-dir".to_owned(),
tui_product_target_dir()?.to_string_lossy().into_owned(),
];
if test_profile() == "release" {
args.push("--release".to_owned());
@@ -1648,7 +1656,9 @@ fn resolve_yoi_binary() -> Result<BinaryProviderInfo> {
});
}
let binary = current_target_profile_dir()?.join(binary_name());
let binary = tui_product_target_dir()?
.join(test_profile())
.join(binary_name());
let info = BinaryProviderInfo {
provider: "cargo-build".to_owned(),
binary,
@@ -1675,6 +1685,26 @@ fn workspace_root() -> Result<PathBuf> {
.ok_or_else(|| HarnessError::Protocol("could not resolve workspace root".to_owned()))
}
/// Product processes must never reuse the target used by test-support builds.
fn product_target_dir() -> Result<PathBuf> {
dedicated_product_target("e2e-products")
}
// Existing PTY tests need their observer feature, never on restore binaries.
fn tui_product_target_dir() -> Result<PathBuf> {
dedicated_product_target("e2e-tui-products")
}
fn dedicated_product_target(name: &str) -> Result<PathBuf> {
let target = workspace_root()?.join("target").join(name);
if current_target_profile_dir()?.parent() == Some(target.as_path()) {
return Err(HarnessError::Protocol(
"run support tests outside reserved product binary targets".to_owned(),
));
}
Ok(target)
}
fn current_target_profile_dir() -> Result<PathBuf> {
let mut path = std::env::current_exe()?;
while let Some(name) = path.file_name().and_then(|name| name.to_str()) {
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,198 @@
//! A single bounded response-loss injection in the shared product harness.
use std::io::{Read, Write};
use std::net::{IpAddr, TcpListener, TcpStream};
use std::time::Instant;
use reqwest::blocking::{Client, Request};
use serde_json::Value;
use super::{EXIT_WAIT, HTTP_WAIT_STEP, STARTUP_WAIT};
use crate::{HarnessError, Result};
const MAX_HEADER: usize = 16 * 1024;
const MAX_MESSAGE: usize = 2 * 1024 * 1024;
fn failure(message: &str) -> HarnessError {
HarnessError::Protocol(message.into())
}
pub(super) fn drop_response(client: &Client, mut request: Request) -> Result<Value> {
let host: IpAddr = request
.url()
.host_str()
.ok_or_else(|| failure("missing relay host"))?
.parse()
.map_err(|_| failure("relay target must be a literal loopback address"))?;
if !host.is_loopback() || request.url().scheme() != "http" {
return Err(failure(
"response-loss relay only permits owned loopback HTTP",
));
}
let port = request
.url()
.port()
.ok_or_else(|| failure("missing relay port"))?;
let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))?;
listener.set_nonblocking(true)?;
let relay_address = listener.local_addr()?;
request
.url_mut()
.set_port(Some(relay_address.port()))
.map_err(|_| failure("set relay port"))?;
request
.url_mut()
.set_host(Some("127.0.0.1"))
.map_err(|_| failure("set relay host"))?;
std::thread::scope(|scope| {
let relay = scope.spawn(move || -> Result<Value> {
let deadline = Instant::now() + STARTUP_WAIT;
let mut downstream = loop {
match listener.accept() {
Ok((stream, peer)) if peer.ip().is_loopback() => break stream,
Ok(_) => return Err(failure("non-loopback relay peer")),
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
if Instant::now() >= deadline {
return Err(failure("response-loss relay accept deadline exceeded"));
}
std::thread::sleep(HTTP_WAIT_STEP);
}
Err(error) => return Err(error.into()),
}
};
let request_bytes = read_message(&mut DeadlineStream {
stream: &mut downstream,
deadline,
})?;
let mut upstream = TcpStream::connect_timeout(&(host, port).into(), EXIT_WAIT)?;
upstream.set_write_timeout(Some(EXIT_WAIT))?;
// The body, request identity and authorization bytes are forwarded
// unchanged. Neither request nor response headers enter diagnostics.
upstream.write_all(&request_bytes)?;
let response = read_message(&mut DeadlineStream {
stream: &mut upstream,
deadline,
})?;
let boundary =
header_end(&response).ok_or_else(|| failure("missing upstream header"))?;
let status = std::str::from_utf8(&response[..boundary])
.ok()
.and_then(|header| header.lines().next())
.and_then(|line| line.split_whitespace().nth(1));
if status != Some("200") {
return Err(failure("restore relay upstream did not return HTTP 200"));
}
let value = serde_json::from_slice(&response[boundary..])?;
// Intentionally send zero bytes back; the caller must observe loss.
drop(downstream);
Ok(value)
});
let caller_failed = client.execute(request).is_err();
let evidence = relay
.join()
.map_err(|_| failure("response-loss relay panicked"))??;
if !caller_failed {
return Err(failure("caller unexpectedly received a response"));
}
Ok(evidence)
})
}
struct DeadlineStream<'a> {
stream: &'a mut TcpStream,
deadline: Instant,
}
impl Read for DeadlineStream<'_> {
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
let remaining = self.deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
return Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"relay deadline exceeded",
));
}
self.stream.set_read_timeout(Some(remaining))?;
self.stream.read(buffer)
}
}
fn header_end(bytes: &[u8]) -> Option<usize> {
bytes
.windows(4)
.position(|window| window == b"\r\n\r\n")
.map(|i| i + 4)
}
fn read_message(stream: &mut impl Read) -> Result<Vec<u8>> {
let mut bytes = Vec::new();
let mut buffer = [0; 4096];
let mut expected = None;
loop {
let count = stream.read(&mut buffer)?;
if count == 0 {
return Err(failure("relay peer closed before complete HTTP message"));
}
bytes.extend_from_slice(&buffer[..count]);
if bytes.len() > MAX_MESSAGE {
return Err(failure("relay HTTP byte budget exceeded"));
}
if expected.is_none() {
if let Some(boundary) = header_end(&bytes) {
if boundary > MAX_HEADER {
return Err(failure("relay HTTP header budget exceeded"));
}
let header = std::str::from_utf8(&bytes[..boundary])
.map_err(|_| failure("invalid relay HTTP header"))?;
let mut lengths = header
.lines()
.filter_map(|line| line.split_once(':'))
.filter(|(name, _)| name.eq_ignore_ascii_case("content-length"))
.map(|(_, value)| value.trim().parse::<usize>());
let length = lengths
.next()
.ok_or_else(|| failure("relay requires Content-Length"))?
.map_err(|_| failure("invalid relay Content-Length"))?;
if lengths.next().is_some()
|| length > MAX_MESSAGE - boundary
|| header
.lines()
.filter_map(|line| line.split_once(':'))
.any(|(name, _)| name.eq_ignore_ascii_case("transfer-encoding"))
{
return Err(failure("unsupported relay HTTP framing"));
}
expected = Some(boundary + length);
} else if bytes.len() > MAX_HEADER {
return Err(failure("relay HTTP header budget exceeded"));
}
}
if let Some(expected) = expected {
if bytes.len() == expected {
return Ok(bytes);
}
if bytes.len() > expected {
return Err(failure("unexpected relay trailing bytes"));
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn framing_rejects_incomplete_ambiguous_or_unbounded_messages() {
for bytes in [
b"HTTP/1.1 200 OK\r\n\r\n".as_slice(),
b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nshort",
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nContent-Length: 0\r\n\r\n",
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nTransfer-Encoding: chunked\r\n\r\n",
b"HTTP/1.1 200 OK\r\nContent-Length: 999999999\r\n\r\n",
] {
assert!(read_message(&mut &bytes[..]).is_err());
}
let valid = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\n{}";
assert_eq!(read_message(&mut &valid[..]).unwrap(), valid);
}
}
+284
View File
@@ -0,0 +1,284 @@
//! Live public Worker protocol assertions. No Submit/Resume/model request is sent.
use std::net::TcpStream;
use std::time::{Duration, Instant};
use protocol::{Event, SessionSnapshot, WorkerState};
use tungstenite::{Message, WebSocket, client::IntoClientRequest};
use crate::{HarnessError, Result, WorkerE2eEnvironment};
const WS_WAIT: Duration = Duration::from_secs(10);
pub struct WorkerSessionSubscription {
socket: WebSocket<TcpStream>,
}
impl WorkerSessionSubscription {
pub fn connect(
environment: &WorkerE2eEnvironment,
worker_id: &str,
expected: &SessionSnapshot,
) -> Result<Self> {
let authorized = environment.worker_protocol_request(worker_id)?;
let mut url = authorized.url().clone();
url.set_scheme("ws")
.map_err(|_| error("invalid protocol URL"))?;
let mut request = url
.as_str()
.into_client_request()
.map_err(|_| error("build protocol handshake"))?;
request.headers_mut().extend(authorized.headers().clone());
// Only this environment's loopback endpoint is permitted, never a proxy.
let address = url
.socket_addrs(|| None)
.map_err(|_| error("resolve protocol address"))?
.into_iter()
.find(|address| address.ip().is_loopback())
.ok_or_else(|| error("protocol address is not loopback"))?;
let stream = TcpStream::connect_timeout(&address, WS_WAIT)?;
stream.set_read_timeout(Some(WS_WAIT))?;
stream.set_write_timeout(Some(WS_WAIT))?;
let (socket, _) =
tungstenite::client(request, stream).map_err(|_| error("protocol handshake failed"))?;
let mut subscription = Self { socket };
let deadline = Instant::now() + WS_WAIT;
loop {
match subscription.read(deadline)? {
Message::Text(text) => {
let raw: serde_json::Value = serde_json::from_str(&text)?;
validate_full_snapshot(&raw, expected)?;
// Only validated public content is retained, never request headers.
std::fs::write(
environment.artifacts_dir.join("restored-session.json"),
serde_json::to_vec_pretty(&raw)?,
)?;
return Ok(subscription);
}
Message::Ping(_) => subscription
.socket
.flush()
.map_err(|_| error("flush protocol Pong"))?,
Message::Pong(_) => {}
_ => return Err(error("protocol ended before its initial full Snapshot")),
}
}
}
/// Prove an already-open subscription remains usable after the other
/// environment is stopped and cleaned up. This does not invoke the model.
pub fn assert_alive(&mut self) -> Result<()> {
let payload = b"restore-e2e-isolation".to_vec();
self.socket
.send(Message::Ping(payload.clone().into()))
.map_err(|_| error("send protocol Ping"))?;
let deadline = Instant::now() + WS_WAIT;
loop {
match self.read(deadline)? {
Message::Pong(bytes) if bytes.as_ref() == payload.as_slice() => return Ok(()),
Message::Ping(_) => self
.socket
.flush()
.map_err(|_| error("flush protocol Pong"))?,
Message::Pong(_) => {}
_ => {
return Err(error(
"idle subscription changed or ended during isolation check",
));
}
}
}
}
/// A public stop must terminate the connected execution, not merely update
/// a retained catalog row. Require the shutdown event or websocket close.
pub fn expect_stopped(mut self) -> Result<()> {
let deadline = Instant::now() + WS_WAIT;
loop {
match self.read(deadline)? {
Message::Close(_) => return Ok(()),
Message::Text(text) => match serde_json::from_str::<Event>(&text)? {
Event::Shutdown => return Ok(()),
Event::WorkerState { .. } => {}
_ => return Err(error("unexpected protocol event after Worker stop")),
},
Message::Ping(_) => self
.socket
.flush()
.map_err(|_| error("flush protocol Pong"))?,
Message::Pong(_) => {}
_ => return Err(error("unexpected protocol frame after Worker stop")),
}
}
}
fn read(&mut self, deadline: Instant) -> Result<Message> {
let remaining = deadline
.checked_duration_since(Instant::now())
.filter(|duration| !duration.is_zero())
.ok_or_else(|| error("protocol deadline exceeded"))?;
self.socket.get_mut().set_read_timeout(Some(remaining))?;
self.socket
.read()
.map_err(|_| error("protocol read failed or timed out"))
}
}
fn error(message: &str) -> HarnessError {
HarnessError::Protocol(message.into())
}
pub fn validate_full_snapshot(raw: &serde_json::Value, expected: &SessionSnapshot) -> Result<()> {
let event: Event = serde_json::from_value(raw.clone())?;
// Product DTOs intentionally tolerate unknown fields for compatibility. This
// fixture boundary does not: anything discarded by typed decoding must never
// be persisted to an artifact, including fields outside the Session payload.
if serde_json::to_value(&event)? != *raw {
return Err(error(
"noncanonical or unknown fields in full protocol Snapshot envelope",
));
}
let Event::Snapshot {
session,
state,
in_flight,
internal_workers,
..
} = event
else {
return Err(error("first protocol event is not a full Snapshot"));
};
if state.state != WorkerState::Idle || !in_flight.is_empty() || !internal_workers.is_empty() {
return Err(error(
"restored Worker is not an idle, completed standalone Session",
));
}
// Typed equality covers entry order/ID/timestamp/provenance/content; raw
// equality also refuses unknown fields that could leak private data.
if session != *expected
|| raw.pointer("/data/session") != Some(&serde_json::to_value(expected)?)
{
return Err(error(
"full Session snapshot differs from supplied expected rows",
));
}
if !session.pending_submissions.submissions.is_empty() {
return Err(error("fixture has pending submissions"));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use protocol::{
Greeting, InFlightSnapshot, Segment, SessionEntryProvenance, SessionSnapshotEntry,
SessionSnapshotEntryData, WorkerStateSnapshot, WorkerStatus,
};
// Protocol-validation samples only, not a substitute saved Worker fixture.
fn sample() -> (serde_json::Value, SessionSnapshot) {
let session = SessionSnapshot {
pending_submissions: Default::default(),
entries: vec![SessionSnapshotEntry {
entry_id: "E-unit-1".into(),
timestamp: 123,
provenance: SessionEntryProvenance::HumanInput,
derived_from: vec![],
use_policy: None,
data: SessionSnapshotEntryData::UserInput {
segments: vec![Segment::text("unit input")],
},
}],
};
let event = Event::Snapshot {
session: session.clone(),
state: WorkerStateSnapshot::from(WorkerStatus::Idle),
in_flight: InFlightSnapshot::default(),
internal_workers: vec![],
greeting: Greeting {
worker_name: "unit".into(),
cwd: ".".into(),
provider: "unit".into(),
model: "unit".into(),
reasoning: None,
scope_summary: "none".into(),
tools: vec![],
context_window: 0,
context_tokens: 0,
context_usage: None,
},
};
(serde_json::to_value(event).unwrap(), session)
}
#[test]
fn full_snapshot_requires_exact_ids_timestamps_contents_and_no_unknown_fields() {
let (raw, expected) = sample();
validate_full_snapshot(&raw, &expected).unwrap();
for (key, value) in [
("entry_id", serde_json::json!("different")),
("timestamp", serde_json::json!(124)),
(
"hidden_reasoning",
serde_json::json!("must never be exposed"),
),
("segments", serde_json::json!([])),
] {
let mut changed = raw.clone();
changed["data"]["session"]["entries"][0][key] = value;
assert!(
validate_full_snapshot(&changed, &expected).is_err(),
"{key}"
);
}
let mut missing = raw.clone();
missing["data"]["session"]["entries"] = serde_json::json!([]);
assert!(validate_full_snapshot(&missing, &expected).is_err());
let mut extra = raw;
let entry = extra["data"]["session"]["entries"][0].clone();
extra["data"]["session"]["entries"]
.as_array_mut()
.unwrap()
.push(entry);
assert!(validate_full_snapshot(&extra, &expected).is_err());
}
#[test]
fn full_snapshot_rejects_unknown_fields_outside_session_before_artifact_write() {
let (raw, expected) = sample();
for pointer in [
"",
"/data",
"/data/greeting",
"/data/state",
"/data/state/state",
] {
let mut changed = raw.clone();
changed
.pointer_mut(pointer)
.unwrap()
.as_object_mut()
.unwrap()
.insert(
"hidden_reasoning".into(),
serde_json::json!("must not reach artifact"),
);
assert!(
validate_full_snapshot(&changed, &expected).is_err(),
"{pointer}"
);
}
let mut changed = raw;
changed["data"]["greeting"]["private_key"] = serde_json::json!("must not reach artifact");
assert!(validate_full_snapshot(&changed, &expected).is_err());
}
#[test]
fn non_snapshot_event_cannot_stand_in_for_live_snapshot() {
let (_, expected) = sample();
assert!(
validate_full_snapshot(&serde_json::to_value(Event::Shutdown).unwrap(), &expected)
.is_err()
);
}
}
+5 -5
View File
@@ -41,13 +41,13 @@ fn backend_panel_loads_backend_owned_tickets_and_supports_reload_navigation_and_
&event.data,
&ready_ticket_id,
"Planning E2E Ticket A",
"planning",
"open",
)
&& has_backend_ticket(
&event.data,
&planning_ticket_id,
"Planning E2E Ticket B",
"planning",
"open",
)
},
)?;
@@ -82,13 +82,13 @@ fn backend_panel_loads_backend_owned_tickets_and_supports_reload_navigation_and_
&event.data,
&ready_ticket_id,
"Planning E2E Ticket A",
"planning",
"open",
)
&& has_backend_ticket(
&event.data,
&planning_ticket_id,
"Planning E2E Ticket B",
"planning",
"open",
)
},
)?;
@@ -137,7 +137,7 @@ fn backend_panel_shell_enter_path_uses_the_same_backend_authority() -> yoi_e2e::
&event.data,
&ready_ticket_id,
"Planning E2E Ticket A",
"planning",
"open",
)
},
)?;
+457
View File
@@ -0,0 +1,457 @@
//! Fixed, externally supplied Worker restore. Never generates a fixture or sends model input.
#[cfg(feature = "restore-checkpoints")]
#[path = "worker_restore/checkpoint.rs"]
mod checkpoint;
#[path = "worker_restore/fixture.rs"]
mod fixture;
#[path = "worker_restore/seed.rs"]
mod seed;
use std::fs;
use std::time::{Duration, Instant};
use fixture::{Fixture, failure};
use seed::Installed;
use server_api::{
RuntimeWorkerLifecycleResult, RuntimeWorkerShowResponse, WorkerListResponse,
WorkerOperationState, WorkerRestoreResponse, WorkerRestoreState, WorkerSummary,
};
use yoi_e2e::{FixtureSource, Result, WorkerE2eEnvironment, WorkerSessionSubscription};
#[test]
fn supplied_stopped_worker_restores_exact_live_session_and_cleans_up() -> Result<()> {
let fixture = Fixture::load()?;
let result = (|| {
let installed = seed::start(&fixture)?;
let subscription = restore(&installed, &fixture, "fixed-fixture-restore-single")?;
stop_and_cleanup(installed, vec![subscription], &fixture)
})();
fixture.source.assert_unchanged()?;
result
}
#[test]
fn concurrent_fixed_id_restores_keep_sessions_storage_and_cleanup_isolated() -> Result<()> {
let fixture = Fixture::load()?;
let result = (|| {
let (first, second) = std::thread::scope(|scope| {
let first = scope.spawn(|| seed::start(&fixture));
let second = seed::start(&fixture);
(
first
.join()
.map_err(|_| failure("first environment startup panicked"))
.and_then(|result| result),
second,
)
});
let first = first?;
let second = second?;
assert_eq!(
first.environment.workspace_id,
second.environment.workspace_id
);
assert_eq!(first.environment.runtime_id, second.environment.runtime_id);
assert_ne!(first.environment.root, second.environment.root);
assert_ne!(
first.environment.server_data_dir,
second.environment.server_data_dir
);
assert_ne!(
first.environment.runtime_data_dir,
second.environment.runtime_data_dir
);
assert_ne!(
first.environment.artifacts_dir,
second.environment.artifacts_dir
);
assert_ne!(
first.environment.account_handle,
second.environment.account_handle
);
assert_ne!(first.runtime_public_key, second.runtime_public_key);
assert_ne!(first.workspace_public_key, second.workspace_public_key);
assert_ne!(
first.environment.server_port()?,
second.environment.server_port()?
);
assert_ne!(
first.environment.runtime_port()?,
second.environment.runtime_port()?
);
assert_ne!(
first.environment.server_pid(),
second.environment.server_pid()
);
assert_ne!(
first.environment.runtime_pid(),
second.environment.runtime_pid()
);
first
.environment
.assert_authentication_isolated_from(&second.environment)?;
second
.environment
.assert_authentication_isolated_from(&first.environment)?;
let (first_subscription, second_subscription) = std::thread::scope(|scope| {
// Deliberately reuse request identity in isolated stores, just like fixed IDs.
let a = scope.spawn(|| restore(&first, &fixture, "fixed-fixture-restore-concurrent"));
let b = restore(&second, &fixture, "fixed-fixture-restore-concurrent");
(
a.join()
.map_err(|_| failure("first restore panicked"))
.and_then(|result| result),
b,
)
});
let first_subscription = first_subscription?;
let mut second_subscription = second_subscription?;
let second_session_root = second
.environment
.runtime_data_dir
.join("runtime/workers")
.join(&fixture.manifest.worker_id)
.join("session");
let second_session = FixtureSource::capture(&second_session_root)?;
let first_root = first.environment.root.clone();
let first_artifacts = first.environment.artifacts_dir.clone();
stop_and_cleanup(first, vec![first_subscription], &fixture)?;
assert!(!first_root.exists());
assert!(first_artifacts.join("cleanup.json").is_file());
assert!(second.environment.root.is_dir());
assert!(second.environment.server_data_dir.is_dir());
assert!(second.environment.runtime_data_dir.is_dir());
assert!(
second
.environment
.artifacts_dir
.join("restored-session.json")
.is_file()
);
second.environment.assert_connected()?;
assert_single_state(&second.environment, &fixture, "idle")?;
second_subscription.assert_alive()?;
// A second live connection independently verifies content after peer cleanup.
let mut reconnected = WorkerSessionSubscription::connect(
&second.environment,
&fixture.manifest.worker_id,
&second.snapshot,
)?;
reconnected.assert_alive()?;
second_session.assert_unchanged()?;
stop_and_cleanup(second, vec![second_subscription, reconnected], &fixture)?;
Ok(())
})();
fixture.source.assert_unchanged()?;
result
}
#[test]
fn lost_http_response_recovers_exact_request_and_rejects_fresh_stale_intent() -> Result<()> {
response_loss_recovery(false)
}
#[test]
fn committed_response_loss_survives_owned_runtime_process_kill_and_restart() -> Result<()> {
response_loss_recovery(true)
}
fn response_loss_recovery(crash_runtime: bool) -> Result<()> {
let fixture = Fixture::load()?;
let result = (|| {
let mut installed = seed::start(&fixture)?;
let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?;
let token = stopped
.restore_observation_token
.as_deref()
.filter(|token| !token.is_empty())
.ok_or_else(|| failure("missing stopped observation token"))?;
let request_id = "fixed-fixture-response-loss";
phase(
&installed.environment,
"drop-http-response-after-complete-upstream",
)?;
let observed = installed.environment.restore_worker_losing_response(
&fixture.manifest.worker_id,
request_id,
token,
)?;
let receipt: WorkerRestoreResponse = serde_json::from_value(observed.clone())?;
if receipt.result.state != WorkerRestoreState::Accepted {
return Err(failure(
"upstream did not reach Accepted before response loss",
));
}
fs::write(
installed
.environment
.artifacts_dir
.join("response-loss.json"),
serde_json::to_vec_pretty(&serde_json::json!({
"caller_received_response": false,
"relay_upstream_state": receipt.result.state,
"request_id": request_id,
"crash_after_committed_terminal_receipt": crash_runtime,
}))?,
)?;
let owner_before = restore_owner_evidence(&installed.environment, &fixture, request_id)?;
if crash_runtime {
phase(
&installed.environment,
"kill-owned-runtime-after-committed-receipt",
)?;
installed.environment.crash_and_restart_runtime()?;
}
phase(
&installed.environment,
"same-request-original-token-recovery",
)?;
let recovered =
installed
.environment
.restore_worker(&fixture.manifest.worker_id, request_id, token)?;
if crash_runtime {
// The public response projects the current observation fence. Runtime
// startup invalidates prior active observations; this is not a change
// to the original request token or its saved outcome.
let current = installed
.environment
.worker_detail(&fixture.manifest.worker_id)?;
assert_eq!(
recovered.pointer("/result/worker/restore_observation_token"),
current.get("restore_observation_token")
);
let mut expected = observed.clone();
expected["result"]["worker"]["restore_observation_token"] =
recovered["result"]["worker"]["restore_observation_token"].clone();
assert_eq!(
recovered, expected,
"restart may change only the public observation fence, not the saved outcome"
);
} else {
assert_eq!(
recovered, observed,
"exact request must recover the saved receipt"
);
}
assert_single_state(&installed.environment, &fixture, "idle")?;
let mut subscription = WorkerSessionSubscription::connect(
&installed.environment,
&fixture.manifest.worker_id,
&installed.snapshot,
)?;
subscription.assert_alive()?;
let stale = installed.environment.restore_worker(
&fixture.manifest.worker_id,
"fixed-fixture-fresh-stale-intent",
token,
);
match stale {
Err(error) if error.to_string().contains("HTTP 409") => {}
_ => {
return Err(failure(
"fresh intent with old observation was not HTTP 409",
));
}
}
let owner_after = restore_owner_evidence(&installed.environment, &fixture, request_id)?;
assert_eq!(
owner_after, owner_before,
"saved owner/operation/receipt must not change"
);
fs::write(
installed
.environment
.artifacts_dir
.join("restore-owner-evidence.json"),
serde_json::to_vec_pretty(&serde_json::json!({
"before": owner_before, "after": owner_after,
"request_token_replaced": false,
}))?,
)?;
installed.environment.assert_connected()?;
subscription.assert_alive()?;
stop_and_cleanup(installed, vec![subscription], &fixture)
})();
fixture.source.assert_unchanged()?;
result
}
// Read-only diagnostic of the product-written aggregate; never seeds or edits it.
// Persist only a digest, operation ID and owner count, not prompts or credentials.
fn restore_owner_evidence(
environment: &WorkerE2eEnvironment,
fixture: &Fixture,
request_id: &str,
) -> Result<serde_json::Value> {
use sha2::{Digest, Sha256};
use std::io::Read;
let path = environment
.runtime_data_dir
.join("runtime/workers")
.join(&fixture.manifest.worker_id)
.join("worker.json");
let mut bytes = Vec::new();
fs::File::open(path)?
.take(8 * 1024 * 1024 + 1)
.read_to_end(&mut bytes)?;
if bytes.len() > 8 * 1024 * 1024 {
return Err(failure("Worker aggregate byte budget exceeded"));
}
let aggregate: serde_json::Value = serde_json::from_slice(&bytes)?;
let owners = aggregate
.pointer("/restore_guard/owners")
.and_then(serde_json::Value::as_object)
.ok_or_else(|| failure("canonical Worker omitted restore owners"))?;
let owner = owners
.get(request_id)
.ok_or_else(|| failure("canonical owner missing"))?;
if owner
.pointer("/receipt/state")
.and_then(serde_json::Value::as_str)
!= Some("accepted")
{
return Err(failure("canonical owner has no Accepted receipt"));
}
let operation = owner
.get("operation_id")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| failure("canonical owner omitted operation ID"))?;
let digest: String = Sha256::digest(serde_json::to_vec(owner)?)
.iter()
.map(|byte| format!("{byte:02x}"))
.collect();
Ok(
serde_json::json!({"operation_id": operation, "owner_sha256": digest, "owner_count": owners.len()}),
)
}
fn restore(
installed: &Installed,
fixture: &Fixture,
request_id: &str,
) -> Result<WorkerSessionSubscription> {
phase(&installed.environment, "observe-stopped")?;
let stopped = assert_single_state(&installed.environment, fixture, "stopped")?;
let token = stopped
.restore_observation_token
.as_deref()
.filter(|token| !token.is_empty())
.ok_or_else(|| failure("stopped Worker did not expose a restore observation token"))?;
phase(&installed.environment, "restore")?;
let result: WorkerRestoreResponse = serde_json::from_value(
installed
.environment
.restore_worker(&fixture.manifest.worker_id, request_id, token)?,
)?;
if result.workspace_id != fixture.manifest.workspace_id
|| result.runtime_id != fixture.manifest.runtime_id
|| result.worker_id != fixture.manifest.worker_id
|| result.result.state != WorkerRestoreState::Accepted
{
return Err(failure(format!(
"restore did not commit accepted: {:?}",
result.result.state
)));
}
assert_single_state(&installed.environment, fixture, "idle")?;
let detail: RuntimeWorkerShowResponse = serde_json::from_value(
installed
.environment
.worker_detail(&fixture.manifest.worker_id)?,
)?;
if detail.worker.worker_id != fixture.manifest.worker_id
|| detail.worker.runtime_id != fixture.manifest.runtime_id
|| detail.worker.state != "idle"
|| detail
.worker
.worker_state
.as_ref()
.is_none_or(|state| state.state != protocol::WorkerState::Idle)
{
return Err(failure(
"restored Worker detail is not the single live idle execution",
));
}
phase(&installed.environment, "subscribe-live-session")?;
let mut subscription = WorkerSessionSubscription::connect(
&installed.environment,
&fixture.manifest.worker_id,
&installed.snapshot,
)?;
subscription.assert_alive()?;
phase(&installed.environment, "live-session-verified")?;
Ok(subscription)
}
fn assert_single_state(
environment: &WorkerE2eEnvironment,
fixture: &Fixture,
state: &str,
) -> Result<WorkerSummary> {
let deadline = Instant::now() + Duration::from_secs(15);
loop {
let list: WorkerListResponse = serde_json::from_value(environment.list_workers()?)?;
if list.workspace_id != fixture.manifest.workspace_id || list.items.len() > 1 {
return Err(failure(
"Workspace Worker list has wrong identity or duplicate Workers",
));
}
if let Some(worker) = list.items.into_iter().next() {
if worker.worker_id != fixture.manifest.worker_id
|| worker.runtime_id != fixture.manifest.runtime_id
{
return Err(failure("Workspace listed an unexpected Worker identity"));
}
if worker.state == state {
return Ok(worker);
}
}
if Instant::now() >= deadline {
return Err(failure(format!(
"Worker list did not become exactly one {state} Worker"
)));
}
std::thread::sleep(Duration::from_millis(50));
}
}
fn stop_and_cleanup(
installed: Installed,
subscriptions: Vec<WorkerSessionSubscription>,
fixture: &Fixture,
) -> Result<()> {
phase(&installed.environment, "stop")?;
let result: RuntimeWorkerLifecycleResult = serde_json::from_value(
installed
.environment
.stop_worker(&fixture.manifest.worker_id)?,
)?;
if result.state != WorkerOperationState::Accepted {
return Err(failure("Worker stop was not accepted"));
}
installed.environment.assert_connected()?;
for subscription in subscriptions {
subscription.expect_stopped()?;
// A transport close caused by Server/Runtime shutdown is not stop evidence.
installed.environment.assert_connected()?;
}
assert_single_state(&installed.environment, fixture, "stopped")?;
phase(&installed.environment, "shutdown-cleanup")?;
let report = installed.environment.cleanup()?;
if !report.cleanup_success || report.fixture_root_exists_after {
return Err(failure("environment cleanup failed"));
}
if !report.report_path.is_file() {
return Err(failure("cleanup evidence is missing"));
}
Ok(())
}
fn phase(environment: &WorkerE2eEnvironment, phase: &str) -> Result<()> {
fs::write(
environment.artifacts_dir.join("restore-phase.json"),
serde_json::to_vec_pretty(&serde_json::json!({"phase":phase}))?,
)?;
Ok(())
}
@@ -0,0 +1,498 @@
//! Opt-in real-process checkpoint cases; never substitute the featureless lane.
#[path = "repair.rs"]
mod repair;
use super::*;
use std::collections::BTreeMap;
use std::io::{Read, Write};
use std::net::{TcpListener, TcpStream};
use std::path::{Path, PathBuf};
use worker_runtime::restore_checkpoints::{
MAX_RESTORE_CHECKPOINT_ACK_BYTES, RestoreCheckpointAck, RestoreCheckpointCommit,
RestoreCheckpointConfig, RestoreCheckpointStage,
};
struct Peer {
listener: TcpListener,
config: RestoreCheckpointConfig,
connection: Option<TcpStream>,
}
impl Peer {
fn configure(
environment: &mut WorkerE2eEnvironment,
fixture: &Fixture,
stage: RestoreCheckpointStage,
) -> Result<Self> {
let listener = TcpListener::bind("127.0.0.1:0")?;
listener.set_nonblocking(true)?;
let config = RestoreCheckpointConfig {
endpoint: listener.local_addr()?,
worker_id: fixture::support(fixture.manifest.worker_id.parse())?,
stage,
nonce: format!(
"owned-checkpoint-{}-{}",
std::process::id(),
listener.local_addr()?.port()
),
};
let path = environment.root.join("restore-checkpoint.json");
fs::write(&path, serde_json::to_vec(&config)?)?;
let provider = yoi_e2e::worker_binary_provider_info()?;
let binary = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../../target/e2e-checkpoint-products")
.join(provider.profile)
.join("yoi-runtime");
environment.use_checkpoint_runtime(binary, &path)?;
Ok(Self {
listener,
config,
connection: None,
})
}
fn acknowledge(&mut self, environment: &WorkerE2eEnvironment) -> Result<RestoreCheckpointAck> {
let deadline = Instant::now() + Duration::from_secs(15);
let (mut stream, address) = loop {
match self.listener.accept() {
Ok(value) => break value,
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
if Instant::now() >= deadline {
return Err(failure("checkpoint ack deadline exceeded"));
}
std::thread::sleep(Duration::from_millis(10));
}
Err(error) => return Err(error.into()),
}
};
if !address.ip().is_loopback() {
return Err(failure("non-loopback checkpoint peer"));
}
let mut bytes = Vec::new();
loop {
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
return Err(failure("checkpoint ack read deadline exceeded"));
}
stream.set_read_timeout(Some(remaining))?;
let mut byte = [0];
stream.read_exact(&mut byte)?;
if byte == [b'\n'] {
break;
}
bytes.push(byte[0]);
if bytes.len() > MAX_RESTORE_CHECKPOINT_ACK_BYTES {
return Err(failure("checkpoint ack oversized"));
}
}
let ack: RestoreCheckpointAck = serde_json::from_slice(&bytes)?;
if ack.nonce != self.config.nonce
|| ack.stage != self.config.stage
|| ack.worker_id != self.config.worker_id
|| ack.runtime_id != environment.runtime_id
|| Some(ack.pid) != environment.runtime_pid()
|| !ack.receipt_absent
{
return Err(failure(
"checkpoint ack does not match owned process and stage",
));
}
fs::write(
environment.artifacts_dir.join("checkpoint-ack.json"),
serde_json::to_vec_pretty(&ack)?,
)?;
self.connection = Some(stream);
Ok(ack)
}
fn release(&mut self) -> Result<()> {
let mut stream = self
.connection
.take()
.ok_or_else(|| failure("checkpoint not acknowledged"))?;
stream.set_write_timeout(Some(Duration::from_secs(1)))?;
stream.write_all(b"R")?;
Ok(())
}
}
fn aggregate_path(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> PathBuf {
environment
.runtime_data_dir
.join("runtime/workers")
.join(&fixture.manifest.worker_id)
}
fn canonical(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> Result<serde_json::Value> {
let mut bytes = Vec::new();
fs::File::open(aggregate_path(environment, fixture).join("worker.json"))?
.take(8 * 1024 * 1024 + 1)
.read_to_end(&mut bytes)?;
if bytes.len() > 8 * 1024 * 1024 {
return Err(failure("canonical aggregate oversized"));
}
Ok(serde_json::from_slice(&bytes)?)
}
fn saved_source_hashes(
environment: &WorkerE2eEnvironment,
fixture: &Fixture,
) -> Result<BTreeMap<String, String>> {
fn visit(root: &Path, path: &Path, files: &mut BTreeMap<String, String>) -> Result<()> {
if files.len() > 128 {
return Err(failure("saved source file budget exceeded"));
}
let metadata = fs::symlink_metadata(path)?;
if metadata.file_type().is_symlink() {
return Err(failure("unexpected saved source symlink"));
}
if metadata.is_dir() {
for entry in fs::read_dir(path)? {
visit(root, &entry?.path(), files)?;
}
} else if metadata.is_file() {
if metadata.len() > 8 * 1024 * 1024 {
return Err(failure("saved source byte budget exceeded"));
}
use sha2::{Digest, Sha256};
let digest: String = Sha256::digest(fs::read(path)?)
.iter()
.map(|byte| format!("{byte:02x}"))
.collect();
files.insert(
path.strip_prefix(root)
.unwrap()
.to_string_lossy()
.into_owned(),
digest,
);
} else {
return Err(failure("unexpected saved source entry type"));
}
Ok(())
}
let root = aggregate_path(environment, fixture);
let mut files = BTreeMap::new();
for name in ["metadata.json", "session"] {
visit(&root, &root.join(name), &mut files)?;
}
if files.len() < 2 {
return Err(failure("saved source inventory is incomplete"));
}
Ok(files)
}
fn restore_runs(environment: &WorkerE2eEnvironment, fixture: &Fixture) -> Result<Vec<String>> {
let runs = aggregate_path(environment, fixture).join("runs");
if !runs.exists() {
return Ok(Vec::new());
}
let mut names = Vec::new();
for entry in fs::read_dir(runs)? {
let entry = entry?;
if entry.file_name().to_string_lossy().starts_with("restore-") {
if !entry.file_type()?.is_dir() {
return Err(failure("restore run is not a directory"));
}
names.push(entry.file_name().to_string_lossy().into_owned());
}
}
names.sort();
Ok(names)
}
#[test]
fn acknowledged_not_committed_candidate_process_death_recovers_same_request_as_rollback()
-> Result<()> {
crash_window(RestoreCheckpointStage::PreparedNotCommitted)
}
#[test]
fn acknowledged_committed_preterminal_process_death_reconstructs_same_decision() -> Result<()> {
crash_window(RestoreCheckpointStage::CommittedPreterminal)
}
fn crash_window(stage: RestoreCheckpointStage) -> Result<()> {
let fixture = Fixture::load()?;
let result = (|| {
let mut environment = WorkerE2eEnvironment::new()?;
let mut peer = Peer::configure(&mut environment, &fixture, stage)?;
let mut installed = seed::start_in(&fixture, environment)?;
let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?;
let token = stopped
.restore_observation_token
.ok_or_else(|| failure("missing original observation"))?;
let original_sources = saved_source_hashes(&installed.environment, &fixture)?;
let request_id = "owned-process-checkpoint-request";
let call = installed.environment.pending_restore_call(
&fixture.manifest.worker_id,
request_id,
&token,
)?;
let ack = std::thread::scope(|scope| -> Result<RestoreCheckpointAck> {
let request = scope.spawn(call);
let ack = peer.acknowledge(&installed.environment)?;
installed.environment.crash_runtime()?;
// With the actual child reaped there can be no deadline race while
// inspecting the exact decision and nonterminal owner at process death.
let saved = canonical(&installed.environment, &fixture)?;
let op = saved
.pointer("/execution_state/execution/intent")
.ok_or_else(|| failure("checkpoint lacks canonical pending operation"))?;
let owner = &saved["restore_guard"]["owners"][request_id];
assert_eq!(op["operation_id"], serde_json::to_value(ack.operation_id)?);
assert_eq!(owner["operation_id"], op["operation_id"]);
assert!(owner["receipt"].is_null());
assert_eq!(owner["request"]["expected_observation_token"], token);
assert_eq!(op["source_repair_possible"], false);
let committed = stage == RestoreCheckpointStage::CommittedPreterminal;
assert_eq!(
op["commit"],
if committed {
"committed"
} else {
"not_committed"
}
);
assert_eq!(
ack.commit,
if committed {
RestoreCheckpointCommit::Committed
} else {
RestoreCheckpointCommit::NotCommitted
}
);
assert_eq!(
saved_source_hashes(&installed.environment, &fixture)?,
original_sources
);
assert_eq!(
restore_runs(&installed.environment, &fixture)?,
vec![format!("restore-{}", ack.operation_id)]
);
fs::write(
installed
.environment
.artifacts_dir
.join("checkpoint-at-death.json"),
serde_json::to_vec_pretty(&serde_json::json!({
"runtime_reaped_before_inspection": installed.environment.runtime_pid().is_none(),
"operation_id": op["operation_id"], "commit": op["commit"],
"receipt_absent": owner["receipt"].is_null(),
"source_repair_possible": op["source_repair_possible"],
"same_original_request_and_token": true,
"restore_runs": restore_runs(&installed.environment, &fixture)?,
"saved_source_hashes": saved_source_hashes(&installed.environment, &fixture)?,
}))?,
)?;
let outcome = request
.join()
.map_err(|_| failure("public restore caller panicked"))?;
if let Ok(value) = outcome {
let receipt: WorkerRestoreResponse = serde_json::from_value(value)?;
assert_ne!(
receipt.result.state,
WorkerRestoreState::Accepted,
"no caller Accepted may precede either checkpoint kill"
);
}
installed.environment.restart_runtime_after_crash()?;
assert_ne!(installed.environment.runtime_pid(), Some(ack.pid));
Ok(ack)
})?;
drop(peer);
let recovered = installed.environment.restore_worker(
&fixture.manifest.worker_id,
request_id,
&token,
)?;
let receipt: WorkerRestoreResponse = serde_json::from_value(recovered.clone())?;
let committed = stage == RestoreCheckpointStage::CommittedPreterminal;
assert_eq!(
receipt.result.state,
if committed {
WorkerRestoreState::Accepted
} else {
WorkerRestoreState::RolledBack
}
);
assert_eq!(
installed.environment.restore_worker(
&fixture.manifest.worker_id,
request_id,
&token
)?,
recovered
);
let final_saved = canonical(&installed.environment, &fixture)?;
let owners = final_saved["restore_guard"]["owners"]
.as_object()
.ok_or_else(|| failure("missing canonical owners"))?;
assert_eq!(owners.len(), 1);
assert_eq!(
owners[request_id]["operation_id"],
serde_json::to_value(ack.operation_id)?
);
assert_eq!(
owners[request_id]["request"]["expected_observation_token"],
token
);
assert_eq!(
owners[request_id]["receipt"]["state"],
if committed { "accepted" } else { "rolled_back" }
);
let stale = installed.environment.restore_worker(
&fixture.manifest.worker_id,
"new-stale-checkpoint-intent",
&token,
);
if !matches!(stale, Err(error) if error.to_string().contains("HTTP 409")) {
return Err(failure("fresh stale intent was not rejected"));
}
fs::write(
installed
.environment
.artifacts_dir
.join("checkpoint-recovery.json"),
serde_json::to_vec_pretty(&serde_json::json!({
"ack": ack, "recovered_state": receipt.result.state,
"canonical_operation_id": owners[request_id]["operation_id"],
"owner_count": owners.len(), "same_original_request_and_token": true,
"saved_source_hashes_before": original_sources,
"saved_source_hashes_after": saved_source_hashes(&installed.environment, &fixture)?,
"restore_runs_after": restore_runs(&installed.environment, &fixture)?,
}))?,
)?;
if committed {
assert_single_state(&installed.environment, &fixture, "idle")?;
assert_eq!(
restore_runs(&installed.environment, &fixture)?,
vec![format!("restore-{}", ack.operation_id)]
);
let mut subscription = WorkerSessionSubscription::connect(
&installed.environment,
&fixture.manifest.worker_id,
&installed.snapshot,
)?;
subscription.assert_alive()?;
stop_and_cleanup(installed, vec![subscription], &fixture)
} else {
assert_single_state(&installed.environment, &fixture, "stopped")?;
assert!(restore_runs(&installed.environment, &fixture)?.is_empty());
assert_eq!(
saved_source_hashes(&installed.environment, &fixture)?,
original_sources
);
let report = installed.environment.cleanup()?;
assert!(report.cleanup_success && !report.fixture_root_exists_after);
Ok(())
}
})();
fixture.source.assert_unchanged()?;
result
}
#[test]
fn acknowledged_checkpoint_peer_disconnect_resumes_original_operation_and_tears_down() -> Result<()>
{
observer_completion(false)
}
#[test]
fn acknowledged_checkpoint_explicit_release_resumes_original_operation_and_tears_down() -> Result<()>
{
observer_completion(true)
}
fn observer_completion(release: bool) -> Result<()> {
let fixture = Fixture::load()?;
let result = (|| {
let mut environment = WorkerE2eEnvironment::new()?;
let mut peer = Peer::configure(
&mut environment,
&fixture,
RestoreCheckpointStage::PreparedNotCommitted,
)?;
let installed = seed::start_in(&fixture, environment)?;
let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?;
let token = stopped
.restore_observation_token
.ok_or_else(|| failure("missing observation"))?;
let call = installed.environment.pending_restore_call(
&fixture.manifest.worker_id,
"peer-disconnect-request",
&token,
)?;
std::thread::scope(|scope| -> Result<()> {
let request = scope.spawn(call);
peer.acknowledge(&installed.environment)?;
// EOF models an abandoned observer, not a successful restore result.
if release {
peer.release()?;
} else {
drop(peer);
}
let value = request
.join()
.map_err(|_| failure("public caller panicked"))??;
let response: WorkerRestoreResponse = serde_json::from_value(value)?;
assert_eq!(response.result.state, WorkerRestoreState::Accepted);
Ok(())
})?;
let mut subscription = WorkerSessionSubscription::connect(
&installed.environment,
&fixture.manifest.worker_id,
&installed.snapshot,
)?;
subscription.assert_alive()?;
stop_and_cleanup(installed, vec![subscription], &fixture)
})();
fixture.source.assert_unchanged()?;
result
}
#[test]
fn acknowledged_checkpoint_failed_observer_scope_has_bounded_owned_cleanup() -> Result<()> {
let fixture = Fixture::load()?;
let environment = WorkerE2eEnvironment::new()?;
let root = environment.root.clone();
let artifacts = environment.artifacts_dir.clone();
let started = Instant::now();
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| -> Result<()> {
let mut environment = environment;
let mut peer = Peer::configure(
&mut environment,
&fixture,
RestoreCheckpointStage::PreparedNotCommitted,
)?;
let installed = seed::start_in(&fixture, environment)?;
let stopped = assert_single_state(&installed.environment, &fixture, "stopped")?;
let token = stopped
.restore_observation_token
.ok_or_else(|| failure("missing observation"))?;
let call = installed.environment.pending_restore_call(
&fixture.manifest.worker_id,
"failed-observer-request",
&token,
)?;
std::thread::scope(|scope| -> Result<()> {
scope.spawn(call);
peer.acknowledge(&installed.environment)?;
// Leave the peer connected while unwinding the scoped HTTP caller.
// The Runtime's own deadline must unblock it, then RAII must reap
// both owned processes and remove the environment, not wait forever.
panic!("intentional checkpoint observer failure");
})
}));
assert!(outcome.is_err(), "the observer failure was not exercised");
assert!(
started.elapsed() < Duration::from_secs(45),
"owned cleanup exceeded scenario budget"
);
assert!(!root.exists(), "failed observer left its isolated root");
let cleanup: serde_json::Value =
serde_json::from_slice(&fs::read(artifacts.join("cleanup.json"))?)?;
assert_eq!(cleanup["cleanup_success"], true);
assert_eq!(cleanup["fixture_root_exists_after"], false);
fixture.source.assert_unchanged()?;
Ok(())
}
+257
View File
@@ -0,0 +1,257 @@
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use protocol::{SessionMessageRole, SessionSnapshot, SessionSnapshotEntryData};
use serde::Deserialize;
use serde_json::{Value, json};
use worker_runtime::test_support::StoppedWorkerTemplate;
use yoi_e2e::{FixtureSource, HarnessError, Result};
pub const PROFILE: &str = "project:fixture-chat";
const ASSETS: [&str; 3] = [
"expected.json",
"worker-template.json",
"workspace-config.dcdl",
];
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Manifest {
pub format_version: u32,
#[serde(rename = "source_revision")]
pub source_commit: String,
pub runtime_schema_version: u32,
pub session_schema_version: u32,
pub workspace_id: String,
pub runtime_id: String,
pub worker_id: String,
pub session_id: String,
pub segment_id: String,
pub model: String,
pub state: String,
pub acquisition: String,
pub source_file_sha256: BTreeMap<String, String>,
pub files: BTreeMap<String, Asset>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Asset {
pub bytes: u64,
pub sha256: String,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Expected {
#[serde(rename = "source_revision")]
source_commit: String,
model: String,
worker_id: String,
session_id: String,
rows: Vec<Value>,
}
pub struct Fixture {
pub root: PathBuf,
pub source: FixtureSource,
pub manifest: Manifest,
expected: Expected,
}
pub fn failure(message: impl Into<String>) -> HarnessError {
HarnessError::Protocol(message.into())
}
pub fn support<T, E: std::fmt::Display>(result: std::result::Result<T, E>) -> Result<T> {
result.map_err(|error| failure(error.to_string()))
}
impl Fixture {
pub fn load() -> Result<Self> {
// Candidate override is explicit and read-only. Missing assets are failures,
// never skips, and no test generates or repairs a fixture.
let root = std::env::var_os("YOI_E2E_RESTORE_FIXTURE")
.map(PathBuf::from)
.map(|path| {
if path.is_absolute() {
path
} else {
Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.parent()
.unwrap()
.join(path)
}
})
.unwrap_or_else(|| {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/worker-codex-v1")
});
let file_names: Vec<_> = std::iter::once("manifest.json").chain(ASSETS).collect();
let source = FixtureSource::capture_flat(&root, &file_names)?;
let manifest: Manifest =
serde_json::from_slice(&bounded_read(&root.join("manifest.json"))?)?;
if manifest.format_version != 1
|| manifest.state != "stopped"
|| manifest.runtime_schema_version == 0
|| manifest.session_schema_version == 0
|| manifest.acquisition.is_empty()
|| manifest.source_file_sha256.is_empty()
|| manifest.source_commit.len() != 40
|| !manifest
.source_commit
.bytes()
.all(|c| c.is_ascii_hexdigit())
{
return Err(failure("unsupported/incomplete fixture manifest"));
}
let names: BTreeSet<_> = manifest.files.keys().map(String::as_str).collect();
if names != BTreeSet::from(ASSETS) {
return Err(failure("unexpected fixture asset set"));
}
for (name, asset) in &manifest.files {
if asset.bytes != fs::metadata(root.join(name))?.len()
|| source.hashes().get(Path::new(name)) != Some(&format!("sha256:{}", asset.sha256))
{
return Err(failure(format!("fixture asset hash/size mismatch: {name}")));
}
}
let expected: Expected =
serde_json::from_slice(&bounded_read(&root.join("expected.json"))?)?;
if expected.source_commit != manifest.source_commit
|| expected.model != manifest.model
|| expected.worker_id != manifest.worker_id
|| expected.session_id != manifest.session_id
|| expected.rows.is_empty()
{
return Err(failure(
"expected conversation identity differs from manifest",
));
}
verify_builtin_commit(&manifest.source_commit)?;
let fixture = Self {
root,
source,
manifest,
expected,
};
fixture.template(&fixture.root)?;
fixture.source.assert_unchanged()?;
Ok(fixture)
}
pub fn template(&self, copy: &Path) -> Result<StoppedWorkerTemplate> {
let bytes = bounded_read(&copy.join("worker-template.json"))?;
let template: StoppedWorkerTemplate = serde_json::from_slice(&bytes)?;
let shape: Value = serde_json::from_slice(&bytes)?;
let session_manifest_bytes: Vec<u8> =
serde_json::from_value(shape["session_manifest"].clone())?;
let segment_bytes: Vec<u8> = serde_json::from_value(shape["segment_jsonl"].clone())?;
for (name, bytes) in [
(
"session/session.json".to_owned(),
session_manifest_bytes.as_slice(),
),
(
format!("session/segments/{}.jsonl", self.manifest.segment_id),
segment_bytes.as_slice(),
),
] {
use sha2::{Digest, Sha256};
let digest = Sha256::digest(bytes)
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>();
if self.manifest.source_file_sha256.get(&name) != Some(&digest) {
return Err(failure(
"exported Session bytes differ from acquisition hash evidence",
));
}
}
let session_manifest: Value = serde_json::from_slice(&session_manifest_bytes)?;
let m = &self.manifest;
if template.workspace_id() != m.workspace_id
|| template.runtime_id() != m.runtime_id
|| template.worker_id().to_string() != m.worker_id
|| template.session_id().to_string() != m.session_id
|| template.segment_id().to_string() != m.segment_id
|| shape["runtime_schema_version"] != m.runtime_schema_version
|| session_manifest["schema_version"] != m.session_schema_version
{
return Err(failure("template schema/identity differs from manifest"));
}
Ok(template)
}
pub fn verify_conversation(&self, snapshot: &SessionSnapshot) -> Result<()> {
let mut rows = Vec::new();
for entry in &snapshot.entries {
let (role, parts) = match &entry.data {
SessionSnapshotEntryData::UserInput { segments } => {
("user", serde_json::to_value(segments)?)
}
SessionSnapshotEntryData::Message {
role: SessionMessageRole::Assistant,
content,
} => ("assistant", serde_json::to_value(content)?),
// Display-only accounting is not a conversation row. Its exact
// IDs, timestamp and values are still checked in the full WS snapshot.
SessionSnapshotEntryData::RunStats { .. } => continue,
_ => {
return Err(failure(
"unexpected non-conversation entry in full Session snapshot",
));
}
};
rows.push(json!({"entry_id":entry.entry_id,"timestamp":entry.timestamp,"role":role,"parts":parts}));
}
if rows != self.expected.rows {
return Err(failure(
"full public conversation rows differ from expected IDs/time/roles/text/order",
));
}
Ok(())
}
}
fn bounded_read(path: &Path) -> Result<Vec<u8>> {
FixtureSource::read_bounded(path)
}
fn verify_builtin_commit(source_commit: &str) -> Result<()> {
let workspace = Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.parent()
.unwrap();
// Resolve the immutable commit before diff; unavailable commits must not be treated
// as an empty diff. All resources are checked, including transitive imports.
let commit = Command::new("git")
.args(["cat-file", "-t", source_commit])
.current_dir(workspace)
.output()?;
if !commit.status.success() || commit.stdout != b"commit\n" {
return Err(failure(
"fixture source commit is unavailable for builtin verification",
));
}
let status = Command::new("git")
.args([
"diff",
"--quiet",
source_commit,
"--",
"resources",
"crates/builtin-source",
])
.current_dir(workspace)
.status()?;
if !status.success() {
return Err(failure(
"builtin/config resources differ from fixture source commit; provide a new version or frozen references",
));
}
Ok(())
}
+246
View File
@@ -0,0 +1,246 @@
//! Real orphan-pin repair on an installed fixed-asset copy, not a synthetic
//! pending head. Only the postcommit activation path can reconcile these pins.
use super::*;
use session_store::{Store, UploadedFileLimits, WorkerSessionStore};
const PIN_OWNER: &str = "interrupted-pin-before-queue-checkpoint";
const UPLOAD: &[u8] = b"isolated orphan upload; never model input";
#[test]
fn orphan_pin_not_committed_process_death_preserves_unrepaired_source() -> Result<()> {
repair_crash(RestoreCheckpointStage::PreparedNotCommitted)
}
#[test]
fn orphan_pin_activated_process_death_preserves_real_repair_and_committed_decision() -> Result<()> {
repair_crash(RestoreCheckpointStage::ActivatedPreterminal)
}
fn repair_crash(stage: RestoreCheckpointStage) -> Result<()> {
let fixture = Fixture::load()?;
let result = (|| {
let mut environment = WorkerE2eEnvironment::new()?;
let mut peer = Peer::configure(&mut environment, &fixture, stage)?;
let session_id = fixture::support(fixture.manifest.session_id.parse())?;
let mut upload = None;
let mut installed = seed::start_in_with_source_setup(&fixture, environment, |aggregate| {
let store = fixture::support(WorkerSessionStore::new(aggregate.join("session")))?;
let reference = fixture::support(store.write_uploaded_file(
session_id,
"orphan.txt",
"text/plain",
UPLOAD,
UploadedFileLimits::default(),
))?;
fixture::support(store.pin_uploaded_file(session_id, &reference, PIN_OWNER))?;
upload = Some(reference);
Ok(())
})?;
let upload = upload.ok_or_else(|| failure("offline upload setup not called"))?;
let relative = format!("session/artifacts/paste/{}.file.json", upload.artifact_id);
let pin_path = aggregate_path(&installed.environment, &fixture).join(&relative);
let pin_before: serde_json::Value = serde_json::from_slice(&fs::read(&pin_path)?)?;
assert_eq!(pin_before["pending_owner_id"], PIN_OWNER);
let before = saved_source_hashes(&installed.environment, &fixture)?;
let initial = canonical(&installed.environment, &fixture)?;
let token = assert_single_state(&installed.environment, &fixture, "stopped")?
.restore_observation_token
.ok_or_else(|| failure("missing original token"))?;
let request_id = "real-orphan-repair-crash-request";
let committed = stage == RestoreCheckpointStage::ActivatedPreterminal;
let call = installed.environment.pending_restore_call(
&fixture.manifest.worker_id,
request_id,
&token,
)?;
let (ack, at_death) = std::thread::scope(|scope| -> Result<_> {
let request = scope.spawn(call);
let ack = peer.acknowledge(&installed.environment)?;
installed.environment.crash_runtime()?;
let saved = canonical(&installed.environment, &fixture)?;
let op = &saved["execution_state"]["execution"]["intent"];
let owner = &saved["restore_guard"]["owners"][request_id];
assert_eq!(saved["restore_guard"]["active_request_id"], request_id);
assert_eq!(op["operation_id"], serde_json::to_value(ack.operation_id)?);
assert_eq!(owner["operation_id"], op["operation_id"]);
assert_eq!(owner["request"]["expected_observation_token"], token);
assert!(owner["receipt"].is_null());
assert_eq!(
op["commit"],
if committed {
"committed"
} else {
"not_committed"
}
);
assert_eq!(
ack.commit,
if committed {
RestoreCheckpointCommit::Committed
} else {
RestoreCheckpointCommit::NotCommitted
}
);
// The durable Committed decision authorizes activation effects. The
// precommit repair callback is not used for a materialized head.
assert_eq!(op["source_repair_possible"], false);
let at_death = saved_source_hashes(&installed.environment, &fixture)?;
let pin_after: serde_json::Value = serde_json::from_slice(&fs::read(&pin_path)?)?;
let mut expected_pin = pin_before.clone();
if committed {
expected_pin
.as_object_mut()
.unwrap()
.remove("pending_owner_id");
}
assert_eq!(
pin_after, expected_pin,
"real repair changes only orphan ownership"
);
if committed {
assert_ne!(
at_death[&relative], before[&relative],
"actual pin bytes must change"
);
} else {
assert_eq!(
at_death, before,
"preparation cannot repair pin or metadata"
);
}
// In both windows the original conversation is immutable.
for (path, hash) in &before {
if path == "session/session.json" || path.starts_with("session/segments/") {
assert_eq!(at_death.get(path), Some(hash), "{path}");
}
}
let store = fixture::support(WorkerSessionStore::open_read_only(
aggregate_path(&installed.environment, &fixture).join("session"),
))?;
assert_eq!(
fixture::support(store.read_uploaded_file(session_id, &upload))?,
UPLOAD
);
assert_eq!(
restore_runs(&installed.environment, &fixture)?,
vec![format!("restore-{}", ack.operation_id)]
);
fs::write(
installed
.environment
.artifacts_dir
.join("repair-at-death.json"),
serde_json::to_vec_pretty(&serde_json::json!({
"ack":ack, "initial_execution":initial["execution_state"],
"canonical_commit":op["commit"], "source_repair_possible":op["source_repair_possible"],
"operation_id":op["operation_id"], "owner_operation_id":owner["operation_id"],
"receipt_absent":owner["receipt"].is_null(), "original_request_and_token":true,
"pin_pending_owner_before":pin_before["pending_owner_id"],
"pin_pending_owner_after":pin_after["pending_owner_id"],
"hashes_before":before, "hashes_at_death":at_death,
"restore_runs":restore_runs(&installed.environment, &fixture)?,
}))?,
)?;
let outcome = request
.join()
.map_err(|_| failure("restore caller panicked"))?;
if let Ok(value) = outcome {
let receipt: WorkerRestoreResponse = serde_json::from_value(value)?;
assert_ne!(receipt.result.state, WorkerRestoreState::Accepted);
}
installed.environment.restart_runtime_after_crash()?;
assert_ne!(installed.environment.runtime_pid(), Some(ack.pid));
Ok((ack, at_death))
})?;
drop(peer);
let recovered = installed.environment.restore_worker(
&fixture.manifest.worker_id,
request_id,
&token,
)?;
let receipt: WorkerRestoreResponse = serde_json::from_value(recovered.clone())?;
fs::write(
installed
.environment
.artifacts_dir
.join("repair-first-recovery.json"),
serde_json::to_vec_pretty(&recovered)?,
)?;
assert_eq!(
receipt.result.state,
if committed {
WorkerRestoreState::Accepted
} else {
WorkerRestoreState::RolledBack
}
);
assert_eq!(
installed.environment.restore_worker(
&fixture.manifest.worker_id,
request_id,
&token
)?,
recovered
);
let final_saved = canonical(&installed.environment, &fixture)?;
let owners = final_saved["restore_guard"]["owners"]
.as_object()
.ok_or_else(|| failure("missing owners"))?;
assert_eq!(owners.len(), 1);
let owner = &owners[request_id];
assert_eq!(
owner["operation_id"],
serde_json::to_value(ack.operation_id)?
);
assert_eq!(owner["request"]["expected_observation_token"], token);
assert_eq!(
owner["receipt"]["state"],
if committed { "accepted" } else { "rolled_back" }
);
let after = saved_source_hashes(&installed.environment, &fixture)?;
assert_eq!(
after, at_death,
"restart/replay cannot restore the orphan pin or change saved source"
);
let stale = installed.environment.restore_worker(
&fixture.manifest.worker_id,
"fresh-stale-repair-intent",
&token,
);
assert!(matches!(stale, Err(error) if error.to_string().contains("HTTP 409")));
fs::write(
installed
.environment
.artifacts_dir
.join("repair-recovery.json"),
serde_json::to_vec_pretty(&serde_json::json!({
"ack":ack, "state":receipt.result.state, "diagnostics":receipt.result.diagnostics,
"operation_id":owner["operation_id"], "owner_count":owners.len(),
"hashes_after_recovery":after, "restore_runs":restore_runs(&installed.environment, &fixture)?,
"same_original_request_and_token":true, "fresh_stale_rejected":true,
}))?,
)?;
if committed {
assert_single_state(&installed.environment, &fixture, "idle")?;
assert_eq!(
restore_runs(&installed.environment, &fixture)?,
vec![format!("restore-{}", ack.operation_id)]
);
let mut subscription = WorkerSessionSubscription::connect(
&installed.environment,
&fixture.manifest.worker_id,
&installed.snapshot,
)?;
subscription.assert_alive()?;
stop_and_cleanup(installed, vec![subscription], &fixture)
} else {
assert_single_state(&installed.environment, &fixture, "stopped")?;
assert!(restore_runs(&installed.environment, &fixture)?.is_empty());
let report = installed.environment.cleanup()?;
assert!(report.cleanup_success && !report.fixture_root_exists_after);
Ok(())
}
})();
fixture.source.assert_unchanged()?;
result
}
+501
View File
@@ -0,0 +1,501 @@
use std::fs;
use std::net::TcpListener;
use std::path::Path;
use std::sync::Arc;
use config_source::{ConfigTreeChange, VirtualPath};
use protocol::SessionSnapshot;
use serde_json::Value;
use worker_runtime::FsRuntimeStoreOptions;
use worker_runtime::catalog::{
ConfigBundleRef, CreateWorkerRequest, ProfileSelector, ProfileSourceArchiveSource,
WorkspaceApiRef,
};
use yoi_e2e::{Result, WorkerE2eEnvironment};
use yoi_workspace_server::auth::{mint_secret, new_id, now_rfc3339, token_hash};
use yoi_workspace_server::config_source::{ConfigCommitRequest, WorkspaceConfigSchemaRegistry};
use yoi_workspace_server::hosts::{
WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest,
};
use yoi_workspace_server::profile_settings::{
ProfileConfigSchemaProvider, build_virtual_profile_config_bundle,
project_profiles_from_workspace_config,
};
use yoi_workspace_server::store::{
AccountRecord, ApiTokenRecord, UserRecord, WorkspaceBootstrapRecord,
WorkspaceRuntimeAuthenticationMode, WorkspaceRuntimeBinding,
};
use yoi_workspace_server::test_support::{FixedWorkerSeed, seed_fixed_worker};
use yoi_workspace_server::workspace_signing_identity::{
FsWorkspaceSigningMaterialStore, WorkspaceSigningIdentityService,
workspace_signing_material_root,
};
use yoi_workspace_server::{ControlPlaneStore, SqliteWorkspaceStore, WorkspaceRecord};
use super::fixture::{Fixture, PROFILE, failure, support};
pub struct Installed {
pub environment: WorkerE2eEnvironment,
pub snapshot: SessionSnapshot,
pub runtime_public_key: String,
pub workspace_public_key: String,
}
pub fn start(fixture: &Fixture) -> Result<Installed> {
start_in(fixture, WorkerE2eEnvironment::new()?)
}
pub fn start_in(fixture: &Fixture, environment: WorkerE2eEnvironment) -> Result<Installed> {
start_in_with_source_setup(fixture, environment, |_| Ok(()))
}
// The callback runs only on the installed isolation copy, while Runtime is offline.
// Restart never calls the seed or this setup again.
pub fn start_in_with_source_setup(
fixture: &Fixture,
mut environment: WorkerE2eEnvironment,
setup: impl FnOnce(&Path) -> Result<()>,
) -> Result<Installed> {
super::phase(&environment, "fixture-copy-verify")?;
let copy = environment.root.join("provided-fixture");
fixture.source.copy_to(&copy)?;
fs::write(
environment.artifacts_dir.join("fixture-manifest.json"),
fs::read(copy.join("manifest.json"))?,
)?;
let template = fixture.template(&copy)?;
let runtime_identity =
environment.initialize_runtime_identity(fixture.manifest.runtime_id.clone())?;
let runtime_public_key = runtime_identity["public_key"]
.as_str()
.ok_or_else(|| failure("Runtime public bundle omitted key"))?
.to_owned();
// Reserve an environment-local placeholder endpoint only while the Runtime is
// offline. Its real :0 endpoint replaces this binding through public API CAS.
let reserved_endpoint = TcpListener::bind("127.0.0.1:0")?;
super::phase(
&environment,
"offline-backend-bootstrap-profile-worker-seed",
)?;
let bootstrap = bootstrap(
&environment,
fixture,
&copy,
&runtime_identity,
&format!("http://{}", reserved_endpoint.local_addr()?),
&template.worker_name(),
)?;
// Close the placeholder before Backend probes it: an accepting TCP listener
// with no HTTP service would stall discovery until its transport timeout.
drop(reserved_endpoint);
super::phase(&environment, "start-seeded-backend")?;
environment.start_seeded_backend(
fixture.manifest.workspace_id.clone(),
fixture.manifest.runtime_id.clone(),
bootstrap.token,
)?;
let mut snapshot = None;
super::phase(&environment, "initialize-runtime-trust")?;
environment.start_runtime_with_seed(|environment| {
super::phase(environment, "install-runtime-fixture-and-validate-session")?;
let options = FsRuntimeStoreOptions::new(environment.runtime_data_dir.join("runtime"))
.with_runtime_id(&fixture.manifest.runtime_id);
let request = CreateWorkerRequest {
worker_id: template.worker_id(),
create_fingerprint: bootstrap.create_fingerprint.clone(),
profile: ProfileSelector::Named(PROFILE.into()),
display_name: Some("Restore E2E fixture".into()),
profile_source: ProfileSourceArchiveSource::WorkspaceConfig {
archive: bootstrap.archive.clone(),
},
config_bundle: Some(bootstrap.config_bundle.clone()),
workspace_api: Some(WorkspaceApiRef {
workspace_id: fixture.manifest.workspace_id.clone(),
base_url: environment.backend_target()?.0.into(),
}),
initial_input: None,
workdir_attachment_requests: vec![],
workdir_attachments: vec![],
worker_observation_enabled: false,
worker_observation_grants: vec![],
memory_settings: Some(bootstrap.memory_settings.clone()),
subjektiv_attached: false,
backend_job: None,
};
support(template.install(&options, request, &bootstrap.manifest))?;
setup(
&options
.root
.join("workers")
.join(template.worker_id().to_string()),
)?;
let sessions = support(session_store::WorkerSessionStore::open_read_only(
options
.root
.join("workers")
.join(template.worker_id().to_string())
.join("session"),
))?;
let (logs, _) = support(sessions.read_all_read_only_bounded(
template.session_id(),
template.segment_id(),
8 * 1024 * 1024,
))?;
let projected = session_store::project_session_snapshot(template.session_id(), &logs);
fixture.verify_conversation(&projected)?;
snapshot = Some(full_restored_snapshot(&sessions, &logs));
Ok(())
})?;
fixture.source.assert_unchanged()?;
Ok(Installed {
environment,
snapshot: snapshot.ok_or_else(|| failure("Runtime seed was not invoked"))?,
runtime_public_key,
workspace_public_key: bootstrap.workspace_public_key,
})
}
fn full_restored_snapshot(
store: &dyn session_store::Store,
logs: &[session_store::LogEntry],
) -> SessionSnapshot {
session_store::public_snapshot::project_current_session_snapshot_with_accounting(store, logs)
}
#[cfg(test)]
mod tests {
use super::*;
use protocol::{
Event, Greeting, InFlightSnapshot, InvokeKind, SessionSnapshotEntryData,
WorkerStateSnapshot, WorkerStatus,
};
use session_store::{
LogEntry, LoggedContentPart, LoggedHistoryEntry, LoggedItem, LoggedRole,
LoggedSessionHistoryEntryId, LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin,
};
#[test]
fn completed_run_full_snapshot_retains_invoke_and_usage_accounting() -> Result<()> {
// Projection regression only: no saved Worker or substitute E2E fixture.
let temp = tempfile::tempdir()?;
let store = support(session_store::WorkerSessionStore::new(temp.path()))?;
let session_id = session_store::new_session_id();
let logs = vec![
LogEntry::AnnotatedSegmentStart {
ts: 1,
session_id,
system_prompt: None,
config: Default::default(),
history: vec![],
forked_from: None,
compacted_from: None,
},
LogEntry::Invoke {
ts: 10,
trigger: InvokeKind::UserSend,
},
LogEntry::AnnotatedAssistantItem {
ts: 20,
entry: LoggedHistoryEntry {
item: LoggedItem::Message {
role: LoggedRole::Assistant,
content: vec![LoggedContentPart::Text {
text: "unit response".into(),
}],
},
metadata: LoggedSessionHistoryMetadata {
entry_id: LoggedSessionHistoryEntryId("unit-response".into()),
origin: LoggedSessionHistoryOrigin::LegacyUnknown,
derivation: None,
input_metadata: None,
},
},
},
LogEntry::LlmUsage {
ts: 30,
history_len: 1,
input_total_tokens: 100,
cache_read_tokens: 20,
cache_write_tokens: 0,
output_tokens: 5,
},
serde_json::from_value(serde_json::json!({"kind":"run_completed", "ts":40,
"interrupted":false,"result":"finished"}))?,
];
let conversation = session_store::project_session_snapshot(session_id, &logs);
let expected = full_restored_snapshot(&store, &logs);
assert_eq!(expected.entries.len(), conversation.entries.len() + 2);
assert!(matches!(
expected.entries[expected.entries.len() - 2].data,
SessionSnapshotEntryData::Invoke {
trigger: InvokeKind::UserSend
}
));
assert_eq!(expected.entries[expected.entries.len() - 2].timestamp, 10);
assert!(matches!(
expected.entries.last().unwrap().data,
SessionSnapshotEntryData::Usage {
input_tokens: 100,
cache_read_input_tokens: 20,
output_tokens: 5
}
));
assert_eq!(expected.entries.last().unwrap().timestamp, 30);
let live = session_store::public_snapshot::project_current_session_snapshot_with_accounting(
&store, &logs,
);
let event = Event::Snapshot {
session: live,
state: WorkerStateSnapshot::from(WorkerStatus::Idle),
in_flight: InFlightSnapshot::default(),
internal_workers: vec![],
greeting: Greeting {
worker_name: "unit".into(),
cwd: ".".into(),
provider: "unit".into(),
model: "unit".into(),
reasoning: None,
scope_summary: "none".into(),
tools: vec![],
context_window: 0,
context_tokens: 0,
context_usage: None,
},
};
let raw = serde_json::to_value(event)?;
yoi_e2e::validate_full_snapshot(&raw, &expected)?;
assert!(yoi_e2e::validate_full_snapshot(&raw, &conversation).is_err());
Ok(())
}
}
struct Bootstrap {
token: String,
workspace_public_key: String,
create_fingerprint: String,
archive: worker_runtime::profile_archive::ProfileSourceArchiveRef,
config_bundle: ConfigBundleRef,
manifest: manifest::WorkerManifest,
memory_settings: manifest::WorkspaceMemorySettingsSnapshot,
}
fn bootstrap(
environment: &WorkerE2eEnvironment,
fixture: &Fixture,
copy: &Path,
runtime_identity: &Value,
endpoint: &str,
worker_name: &str,
) -> Result<Bootstrap> {
fs::create_dir_all(&environment.server_data_dir)?;
let database = environment.server_data_dir.join("server.db");
if database.exists() {
return Err(failure("Backend seed refuses an existing database"));
}
let store = support(SqliteWorkspaceStore::open(&database))?;
let now = now_rfc3339();
let account_id = new_id("acct-user");
let user_id = new_id("user");
support(store.upsert_account(&AccountRecord {
account_id: account_id.clone(),
kind: "user".into(),
handle: environment.account_handle.clone(),
display_name: "Restore E2E owner".into(),
created_at: now.clone(),
updated_at: now.clone(),
}))?;
support(store.upsert_user(&UserRecord {
user_id: user_id.clone(),
account_id: account_id.clone(),
handle: environment.account_handle.clone(),
display_name: "Restore E2E owner".into(),
created_at: now.clone(),
updated_at: now.clone(),
}))?;
let token = mint_secret("yoi_api");
support(store.create_api_token(&ApiTokenRecord {
token_id: new_id("api-token"),
token_hash: token_hash(&token),
user_id,
label: "restore-e2e".into(),
created_at: now.clone(),
expires_at: None,
revoked_at: None,
last_used_at: None,
}))?;
let identities = WorkspaceSigningIdentityService::new(
Arc::new(store.clone()),
Arc::new(FsWorkspaceSigningMaterialStore::new(
workspace_signing_material_root(&database),
)),
);
let (identity, operation) = support(identities.prepare_workspace_creation(
"restore-e2e-bootstrap",
"restore-e2e-bootstrap",
&fixture.manifest.workspace_id,
&account_id,
))?;
let workspace_public_key = identity.public_key.clone();
support(store.create_workspace_bootstrap(
&WorkspaceBootstrapRecord {
operation_key: "restore-e2e-bootstrap".into(),
request_fingerprint: "restore-e2e-bootstrap".into(),
workspace: WorkspaceRecord {
workspace_id: fixture.manifest.workspace_id.clone(),
owner_account_id: account_id,
display_name: "Restore E2E".into(),
state: "active".into(),
created_at: now.clone(),
updated_at: now.clone(),
},
repository: None,
},
&identity,
&operation,
))?;
support(
store.upsert_workspace_runtime_binding(
WorkspaceRuntimeBinding {
workspace_id: fixture.manifest.workspace_id.clone(),
runtime_id: fixture.manifest.runtime_id.clone(),
display_name: format!("E2E {}", fixture.manifest.runtime_id),
base_url: endpoint.into(),
public_key: runtime_identity["public_key"]
.as_str()
.ok_or_else(|| failure("missing Runtime public key"))?
.into(),
public_key_fingerprint: String::new(),
binding_id: String::new(),
authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity,
created_at: now.clone(),
updated_at: now.clone(),
revoked_at: None,
},
false,
),
)?;
let schema = support(
WorkspaceConfigSchemaRegistry::default()
.with_provider(Arc::new(ProfileConfigSchemaProvider))
.with_provider(Arc::new(
yoi_workspace_server::prompt_settings::PromptConfigSchemaProvider,
))
.with_provider(Arc::new(
yoi_workspace_server::runtime_settings::RuntimeConfigSchemaProvider,
))
.with_provider(Arc::new(
yoi_workspace_server::repository_access::RepositoryAccessConfigSchemaProvider,
))
.with_provider(Arc::new(
yoi_workspace_server::skills::SkillConfigSchemaProvider,
))
.compose(),
)?;
let state = support(store.ensure_workspace_config_materialized_with_schema(
&fixture.manifest.workspace_id,
&now,
schema.clone(),
))?;
let main = support(VirtualPath::parse("main.dcdl"))?;
let current = state
.snapshot
.get(&main)
.ok_or_else(|| failure("Workspace config omitted main.dcdl"))?;
let candidate = support(store.evaluate_workspace_config_candidate_with_schema(
&fixture.manifest.workspace_id,
&ConfigCommitRequest {
base_digest: state.snapshot.digest.clone(),
changes: vec![ConfigTreeChange::Update {
path: main.clone(),
expected_digest: current.content_digest.clone(),
content: fs::read_to_string(copy.join("workspace-config.dcdl"))?,
}],
entrypoints: vec![main],
},
schema,
))?;
let state = support(
store.commit_evaluated_workspace_config(&fixture.manifest.workspace_id, &candidate),
)?;
let projection = support(project_profiles_from_workspace_config(
&fixture.manifest.workspace_id,
&state,
))?;
let bundle = support(build_virtual_profile_config_bundle(
&projection,
&state,
&fixture.manifest.workspace_id,
&now,
PROFILE,
))?
.ok_or_else(|| failure("fixture Profile is not registered"))?;
let archive = bundle
.profile_source_archive
.as_ref()
.ok_or_else(|| failure("fixture Profile archive is absent"))?;
let mut resolved = support(
support(archive.verify())?.resolve_profile(
PROFILE,
&environment
.runtime_data_dir
.join("runtime/workers")
.join(&fixture.manifest.worker_id),
worker_name,
),
)?;
if resolved.model.ref_.as_deref() != Some(fixture.manifest.model.as_str()) {
return Err(failure("resolved Profile changed the supplied model"));
}
let request = WorkerSpawnRequest {
intent: WorkerSpawnIntent::WorkspaceCoding,
requested_worker_name: Some("Restore E2E fixture".into()),
singleton_key: None,
acceptance: WorkerSpawnAcceptanceRequirement::SocketReady,
profile: ProfileSelector::Named(PROFILE.into()),
ticket_assignment: None,
initial_submit: vec![],
workdir_attachment_requests: vec![],
resolved_workdir_attachment_requests: vec![],
resolved_workdir_attachments: vec![],
resolved_config_bundle: None,
resolved_workspace_api: None,
resolved_memory_settings: None,
resolved_subjektiv_attached: false,
resolved_worker_observation_enabled: false,
resolved_worker_observation_grants: vec![],
resolved_control_operation: None,
resolved_execution_grant: None,
};
let reservation = support(seed_fixed_worker(
&store,
FixedWorkerSeed {
workspace_id: &fixture.manifest.workspace_id,
runtime_id: &fixture.manifest.runtime_id,
worker_id: support(fixture.manifest.worker_id.parse())?,
allocation_key: "restore-e2e-worker",
request: &request,
},
))?;
if resolved.feature.memory.profile.enabled {
support(
resolved
.feature
.memory
.bind_workspace_settings(reservation.memory_settings.clone()),
)?;
}
Ok(Bootstrap {
manifest: resolved,
memory_settings: reservation.memory_settings,
token,
workspace_public_key,
create_fingerprint: reservation.create_fingerprint,
archive: archive.reference.clone(),
config_bundle: ConfigBundleRef {
id: bundle.metadata.id,
digest: bundle.metadata.digest,
},
})
}
+129
View File
@@ -0,0 +1,129 @@
//! Fail-closed preflight shared by restore fixtures across Worker, Runtime and Server.
//!
//! The normal runner supplies a separate namespace per process; a manually
//! configured invocation may use the checkout-owned base itself. This module
//! never sets process-wide environment or creates directories/tables.
use std::fs;
use std::path::{Component, Path, PathBuf};
fn validate_restore_test_paths(
base: &Path,
resolved: [Option<PathBuf>; 5],
) -> Result<PathBuf, &'static str> {
let root = resolved[0]
.as_deref()
.filter(|config| config.file_name().is_some_and(|name| name == "config"))
.and_then(Path::parent)
.ok_or("config")?;
let namespace = root.strip_prefix(base).map_err(|_| "config")?;
if namespace
.components()
.any(|part| !matches!(part, Component::Normal(_)))
{
return Err("config");
}
for ((label, suffix), actual) in [
("config", "config"),
("data", "data"),
("secrets", "data"),
("runtime", "run"),
("allocation", "run/workers.json"),
]
.into_iter()
.zip(resolved.iter())
{
if actual.as_deref() != Some(root.join(suffix).as_path()) {
return Err(label);
}
}
Ok(root.to_path_buf())
}
fn require_owned_directory(checkout: &Path, directory: &Path) {
let mut checked = checkout.to_path_buf();
for part in directory.strip_prefix(checkout).unwrap().components() {
assert!(matches!(part, Component::Normal(_)));
checked.push(part);
let metadata = fs::symlink_metadata(&checked)
.expect("create isolated restore test directories before launching the test process");
assert!(
metadata.is_dir() && !metadata.file_type().is_symlink(),
"restore test directory must not traverse symlinks"
);
}
}
pub(crate) fn require_restore_test_environment() {
let checkout = Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.parent()
.unwrap();
let base = checkout.join("target/restore-test-environment");
// Compare all resolver results before inspecting even an absent allocation
// table (its lookup can create a lock). Never inspect a live fallback path.
let root = validate_restore_test_paths(
&base,
[
manifest::paths::config_dir(),
manifest::paths::data_dir(),
manifest::paths::secret_data_dir(),
manifest::paths::runtime_dir(),
manifest::paths::worker_allocation_path(),
],
)
.unwrap_or_else(|label| {
panic!(
"restore fixture requires isolated {label}: run deno task test, or configure YOI_CONFIG_DIR=<checkout>/target/restore-test-environment/config, YOI_DATA_DIR=<checkout>/target/restore-test-environment/data, YOI_RUNTIME_DIR=<checkout>/target/restore-test-environment/run before launch"
)
});
assert_eq!(fs::canonicalize(checkout).unwrap(), checkout);
for directory in [
root.clone(),
root.join("config"),
root.join("data"),
root.join("run"),
] {
require_owned_directory(checkout, &directory);
}
for name in ["workers.json", "workers.json.lock"] {
match fs::symlink_metadata(root.join("run").join(name)) {
Ok(metadata) => assert!(
metadata.is_file() && !metadata.file_type().is_symlink(),
"owned allocation paths must be regular files, not symlinks"
),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => panic!("inspect owned restore allocation path: {error}"),
}
}
}
#[test]
fn restore_fixture_path_validation_rejects_fallback_before_resource_access() {
let base = Path::new("/owned/test-environment");
for root in [base.to_path_buf(), base.join("run-1/worker-shard-2")] {
let safe = ["config", "data", "data", "run", "run/workers.json"]
.map(|suffix| Some(root.join(suffix)));
assert_eq!(
validate_restore_test_paths(base, safe.clone()),
Ok(root.clone())
);
for index in 0..safe.len() {
for fallback in [None, Some(PathBuf::from("/not-a-fixture"))] {
let mut unsafe_paths = safe.clone();
unsafe_paths[index] = fallback;
assert!(validate_restore_test_paths(base, unsafe_paths).is_err());
}
}
}
for root in [
base.join("../escape"),
base.join("safe/../../escape"),
PathBuf::from("/outside"),
] {
let paths = ["config", "data", "data", "run", "run/workers.json"]
.map(|suffix| Some(root.join(suffix)));
assert!(validate_restore_test_paths(base, paths).is_err());
}
}
+33
View File
@@ -1,4 +1,37 @@
[
{
"path": "tests/e2e/fixtures/worker-codex-v1/manifest.json",
"kind": "saved_data",
"reason": "ユーザー提供済v1固定資産の生成元Git commitの保存key。原本の取得証拠とSHA-256を保持するため変更しない。Worker実行状態や競合制御の番号ではなく、生成元resources照合のみに使う。",
"lines": [
{
"sha256": "fbf8f92576eb6afbb03502d2f95144651373ce8cd7b4401bc06f801fdb4db28d",
"count": 1
}
]
},
{
"path": "tests/e2e/fixtures/worker-codex-v1/expected.json",
"kind": "saved_data",
"reason": "ユーザー提供済v1固定資産の生成元Git commitの保存key。原本の取得証拠とSHA-256を保持するため変更しない。Worker実行状態や競合制御の番号ではなく、生成元resources照合のみに使う。",
"lines": [
{
"sha256": "fbf8f92576eb6afbb03502d2f95144651373ce8cd7b4401bc06f801fdb4db28d",
"count": 1
}
]
},
{
"path": "tests/e2e/tests/worker_restore/fixture.rs",
"kind": "saved_data",
"reason": "提供済v1原本の生成元Git commitを読むための固定保存key対応。原本bytesとhashを変更しない。実行状態の観測tokenや更新番号に利用せず、現行Rust内ではsource_commitとして扱う。",
"lines": [
{
"sha256": "4dd4810d1a31b812596b9d6cc8a8b27f398476174dfa6f0c3c1881aeadb5ade4",
"count": 2
}
]
},
{
"path": "crates/subjektiv/src/journal/memory_links/tests.rs",
"kind": "saved_data",
+367 -2
View File
@@ -6,6 +6,7 @@ import {
mapConcurrent,
parseJobs,
parseRustListing,
prepareTestIsolation,
resultSummary,
rustTestCommands,
shardRustCommand,
@@ -202,6 +203,39 @@ Deno.test("round-robin shards retain every listed name exactly once at threshold
assertThrows(() => shardRustCommand(task, ["same", "same"]));
});
Deno.test("global runtime-path packages serialize every normal target without losing names", () => {
for (const pkg of ["worker", "worker-runtime", "yoi-workspace-server"]) {
for (const target of ["lib", "controller_restore_test"]) {
const task = {
name: `${pkg}/${target}`,
command: "/test-binary",
args: [],
cwd: "/pkg",
};
for (const count of [0, 1, 59, 72, 73, 1135]) {
const names = listedNames(count);
const shards = shardRustCommand(task, names);
assert(
shards.flatMap((shard) => shard.selectedNames).sort().join() ===
[...names].sort().join(),
);
for (const shard of shards) {
const args = shard.task.args;
assert(args[args.indexOf("--test-threads") + 1] === "1");
if (count > 72) {
assert(args[0] === "--exact");
assert(args.slice(3).join() === shard.selectedNames.join());
} else {
assert(args.join() === "--test-threads,1");
}
assert(!args.includes("--ignored") && !args.includes("--skip"));
}
}
assert(task.args.length === 0, "do not mutate discovered commands");
}
}
});
Deno.test("scheduler drains active workers before propagating a callback error", async () => {
let drained = false;
let failed = false;
@@ -266,6 +300,276 @@ function emergencyKill(pid: number) {
}
}
async function assertRejects(
run: () => Promise<unknown>,
message: string,
) {
try {
await run();
} catch (error) {
assert(String(error).includes(message), String(error));
return;
}
throw new Error(`expected rejection containing ${message}`);
}
Deno.test({
name:
"isolated real children use bounded disjoint tables and sanitized Git/compiler environments",
ignore: Deno.build.os !== "linux",
async fn() {
const checkout = await Deno.realPath(await Deno.makeTempDir());
const environmentSnapshot = () =>
JSON.stringify(
Object.entries(process.env).sort(([a], [b]) => a.localeCompare(b)),
);
const before = environmentSnapshot();
try {
// An ancestor repository must not make an uninitialized tempfile look like
// a repository. No developer repository or configuration is borrowed.
for (const dir of [".git/objects", ".git/refs"]) {
await Deno.mkdir(`${checkout}/${dir}`, { recursive: true });
}
await Deno.writeTextFile(
`${checkout}/.git/HEAD`,
"ref: refs/heads/main\n",
);
const [first, second] = await Promise.all([
prepareTestIsolation(checkout, "same/output/id"),
prepareTestIsolation(checkout, "same/output/id"),
]);
assert(
first.root !== second.root,
"concurrent runner ids must not collide",
);
const task = {
...fixtureTask("isolation-env", checkout),
env: {
CARGO_MANIFEST_DIR: `${checkout}/crates/example`,
CARGO_HOME: `${checkout}/compiler-cache`,
RUSTUP_HOME: `${checkout}/toolchain-cache`,
YOI_HOME: "/live/yoi",
YOI_CONFIG_DIR: "/live/config",
YOI_DATA_DIR: "/live/data",
YOI_RUNTIME_DIR: "/live/run",
HOME: "/live/home",
TMPDIR: "/deleted/compiler-tmp",
GIT_DIR: "/live/git",
GIT_WORK_TREE: "/live/tree",
GIT_CONFIG: "/live/gitconfig",
GIT_CONFIG_GLOBAL: "/live/gitconfig",
GIT_CONFIG_SYSTEM: "/live/system-gitconfig",
GIT_TEMPLATE_DIR: "/live/template",
GIT_CONFIG_PARAMETERS: "'core.hooksPath'='/live/hooks'",
GIT_CONFIG_COUNT: "1",
GIT_CONFIG_KEY_0: "core.hooksPath",
GIT_CONFIG_VALUE_0: "/live/hooks",
HTTP_PROXY: "http://live.invalid",
https_proxy: "http://live.invalid",
ALL_PROXY: "http://live.invalid",
NO_PROXY: "live.invalid",
RUSTC_WRAPPER: "/live/sccache",
RUSTC_WORKSPACE_WRAPPER: "/live/wrapper",
SCCACHE_DISABLE: "0",
},
};
const taskBefore = JSON.stringify(task);
const commands = await Promise.all([
first.wrapCommand(task, "pkg/a"),
first.wrapCommand(task, "pkg?a"), // same sanitized label
first.wrapCommand(task, "pkg/a"), // same exact label
second.wrapCommand(task, "pkg/a"),
]);
assert(JSON.stringify(task) === taskBefore);
assert(
environmentSnapshot() === before,
"host environment changed",
);
const results = await Promise.all(
commands.map((command, index) =>
execute(command, {
output: checkout,
label: `environment-${index}`,
timeoutMs: 10_000,
})
),
);
const tables = new Set<string>();
for (const [index, result] of results.entries()) {
assert(result.code === 0 && !result.error, JSON.stringify(result));
const observed = JSON.parse(result.stdout);
const env = observed.env as Record<string, string>;
assert(observed.cwd === checkout && commands[index].cwd === task.cwd);
assert(commands[index].args === task.args);
assert(env.CARGO_MANIFEST_DIR === task.env.CARGO_MANIFEST_DIR);
assert(env.CARGO_HOME === task.env.CARGO_HOME);
assert(env.RUSTUP_HOME === task.env.RUSTUP_HOME);
const namespace = env.YOI_RUNTIME_DIR.replace(/\/run$/, "");
const prefix = `${checkout}/target/restore-test-environment/`;
assert(namespace.startsWith(prefix));
const suffix = namespace.slice(prefix.length);
assert(
suffix.split("/").every((part) => /^[a-zA-Z0-9_-]+$/.test(part)),
);
assert(env.YOI_CONFIG_DIR === `${namespace}/config`);
assert(env.YOI_DATA_DIR === `${namespace}/data`);
assert(
env.YOI_HOME === env.YOI_DATA_DIR,
"secret data must equal data",
);
assert(env.HOME === `${namespace}/home`);
for (
const dir of [
namespace,
env.YOI_CONFIG_DIR,
env.YOI_DATA_DIR,
env.YOI_RUNTIME_DIR,
env.HOME,
]
) {
assert((await Deno.lstat(dir)).isDirectory);
assert(await Deno.realPath(dir) === dir);
}
for (const key of ["TMPDIR", "TMP", "TEMP"]) {
assert(env[key] === `${checkout}/target/t`);
}
for (
const key of [
"GIT_DIR",
"GIT_WORK_TREE",
"GIT_CONFIG",
"GIT_CONFIG_PARAMETERS",
"GIT_CONFIG_COUNT",
"GIT_CONFIG_KEY_0",
"GIT_CONFIG_VALUE_0",
"HTTP_PROXY",
"https_proxy",
"ALL_PROXY",
"NO_PROXY",
]
) {
assert(!(key in env), `${key} leaked to child`);
}
assert(env.GIT_CONFIG_GLOBAL === "/dev/null");
assert(env.GIT_CONFIG_SYSTEM === "/dev/null");
assert(env.GIT_CONFIG_NOSYSTEM === "1");
assert(env.GIT_CEILING_DIRECTORIES === checkout);
assert(env.GIT_TEMPLATE_DIR === `${namespace}/git-template`);
assert(observed.globalConfig === "");
assert(observed.ancestorDiscovered === false);
assert(observed.hooks.length === 0 && observed.info.length === 0);
assert(observed.author.includes("normal-tests@example.invalid"));
assert(env.GIT_AUTHOR_NAME === env.GIT_COMMITTER_NAME);
assert(env.GIT_AUTHOR_EMAIL === env.GIT_COMMITTER_EMAIL);
assert(env.RUSTC_WRAPPER === "" && env.RUSTC_WORKSPACE_WRAPPER === "");
assert(env.SCCACHE_DISABLE === "1");
const table = `${env.YOI_RUNTIME_DIR}/workers.json`;
tables.add(table);
assert(await Deno.readTextFile(table) === String(observed.pid));
}
assert(
tables.size === commands.length,
"independent commands shared a table",
);
assert(environmentSnapshot() === before);
} finally {
await Deno.remove(checkout, { recursive: true });
}
},
});
Deno.test("isolation refuses lexical aliases and symlinked owned paths before external writes", async () => {
for (
const unsafe of ["target", "target/restore-test-environment", "target/t"]
) {
const checkout = await Deno.realPath(await Deno.makeTempDir());
const outside = await Deno.realPath(await Deno.makeTempDir());
try {
await Deno.writeTextFile(`${outside}/sentinel`, "untouched");
if (unsafe !== "target") await Deno.mkdir(`${checkout}/target`);
await Deno.symlink(outside, `${checkout}/${unsafe}`);
await assertRejects(
() => prepareTestIsolation(checkout, "run"),
"symlink",
);
assert(
Array.from(Deno.readDirSync(outside)).map((entry) => entry.name)
.join() ===
"sentinel",
);
assert(await Deno.readTextFile(`${outside}/sentinel`) === "untouched");
} finally {
await Deno.remove(checkout, { recursive: true });
await Deno.remove(outside, { recursive: true });
}
}
const checkout = await Deno.realPath(await Deno.makeTempDir());
try {
await Deno.symlink(checkout, `${checkout}/alias`);
await assertRejects(
() => prepareTestIsolation(`${checkout}/alias`, "run"),
"symlink",
);
for (const suffix of ["/../escape", "/./child", "//child"]) {
await assertRejects(
() => prepareTestIsolation(`${checkout}${suffix}`, "run"),
"normal-only",
);
}
assert(
Array.from(Deno.readDirSync(checkout)).map((entry) => entry.name)
.join() ===
"alias",
);
} finally {
await Deno.remove(checkout, { recursive: true });
}
});
Deno.test("command wrapping rechecks owned ancestors and refuses symlinked namespace children", async () => {
for (
const unsafe of [
"ancestor",
"tmp",
"config",
"data",
"run",
"home",
"git-template",
"git-template/info",
"git-template/hooks",
]
) {
const checkout = await Deno.realPath(await Deno.makeTempDir());
const outside = await Deno.realPath(await Deno.makeTempDir());
try {
const isolation = await prepareTestIsolation(checkout, "run");
let path: string;
if (unsafe === "ancestor") {
path = isolation.root;
await Deno.remove(path);
} else if (unsafe === "tmp") {
path = `${checkout}/target/t`;
await Deno.remove(path);
} else {
const namespace = `${isolation.root}/unsafe-0`;
await Deno.mkdir(namespace);
if (unsafe.includes("/")) await Deno.mkdir(`${namespace}/git-template`);
path = `${namespace}/${unsafe}`;
}
await Deno.symlink(outside, path);
await assertRejects(
() => isolation.wrapCommand(fixtureTask("success", checkout), "unsafe"),
"symlink",
);
assert(Array.from(Deno.readDirSync(outside)).length === 0);
} finally {
await Deno.remove(checkout, { recursive: true });
await Deno.remove(outside, { recursive: true });
}
}
});
for (const [mode, code] of [["success", 0], ["nonzero", 7]] as const) {
Deno.test({
name: `execute streams real ${mode} child output and records its exit`,
@@ -436,15 +740,28 @@ Deno.test({
"runtime listing persists complete name inventory for large and small binaries",
ignore: Deno.build.os !== "linux",
async fn() {
const output = await Deno.makeTempDir();
const output = await Deno.realPath(await Deno.makeTempDir());
try {
const isolation = await prepareTestIsolation(output, "discovery");
const wrappedTables = new Set<string>();
const discovery = await discoverRustCommands(
[
fixtureTask("list-large", output),
fixtureTask("list-small", output),
],
2,
{ output },
{
output,
run: async (task, label) => {
const wrapped = await isolation.wrapCommand(task, label);
wrappedTables.add(`${wrapped.env!.YOI_RUNTIME_DIR}/workers.json`);
return await execute(wrapped, { output, label, timeoutMs: 10_000 });
},
},
);
assert(
wrappedTables.size === 2,
"all discovery launches must use the callback",
);
assert(discovery.inventory.complete);
assert(
@@ -635,6 +952,54 @@ if (import.meta.main) {
);
}
Deno.exit(mode === "list-nonzero" ? 7 : 0);
} else if (mode === "isolation-env") {
const env = Deno.env.toObject();
const scratch = await Deno.makeTempDir({ dir: env.TMPDIR });
const text = (bytes: Uint8Array) => new TextDecoder().decode(bytes);
async function git(args: string[]) {
return await new Deno.Command("git", {
args,
cwd: scratch,
stdout: "piped",
stderr: "piped",
}).output();
}
try {
const ownedRepo = await git([
"-C",
output,
"rev-parse",
"--show-toplevel",
]);
assert(
ownedRepo.success && text(ownedRepo.stdout).trim() === output,
"owned ancestor repository fixture must be discoverable at its root",
);
const ancestor = await git(["rev-parse", "--show-toplevel"]);
const init = await git(["init", "--quiet"]);
assert(init.success, text(init.stderr));
const global = await git(["config", "--global", "--list"]);
assert(global.success, text(global.stderr));
const author = await git(["var", "GIT_AUTHOR_IDENT"]);
assert(author.success, text(author.stderr));
await Deno.writeTextFile(
`${env.YOI_RUNTIME_DIR}/workers.json`,
String(Deno.pid),
);
console.log(JSON.stringify({
env,
cwd: Deno.cwd(),
pid: Deno.pid,
ancestorDiscovered: ancestor.success,
globalConfig: text(global.stdout),
author: text(author.stdout),
hooks: Array.from(Deno.readDirSync(`${scratch}/.git/hooks`)),
info: Array.from(Deno.readDirSync(`${scratch}/.git/info`)),
}));
} finally {
await Deno.remove(scratch, { recursive: true });
}
Deno.exit(0);
} else if (mode === "runner-parent") {
const cancellation = installCancellationHandlers();
try {
+196 -18
View File
@@ -1,9 +1,10 @@
// Bound top-level suites/shards, not total processes; large shards use two test threads.
// Bound top-level suites/shards, not total processes; large shards normally use two threads.
// Cargo selects targets/features. Retain all listed names; never skip cached successes.
import { spawn } from "node:child_process";
import process from "node:process";
import { availableParallelism } from "node:os";
import { fileURLToPath } from "node:url";
import { posix } from "node:path";
// Safety limits, NOT the separate 60s performance goal.
export const SUITE_TIMEOUT_MS = 180_000;
@@ -65,7 +66,19 @@ export function shardRustCommand(
if (new Set(names).size !== names.length) {
throw new Error("cannot shard duplicate test names");
}
if (names.length <= 72) return [{ task, selectedNames: [...names] }];
// RuntimeDirSandbox changes process-global paths. Do not overlap its writes
// with restore guards in the same process, including small integration targets.
const serial = ["worker", "worker-runtime", "yoi-workspace-server"].includes(
task.name.split("/")[0],
);
if (names.length <= 72) {
return [{
task: serial
? { ...task, args: [...task.args, "--test-threads", "1"] }
: task,
selectedNames: [...names],
}];
}
const count = Math.min(32, Math.ceil(names.length / 36));
const shards: string[][] = Array.from({ length: count }, () => []);
names.forEach((name, index) => shards[index % count].push(name));
@@ -75,7 +88,13 @@ export function shardRustCommand(
name: `${task.name}/shard-${
String(index + 1).padStart(2, "0")
}-of-${count}`,
args: [...task.args, "--exact", "--test-threads", "2", ...selectedNames],
args: [
...task.args,
"--exact",
"--test-threads",
serial ? "1" : "2",
...selectedNames,
],
},
selectedNames,
}));
@@ -84,18 +103,27 @@ export function shardRustCommand(
export async function discoverRustCommands(
commands: TestCommand[],
jobs: number,
options: { output: string; signal?: AbortSignal },
options: {
output: string;
signal?: AbortSignal;
run?: (task: TestCommand, label: string) => Promise<ExecutionResult>;
},
) {
const binaries = await mapConcurrent(commands, jobs, async (task, index) => {
const listing = await execute({
const listingTask = {
...task,
name: `${task.name}/list`,
args: [...task.args, "--list", "--format", "terse"],
}, {
...options,
label: `list-${String(index).padStart(3, "0")}`,
};
const label = `list-${String(index).padStart(3, "0")}`;
const listing = await (options.run
? options.run(listingTask, label)
: execute(listingTask, {
output: options.output,
signal: options.signal,
label,
timeoutMs: SUITE_TIMEOUT_MS,
});
}));
let shards: RustShard[] = [];
let error: string | undefined;
try {
@@ -124,7 +152,8 @@ export async function discoverRustCommands(
error,
selectedCases: error
? null
: shards.reduce((sum, shard) => sum + shard.selectedNames.length, 0),
: shards.reduce((sum, shard) =>
sum + shard.selectedNames.length, 0),
shards,
};
});
@@ -426,12 +455,149 @@ export interface TestCommand {
command: string;
args: string[];
cwd: string;
env?: Record<string, string>;
env?: Record<string, string | undefined>;
// Exact link from each execution result into inventory.json, even when two
// Cargo targets happen to share the same package/target display name.
inventoryRef?: { binaryIndex: number; shardIndex: number };
}
// Walk with lstat before creating any child: recursive mkdir/realPath alone
// would follow an existing symlink first. Reject aliases rather than silently
// canonicalizing a caller's checkout. As with the Rust guard, this assumes the
// owned checkout is not concurrently replaced by an adversarial filesystem actor.
async function ownedTestDirectory(checkout: string, relative: string) {
if (
!posix.isAbsolute(checkout) || checkout === "/" ||
posix.normalize(checkout) !== checkout
) throw new Error("test checkout must be an absolute normal-only path");
async function checkDirectory(path: string, owned: boolean) {
const stat = await Deno.lstat(path);
if (!stat.isDirectory || stat.isSymlink) {
throw new Error(
`test isolation refuses non-directory or symlink: ${path}`,
);
}
if (owned && process.getuid && stat.uid !== process.getuid()) {
throw new Error(
`test isolation directory is not owned by this user: ${path}`,
);
}
if (await Deno.realPath(path) !== path) {
throw new Error(
`test isolation refuses non-canonical directory: ${path}`,
);
}
}
let checkoutPath = "";
for (const part of checkout.slice(1).split("/")) {
checkoutPath += `/${part}`;
await checkDirectory(checkoutPath, false);
}
let path = checkout;
for (const part of relative.split("/")) {
if (!part || part === "." || part === "..") {
throw new Error("invalid owned test directory component");
}
path += `/${part}`;
try {
await Deno.mkdir(path, { mode: 0o700 });
} catch (error) {
if (!(error instanceof Deno.errors.AlreadyExists)) throw error;
}
await checkDirectory(path, true);
}
return path;
}
/** Prepare child-only isolation for the maintained normal-test entrypoint.
* Each wrap gets a fresh allocation table, even for repeated/sanitized labels or
* concurrent runners using the same output id. Resources stay under target for
* diagnosis; callers with temporary checkouts own their recursive cleanup.
*/
export async function prepareTestIsolation(checkout: string, runId: string) {
const ownedDirectory = (relative: string) =>
ownedTestDirectory(checkout, relative);
const prefix = "target/restore-test-environment";
await ownedDirectory(prefix);
// Keep tempfile Unix-socket references short. tempfile owns unique resources
// here; neither runners nor suites remove/replace this shared directory.
const tmp = await ownedDirectory("target/t");
const digest = new Uint8Array(
await crypto.subtle.digest("SHA-256", new TextEncoder().encode(runId)),
);
const id = Array.from(
digest.slice(0, 6),
(byte) => byte.toString(16).padStart(2, "0"),
).join("");
const relativeRoot = `${prefix}/runner-${id}-${crypto.randomUUID()}`;
const root = await ownedDirectory(relativeRoot);
let next = 0;
return {
root,
async wrapCommand(task: TestCommand, label: string): Promise<TestCommand> {
const safeLabel = label.replaceAll(/[^a-zA-Z0-9_-]/g, "-").slice(0, 40) ||
"command";
const relative = `${relativeRoot}/${safeLabel}-${next++}`;
// Recheck shared ancestors as well as the command namespace on every wrap.
await ownedDirectory("target/t");
const namespace = await ownedDirectory(relative);
for (
const dir of [
"config",
"data",
"run",
"home",
"git-template/info",
"git-template/hooks",
]
) {
await ownedDirectory(`${relative}/${dir}`);
}
const inherited = { ...process.env, ...task.env };
const env: NonNullable<TestCommand["env"]> = { ...task.env };
// Undefined shadows inherited entries in execute's merge; node spawn omits
// them. Strip indexed Git config and less common Git path overrides too.
for (const key of Object.keys(inherited)) {
if (/^GIT_/i.test(key) || /(^|_)proxy$/i.test(key)) {
env[key] = undefined;
}
}
Object.assign(env, {
HOME: `${namespace}/home`,
YOI_HOME: `${namespace}/data`,
YOI_CONFIG_DIR: `${namespace}/config`,
YOI_DATA_DIR: `${namespace}/data`,
YOI_RUNTIME_DIR: `${namespace}/run`,
TMPDIR: tmp,
TMP: tmp,
TEMP: tmp,
GIT_TEMPLATE_DIR: `${namespace}/git-template`,
GIT_CONFIG_GLOBAL: "/dev/null",
GIT_CONFIG_SYSTEM: "/dev/null",
GIT_CONFIG_NOSYSTEM: "1",
GIT_ATTR_NOSYSTEM: "1",
GIT_CEILING_DIRECTORIES: checkout,
GIT_AUTHOR_NAME: "Normal test runner",
GIT_AUTHOR_EMAIL: "normal-tests@example.invalid",
GIT_COMMITTER_NAME: "Normal test runner",
GIT_COMMITTER_EMAIL: "normal-tests@example.invalid",
RUSTC_WRAPPER: "",
RUSTC_WORKSPACE_WRAPPER: "",
SCCACHE_DISABLE: "1",
});
// HOME is isolated, but toolchain/cache discovery must keep its ordinary
// host locations (these are compiler settings, not live product state).
for (const key of ["CARGO_HOME", "RUSTUP_HOME"] as const) {
const fallback = key === "CARGO_HOME" ? ".cargo" : ".rustup";
env[key] = inherited[key] ||
(process.env.HOME ? `${process.env.HOME}/${fallback}` : undefined);
}
return { ...task, env };
},
};
}
export function rustTestCommands(
artifacts: CargoArtifact[],
packages: { id: string; name: string; manifest_path: string }[],
@@ -497,16 +663,26 @@ async function main() {
"",
);
const jobs = parseJobs(Deno.args);
const output = `${root}/target/normal-tests/${
const outputId = `${
new Date().toISOString().replaceAll(":", "-")
}`;
await Deno.mkdir(output, { recursive: true });
}-${crypto.randomUUID()}`;
const isolation = await prepareTestIsolation(root, outputId);
const output = await ownedTestDirectory(
root,
`target/normal-tests/${outputId}`,
);
const cancellation = installCancellationHandlers();
const run = (
const run = async (
task: TestCommand,
label: string,
timeoutMs = SUITE_TIMEOUT_MS,
) => execute(task, { output, label, timeoutMs, signal: cancellation.signal });
) =>
execute(await isolation.wrapCommand(task, label), {
output,
label,
timeoutMs,
signal: cancellation.signal,
});
const summary: Record<string, unknown> = { phase: "metadata", failed: false };
let executionStart: number | undefined;
const writeSummary = () =>
@@ -516,7 +692,7 @@ async function main() {
{
jobs,
concurrency:
"top-level suites/shards; large Rust shards use 2 test threads, other suites retain internal parallelism",
"top-level suites/shards; worker, worker-runtime and yoi-workspace-server use 1 test thread; other large Rust shards use 2, other suites retain internal parallelism",
platform: "linux",
suiteTimeoutMs: SUITE_TIMEOUT_MS,
buildTimeoutMs: BUILD_TIMEOUT_MS,
@@ -600,6 +776,7 @@ async function main() {
const discovery = await discoverRustCommands(rust, jobs, {
output,
signal: cancellation.signal,
run,
});
summary.discoverySeconds = (performance.now() - discoveryStart) / 1000;
summary.inventory = "inventory.json";
@@ -655,6 +832,7 @@ async function main() {
"--allow-write",
"--allow-run",
"--allow-env",
"--allow-sys=uid",
"tools/test.test.ts",
"tools/check-state-contract-names.test.ts",
],
@@ -663,7 +841,7 @@ async function main() {
...discovery.tasks,
];
console.log(
`Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (large Rust shards: 2 test threads; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`,
`Running ${rust.length} Rust binaries (${discovery.inventory.selectedCases} selected names, including ignored; ${discovery.tasks.length} suites/shards), doctests, Web and runner suites; at most ${jobs} top-level suites/shards (worker/worker-runtime/yoi-workspace-server: 1 test thread; other large Rust shards: 2; other internal parallelism unchanged; NOT a total process limit). Logs: ${output}`,
);
const results: ReturnType<typeof resultSummary>[] = [];
summary.results = results;
@@ -1,6 +1,6 @@
// @generated by yoi-companion-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7)
// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e)
// Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/companion-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check
@@ -1,6 +1,6 @@
// @generated by yoi-repository-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7)
// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e)
// Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output
// Output: web/workspace/src/lib/generated/repository-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check
@@ -1,6 +1,6 @@
// @generated by yoi-runtime-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7)
// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e)
// Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/runtime-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check
@@ -181,7 +181,7 @@ export type WorkerRestoreResponse = { result: WorkerRestoreResult; runtime_id: s
export type WorkerRestoreResult = { diagnostics?: Array<Diagnostic>; state: WorkerRestoreState; worker?: WorkerSummary | null; };
export type WorkerRestoreState = "accepted" | "rejected" | "rolled_back" | "reconciliation_required";
export type WorkerRestoreState = "accepted" | "rejected" | "rolled_back" | "reconciliation_required" | "superseded";
export type WorkerRetentionResponse = { pinned: boolean; retention_state: string; runtime_id: string; worker_id: string; workspace_id: string; };
@@ -1,6 +1,6 @@
// @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7)
// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e)
// Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/ticket-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check
@@ -1,6 +1,6 @@
// @generated by yoi-workdir-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7)
// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e)
// Options: workdir-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/workdir-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_workdir_api_types -- --check
@@ -1,6 +1,6 @@
// @generated by yoi-worker-openapi-typescript@1. Do not edit.
// Input: openapi/server-api.json
// Input contract: 0.1.0 (sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7)
// Input contract: 0.1.0 (sha256:4159d6baa918c9d7d3cb6ae02207aad877a69ad1dd4207beb63b271deb24176e)
// Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output
// Output: web/workspace/src/lib/generated/worker-launch-api.ts
// Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check
@@ -363,8 +363,13 @@ export function parseWorkerRestoreResponse(
exactKeys(result, ["state"], ["diagnostics", "worker"], `${path}.result`);
const state = boundedString(result.state, `${path}.result.state`);
if (
!["accepted", "rejected", "rolled_back", "reconciliation_required"]
.includes(state)
![
"accepted",
"rejected",
"rolled_back",
"superseded",
"reconciliation_required",
].includes(state)
) {
fail(`${path}.result.state`, "has an unknown value");
}
@@ -427,7 +427,12 @@ test.each(["switch", "dispose"] as const)(
);
function restoreResponse(
state: "accepted" | "rejected" | "rolled_back" | "reconciliation_required",
state:
| "accepted"
| "rejected"
| "rolled_back"
| "superseded"
| "reconciliation_required",
extra = {},
) {
return Response.json({
@@ -461,7 +466,13 @@ test.each(["stopped", "idle", "running", "paused", "unavailable"] as const)(
);
test.each(
["accepted", "rejected", "rolled_back", "reconciliation_required"] as const,
[
"accepted",
"rejected",
"rolled_back",
"superseded",
"reconciliation_required",
] as const,
)(
"Restore classifies %s through the shared helper, without treating HTTP 200 as success",
async (state) => {
@@ -470,10 +481,11 @@ test.each(
const unsubscribe = workspaceAlerts.subscribe((value) => {
alerts = value;
});
const request = vi.fn(async (input: RequestInfo | URL) =>
const request = vi.fn(
async (input: RequestInfo | URL, _init?: RequestInit) =>
String(input).endsWith("/restore")
? restoreResponse(state)
: Response.json(plan())
: Response.json(plan()),
);
vi.stubGlobal("fetch", request);
render(WorkersPage, { data: data() });
@@ -498,6 +510,16 @@ test.each(
screen.queryByRole("button", { name: "Retry Restore Catalog worker" }),
),
).toBe(state === "reconciliation_required");
expect(
request.mock.calls.filter(([url]) => String(url).endsWith("/restore")),
).toHaveLength(1);
expect(
request.mock.calls
.filter(([, init]) => init?.method && init.method !== "GET")
.map(([url]) => String(url)),
).toEqual([
"/api/w/page-test/runtimes/runtime-a/workers/worker-a/restore",
]);
// The response is never projected into the catalog by the action.
expect(screen.getByRole("cell", { name: "stopped" })).toBeTruthy();
unsubscribe();
@@ -305,6 +305,7 @@ Deno.test("Restore classification never promotes nonaccepted results and preserv
"accepted",
"rejected",
"rolled_back",
"superseded",
"reconciliation_required",
] as const
) {
+2 -1
View File
@@ -173,12 +173,13 @@ function restoredWorker() {
restore_observation_token: "generation-b",
};
}
Deno.test("Restore parser preserves all four states and optional Workspace summary", () => {
Deno.test("Restore parser preserves typed outcomes and optional Workspace summary", () => {
for (
const state of [
"accepted",
"rejected",
"rolled_back",
"superseded",
"reconciliation_required",
]
) {