fix: merge develop and reconcile schema histories

Preserve both published schema-103 histories and converge their Passkey results and Job input retention features transactionally at schema 104. Retain local Reviewer, integration-event, auth and sidebar fixes while adopting develop restore and E2E updates; regenerate API contracts.

Validation: schema convergence regressions, full Server/Worker/Runtime suites, server-api and merge-request tests, Web unit/component tests, root cargo check, E2E harness tests and restore/auth target compilation, generated artifacts, fmt and naming checks. Product-process E2E not rerun; live services and databases untouched.
This commit is contained in:
2026-10-28 04:01:34 +09:00
218 changed files with 48985 additions and 11155 deletions
Generated
+7 -2
View File
@@ -3990,6 +3990,7 @@ dependencies = [
"protocol",
"serde",
"serde_json",
"session-store",
"tokio",
"workdir",
]
@@ -4783,12 +4784,14 @@ version = "0.1.0"
dependencies = [
"chrono",
"feature-storage",
"job",
"manifest",
"memory",
"rusqlite",
"serde",
"serde_json",
"server-api",
"session-store",
"sha2 0.11.0",
"tempfile",
"thiserror 2.0.18",
@@ -6497,6 +6500,7 @@ name = "workdir"
version = "0.1.0"
dependencies = [
"async-trait",
"axum",
"fs-operation",
"libc",
"manifest",
@@ -6520,7 +6524,6 @@ dependencies = [
"axum",
"base64 0.22.1",
"chrono",
"clap",
"client",
"config-source",
"dotenv",
@@ -6558,6 +6561,7 @@ dependencies = [
"toml",
"tools",
"tracing",
"tracing-subscriber",
"tungstenite 0.28.0",
"uuid",
"wip-client",
@@ -6597,6 +6601,7 @@ dependencies = [
"tokio",
"tokio-tungstenite 0.29.0",
"toml",
"tools",
"tower",
"tracing",
"tracing-subscriber",
@@ -6684,6 +6689,7 @@ dependencies = [
"futures",
"manifest",
"project-record",
"protocol",
"reqwest",
"serde",
"serde_json",
@@ -6699,7 +6705,6 @@ dependencies = [
"tui",
"uuid",
"workdir",
"worker",
]
[[package]]
+1 -38
View File
@@ -100,21 +100,6 @@ pub fn worker_allocation_path() -> Option<PathBuf> {
worker_allocation_path_from_runtime_dir(runtime_dir())
}
/// `<runtime_dir>/<worker_name>/` — per-Worker runtime directory.
pub fn worker_runtime_dir(worker_name: &str) -> Option<PathBuf> {
worker_runtime_dir_from_runtime_dir(runtime_dir(), worker_name)
}
/// `<runtime_dir>/<worker_name>/sock` — Worker's Unix socket path.
///
/// The `worker` crate's `RuntimeDir::socket_path()` creates the socket inside the
/// Worker process. This function **predicts** the path for external callers that
/// know the Worker name, such as the TUI attach flow. Both should return the same
/// path.
pub fn worker_socket_path(worker_name: &str) -> Option<PathBuf> {
worker_socket_path_from_runtime_dir(runtime_dir(), worker_name)
}
// ---- internals --------------------------------------------------------------
fn resolve_config_dir_from_parts(
@@ -204,20 +189,6 @@ fn worker_allocation_path_from_runtime_dir(runtime_dir: Option<PathBuf>) -> Opti
Some(runtime_dir?.join("workers.json"))
}
fn worker_runtime_dir_from_runtime_dir(
runtime_dir: Option<PathBuf>,
worker_name: &str,
) -> Option<PathBuf> {
Some(runtime_dir?.join(worker_name))
}
fn worker_socket_path_from_runtime_dir(
runtime_dir: Option<PathBuf>,
worker_name: &str,
) -> Option<PathBuf> {
Some(worker_runtime_dir_from_runtime_dir(runtime_dir, worker_name)?.join("sock"))
}
/// Treats empty environment values as unset. `std::env::var` distinguishes
/// `Ok("")` from `Err(NotPresent)`, but path resolution treats both as unset
/// for intuitive behavior.
@@ -455,16 +426,8 @@ mod tests {
PathBuf::from("/sand/sessions")
);
assert_eq!(
worker_allocation_path_from_runtime_dir(runtime_dir.clone()).unwrap(),
worker_allocation_path_from_runtime_dir(runtime_dir).unwrap(),
PathBuf::from("/sand/run/workers.json")
);
assert_eq!(
worker_runtime_dir_from_runtime_dir(runtime_dir.clone(), "foo").unwrap(),
PathBuf::from("/sand/run/foo")
);
assert_eq!(
worker_socket_path_from_runtime_dir(runtime_dir, "foo").unwrap(),
PathBuf::from("/sand/run/foo/sock")
);
}
}
+184 -83
View File
@@ -617,7 +617,8 @@ impl fmt::Display for McpRequestStage {
}
}
/// A running initialized stdio MCP client.
/// An owned stdio MCP process, initialized by `connect` or explicitly after
/// `spawn_uninitialized` when startup cleanup must retain it before any await.
pub struct McpStdioClient {
server_name: String,
limits: McpStdioLimits,
@@ -636,6 +637,9 @@ pub struct McpStdioClient {
impl McpStdioClient {
/// Spawn, initialize, negotiate capabilities, and send notifications/initialized.
/// Initialization errors attempt shutdown, but this convenience API cannot
/// return an unreaped client on failure/cancellation. Its error is not proof
/// of cleanup; use `spawn_uninitialized` with retained ownership when needed.
pub async fn connect(
spec: McpStdioServerSpec,
limits: McpStdioLimits,
@@ -667,6 +671,21 @@ impl McpStdioClient {
async fn spawn(
spec: McpStdioServerSpec,
limits: McpStdioLimits,
) -> Result<Self, McpClientError> {
Self::spawn_uninitialized(spec, limits)
}
/// Spawn without awaiting the handshake, so a cleanup owner can retain the
/// client before initialization can fail or be cancelled. Call
/// `initialize_with_timeout` before discovery; on error/cancellation, shut
/// down this same client rather than spawning a replacement.
///
/// Once the process spawns, this returns its owner even if an expected pipe
/// is missing. Initialization then fails, but cleanup can still reap the
/// actual child rather than relying on kill-on-drop for a post-spawn error.
pub fn spawn_uninitialized(
spec: McpStdioServerSpec,
limits: McpStdioLimits,
) -> Result<Self, McpClientError> {
let redactor = spec.redactor();
let mut command = Command::new(&spec.command);
@@ -688,29 +707,23 @@ impl McpStdioClient {
McpErrorKind::Io(redactor.redact(&err.to_string())),
)
})?;
let stdin = child.stdin.take().ok_or_else(|| {
McpClientError::new(
&spec.name,
McpPhase::Spawn,
McpErrorKind::Protocol("child stdin was not piped".into()),
)
})?;
let stdout = child.stdout.take().ok_or_else(|| {
McpClientError::new(
&spec.name,
McpPhase::Spawn,
McpErrorKind::Protocol("child stdout was not piped".into()),
)
})?;
let stderr = child.stderr.take().ok_or_else(|| {
McpClientError::new(
&spec.name,
McpPhase::Spawn,
McpErrorKind::Protocol("child stderr was not piped".into()),
)
})?;
let stdin = child.stdin.take();
let stdout = child.stdout.take();
let stderr = child.stderr.take();
let missing_pipe = if stdin.is_none() {
Some("stdin")
} else if stdout.is_none() {
Some("stdout")
} else if stderr.is_none() {
Some("stderr")
} else {
None
};
let stdin: SharedStdin = Arc::new(Mutex::new(Some(Box::new(stdin))));
let stdin: SharedStdin =
Arc::new(Mutex::new(stdin.map(|stdin| {
Box::new(stdin) as Box<dyn AsyncWrite + Send + Unpin>
})));
let diagnostics = Arc::new(Mutex::new(BoundedDiagnostics::new(
spec.name.clone(),
limits.max_diagnostic_lines,
@@ -718,16 +731,32 @@ impl McpStdioClient {
)));
let list_changes = Arc::new(Mutex::new(BoundedListChanged::new(spec.name.clone())));
let (tx, rx) = mpsc::channel(16);
let reader_task = spawn_stdout_reader(
spec.name.clone(),
stdout,
stdin.clone(),
tx,
limits.clone(),
redactor.clone(),
list_changes.clone(),
);
let stderr_task = spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone());
// Do not return an error that discards an already spawned child. Queue
// setup failure before reader events so initialization cannot succeed.
if let Some(pipe) = missing_pipe {
tx.try_send(ReaderEvent::Error(McpClientError::new(
&spec.name,
McpPhase::Spawn,
McpErrorKind::Protocol(format!("child {pipe} was not piped")),
)))
.expect("new response channel has capacity");
}
let reader_task = match stdout {
Some(stdout) => spawn_stdout_reader(
spec.name.clone(),
stdout,
stdin.clone(),
tx,
limits.clone(),
redactor.clone(),
list_changes.clone(),
),
None => tokio::spawn(async {}),
};
let stderr_task = match stderr {
Some(stderr) => spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone()),
None => tokio::spawn(async {}),
};
Ok(Self {
server_name: spec.name,
@@ -746,6 +775,25 @@ impl McpStdioClient {
})
}
/// Perform the bounded handshake on an already retained client. Failure or
/// cancellation does not drop or shut down the client: its owner must do so.
/// Do not retry initialization on a partially initialized connection.
pub async fn initialize_with_timeout(&mut self) -> Result<(), McpClientError> {
let started = Instant::now();
match timeout(self.limits.startup_timeout, self.initialize()).await {
Ok(result) => result,
Err(_) => Err(McpClientError::new(
&self.server_name,
McpPhase::Initialize,
McpErrorKind::Timeout {
operation: "startup".to_string(),
elapsed: started.elapsed(),
},
)
.with_diagnostics(self.snapshot_diagnostics().await)),
}
}
async fn initialize(&mut self) -> Result<(), McpClientError> {
let result: InitializeResult = self
.request(
@@ -1080,6 +1128,8 @@ impl McpStdioClient {
}
/// Close stdin and wait for process exit, falling back to terminate and kill.
/// Errors and cancellation retain the child for another attempt on this
/// client. Only confirmed reaping releases the process handle.
pub async fn shutdown(&mut self) -> Result<ShutdownReport, McpClientError> {
self.shutdown_started = true;
let deadline = Instant::now() + self.limits.shutdown_timeout;
@@ -1091,64 +1141,55 @@ impl McpStdioClient {
.await
.is_ok();
let mut child = match self.child.take() {
Some(child) => child,
None => return Ok(ShutdownReport::already_finished()),
};
// Keep the actual process handle in the client across every await and
// error. Taking it before wait loses retry/reaping authority on cancel.
if self.child.is_none() {
return Ok(ShutdownReport::already_finished());
}
if !closed {
self.reader_task.abort();
return self.terminate_then_kill(child).await;
return self.terminate_then_kill().await;
}
let child = self.child.as_mut().expect("checked child");
if let Ok(Some(status)) = child.try_wait() {
self.reader_task.abort();
self.stderr_task.abort();
return Ok(ShutdownReport {
exit_status: Some(status),
terminated: false,
killed: false,
});
return Ok(self.finish_shutdown(status, false, false));
}
match timeout_at(deadline, child.wait()).await {
Ok(Ok(status)) => {
self.reader_task.abort();
self.stderr_task.abort();
Ok(ShutdownReport {
exit_status: Some(status),
terminated: false,
killed: false,
})
}
Ok(Ok(status)) => Ok(self.finish_shutdown(status, false, false)),
Ok(Err(err)) => Err(McpClientError::new(
&self.server_name,
McpPhase::Shutdown,
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
)
.with_diagnostics(self.snapshot_diagnostics().await)),
Err(_) => self.terminate_then_kill(child).await,
Err(_) => self.terminate_then_kill().await,
}
}
async fn terminate_then_kill(
fn finish_shutdown(
&mut self,
mut child: Child,
) -> Result<ShutdownReport, McpClientError> {
let mut terminated = false;
let mut killed = false;
if send_terminate(&mut child).is_ok() {
terminated = true;
status: ExitStatus,
terminated: bool,
killed: bool,
) -> ShutdownReport {
// No await separates confirmed reaping from releasing the handle.
drop(self.child.take());
self.reader_task.abort();
self.stderr_task.abort();
ShutdownReport {
exit_status: Some(status),
terminated,
killed,
}
}
async fn terminate_then_kill(&mut self) -> Result<ShutdownReport, McpClientError> {
let child = self.child.as_mut().expect("shutdown retains child");
let terminated = send_terminate(child).is_ok();
match timeout(self.limits.kill_timeout, child.wait()).await {
Ok(Ok(status)) => {
self.reader_task.abort();
self.stderr_task.abort();
Ok(ShutdownReport {
exit_status: Some(status),
terminated,
killed,
})
}
Ok(Ok(status)) => Ok(self.finish_shutdown(status, terminated, false)),
Ok(Err(err)) => Err(McpClientError::new(
&self.server_name,
McpPhase::Shutdown,
@@ -1163,7 +1204,6 @@ impl McpStdioClient {
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
)
})?;
killed = true;
let status = timeout(self.limits.kill_timeout, child.wait())
.await
.map_err(|_| {
@@ -1183,13 +1223,7 @@ impl McpStdioClient {
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
)
})?;
self.reader_task.abort();
self.stderr_task.abort();
Ok(ShutdownReport {
exit_status: Some(status),
terminated,
killed,
})
Ok(self.finish_shutdown(status, terminated, true))
}
}
}
@@ -1197,10 +1231,10 @@ impl McpStdioClient {
impl Drop for McpStdioClient {
fn drop(&mut self) {
if !self.shutdown_started {
if let Some(child) = &mut self.child {
let _ = child.start_kill();
}
// An attempted shutdown is not proof of exit. Errors and cancellation
// leave the child here so Drop must still make a best-effort kill.
if let Some(child) = &mut self.child {
let _ = child.start_kill();
}
self.reader_task.abort();
self.stderr_task.abort();
@@ -1784,3 +1818,70 @@ fn send_terminate(child: &mut Child) -> Result<(), ()> {
#[cfg(test)]
mod deadline_tests;
#[cfg(all(test, unix))]
mod shutdown_retention_tests {
//! The real client and OS process are used; only the external server is a
//! local shell that acknowledges initialization and ignores TERM/EOF.
use super::*;
use std::future::{Future, poll_fn};
use std::task::Poll;
#[tokio::test]
async fn cancelled_graceful_or_terminate_wait_retries_reaping_the_same_child() {
for cancel_in_fallback in [false, true] {
let script = r#"trap '' TERM
read init || exit 1
printf '%s\n' '{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-11-25","capabilities":{},"serverInfo":{"name":"retained","version":"1"}}}'
while :; do :; done
"#;
let mut client = McpStdioClient::connect(
McpStdioServerSpec::new("retained", "/bin/sh").args(["-c", script]),
McpStdioLimits {
shutdown_timeout: if cancel_in_fallback {
Duration::ZERO
} else {
Duration::from_secs(30)
},
kill_timeout: Duration::from_secs(30),
..Default::default()
},
)
.await
.expect("initialize after installing TERM handler");
let pid = client.child.as_ref().unwrap().id();
let mut shutdown = Box::pin(client.shutdown());
poll_fn(|cx| {
assert!(shutdown.as_mut().poll(cx).is_pending());
Poll::Ready(())
})
.await;
drop(shutdown);
assert_eq!(client.child.as_ref().and_then(Child::id), pid);
assert!(client.shutdown_started);
client.limits.shutdown_timeout = Duration::from_millis(20);
client.limits.kill_timeout = Duration::from_millis(50);
let report = timeout(Duration::from_secs(2), client.shutdown())
.await
.expect("retry bounded")
.expect("retry reaps original child");
assert!(
report.exit_status.is_some(),
"not an already-finished receipt"
);
assert!(report.killed, "TERM was ignored");
assert!(client.child.is_none());
let mut status = 0;
assert_eq!(
unsafe { libc::waitpid(pid.unwrap() as libc::pid_t, &mut status, libc::WNOHANG) },
-1
);
assert_eq!(
std::io::Error::last_os_error().raw_os_error(),
Some(libc::ECHILD),
"shutdown, not a later observer, reaped the original process"
);
}
}
}
+8 -19
View File
@@ -52,7 +52,7 @@ pub enum AuthenticatedInputSource {
/// Assigned whenever a serialized tracked method crosses an untrusted
/// protocol boundary. Receivers must handle it exactly like public input.
UntrustedWire,
/// Stamped only by the operator-owned standalone in-process transport.
/// Stamped only by the operator-owned Standalone Host after local admission.
/// This is not a Backend Account or authority accepted from wire fields.
LocalOperator,
Account {
@@ -399,15 +399,12 @@ pub enum Method {
/// Typed lifecycle events sent from a child Worker to its parent.
///
/// Delivered as `Method::WorkerEvent` over the parent's Unix socket. The
/// parent Controller always applies variant-specific side effects
/// (registry / worker-allocation updates). Agent-visible variants are also
/// queued into the notification buffer; control-plane-only variants are
/// not injected into the parent's LLM context.
/// Delivered in-process to the parent Controller's protocol intake. Agent-visible variants
/// are queued into the notification buffer. Reports are advisory: receivers must check the
/// actual child session before treating its work as complete.
///
/// Transport is fire-and-forget; receivers must tolerate out-of-order
/// delivery (e.g. `TurnEnded` arriving after `ShutDown` for the same
/// child Worker).
/// Retained legacy control-only variants can be decoded from saved history, but do not
/// register live children, delegate scope, or establish a socket callback.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
@@ -429,16 +426,8 @@ pub enum WorkerEvent {
/// Child has stopped (controller loop is exiting).
ShutDown { worker_name: String },
/// Child SubWorker sub-delegated scope to a grandchild SubWorker via `SubWorkerSpawn`.
///
/// Control-plane only: receivers apply registry side effects and
/// propagate upward, but do not expose this as an agent notification.
///
/// The parent uses this to add the grandchild to its own
/// `spawned_workers.json` so it can manage the grandchild directly
/// even if the intermediate child dies. The parent then re-fires
/// this event upward (if it has a parent of its own) to maintain
/// the chain to root.
/// Legacy process-child scope report retained for saved Session decoding only.
/// Live Controllers ignore it; only the Host-owned Internal registry grants child scope.
ScopeSubDelegated {
/// Sub-delegating Worker (= the sender itself).
parent_worker: String,
+1
View File
@@ -9,6 +9,7 @@ license.workspace = true
api-macros = { workspace = true, features = ["reqwest", "axum"] }
manifest.workspace = true
protocol.workspace = true
session-store.workspace = true
serde = { workspace = true, features = ["derive"] }
workdir.workspace = true
+64 -6
View File
@@ -86,6 +86,8 @@ pub enum WorkerRestoreState {
Accepted,
Rejected,
RolledBack,
/// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.
Superseded,
ReconciliationRequired,
}
@@ -836,6 +838,34 @@ pub struct SessionMaterialPinResponse {
pub pin_id: String,
}
/// One body-bearing immutable slice. The byte budget includes slice metadata,
/// but not the enclosing availability enum or the caller's durable Job envelope.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SessionRecordingCaptureRequest {
pub workspace_id: String,
pub source: SessionPublicSource,
pub expected_session_id: String,
pub start: session_store::capture::RecordingPosition,
pub max_entries: usize,
pub max_serialized_bytes: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(tag = "availability", rename_all = "snake_case")]
pub enum SessionRecordingCaptureAvailability {
NotMaterialized {
session_id: String,
},
Captured {
slice: session_store::capture::RecordingCaptureSlice,
},
Unavailable {
reason: SessionPublicUnavailableReason,
message: String,
},
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SessionMaterialCatalogRequest {
@@ -1246,6 +1276,16 @@ pub trait RuntimeApi {
#[body] request: SessionMaterialPinRequest,
) -> Result<SessionMaterialPinResponse, RuntimeApiError>;
#[post(
"/v1/session-public/recording-capture",
status = 200,
error_status = 400
)]
async fn session_recording_capture(
&self,
#[body] request: SessionRecordingCaptureRequest,
) -> Result<SessionRecordingCaptureAvailability, RuntimeApiError>;
#[post(
"/v1/session-public/material/catalog",
status = 200,
@@ -1540,13 +1580,24 @@ mod tests {
)
.unwrap();
assert!(request.preparation.is_none());
for state in [
"accepted",
"rejected",
"rolled_back",
"reconciliation_required",
for (state, wire) in [
(WorkerRestoreState::Accepted, "accepted"),
(WorkerRestoreState::Rejected, "rejected"),
(WorkerRestoreState::RolledBack, "rolled_back"),
(WorkerRestoreState::Superseded, "superseded"),
(
WorkerRestoreState::ReconciliationRequired,
"reconciliation_required",
),
] {
assert!(serde_json::from_value::<WorkerRestoreState>(serde_json::json!(state)).is_ok());
assert_eq!(
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(wire)).unwrap(),
state
);
assert_eq!(
serde_json::to_value(state).unwrap(),
serde_json::json!(wire)
);
}
assert!(
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(
@@ -1730,6 +1781,13 @@ mod tests {
Err(test_error(501))
}
async fn session_recording_capture(
&self,
_request: SessionRecordingCaptureRequest,
) -> Result<SessionRecordingCaptureAvailability, RuntimeApiError> {
Err(test_error(501))
}
async fn session_material_catalog(
&self,
_request: SessionMaterialCatalogRequest,
+19 -5
View File
@@ -125,6 +125,8 @@ pub struct WorkdirOperationApiError {
pub message: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub diagnostics: Option<Vec<Diagnostic>>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub audit_failures: Vec<workdir::http::WorkdirAuditFailure>,
#[serde(skip, default = "default_repository_error_status")]
status: u16,
/// Server-local metadata retained by generated error response extensions, never the wire body.
@@ -156,6 +158,7 @@ impl WorkdirOperationApiError {
code: None,
message: error.message,
diagnostics: Some(error.diagnostics),
audit_failures: Vec::new(),
status: error.status,
workdir_log: None,
denial_reason: None,
@@ -170,6 +173,7 @@ impl WorkdirOperationApiError {
code: Some(error.code),
message: error.message,
diagnostics: None,
audit_failures: Vec::new(),
status,
workdir_log: None,
denial_reason: error.denial_reason,
@@ -552,8 +556,9 @@ pub struct WorkspaceWorkerSessionHistoryResponse {
/// comparisons do not replace admission checks under the Runtime operation lock; conflicts do
/// not trigger automatic token replacement.
/// [`WorkerRestoreState`] Accepted means Restore committed, not that the subsequent run succeeded.
/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects, and
/// ReconciliationRequired needing settled-state checks. Response loss does not imply rollback.
/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects,
/// Superseded by a later Stop, and ReconciliationRequired needing settled-state checks.
/// Response loss does not imply rollback.
#[api(reqwest, axum, openapi)]
pub trait ServerApi {
#[get("/health", status = 200, error_status = 400)]
@@ -7593,6 +7598,9 @@ pub struct CurrentWorkerWorkdirAttachmentResponse {
pub alias: String,
pub working_directory_id: String,
pub capabilities: workdir::WorkdirSessionCapabilities,
/// Present for attachment responses, absent after detach.
#[serde(skip_serializing_if = "Option::is_none")]
pub environment: Option<workdir::WorkdirSessionEnvironment>,
pub attached: bool,
}
@@ -7605,7 +7613,9 @@ pub struct CurrentWorkerWorkdirOperationRequest {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
#[serde(transparent)]
pub struct CurrentWorkerWorkdirOperationResponse(pub workdir::http::WorkdirSessionOperationResult);
pub struct CurrentWorkerWorkdirOperationResponse(
pub workdir::http::WorkdirSessionOperationResponse,
);
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
@@ -10653,8 +10663,8 @@ pub enum WorkerOperationState {
/// Cross-layer outcome of an explicit Worker restore operation.
///
/// `Rejected` is reserved for read-only preflight failures. Once live restore
/// work starts, failure is either a confirmed `RolledBack` operation or a
/// `ReconciliationRequired` result whose commit state must be reread/retried.
/// work starts, distinguish confirmed `RolledBack`, unresolved
/// `ReconciliationRequired`, and a later Stop's `Superseded` outcome.
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
#[serde(rename_all = "snake_case")]
@@ -10662,6 +10672,8 @@ pub enum WorkerRestoreState {
Accepted,
Rejected,
RolledBack,
/// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.
Superseded,
ReconciliationRequired,
}
@@ -12759,6 +12771,7 @@ mod tests {
(WorkerRestoreState::Accepted, "accepted"),
(WorkerRestoreState::Rejected, "rejected"),
(WorkerRestoreState::RolledBack, "rolled_back"),
(WorkerRestoreState::Superseded, "superseded"),
(
WorkerRestoreState::ReconciliationRequired,
"reconciliation_required",
@@ -13367,6 +13380,7 @@ mod tests {
alias: "checkout".to_string(),
working_directory_id: "workdir".to_string(),
capabilities: workdir::WorkdirSessionCapabilities::EMPTY,
environment: None,
attached: false,
};
let value = serde_json::to_value(&response).unwrap();
+2
View File
@@ -11,5 +11,7 @@ checks or Runtime catalog lifecycle management.
- [`worker_session_store`]: storage boundary for a Session bound to a Worker
- [`worker_metadata`]: current Worker state and active Segment switching
- [`system_item`]: records of system input presented to the model
- [`capture`]: Worker-independent immutable public values and bounded body transport;
authorization, source I/O, and delivery lifecycle stay with the Host
- [`public_snapshot`] / [`public_index`]: public projections of stored history
- [`event_trace`]: diagnostic records independent of history replay
+967
View File
@@ -0,0 +1,967 @@
//! Exploration domain for immutable Session captures, with no Workspace/Memory mutation capability.
//!
//! A capture is an ordered projection of committed entries; exploration excludes reasoning and
//! system prompts, including Host compaction summaries and other derived context. Full inherited
//! summary input belongs to compaction, not public observation or Memory evidence. Overview,
//! search, bounded reads, and evidence conversion share `SessionEntryRef`.
//! `SessionEntryRef` is a contextual UI reference. `ScopedSessionEntryRef` additionally binds
//! the Session and exact immutable capture, and is required for portable checked operations.
//! Appends preserve existing entry identities. Sparse real user/assistant anchors and intervening
//! entry counts index long histories. Search with an empty filter also provides bounded range lists.
//!
//! Host-fixed snapshot exploration, Memory candidate evidence validation, and current Worker
//! observation share only this domain; they do not enable or modify each other's Features.
//! The provider owns authorization to recapture a current Worker. A capture itself proves neither
//! completion nor approval and grants no business-mutation authority.
//!
//! Recording Hosts use the common committed replay through the guarded APIs below.
//! [`recording_catalog_worker_guarded`] and [`recording_catalog_fs_guarded`] expose
//! descriptors only to an admission callback under the caller's existing publication/
//! erasure guard and persisted active selector, without allocating a full body DTO.
//! [`capture_recording_slice_worker_guarded`] and [`capture_recording_slice_fs_guarded`]
//! separately construct bounded body-bearing inputs. Admission descriptors are not
//! sufficient Job inputs; saved recording DTOs are read/sliced without live storage.
use std::sync::Arc;
use crate::{
LoggedSessionHistoryMetadata as SessionHistoryMetadata,
LoggedSessionHistoryOrigin as HistoryOrigin,
};
mod body;
mod recording;
use agen::{HistoryEntry, Item, Role};
pub use body::*;
use protocol::{
SessionContentPart, SessionEntryProvenance, SessionMessageRole, SessionSnapshot,
SessionSnapshotEntryData,
};
pub use recording::*;
use serde::{Deserialize, Serialize};
const DEFAULT_SEARCH_LIMIT: usize = 20;
const MAX_SEARCH_LIMIT: usize = 50;
const DEFAULT_READ_MAX_ITEMS: usize = 40;
const MAX_READ_MAX_ITEMS: usize = 80;
const DEFAULT_READ_MAX_BYTES: usize = 32 * 1024;
const OVERVIEW_ANCHOR_STRIDE: usize = 8;
/// Append-stable contextual UI reference derived from a logical entry identity.
/// Portable consumers must bind it with `SessionCapture::scoped_ref` before use.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(transparent)]
pub struct SessionEntryRef(String);
impl SessionEntryRef {
pub fn from_history_entry_id(entry_id: &crate::LoggedSessionHistoryEntryId) -> Self {
Self(format!("E{}", entry_id.0))
}
pub fn parse(value: &str) -> Option<Self> {
let suffix = value.strip_prefix('E')?;
if suffix.is_empty()
// Legacy Host boundary identities contain both Session and Segment UUIDs.
|| suffix.len() > 256
|| !suffix
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
{
return None;
}
Some(Self(value.to_string()))
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl std::fmt::Display for SessionEntryRef {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.0)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReferenceKind {
User,
Assistant,
System,
Tool,
}
impl ReferenceKind {
pub fn as_str(self) -> &'static str {
match self {
Self::User => "user",
Self::Assistant => "assistant",
Self::System => "system",
Self::Tool => "tool",
}
}
pub fn parse(value: &str) -> Option<Self> {
match value {
"user" => Some(Self::User),
"assistant" | "agent" => Some(Self::Assistant),
"system" => Some(Self::System),
"tool" => Some(Self::Tool),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ToolPart {
Input,
Output,
Both,
}
impl ToolPart {
pub fn parse(value: &str) -> Option<Self> {
match value {
"input" => Some(Self::Input),
"output" => Some(Self::Output),
"both" => Some(Self::Both),
_ => None,
}
}
pub fn as_str(self) -> &'static str {
match self {
Self::Input => "input",
Self::Output => "output",
Self::Both => "both",
}
}
pub fn matches(self, actual: ToolPart) -> bool {
matches!(self, ToolPart::Both) || self == actual
}
}
#[derive(Debug, Clone)]
pub struct OverviewItem {
pub id: SessionEntryRef,
pub origin: SessionEntryProvenance,
pub entry_range: [u64; 2],
pub kind: ReferenceKind,
pub label: String,
pub text: String,
pub intervening_entries: usize,
}
#[derive(Debug, Clone)]
pub struct ReferenceEntry {
pub attachment_count: usize,
use_policy: Option<protocol::SessionEntryUsePolicy>,
pub id: SessionEntryRef,
pub origin: SessionEntryProvenance,
pub entry_range: [u64; 2],
pub kind: ReferenceKind,
pub tool_part: Option<ToolPart>,
pub tool_name: Option<String>,
pub label: String,
pub summary: String,
search_text: String,
}
#[derive(Debug, Clone, Default)]
pub struct SearchOptions {
pub query: String,
pub kind: Option<ReferenceKind>,
pub tool_part: Option<ToolPart>,
pub tool_name: Option<String>,
pub limit: Option<usize>,
pub min_entry_index: Option<u64>,
pub from: Option<SessionEntryRef>,
pub through: Option<SessionEntryRef>,
pub offset: usize,
}
#[derive(Debug, Clone)]
pub struct SearchHit {
pub id: SessionEntryRef,
pub origin: SessionEntryProvenance,
pub kind: ReferenceKind,
pub tool_part: Option<ToolPart>,
pub tool_name: Option<String>,
pub entry_range: [u64; 2],
pub label: String,
pub summary: String,
}
#[derive(Debug, Clone, Copy)]
pub enum ReadSelector<'a> {
Id(&'a str),
EntryRange([u64; 2]),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReadDetail {
Compact,
Full,
}
#[derive(Debug, Clone)]
pub struct ReadOptions {
pub include_tools: bool,
pub tool_part: ToolPart,
pub detail: ReadDetail,
pub max_items: usize,
pub max_bytes: usize,
}
impl Default for ReadOptions {
fn default() -> Self {
Self {
include_tools: true,
tool_part: ToolPart::Both,
detail: ReadDetail::Compact,
max_items: DEFAULT_READ_MAX_ITEMS,
max_bytes: DEFAULT_READ_MAX_BYTES,
}
}
}
#[derive(Debug, Clone)]
pub struct ReadEntry {
pub id: SessionEntryRef,
pub origin: SessionEntryProvenance,
pub kind: ReferenceKind,
pub tool_part: Option<ToolPart>,
pub tool_name: Option<String>,
pub entry_range: [u64; 2],
pub label: String,
pub text: String,
}
#[derive(Debug, Clone)]
pub struct ReadResult {
pub entries: Vec<ReadEntry>,
pub truncated: bool,
}
#[derive(Debug, Clone)]
pub struct SessionEntryEvidence {
pub segment_id: String,
pub entry_ref: SessionEntryRef,
pub origin: SessionEntryProvenance,
/// Inclusive original-source positions in `segment_id`, not view/UI order.
pub entry_range: [u64; 2],
pub kind: ReferenceKind,
pub tool_part: Option<ToolPart>,
pub label: String,
pub summary: String,
pub excerpt: String,
}
#[derive(Debug, Clone)]
struct CapturedHistoryEntry {
use_policy: Option<protocol::SessionEntryUsePolicy>,
item: Item,
entry_id: crate::LoggedSessionHistoryEntryId,
origin: SessionEntryProvenance,
dto: CaptureEntryDto,
}
#[derive(Debug, Clone)]
pub struct SessionCapture {
session_id: String,
segment_id: String,
capture_id: String,
view_entry_count: u64,
entries: Arc<Vec<CapturedHistoryEntry>>,
overview: Vec<OverviewItem>,
index: Vec<ReferenceEntry>,
}
impl SessionCapture {
pub fn from_session_snapshot(
session_id: impl Into<String>,
segment_id: impl Into<String>,
snapshot: SessionSnapshot,
) -> Self {
let segment_id = segment_id.into();
let view_entry_count = snapshot.entries.len() as u64;
let entries = snapshot
.entries
.into_iter()
.enumerate()
.filter_map(|(index, entry)| {
let policy = CaptureUsePolicy {
recording_material: false,
memory_evidence: entry.use_policy.is_none_or(|p| p.memory_evidence),
observation_readable: entry.use_policy.is_none_or(|p| p.observation_readable),
};
let explicit = entry.use_policy.is_some();
let item = match entry.data {
SessionSnapshotEntryData::UserInput { segments } => {
Item::user_message(protocol::Segment::flatten_to_text(&segments))
}
SessionSnapshotEntryData::Message { role, content } => Item::Message {
id: None,
status: None,
role: match role {
SessionMessageRole::User => Role::User,
SessionMessageRole::Assistant => Role::Assistant,
},
content: content
.into_iter()
.map(|part| match part {
SessionContentPart::Text { text } => {
agen::ContentPart::Text { text }
}
SessionContentPart::Refusal { refusal } => {
agen::ContentPart::Refusal { refusal }
}
})
.collect(),
},
SessionSnapshotEntryData::ToolCall {
call_id,
name,
arguments,
} => Item::tool_call(call_id, name, arguments),
SessionSnapshotEntryData::ToolResult {
call_id,
summary,
content,
is_error,
..
} => Item::tool_result_item(call_id, summary, content, is_error),
SessionSnapshotEntryData::SystemItem { content, .. } if explicit => {
Item::system_message(content)
}
_ => return None,
};
let body = CaptureBody::project(&item, &entry.provenance, explicit)?;
if !policy.any() {
return None;
}
Some(CaptureEntryDto {
entry_id: entry.entry_id,
order: index as u64,
source: CaptureSource {
segment_id: segment_id.clone(),
entry_index: index as u64,
provenance: entry.provenance,
history_origin: None,
derived_from: entry.derived_from,
},
policy,
body,
})
})
.collect();
Self::from_projected(session_id.into(), segment_id, view_entry_count, entries)
}
pub fn new(
session_id: impl Into<String>,
segment_id: impl Into<String>,
items: Vec<Item>,
) -> Self {
let segment_id = segment_id.into();
let view_entry_count = items.len() as u64;
let entries = items
.into_iter()
.enumerate()
.filter_map(|(index, item)| {
let origin = SessionEntryProvenance::LegacyUnknown;
let body = CaptureBody::project(&item, &origin, false)?;
Some(CaptureEntryDto {
entry_id: format!("{index:08}"),
order: index as u64,
source: CaptureSource {
segment_id: segment_id.clone(),
entry_index: index as u64,
provenance: origin,
history_origin: None,
derived_from: Vec::new(),
},
policy: CaptureUsePolicy::PUBLIC,
body,
})
})
.collect();
Self::from_projected(session_id.into(), segment_id, view_entry_count, entries)
}
/// Project a complete Host-fixed history whose source positions start at zero.
/// Select a suffix with `select_order_range` on the resulting value instead
/// of passing a rebased history vector: annotations do not carry source positions.
pub fn from_complete_history(
session_id: impl Into<String>,
segment_id: impl Into<String>,
entries: Vec<HistoryEntry<SessionHistoryMetadata>>,
) -> Self {
let segment_id = segment_id.into();
let view_entry_count = entries.len() as u64;
let entries = entries
.into_iter()
.enumerate()
.filter_map(|(index, entry)| {
let origin = public_provenance(&entry.annotation.origin);
let mut item = entry.item;
let mut explicit = false;
let policy = if let Some(input) = entry.annotation.input_metadata {
explicit = true;
if matches!(
item,
Item::Message {
role: Role::System,
..
}
) {
// Only the explicit public projection crosses this boundary.
let public = input.public_item()?;
item = Item::system_message(public.body.clone());
} else if input.feature_payload().is_some() || input.public_item().is_some() {
return None;
}
CaptureUsePolicy::from(input.usage())
} else {
if matches!(entry.annotation.origin, HistoryOrigin::Provided { .. }) {
return None;
}
CaptureUsePolicy::PUBLIC
};
if !policy.any() {
return None;
}
let body = CaptureBody::project(&item, &origin, explicit)?;
Some(CaptureEntryDto {
entry_id: entry.annotation.entry_id.0,
order: index as u64,
source: CaptureSource {
segment_id: segment_id.clone(),
entry_index: index as u64,
provenance: origin,
history_origin: Some(entry.annotation.origin),
derived_from: entry
.annotation
.derivation
.map(|d| d.sources.into_iter().map(|id| id.0).collect())
.unwrap_or_default(),
},
policy,
body,
})
})
.collect();
Self::from_projected(session_id.into(), segment_id, view_entry_count, entries)
}
fn from_projected(
session_id: String,
segment_id: String,
view_entry_count: u64,
dtos: Vec<CaptureEntryDto>,
) -> Self {
let dto = CaptureDto {
version: CAPTURE_VERSION,
session_id: session_id.clone(),
segment_id: segment_id.clone(),
view_entry_count,
entries: dtos,
};
let capture_id = dto.fingerprint();
let entries: Vec<_> = dto
.entries
.into_iter()
.map(|dto| CapturedHistoryEntry {
use_policy: Some(protocol::SessionEntryUsePolicy {
observation_readable: dto.policy.observation_readable,
memory_evidence: dto.policy.memory_evidence,
}),
item: dto.body.to_item(),
entry_id: crate::LoggedSessionHistoryEntryId(dto.entry_id.clone()),
origin: dto.source.provenance.clone(),
dto,
})
.collect();
let mut overview = Vec::new();
let mut index = Vec::new();
for entry in &entries {
let item = &entry.item;
let entry_range = [entry.dto.order; 2];
let attachment_count = match entry.dto.body {
CaptureBody::ToolResult {
attachment_count, ..
} => attachment_count,
_ => 0,
};
match item {
Item::Message { role, content, .. } => {
let Some(kind) = message_reference_kind(&entry.origin, role, entry.use_policy)
else {
continue;
};
let text = content
.iter()
.map(|p| p.as_text())
.collect::<Vec<_>>()
.join("");
let label = format!("{} message", kind.as_str());
let summary = truncate_chars(&text, 240);
let id = SessionEntryRef::from_history_entry_id(&entry.entry_id);
index.push(ReferenceEntry {
use_policy: entry.use_policy,
attachment_count,
id: id.clone(),
origin: entry.origin.clone(),
entry_range,
kind,
tool_part: None,
tool_name: None,
label: label.clone(),
summary: summary.clone(),
search_text: text.clone(),
});
if entry.use_policy.is_none_or(|p| p.observation_readable)
&& matches!(kind, ReferenceKind::User | ReferenceKind::Assistant)
{
overview.push(OverviewItem {
id: id.clone(),
origin: entry.origin.clone(),
entry_range,
kind,
label,
text,
intervening_entries: 0,
});
}
}
Item::ToolCall {
name, arguments, ..
} => {
let text = format!("{name}\n{arguments}");
index.push(ReferenceEntry {
use_policy: entry.use_policy,
attachment_count,
id: SessionEntryRef::from_history_entry_id(&entry.entry_id),
origin: entry.origin.clone(),
entry_range,
kind: ReferenceKind::Tool,
tool_part: Some(ToolPart::Input),
tool_name: Some(name.clone()),
label: format!("tool input: {name}"),
summary: format!("Tool call {name}"),
search_text: text,
});
}
Item::ToolResult {
summary,
content,
attachments: _,
..
} => {
let attachment_marker = if attachment_count == 0 {
String::new()
} else {
format!("\n[{} image attachment(s)]", attachment_count)
};
let text = format!(
"{summary}\n{}{attachment_marker}",
content.as_deref().unwrap_or_default(),
);
index.push(ReferenceEntry {
use_policy: entry.use_policy,
attachment_count,
id: SessionEntryRef::from_history_entry_id(&entry.entry_id),
origin: entry.origin.clone(),
entry_range,
kind: ReferenceKind::Tool,
tool_part: Some(ToolPart::Output),
tool_name: None,
label: "tool output".to_string(),
summary: truncate_chars(summary, 240),
search_text: text,
});
}
Item::AssistantResponseBoundary { .. } | Item::Reasoning { .. } => {}
}
}
if overview.len() > 2 {
let last = overview.len() - 1;
overview = overview
.into_iter()
.enumerate()
.filter_map(|(index, entry)| {
(index == 0 || index == last || index % OVERVIEW_ANCHOR_STRIDE == 0)
.then_some(entry)
})
.collect();
}
for overview_index in 0..overview.len().saturating_sub(1) {
let current_entry = overview[overview_index].entry_range[0];
let next_entry = overview[overview_index + 1].entry_range[0];
overview[overview_index].intervening_entries = index
.iter()
.filter(|entry| {
let entry_index = entry.entry_range[0];
entry_index > current_entry && entry_index < next_entry
})
.count();
}
Self {
session_id,
segment_id,
capture_id,
view_entry_count,
entries: Arc::new(entries),
overview,
index,
}
}
/// Public source entries are immutable; callers retain their own business decisions.
pub fn entries(&self) -> impl ExactSizeIterator<Item = &CaptureEntryDto> {
self.entries.iter().map(|entry| &entry.dto)
}
pub fn overview(&self) -> &[OverviewItem] {
&self.overview
}
/// Capture-local UI position, not the original source position used by evidence.
pub fn view_order_for_ref(&self, reference: &SessionEntryRef) -> Option<u64> {
self.index
.iter()
.find(|entry| entry.id == *reference)
.map(|entry| entry.entry_range[0])
}
pub fn search(&self, options: &SearchOptions) -> Vec<SearchHit> {
if self.validate_search(options).is_err() {
return Vec::new();
}
let query = options.query.trim().to_lowercase();
let limit = options
.limit
.unwrap_or(DEFAULT_SEARCH_LIMIT)
.clamp(1, MAX_SEARCH_LIMIT);
let tool_name = options.tool_name.as_deref();
let min_entry_index = options
.from
.as_ref()
.and_then(|reference| self.view_order_for_ref(reference))
.unwrap_or_else(|| options.min_entry_index.unwrap_or(0));
let max_entry_index = options
.through
.as_ref()
.and_then(|reference| self.view_order_for_ref(reference))
.unwrap_or(u64::MAX);
let mut skipped = 0usize;
let mut hits = Vec::new();
for entry in &self.index {
if entry.use_policy.is_some_and(|p| !p.observation_readable) {
continue;
}
if entry.entry_range[0] < min_entry_index || entry.entry_range[0] > max_entry_index {
continue;
}
if let Some(kind) = options.kind {
if entry.kind != kind {
continue;
}
}
if let Some(part) = options.tool_part {
if entry.kind != ReferenceKind::Tool {
continue;
}
let Some(actual) = entry.tool_part else {
continue;
};
if !part.matches(actual) {
continue;
}
}
if let Some(name) = tool_name {
if entry.tool_name.as_deref() != Some(name) {
continue;
}
}
if !query.is_empty() && !entry.search_text.to_lowercase().contains(&query) {
continue;
}
if skipped < options.offset {
skipped += 1;
continue;
}
hits.push(SearchHit {
id: entry.id.clone(),
origin: entry.origin.clone(),
kind: entry.kind,
tool_part: entry.tool_part,
tool_name: entry.tool_name.clone(),
entry_range: entry.entry_range,
label: entry.label.clone(),
summary: entry.summary.clone(),
});
if hits.len() >= limit {
break;
}
}
hits
}
pub fn read(&self, selector: ReadSelector<'_>, options: ReadOptions) -> ReadResult {
let max_items = options.max_items.clamp(1, MAX_READ_MAX_ITEMS);
let max_bytes = options.max_bytes.max(1);
let mut entries = Vec::new();
let mut bytes = 0usize;
let mut truncated = false;
let selected: Vec<&ReferenceEntry> = match selector {
ReadSelector::Id(id) => self
.index
.iter()
.filter(|entry| entry.id.as_str() == id)
.collect(),
ReadSelector::EntryRange([start, end]) => self
.index
.iter()
.filter(|entry| entry.entry_range[0] >= start && entry.entry_range[0] <= end)
.collect(),
};
for entry in selected {
if entry.use_policy.is_some_and(|p| !p.observation_readable) {
continue;
}
if entries.len() >= max_items {
truncated = true;
break;
}
if entry.kind == ReferenceKind::Tool {
if !options.include_tools {
continue;
}
if let Some(actual) = entry.tool_part {
if !options.tool_part.matches(actual) {
continue;
}
}
}
let Some(item) = self
.entries
.iter()
.find(|captured| captured.dto.order == entry.entry_range[0])
.map(|entry| &entry.item)
else {
continue;
};
let remaining = max_bytes.saturating_sub(bytes);
if remaining == 0 {
truncated = true;
break;
}
let rendered = render_item(item, entry, options.detail, usize::MAX);
truncated |= rendered.len() > remaining;
let text = truncate_bytes(&rendered, remaining);
bytes = bytes.saturating_add(text.len());
entries.push(ReadEntry {
id: entry.id.clone(),
origin: entry.origin.clone(),
kind: entry.kind,
tool_part: entry.tool_part,
tool_name: entry.tool_name.clone(),
entry_range: entry.entry_range,
label: entry.label.clone(),
text,
});
if truncated {
break;
}
}
ReadResult { entries, truncated }
}
/// Resolves the committed tool-call input for a provider call id. This is
/// used by explicit-memory receipts only after the containing run is durable,
/// so a model-authored claim can cite its own committed tool invocation
/// without inventing a SessionEntryRef.
pub fn evidence_for_tool_call(&self, call_id: &str) -> Option<SessionEntryEvidence> {
let captured = self.entries.iter().find(|entry| {
matches!(
&entry.item,
Item::ToolCall {
call_id: stored_call_id,
..
} if stored_call_id == call_id
)
})?;
let reference = self.index.iter().find(|entry| {
entry.entry_range[0] == captured.dto.order
&& entry.kind == ReferenceKind::Tool
&& entry.tool_part == Some(ToolPart::Input)
})?;
self.evidence_for(reference.id.as_str())
}
pub fn evidence_for(&self, id: &str) -> Option<SessionEntryEvidence> {
let entry = self.index.iter().find(|entry| entry.id.as_str() == id)?;
if entry.use_policy.is_some_and(|p| !p.memory_evidence) {
return None;
}
let captured = self
.entries
.iter()
.find(|captured| captured.dto.order == entry.entry_range[0])?;
let excerpt = render_item(&captured.item, entry, ReadDetail::Compact, 2 * 1024);
Some(SessionEntryEvidence {
segment_id: captured.dto.source.segment_id.clone(),
entry_ref: entry.id.clone(),
origin: entry.origin.clone(),
entry_range: [captured.dto.source.entry_index; 2],
kind: entry.kind,
tool_part: entry.tool_part,
label: entry.label.clone(),
summary: entry.summary.clone(),
excerpt,
})
}
}
pub(crate) fn public_provenance(origin: &HistoryOrigin) -> SessionEntryProvenance {
match origin {
HistoryOrigin::HumanInput { .. } | HistoryOrigin::LocalHumanInput => {
SessionEntryProvenance::HumanInput
}
HistoryOrigin::WorkerInput { .. } => SessionEntryProvenance::WorkerInput,
HistoryOrigin::FlowInstruction { .. } => SessionEntryProvenance::FlowInstruction,
HistoryOrigin::BackendInstruction { .. } | HistoryOrigin::Provided { .. } => {
SessionEntryProvenance::BackendInstruction
}
HistoryOrigin::ModelOutput { .. } => SessionEntryProvenance::ModelOutput,
HistoryOrigin::ToolOutput { .. } => SessionEntryProvenance::ToolOutput,
HistoryOrigin::CompactionSummary
| HistoryOrigin::DerivedSummary
| HistoryOrigin::JournalContext => SessionEntryProvenance::DerivedSummary,
HistoryOrigin::LegacyUnknown => SessionEntryProvenance::LegacyUnknown,
}
}
fn message_reference_kind(
origin: &SessionEntryProvenance,
provider_role: &Role,
use_policy: Option<protocol::SessionEntryUsePolicy>,
) -> Option<ReferenceKind> {
if use_policy.is_some() {
return match provider_role {
Role::System => Some(ReferenceKind::System),
Role::User => Some(ReferenceKind::User),
Role::Assistant => Some(ReferenceKind::Assistant),
};
}
match origin {
SessionEntryProvenance::HumanInput | SessionEntryProvenance::WorkerInput => {
Some(ReferenceKind::User)
}
SessionEntryProvenance::ModelOutput => Some(ReferenceKind::Assistant),
SessionEntryProvenance::ToolOutput => Some(ReferenceKind::Tool),
SessionEntryProvenance::LegacyUnknown => match provider_role {
Role::User => Some(ReferenceKind::User),
Role::Assistant => Some(ReferenceKind::Assistant),
Role::System => None,
},
// Flow/backend/system content remains out of the observation surface
// even when represented with a provider user/system role.
SessionEntryProvenance::FlowInstruction
| SessionEntryProvenance::BackendInstruction
| SessionEntryProvenance::DerivedSummary => None,
}
}
fn render_item(
item: &Item,
entry: &ReferenceEntry,
detail: ReadDetail,
max_bytes: usize,
) -> String {
let text = match item {
Item::Message { role, content, .. } => {
let text = content
.iter()
.map(|p| p.as_text())
.collect::<Vec<_>>()
.join("");
format!("[{} {:?}] {text}", entry.id, role)
}
Item::ToolCall {
name, arguments, ..
} => match detail {
ReadDetail::Compact => format!("[{} ToolInput {name}] (arguments omitted)", entry.id),
ReadDetail::Full => format!("[{} ToolInput {name}]\narguments: {arguments}", entry.id),
},
Item::ToolResult {
summary,
content,
attachments: _,
is_error,
..
} => {
let attachment_line = if entry.attachment_count == 0 {
String::new()
} else {
format!("\nattachments: {} image(s)", entry.attachment_count)
};
match detail {
ReadDetail::Compact => format!(
"[{} ToolOutput{}]\nsummary: {summary}\ncontent: (omitted){attachment_line}",
entry.id,
if *is_error { " error" } else { "" },
),
ReadDetail::Full => format!(
"[{} ToolOutput{}]\nsummary: {summary}\ncontent: {}{attachment_line}",
entry.id,
if *is_error { " error" } else { "" },
content.as_deref().unwrap_or_default(),
),
}
}
Item::AssistantResponseBoundary { .. } => {
format!("[{} Assistant response boundary omitted]", entry.id)
}
Item::Reasoning { .. } => format!("[{} Reasoning omitted]", entry.id),
};
truncate_bytes(&text, max_bytes)
}
fn truncate_chars(text: &str, max_chars: usize) -> String {
if max_chars == 0 {
return "… [truncated]".to_string();
}
if text.chars().count() <= max_chars {
return text.to_string();
}
let mut out = text.chars().take(max_chars).collect::<String>();
out.push_str("… [truncated]");
out
}
fn truncate_bytes(text: &str, max_bytes: usize) -> String {
if text.len() <= max_bytes {
return text.to_owned();
}
let marker = "… [truncated]";
let mut end = max_bytes.saturating_sub(marker.len());
while !text.is_char_boundary(end) {
end -= 1;
}
let mut out = text[..end].to_owned();
if max_bytes >= marker.len() {
out.push_str(marker);
}
out
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,425 @@
//! Host capture and pure canonical slicing. No Job state or live-reader fallback lives here.
use super::*;
use crate::committed_replay::CommittedPublicReplay;
use crate::public_material::{
PublicMaterialDescriptor, PublicMaterialError, PublicMaterialLimits, PublicMaterialUse,
};
use crate::retention::SessionRetentionStore;
use crate::{SegmentId, SessionId};
use std::path::Path;
/// Continuation in canonical Journal order, not an observation/Memory view cursor.
/// A Host must reconcile the descriptor at this position with its saved checkpoint.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RecordingPosition {
pub canonical_order: u64,
pub byte_offset: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RecordingCaptureSlice {
pub capture: CaptureDto,
pub next: Option<RecordingPosition>,
}
/// Inspect recording admission descriptors without constructing a body-bearing DTO.
///
/// The Host must authorize the source and hold the **same** Session publication/erasure
/// guard from before reading its persisted active pointer until this function returns
/// (including the callback). Pass that pointer's Segment, not a filename/mtime-derived
/// selector. This function verifies the guard and stored Session identities but does
/// not authorize the caller or establish that the supplied guard fences this root.
/// No lock is recursively acquired, no active pointer is reread, and no replay handle
/// or body-read capability is exposed to `inspect`.
///
/// The common committed recording replay fixes per-file ends before decoding. It
/// retains only descriptors/locations, decoding one record at a time and streaming
/// canonical bodies into their digests. `replay_limits` bound record allocation,
/// aggregate storage extent, entries and Segments; there is no full-capture text/JSON
/// allocation or `CaptureLimits` ceiling. The callback receives only descriptors in
/// canonical order and may return a Host-specific mapping (including a `Result`). It
/// runs once on success and not at all on replay/identity/resource errors.
///
/// This catalog is for Host admission, **not** a descriptor-only Job input. Construct
/// the Job's bounded body input with the guarded slice API at its own authorized
/// boundary; after saving it, execute only that immutable value. Neither descriptors
/// nor a completed callback authorize delayed adoption after erasure.
///
/// ```no_run
/// # use std::path::Path;
/// # use session_store::{SessionId, SegmentId};
/// # use session_store::retention::SessionRetentionGuard;
/// # use session_store::public_material::PublicMaterialError;
/// # fn admit(root: &Path, session: SessionId, active: SegmentId,
/// # guard: &SessionRetentionGuard) -> Result<Vec<u64>, PublicMaterialError> {
/// // `active` was resolved from persisted Host metadata while holding `guard`.
/// session_store::capture::recording_catalog_worker_guarded(
/// root, session, active, Default::default(), guard,
/// |descriptors| descriptors.iter().map(|d| d.total_bytes).collect(),
/// )
/// # }
/// ```
pub fn recording_catalog_worker_guarded<T>(
session_root: &Path,
expected_session: SessionId,
active_segment: SegmentId,
replay_limits: PublicMaterialLimits,
guard: &crate::retention::SessionRetentionGuard,
inspect: impl FnOnce(&[PublicMaterialDescriptor]) -> T,
) -> Result<T, PublicMaterialError> {
if guard.session_id() != expected_session {
return Err(PublicMaterialError::Inconsistent);
}
let replay = CommittedPublicReplay::open_worker_guarded(
session_root,
active_segment,
replay_limits,
None,
PublicMaterialUse::Recording,
)?;
if replay.session_id() != expected_session {
return Err(PublicMaterialError::Inconsistent);
}
Ok(inspect(&replay.into_recording_descriptors()))
}
/// FsStore-layout counterpart of [`recording_catalog_worker_guarded`].
///
/// `sessions_root` is the directory containing per-Session directories. Invoke this
/// inside the Host's `with_recording_boundary` callback, passing **its existing guard
/// and persisted active Segment**. That same external retention control store must
/// fence publishers, pointer changes, archive moves and erasure. Do not reacquire the
/// guard or use a separately opened legacy public reader inside the boundary.
/// All descriptor-only, allocation and callback guarantees of the Worker variant
/// apply; this function does not construct an unbounded transient `CaptureDto`.
pub fn recording_catalog_fs_guarded<T>(
sessions_root: &Path,
session: SessionId,
active_segment: SegmentId,
replay_limits: PublicMaterialLimits,
guard: &crate::retention::SessionRetentionGuard,
inspect: impl FnOnce(&[PublicMaterialDescriptor]) -> T,
) -> Result<T, PublicMaterialError> {
if guard.session_id() != session {
return Err(PublicMaterialError::Inconsistent);
}
let replay = CommittedPublicReplay::open_fs_guarded(
sessions_root,
session,
active_segment,
replay_limits,
)?;
Ok(inspect(&replay.into_recording_descriptors()))
}
/// Capture the selected original lineage, including typed Host run boundaries.
/// The caller must hold the Session publication/erasure guard while selecting the
/// active pointer and through this call. Runtime publishers use this same lock
/// for appends and Segment creation. No lock is recursively acquired here.
pub fn capture_recording_worker_guarded(
session_root: &Path,
expected_session: SessionId,
active_segment: SegmentId,
replay_limits: PublicMaterialLimits,
capture_limits: CaptureLimits,
guard: &crate::retention::SessionRetentionGuard,
) -> Result<CaptureDto, PublicMaterialError> {
if guard.session_id() != expected_session {
return Err(PublicMaterialError::Inconsistent);
}
let replay = CommittedPublicReplay::open_worker_guarded(
session_root,
active_segment,
replay_limits,
None,
PublicMaterialUse::Recording,
)?;
if replay.session_id() != expected_session {
return Err(PublicMaterialError::Inconsistent);
}
replay.into_recording_dto(capture_limits)
}
/// Standalone convenience entry; Hosts resolving a mutable active pointer should
/// instead hold the fence while resolving it and call the guarded entry.
pub fn capture_recording_worker(
session_root: &Path,
expected_session: SessionId,
active_segment: SegmentId,
retention: &SessionRetentionStore,
replay_limits: PublicMaterialLimits,
capture_limits: CaptureLimits,
) -> Result<CaptureDto, PublicMaterialError> {
let guard = retention
.lock_live(expected_session)
.map_err(|error| match error {
crate::StoreError::SessionErased(_) => PublicMaterialError::Erased,
_ => PublicMaterialError::Unavailable,
})?;
capture_recording_worker_guarded(
session_root,
expected_session,
active_segment,
replay_limits,
capture_limits,
&guard,
)
}
/// Bounded construction for one Job input: descriptor replay is bounded separately;
/// only at most the requested body budget is materialized from the fixed files.
pub fn capture_recording_slice_worker_guarded(
session_root: &Path,
expected_session: SessionId,
active_segment: SegmentId,
replay_limits: PublicMaterialLimits,
start: RecordingPosition,
max_entries: usize,
max_serialized_bytes: usize,
guard: &crate::retention::SessionRetentionGuard,
) -> Result<RecordingCaptureSlice, PublicMaterialError> {
if guard.session_id() != expected_session {
return Err(PublicMaterialError::Inconsistent);
}
let replay = CommittedPublicReplay::open_worker_guarded(
session_root,
active_segment,
replay_limits,
None,
PublicMaterialUse::Recording,
)?;
if replay.session_id() != expected_session {
return Err(PublicMaterialError::Inconsistent);
}
replay.recording_slice(start, max_entries, max_serialized_bytes)
}
/// The standalone FsStore layout uses the same committed replay and fixed values.
pub fn capture_recording_fs_guarded(
sessions_root: &Path,
session: SessionId,
active_segment: SegmentId,
replay_limits: PublicMaterialLimits,
capture_limits: CaptureLimits,
guard: &crate::retention::SessionRetentionGuard,
) -> Result<CaptureDto, PublicMaterialError> {
if guard.session_id() != session {
return Err(PublicMaterialError::Inconsistent);
}
CommittedPublicReplay::open_fs_guarded(sessions_root, session, active_segment, replay_limits)?
.into_recording_dto(capture_limits)
}
/// Bounded standalone capture; the caller holds its FsStore retention fence.
pub fn capture_recording_slice_fs_guarded(
sessions_root: &Path,
session: SessionId,
active_segment: SegmentId,
replay_limits: PublicMaterialLimits,
start: RecordingPosition,
max_entries: usize,
max_serialized_bytes: usize,
guard: &crate::retention::SessionRetentionGuard,
) -> Result<RecordingCaptureSlice, PublicMaterialError> {
if guard.session_id() != session {
return Err(PublicMaterialError::Inconsistent);
}
CommittedPublicReplay::open_fs_guarded(sessions_root, session, active_segment, replay_limits)?
.recording_slice(start, max_entries, max_serialized_bytes)
}
impl CaptureDto {
/// Select the oldest canonical prefix fitting both budgets, including JSON
/// escaping, metadata and continuation. Only immutable owned bytes are read.
/// A later Job can recapture at `next`; never combine live pages into this value.
pub fn recording_slice(
&self,
start: RecordingPosition,
max_entries: usize,
max_serialized_bytes: usize,
) -> Result<RecordingCaptureSlice, CaptureError> {
if max_entries == 0 {
return Err(CaptureError::BudgetTooSmall);
}
if start.canonical_order > self.view_entry_count {
return Err(CaptureError::InvalidRange);
}
let mut result = RecordingCaptureSlice {
capture: CaptureDto {
version: self.version,
session_id: self.session_id.clone(),
segment_id: self.segment_id.clone(),
view_entry_count: self.view_entry_count,
entries: Vec::new(),
},
next: (start.canonical_order < self.view_entry_count).then_some(start),
};
if body::json_size(&result, max_serialized_bytes).is_err() {
return Err(CaptureError::BudgetTooSmall);
}
if start.canonical_order == self.view_entry_count {
if start.byte_offset != 0 {
return Err(CaptureError::InvalidRange);
}
result.capture.validate(CaptureLimits {
max_bytes: max_serialized_bytes,
max_text_bytes: max_serialized_bytes,
max_entries,
..Default::default()
})?;
return Ok(result);
}
let first = self
.entries
.iter()
.position(|entry| entry.order == start.canonical_order)
.ok_or(CaptureError::InvalidRange)?;
let mut expected_order = start.canonical_order;
for entry in self.entries.iter().skip(first).take(max_entries) {
if entry.order != expected_order {
return Err(CaptureError::InvalidRange);
}
let CaptureBody::CanonicalPublic {
descriptor,
byte_offset,
text,
boundary,
} = &entry.body
else {
return Err(CaptureError::Invalid("not a recording capture"));
};
if !entry.policy.recording_material {
return Err(CaptureError::PolicyDenied);
}
let offset = if entry.order == start.canonical_order {
start.byte_offset
} else {
*byte_offset
};
let local = offset
.checked_sub(*byte_offset)
.and_then(|o| usize::try_from(o).ok())
.ok_or(CaptureError::InvalidRange)?;
if local > text.len()
|| !text.is_char_boundary(local)
|| offset >= descriptor.total_bytes
{
return Err(CaptureError::InvalidRange);
}
let item = CaptureEntryDto {
entry_id: entry.entry_id.clone(),
order: entry.order,
source: entry.source.clone(),
policy: entry.policy,
body: CaptureBody::CanonicalPublic {
descriptor: descriptor.clone(),
byte_offset: offset,
text: String::new(),
boundary: *boundary,
},
};
result.capture.entries.push(item);
// At most budget bytes are considered before allocating a trial fragment.
let mut low = 0;
let mut high = text.len().saturating_sub(local).min(max_serialized_bytes);
let prior_next = result.next;
while low < high {
let mid = low + (high - low).div_ceil(2);
let mut end = local + mid;
while !text.is_char_boundary(end) {
end -= 1;
}
if let CaptureBody::CanonicalPublic { text: fragment, .. } =
&mut result.capture.entries.last_mut().unwrap().body
{
*fragment = text[local..end].to_owned();
}
let next_offset = offset + (end - local) as u64;
result.next = if next_offset < descriptor.total_bytes {
Some(RecordingPosition {
canonical_order: entry.order,
byte_offset: next_offset,
})
} else {
(entry.order + 1 < self.view_entry_count).then_some(RecordingPosition {
canonical_order: entry.order + 1,
byte_offset: 0,
})
};
if body::json_size(&result, max_serialized_bytes).is_ok() {
low = mid;
} else {
high = mid - 1;
}
}
let mut end = local + low;
while !text.is_char_boundary(end) {
end -= 1;
}
if end == local {
result.capture.entries.pop();
result.next = prior_next;
if result.capture.entries.is_empty() {
return Err(CaptureError::BudgetTooSmall);
}
break;
}
if let CaptureBody::CanonicalPublic { text: fragment, .. } =
&mut result.capture.entries.last_mut().unwrap().body
{
*fragment = text[local..end].to_owned();
}
let next_offset = offset + (end - local) as u64;
result.next = if next_offset < descriptor.total_bytes {
Some(RecordingPosition {
canonical_order: entry.order,
byte_offset: next_offset,
})
} else {
(entry.order + 1 < self.view_entry_count).then_some(RecordingPosition {
canonical_order: entry.order + 1,
byte_offset: 0,
})
};
if next_offset < descriptor.total_bytes {
break;
}
expected_order = entry.order + 1;
}
body::json_size(&result, max_serialized_bytes).map_err(|_| CaptureError::BudgetTooSmall)?;
result.capture.validate(CaptureLimits {
max_bytes: max_serialized_bytes,
max_text_bytes: max_serialized_bytes,
max_entries,
..Default::default()
})?;
Ok(result)
}
/// Resolve canonical text already in the value. No original storage is consulted.
pub fn recording_text(
&self,
order: u64,
) -> Result<(&crate::public_material::PublicMaterialDescriptor, u64, &str), CaptureError> {
let entry = self
.entries
.iter()
.find(|entry| entry.order == order)
.ok_or(CaptureError::UnknownReference)?;
if !entry.policy.recording_material {
return Err(CaptureError::PolicyDenied);
}
match &entry.body {
CaptureBody::CanonicalPublic {
descriptor,
byte_offset,
text,
..
} => Ok((descriptor, *byte_offset, text)),
_ => Err(CaptureError::Invalid("not canonical recording text")),
}
}
}
File diff suppressed because it is too large Load Diff
+2
View File
@@ -38,6 +38,8 @@ pub(crate) fn read_without_atime_bounded(path: &Path, max_bytes: u64) -> io::Res
}
}
pub mod capture;
mod committed_replay;
pub mod event_trace;
pub mod fs_store;
pub mod history;
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+11 -1
View File
@@ -46,6 +46,13 @@ pub struct SessionRetentionStore {
/// Do not recursively acquire the same Session lock inside a guarded callback.
pub struct SessionRetentionGuard {
_file: File,
session_id: SessionId,
}
impl SessionRetentionGuard {
pub fn session_id(&self) -> SessionId {
self.session_id
}
}
impl SessionRetentionStore {
@@ -71,7 +78,10 @@ impl SessionRetentionStore {
}
let file = options.open(self.root.join(format!("{session}.lock")))?;
FileExt::lock_exclusive(&file)?;
Ok(SessionRetentionGuard { _file: file })
Ok(SessionRetentionGuard {
_file: file,
session_id: session,
})
}
fn read(&self, session: SessionId) -> Result<RetentionState, StoreError> {
@@ -194,6 +194,21 @@ impl WorkerSessionStore {
Ok(store)
}
/// Open a current retained Session without creating or migrating anything.
/// The live constraint is checked here and again by every later publication;
/// attaching the control store itself does not acquire publication authority.
pub fn open_existing_with_retention(
root: impl Into<PathBuf>,
retention: crate::retention::SessionRetentionStore,
) -> Result<Self, StoreError> {
let mut store = Self::open_read_only(root)?;
if let Some(session) = *store.session_id.lock().unwrap() {
retention.ensure_live(session)?;
}
store.retention = Some(retention);
Ok(store)
}
fn lock_publication(
&self,
session: SessionId,
@@ -0,0 +1,711 @@
//! Public consumer boundary: real capture adapters, checked DTO codec and pure readers.
//! No Worker, persistence, live Session provider, or substituted capture implementation.
use agen::{HistoryEntry, Item};
use protocol::{
SessionEntryProvenance, SessionSnapshot, SessionSnapshotEntry, SessionSnapshotEntryData,
};
use session_store::capture::*;
use session_store::{
FeatureInputPayload, HistoryInputMetadata, HistoryInputUsePolicy, InputCompactionUse,
InputEvidenceUse, InputObservationUse, LoggedHistoryDerivation, LoggedSessionHistoryEntryId,
LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, PublicSystemItem,
PublicSystemPresentation,
};
fn history(
id: &str,
item: Item,
input_metadata: Option<HistoryInputMetadata>,
) -> HistoryEntry<LoggedSessionHistoryMetadata> {
HistoryEntry::new(
item,
LoggedSessionHistoryMetadata {
entry_id: LoggedSessionHistoryEntryId(id.into()),
origin: LoggedSessionHistoryOrigin::LocalHumanInput,
derivation: Some(LoggedHistoryDerivation {
sources: vec![LoggedSessionHistoryEntryId("earlier-entry".into())],
}),
input_metadata,
},
)
}
fn usage(recording: bool, memory: bool, observation: bool) -> HistoryInputUsePolicy {
HistoryInputUsePolicy {
recording_material: if recording {
InputEvidenceUse::Include
} else {
InputEvidenceUse::Exclude
},
memory_evidence: if memory {
InputEvidenceUse::Include
} else {
InputEvidenceUse::Exclude
},
observation: if observation {
InputObservationUse::Readable
} else {
InputObservationUse::Hidden
},
compaction: InputCompactionUse::Retain,
}
}
fn reference(capture: &SessionCapture, id: &str) -> ScopedSessionEntryRef {
capture
.scoped_ref(&SessionEntryRef::parse(id).unwrap())
.unwrap()
}
#[test]
fn public_consumer_roundtrip_retains_body_original_source_order_origin_and_independent_policy() {
let entries = vec![
history(
"recording",
Item::user_message("record-only"),
Some(HistoryInputMetadata::new(usage(true, false, false), None, None).unwrap()),
),
history(
"memory",
Item::user_message("memory-only"),
Some(HistoryInputMetadata::new(usage(false, true, false), None, None).unwrap()),
),
history(
"observation",
Item::user_message("observation-only"),
Some(HistoryInputMetadata::new(usage(false, false, true), None, None).unwrap()),
),
];
let capture = SessionCapture::from_complete_history("synthetic-session", "selected", entries);
let mut dto = capture.to_dto();
// The checked boundary also accepts entries retained from different original segments.
dto.entries[0].source.segment_id = "original-a".into();
dto.entries[0].source.entry_index = 80;
dto.entries[1].source.segment_id = "original-b".into();
dto.entries[1].source.entry_index = 3;
let capture = SessionCapture::try_from_dto(dto.clone(), CaptureLimits::default()).unwrap();
let encoded = capture.encode(CaptureLimits::default()).unwrap();
let restored = SessionCapture::decode(&encoded, CaptureLimits::default()).unwrap();
assert_eq!(restored.to_dto(), dto);
assert_eq!(restored.capture_id(), capture.capture_id());
for (id, allowed) in [
("Erecording", CapturePurpose::Recording),
("Ememory", CapturePurpose::MemoryEvidence),
("Eobservation", CapturePurpose::Observation),
] {
let reference = reference(&capture, id);
assert_eq!(
restored.resolve_ref(&reference, allowed).unwrap(),
capture.resolve_ref(&reference, allowed).unwrap()
);
for purpose in [
CapturePurpose::Recording,
CapturePurpose::MemoryEvidence,
CapturePurpose::Observation,
] {
assert_eq!(
restored.resolve_ref(&reference, purpose).is_ok(),
purpose == allowed
);
}
}
assert_eq!(restored.overview().len(), 1);
assert_eq!(restored.search(&SearchOptions::default()).len(), 1);
assert!(restored.evidence_for("Erecording").is_none());
let evidence = restored.evidence_for("Ememory").unwrap();
assert_eq!(evidence.segment_id, "original-b");
assert_eq!(evidence.entry_range, [3, 3]);
assert!(evidence.excerpt.contains("memory-only"));
let recorded = restored
.slice(CapturePurpose::Recording, None, 10, 16_000)
.unwrap();
assert_eq!(recorded.slice().items.len(), 1);
assert_eq!(recorded.slice().items[0].text, "record-only");
for (purpose, expected) in [
(CapturePurpose::MemoryEvidence, "memory-only"),
(CapturePurpose::Observation, "observation-only"),
] {
let slice = restored.slice(purpose, None, 10, 16_000).unwrap();
assert_eq!(slice.slice().items.len(), 1);
assert_eq!(slice.slice().items[0].text, expected);
assert!(
restored
.read_raw(
&reference(&restored, "Erecording"),
purpose,
RawField::MessagePart(0),
0,
100
)
.is_err()
);
}
}
#[test]
fn common_data_drops_reasoning_raw_system_and_private_payload_even_when_unindexed() {
let payload = FeatureInputPayload::encode(
"capture-test",
1,
&serde_json::json!({"secret": "PRIVATE-PAYLOAD"}),
)
.unwrap();
let public = PublicSystemItem {
kind: "notice".into(),
body: "public notice".into(),
presentation: PublicSystemPresentation::Ignored,
};
let system =
HistoryInputMetadata::new(usage(true, true, true), Some(payload.clone()), Some(public))
.unwrap();
let private = HistoryInputMetadata::new(usage(true, true, true), Some(payload), None).unwrap();
let capture = SessionCapture::from_complete_history(
"s",
"segment",
vec![
history("raw", Item::system_message("RAW-SYSTEM-PROMPT"), None),
history("reasoning", Item::reasoning("HIDDEN-REASONING"), None),
history(
"private",
Item::system_message("PRIVATE-SYSTEM-BODY"),
Some(private),
),
history(
"public",
Item::system_message("MODEL-SYSTEM-BODY"),
Some(system),
),
history("ordinary", Item::user_message("ordinary"), None),
],
);
let encoded = String::from_utf8(capture.encode(CaptureLimits::default()).unwrap()).unwrap();
let retained_debug = format!("{capture:?}");
for secret in [
"RAW-SYSTEM-PROMPT",
"HIDDEN-REASONING",
"PRIVATE-SYSTEM-BODY",
"MODEL-SYSTEM-BODY",
"PRIVATE-PAYLOAD",
] {
assert!(!encoded.contains(secret));
assert!(!retained_debug.contains(secret));
}
assert_eq!(capture.to_dto().entries.len(), 2);
assert!(encoded.contains("public notice"));
// Filtered entries do not renumber the public source positions.
assert_eq!(capture.to_dto().entries[0].order, 3);
assert_eq!(
capture.view_order_for_ref(&SessionEntryRef::parse("Eordinary").unwrap()),
Some(4)
);
}
#[test]
fn protocol_projection_does_not_invent_recording_permission() {
let capture = SessionCapture::from_session_snapshot(
"s",
"seg",
SessionSnapshot {
pending_submissions: Default::default(),
entries: vec![SessionSnapshotEntry {
entry_id: "projected".into(),
timestamp: 0,
provenance: SessionEntryProvenance::HumanInput,
derived_from: vec!["source".into()],
use_policy: Some(protocol::SessionEntryUsePolicy {
observation_readable: true,
memory_evidence: false,
}),
data: SessionSnapshotEntryData::Message {
role: protocol::SessionMessageRole::User,
content: vec![protocol::SessionContentPart::Text {
text: "projected".into(),
}],
},
}],
},
);
let reference = reference(&capture, "Eprojected");
assert!(matches!(
capture.resolve_ref(&reference, CapturePurpose::Recording),
Err(CaptureError::PolicyDenied)
));
assert!(
capture
.resolve_ref(&reference, CapturePurpose::Observation)
.is_ok()
);
assert!(capture.evidence_for("Eprojected").is_none());
}
#[test]
fn malformed_dtos_and_oversized_representations_are_rejected() {
let capture = SessionCapture::new("s", "seg", vec![Item::user_message("body")]);
let original = serde_json::to_value(capture.to_dto()).unwrap();
let mut invalids = Vec::new();
let mut value = original.clone();
value["version"] = 999.into();
invalids.push(value);
let mut value = original.clone();
value["unexpected"] = true.into();
invalids.push(value);
let mut value = original.clone();
value["entries"][0]["policy"]["recording_material"] = "include".into();
invalids.push(value);
let mut value = original.clone();
value["entries"][0]["policy"]
.as_object_mut()
.unwrap()
.remove("memory_evidence");
invalids.push(value);
let mut value = original.clone();
value["entries"][0]["body"] = serde_json::json!({"kind":"reasoning","text":"secret"});
invalids.push(value);
let mut value = original.clone();
value["entries"][0]["body"]["feature_payload"] = serde_json::json!({"private":true});
invalids.push(value);
let mut value = original.clone();
value["entries"][0]["body"]["role"] = "system".into();
invalids.push(value);
let mut value = original.clone();
value["entries"][0]["source"]["unknown"] = true.into();
invalids.push(value);
let mut value = original.clone();
value["entries"][0]["source"]["history_origin"] =
serde_json::json!({"kind":"legacy_unknown","unknown":true});
invalids.push(value);
let mut value = original.clone();
value["entries"][0]["entry_id"] = "bad/id".into();
invalids.push(value);
let mut value = original.clone();
value["entries"]
.as_array_mut()
.unwrap()
.push(original["entries"][0].clone());
invalids.push(value);
for invalid in invalids {
let bytes = serde_json::to_vec(&invalid).unwrap();
assert!(
SessionCapture::decode(&bytes, CaptureLimits::default()).is_err(),
"accepted {invalid}"
);
}
let bytes = capture.encode(CaptureLimits::default()).unwrap();
assert!(
SessionCapture::decode(
&bytes,
CaptureLimits {
max_bytes: bytes.len() - 1,
..CaptureLimits::default()
}
)
.is_err()
);
assert!(
capture
.encode(CaptureLimits {
max_entries: 0,
..CaptureLimits::default()
})
.is_err()
);
assert!(
capture
.encode(CaptureLimits {
max_text_bytes: 3,
..CaptureLimits::default()
})
.is_err()
);
let mut reversed = capture.to_dto();
reversed.view_entry_count = 2;
let mut second = reversed.entries[0].clone();
second.entry_id = "second".into();
second.source.entry_index = 4;
second.order = 0;
reversed.entries[0].order = 1;
reversed.entries.push(second);
assert!(SessionCapture::try_from_dto(reversed, CaptureLimits::default()).is_err());
let mut duplicate_source = capture.to_dto();
duplicate_source.view_entry_count = 2;
let mut second = duplicate_source.entries[0].clone();
second.entry_id = "distinct-id".into();
second.order = 1;
duplicate_source.entries.push(second);
assert!(matches!(
SessionCapture::try_from_dto(duplicate_source, CaptureLimits::default()),
Err(CaptureError::Invalid("duplicate source position"))
));
}
#[test]
fn raw_utf8_tool_input_and_result_are_lossless_without_preview_markers() {
let arguments = "\"quoted\"\\path\n日本語🙂".repeat(300);
let result = "result\n\t\\\"λ🙂".repeat(400);
let capture = SessionCapture::new(
"s",
"seg",
vec![
Item::tool_call("call", "tool", arguments.clone()),
Item::tool_result_item("call", "summary", Some(result.clone()), false),
],
);
for (id, field, expected) in [
("E00000000", RawField::ToolArguments, arguments),
("E00000001", RawField::ToolContent, result),
] {
let reference = reference(&capture, id);
let mut offset = 0;
let mut actual = String::new();
loop {
let part = capture
.read_raw(&reference, CapturePurpose::Observation, field, offset, 17)
.unwrap();
assert!(part.text.len() <= 17);
assert_eq!(part.offset, offset);
actual.push_str(&part.text);
match part.next_offset {
Some(next) => {
assert!(next > offset);
offset = next;
}
None => break,
}
}
assert_eq!(actual, expected);
}
let unicode = SessionCapture::new("s", "seg", vec![Item::tool_call("call", "tool", "🙂")]);
let reference = reference(&unicode, "E00000000");
assert!(matches!(
unicode.read_raw(
&reference,
CapturePurpose::Observation,
RawField::ToolArguments,
1,
8
),
Err(CaptureError::InvalidRange)
));
assert!(matches!(
unicode.read_raw(
&reference,
CapturePurpose::Observation,
RawField::ToolArguments,
0,
3
),
Err(CaptureError::BudgetTooSmall)
));
assert!(matches!(
unicode.read_raw(
&reference,
CapturePurpose::Observation,
RawField::ToolArguments,
5,
8
),
Err(CaptureError::InvalidRange)
));
}
#[test]
fn escaped_json_slice_budget_includes_metadata_and_bound_continuation() {
let body = "\"\\\n\t🙂日本語".repeat(500);
let capture = SessionCapture::new(
"synthetic-session",
"segment",
vec![Item::tool_call("call", "tool", body.clone())],
);
let other_session = SessionCapture::new(
"other-session",
"segment",
vec![Item::tool_call("call", "tool", body.clone())],
);
let changed = SessionCapture::new(
"synthetic-session",
"segment",
vec![Item::tool_call("call", "tool", "changed")],
);
let mut cursor = None;
let mut actual = String::new();
let budget = 1400;
loop {
let part = capture
.slice(CapturePurpose::Recording, cursor.as_ref(), 1, budget)
.unwrap();
assert!(part.encoded_len() <= budget);
assert_eq!(part.encoded_len(), part.as_json().len());
assert_eq!(serde_json::to_vec(part.slice()).unwrap(), part.as_json());
assert_eq!(part.slice().items.len(), 1);
let item = &part.slice().items[0];
assert_eq!(item.offset, actual.len());
actual.push_str(&item.text);
cursor = part.slice().next_cursor.clone();
if let Some(cursor) = &cursor {
assert!(matches!(
other_session.slice(CapturePurpose::Recording, Some(cursor), 1, budget),
Err(CaptureError::ForeignReference)
));
assert!(matches!(
changed.slice(CapturePurpose::Recording, Some(cursor), 1, budget),
Err(CaptureError::ForeignReference)
));
assert!(matches!(
capture.slice(CapturePurpose::Observation, Some(cursor), 1, budget),
Err(CaptureError::ForeignReference)
));
} else {
break;
}
}
assert_eq!(actual, body);
assert!(matches!(
capture.slice(CapturePurpose::Recording, None, 1, 8),
Err(CaptureError::BudgetTooSmall)
));
assert!(matches!(
capture.slice(CapturePurpose::Recording, None, 0, budget),
Err(CaptureError::BudgetTooSmall)
));
}
#[test]
fn snapshot_immutability_and_scoped_operations_reject_foreign_or_unknown_refs_and_bad_ranges() {
let mut source = vec![
Item::user_message("before"),
Item::assistant_message("answer"),
];
let capture = SessionCapture::new("s", "seg", source.clone());
let fixed_bytes = capture.encode(CaptureLimits::default()).unwrap();
let first = reference(&capture, "E00000000");
let second = reference(&capture, "E00000001");
source.push(Item::user_message("appended"));
source[0] = Item::user_message("compacted");
source.clear();
assert_eq!(
capture.encode(CaptureLimits::default()).unwrap(),
fixed_bytes
);
assert_eq!(
capture.clone().encode(CaptureLimits::default()).unwrap(),
fixed_bytes
);
let foreign = SessionCapture::new(
"different",
"seg",
vec![
Item::user_message("before"),
Item::assistant_message("answer"),
],
);
assert!(matches!(
foreign.resolve_ref(&first, CapturePurpose::Observation),
Err(CaptureError::ForeignReference)
));
assert!(matches!(
capture.checked_range(&second, &first, CapturePurpose::Observation),
Err(CaptureError::InvalidRange)
));
assert_eq!(
capture
.checked_range(&first, &second, CapturePurpose::Observation)
.unwrap()
.len(),
2
);
let unknown = SessionEntryRef::parse("E99999999").unwrap();
assert_eq!(capture.view_order_for_ref(&unknown), None);
assert!(matches!(
capture.scoped_ref(&unknown),
Err(CaptureError::UnknownReference)
));
assert!(
capture
.search_checked(&SearchOptions {
from: Some(unknown.clone()),
..Default::default()
})
.is_err()
);
assert!(
capture
.search(&SearchOptions {
through: Some(unknown),
..Default::default()
})
.is_empty()
);
assert!(
capture
.read_checked(ReadSelector::EntryRange([1, 0]), ReadOptions::default())
.is_err()
);
assert!(
capture
.read_checked(ReadSelector::EntryRange([0, 99]), ReadOptions::default())
.is_err()
);
}
#[test]
fn slices_preserve_empty_and_long_result_fields_under_item_and_metadata_budgets() {
let summary = "summary\n\"🙂".repeat(100);
let content = "content\t\\日本語".repeat(200);
let capture = SessionCapture::new(
"s",
"seg",
vec![
Item::tool_result_item("first", summary.clone(), Some(content.clone()), true),
Item::tool_result_item("empty", "", Some(String::new()), false),
],
);
let mut cursor = None;
let mut fields: Vec<(String, RawField, String)> = Vec::new();
loop {
let part = capture
.slice(CapturePurpose::Recording, cursor.as_ref(), 2, 1500)
.unwrap();
assert!(part.slice().items.len() <= 2);
assert!(serde_json::to_vec(part.slice()).unwrap().len() <= 1500);
for item in &part.slice().items {
let id = item.reference.entry_ref.to_string();
if let Some((last_id, last_field, text)) = fields.last_mut() {
if *last_id == id && *last_field == item.field {
assert_eq!(text.len(), item.offset);
text.push_str(&item.text);
continue;
}
}
assert_eq!(item.offset, 0);
fields.push((id, item.field, item.text.clone()));
}
cursor = part.slice().next_cursor.clone();
if cursor.is_none() {
break;
}
}
assert_eq!(
fields,
vec![
("E00000000".into(), RawField::ToolSummary, summary),
("E00000000".into(), RawField::ToolContent, content),
("E00000001".into(), RawField::ToolSummary, String::new()),
("E00000001".into(), RawField::ToolContent, String::new()),
]
);
let metadata_heavy = SessionCapture::new(
"s",
"seg",
vec![Item::tool_call("call", "name".repeat(200), "x")],
);
assert!(matches!(
metadata_heavy.slice(CapturePurpose::Recording, None, 1, 700),
Err(CaptureError::BudgetTooSmall)
));
}
#[test]
fn evidence_uses_original_segment_positions_after_roundtrip_and_view_selection() {
let full = SessionCapture::new(
"s",
"selected",
vec![
Item::user_message("first"),
Item::assistant_message("second"),
],
);
let mut dto = full.to_dto();
dto.entries[0].source.segment_id = "original-a".into();
dto.entries[0].source.entry_index = 80;
dto.entries[1].source.segment_id = "original-b".into();
dto.entries[1].source.entry_index = 3;
let encoded = dto.encode(CaptureLimits::default()).unwrap();
let restored = SessionCapture::decode(&encoded, CaptureLimits::default()).unwrap();
for (id, segment, index, order) in [
("E00000000", "original-a", 80, 0),
("E00000001", "original-b", 3, 1),
] {
let evidence = restored.evidence_for(id).unwrap();
assert_eq!(evidence.segment_id, segment);
assert_eq!(evidence.entry_range, [index; 2]);
let read = restored.read(ReadSelector::Id(id), ReadOptions::default());
assert_eq!(read.entries[0].entry_range, [order; 2]);
}
let selected = restored.select_order_range(1..2).unwrap();
assert_eq!(selected.overview()[0].entry_range, [0, 0]);
let evidence = selected.evidence_for("E00000001").unwrap();
assert_eq!(evidence.segment_id, "original-b");
assert_eq!(evidence.entry_range, [3, 3]);
let roundtrip = SessionCapture::decode(
&selected.encode(CaptureLimits::default()).unwrap(),
CaptureLimits::default(),
)
.unwrap();
assert_eq!(roundtrip.to_dto(), selected.to_dto());
assert_eq!(
roundtrip.evidence_for("E00000001").unwrap().entry_range,
[3, 3]
);
}
#[test]
fn fixed_history_selection_preserves_original_coordinates_and_checks_private_position_bounds() {
let history = vec![
history("old", Item::user_message("old"), None),
history("hidden", Item::reasoning("PRIVATE"), None),
history("selected", Item::user_message("selected"), None),
history("private-tail", Item::system_message("PRIVATE"), None),
];
let full = SessionCapture::from_complete_history("s", "seg", history);
let suffix = full.select_order_range(1..4).unwrap();
let original = full
.entries()
.find(|entry| entry.entry_id == "selected")
.unwrap();
let selected = suffix.entries().next().unwrap();
assert_eq!(selected.entry_id, original.entry_id);
assert_eq!(selected.source, original.source);
assert_eq!(selected.body, original.body);
assert_eq!(selected.policy, original.policy);
assert!(suffix.evidence_for("Eold").is_none());
assert_eq!(suffix.search(&SearchOptions::default()).len(), 1);
assert_eq!(
suffix.view_order_for_ref(&SessionEntryRef::parse("Eselected").unwrap()),
Some(1)
);
assert_eq!(selected.source.entry_index, 2);
assert_eq!(selected.order, 1);
assert_eq!(suffix.to_dto().view_entry_count, 3);
assert_eq!(
suffix.evidence_for("Eselected").unwrap().entry_range,
[2, 2]
);
let nested = suffix.select_order_range(1..2).unwrap();
assert_eq!(nested.entries().next().unwrap().order, 0);
assert_eq!(nested.entries().next().unwrap().source, original.source);
let restored = SessionCapture::decode(
&suffix.encode(CaptureLimits::default()).unwrap(),
CaptureLimits::default(),
)
.unwrap();
assert_eq!(restored.to_dto(), suffix.to_dto());
assert!(full.select_order_range(4..5).is_err());
let reversed = std::ops::Range { start: 3, end: 2 };
assert!(full.select_order_range(reversed).is_err());
let hidden_only = full.select_order_range(3..4).unwrap();
assert_eq!(hidden_only.entries().len(), 0);
assert_eq!(hidden_only.to_dto().view_entry_count, 1);
let mut invalid = full.to_dto();
invalid.view_entry_count = 2; // the selected entry's view order is outside this extent
assert!(SessionCapture::try_from_dto(invalid, CaptureLimits::default()).is_err());
assert_eq!(
full.retain_entries(|entry| entry.entry_id == "selected")
.entries()
.next()
.unwrap()
.source,
original.source
);
assert_eq!(full.to_dto().view_entry_count, 4);
}
+1 -1
View File
@@ -23,7 +23,7 @@ serde.workspace = true
serde_json.workspace = true
session-store.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["rt", "sync", "time"] }
tokio = { workspace = true, features = ["rt", "sync", "time", "net", "io-util", "io-std", "signal"] }
uuid = { workspace = true, features = ["v7"] }
worker.workspace = true
+2 -1
View File
@@ -4,7 +4,8 @@ A Host that starts existing Workers in the same process. Assembles interactive W
## Reference index
- [`host`] — bootstrap and confirmed shutdown.
- [`host`] — bootstrap and confirmed shutdown, with or without a TUI.
- [`unix`] — optional operator transport; disconnect does not release execution ownership.
- [`launch`] — Profile, cwd, and Subject validation before launch.
- [`store`] — Worker records and leases.
- [`jobs`] — Job execution and independent result and cleanup lifetimes.
+39 -10
View File
@@ -24,7 +24,6 @@ use session_store::{
};
use thiserror::Error;
use worker::bootstrap::{WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout};
use worker::controller::WorkerControllerTransport;
use worker::ipc::protocol_session::{
WorkerProtocolSessionStreams, dispatch_worker_protocol_method, live_log_entry_event,
subscribe_worker_protocol_session,
@@ -222,7 +221,6 @@ impl StandaloneHost {
workspace_context,
filesystem_authority,
WorkerBootstrapLayout::Direct { runtime_base },
WorkerControllerTransport::InProcess,
);
if let Some(subject) = &subject {
bootstrap = bootstrap.with_subjektiv_host(subject.host.clone());
@@ -430,7 +428,6 @@ impl StandaloneHost {
workspace_context,
filesystem_authority,
WorkerBootstrapLayout::Direct { runtime_base },
WorkerControllerTransport::InProcess,
);
if let Some(subject) = &subject {
bootstrap = bootstrap.with_subjektiv_host(subject.host.clone());
@@ -520,10 +517,7 @@ impl StandaloneHost {
/// Working events, committed session entries, alert snapshots, and the
/// initial history snapshot are merged behind the client boundary.
pub fn connect(&self) -> Client<InProcessSocket> {
let streams = subscribe_worker_protocol_session(&self.handle);
let (socket, peer) = InProcessSocket::pair();
tokio::spawn(run_protocol_session(self.handle.clone(), streams, peer));
Client::new(socket)
Client::new(connect_protocol(self.handle.clone()))
}
/// Logical Subject disconnection, not Worker shutdown or completion of its represented work.
@@ -565,6 +559,35 @@ impl StandaloneHost {
}
}
/// Run without a TUI until an admitted protocol Shutdown, controller exit, or process signal.
/// Client EOF only disconnects that client. All exits confirm the same production shutdown
/// and lease release as the in-process UI; an unconfirmed shutdown retains the lease.
pub async fn serve_unix(
self,
endpoint: crate::unix::UnixEndpoint,
) -> Result<(), crate::HeadlessError> {
let mut adapter = crate::unix::UnixAdapter::start(endpoint, self.handle.clone());
let signal = async {
let mut terminate =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
tokio::select! {
result = tokio::signal::ctrl_c() => result,
_ = terminate.recv() => Ok(()),
}
};
let result = tokio::select! {
_ = self.handle.protocol_closed() => Ok(()),
result = signal => result,
result = adapter.ended() => result,
};
// Reclaim the public entry and every connection before releasing execution ownership.
let transport = adapter.shutdown().await;
self.shutdown().await?;
result?;
transport?;
Ok(())
}
pub fn with_shutdown_timeout(mut self, shutdown_timeout: Duration) -> Self {
self.shutdown_timeout = shutdown_timeout;
self
@@ -647,9 +670,8 @@ impl Drop for StandaloneHost {
}
}
// The in-process channel is owned by the local operator's Host, not an
// arbitrary socket or Backend account. decode_method discards wire provenance;
// only this boundary can stamp accepted local human input.
// Both in-process views and credential-checked Unix clients are admitted by this operator's
// Host. decode_method discards wire provenance; only this boundary stamps local human input.
fn authorize_local_input(method: protocol::Method) -> protocol::Method {
use protocol::{AuthenticatedInputSource, Method};
match method {
@@ -692,6 +714,13 @@ fn authorize_local_input(method: protocol::Method) -> protocol::Method {
}
}
pub(crate) fn connect_protocol(handle: worker::WorkerHandle) -> InProcessSocket {
let streams = subscribe_worker_protocol_session(&handle);
let (socket, peer) = InProcessSocket::pair();
tokio::spawn(run_protocol_session(handle, streams, peer));
socket
}
async fn run_protocol_session(
handle: worker::WorkerHandle,
streams: WorkerProtocolSessionStreams,
+529 -20
View File
@@ -25,7 +25,46 @@ use thiserror::Error;
#[path = "job_store/legacy_migrations.rs"]
mod legacy_migrations;
const SCHEMA_VERSION: i64 = 4;
const SCHEMA_VERSION: i64 = 5;
/// Availability of the immutable input body, independent of execution outcome.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum JobInputDisposition {
Available,
Reclaimed,
Erased,
Invalidated,
}
impl JobInputDisposition {
pub fn as_str(self) -> &'static str {
match self {
Self::Available => "available",
Self::Reclaimed => "reclaimed",
Self::Erased => "erased",
Self::Invalidated => "invalidated",
}
}
}
/// Host-authorized retirement; erasure and legacy invalidation do not change outcomes.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum JobInputRetirementReason {
Completed,
ExplicitErasure,
LegacyInvalidation,
}
impl JobInputRetirementReason {
pub fn disposition(self) -> JobInputDisposition {
match self {
Self::Completed => JobInputDisposition::Reclaimed,
Self::ExplicitErasure => JobInputDisposition::Erased,
Self::LegacyInvalidation => JobInputDisposition::Invalidated,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
@@ -67,7 +106,11 @@ impl JobAttemptState {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct JobSnapshot {
/// Receipt metadata. After retirement `input` is null; use the verified loader to execute.
pub request: JobRequest,
pub input_digest: String,
pub intent_fingerprint: String,
pub input_disposition: JobInputDisposition,
/// Durable Host-issued domain delegation, not model input or Profile identity.
pub domain_grant: Option<Value>,
pub state: JobState,
@@ -77,14 +120,27 @@ pub struct JobSnapshot {
}
impl JobSnapshot {
/// Load only the original available body, checking both content and immutable intent.
pub fn load_executable_request(&self) -> Result<&JobRequest, JobStoreError> {
if self.input_disposition != JobInputDisposition::Available {
return Err(JobStoreError::InvalidTransition("Job input is retired"));
}
self.request.validate()?;
if self.request.input_digest()? != self.input_digest
|| self.request.fingerprint()? != self.intent_fingerprint
{
return Err(JobStoreError::Corrupt(
"immutable input or intent fingerprint mismatch".into(),
));
}
Ok(&self.request)
}
/// Shared Feature-facing read contract; no Workspace or execution identity leaks.
pub fn outcome(&self) -> job::JobOutcome {
job::JobOutcome {
job_id: self.request.job_id.clone(),
input_digest: self
.request
.input_digest()
.expect("validated immutable Job input"),
input_digest: self.input_digest.clone(),
attempt_id: self.attempt.attempt_id.clone(),
attempt: self.attempt.number,
state: match self.state {
@@ -204,9 +260,23 @@ impl JobStore {
tx.execute_batch(RECONCILIATION_SCHEMA)?;
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
}
SCHEMA_VERSION => {}
4 | SCHEMA_VERSION => {}
other => return Err(JobStoreError::UnsupportedSchema(other)),
}
if version < 5 {
tx.execute_batch(INPUT_RETENTION_SCHEMA)?;
let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?;
if has_legacy_archive {
tx.execute_batch("DROP TRIGGER legacy_job_archive_immutable;
CREATE TRIGGER legacy_job_archive_immutable BEFORE UPDATE OF job_id,fingerprint,attempts_json ON legacy_job_archive
BEGIN SELECT RAISE(ABORT, 'immutable legacy Job archive'); END;
CREATE TRIGGER legacy_job_archive_input_retirement BEFORE UPDATE OF request_json ON legacy_job_archive
WHEN NOT EXISTS (SELECT 1 FROM job_intents WHERE job_id=OLD.job_id AND input_disposition<>'available')
OR (json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') AND json_type(NEW.request_json,'$.input')='null') IS NOT TRUE
BEGIN SELECT RAISE(ABORT, 'immutable legacy Job input'); END;")?;
}
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
}
// An accepted result is the only evidence of success. Reserved attempts
// were never started and remain pending; cancelled attempts never resume.
tx.execute_batch(
@@ -249,10 +319,7 @@ impl JobStore {
[&request.job_id],
|row| row.get(0),
)?;
if stored != fingerprint
|| snapshot.request != request
|| snapshot.domain_grant != grant
{
if stored != fingerprint || snapshot.domain_grant != grant {
return Err(JobStoreError::IntentConflict(request.job_id));
}
tx.commit()?;
@@ -334,6 +401,7 @@ impl JobStore {
.conn
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let previous = require_snapshot(&tx, job_id)?;
previous.load_executable_request()?;
if previous.state != JobState::Failed || previous.attempt.state != JobAttemptState::Failed {
return Err(JobStoreError::InvalidTransition(
"retry requires known failure",
@@ -354,6 +422,81 @@ impl JobStore {
Ok(snapshot)
}
/// Historical snapshots for one domain Session, including terminal and retired Jobs.
pub(crate) fn list_session_snapshots(
&self,
session_id: &str,
after: Option<&str>,
limit: usize,
) -> Result<Vec<JobSnapshot>, JobStoreError> {
let mut stmt = self.conn.prepare("SELECT i.job_id FROM job_intents i JOIN job_domain_grants g ON g.job_id=i.job_id WHERE json_extract(g.grant_json,'$.session_id')=?1 AND (?2 IS NULL OR i.job_id>?2) ORDER BY i.job_id LIMIT ?3")?;
let ids = stmt
.query_map(
params![session_id, after, limit.clamp(1, 100) as i64],
|r| r.get::<_, String>(0),
)?
.collect::<rusqlite::Result<Vec<_>>>()?;
ids.into_iter()
.map(|id| require_snapshot(&self.conn, &id))
.collect()
}
/// The Host must fence domain mutation and delivery before erasure/invalidation.
/// This atomically prevents dispatch/retry/result acceptance, without rewriting outcome.
pub(crate) fn reclaim_input(
&mut self,
job_id: &str,
expected_input_digest: &str,
reason: JobInputRetirementReason,
) -> Result<JobSnapshot, JobStoreError> {
let tx = self
.conn
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let snapshot = require_snapshot(&tx, job_id)?;
if snapshot.input_digest != expected_input_digest {
return Err(JobStoreError::DigestMismatch);
}
let desired = reason.disposition();
if snapshot.input_disposition == desired {
tx.commit()?;
reclaim_retired_input_storage(&self.conn)?;
return Ok(snapshot);
}
if reason == JobInputRetirementReason::Completed
&& (snapshot.state != JobState::Completed
|| snapshot.attempt.state != JobAttemptState::Completed
|| snapshot.result.is_none())
{
return Err(JobStoreError::InvalidTransition(
"reclaim requires known completed result",
));
}
if snapshot.input_disposition != JobInputDisposition::Available
&& reason != JobInputRetirementReason::ExplicitErasure
{
return Err(JobStoreError::InvalidTransition(
"retired input cannot be restored or reclassified",
));
}
if reason == JobInputRetirementReason::Completed {
snapshot.load_executable_request()?;
}
let mut metadata = snapshot.request.clone();
metadata.input = Value::Null;
tx.execute(
"UPDATE job_intents SET request_json=?2, input_disposition=?3 WHERE job_id=?1",
params![job_id, serde_json::to_string(&metadata)?, desired.as_str()],
)?;
let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?;
if has_legacy_archive {
tx.execute("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input',json('null')) WHERE job_id=?1", [job_id])?;
}
let snapshot = require_snapshot(&tx, job_id)?;
tx.commit()?;
reclaim_retired_input_storage(&self.conn)?;
Ok(snapshot)
}
pub(crate) fn get(&self, job_id: &str) -> Result<JobSnapshot, JobStoreError> {
require_snapshot(&self.conn, job_id)
}
@@ -383,7 +526,7 @@ impl JobStore {
let mut statement = self.conn.prepare(
"SELECT i.job_id FROM job_intents i JOIN job_attempts a
ON a.job_id = i.job_id AND a.number = i.current_attempt
WHERE i.state = 'pending' AND a.state = 'reserved' ORDER BY i.rowid",
WHERE i.state = 'pending' AND i.input_disposition='available' AND a.state = 'reserved' ORDER BY i.rowid",
)?;
let ids = statement
.query_map([], |row| row.get::<_, String>(0))?
@@ -406,6 +549,7 @@ impl JobStore {
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let snapshot = require_snapshot(&tx, job_id)?;
check_attempt(&snapshot, attempt_id)?;
snapshot.load_executable_request()?;
if snapshot.state != JobState::Pending
|| snapshot.attempt.state != JobAttemptState::Reserved
{
@@ -447,7 +591,7 @@ impl JobStore {
.transaction_with_behavior(TransactionBehavior::Immediate)?;
let snapshot = require_snapshot(&tx, &bound.job_id)?;
check_attempt(&snapshot, &bound.attempt_id)?;
if snapshot.request.input_digest()? != bound.input_digest {
if snapshot.input_digest != bound.input_digest {
return Err(JobStoreError::DigestMismatch);
}
let (digest, encoded) =
@@ -466,6 +610,7 @@ impl JobStore {
tx.commit()?;
return Ok(snapshot);
}
snapshot.load_executable_request()?;
if snapshot.state != JobState::Pending
|| snapshot.attempt.state != JobAttemptState::Dispatched
{
@@ -649,7 +794,7 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
let row = conn
.query_row(
"SELECT i.request_json, i.state, i.acknowledged, a.attempt_id, a.number,
a.state, a.failure, a.result_json FROM job_intents i JOIN job_attempts a
a.state, a.failure, a.result_json, a.input_digest, i.fingerprint, i.input_disposition FROM job_intents i JOIN job_attempts a
ON a.job_id = i.job_id AND a.number = i.current_attempt WHERE i.job_id = ?1",
[job_id],
|row| {
@@ -662,12 +807,27 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
row.get::<_, String>(5)?,
row.get::<_, Option<String>>(6)?,
row.get::<_, Option<String>>(7)?,
row.get::<_, String>(8)?,
row.get::<_, String>(9)?,
row.get::<_, String>(10)?,
))
},
)
.optional()?;
row.map(
|(request, state, acknowledged, attempt_id, number, attempt_state, failure, result)| {
|(
request,
state,
acknowledged,
attempt_id,
number,
attempt_state,
failure,
result,
input_digest,
intent_fingerprint,
disposition,
)| {
let state = match state.as_str() {
"pending" => JobState::Pending,
"completed" => JobState::Completed,
@@ -693,8 +853,17 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
)));
}
};
let request_value: Value = serde_json::from_str(&request)?;
if request_value.get("input").is_none() {
return Err(JobStoreError::Corrupt(
"stored Job input field is missing".into(),
));
}
Ok(JobSnapshot {
request: serde_json::from_str(&request)?,
request: serde_json::from_value(request_value)?,
input_digest,
intent_fingerprint,
input_disposition: serde_json::from_value(Value::String(disposition))?,
domain_grant: conn
.query_row(
"SELECT grant_json FROM job_domain_grants WHERE job_id = ?1",
@@ -719,6 +888,37 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
.transpose()
}
// Retirement is already durable before this maintenance step. Failure must be
// retried with the same Job/digest/reason; it must never restore input. VACUUM
// removes historical free-page copies, including stores written before secure_delete.
fn reclaim_retired_input_storage(conn: &Connection) -> Result<(), JobStoreError> {
conn.execute_batch("PRAGMA secure_delete=ON; VACUUM;")?;
let busy: i64 = conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| r.get(0))?;
if busy != 0 {
return Err(JobStoreError::InvalidTransition(
"input retired; SQLite body cleanup blocked by a reader, retry retirement",
));
}
Ok(())
}
const INPUT_RETENTION_SCHEMA: &str = "
ALTER TABLE job_intents ADD COLUMN input_disposition TEXT NOT NULL DEFAULT 'available'
CHECK (input_disposition IN ('available','reclaimed','erased','invalidated'));
DROP TRIGGER job_intent_immutable;
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF
job_id, fingerprint, serialization_key, max_concurrent ON job_intents
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
CREATE TRIGGER job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON job_intents
WHEN (
((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated'))
OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased'))
AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input')
AND json_type(NEW.request_json,'$.input') = 'null'
) IS NOT TRUE
BEGIN SELECT RAISE(ABORT, 'immutable Job input retirement'); END;
";
const RECONCILIATION_SCHEMA: &str = "
DROP INDEX job_serialization_singleflight;
CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key)
@@ -885,7 +1085,11 @@ mod tests {
Some("unconfirmed"),
)
.unwrap();
store.conn.execute_batch("DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap();
store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable;
ALTER TABLE job_intents DROP COLUMN input_disposition;
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap();
drop(store);
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
assert_eq!(reopened.get("first").unwrap(), unknown);
@@ -911,10 +1115,7 @@ mod tests {
JobResultSubmission {
job_id: snapshot.request.job_id.clone(),
attempt_id: snapshot.attempt.attempt_id.clone(),
input_digest: snapshot
.request
.input_digest()
.expect("validated immutable Job input"),
input_digest: snapshot.input_digest.clone(),
result,
}
}
@@ -926,6 +1127,314 @@ mod tests {
.unwrap()
}
#[test]
fn input_reclaim_reopens_as_receipt_and_exact_replay_never_rehydrates() {
let (dir, mut store) = fixture();
let original = request("retained");
let grant = Some(json!({"session_id":"session-a", "consumer":"fixture"}));
let reserved = store
.reserve_granted(original.clone(), grant.clone())
.unwrap();
let running = store
.dispatch("retained", &reserved.attempt.attempt_id, 8)
.unwrap();
let result = submission(&running, json!({"saved":true}));
let completed = store.accept(&result).unwrap();
store.ack("retained").unwrap();
let retired = store
.reclaim_input(
"retained",
&completed.input_digest,
JobInputRetirementReason::Completed,
)
.unwrap();
assert_eq!(retired.input_disposition, JobInputDisposition::Reclaimed);
assert!(retired.request.input.is_null());
assert!(retired.load_executable_request().is_err());
assert_eq!(
retired.outcome().input_digest,
original.input_digest().unwrap()
);
assert_eq!(retired.result, completed.result);
drop(store);
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
assert_eq!(
reopened
.reserve_granted(original.clone(), grant.clone())
.unwrap(),
retired
);
assert_eq!(reopened.accept(&result).unwrap(), retired);
assert_eq!(
reopened
.list_session_snapshots("session-a", None, 1)
.unwrap(),
vec![retired.clone()]
);
assert!(
reopened
.list_session_snapshots("session-b", None, 100)
.unwrap()
.is_empty()
);
assert!(
reopened
.list_session_snapshots("session-a", Some("retained"), 100)
.unwrap()
.is_empty()
);
let mut changed = original.clone();
changed.input = json!({"changed":true});
assert!(matches!(
reopened.reserve_granted(changed, grant),
Err(JobStoreError::IntentConflict(_))
));
assert!(reopened.retry("retained").is_err());
assert!(
reopened
.dispatch("retained", &retired.attempt.attempt_id, 8)
.is_err()
);
assert!(reopened.conn.execute("UPDATE job_intents SET request_json=?1,input_disposition='available' WHERE job_id='retained'", [serde_json::to_string(&original).unwrap()]).is_err());
let erased = reopened
.reclaim_input(
"retained",
&retired.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
assert_eq!(erased.state, JobState::Completed);
assert!(erased.acknowledged);
assert_eq!(
reopened
.reserve_granted(original, erased.domain_grant.clone())
.unwrap(),
erased
);
}
#[test]
fn unfinished_unknown_and_failed_inputs_are_kept_until_explicit_invalidation() {
let (_dir, mut store) = fixture();
for (id, state) in [
("pending", None),
("unknown", Some(JobAttemptState::Unknown)),
("failed", Some(JobAttemptState::Failed)),
] {
let original = request(id);
let snapshot = store.reserve(original.clone()).unwrap();
if let Some(state) = state {
store.dispatch(id, &snapshot.attempt.attempt_id, 8).unwrap();
store
.finish(
id,
&snapshot.attempt.attempt_id,
state,
Some("not complete"),
)
.unwrap();
}
let before = store.get(id).unwrap();
assert!(
store
.reclaim_input(
id,
&before.input_digest,
JobInputRetirementReason::Completed
)
.is_err()
);
assert_eq!(store.get(id).unwrap(), before);
assert_eq!(before.load_executable_request().unwrap(), &original);
assert!(
store
.reclaim_input(id, "wrong", JobInputRetirementReason::ExplicitErasure)
.is_err()
);
let retired = store
.reclaim_input(
id,
&before.input_digest,
JobInputRetirementReason::LegacyInvalidation,
)
.unwrap();
assert_eq!(retired.state, before.state);
assert_eq!(retired.attempt, before.attempt);
assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated);
assert!(store.retry(id).is_err());
assert!(store.dispatch(id, &before.attempt.attempt_id, 8).is_err());
assert_eq!(store.reserve(original).unwrap(), retired);
}
}
#[test]
fn erased_dispatched_input_refuses_delayed_result_without_rewriting_unknown() {
let (dir, mut store) = fixture();
let running = dispatched(&mut store, request("erase-running"));
let result = submission(&running, json!({"late":true}));
let erased = store
.reclaim_input(
"erase-running",
&running.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
assert!(store.accept(&result).is_err());
assert_eq!(erased.state, JobState::Pending);
drop(store);
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
let recovered = reopened.get("erase-running").unwrap();
assert_eq!(recovered.state, JobState::Unknown);
assert_eq!(recovered.input_disposition, JobInputDisposition::Erased);
assert_eq!(recovered.input_digest, running.input_digest);
assert!(reopened.accept(&result).is_err());
assert!(reopened.retry("erase-running").is_err());
}
#[test]
fn available_input_loader_refuses_tamper_missing_body_and_envelope_changes() {
for mutation in [
"json_set(request_json,'$.input',json('{\"tampered\":true}'))",
"json_remove(request_json,'$.input')",
"json_set(request_json,'$.instruction','changed')",
] {
let (_dir, mut store) = fixture();
store.reserve(request("tamper")).unwrap();
// Simulate external corruption bypassing SQLite's normal write guard.
store
.conn
.execute_batch("DROP TRIGGER job_input_retirement")
.unwrap();
store
.conn
.execute_batch(&format!("UPDATE job_intents SET request_json={mutation}"))
.unwrap();
match store.get("tamper") {
Ok(snapshot) => {
assert!(snapshot.load_executable_request().is_err());
assert!(
store
.dispatch("tamper", &snapshot.attempt.attempt_id, 8)
.is_err()
);
}
Err(JobStoreError::Json(_) | JobStoreError::Corrupt(_)) => {}
other => panic!("unexpected corrupt input read: {other:?}"),
}
}
}
#[test]
fn input_retirement_cleanup_failure_is_durable_and_exact_retry_finishes_without_rehydration() {
let (dir, mut store) = fixture();
let mut original = request("physical-erasure");
let private_body = "private-input-retirement-original-bytes".repeat(1000);
original.input = json!({"body":private_body});
let reserved = store.reserve(original.clone()).unwrap();
// A reader retaining the old SQLite snapshot prevents truncation of the old WAL.
let reader = Connection::open(dir.path().join("jobs.sqlite3")).unwrap();
reader
.execute_batch("BEGIN; SELECT request_json FROM job_intents;")
.unwrap();
store.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
assert!(
store
.reclaim_input(
"physical-erasure",
&reserved.input_digest,
JobInputRetirementReason::ExplicitErasure
)
.is_err()
);
let retired = store.get("physical-erasure").unwrap();
assert_eq!(retired.input_disposition, JobInputDisposition::Erased);
assert!(retired.load_executable_request().is_err());
reader.execute_batch("ROLLBACK").unwrap();
drop(reader);
let replay = store
.reclaim_input(
"physical-erasure",
&reserved.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
assert_eq!(replay, retired);
assert_eq!(store.reserve(original).unwrap(), retired);
let needle = b"private-input-retirement-original-bytes";
for entry in std::fs::read_dir(dir.path()).unwrap() {
let path = entry.unwrap().path();
if path.is_file() {
let bytes = std::fs::read(&path).unwrap();
assert!(
!bytes.windows(needle.len()).any(|bytes| bytes == needle),
"retired body remained in {}",
path.display()
);
}
}
}
#[test]
fn missing_original_null_input_is_not_treated_as_available_null() {
let (_dir, mut store) = fixture();
let mut original = request("null-original");
original.input = Value::Null;
let reserved = store.reserve(original).unwrap();
assert!(reserved.load_executable_request().is_ok());
store.conn.execute_batch("DROP TRIGGER job_input_retirement; UPDATE job_intents SET request_json=json_remove(request_json,'$.input')").unwrap();
assert!(store.get("null-original").is_err());
}
#[test]
fn input_retirement_migration_preserves_receipts_and_rolls_back_failed_ddl() {
let (dir, mut store) = fixture();
let original = store.reserve(request("upgrade")).unwrap();
store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable;
ALTER TABLE job_intents DROP COLUMN input_disposition;
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
PRAGMA user_version=4;
CREATE TRIGGER job_input_retirement BEFORE UPDATE ON job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;").unwrap();
drop(store);
let path = dir.path().join("jobs.sqlite3");
assert!(JobStore::open(&path).is_err());
let conn = Connection::open(&path).unwrap();
let version: i64 = conn
.pragma_query_value(None, "user_version", |r| r.get(0))
.unwrap();
assert_eq!(version, 4);
let present: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM pragma_table_info('job_intents') WHERE name='input_disposition')",[],|r|r.get(0)).unwrap();
assert!(!present);
conn.execute_batch("DROP TRIGGER job_input_retirement")
.unwrap();
drop(conn);
let mut reopened = JobStore::open(&path).unwrap();
assert_eq!(reopened.get("upgrade").unwrap(), original);
assert_eq!(
reopened
.get("upgrade")
.unwrap()
.load_executable_request()
.unwrap(),
&request("upgrade")
);
let retired = reopened
.reclaim_input(
"upgrade",
&original.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
assert_eq!(retired.state, JobState::Pending);
assert!(
reopened
.conn
.execute_batch("UPDATE job_intents SET input_disposition='available'")
.is_err()
);
}
#[test]
fn database_configuration_schema_and_identity_constraints() {
let (_dir, mut store) = fixture();
@@ -143,6 +143,42 @@ mod tests {
(original, receipt, grant)
}
#[test]
fn legacy_archive_input_erasure_preserves_fingerprints_and_attempt_receipts() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("jobs.sqlite3");
seed(&path, 2);
let mut store = JobStore::open(&path).unwrap();
let before: (String, String) = store
.conn
.query_row(
"SELECT fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.unwrap();
let snapshot = store.get("done").unwrap();
store
.reclaim_input(
"done",
&snapshot.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
let (body, fingerprint, attempts): (String,String,String) = store.conn.query_row("SELECT request_json,fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
assert!(serde_json::from_str::<Value>(&body).unwrap()["input"].is_null());
assert_eq!((fingerprint, attempts), before);
assert!(store.conn.execute_batch("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input','restored') WHERE job_id='done'").is_err());
drop(store);
let mut reopened = JobStore::open(&path).unwrap();
let receipt = reopened
.reserve_granted(request("done"), snapshot.domain_grant)
.unwrap();
assert_eq!(receipt.state, JobState::Completed);
assert_eq!(receipt.input_disposition, JobInputDisposition::Erased);
assert!(receipt.load_executable_request().is_err());
}
#[test]
fn legacy_versions_preserve_intent_receipts_attempts_grants_and_replay() {
for version in [1, 2] {
+113 -48
View File
@@ -26,7 +26,10 @@ use job::{JobRequest, JobResultSubmission};
use tokio::sync::watch;
use crate::job_store::JobStore;
pub use crate::job_store::{JobAttempt, JobAttemptState, JobSnapshot, JobState, JobStoreError};
pub use crate::job_store::{
JobAttempt, JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot,
JobState, JobStoreError,
};
#[derive(Debug, thiserror::Error)]
pub enum JobServiceError {
@@ -40,6 +43,8 @@ pub enum JobServiceError {
Capacity,
#[error("standalone Job cleanup failed: {0}")]
Cleanup(String),
#[error("standalone Job shutdown is incomplete; execution resources are still owned")]
ShutdownIncomplete,
}
pub(crate) trait JobAttemptFence: Send + Sync {
@@ -173,6 +178,36 @@ impl StandaloneJobs {
Ok(self.with_store(|store| store.reserve(request))?)
}
/// Enumerate historical domain snapshots for a Session, including retired input receipts.
pub fn list_session_snapshots(
&self,
session_id: &str,
after: Option<&str>,
limit: usize,
) -> Result<Vec<JobSnapshot>, JobServiceError> {
self.with_store(|store| store.list_session_snapshots(session_id, after, limit))
}
/// Retire body only. The caller fences domain effects/delivered snapshots before erasure.
/// Normal reclamation also requires that execution cleanup has finished.
pub fn reclaim_input(
&self,
job_id: &str,
expected_input_digest: &str,
reason: JobInputRetirementReason,
) -> Result<JobSnapshot, JobServiceError> {
let executions = self.executions.lock().expect("Job executions poisoned");
if reason == JobInputRetirementReason::Completed
&& executions
.jobs
.get(job_id)
.is_some_and(|execution| !execution.task.is_finished())
{
return Err(JobServiceError::Capacity);
}
self.with_store(|store| store.reclaim_input(job_id, expected_input_digest, reason))
}
pub fn get(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
Ok(self.with_store(|store| store.get(job_id))?)
}
@@ -274,6 +309,12 @@ impl StandaloneJobs {
job_id: &str,
client: Option<Box<dyn LlmClient>>,
) -> Result<(), JobServiceError> {
// Preparation owns copies of the immutable input and domain adapter too. Exclude
// owner shutdown before creating them, not only before publishing the task.
let mut executions = self.executions.lock().expect("Job executions poisoned");
if executions.closed {
return Err(JobServiceError::Closed);
}
let client = client.or_else(|| {
self.model_client
.lock()
@@ -282,16 +323,14 @@ impl StandaloneJobs {
.map(|c| c.clone_boxed())
});
let snapshot = self.get(job_id)?;
snapshot.load_executable_request()?;
let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds));
let deadline = tokio::time::Instant::now() + timeout;
let fence: Arc<dyn JobAttemptFence> = Arc::new(BoundAttemptFence {
store: self.store.clone(),
job_id: job_id.into(),
attempt_id: snapshot.attempt.attempt_id.clone(),
input_digest: snapshot
.request
.input_digest()
.map_err(JobStoreError::from)?,
input_digest: snapshot.input_digest.clone(),
deadline,
});
let grant = match (
@@ -311,10 +350,6 @@ impl StandaloneJobs {
let prepared =
worker::job::prepare_job_with_grant(&snapshot.request, &self.cwd, client, grant)
.map_err(|error| JobServiceError::Configuration(error.to_string()))?;
let mut executions = self.executions.lock().expect("Job executions poisoned");
if executions.closed {
return Err(JobServiceError::Closed);
}
executions.jobs.retain(|_, run| {
!(run.task.is_finished() && matches!(run.complete.borrow().as_ref(), Some(Ok(()))))
});
@@ -367,10 +402,7 @@ impl StandaloneJobs {
store: store.clone(),
job_id: id.clone(),
attempt_id: attempt_id.clone(),
input_digest: snapshot
.request
.input_digest()
.map_err(JobStoreError::from)?,
input_digest: snapshot.input_digest.clone(),
deadline,
});
let (cancel, mut cancellation) = watch::channel(false);
@@ -446,23 +478,25 @@ impl StandaloneJobs {
.clone()
.expect("completion");
result.map_err(JobServiceError::Cleanup)?;
let task = {
let mut executions = self.executions.lock().expect("Job executions poisoned");
if executions
.jobs
.get(job_id)
.is_some_and(|run| run.attempt_id == attempt)
{
executions.jobs.remove(job_id)
} else {
None
}
};
if let Some(execution) = task {
execution
.task
.await
.map_err(|error| JobServiceError::Cleanup(error.to_string()))??;
// Completion is published immediately before the task returns. Do not remove
// its ownership record until the runtime confirms that return (and future drop).
while self
.executions
.lock()
.expect("Job executions poisoned")
.jobs
.get(job_id)
.is_some_and(|run| run.attempt_id == attempt && !run.task.is_finished())
{
tokio::task::yield_now().await;
}
let mut executions = self.executions.lock().expect("Job executions poisoned");
if executions
.jobs
.get(job_id)
.is_some_and(|run| run.attempt_id == attempt)
{
executions.jobs.remove(job_id);
}
}
self.get(job_id)
@@ -476,33 +510,63 @@ impl StandaloneJobs {
}
}
pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> {
/// Initiate owner shutdown without blocking a synchronous caller. Success confirms all
/// local executions have returned, including request/history and Feature cleanup. An
/// incomplete/error result retains the service resources and must retain the Host lease.
pub(crate) fn try_shutdown(&self) -> Result<(), JobServiceError> {
self.close();
let tasks = {
let mut executions = self.executions.lock().expect("Job executions poisoned");
std::mem::take(&mut executions.jobs)
};
let mut error = None;
for execution in tasks.into_values() {
match execution.task.await {
Ok(Ok(())) => {}
Ok(Err(failure)) => {
error = Some(failure);
}
Err(failure) => {
error = Some(JobServiceError::Cleanup(failure.to_string()));
let mut executions = self.executions.lock().expect("Job executions poisoned");
for execution in executions.jobs.values() {
if !execution.task.is_finished() {
return Err(JobServiceError::ShutdownIncomplete);
}
match execution.complete.borrow().as_ref() {
Some(Ok(())) => {}
Some(Err(error)) => return Err(JobServiceError::Cleanup(error.clone())),
None => {
return Err(JobServiceError::Cleanup(
"Internal Worker ended without cleanup confirmation".into(),
));
}
}
}
// Close SQLite before the Host releases its lease. Cloned handles may
// outlive the Host but cannot mutate or retain the live connection.
executions.jobs.clear();
// Retained external handles share these slots; they cannot retain a live store,
// provider or model client after successful owner shutdown.
self.store.lock().expect("Job store poisoned").take();
self.domain.lock().expect("Job domain poisoned").take();
self.model_client
.lock()
.expect("Job model transport poisoned")
.take();
error.map_or(Ok(()), Err)
Ok(())
}
pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> {
self.close();
let completions: Vec<_> = self
.executions
.lock()
.expect("Job executions poisoned")
.jobs
.values()
.map(|execution| execution.complete.clone())
.collect();
// Never take execution ownership out of the shared service across an await: a
// cancelled shutdown future or a concurrent close must still see the barrier.
for mut completion in completions {
completion
.wait_for(Option::is_some)
.await
.map_err(|error| JobServiceError::Cleanup(error.to_string()))?;
}
loop {
match self.try_shutdown() {
// Only the task's final return follows its completion notification.
Err(JobServiceError::ShutdownIncomplete) => tokio::task::yield_now().await,
result => return result,
}
}
}
}
@@ -574,7 +638,8 @@ impl JobAttemptFence for BoundAttemptFence {
if snapshot.state != JobState::Pending
|| snapshot.attempt.state != JobAttemptState::Dispatched
|| snapshot.attempt.attempt_id != self.attempt_id
|| snapshot.request.input_digest().map_err(|e| e.to_string())? != self.input_digest
|| snapshot.input_digest != self.input_digest
|| snapshot.input_disposition != JobInputDisposition::Available
{
return Err("Job domain grant is not bound to the current live attempt".into());
}
+101
View File
@@ -468,3 +468,104 @@ async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_han
assert!(matches!(operation, Err(JobServiceError::Closed)));
}
}
#[tokio::test]
async fn cancelled_shutdown_future_keeps_execution_barrier_and_unknown_input() {
// Actual generic runner + SQLite; only the provider stream is scripted/blocked.
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("jobs.sqlite3");
let jobs = StandaloneJobs::open(&path, temp.path().into()).unwrap();
let reserved = jobs.request(request("cancel-shutdown-wait")).unwrap();
let model = DomainTestClient {
stall: true,
..Default::default()
};
jobs.start_with_model_client(&reserved.request.job_id, model.clone())
.await
.unwrap();
await_model(&model).await;
{
let shutdown = jobs.shutdown();
tokio::pin!(shutdown);
assert!(futures::poll!(shutdown.as_mut()).is_pending());
// Drop the caller's waiter while cancellation/cleanup is still pending.
}
assert!(matches!(
jobs.try_shutdown(),
Err(JobServiceError::ShutdownIncomplete)
));
assert!(jobs.store.lock().unwrap().is_some());
assert!(matches!(
jobs.request(request("after-shutdown")),
Err(JobServiceError::Closed)
));
tokio::time::timeout(Duration::from_secs(5), jobs.shutdown())
.await
.unwrap()
.unwrap();
assert!(jobs.store.lock().unwrap().is_none());
let reopened = StandaloneJobs::open(&path, temp.path().into()).unwrap();
let unknown = reopened.get(&reserved.request.job_id).unwrap();
assert_eq!(unknown.attempt.state, JobAttemptState::Unknown);
assert_eq!(unknown.input_disposition, JobInputDisposition::Available);
assert_eq!(unknown.request, reserved.request);
assert_eq!(unknown.input_digest, reserved.input_digest);
assert!(reopened.retry(&unknown.request.job_id).is_err());
reopened.shutdown().await.unwrap();
}
#[tokio::test]
async fn input_retirement_service_fences_live_attempt_and_preserves_session_receipts() {
// Production service + SQLite + attempt/result fences, without a model or Worker fixture.
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("jobs.sqlite3");
let jobs = StandaloneJobs::open(&path, temp.path().to_path_buf()).unwrap();
let original = request("input-retirement");
let grant = json!({"session_id":"session-a"});
let reserved = jobs
.request_granted(original.clone(), grant.clone())
.unwrap();
jobs.with_store(|store| store.dispatch("input-retirement", &reserved.attempt.attempt_id, 8))
.unwrap();
let fence = BoundAttemptFence {
store: jobs.store.clone(),
job_id: "input-retirement".into(),
attempt_id: reserved.attempt.attempt_id.clone(),
input_digest: reserved.input_digest.clone(),
deadline: tokio::time::Instant::now() + Duration::from_secs(10),
};
assert!(fence.ensure_live().is_ok());
assert!(
jobs.reclaim_input(
"input-retirement",
&reserved.input_digest,
JobInputRetirementReason::Completed
)
.is_err()
);
let erased = jobs
.reclaim_input(
"input-retirement",
&reserved.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.unwrap();
assert!(fence.ensure_live().is_err());
assert!(
sink(&jobs, "input-retirement", &reserved.attempt.attempt_id)
.submit(result("input-retirement", &reserved.attempt.attempt_id))
.is_err()
);
assert!(jobs.start("input-retirement").await.is_err());
assert_eq!(
jobs.list_session_snapshots("session-a", None, 100).unwrap(),
vec![erased.clone()]
);
assert!(
jobs.list_session_snapshots("session-b", None, 100)
.unwrap()
.is_empty()
);
assert_eq!(jobs.request_granted(original, grant).unwrap(), erased);
jobs.shutdown().await.unwrap();
}
+38
View File
@@ -7,6 +7,44 @@ pub mod launch;
pub mod store;
pub mod subjektiv;
mod subjektiv_sessions;
pub mod unix;
pub use unix::{UnixEndpoint, relay_unix_client, run_unix_client};
/// Restore reads the saved cwd and Profile, not a newly resolved launch recipe.
pub enum HeadlessLaunch {
Fresh(ResolvedStandaloneLaunch),
Restore {
state_dir: std::path::PathBuf,
worker_id: WorkerId,
},
}
#[derive(Debug, thiserror::Error)]
pub enum HeadlessError {
#[error(transparent)]
Transport(#[from] std::io::Error),
#[error(transparent)]
Startup(#[from] StandaloneStartupError),
#[error(transparent)]
Shutdown(#[from] StandaloneShutdownError),
}
/// Reserve the optional public transport before creating or restoring execution state.
pub async fn run_headless(
launch: HeadlessLaunch,
endpoint: std::path::PathBuf,
) -> Result<(), HeadlessError> {
let endpoint = UnixEndpoint::bind(endpoint)?;
let host = match launch {
HeadlessLaunch::Fresh(launch) => StandaloneHost::start(launch).await?,
HeadlessLaunch::Restore {
state_dir,
worker_id,
} => StandaloneHost::restore(state_dir, worker_id).await?,
};
host.serve_unix(endpoint).await
}
pub use host::{
StandaloneHost, StandaloneShutdownError, StandaloneStartupError,
+11 -1
View File
@@ -54,6 +54,8 @@ pub enum SubjectError {
Journal(#[from] subjektiv::journal::JournalError),
#[error(transparent)]
Storage(#[from] feature_storage::FeatureStorageError),
#[error("local Subject Job shutdown: {0}")]
JobShutdown(#[from] crate::jobs::JobServiceError),
#[error("local Subject storage metadata: {0}")]
Json(#[from] serde_json::Error),
#[error("local Subject storage permission or identity is invalid: {0}")]
@@ -338,8 +340,16 @@ impl SubjectConnection {
}
}
pub fn close(&mut self) -> Result<(), SubjectError> {
self.host.jobs.lock().expect("Subject Jobs poisoned").take();
self.host.live.store(false, Ordering::Release);
{
let mut jobs = self.host.jobs.lock().expect("Subject Jobs poisoned");
if let Some(service) = jobs.as_ref() {
// Signal cancellation, but retain the handle and exclusive lease until
// the generic owner confirms local request/history/resource teardown.
service.try_shutdown()?;
}
jobs.take();
}
self.host.catalog.manager.shutdown()?;
if let Some(file) = &self.lease {
file.unlock()?;
+498 -231
View File
@@ -5,9 +5,12 @@
//! recovery authority; shutdown does not delete either original Sessions or pending requests.
use super::*;
use crate::jobs::{JobAttemptState, JobSnapshot, JobState};
use crate::jobs::{
JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot, JobState,
};
use session_store::capture::{CaptureDto, CaptureLimits, RecordingPosition};
use subjektiv::api::JobAttemptBinding;
use subjektiv::journal::recording::JournalMaterial;
use subjektiv::journal::recording::JournalInput;
use subjektiv::journal::transport::*;
use subjektiv::journal::*;
use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeature};
@@ -16,7 +19,6 @@ use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeat
mod tests;
pub(super) const PURPOSE: &str = "subjektiv_journal";
const INPUT_BUDGET: usize = 48 * 1024;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
@@ -27,6 +29,7 @@ struct LocalJournalGrant {
session_id: String,
journal_id: String,
ranges: Vec<JournalSourceRange>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
origins: Vec<SourceOrigin>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -35,11 +38,6 @@ struct SourceOrigin {
pointer: JournalSourcePointer,
origin: session_store::public_material::PublicMaterialOrigin,
}
impl LocalJournalGrant {
fn input(&self) -> serde_json::Value {
serde_json::json!({"selected_ranges": self.ranges, "origins": self.origins})
}
}
impl StandaloneSubjects {
/// Explicit operator erasure, separate from ordinary Worker removal. Requires the
@@ -84,32 +82,29 @@ impl StandaloneSubjects {
self.state_root.clone(),
)
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
for snapshot in jobs
.pending()
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
.into_iter()
.chain(
jobs.unknown_granted()
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?,
)
{
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
}) {
let mut after = None;
loop {
let page = jobs
.list_session_snapshots(session_id, after.as_deref(), 100)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
if page.is_empty() {
break;
}
for snapshot in &page {
jobs.cancel(&snapshot.request.job_id)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
jobs.reclaim_input(
&snapshot.request.job_id,
&snapshot.input_digest,
JobInputRetirementReason::ExplicitErasure,
)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
if snapshot.state == JobState::Completed && !snapshot.acknowledged {
jobs.acknowledge(&snapshot.request.job_id)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
}
}
}
for snapshot in jobs
.unacknowledged_results()
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
{
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
}) {
jobs.acknowledge(&snapshot.request.job_id)
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
}
after = page.last().map(|job| job.request.job_id.clone());
}
session_store::FsStore::new_with_retention(
path,
@@ -590,17 +585,28 @@ impl LocalSubjectHost {
self.ensure_live()?;
let jobs = self.jobs()?;
for saved in jobs.unacknowledged_results().map_err(service_error)? {
if self.journal_grant(&saved).is_ok() {
if self.journal_grant_scope(&saved).is_ok() {
self.complete_journal_job(&jobs, &saved)?;
}
}
for pending in jobs.pending().map_err(service_error)? {
if self.journal_grant(&pending).is_ok() {
if pending.request.purpose != PURPOSE {
continue;
}
self.journal_grant_scope(&pending)?;
if pending.request.input.get("capture").is_some() {
self.journal_grant(&pending)?;
return Ok(Some((jobs, pending.request.job_id)));
}
// Old descriptor jobs never acquire new execution authority. Their immutable
// identity/input is retired, not upgraded in place; saved checkpoint wins.
let replacement = self.replace_legacy_journal_job(&jobs, &pending)?;
if let Some(id) = replacement {
return Ok(Some((jobs, id)));
}
}
for unknown in jobs.unknown_granted().map_err(service_error)? {
let Ok(grant) = self.journal_grant(&unknown) else {
let Ok(grant) = self.journal_grant_scope(&unknown) else {
continue;
};
let context = authority(self, &grant.session_id)?;
@@ -685,32 +691,96 @@ impl LocalSubjectHost {
}
}
}
let materials = self.select_journal_materials(&context, &journal.id)?;
if materials.is_empty() {
let mut retained = None;
// Saved body snapshots always win over descriptor migration. In particular,
// a failed continuation must not recapture its original text even if an
// invalidated legacy grant still bounds a larger unprocessed suffix.
for legacy_pass in [false, true] {
let mut after_job = None;
loop {
let page = jobs
.list_session_snapshots(
&journal.source_session_id,
after_job.as_deref(),
100,
)
.map_err(service_error)?;
if page.is_empty() {
break;
}
for saved in &page {
if saved.request.purpose != PURPOSE {
continue;
}
let legacy = saved.input_disposition
== JobInputDisposition::Invalidated
|| (saved.input_disposition == JobInputDisposition::Available
&& saved.request.input.get("capture").is_none());
if legacy != legacy_pass
|| (!legacy
&& saved.input_disposition != JobInputDisposition::Available)
{
continue;
}
let saved_grant = self.journal_grant_scope(saved)?;
if saved_grant.journal_id != journal.id {
continue;
}
if legacy {
if let Some(id) = self.replace_legacy_journal_job(&jobs, saved)? {
return Ok(Some((jobs, id)));
}
continue;
}
self.journal_grant(saved)?;
let original = JournalInput::decode(&saved.request.input)
.map_err(journal_error)?;
let continuation = self.build_journal_input(
&context,
&journal.id,
Some(&original.capture),
)?;
if !continuation.selected_ranges.is_empty() {
retained = Some(continuation);
break;
}
if saved.state == JobState::Completed && saved.acknowledged {
// Recover the acknowledgement-before-reclamation gap without
// turning a retained receipt into an executable empty input.
jobs.reclaim_input(
&saved.request.job_id,
&saved.input_digest,
JobInputRetirementReason::Completed,
)
.map_err(service_error)?;
}
}
if retained.is_some() {
break;
}
after_job = page.last().map(|job| job.request.job_id.clone());
}
if retained.is_some() {
break;
}
}
let input = match retained {
Some(input) => input,
None => self.build_journal_input(&context, &journal.id, None)?,
};
if input.selected_ranges.is_empty() {
continue;
}
let ranges = materials
.into_iter()
.map(|material| material.range)
.collect::<Vec<_>>();
let origins = self.journal_origins(&ranges)?;
let grant = LocalJournalGrant {
writer_binding_id: Some(self.writer.binding_id.clone()),
binding: self.binding.clone(),
session_id: journal.source_session_id,
journal_id: journal.id.clone(),
ranges,
origins,
};
let request = job::JobRequest {
job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()),
purpose: PURPOSE.into(), input_ref: format!("subjektiv://{}/journal/{}", self.binding.subject_id, journal.id),
input: grant.input(),
instruction: "ReadJournalState first. Input descriptors are the frozen canonical public ranges authorized for this Job, not already read material. Read every selected source byte with bounded ReadJournalText before recording; then read initial/current position and open text in bounded pages. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(),
profile: "builtin:subjektiv-journal-recording".into(),
serialization_key: Some(format!("subjektiv-journal:{}", journal.id)),
limits: job::JobLimits { timeout_seconds: 600, ..Default::default() },
ranges: input.selected_ranges.clone(),
origins: vec![],
};
let request = self.journal_request(&journal.id, &input)?;
let saved = jobs
.request_granted(
request,
@@ -727,6 +797,138 @@ impl LocalSubjectHost {
}
}
fn replace_legacy_journal_job(
&self,
jobs: &crate::jobs::StandaloneJobs,
old: &JobSnapshot,
) -> Result<Option<String>, SubjektivHostError> {
let grant = self.journal_grant_scope(old)?;
if old.input_disposition == JobInputDisposition::Available {
old.load_executable_request()
.map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?;
if old.request.input
!= serde_json::json!({"selected_ranges":grant.ranges,"origins":grant.origins})
{
return Err(SubjektivHostError::InvalidArgument(
"invalid legacy Journal descriptor input".into(),
));
}
} else if old.input_disposition != JobInputDisposition::Invalidated {
return Err(SubjektivHostError::InvalidArgument(
"legacy Journal input is unavailable".into(),
));
}
let context = authority(self, &grant.session_id)?;
if let Some(active) = self
.catalog
.store
.active_journal_generation(&context, &grant.journal_id)
.map_err(journal_error)?
{
if active.job.job_id != old.request.job_id {
return Err(SubjektivHostError::InvalidArgument(
"another generation owns legacy migration".into(),
));
}
self.catalog
.store
.close_journal_generation(&context, &active.id, &active.job)
.map_err(journal_error)?;
}
jobs.cancel(&old.request.job_id).map_err(service_error)?;
let current = self
.catalog
.store
.read_journal(&context, &grant.journal_id)
.map_err(journal_error)?;
let end = grant
.ranges
.last()
.ok_or_else(|| SubjektivHostError::InvalidArgument("empty legacy selection".into()))?;
let end_position = if end.bytes.end == end.source.total_bytes {
(end.source.order + 1, 0)
} else {
(end.source.order, end.bytes.end)
};
let replacement = if (current.process_order, current.process_byte) >= end_position {
None
} else {
let mut capture = self.recording_capture(&grant.session_id)?;
capture
.entries
.retain(|entry| grant.ranges.iter().any(|r| r.source.order == entry.order));
for entry in &mut capture.entries {
let range = grant
.ranges
.iter()
.find(|r| r.source.order == entry.order)
.unwrap();
let session_store::capture::CaptureBody::CanonicalPublic {
descriptor,
byte_offset,
text,
..
} = &mut entry.body
else {
unreachable!()
};
if source_descriptor(&grant.session_id, descriptor) != range.source
|| !grant.origins.iter().any(|origin| {
origin.pointer == range.source.pointer && origin.origin == descriptor.origin
})
|| !text.is_char_boundary(range.bytes.start as usize)
|| !text.is_char_boundary(range.bytes.end as usize)
{
return Err(SubjektivHostError::InvalidArgument(
"legacy original source identity/range changed".into(),
));
}
*text = text[range.bytes.start as usize..range.bytes.end as usize].to_owned();
*byte_offset = range.bytes.start;
}
let input = self.build_journal_input(&context, &grant.journal_id, Some(&capture))?;
let request = self.journal_request(&grant.journal_id, &input)?;
let next_grant = LocalJournalGrant {
ranges: input.selected_ranges.clone(),
origins: vec![],
..grant
};
Some(
jobs.request_granted(
request,
serde_json::to_value(next_grant)
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?,
)
.map_err(service_error)?
.request
.job_id,
)
};
jobs.reclaim_input(
&old.request.job_id,
&old.input_digest,
JobInputRetirementReason::LegacyInvalidation,
)
.map_err(service_error)?;
Ok(replacement)
}
fn journal_request(
&self,
journal_id: &str,
input: &JournalInput,
) -> Result<job::JobRequest, SubjektivHostError> {
Ok(job::JobRequest {
job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()), purpose: PURPOSE.into(),
input_ref: format!("subjektiv://{}/journal/{journal_id}", self.binding.subject_id),
input: serde_json::to_value(input).map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?,
instruction: "Input contains the frozen canonical public source bodies, provenance and byte ranges. Use only this saved input, never re-fetch Source. ReadJournalState first and read initial/current Journal position/open text in bounded pages if needed. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(),
profile: "builtin:subjektiv-journal-recording".into(),
serialization_key: Some(format!("subjektiv-journal:{journal_id}")),
limits: job::JobLimits { timeout_seconds: 600, ..Default::default() },
})
}
fn pin_journal_session(&self, session: &str) -> Result<(), SubjektivHostError> {
let path = crate::subjektiv_sessions::authorized_sessions_path(
&self.catalog.state_root,
@@ -869,11 +1071,16 @@ impl LocalSubjectHost {
Ok(())
}
fn material_reader(
fn with_recording_boundary<T>(
&self,
session: &str,
) -> Result<session_store::public_material::SessionPublicMaterialReader, SubjektivHostError>
{
read: impl FnOnce(
&Path,
session_store::SessionId,
session_store::SegmentId,
&session_store::retention::SessionRetentionGuard,
) -> Result<T, SubjektivHostError>,
) -> Result<T, SubjektivHostError> {
let sessions = crate::subjektiv_sessions::authorized_sessions_path(
&self.catalog.state_root,
&self.binding.scope_id,
@@ -906,6 +1113,11 @@ impl LocalSubjectHost {
.ok_or_else(|| SubjektivHostError::InvalidArgument("invalid Session storage".into()))?
.join("worker")
.join(&record.storage_key);
let session_id = session
.parse()
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?;
let retention = crate::store::session_retention(&self.catalog.state_root);
let guard = retention.lock_live(session_id).map_err(source_error)?;
// This is the persisted active selector, not the newest filename/mtime or a model claim.
let metadata =
session_store::WorkerAggregateStore::open_read_only(metadata_root, &record.storage_key)
@@ -933,154 +1145,203 @@ impl LocalSubjectHost {
"no attested active lineage for the retained Session".into(),
)
})?;
session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention(
&sessions,
session
.parse()
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
active,
session_store::public_material::PublicMaterialLimits {
max_segments: 256,
max_entries: 100_000,
max_record_bytes: 64 * 1024 * 1024,
},
crate::store::session_retention(&self.catalog.state_root),
)
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
read(&sessions, session_id, active, &guard)
}
fn recording_capture(&self, session: &str) -> Result<CaptureDto, SubjektivHostError> {
self.with_recording_boundary(session, |sessions, session_id, active, guard| {
session_store::capture::capture_recording_fs_guarded(
sessions,
session_id,
active,
session_store::public_material::PublicMaterialLimits::default(),
CaptureLimits {
max_bytes: 128 * 1024 * 1024,
max_entries: 100_000,
max_text_bytes: 64 * 1024 * 1024,
..Default::default()
},
guard,
)
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
})
}
fn journal_sources(
&self,
session: &str,
) -> Result<Vec<JournalSourceRange>, SubjektivHostError> {
let reader = self.material_reader(session)?;
Ok(reader
.descriptors()
.map(|descriptor| JournalSourceRange {
source: source_descriptor(session, descriptor),
bytes: JournalByteRange {
start: 0,
end: descriptor.total_bytes,
self.with_recording_boundary(session, |sessions, session_id, active, guard| {
session_store::capture::recording_catalog_fs_guarded(
sessions,
session_id,
active,
session_store::public_material::PublicMaterialLimits::default(),
guard,
|descriptors| {
descriptors
.iter()
.map(|descriptor| JournalSourceRange {
source: source_descriptor(session, descriptor),
bytes: JournalByteRange {
start: 0,
end: descriptor.total_bytes,
},
})
.collect()
},
})
.collect())
}
fn source_reader(
&self,
source: &JournalSource,
) -> Result<session_store::public_material::SessionPublicMaterialReader, SubjektivHostError>
{
let sessions = crate::subjektiv_sessions::authorized_sessions_path(
&self.catalog.state_root,
&self.binding.scope_id,
&self.binding.subject_id,
&self.catalog.store,
&source.session_id,
)?;
// Frozen, already admitted original pointers do not follow the Worker's later active
// Session/Segment. This reads that original lineage and still checks descriptor identity,
// canonical order and digest below; the shared service enforces the frozen byte range.
session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention(
&sessions,
source
.session_id
.parse()
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
source.pointer.segment_id,
session_store::public_material::PublicMaterialLimits {
max_segments: 256,
max_entries: 100_000,
max_record_bytes: 64 * 1024 * 1024,
},
crate::store::session_retention(&self.catalog.state_root),
)
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
}
fn journal_origins(
&self,
ranges: &[JournalSourceRange],
) -> Result<Vec<SourceOrigin>, SubjektivHostError> {
if ranges.is_empty() {
return Err(SubjektivHostError::InvalidArgument(
"empty material selection".into(),
));
}
ranges
.iter()
.map(|range| {
let reader = self.source_reader(&range.source)?;
reader
.descriptors()
.find(|descriptor| {
source_descriptor(&range.source.session_id, descriptor) == range.source
})
.map(|descriptor| SourceOrigin {
pointer: range.source.pointer.clone(),
origin: descriptor.origin.clone(),
})
.ok_or_else(|| {
SubjektivHostError::InvalidArgument(
"source provenance/identity changed".into(),
)
})
})
.collect()
}
fn journal_source_window(
&self,
source: &JournalSource,
offset: u64,
max_bytes: usize,
) -> Result<String, SubjektivHostError> {
let reader = self.source_reader(source)?;
let descriptor = reader.descriptors().find(|descriptor| source_descriptor(&source.session_id, descriptor) == *source)
.ok_or_else(|| SubjektivHostError::InvalidArgument("canonical source identity/order/digest changed or is outside the authorized Session".into()))?;
reader
.read_content(descriptor, offset, max_bytes)
.map(|window| window.content)
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
}
fn journal_source_material(
&self,
source: &JournalSource,
start: u64,
limit: usize,
) -> JournalResult<JournalMaterial> {
let text = self
.journal_source_window(source, start, limit.max(4))
.map_err(|error| JournalError::Conflict(error.to_string()))?;
Ok(JournalMaterial {
range: JournalSourceRange {
source: source.clone(),
bytes: JournalByteRange {
start,
end: start + text.len() as u64,
},
},
text,
)
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
})
}
fn select_journal_materials(
fn build_journal_input(
&self,
context: &JournalHostContext,
id: &str,
) -> Result<Vec<JournalMaterial>, SubjektivHostError> {
subjektiv::journal::recording::select_materials_bounded(
&self.catalog.store,
context,
id,
INPUT_BUDGET,
|source, start, limit| self.journal_source_material(source, start, limit),
)
saved: Option<&CaptureDto>,
) -> Result<JournalInput, SubjektivHostError> {
let session = self
.catalog
.store
.read_journal(context, id)
.map_err(journal_error)?
.source_session_id;
JournalInput::build(&self.catalog.store, context, id, |order, offset, budget| {
let capture = match saved {
Some(capture) => capture.clone(),
None => {
let capture = self
.with_recording_boundary(&session, |sessions, session_id, active, guard| {
session_store::capture::capture_recording_slice_fs_guarded(
sessions,
session_id,
active,
session_store::public_material::PublicMaterialLimits::default(),
RecordingPosition {
canonical_order: order,
byte_offset: offset,
},
MAX_JOURNAL_RANGES,
budget,
guard,
)
.map(|slice| slice.capture)
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
})
.map_err(|e| JournalError::Conflict(e.to_string()))?;
// Admission and capture are separate saves. An append between them
// may be in replay but has not yet been accepted for this Journal.
return self.accepted_capture_prefix(context, id, capture);
}
};
let end = capture
.entries
.last()
.map(|entry| {
capture
.recording_text(entry.order)
.map(|(descriptor, start, text)| {
if start + text.len() as u64 == descriptor.total_bytes {
RecordingPosition {
canonical_order: entry.order + 1,
byte_offset: 0,
}
} else {
RecordingPosition {
canonical_order: entry.order,
byte_offset: start + text.len() as u64,
}
}
})
})
.transpose()
.map_err(|e| JournalError::Conflict(e.to_string()))?;
if end.is_none_or(|end| (order, offset) >= (end.canonical_order, end.byte_offset)) {
return Ok(CaptureDto {
entries: vec![],
..capture
});
}
capture
.recording_slice(
RecordingPosition {
canonical_order: order,
byte_offset: offset,
},
MAX_JOURNAL_RANGES,
budget,
)
.map(|slice| slice.capture)
.map_err(|e| JournalError::Conflict(e.to_string()))
})
.map_err(journal_error)
}
fn journal_grant(
fn accepted_capture_prefix(
&self,
context: &JournalHostContext,
id: &str,
mut capture: CaptureDto,
) -> JournalResult<CaptureDto> {
let mut selected = Vec::new();
for mut entry in capture.entries {
let status = match self
.catalog
.store
.read_journal_source(context, id, entry.order)
{
Ok(status) => status,
Err(JournalError::NotFound(_)) => break,
Err(error) => return Err(error),
};
let session_store::capture::CaptureBody::CanonicalPublic {
descriptor,
byte_offset,
text,
..
} = &mut entry.body
else {
return Err(JournalError::InvalidInput("non-recording capture".into()));
};
if source_descriptor(&capture.session_id, descriptor) != status.source {
return Err(JournalError::Conflict(
"captured source differs from admitted identity".into(),
));
}
let Some(accepted) = status
.accepted
.iter()
.find(|range| range.start <= *byte_offset && *byte_offset < range.end)
else {
break;
};
let end = status
.unavailable
.iter()
.filter(|range| range.end > *byte_offset)
.fold(accepted.end, |end, range| {
end.min(range.start.max(*byte_offset))
});
let mut length = text.len().min(end.saturating_sub(*byte_offset) as usize);
while !text.is_char_boundary(length) {
length -= 1;
}
if length == 0 {
break;
}
text.truncate(length);
let partial = *byte_offset + (text.len() as u64) < descriptor.total_bytes;
selected.push(entry);
if partial {
break;
}
}
capture.entries = selected;
Ok(capture)
}
fn journal_grant_scope(
&self,
snapshot: &JobSnapshot,
) -> Result<LocalJournalGrant, SubjektivHostError> {
@@ -1097,7 +1358,6 @@ impl LocalSubjectHost {
|| snapshot.request.purpose != PURPOSE
|| snapshot.request.serialization_key.as_deref()
!= Some(&format!("subjektiv-journal:{}", grant.journal_id))
|| snapshot.request.input != grant.input()
{
return Err(SubjektivHostError::InvalidArgument(
"Job input/grant does not match this Journal capability".into(),
@@ -1135,12 +1395,33 @@ impl LocalSubjectHost {
Ok(grant)
}
fn journal_grant(
&self,
snapshot: &JobSnapshot,
) -> Result<LocalJournalGrant, SubjektivHostError> {
snapshot
.load_executable_request()
.map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?;
let grant = self.journal_grant_scope(snapshot)?;
let input = JournalInput::decode(&snapshot.request.input).map_err(journal_error)?;
if input.subject_id != self.binding.subject_id
|| input.journal_id != grant.journal_id
|| input.capture.session_id != grant.session_id
|| input.selected_ranges != grant.ranges
{
return Err(SubjektivHostError::InvalidArgument(
"saved Journal body identity/ranges differ from grant".into(),
));
}
Ok(grant)
}
fn complete_journal_job(
&self,
jobs: &crate::jobs::StandaloneJobs,
snapshot: &JobSnapshot,
) -> Result<(), SubjektivHostError> {
let grant = self.journal_grant(snapshot)?;
let grant = self.journal_grant_scope(snapshot)?;
let context = authority(self, &grant.session_id)?;
let completion: JournalJobCompletion =
serde_json::from_value(snapshot.result.clone().ok_or_else(|| {
@@ -1174,6 +1455,12 @@ impl LocalSubjectHost {
}
jobs.acknowledge(&snapshot.request.job_id)
.map_err(service_error)?;
jobs.reclaim_input(
&snapshot.request.job_id,
&snapshot.input_digest,
JobInputRetirementReason::Completed,
)
.map_err(service_error)?;
if let Err(error) = self.prune_retained_workers() {
// Pruning is independent of a successful checkpoint/result. Keep the originals
// and continue draining other saved prefixes; a retention error is not model failure.
@@ -1192,47 +1479,18 @@ pub(super) fn grant(
.journal_grant(snapshot)
.map_err(|error| error.to_string())?;
let context = authority(&host, &grant.session_id).map_err(|error| error.to_string())?;
// Re-fetch selected identities before opening mutation authority. Only descriptors are
// retained in Job storage; the model must fetch actual source bytes with bounded tools.
for range in &grant.ranges {
let length = usize::try_from(
range
.bytes
.end
.checked_sub(range.bytes.start)
.ok_or("invalid source range")?,
)
.map_err(|_| "source range overflow")?;
if length > INPUT_BUDGET {
return Err("source range exceeds frozen input budget".into());
}
let text = host
.journal_source_window(&range.source, range.bytes.start, length.max(4))
.map_err(|error| error.to_string())?;
if text.len() != length {
return Err("frozen Journal source byte range changed".into());
}
}
if host
.journal_origins(&grant.ranges)
.map_err(|error| error.to_string())?
!= grant.origins
{
return Err(
"frozen Journal provenance does not match the authorized original Session".into(),
);
}
let input = JournalInput::decode(&snapshot.request.input).map_err(|e| e.to_string())?;
let input_digest = snapshot.input_digest.clone();
let generation = host
.catalog
.store
.prepare_journal_generation(
.prepare_journal_input(
&context,
&grant.journal_id,
&input,
JobAttemptBinding {
job_id: snapshot.request.job_id.clone(),
attempt_id: snapshot.attempt.attempt_id.clone(),
},
grant.ranges.clone(),
)
.map_err(|error| error.to_string())?;
let job_context = JournalJobContext::validated(
@@ -1242,12 +1500,17 @@ pub(super) fn grant(
generation.job.clone(),
)
.map_err(|error| error.to_string())?;
input
.validate_generation(&generation)
.map_err(|e| e.to_string())?;
let adapter = Arc::new(LocalJournalHost {
host,
grant,
context,
job_context,
#[cfg(test)]
generation,
input_digest,
fence,
});
Ok(worker::job::JobFeatureGrant {
@@ -1263,7 +1526,9 @@ struct LocalJournalHost {
grant: LocalJournalGrant,
context: JournalHostContext,
job_context: JournalJobContext,
#[cfg(test)]
generation: JournalGeneration,
input_digest: String,
fence: Arc<dyn crate::jobs::JobAttemptFence>,
}
impl LocalJournalHost {
@@ -1296,18 +1561,20 @@ impl JournalJobHost for LocalJournalHost {
request: JournalJobRequest,
) -> Result<JournalJobResponse, SubjektivHostError> {
let context = self.live()?;
subjektiv::journal::service::execute_bounded(
// The generic runner has supplied this exact immutable input before it can
// invoke this bound tool. Preparation alone is not evidence of delivery.
subjektiv::journal::service::attest_input_delivery(
&self.host.catalog.store,
&context,
&self.input_digest,
&self.grant.ranges,
)
.map_err(journal_error)?;
subjektiv::journal::service::execute_snapshot(
&self.host.catalog.store,
&context,
&self.input_digest,
request,
|source, start, limit| {
if source.session_id != self.generation.initial_journal.source_session_id {
return Err(JournalError::PermissionDenied(
"foreign original Session".into(),
));
}
self.host.journal_source_material(source, start, limit)
},
)
.map_err(journal_error)
}
+747 -112
View File
@@ -1,6 +1,6 @@
//! Integration of standalone attribution, original FsStore material, shared Journal storage
//! and the local Job adapter. Only the outer model/execution lease is replaced in these cases.
//! The generic Job runner is exercised separately by the standalone Jobs integration suite.
//! The scripted model also exercises saved direct input through the actual generic Job runner.
use super::*;
use crate::store::{StaleLeasePolicy, StandaloneWorkerAllocation, StandaloneWorkerStore};
@@ -144,8 +144,8 @@ impl Fixture {
.unwrap()
.contains(sentinel)
);
// Include SQLite/WAL/journal files: immutable request/grant storage must never hold
// original text, even in freed pages left behind by explicit source erasure.
// Include SQLite/WAL/journal files: erasure must scrub retired original bodies,
// including any freed pages or journal left by the immutable request store.
for entry in fs::read_dir(self.connection.jobs_path.parent().unwrap()).unwrap() {
let entry = entry.unwrap();
if entry.file_type().unwrap().is_file() {
@@ -194,9 +194,9 @@ impl Fixture {
context,
job_context,
generation,
input_digest: snapshot.input_digest.clone(),
fence: fence.clone(),
};
adapter.read_all_sources();
(adapter, fence, snapshot)
}
}
@@ -220,37 +220,6 @@ fn message(id: &str, text: &str) -> session_store::LoggedHistoryEntry {
}
}
impl LocalJournalHost {
fn read_all_sources(&self) {
for range in &self.generation.selected_ranges {
let mut offset = range.bytes.start as usize;
let mut content_token = None;
loop {
let response = self
.execute(JournalJobRequest::ReadText {
read: JournalTextRead {
target: JournalTextTarget::Source {
pointer: range.source.pointer.clone(),
},
offset,
limit: MAX_JOURNAL_READ_BYTES,
content_token: content_token.clone(),
},
})
.unwrap();
let JournalJobResponse::Text(text) = response else {
panic!()
};
match text.next_byte {
Some(next) => {
assert!(next > offset);
offset = next;
content_token = Some(text.content_token);
}
None => break,
}
}
}
}
fn state(&self) -> Result<JournalJobState, SubjektivHostError> {
match self.execute(JournalJobRequest::ReadState)? {
JournalJobResponse::State(state) => Ok(state),
@@ -362,8 +331,8 @@ fn duplicate_admission_reserves_one_job_and_excludes_private_prompt() {
assert_eq!(jobs.pending().unwrap().len(), 1);
let input = jobs.get(&id).unwrap().request.input.to_string();
assert!(
!input.contains("short request"),
"Job storage retains descriptors, never a second copy of original text"
input.contains("short request"),
"Job storage supplies the canonical body directly"
);
assert!(!input.contains("PRIVATE PROMPT"));
let foreign = SubjektivHostContext {
@@ -498,14 +467,16 @@ fn generation_input_stays_frozen_and_next_prefix_includes_last_input() {
&adapter.generation.job,
)
.unwrap();
let materials = fixture
let next = fixture
.host()
.select_journal_materials(&adapter.context, &adapter.grant.journal_id)
.build_journal_input(&adapter.context, &adapter.grant.journal_id, None)
.unwrap();
assert_eq!(materials[0].range.source.pointer.entry_ref, "Elast");
assert_eq!(next.selected_ranges[0].source.pointer.entry_ref, "Elast");
assert!(
materials[0]
.text
next.capture
.recording_text(next.selected_ranges[0].source.order)
.unwrap()
.2
.contains("last input while generation runs")
);
}
@@ -547,7 +518,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() {
let (adapter, _, _) = fixture.adapter();
let range = &adapter.grant.ranges[0];
assert!(range.bytes.end < range.source.total_bytes);
assert!(range.bytes.end - range.bytes.start < INPUT_BUDGET as u64);
assert!(range.bytes.end - range.bytes.start < job::MAX_JOB_INPUT_BYTES as u64);
let snapshot = fixture
.host()
.jobs()
@@ -557,7 +528,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() {
.pop()
.unwrap();
assert!(serde_json::to_vec(&snapshot.request.input).unwrap().len() < 128 * 1024);
assert!(!snapshot.request.input.to_string().contains("漢字"));
assert!(snapshot.request.input.to_string().contains("漢字"));
assert!(adapter.finish().is_err());
}
@@ -628,43 +599,35 @@ impl JournalScript {
)
} else if let Some(state) = result("state") {
let state = &state["value"];
for (index, range) in state["selected_ranges"]
.as_array()
.unwrap()
let input: serde_json::Value = request
.items
.iter()
.enumerate()
{
let mut offset = range["bytes"]["start"].as_u64().unwrap();
let mut token = Value::Null;
loop {
let id = format!("source-{index}-{offset}");
if let Some(page) = result(&id) {
let page = &page["value"];
let key = format!("{}:{id}", state["generation_id"].as_str().unwrap());
let text = page["text"].as_str().unwrap().to_owned();
if let Some(previous) =
self.source_reads.lock().unwrap().insert(key, text.clone())
{
assert_eq!(
previous, text,
"frozen source read changed within a generation"
);
}
if let Some(next) = page["next_byte"].as_u64() {
assert!(next > offset);
offset = next;
token = page["content_token"].clone();
} else {
break;
}
} else {
let input = json!({"target":{"target":"source","pointer":range["source"]["pointer"]},"offset":offset,"limit":MAX_JOURNAL_READ_BYTES,"content_token":token});
return vec![
Event::tool_use_start(0, &id, "ReadJournalText"),
Event::tool_input_delta(0, input.to_string()),
Event::tool_use_stop(0),
];
}
.filter_map(Item::as_text)
.find_map(|text| {
text.lines()
.find_map(|line| line.strip_prefix("input_json: "))
})
.map(|json| serde_json::from_str(json).unwrap())
.expect("direct saved Journal input");
let input = JournalInput::decode(&input).unwrap();
assert_eq!(
serde_json::to_value(&input.selected_ranges).unwrap(),
state["selected_ranges"]
);
for entry in &input.capture.entries {
let (_, offset, text) = input.capture.recording_text(entry.order).unwrap();
let key = format!(
"{}:{}:{offset}",
state["generation_id"].as_str().unwrap(),
entry.order
);
if let Some(previous) = self
.source_reads
.lock()
.unwrap()
.insert(key, text.to_owned())
{
assert_eq!(previous, text, "saved source changed within generation");
}
}
let (name, action) = if state["current_open"].is_null() {
@@ -1166,6 +1129,11 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
.items;
assert_eq!(entries.len(), 1);
let saved_entry = entries[0].id.clone();
let original_input = jobs.get(&id).unwrap().request.input;
fixture.append(
"newer-than-snapshot",
"NEW INPUT MUST NOT MIX INTO RESUMED SNAPSHOT",
);
jobs.shutdown().await.unwrap();
fixture.connection.close().unwrap();
let mut reopened = StandaloneSubjects::open_existing(&state)
@@ -1176,8 +1144,29 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
.unwrap();
reopened.host.bind_jobs(jobs.clone()).unwrap();
assert_eq!(jobs.get(&id).unwrap().state, JobState::Unknown);
// Domain reconciliation inspects the now-fenced prior attempt and preserves its receipts.
// Deny every original-source read while preparing and running the replacement.
let source_path = fixture
.sessions
.root_dir()
.join(&fixture.context.session_id);
let denied = fixture
._temp
.path()
.join("temporarily-unavailable-original");
fs::rename(&source_path, &denied).unwrap();
let (_, replacement) = reopened.host.next_journal_job().unwrap().unwrap();
let replacement_input = jobs.get(&replacement).unwrap().request.input;
assert_eq!(replacement_input["capture"], original_input["capture"]);
assert!(
!replacement_input
.to_string()
.contains("NEW INPUT MUST NOT MIX")
);
assert_eq!(jobs.get(&id).unwrap().request.input, original_input);
assert_eq!(
jobs.get(&id).unwrap().input_disposition,
JobInputDisposition::Available
);
assert_ne!(
replacement, id,
"unknown external execution is not blindly replayed"
@@ -1205,8 +1194,26 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
.items;
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].id, saved_entry);
fs::rename(&denied, &source_path).unwrap();
let journal = reopened
.host
.catalog
.store
.read_journal(&context, &journal_id)
.unwrap();
let original = JournalInput::decode(&original_input).unwrap();
let end = original.selected_ranges.last().unwrap();
assert_eq!(
(journal.process_order, journal.process_byte),
if end.bytes.end == end.source.total_bytes {
(end.source.order + 1, 0)
} else {
(end.source.order, end.bytes.end)
}
);
assert!(reopened.host.admit_journal(&fixture.context).unwrap());
assert!(
reopened
!reopened
.host
.session_fully_reflected(&fixture.context.session_id)
.unwrap()
@@ -1302,9 +1309,9 @@ fn missing_original_material_blocks_worker_removal_without_erasing_state() {
}
#[test]
fn frozen_original_remains_readable_after_worker_switches_its_active_session() {
fn frozen_input_survives_active_session_change_and_source_tool_remains_refused() {
let fixture = Fixture::new("old Session original");
let (adapter, _, _) = fixture.adapter();
let (adapter, _, snapshot) = fixture.adapter();
let workers = StandaloneWorkerStore::open_existing(&fixture.host().catalog.state_root).unwrap();
let worker = fixture._allocation.as_ref().unwrap().worker_id();
let record = workers.load(worker).unwrap();
@@ -1318,22 +1325,30 @@ fn frozen_original_remains_readable_after_worker_switches_its_active_session() {
)),
))
.unwrap();
let response = adapter
.execute(JournalJobRequest::ReadText {
read: JournalTextRead {
target: JournalTextTarget::Source {
pointer: adapter.generation.selected_ranges[0].source.pointer.clone(),
},
offset: 0,
limit: MAX_JOURNAL_READ_BYTES,
content_token: None,
},
})
.unwrap();
let JournalJobResponse::Text(page) = response else {
panic!()
};
assert!(page.text.contains("old Session original"));
assert!(
adapter
.execute(JournalJobRequest::ReadText {
read: JournalTextRead {
target: JournalTextTarget::Source {
pointer: adapter.generation.selected_ranges[0].source.pointer.clone()
},
offset: 0,
limit: MAX_JOURNAL_READ_BYTES,
content_token: None
}
})
.is_err(),
"original Source lookup is explicitly unavailable"
);
let input = JournalInput::decode(&snapshot.request.input).unwrap();
assert!(
input
.capture
.recording_text(0)
.unwrap()
.2
.contains("old Session original")
);
assert!(
fixture
.host()
@@ -1512,14 +1527,18 @@ fn relocated_erasure_fences_old_and_retained_readers_publishers_and_late_pins()
const SENTINEL: &str = "original-plaintext-sentinel-must-never-enter-immutable-job-storage";
let mut fixture = Fixture::new(SENTINEL);
let (_, _, snapshot) = fixture.adapter();
fixture.assert_job_storage_excludes(&snapshot, SENTINEL);
assert!(snapshot.request.input.to_string().contains(SENTINEL));
let state = fixture.host().catalog.state_root.clone();
let binding = fixture.connection.binding.clone();
let session: session_store::SessionId = fixture.context.session_id.parse().unwrap();
let old_reader = fixture
.host()
.material_reader(&fixture.context.session_id)
.unwrap();
let old_reader = SessionPublicMaterialReader::open_fs_with_retention(
fixture.sessions.root_dir(),
session,
fixture.segment,
PublicMaterialLimits::default(),
crate::store::session_retention(&state),
)
.unwrap();
let descriptor = old_reader.descriptors().next().unwrap().clone();
let (worker, workers) = fixture.stop_worker();
assert!(workers.delete(worker).is_err());
@@ -1759,14 +1778,12 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job(
let snapshot = jobs.get(&next).unwrap();
let grant = fixture.host().journal_grant(&snapshot).unwrap();
assert_eq!(grant.ranges.len(), 1);
assert_eq!(
grant.origins[0].origin,
PublicMaterialOrigin::HostRunBoundary
);
let content = fixture
.host()
.journal_source_window(&grant.ranges[0].source, 0, 4096)
let input = JournalInput::decode(&snapshot.request.input).unwrap();
let (descriptor, _, content) = input
.capture
.recording_text(grant.ranges[0].source.order)
.unwrap();
assert_eq!(descriptor.origin, PublicMaterialOrigin::HostRunBoundary);
let value: serde_json::Value = serde_json::from_str(&content).unwrap();
assert_eq!(value["kind"], "run_boundary");
assert_eq!(value["boundary"]["kind"], "errored");
@@ -1791,3 +1808,621 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job(
assert!(fixture.host().next_journal_job().unwrap().is_none());
jobs.shutdown().await.unwrap();
}
/// Original FsStore -> bounded common capture -> persisted Job -> actual generic runner ->
/// sequential Journal tools. Only the LLM transport is scripted. Source files are deliberately
/// unavailable after persistence, and input bodies are inspected in the provider request.
#[tokio::test]
async fn saved_body_jobs_process_over_a_hundred_entries_and_utf8_without_live_source_reads() {
let fixture = Fixture::new("oldest original");
for index in 0..115 {
fixture.append(
&format!("entry-{index}"),
&format!("public original {index}"),
);
}
fixture.append("long-utf8", &"漢字🙂\n\"".repeat(24_000));
fixture.host().admit_journal(&fixture.context).unwrap();
let expected = fixture
.host()
.recording_capture(&fixture.context.session_id)
.unwrap();
let source_path = fixture
.sessions
.root_dir()
.join(&fixture.context.session_id);
let denied = fixture._temp.path().join("denied-original-source");
let mut seen: std::collections::BTreeMap<u64, String> = Default::default();
let mut jobs_processed = 0;
loop {
let Some((jobs, id)) = fixture.host().next_journal_job().unwrap() else {
break;
};
let saved = jobs.get(&id).unwrap();
let input = JournalInput::decode(&saved.request.input).unwrap();
assert!(
serde_json::to_vec(&saved.request.input).unwrap().len() <= job::MAX_JOB_INPUT_BYTES
);
assert!(input.capture.entries.len() <= MAX_JOURNAL_RANGES);
for entry in &input.capture.entries {
let (_, offset, text) = input.capture.recording_text(entry.order).unwrap();
let previous = seen.entry(entry.order).or_default();
assert_eq!(
previous.len() as u64,
offset,
"no gap, preview or duplicate prefix"
);
previous.push_str(text);
}
// Deny all catalogs/reads, including grant preparation. The model still receives
// exactly the persisted body, and the Host must not follow the active pointer.
fs::rename(&source_path, &denied).unwrap();
let model = JournalScript::default();
jobs.start_with_model_client(&id, model.clone())
.await
.unwrap();
let result = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
.await
.unwrap()
.unwrap();
assert_eq!(result.state, JobState::Completed);
assert!(
!model.source_reads.lock().unwrap().is_empty(),
"provider saw saved source bodies"
);
fixture.host().complete_journal_job(&jobs, &result).unwrap();
assert_eq!(
jobs.get(&id).unwrap().input_disposition,
JobInputDisposition::Reclaimed
);
fs::rename(&denied, &source_path).unwrap();
jobs_processed += 1;
assert!(jobs_processed < 20, "bounded prefix drain must progress");
}
assert!(
jobs_processed > 2,
"entry count and long text both require continuation"
);
assert_eq!(seen.len(), expected.entries.len());
for entry in &expected.entries {
assert_eq!(
seen[&entry.order],
expected.recording_text(entry.order).unwrap().2
);
}
fixture.host().jobs().unwrap().shutdown().await.unwrap();
}
#[tokio::test]
async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful_records() {
let fixture = Fixture::new("original descriptor-only input");
let (adapter, _, modern) = fixture.adapter();
let jobs = fixture.host().jobs().unwrap();
fixture
.host()
.catalog
.store
.close_journal_generation(
&adapter.context,
&adapter.generation.id,
&adapter.generation.job,
)
.unwrap();
jobs.cancel(&modern.request.job_id).unwrap();
let input = JournalInput::decode(&modern.request.input).unwrap();
let origins: Vec<_> = input
.capture
.entries
.iter()
.map(|entry| {
let (descriptor, _, _) = input.capture.recording_text(entry.order).unwrap();
SourceOrigin {
pointer: source_descriptor(&fixture.context.session_id, descriptor).pointer,
origin: descriptor.origin.clone(),
}
})
.collect();
let mut old_grant = adapter.grant.clone();
old_grant.origins = origins;
old_grant.writer_binding_id = None;
let mut request = modern.request.clone();
request.job_id = format!("legacy-descriptor:{}", uuid::Uuid::now_v7());
request.input =
serde_json::json!({"selected_ranges":old_grant.ranges,"origins":old_grant.origins});
let old = jobs
.request_granted(request, serde_json::to_value(&old_grant).unwrap())
.unwrap();
assert!(
fixture.host().journal_grant(&old).is_err(),
"descriptor jobs must not execute as snapshots"
);
let generation = fixture
.host()
.catalog
.store
.prepare_journal_generation(
&adapter.context,
&old_grant.journal_id,
JobAttemptBinding {
job_id: old.request.job_id.clone(),
attempt_id: old.attempt.attempt_id.clone(),
},
old_grant.ranges.clone(),
)
.unwrap();
let context = JournalJobContext::validated(
&fixture.host().catalog.store,
&adapter.context,
&generation.id,
generation.job.clone(),
)
.unwrap();
let saved = fixture
.host()
.catalog
.store
.execute_journal(
&context,
JournalOperation {
request_id: "legacy-record-success-response-lost".into(),
expected_state: generation.initial_journal.state_token.clone(),
action: JournalAction::Create {
title: "already saved".into(),
body: "legacy successful operation".into(),
ongoing: "unfinished".into(),
state: JournalEntryState::Open,
source_end: None,
correction_refs: vec![],
},
covered_ranges: generation.selected_ranges.clone(),
},
)
.unwrap();
fixture.append(
"new-not-selected",
"LATEST SOURCE MUST WAIT UNTIL OLD INPUT FINISHES",
);
fixture.host().admit_journal(&fixture.context).unwrap();
let (_, next) = fixture.host().next_journal_job().unwrap().unwrap();
assert_ne!(next, old.request.job_id);
let replacement = jobs.get(&next).unwrap();
let replacement_input = JournalInput::decode(&replacement.request.input).unwrap();
assert!(
!replacement
.request
.input
.to_string()
.contains("LATEST SOURCE MUST WAIT")
);
assert_eq!(replacement_input.selected_ranges, old_grant.ranges);
let retired = jobs.get(&old.request.job_id).unwrap();
assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated);
assert_eq!(retired.input_digest, old.input_digest);
assert_eq!(retired.intent_fingerprint, old.intent_fingerprint);
assert!(retired.request.input.is_null());
assert!(
fixture
.host()
.catalog
.store
.execute_journal(
&context,
JournalOperation {
request_id: "legacy-delayed".into(),
expected_state: saved.state_token,
action: JournalAction::ConfirmUnchanged,
covered_ranges: old_grant.ranges,
}
)
.is_err()
);
jobs.start_with_model_client(&next, JournalScript::default())
.await
.unwrap();
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&next))
.await
.unwrap()
.unwrap();
assert_eq!(completed.state, JobState::Completed);
fixture
.host()
.complete_journal_job(&jobs, &completed)
.unwrap();
let entries = fixture
.host()
.catalog
.store
.list_journal_entries(&adapter.context, &adapter.grant.journal_id, None, 32)
.unwrap();
assert_eq!(entries.items.len(), 1);
assert_eq!(entries.items[0].id, saved.entry_id.unwrap());
jobs.shutdown().await.unwrap();
}
#[test]
fn explicit_erase_retires_every_historical_body_across_job_history_pages() {
const SENTINEL: &str = "historical-input-erasure-sentinel-781";
let mut fixture = Fixture::new("ordinary original");
let (adapter, _, first) = fixture.adapter();
let jobs = fixture.host().jobs().unwrap();
for index in 0..105 {
let mut request = first.request.clone();
request.job_id = format!("historical-{index:03}");
request.purpose = "historical-other-domain-job".into();
request.serialization_key = None;
request.input = serde_json::json!({"body":SENTINEL, "index":index});
let saved = jobs
.request_granted(
request,
serde_json::json!({"session_id":fixture.context.session_id}),
)
.unwrap();
jobs.cancel(&saved.request.job_id).unwrap();
}
let state = fixture.host().catalog.state_root.clone();
let subject = fixture.connection.binding.subject_id.clone();
fixture.connection.close().unwrap();
StandaloneSubjects::open_existing(&state)
.unwrap()
.erase_session_completely(&subject, &fixture.context.session_id, "erase-all-history")
.unwrap();
let jobs = crate::jobs::StandaloneJobs::open(
&fixture.connection.jobs_path,
fixture._temp.path().into(),
)
.unwrap();
let mut after = None;
let mut count = 0;
loop {
let page = jobs
.list_session_snapshots(&fixture.context.session_id, after.as_deref(), 100)
.unwrap();
if page.is_empty() {
break;
}
for saved in &page {
assert_eq!(saved.input_disposition, JobInputDisposition::Erased);
assert!(saved.request.input.is_null());
assert!(saved.load_executable_request().is_err());
}
count += page.len();
after = page.last().map(|job| job.request.job_id.clone());
}
assert_eq!(count, 106);
assert!(
adapter.execute(JournalJobRequest::ReadState).is_err(),
"prepared local adapter is invalidated"
);
fixture.assert_job_storage_excludes(&jobs.get(&first.request.job_id).unwrap(), SENTINEL);
}
#[test]
fn saved_snapshot_missing_tampered_foreign_or_changed_initial_state_is_not_recaptured() {
let fixture = Fixture::new("saved immutable original");
fixture.host().admit_journal(&fixture.context).unwrap();
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
let saved = jobs.get(&id).unwrap();
for changed in [serde_json::Value::Null, serde_json::json!({"capture":{}})] {
let mut malformed = saved.clone();
malformed.request.input = changed;
assert!(fixture.host().journal_grant(&malformed).is_err());
}
let original = JournalInput::decode(&saved.request.input).unwrap();
for field in ["subject_id", "journal_id", "initial_state_token"] {
let mut changed = saved.request.clone();
changed.job_id = format!("changed-{field}");
changed.serialization_key = None;
changed.input[field] = serde_json::json!("foreign-or-altered");
// Even a separately reserved immutable Job may not adopt the original capability.
let changed = jobs
.request_granted(changed, saved.domain_grant.clone().unwrap())
.unwrap();
assert!(
grant(
fixture.host().clone(),
&changed,
Arc::new(TestFence(AtomicBool::new(true)))
)
.is_err()
);
}
let mut generation = fixture
.host()
.catalog
.store
.prepare_journal_generation(
&authority(fixture.host(), &fixture.context.session_id).unwrap(),
&original.journal_id,
JobAttemptBinding {
job_id: id,
attempt_id: saved.attempt.attempt_id.clone(),
},
original.selected_ranges.clone(),
)
.unwrap();
generation.initial_journal.state_token = "changed-after-snapshot".into();
assert!(original.validate_generation(&generation).is_err());
assert_eq!(
saved.request.input,
jobs.get(&saved.request.job_id).unwrap().request.input
);
}
#[tokio::test]
async fn append_between_admission_and_body_capture_does_not_publish_unaccepted_input() {
let fixture = Fixture::new("accepted oldest prefix");
fixture.host().admit_journal(&fixture.context).unwrap();
fixture.append("not-yet-admitted", "newly saved but not accepted");
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
let input = jobs.get(&id).unwrap().request.input;
assert!(input.to_string().contains("accepted oldest prefix"));
assert!(!input.to_string().contains("newly saved but not accepted"));
jobs.start_with_model_client(&id, JournalScript::default())
.await
.unwrap();
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
.await
.unwrap()
.unwrap();
assert_eq!(completed.state, JobState::Completed);
fixture
.host()
.complete_journal_job(&jobs, &completed)
.unwrap();
fixture.host().admit_journal(&fixture.context).unwrap();
let (_, next) = fixture.host().next_journal_job().unwrap().unwrap();
let remaining = jobs.get(&next).unwrap().request.input;
assert!(
remaining
.to_string()
.contains("newly saved but not accepted")
);
assert!(!remaining.to_string().contains("accepted oldest prefix"));
jobs.shutdown().await.unwrap();
}
/// Generic Job preparation must not attest delivery. The saved body is supplied by the
/// runner before a bound Journal tool may execute; no model-provided digest is trusted.
#[tokio::test]
async fn actual_bound_tool_attests_delivery_but_prepared_blocked_job_does_not() {
let fixture = Fixture::new("body delivered through generic runner");
fixture.host().admit_journal(&fixture.context).unwrap();
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
let saved = jobs.get(&id).unwrap();
let model = JournalScript {
gate_first: true,
pause_after_record: true,
..Default::default()
};
jobs.start_with_model_client(&id, model.clone())
.await
.unwrap();
tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified())
.await
.unwrap();
let host_context = authority(fixture.host(), &fixture.context.session_id).unwrap();
let journal_id = fixture
.host()
.catalog
.store
.journal_for_session(&host_context)
.unwrap()
.unwrap();
let generation = fixture
.host()
.catalog
.store
.active_journal_generation(&host_context, &journal_id)
.unwrap()
.unwrap();
let context = JournalJobContext::validated(
&fixture.host().catalog.store,
&host_context,
&generation.id,
generation.job.clone(),
)
.unwrap();
let undelivered = subjektiv::journal::service::execute_snapshot(
&fixture.host().catalog.store,
&context,
&saved.input_digest,
JournalJobRequest::ReadState,
)
.unwrap_err();
assert!(
undelivered.to_string().contains("delivery"),
"prepared grant must not satisfy delivery gate: {undelivered}"
);
let before = fixture
.host()
.catalog
.store
.read_journal(&host_context, &journal_id)
.unwrap();
assert_eq!((before.process_order, before.process_byte), (0, 0));
model.release.notify_one();
tokio::time::timeout(
std::time::Duration::from_secs(20),
model.record_saved.notified(),
)
.await
.unwrap();
assert!(
subjektiv::journal::service::execute_snapshot(
&fixture.host().catalog.store,
&context,
&saved.input_digest,
JournalJobRequest::ReadState
)
.is_ok(),
"actual bound tool execution attests exact saved input"
);
model.release.notify_one();
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
.await
.unwrap()
.unwrap();
assert_eq!(completed.state, JobState::Completed);
fixture
.host()
.complete_journal_job(&jobs, &completed)
.unwrap();
jobs.shutdown().await.unwrap();
}
#[derive(Clone, Default)]
struct BlockedJournalRequest {
entered: Arc<tokio::sync::Notify>,
dropped: Arc<tokio::sync::Notify>,
request_dropped: Arc<AtomicBool>,
}
struct OwnedBlockedJournalRequest {
// A local request copy, including the saved input, must be dropped before erasure
// can acquire the lease. This does not claim remote/provider bytes are retractable.
_request: agen::llm_client::Request,
model: BlockedJournalRequest,
}
impl Drop for OwnedBlockedJournalRequest {
fn drop(&mut self) {
self.model.request_dropped.store(true, Ordering::Release);
self.model.dropped.notify_one();
}
}
#[async_trait::async_trait]
impl agen::llm_client::LlmClient for BlockedJournalRequest {
async fn stream(
&self,
request: agen::llm_client::Request,
) -> Result<
std::pin::Pin<
Box<
dyn futures::Stream<
Item = Result<
agen::llm_client::event::Event,
agen::llm_client::error::ClientError,
>,
> + Send,
>,
>,
agen::llm_client::error::ClientError,
> {
let owned = OwnedBlockedJournalRequest {
_request: request,
model: self.clone(),
};
assert!(
owned
._request
.items
.iter()
.filter_map(agen::llm_client::types::Item::as_text)
.any(|text| text.contains("local blocked source body"))
);
self.entered.notify_one();
std::future::pending::<()>().await;
drop(owned);
unreachable!()
}
fn clone_boxed(&self) -> Box<dyn agen::llm_client::LlmClient> {
Box::new(self.clone())
}
}
/// Real generic Jobs and a retained public handle, with only the provider transport blocked.
/// Connection close must initiate cancellation and preserve the exclusive lease until local
/// request/history ownership is gone (or return an explicit incomplete barrier).
#[tokio::test]
async fn close_cancels_retained_local_job_before_erase_can_acquire_exclusive_lease() {
let mut fixture = Fixture::new("local blocked source body");
fixture.host().admit_journal(&fixture.context).unwrap();
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
let retained = jobs.clone();
let model = BlockedJournalRequest::default();
retained
.start_with_model_client(&id, model.clone())
.await
.unwrap();
tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified())
.await
.unwrap();
let state = fixture.host().catalog.state_root.clone();
let subject = fixture.connection.binding.subject_id.clone();
let close_error = fixture.connection.close().unwrap_err();
assert!(matches!(
close_error,
SubjectError::JobShutdown(crate::jobs::JobServiceError::ShutdownIncomplete)
));
assert!(fixture.connection.lease.is_some());
assert!(fixture.host().jobs.lock().unwrap().is_some());
assert!(matches!(
retained.start(&id).await,
Err(crate::jobs::JobServiceError::Closed)
));
let locally_dropped_before_erase = model.request_dropped.load(Ordering::Acquire);
let early_erasure = StandaloneSubjects::open_existing(&state)
.unwrap()
.erase_session_completely(
&subject,
&fixture.context.session_id,
"erase-blocked-local-run",
);
let early_lease_was_retained = matches!(early_erasure, Err(SubjectError::Active(_)));
let teardown_from_close = model.request_dropped.load(Ordering::Acquire)
|| tokio::time::timeout(std::time::Duration::from_secs(5), model.dropped.notified())
.await
.is_ok();
// Always tear down before asserting the observed violation, so a failed regression
// cannot leak the blocked model future into another test.
let _ = retained.shutdown().await;
assert!(model.request_dropped.load(Ordering::Acquire));
fixture.connection.close().unwrap();
assert!(fixture.connection.lease.is_none());
assert!(fixture.host().jobs.lock().unwrap().is_none());
// Cancellation is not completion/erasure. Reopening retains the same unknown
// attempt's immutable body until the explicit erasure transaction retires it.
let reopened = crate::jobs::StandaloneJobs::open(
&fixture.connection.jobs_path,
fixture._temp.path().into(),
)
.unwrap();
let unknown = reopened.get(&id).unwrap();
assert_eq!(unknown.attempt.state, crate::jobs::JobAttemptState::Unknown);
assert_eq!(unknown.input_disposition, JobInputDisposition::Available);
assert!(
unknown
.request
.input
.to_string()
.contains("local blocked source body")
);
reopened.shutdown().await.unwrap();
StandaloneSubjects::open_existing(&state)
.unwrap()
.erase_session_completely(
&subject,
&fixture.context.session_id,
"erase-blocked-local-run",
)
.unwrap();
let erased_store = crate::jobs::StandaloneJobs::open(
&fixture.connection.jobs_path,
fixture._temp.path().into(),
)
.unwrap();
let erased = erased_store.get(&id).unwrap();
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
fixture.assert_job_storage_excludes(&erased, "local blocked source body");
erased_store.shutdown().await.unwrap();
assert!(
locally_dropped_before_erase || early_lease_was_retained,
"close released exclusive lease while a retained generic Job still owned local input/history"
);
assert!(
teardown_from_close,
"connection close did not initiate cancellation of the externally retained local Job"
);
assert!(
retained.get(&id).is_err(),
"retained handles are closed after teardown"
);
}
@@ -127,7 +127,6 @@ async fn local_host_compaction_freezes_pending_journal_through_request_and_durab
worker::WorkerWorkspaceContext::no_workspace(),
worker::WorkerFilesystemAuthority::None,
worker::WorkerBootstrapLayout::Direct { runtime_base: fixture._temp.path().join("runtime") },
worker::WorkerControllerTransport::InProcess,
).with_model_client(model.clone()).with_subjektiv_host(fixture.host().clone()).prepare_restored("journal-test").await.unwrap();
assert_eq!(prepared.worker().session_id().to_string(), fixture.context.session_id);
let source_segment = prepared.worker().segment_id();
+296
View File
@@ -0,0 +1,296 @@
//! Optional operator transport owned by the Standalone Host, never by a Worker.
//!
//! The endpoint's existing parent must be private to the current Unix user. Binding never
//! removes an existing entry, including stale sockets. Recovery is an explicit operator action.
//! Peers must have that same uid; admitted methods still cross the normal local protocol
//! authorization boundary. Disconnect only drops a view, not the Host or its execution lease.
use std::io;
use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
use std::path::{Path, PathBuf};
use client::transport::Socket;
use protocol::stream::{JsonLineWriter, decode_event, decode_method};
use tokio::io::{AsyncBufReadExt, BufReader};
use tokio::net::{UnixListener, UnixStream};
use tokio::sync::oneshot;
use tokio::task::{JoinHandle, JoinSet};
/// A bound endpoint reservation. Drop removes only the exact socket created by this bind.
/// It can be reserved before starting a Host, so bind failures do not allocate Workers.
pub struct UnixEndpoint {
listener: UnixListener,
path: PathBuf,
device: u64,
inode: u64,
}
impl UnixEndpoint {
pub fn bind(path: impl AsRef<Path>) -> io::Result<Self> {
let path = path.as_ref();
let name = path.file_name().ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidInput, "endpoint requires a filename")
})?;
let parent = std::fs::canonicalize(
path.parent()
.filter(|p| !p.as_os_str().is_empty())
.unwrap_or(Path::new(".")),
)?;
let metadata = std::fs::metadata(&parent)?;
// SAFETY: geteuid has no preconditions and does not mutate process identity.
let uid = unsafe { libc::geteuid() };
if !metadata.is_dir() || metadata.uid() != uid || metadata.mode() & 0o077 != 0 {
return Err(io::Error::new(
io::ErrorKind::PermissionDenied,
"endpoint parent must be a current-user-owned private directory (mode 0700)",
));
}
let path = parent.join(name);
let listener = UnixListener::bind(&path)?;
let metadata = std::fs::symlink_metadata(&path)?;
let endpoint = Self {
listener,
path,
device: metadata.dev(),
inode: metadata.ino(),
};
std::fs::set_permissions(&endpoint.path, std::fs::Permissions::from_mode(0o600))?;
Ok(endpoint)
}
pub fn path(&self) -> &Path {
&self.path
}
}
impl Drop for UnixEndpoint {
fn drop(&mut self) {
if let Ok(current) = std::fs::symlink_metadata(&self.path)
&& current.file_type().is_socket()
&& current.dev() == self.device
&& current.ino() == self.inode
{
let _ = std::fs::remove_file(&self.path);
}
}
}
pub(crate) struct UnixAdapter {
task: Option<JoinHandle<io::Result<()>>>,
stop: Option<oneshot::Sender<()>>,
}
impl UnixAdapter {
pub(crate) fn start(endpoint: UnixEndpoint, handle: worker::WorkerHandle) -> Self {
let (stop, mut stopped) = oneshot::channel();
let task = tokio::spawn(async move {
let mut connections = JoinSet::new();
let result = loop {
tokio::select! {
_ = &mut stopped => break Ok(()),
accepted = endpoint.listener.accept(), if connections.len() < 64 => {
match accepted {
Ok((stream, _)) => {
// The filesystem gate protects connection establishment; credentials
// also fence open connections to the Host's operator identity.
let allowed = stream.peer_cred().is_ok_and(|cred| {
// SAFETY: geteuid has no preconditions.
cred.uid() == unsafe { libc::geteuid() }
});
if allowed {
let socket = crate::host::connect_protocol(handle.clone());
connections.spawn(bridge(stream, socket));
}
}
Err(error) => break Err(error),
}
}
_ = connections.join_next(), if !connections.is_empty() => {}
}
};
connections.shutdown().await;
// Keep the bound endpoint alive until every connection has been reclaimed.
drop(endpoint);
result
});
Self {
task: Some(task),
stop: Some(stop),
}
}
pub(crate) async fn ended(&mut self) -> io::Result<()> {
let result = self.task.as_mut().expect("adapter task").await;
self.task.take();
result.map_err(io::Error::other)?
}
pub(crate) async fn shutdown(mut self) -> io::Result<()> {
if let Some(stop) = self.stop.take() {
let _ = stop.send(());
}
if self.task.is_some() {
self.ended().await?;
}
Ok(())
}
}
impl Drop for UnixAdapter {
fn drop(&mut self) {
if let Some(stop) = self.stop.take() {
let _ = stop.send(());
}
// Sending stop lets the task join its connection set and drop its endpoint.
// A Host dropped without confirmed shutdown keeps the production execution lease.
}
}
async fn bridge(
stream: UnixStream,
mut socket: client::transport::in_process::Socket,
) -> io::Result<()> {
let (reader, writer) = stream.into_split();
let mut reader = BufReader::new(reader).lines();
let mut writer = JsonLineWriter::new(writer);
loop {
tokio::select! {
line = reader.next_line() => {
let Some(message) = line? else { return Ok(()); };
if message.trim().is_empty() { continue; }
socket.send(message).await.map_err(io::Error::other)?;
}
event = socket.next() => {
let Some(message) = event.map_err(io::Error::other)? else { return Ok(()); };
let event = decode_event(&message).map_err(io::Error::other)?;
writer.write_event(&event).await?;
}
}
}
}
/// Connect a JSONL operator client. EOF disconnects without issuing Shutdown.
/// Input is decoded as Worker Methods and output contains Worker Events only.
pub async fn run_unix_client(endpoint: PathBuf) -> io::Result<()> {
relay_unix_client(endpoint, tokio::io::stdin(), tokio::io::stdout()).await
}
/// The CLI's stream relay, also usable by embedders that own their input/output streams.
pub async fn relay_unix_client<R, W>(endpoint: PathBuf, input: R, output: W) -> io::Result<()>
where
R: tokio::io::AsyncRead + Unpin,
W: tokio::io::AsyncWrite + Unpin,
{
let stream = UnixStream::connect(endpoint).await?;
let (reader, writer) = stream.into_split();
let mut input = BufReader::new(input).lines();
let mut to_host = JsonLineWriter::new(writer);
let mut from_host = BufReader::new(reader).lines();
let mut output = JsonLineWriter::new(output);
loop {
tokio::select! {
line = input.next_line() => {
let Some(line) = line? else { return Ok(()); };
if line.trim().is_empty() { continue; }
let method = decode_method(&line).map_err(io::Error::other)?;
to_host.write(&method).await?;
}
line = from_host.next_line() => {
let Some(line) = line? else { return Ok(()); };
let event = decode_event(&line).map_err(io::Error::other)?;
output.write_event(&event).await?;
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn endpoint_requires_private_parent_and_is_owner_only() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("host.sock");
std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o755)).unwrap();
let error = UnixEndpoint::bind(&path)
.err()
.expect("public parent rejected");
assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
assert!(!path.exists());
std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap();
let endpoint = UnixEndpoint::bind(&path).unwrap();
assert_eq!(
std::fs::metadata(&path).unwrap().permissions().mode() & 0o777,
0o600
);
let client = UnixStream::connect(&path).await.unwrap();
let (server, _) = endpoint.listener.accept().await.unwrap();
assert_eq!(
client.peer_cred().unwrap().uid(),
server.peer_cred().unwrap().uid()
);
drop(endpoint);
assert!(!path.exists());
}
#[tokio::test]
async fn bind_failure_precedes_worker_state_and_startup_failure_releases_only_its_endpoint() {
let directory = tempfile::tempdir().unwrap();
std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap();
let path = directory.path().join("host.sock");
let owner = UnixEndpoint::bind(&path).unwrap();
let state_dir = directory.path().join("must-not-be-opened");
let error = crate::run_headless(
crate::HeadlessLaunch::Restore {
state_dir: state_dir.clone(),
worker_id: protocol::WorkerId::now_v7(),
},
path.clone(),
)
.await
.unwrap_err();
assert!(matches!(error, crate::HeadlessError::Transport(_)));
assert!(!state_dir.exists());
assert!(path.exists());
drop(owner);
let error = crate::run_headless(
crate::HeadlessLaunch::Restore {
state_dir,
worker_id: protocol::WorkerId::now_v7(),
},
path.clone(),
)
.await
.unwrap_err();
assert!(matches!(error, crate::HeadlessError::Startup(_)));
assert!(
!path.exists(),
"a failed Host startup must release its own bind"
);
}
#[tokio::test]
async fn bind_failure_and_old_endpoint_drop_preserve_another_owner() {
let directory = tempfile::tempdir().unwrap();
std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap();
let path = directory.path().join("host.sock");
let first = UnixEndpoint::bind(&path).unwrap();
let first_inode = std::fs::metadata(&path).unwrap().ino();
assert_eq!(
UnixEndpoint::bind(&path).err().unwrap().kind(),
io::ErrorKind::AddrInUse
);
assert_eq!(std::fs::metadata(&path).unwrap().ino(), first_inode);
std::fs::rename(&path, directory.path().join("retained.sock")).unwrap();
let second = UnixEndpoint::bind(&path).unwrap();
let second_inode = std::fs::metadata(&path).unwrap().ino();
assert_ne!(first_inode, second_inode);
drop(first);
assert_eq!(std::fs::metadata(&path).unwrap().ino(), second_inode);
let _client = UnixStream::connect(&path).await.unwrap();
let _connection = second.listener.accept().await.unwrap();
drop(second);
assert!(!path.exists());
}
}
+233
View File
@@ -608,6 +608,239 @@ async fn standalone_metadata_fails_closed_on_incomplete_or_newer_records() -> Te
Ok(())
}
#[tokio::test]
async fn headless_unix_client_uses_production_history_lease_and_local_authority() -> TestResult {
use protocol::stream::{JsonLineReader, encode_method};
use std::os::unix::fs::PermissionsExt;
use tokio::io::AsyncWriteExt;
let temp = tempfile::tempdir()?;
// Shorten only the owned endpoint's components, preserving the harness TMPDIR
// even when it is checkout-local. No process-global cwd or external /tmp fallback.
let socket_dir = tempfile::Builder::new()
.prefix("")
.rand_bytes(4)
.tempdir()?;
std::fs::set_permissions(socket_dir.path(), std::fs::Permissions::from_mode(0o700))?;
let endpoint_path = socket_dir.path().join("s");
let endpoint = standalone::UnixEndpoint::bind(&endpoint_path)?;
let state_dir = temp.path().join("state");
let launch = StandaloneLaunchConfig::new(
temp.path(),
&state_dir,
manifest::ProfileSelector::Default,
"headless-operator",
)
.resolve()?;
let model = ScriptedClient::new(vec![vec![
LlmEvent::text_block_start(0),
LlmEvent::text_delta(0, "headless production response"),
LlmEvent::text_block_stop(0, Some(StopReason::EndTurn)),
]]);
let inspection = model.clone();
let host = StandaloneHost::start_with_model_client(launch, model).await?;
let worker_id = host.worker_id();
let session_id = host.record().active_session_id;
let profile = host.record().manifest.clone();
let serve = tokio::spawn(host.serve_unix(endpoint));
// Exercise the real external client's stdin/stdout relay over the actual Unix adapter.
let (mut input, relay_input) = tokio::io::duplex(64 * 1024);
let (relay_output, output) = tokio::io::duplex(64 * 1024);
let relay = tokio::spawn(standalone::relay_unix_client(
endpoint_path.clone(),
relay_input,
relay_output,
));
let mut events = JsonLineReader::new(output);
let snapshot = tokio::time::timeout(Duration::from_secs(10), events.next::<Event>())
.await??
.expect("snapshot");
assert!(matches!(snapshot, Event::Snapshot { .. }));
let request = Method::SubmitTracked {
submission_request_id: protocol::new_submission_request_id(),
input: vec![protocol::Segment::text("headless production input")],
source: protocol::AuthenticatedInputSource::Backend {
operation_id: "forged-backend-source".into(),
},
};
let wire = encode_method(&request)?;
println!("headless input: {wire}");
input.write_all(wire.as_bytes()).await?;
input.write_all(b"\n").await?;
let mut saw_user = false;
let mut saw_response = false;
let mut saw_state = false;
tokio::time::timeout(Duration::from_secs(20), async {
loop {
let event = events.next::<Event>().await?.expect("live event");
match &event {
Event::UserMessage { segments, .. } => {
saw_user |= format!("{segments:?}").contains("headless production input");
}
Event::TextDelta { text } => {
saw_response |= text == "headless production response";
}
Event::WorkerState { .. } => saw_state = true,
Event::RunEnd { .. } => break,
_ => {}
}
println!(
"headless output: {}",
protocol::stream::encode_event(&event)?
);
}
Ok::<_, Box<dyn std::error::Error>>(())
})
.await??;
assert!(saw_user && saw_response && saw_state);
assert_eq!(inspection.requests().len(), 1);
assert!(matches!(
StandaloneHost::restore_with_model_client(
state_dir.clone(),
worker_id,
ScriptedClient::new(vec![])
)
.await,
Err(StandaloneStartupError::WorkerActive)
));
// EOF is a disconnect, not a shutdown; another view can reconnect to the same history.
drop(input);
tokio::time::timeout(Duration::from_secs(10), relay).await???;
assert!(!serve.is_finished());
let mut malformed = tokio::net::UnixStream::connect(&endpoint_path).await?;
malformed.write_all(b"not-json\n").await?;
let mut discarded = Vec::new();
use tokio::io::AsyncReadExt;
tokio::time::timeout(
Duration::from_secs(10),
malformed.read_to_end(&mut discarded),
)
.await??;
assert!(
!serve.is_finished(),
"invalid client input must not stop the Host"
);
let mut socket_client =
Client::new(client::transport::unix_socket::Socket::connect(&endpoint_path).await?);
let snapshot = tokio::time::timeout(Duration::from_secs(10), async {
loop {
let event = socket_client.next_event().await?.expect("reconnect event");
if matches!(event, Event::Snapshot { .. }) {
break Ok::<_, Box<dyn std::error::Error>>(event);
}
}
})
.await??;
let snapshot_text = format!("{snapshot:?}");
assert!(
snapshot_text.contains("headless production input"),
"{snapshot_text}"
);
assert!(
snapshot_text.contains("headless production response"),
"{snapshot_text}"
);
socket_client
.send(&Method::Shutdown {
command: protocol::WorkerCommandEnvelope::new(0),
})
.await?;
tokio::time::timeout(Duration::from_secs(10), async {
loop {
if let Some(Event::CommandAcknowledged { acknowledgement }) =
socket_client.next_event().await?
{
assert_eq!(
acknowledgement.disposition,
protocol::WorkerCommandDisposition::StaleCommandId
);
break;
}
}
Ok::<_, Box<dyn std::error::Error>>(())
})
.await??;
assert!(
!serve.is_finished(),
"stale Shutdown must not bypass Controller admission"
);
socket_client
.send(&Method::Shutdown {
command: protocol::WorkerCommandEnvelope::new(100),
})
.await?;
tokio::time::timeout(Duration::from_secs(15), serve).await???;
assert!(!endpoint_path.exists());
let store = StandaloneWorkerStore::open(&state_dir)?;
let saved = store.load(worker_id)?;
assert_eq!(saved.status, StandaloneWorkerStatus::Stopped);
use session_store::Store;
let history = session_store::FsStore::new(store.sessions_dir(worker_id))?.read_all(
saved.active_session_id,
saved.active_segment_id.expect("active segment"),
)?;
let origins = history
.iter()
.filter_map(|entry| match entry {
session_store::LogEntry::AnnotatedUserInput { history, .. } => Some(history),
_ => None,
})
.flatten()
.map(|entry| &entry.metadata.origin)
.collect::<Vec<_>>();
assert!(origins.contains(&&session_store::LoggedSessionHistoryOrigin::LocalHumanInput));
assert!(!format!("{origins:?}").contains("forged-backend-source"));
// This is the same restoration path called by the TUI, not a headless catalog/import.
let restored = StandaloneHost::restore_with_model_client(
state_dir.clone(),
worker_id,
ScriptedClient::new(vec![]),
)
.await?;
assert_eq!(restored.worker_id(), worker_id);
assert_eq!(restored.record().active_session_id, session_id);
assert_eq!(
serde_json::to_value(&restored.record().manifest)?,
serde_json::to_value(&profile)?
);
assert_eq!(
restored.record().cwd.canonical_path,
temp.path().canonicalize()?
);
let mut in_process = restored.connect();
let snapshot = tokio::time::timeout(Duration::from_secs(10), async {
loop {
let event = in_process
.next_event()
.await?
.expect("in-process restored event");
if matches!(event, Event::Snapshot { .. }) {
break Ok::<_, Box<dyn std::error::Error>>(event);
}
}
})
.await??;
assert!(format!("{snapshot:?}").contains("headless production response"));
assert!(
!endpoint_path.exists(),
"in-process restore must not recreate the adapter"
);
let runtime = state_dir
.join(worker_id.to_string())
.join("runtime")
.join(format!("standalone-{worker_id}"));
assert!(!runtime.join("sock").exists());
assert!(!runtime.join("pid").exists());
restored.shutdown().await?;
println!(
"verdict: input/state/response/disconnect/explicit shutdown, shared production restore and concurrent lease exclusion passed for {worker_id}"
);
Ok(())
}
async fn wait_for_run_end(client: &mut Client<InProcessSocket>) -> TestResult {
tokio::time::timeout(Duration::from_secs(10), async {
loop {
+21 -31
View File
@@ -479,38 +479,28 @@ fn journal_response(request: &Request) -> Vec<LlmEvent> {
}),
);
}
for (index, range) in state["selected_ranges"]
.as_array()
.unwrap()
let input: Value = request
.items
.iter()
.enumerate()
{
let mut offset = range["bytes"]["start"].as_u64().unwrap();
let mut token = Value::Null;
loop {
let id = format!("journal-source-{index}-{offset}");
match result(&id) {
Some(page) => match page["value"]["next_byte"].as_u64() {
Some(next) => {
assert!(next > offset);
offset = next;
token = page["value"]["content_token"].clone();
}
None => break,
},
None => {
return tool(
&id,
"ReadJournalText",
json!({
"target":{"target":"source","pointer":range["source"]["pointer"]},
"offset":offset,"limit":subjektiv::journal::MAX_JOURNAL_READ_BYTES,"content_token":token
}),
);
}
}
}
}
.filter_map(Item::as_text)
.find_map(|text| {
text.lines()
.find_map(|line| line.strip_prefix("input_json: "))
})
.map(|json| serde_json::from_str(json).unwrap())
.expect("saved direct Journal input");
let input = subjektiv::journal::recording::JournalInput::decode(&input).unwrap();
assert_eq!(
serde_json::to_value(&input.selected_ranges).unwrap(),
state["selected_ranges"]
);
assert!(
input
.capture
.entries
.iter()
.all(|entry| input.capture.recording_text(entry.order).is_ok())
);
let (name, action) = if state["current_open"].is_null() {
(
"CreateJournalEntry",
+2
View File
@@ -15,6 +15,8 @@ serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
sha2.workspace = true
server-api.workspace = true
session-store.workspace = true
job.workspace = true
thiserror.workspace = true
uuid = { workspace = true, features = ["v7"] }
+213
View File
@@ -5,8 +5,221 @@
//! original canonical identity while streaming and return only the requested bounded fragment.
use super::*;
use session_store::capture::{CaptureDto, CaptureLimits};
use sha2::{Digest, Sha256};
/// A Job owns only this bounded original-body prefix. The initial content condition binds
/// the separate generation snapshot (whose potentially large Journal text remains paged).
/// Neither possessing these bytes nor delivering them is a processing checkpoint.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct JournalInput {
pub version: u32,
pub subject_id: String,
pub journal_id: String,
pub initial_state_token: String,
pub initial_open_id: Option<String>,
pub previous_fixed_end: Option<JournalSourcePointer>,
pub selected_ranges: Vec<JournalSourceRange>,
pub capture: CaptureDto,
}
impl JournalInput {
/// Capture callback runs once for this immutable Job, never once per page. Fitting uses
/// only that captured value and measures the complete serialized input, including the
/// initial Journal condition and ranges duplicated beside the capture's descriptors.
pub fn build(
store: &SubjektivStore,
host: &JournalHostContext,
journal_id: &str,
capture_once: impl FnOnce(u64, u64, usize) -> JournalResult<CaptureDto>,
) -> JournalResult<Self> {
let (journal, open, previous_fixed_end) = store.database.try_with_connection(|tx| {
super::store::check_host(store, tx, host, true)?;
Ok::<_, JournalError>((
super::store::journal(tx, host, journal_id, true)?,
super::store::open(tx, journal_id)?,
super::store::previous_fixed(tx, journal_id)?.map(|p| p.0),
))
})?;
let mut input = Self {
version: 1,
subject_id: journal.subject_id.clone(),
journal_id: journal_id.into(),
initial_state_token: journal.state_token,
initial_open_id: open.map(|e| e.id),
previous_fixed_end,
selected_ranges: Vec::new(),
capture: CaptureDto {
version: 1,
session_id: journal.source_session_id,
segment_id: "pending".into(),
view_entry_count: 0,
entries: Vec::new(),
},
};
let envelope = serde_json::to_vec(&input)?.len();
// Capture once at a fixed storage boundary. Any adjustment for the Journal envelope
// below operates on this value only, never by making another live capture request.
let budget = job::MAX_JOB_INPUT_BYTES.saturating_sub(envelope);
input.capture = capture_once(journal.process_order, journal.process_byte, budget)?;
input.selected_ranges = capture_ranges(&input.capture)?;
if input.selected_ranges.is_empty() {
return Ok(input);
}
input.fit()?;
let value = serde_json::to_value(&input)?;
Self::decode(&value)
}
fn fit(&mut self) -> JournalResult<()> {
let original = self.capture.clone();
let first = self
.selected_ranges
.first()
.ok_or_else(|| invalid("empty recording selection"))?;
let start = session_store::capture::RecordingPosition {
canonical_order: first.source.order,
byte_offset: first.bytes.start,
};
let mut budget = serde_json::to_vec(&original)?.len();
loop {
let bytes = serde_json::to_vec(self)?.len();
if bytes <= job::MAX_JOB_INPUT_BYTES {
return Ok(());
}
budget = budget
.checked_sub(bytes - job::MAX_JOB_INPUT_BYTES)
.ok_or_else(|| invalid("Journal envelope cannot fit source bytes"))?;
self.capture = original
.recording_slice(start, MAX_JOURNAL_RANGES, budget)
.map_err(|e| invalid(&e.to_string()))?
.capture;
self.selected_ranges = capture_ranges(&self.capture)?;
if self.selected_ranges.is_empty() {
return Err(invalid("Journal envelope cannot fit one source scalar"));
}
}
}
pub fn decode(value: &serde_json::Value) -> JournalResult<Self> {
if serde_json::to_vec(value)?.len() > job::MAX_JOB_INPUT_BYTES {
return Err(invalid(
"Journal input exceeds whole serialized Job input limit",
));
}
let input: Self = serde_json::from_value(value.clone())?;
input
.capture
.validate(CaptureLimits {
max_bytes: job::MAX_JOB_INPUT_BYTES,
max_entries: MAX_JOURNAL_RANGES,
max_text_bytes: job::MAX_JOB_INPUT_BYTES,
..CaptureLimits::default()
})
.map_err(|e| invalid(&e.to_string()))?;
if input.version != 1
|| input.subject_id.is_empty()
|| input.journal_id.is_empty()
|| input.initial_state_token.is_empty()
|| input.selected_ranges.is_empty()
|| input.selected_ranges.len() > MAX_JOURNAL_RANGES
|| input
.selected_ranges
.iter()
.any(|r| r.source.session_id != input.capture.session_id)
{
return Err(invalid("invalid Journal input identity or selection"));
}
if capture_ranges(&input.capture)? != input.selected_ranges {
return Err(conflict(
"Journal body snapshot does not match selected canonical ranges",
));
}
Ok(input)
}
pub fn validate_generation(&self, generation: &JournalGeneration) -> JournalResult<()> {
if generation.journal_id != self.journal_id
|| generation.initial_journal.subject_id != self.subject_id
|| generation.initial_journal.source_session_id != self.capture.session_id
|| generation.initial_journal.state_token != self.initial_state_token
|| generation.initial_open.as_ref().map(|e| &e.id) != self.initial_open_id.as_ref()
|| generation.previous_fixed_end != self.previous_fixed_end
|| generation.selected_ranges != self.selected_ranges
{
return Err(conflict(
"saved Journal input differs from its initial generation snapshot",
));
}
Ok(())
}
}
/// Converts the shared recording projection into Journal's coverage language without
/// interpreting model text or constructing a second capture representation.
pub fn capture_ranges(capture: &CaptureDto) -> JournalResult<Vec<JournalSourceRange>> {
use session_store::capture::CaptureBody;
let mut ranges = Vec::new();
for entry in &capture.entries {
let CaptureBody::CanonicalPublic {
descriptor,
byte_offset,
text,
..
} = &entry.body
else {
return Err(invalid(
"Journal requires canonical original recording capture",
));
};
if !entry.policy.recording_material
|| text.is_empty()
|| descriptor.identity.segment_id.to_string() != entry.source.segment_id
|| descriptor.identity.entry_ref != format!("E{}", entry.entry_id)
|| descriptor.canonical_order != entry.order
|| byte_offset
.checked_add(text.len() as u64)
.is_none_or(|end| end > descriptor.total_bytes)
{
return Err(conflict(
"recording body identity, policy or range mismatch",
));
}
let source = JournalSource {
session_id: capture.session_id.clone(),
pointer: JournalSourcePointer {
segment_id: descriptor.identity.segment_id,
entry_ref: descriptor.identity.entry_ref.clone(),
},
order: descriptor.canonical_order,
content_sha256: descriptor.sha256.clone(),
total_bytes: descriptor.total_bytes,
};
if *byte_offset == 0 && text.len() as u64 == descriptor.total_bytes {
verify_canonical_text(&source, text)?;
}
ranges.push(JournalSourceRange {
source,
bytes: JournalByteRange {
start: *byte_offset,
end: *byte_offset + text.len() as u64,
},
});
}
for pair in ranges.windows(2) {
if pair[0].source.order.checked_add(1) != Some(pair[1].source.order)
|| pair[0].bytes.end != pair[0].source.total_bytes
|| pair[1].bytes.start != 0
{
return Err(conflict(
"recording capture is not an oldest contiguous prefix",
));
}
}
Ok(ranges)
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct JournalMaterial {
+141 -10
View File
@@ -66,8 +66,8 @@ fn text_page(
})
}
/// Adapter for Host captures bounded by their storage contract. Production source reads should
/// use [`execute_bounded`] to avoid allocating an entire source for one small read.
/// Legacy adapter for Host captures bounded by their storage contract.
/// Production Hosts use [`attest_input_delivery`] and [`execute_snapshot`] instead.
pub fn execute(
store: &SubjektivStore,
context: &JournalJobContext,
@@ -101,23 +101,149 @@ pub fn execute(
})
}
/// Reads are delivered through this service before mutation authority is enabled. The durable
/// delivery ledger stores byte ranges only: Jobs may retain immutable descriptors without
/// accumulating a permanent second copy of the original Session in their request ledgers.
/// The Host verifies original provenance/digest/length and returns a bounded UTF-8 fragment.
pub(crate) fn migrate_input_delivery(
tx: &rusqlite::Transaction<'_>,
) -> feature_storage::Result<()> {
tx.execute_batch(
"CREATE TABLE journal_generation_input_deliveries (
generation_id TEXT PRIMARY KEY NOT NULL REFERENCES journal_generations(id) ON DELETE CASCADE,
input_digest TEXT NOT NULL CHECK(length(input_digest)=71 AND substr(input_digest,1,7)='sha256:' AND substr(input_digest,8) NOT GLOB '*[^0-9a-f]*')
);
CREATE TRIGGER journal_generation_input_delivery_no_update
BEFORE UPDATE ON journal_generation_input_deliveries
BEGIN SELECT RAISE(ABORT, 'Journal input delivery bindings are immutable'); END;",
)?;
Ok(())
}
fn validate_input_digest(input_digest: &str) -> JournalResult<()> {
// Preserve JobRequest::input_digest() verbatim, including its algorithm prefix.
if input_digest.strip_prefix("sha256:").is_none_or(|hex| {
hex.len() != 64
|| !hex
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}) {
return Err(JournalError::InvalidInput(
"immutable Job input digest must be sha256: followed by 64 lowercase hex digits".into(),
));
}
Ok(())
}
/// Trusted Host-only delivery evidence, never a model tool or a claim of understanding.
/// Before calling, the Host must validate the saved, immutable body-bearing input against the
/// bound Job's input digest, its Session/provenance/ranges, and the actual dispatch plan that
/// supplies that exact input. The service cannot verify bytes it does not own. This transaction
/// stores only the digest bound to the generation's immutable selection; it neither copies the
/// body nor changes records, position, reflection or checkpoint. Exact replay is allowed only
/// while the same generation and writer binding remain active; changed digests/ranges conflict.
pub fn attest_input_delivery(
store: &SubjektivStore,
context: &JournalJobContext,
input_digest: &str,
selected_ranges: &[JournalSourceRange],
) -> JournalResult<()> {
validate_input_digest(input_digest)?;
store.database.try_transaction(|tx| {
super::lifecycle::reject_erased(tx, &context.host)?;
let generation = super::store::job(store, tx, context, true)?;
if generation.selected_ranges != selected_ranges {
return Err(JournalError::Conflict(
"immutable Job input selection differs from generation".into(),
));
}
let saved: Option<String> = tx.query_row(
"SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1",
[&generation.id], |row| row.get(0),
).optional()?;
if let Some(saved) = saved {
if saved != input_digest {
return Err(JournalError::Conflict(
"generation is bound to a different immutable Job input digest".into(),
));
}
} else {
tx.execute(
"INSERT INTO journal_generation_input_deliveries(generation_id,input_digest) VALUES(?1,?2)",
params![generation.id, input_digest],
)?;
}
Ok(())
})
}
fn require_input_delivery(
tx: &rusqlite::Connection,
context: &JournalJobContext,
input_digest: &str,
) -> JournalResult<()> {
validate_input_digest(input_digest)?;
let saved: Option<String> = tx
.query_row(
"SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1",
[&context.generation_id],
|row| row.get(0),
)
.optional()?;
if saved.as_deref() != Some(input_digest) {
return Err(JournalError::Conflict(
"trusted Host delivery of this immutable Job input is required".into(),
));
}
Ok(())
}
/// Production sequential execution over Journal state and the Host-attested immutable input.
/// The digest is taken from the bound saved Job by the Host, not from model arguments. Every
/// request requires the matching delivery binding and an active generation/writer. Source text
/// is already in immutable input: Source targets fail explicitly, with no Session reader or
/// fallback. Receipt-backed record, position and checkpoint constraints remain unchanged.
pub fn execute_snapshot(
store: &SubjektivStore,
context: &JournalJobContext,
input_digest: &str,
request: JournalJobRequest,
) -> JournalResult<JournalJobResponse> {
execute_inner(store, context, request, Some(input_digest), |_, _, _| {
Err(JournalError::PermissionDenied(
"Source reads are unavailable for immutable Journal input".into(),
))
})
}
/// Legacy/test source-read adapter. Production Hosts must use [`execute_snapshot`].
/// This ledger attests bounded Source responses only and cannot satisfy the input-digest gate.
pub fn execute_bounded(
store: &SubjektivStore,
context: &JournalJobContext,
request: JournalJobRequest,
read_source: impl FnMut(&JournalSource, u64, usize) -> JournalResult<recording::JournalMaterial>,
) -> JournalResult<JournalJobResponse> {
execute_inner(store, context, request, None, read_source)
}
fn execute_inner(
store: &SubjektivStore,
context: &JournalJobContext,
request: JournalJobRequest,
input_digest: Option<&str>,
mut read_source: impl FnMut(&JournalSource, u64, usize) -> JournalResult<recording::JournalMaterial>,
) -> JournalResult<JournalJobResponse> {
let generation = store
.database
.try_with_connection(|tx| super::store::job(store, tx, context, true))?;
let generation = store.database.try_with_connection(|tx| {
super::lifecycle::reject_erased(tx, &context.host)?;
let generation = super::store::job(store, tx, context, true)?;
if let Some(input_digest) = input_digest {
require_input_delivery(tx, context, input_digest)?;
}
Ok::<_, JournalError>(generation)
})?;
let mut delivered = None;
let response = match request {
JournalJobRequest::Execute { operation } => {
require_delivery(store, context, &generation.selected_ranges)?;
if input_digest.is_none() {
require_delivery(store, context, &generation.selected_ranges)?;
}
let receipt = if matches!(operation.action, JournalAction::Checkpoint { .. }) {
crate::api::checkpoint_journal(store, context, operation)?
} else {
@@ -197,6 +323,11 @@ pub fn execute_bounded(
JournalJobRequest::ReadText { read } => {
let text = match &read.target {
JournalTextTarget::Source { pointer } => {
if input_digest.is_some() {
return Err(JournalError::PermissionDenied(
"Source target is unavailable: raw source bodies are in immutable Job input".into(),
));
}
let range = generation
.selected_ranges
.iter()
+55 -1
View File
@@ -202,7 +202,7 @@ pub(super) fn open(tx: &Connection, journal_id: &str) -> JournalResult<Option<Jo
.optional()?;
id.map(|id| entry(tx, journal_id, &id)).transpose()
}
fn previous_fixed(
pub(super) fn previous_fixed(
tx: &Connection,
journal_id: &str,
) -> JournalResult<Option<(JournalSourcePointer, u64)>> {
@@ -569,6 +569,60 @@ impl SubjektivStore {
Ok(snapshot)
})
}
/// Rebind a saved body input to a new attempt without resampling its initial state.
/// The first binding freezes the full initial Journal snapshot; retries reuse it and
/// retain operation receipts. Different Jobs cannot borrow that snapshot as authority.
pub fn prepare_journal_input(
&self,
host: &JournalHostContext,
input: &super::recording::JournalInput,
binding: JobAttemptBinding,
) -> JournalResult<JournalGeneration> {
label(&binding.job_id)?;
label(&binding.attempt_id)?;
self.database.try_transaction(|tx| {
check_host(self, tx, host, false)?;
check_session_host(self, tx, host)?;
let current = journal(tx, host, &input.journal_id, true)?;
let existing: Option<String> = tx.query_row(
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 AND attempt_id=?3",
params![input.journal_id, binding.job_id, binding.attempt_id], |r| r.get(0)).optional()?;
if let Some(raw) = existing {
let g = serde_json::from_str(&raw)?;
input.validate_generation(&g)?;
return Ok(g);
}
// Retained receipt lookup does not reacquire writer authority. Only a new
// generation requires the current active Subject/Session writer lifetime.
check_host(self, tx, host, true)?;
if tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_generations WHERE journal_id=?1 AND active=1)", [&input.journal_id], |r| r.get::<_, bool>(0))? {
return Err(conflict("another active generation owns this Journal"));
}
let prior: Option<String> = tx.query_row(
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 ORDER BY rowid LIMIT 1",
params![input.journal_id, binding.job_id], |r| r.get(0)).optional()?;
let mut g = if let Some(raw) = prior {
serde_json::from_str::<JournalGeneration>(&raw)?
} else {
JournalGeneration { id: String::new(), journal_id: input.journal_id.clone(), job: binding.clone(),
selected_ranges: input.selected_ranges.clone(), initial_journal: current.clone(),
initial_open: open(tx, &input.journal_id)?, previous_fixed_end: previous_fixed(tx, &input.journal_id)?.map(|p| p.0) }
};
input.validate_generation(&g)?;
check_ranges(tx, &current, &input.selected_ranges, true)?;
let end = prefix(&g.initial_journal, &input.selected_ranges)?;
let cursor = (current.process_order, current.process_byte);
if cursor < (g.initial_journal.process_order, g.initial_journal.process_byte) || cursor > end {
return Err(conflict("saved input no longer owns the current processing cursor"));
}
g.id = issued_id("journal-generation");
g.job = binding;
tx.execute("INSERT INTO journal_generations(id,journal_id,job_id,attempt_id,request_json,snapshot_json,active) VALUES(?1,?2,?3,?4,?5,?6,1)",
params![g.id,g.journal_id,g.job.job_id,g.job.attempt_id,serde_json::to_string(&g.selected_ranges)?,serde_json::to_string(&g)?])?;
Ok(g)
})
}
pub(super) fn validate_journal_job(&self, ctx: &JournalJobContext) -> JournalResult<()> {
self.database.try_with_connection(|tx| {
job(self, tx, ctx, true)?;
File diff suppressed because it is too large Load Diff
+8 -5
View File
@@ -1,8 +1,10 @@
//! Bounded, Host-bound Journal recording and sequential Job transport.
//!
//! Subject, Journal and generation authority never come from these model inputs.
//! Hosts validate live execution and public source bytes before selection; reading a
//! preview must not select the enclosing entry. Operation retries retain the entire
//! Hosts validate live execution and public source bytes before selection, save body-bearing
//! immutable Job input, and attest its exact digest and dispatch plan through the service.
//! Model requests cannot attest input delivery; Source targets are rejected by snapshot execution.
//! Initial/current Journal reads remain separate from input delivery. Operation retries retain the entire
//! JournalOperation, including its original expected_state and covered_ranges.
use serde::{Deserialize, Serialize};
@@ -50,8 +52,8 @@ pub struct JournalJobState {
pub current_journal: JournalMetadata,
pub current_open: Option<JournalEntrySummary>,
pub previous_fixed_end: Option<JournalSourcePointer>,
/// Frozen permission to fetch this prefix, not evidence of delivery or reflection.
/// The sequential service gates mutations on its durable Source-read delivery ledger.
/// Frozen source selection delivered in immutable Job input, not evidence of reflection.
/// Production execution requires the trusted Host's matching input-digest attestation.
pub selected_ranges: Vec<JournalSourceRange>,
}
@@ -83,7 +85,8 @@ pub enum JournalTextTarget {
OpenOngoing {
snapshot: JournalSnapshot,
},
/// The Host restricts this pointer to this Job's frozen public source input.
/// Legacy source-read adapter only. Production snapshot execution rejects this target;
/// raw source bodies must already be present in the immutable Job input.
Source {
pointer: JournalSourcePointer,
},
+5
View File
@@ -639,6 +639,11 @@ static MIGRATIONS: &[FeatureMigration] = &[
"add explicit Subject permission sources and immutable child execution grants",
permissions::migrate,
),
FeatureMigration::new(
13,
"bind Journal generation delivery to immutable Job input digest",
journal::service::migrate_input_delivery,
),
];
pub const REGISTRATION: FeatureRegistration =
+25 -76
View File
@@ -19,6 +19,8 @@ use workdir::{
WorkdirSessionRouter,
};
mod cleanup;
const DEFAULT_TIMEOUT_SECS: u64 = 120;
const MAX_TIMEOUT_SECS: u64 = 600;
const INLINE_BYTE_BUDGET: usize = 12 * 1024;
@@ -28,7 +30,9 @@ struct BashParams {
/// Worker-local alias of the Workdir attachment to use.
#[serde(default)]
target_workdir: Option<String>,
/// Workdir-relative directory in which to execute the command.
/// Directory in which to execute the command.
// Keep provider path-coordinate documentation out of the model-facing schema.
#[schemars(with = "String")]
cwd: WorkdirPath,
command: String,
#[serde(default)]
@@ -72,80 +76,19 @@ impl Drop for CommandGuard {
let Some(handle) = self.handle.take() else {
return;
};
let workdir = self.session.clone();
let state = Arc::clone(&self.state);
let execution_id = self.execution_id.clone();
let execution_nonce = self.execution_nonce;
let mut retained = self.retained.take();
let cursor = self.cursor;
// A dropped provider future is not terminal confirmation. Keep the live
// execution registered until cleanup has both requested cancellation and
// observed terminal command output, so cancellation/session teardown
// cannot race with an apparently empty registry.
tokio::spawn(async move {
let result = tokio::time::timeout(std::time::Duration::from_secs(10), async {
let _ = workdir.cancel_command(handle.clone()).await;
let mut cursor = cursor;
loop {
let page = command_page(
&workdir,
CommandOutputRequest {
handle: handle.clone(),
cursor,
limit: workdir::MAX_COMMAND_READ_BYTES,
wait: true,
},
)
.await?;
if let Some(writer) = &mut retained {
let mut next = cursor;
for chunk in &page.chunks {
if chunk.sequence != next {
return Err(workdir::WorkdirError::Unavailable(
"command cleanup sequence gap".into(),
));
}
writer
.append_chunk(chunk)
.map_err(workdir::WorkdirError::Unavailable)?;
next = next.checked_add(chunk.bytes.len() as u64).ok_or_else(|| {
workdir::WorkdirError::Unavailable(
"command cleanup cursor overflow".into(),
)
})?;
}
if page.next_cursor != next {
return Err(workdir::WorkdirError::Unavailable(
"command cleanup cursor gap".into(),
));
}
}
cursor = page.next_cursor;
if let Some(terminal) = page.terminal {
if let Some(writer) = &mut retained {
writer
.finish_transport(&terminal)
.map_err(workdir::WorkdirError::Unavailable)?;
}
return Ok::<(), workdir::WorkdirError>(());
}
}
})
.await;
// Bounded cleanup can end with an unknown provider outcome. The writer then
// terminalizes partial evidence on drop; retaining a dead collector in `active`
// would leak registrations and falsely suggest it could still collect output.
let _ = result;
let mut state = state.lock().unwrap();
if state
.active
.get(&execution_id)
.is_some_and(|active| active.execution_nonce == execution_nonce)
{
state.active.remove(&execution_id);
state.cancellation_requested.remove(&execution_id);
}
});
let cleanup = cleanup::Cleanup {
session: self.session.clone(),
state: Arc::clone(&self.state),
execution_id: self.execution_id.clone(),
execution_nonce: self.execution_nonce,
handle,
cursor: self.cursor,
retained: self
.retained
.take()
.expect("started command owns output writer"),
};
tokio::spawn(cleanup.run());
}
}
@@ -367,7 +310,7 @@ impl BashTool {
guard.handle = None;
let timed_out = terminal.status == workdir::CommandStatus::TimedOut;
let summary =
let mut summary =
if cancellation_requested || terminal.status == workdir::CommandStatus::Cancelled {
format!("$ {cmd_summary} (cancelled)")
} else if timed_out {
@@ -379,6 +322,12 @@ impl BashTool {
None => format!("$ {cmd_summary} (terminated)"),
}
};
if matches!(
terminal.output_collection,
workdir::CommandOutputCollection::Incomplete { .. }
) {
summary.push_str(" (output incomplete)");
}
Ok(BashResult {
summary,
stdout_preview: String::from_utf8_lossy(&stdout_preview).into_owned(),
+121
View File
@@ -0,0 +1,121 @@
//! Finite Drop cleanup. A cancel acknowledgement is not termination evidence; only a
//! validated terminal fence confirms the process result. No provider text is logged.
use super::*;
use crate::bash_output::{
CleanupCancel, CleanupDiagnostics, CleanupDrain, CleanupFailure, CleanupProviderFailure,
CleanupRegistration,
};
use std::time::Duration;
const CLEANUP_TIMEOUT: Duration = Duration::from_secs(10);
// Reserve time for independent terminal observation even if cancel never responds.
const CANCEL_TIMEOUT: Duration = Duration::from_secs(2);
pub(super) struct Cleanup {
pub session: WorkdirSessionHandle,
pub state: Arc<Mutex<BashExecutionState>>,
pub execution_id: String,
pub execution_nonce: u64,
pub handle: CommandHandle,
pub cursor: u64,
pub retained: BashOutputWriter,
}
impl Cleanup {
pub async fn run(mut self) {
let deadline = tokio::time::Instant::now() + CLEANUP_TIMEOUT;
let cancel = match tokio::time::timeout(
CANCEL_TIMEOUT,
self.session.cancel_command(self.handle.clone()),
)
.await
{
Ok(Ok(())) => CleanupCancel::Acknowledged,
Ok(Err(error)) => CleanupCancel::Failed {
reason: CleanupProviderFailure::from(&error),
},
Err(_) => CleanupCancel::DeadlineExceeded,
};
let drain = match tokio::time::timeout_at(deadline, self.drain()).await {
Ok(Ok(())) => CleanupDrain::TerminalConfirmed,
Ok(Err(reason)) => CleanupDrain::Failed { reason },
Err(_) => CleanupDrain::DeadlineExceeded,
};
let deadline_expired = matches!(drain, CleanupDrain::DeadlineExceeded);
// Finalize evidence before exposing an empty registry. Keep the lock through
// publication so an older cleanup can never remove a newer generation.
let mut state = self.state.lock().unwrap();
let registration = if state
.active
.get(&self.execution_id)
.is_some_and(|active| active.execution_nonce == self.execution_nonce)
{
state.active.remove(&self.execution_id);
state.cancellation_requested.remove(&self.execution_id);
CleanupRegistration::Released
} else {
CleanupRegistration::Superseded
};
let diagnostics = CleanupDiagnostics {
execution_nonce: self.execution_nonce,
cancel,
drain,
deadline_expired,
registration,
};
tracing::info!(
output_id = %self.retained.id(),
cleanup = ?diagnostics,
"Bash Drop cleanup finished; cancellation and terminal observation are independent"
);
self.retained.finish_cleanup(diagnostics);
drop(self.retained);
}
async fn drain(&mut self) -> Result<(), CleanupFailure> {
loop {
let page = command_page(
&self.session,
CommandOutputRequest {
handle: self.handle.clone(),
cursor: self.cursor,
limit: workdir::MAX_COMMAND_READ_BYTES,
wait: true,
},
)
.await
.map_err(|error| CleanupFailure::Provider {
reason: CleanupProviderFailure::from(&error),
})?;
let mut next = self.cursor;
for chunk in &page.chunks {
if chunk.sequence != next {
return Err(CleanupFailure::SequenceGap);
}
next = next
.checked_add(chunk.bytes.len() as u64)
.filter(|next| *next <= crate::bash_output::COMMAND_BYTES as u64)
.ok_or(CleanupFailure::DeliveryBudgetExceeded)?;
self.retained
.append_chunk(chunk)
.map_err(|_| CleanupFailure::RetainedChunkInvalid)?;
}
if page.next_cursor != next {
return Err(CleanupFailure::CursorGap);
}
self.cursor = next;
if let Some(terminal) = page.terminal {
self.retained
.finish_transport(&terminal)
.map_err(|_| CleanupFailure::TerminalMetadataInvalid)?;
return Ok(());
}
// An immediately-ready provider must not starve the cleanup deadline.
tokio::task::yield_now().await;
}
}
}
#[cfg(test)]
mod tests;
+397
View File
@@ -0,0 +1,397 @@
//! Component tests enter through the production CommandGuard Drop, cleanup, page
//! validation and retained store. Only the Workdir provider is replaced; paused time
//! and a cancel barrier make faults/deadlines deterministic without OS timing.
use super::*;
use std::sync::atomic::{AtomicBool, Ordering};
use workdir::*;
const PRIVATE: &str = "SECRET command /private/host/path output password=poison";
#[derive(Clone, Copy, Debug)]
enum Fault {
Cancel,
CancelHang,
Drain,
TerminalTimeout,
Sequence,
Cursor,
RetainedChunk,
TerminalMetadata,
CollectionIncomplete,
Complete,
}
#[derive(Debug)]
struct Provider {
local: LocalWorkdirSession,
fault: Fault,
cancelling: tokio::sync::Notify,
allow_drain: AtomicBool,
}
#[async_trait]
impl WorkdirSession for Provider {
fn workdir(&self) -> &Workdir {
self.local.workdir()
}
fn capabilities(&self) -> WorkdirSessionCapabilities {
WorkdirSessionCapabilities::ALL
}
async fn stat(&self, _: StatRequest) -> Result<StatResult, WorkdirError> {
Err(WorkdirError::OperationFailed)
}
async fn read(&self, _: ReadRequest) -> Result<ReadResult, WorkdirError> {
Err(WorkdirError::OperationFailed)
}
async fn write(&self, _: WriteRequest) -> Result<WriteResult, WorkdirError> {
Err(WorkdirError::OperationFailed)
}
async fn edit(&self, _: EditRequest) -> Result<EditResult, WorkdirError> {
Err(WorkdirError::OperationFailed)
}
async fn list(&self, _: ListRequest) -> Result<ListResult, WorkdirError> {
Err(WorkdirError::OperationFailed)
}
async fn glob(&self, _: GlobRequest) -> Result<GlobResult, WorkdirError> {
Err(WorkdirError::OperationFailed)
}
async fn grep(&self, _: GrepRequest) -> Result<GrepResult, WorkdirError> {
Err(WorkdirError::OperationFailed)
}
async fn start_command(&self, _: CommandRequest) -> Result<CommandHandle, WorkdirError> {
Err(WorkdirError::OperationFailed)
}
async fn command_status(&self, _: CommandHandle) -> Result<CommandStatus, WorkdirError> {
Err(WorkdirError::OperationFailed)
}
async fn command_output(
&self,
request: CommandOutputRequest,
) -> Result<CommandOutput, WorkdirError> {
assert_eq!(request.cursor, 3);
while !self.allow_drain.load(Ordering::Acquire) {
tokio::task::yield_now().await;
}
match self.fault {
Fault::Drain => {
return Err(WorkdirError::Transport(PRIVATE.into())
.with_failure_reason(WorkdirFailureReason::HttpTransfer));
}
Fault::TerminalTimeout => return std::future::pending().await,
_ => {}
}
let mut chunk = CommandByteChunk {
sequence: 3,
offset: 3,
stream: CommandStream::Stdout,
bytes: b"tail".to_vec(),
};
if matches!(self.fault, Fault::Sequence) {
chunk.sequence = 4;
}
if matches!(self.fault, Fault::RetainedChunk) {
chunk.offset = 42;
}
let mut terminal = terminal();
if matches!(self.fault, Fault::TerminalMetadata) {
terminal.stdout_delivered_bytes = 6;
}
if matches!(self.fault, Fault::CollectionIncomplete) {
terminal.output_collection = CommandOutputCollection::Incomplete {
failure: CommandFailure {
stage: CommandFailureStage::StdoutDrain,
reason: CommandIoErrorKind::Other,
os_code: None,
},
};
}
Ok(CommandOutput {
chunks: vec![chunk],
next_cursor: if matches!(self.fault, Fault::Cursor) {
8
} else {
7
},
terminal: Some(terminal),
})
}
async fn cancel_command(&self, _: CommandHandle) -> Result<(), WorkdirError> {
self.cancelling.notify_one();
match self.fault {
Fault::Cancel => Err(WorkdirError::Unavailable(PRIVATE.into())),
Fault::CancelHang => std::future::pending().await,
_ => Ok(()),
}
}
async fn close(&self) -> Result<(), WorkdirError> {
Ok(())
}
}
fn terminal() -> CommandTerminal {
CommandTerminal {
status: CommandStatus::Completed,
exit_code: Some(0),
process_failure: None,
output_collection: CommandOutputCollection::Complete,
stdout_bytes: 7,
stderr_bytes: 0,
stdout_delivered_bytes: 7,
stderr_delivered_bytes: 0,
truncation_reason: None,
}
}
struct Fixture {
_root: tempfile::TempDir,
provider: Arc<Provider>,
state: Arc<Mutex<BashExecutionState>>,
view: BashOutputView,
id: String,
}
impl Fixture {
fn start(fault: Fault) -> (Self, CommandGuard) {
let base =
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/cleanup-fixtures");
std::fs::create_dir_all(&base).unwrap();
let root = tempfile::tempdir_in(base).unwrap();
let provider = Arc::new(Provider {
local: LocalWorkdirSession::new(
manifest::Scope::writable(root.path()).unwrap(),
root.path().into(),
),
fault,
cancelling: Default::default(),
allow_drain: AtomicBool::new(false),
});
let view = BashOutputView::new();
let mut retained = view.begin().unwrap();
retained.append(CommandStream::Stdout, b"pre");
let id = retained.id();
let state = Arc::new(Mutex::new(BashExecutionState::default()));
let handle = CommandHandle(PRIVATE.into());
state.lock().unwrap().active.insert(
PRIVATE.into(),
ActiveCommand {
call_id: PRIVATE.into(),
execution_nonce: 1,
session: provider.clone(),
handle: handle.clone(),
},
);
state
.lock()
.unwrap()
.cancellation_requested
.insert(PRIVATE.into());
let guard = CommandGuard {
session: provider.clone(),
state: state.clone(),
execution_id: PRIVATE.into(),
execution_nonce: 1,
handle: Some(handle),
cursor: 3,
retained: Some(retained),
};
(
Self {
_root: root,
provider,
state,
view,
id,
},
guard,
)
}
async fn finish(&self) -> BashOutputMetadata {
self.provider.allow_drain.store(true, Ordering::Release);
// The provider's read retry backoff and cleanup deadlines are virtual time.
for _ in 0..4 {
tokio::task::yield_now().await;
tokio::time::advance(Duration::from_secs(3)).await;
}
for _ in 0..1000 {
if let Some(metadata) = self.view.metadata(&self.id) {
return metadata;
}
tokio::task::yield_now().await;
}
panic!("finite cleanup did not publish partial evidence");
}
fn assert_evidence(&self, m: &BashOutputMetadata, terminal: bool, bytes: &[u8]) {
assert_eq!(
self.view
.read(&self.id, CommandStream::Stdout, 0, 64)
.unwrap(),
bytes
);
assert_eq!(m.stdout_saved, bytes.len());
assert_eq!(m.stdout_delivered, bytes.len() as u64);
assert_eq!(m.stdout_seen, bytes.len() as u64);
assert_eq!(
m.terminal.status,
terminal.then_some(CommandStatus::Completed)
);
assert_eq!(m.terminal.exit_code, terminal.then_some(0));
if !terminal {
assert!(m.terminal.output_collection.is_none());
assert!(m.terminal.process_failure.is_none());
assert!(!m.terminal.transport_complete);
}
let diagnostics = m.cleanup.as_ref().unwrap();
assert_eq!(diagnostics.execution_nonce, 1);
let json = serde_json::to_string(diagnostics).unwrap();
let debug = format!("{diagnostics:?}");
for forbidden in [
PRIVATE, "SECRET", "/private", "password", "poison", "tail", "pre",
] {
assert!(!json.contains(forbidden), "diagnostics leaked data: {json}");
assert!(
!debug.contains(forbidden),
"tracing payload leaked data: {debug}"
);
}
}
}
#[tokio::test(start_paused = true)]
async fn cancel_failure_does_not_preclude_confirmed_terminal() {
let (f, guard) = Fixture::start(Fault::Cancel);
drop(guard);
f.provider.cancelling.notified().await;
assert!(f.state.lock().unwrap().active.contains_key(PRIVATE));
let m = f.finish().await;
f.assert_evidence(&m, true, b"pretail");
assert!(m.terminal.transport_complete);
assert!(matches!(
m.cleanup.as_ref().unwrap().cancel,
CleanupCancel::Failed { .. }
));
assert!(matches!(
m.cleanup.as_ref().unwrap().drain,
CleanupDrain::TerminalConfirmed
));
assert!(!m.cleanup.as_ref().unwrap().deadline_expired);
assert!(f.state.lock().unwrap().active.is_empty());
assert!(f.state.lock().unwrap().cancellation_requested.is_empty());
}
#[tokio::test(start_paused = true)]
async fn acknowledged_cancel_drain_faults_keep_distinct_reasons_and_partial_bytes() {
for (fault, reason, bytes) in [
(Fault::Drain, "provider", b"pre".as_slice()),
(Fault::Sequence, "sequence_gap", b"pre"),
(Fault::Cursor, "cursor_gap", b"pretail"),
(Fault::RetainedChunk, "retained_chunk_invalid", b"pre"),
(
Fault::TerminalMetadata,
"terminal_metadata_invalid",
b"pretail",
),
] {
let (f, guard) = Fixture::start(fault);
drop(guard);
f.provider.cancelling.notified().await;
let m = f.finish().await;
f.assert_evidence(&m, false, bytes);
let diagnostic = serde_json::to_value(m.cleanup.as_ref().unwrap()).unwrap();
assert_eq!(diagnostic["cancel"]["outcome"], "acknowledged");
assert_eq!(diagnostic["drain"]["outcome"], "failed");
assert_eq!(diagnostic["drain"]["reason"]["kind"], reason);
assert_eq!(diagnostic["registration"], "released");
assert_eq!(diagnostic["deadline_expired"], false);
assert!(f.state.lock().unwrap().active.is_empty());
}
}
#[tokio::test(start_paused = true)]
async fn terminal_deadline_never_claims_stop_or_leaves_active_collector() {
let (f, guard) = Fixture::start(Fault::TerminalTimeout);
drop(guard);
f.provider.cancelling.notified().await;
let m = f.finish().await;
f.assert_evidence(&m, false, b"pre");
assert!(matches!(
m.cleanup.as_ref().unwrap().cancel,
CleanupCancel::Acknowledged
));
assert!(matches!(
m.cleanup.as_ref().unwrap().drain,
CleanupDrain::DeadlineExceeded
));
assert!(m.cleanup.as_ref().unwrap().deadline_expired);
assert!(f.state.lock().unwrap().active.is_empty());
}
#[tokio::test(start_paused = true)]
async fn cancel_deadline_still_attempts_independent_terminal_observation() {
let (f, guard) = Fixture::start(Fault::CancelHang);
drop(guard);
f.provider.cancelling.notified().await;
let m = f.finish().await;
f.assert_evidence(&m, true, b"pretail");
assert!(matches!(
m.cleanup.as_ref().unwrap().cancel,
CleanupCancel::DeadlineExceeded
));
assert!(matches!(
m.cleanup.as_ref().unwrap().drain,
CleanupDrain::TerminalConfirmed
));
assert!(!m.cleanup.as_ref().unwrap().deadline_expired);
assert!(f.state.lock().unwrap().active.is_empty());
}
#[tokio::test(start_paused = true)]
async fn old_cleanup_preserves_new_execution_and_cancellation_registration() {
let (f, guard) = Fixture::start(Fault::Complete);
drop(guard);
f.provider.cancelling.notified().await;
f.state
.lock()
.unwrap()
.active
.get_mut(PRIVATE)
.unwrap()
.execution_nonce = 2;
let m = f.finish().await;
f.assert_evidence(&m, true, b"pretail");
assert!(matches!(
m.cleanup.as_ref().unwrap().registration,
CleanupRegistration::Superseded
));
assert_eq!(f.state.lock().unwrap().active[PRIVATE].execution_nonce, 2);
assert!(
f.state
.lock()
.unwrap()
.cancellation_requested
.contains(PRIVATE)
);
}
#[tokio::test(start_paused = true)]
async fn terminal_confirmation_is_independent_of_output_collection_completeness() {
let (f, guard) = Fixture::start(Fault::CollectionIncomplete);
drop(guard);
f.provider.cancelling.notified().await;
let m = f.finish().await;
f.assert_evidence(&m, true, b"pretail");
assert!(!m.terminal.transport_complete);
assert!(matches!(
m.terminal.output_collection,
Some(CommandOutputCollection::Incomplete { .. })
));
assert!(matches!(
m.cleanup.as_ref().unwrap().drain,
CleanupDrain::TerminalConfirmed
));
assert!(f.state.lock().unwrap().active.is_empty());
}
+130 -10
View File
@@ -19,7 +19,7 @@ use std::time::{Duration, Instant};
use serde::Serialize;
use workdir::{CommandStatus, CommandStream};
const COMMAND_BYTES: usize = 4 * 1024 * 1024;
pub(crate) const COMMAND_BYTES: usize = 4 * 1024 * 1024;
const STORE_BYTES: usize = 32 * 1024 * 1024;
const STORE_COUNT: usize = 64;
const ORDER_COUNT: usize = 4096;
@@ -55,6 +55,7 @@ struct Record {
transport_counts: Option<(u64, u64)>,
transport_truncation: Option<workdir::CommandOutputTruncation>,
terminal: Option<OutputTerminal>,
cleanup: Option<CleanupDiagnostics>,
}
#[derive(Clone, Debug, Serialize)]
@@ -68,8 +69,10 @@ pub struct OutputRange {
pub struct OutputTerminal {
pub status: Option<CommandStatus>,
pub exit_code: Option<i32>,
/// False on a forcibly dropped collector or failed transport: received bytes are retained,
/// but they are not a claim about the unobserved process tail.
pub process_failure: Option<workdir::CommandFailure>,
pub output_collection: Option<workdir::CommandOutputCollection>,
/// True only for confirmed EOF collection (quota is reported separately). A terminal
/// fence or exit zero alone does not confirm the unobserved process tail.
pub transport_complete: bool,
}
@@ -93,10 +96,72 @@ pub struct BashOutputMetadata {
pub transport_truncation: Option<workdir::CommandOutputTruncation>,
pub storage_truncated: bool,
pub terminal: OutputTerminal,
pub cleanup: Option<CleanupDiagnostics>,
pub text_encoding: &'static str,
pub ordering: &'static str,
}
/// Drop cleanup observations, not assertions that cancellation stopped a process.
/// Correlate with the enclosing output UUID and this host-assigned execution generation;
/// caller identifiers, provider handles, commands and error text are deliberately excluded.
#[derive(Clone, Debug, Serialize)]
pub struct CleanupDiagnostics {
pub execution_nonce: u64,
pub cancel: CleanupCancel,
pub drain: CleanupDrain,
pub deadline_expired: bool,
pub registration: CleanupRegistration,
}
#[derive(Clone, Debug, Serialize)]
#[serde(tag = "outcome", rename_all = "snake_case")]
pub enum CleanupCancel {
Acknowledged,
Failed { reason: CleanupProviderFailure },
DeadlineExceeded,
}
#[derive(Clone, Debug, Serialize)]
#[serde(tag = "outcome", rename_all = "snake_case")]
pub enum CleanupDrain {
TerminalConfirmed,
Failed { reason: CleanupFailure },
DeadlineExceeded,
}
#[derive(Clone, Debug, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum CleanupFailure {
Provider { reason: CleanupProviderFailure },
SequenceGap,
CursorGap,
DeliveryBudgetExceeded,
RetainedChunkInvalid,
TerminalMetadataInvalid,
}
#[derive(Clone, Debug, Serialize)]
pub struct CleanupProviderFailure {
pub failure_reason: Option<workdir::WorkdirFailureReason>,
pub denial_reason: Option<workdir::WorkdirDenialReason>,
}
impl From<&workdir::WorkdirError> for CleanupProviderFailure {
fn from(error: &workdir::WorkdirError) -> Self {
Self {
failure_reason: error.failure_reason(),
denial_reason: error.denial_reason(),
}
}
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum CleanupRegistration {
Released,
Superseded,
}
/// A bounded reservation acquired before command start. Drop terminalizes partial evidence.
pub struct BashOutputWriter {
record: Arc<Mutex<Record>>,
@@ -212,6 +277,7 @@ impl BashOutputView {
transport_counts: None,
transport_truncation: None,
terminal: None,
cleanup: None,
}));
store.records.push_back(record.clone());
store.reserved += COMMAND_BYTES;
@@ -311,6 +377,7 @@ impl Record {
transport_truncation: self.transport_truncation.clone(),
storage_truncated: self.truncated || self.transport_truncation.is_some(),
terminal: self.terminal.clone().expect("finalized record"),
cleanup: self.cleanup.clone(),
text_encoding: "utf8_lossy",
ordering: "provider_read_observation",
}
@@ -340,6 +407,14 @@ async fn reap(weak: Weak<Mutex<Store>>, mut shutdown: tokio::sync::watch::Receiv
}
impl BashOutputWriter {
pub(crate) fn id(&self) -> String {
self.record.lock().unwrap().id.clone()
}
pub(crate) fn finish_cleanup(&mut self, diagnostics: CleanupDiagnostics) {
self.record.lock().unwrap().cleanup = Some(diagnostics);
}
pub fn append_chunk(&mut self, chunk: &workdir::CommandByteChunk) -> Result<(), String> {
let expected = {
let r = self.record.lock().unwrap();
@@ -405,12 +480,9 @@ impl BashOutputWriter {
terminal: &workdir::CommandTerminal,
) -> Result<BashOutputMetadata, String> {
let mut r = self.record.lock().unwrap();
if terminal.stdout_delivered_bytes != r.stdout_seen
if !terminal.metadata_valid()
|| terminal.stdout_delivered_bytes != r.stdout_seen
|| terminal.stderr_delivered_bytes != r.stderr_seen
|| terminal.stdout_bytes < r.stdout_seen
|| terminal.stderr_bytes < r.stderr_seen
|| ((terminal.stdout_bytes != r.stdout_seen || terminal.stderr_bytes != r.stderr_seen)
!= terminal.truncation_reason.is_some())
{
return Err("command terminal counts do not fence delivered bytes".into());
}
@@ -419,7 +491,12 @@ impl BashOutputWriter {
r.terminal = Some(OutputTerminal {
status: Some(terminal.status),
exit_code: terminal.exit_code,
transport_complete: true,
transport_complete: matches!(
terminal.output_collection,
workdir::CommandOutputCollection::Complete
),
process_failure: terminal.process_failure,
output_collection: Some(terminal.output_collection),
});
Ok(r.metadata())
}
@@ -439,6 +516,8 @@ impl Drop for BashOutputWriter {
status: None,
exit_code: None,
transport_complete: false,
process_failure: None,
output_collection: None,
});
}
}
@@ -456,6 +535,8 @@ mod tests {
status: Some(CommandStatus::Completed),
exit_code: Some(0),
transport_complete: true,
process_failure: None,
output_collection: Some(workdir::CommandOutputCollection::Complete),
})
.id
}
@@ -514,6 +595,8 @@ mod tests {
status: Some(CommandStatus::Completed),
exit_code: Some(0),
transport_complete: true,
process_failure: None,
output_collection: Some(workdir::CommandOutputCollection::Complete),
});
assert!(root.metadata(&metadata.id).is_some());
let _replacement = sibling.begin().unwrap();
@@ -582,7 +665,7 @@ mod tests {
let root = BashOutputView::new();
let mut writer = root.begin().unwrap();
writer.append(CommandStream::Stdout, b"abc");
let mut terminal = serde_json::json!({"status":"completed","exit_code":0,"stdout_bytes":9,"stderr_bytes":0,"stdout_delivered_bytes":4,"stderr_delivered_bytes":0,"truncation_reason":"byte_budget"});
let mut terminal = serde_json::json!({"status":"completed","exit_code":0,"process_failure":null,"output_collection":{"state":"complete"},"stdout_bytes":9,"stderr_bytes":0,"stdout_delivered_bytes":4,"stderr_delivered_bytes":0,"truncation_reason":"byte_budget"});
assert!(
writer
.finish_transport(&serde_json::from_value(terminal.clone()).unwrap())
@@ -602,4 +685,41 @@ mod tests {
Some(workdir::CommandOutputTruncation::ByteBudget)
);
}
#[test]
fn incomplete_collection_with_quota_retains_both_causes_without_claiming_complete_output() {
let root = BashOutputView::new();
let mut writer = root.begin().unwrap();
writer.append(CommandStream::Stdout, b"abc");
let terminal: workdir::CommandTerminal = serde_json::from_value(serde_json::json!({
"status":"completed", "exit_code":0, "process_failure":null,
"output_collection":{"state":"incomplete","failure":{"stage":"stdout_drain","reason":"other","os_code":null}},
"stdout_bytes":9, "stderr_bytes":0, "stdout_delivered_bytes":3, "stderr_delivered_bytes":0,
"truncation_reason":"byte_budget"
})).unwrap();
let metadata = writer.finish_transport(&terminal).unwrap();
assert!(!metadata.terminal.transport_complete);
assert_eq!(metadata.terminal.exit_code, Some(0));
assert_eq!(
metadata.terminal.output_collection,
Some(terminal.output_collection)
);
assert_eq!(
metadata.transport_truncation,
Some(workdir::CommandOutputTruncation::ByteBudget)
);
assert_eq!(metadata.stdout_seen, 9);
assert_eq!(metadata.stdout_saved, 3);
assert!(metadata.storage_truncated);
assert!(metadata.retention_limit.is_none());
assert_eq!(
root.read(&metadata.id, CommandStream::Stdout, 0, 100)
.unwrap(),
b"abc"
);
assert_eq!(
serde_json::to_value(root.metadata(&metadata.id).unwrap()).unwrap(),
serde_json::to_value(metadata).unwrap()
);
}
}
+1 -1
View File
@@ -21,7 +21,7 @@ pub(crate) struct EditParams {
/// Worker-local alias of the Workdir attachment to use.
#[serde(default)]
pub target_workdir: Option<String>,
/// Logical path relative to the bound Workdir root.
/// Existing file in which to replace text.
pub file_path: String,
#[serde(flatten)]
pub edit: fs_operation::text::EditArgs,
+1
View File
@@ -167,6 +167,7 @@ pub(crate) fn checked_error(error: workdir::WorkdirError) -> ToolError {
| WorkdirError::IsDirectory { .. }
| WorkdirError::InvalidGlob { .. }
| WorkdirError::InvalidRegex { .. } => "checkout_invalid",
WorkdirError::NotDispatched => "checkout_not_dispatched",
WorkdirError::Unavailable { .. } | WorkdirError::SessionClosed => "checkout_unavailable",
// IO/transport failures may arrive after effects have begun. Do not infer
// no-effects from their text or strip explicit OutcomeUnknown evidence.
+2 -2
View File
@@ -17,7 +17,7 @@ pub(crate) struct GlobParams {
pub target_workdir: Option<String>,
/// Glob pattern, for example `**/*.rs` or `src/**/test_*.py`.
pub pattern: String,
/// Logical Workdir-relative directory. Defaults to the Workdir root.
/// Directory to search. Defaults to `.`.
#[serde(default)]
pub path: Option<String>,
}
@@ -106,7 +106,7 @@ pub(crate) fn routed_glob_tool(router: Arc<WorkdirSessionRouter>) -> ToolDefinit
Arc::new(move || {
let schema = schemars::schema_for!(GlobParams);
let meta = ToolMeta::new("Glob")
.description("Find files matching a glob pattern inside the selected Workdir. Results are sorted and capped at 1000 entries. Paths are Workdir-relative.")
.description("Find files matching a glob pattern inside the selected Workdir. Results are sorted and capped at 1000 entries.")
.input_schema(serde_json::to_value(schema).expect("Glob schema serialization"));
let tool: Arc<dyn Tool> = Arc::new(GlobTool {
router: router.clone(),
+2 -2
View File
@@ -27,7 +27,7 @@ pub(crate) struct GrepParams {
#[serde(default)]
target_workdir: Option<String>,
pattern: String,
/// Workdir-relative path, or an absolute path covered by readable scope. Defaults to the Workdir root.
/// File or directory to search. Defaults to `.`.
#[serde(default)]
pub path: Option<String>,
#[serde(default)]
@@ -160,7 +160,7 @@ pub(crate) fn routed_grep_tool(router: Arc<WorkdirSessionRouter>) -> ToolDefinit
Arc::new(move || {
let schema = schemars::schema_for!(GrepParams);
let meta = ToolMeta::new("Grep")
.description("Search a file or directory in the selected Workdir with a regex. Content results group lines by file; `>` marks matching lines and unmarked lines are context. Directory traversal executes inside the WorkdirSession provider. Results are bounded and Workdir-relative.")
.description("Search a file or directory in the selected Workdir with a regex. Content results group lines by file; `>` marks matching lines and unmarked lines are context. Directory traversal executes inside the WorkdirSession provider. Results are bounded.")
.input_schema(serde_json::to_value(schema).expect("Grep schema serialization"));
let tool: Arc<dyn Tool> = Arc::new(GrepTool {
router: router.clone(),
+2 -2
View File
@@ -13,7 +13,7 @@ use workdir::{WorkdirPath, WorkdirSessionHandle, WorkdirSessionRouter};
const DESCRIPTION: &str = "Read a text file from a Workdir attachment selected by its Worker-local alias. \
Supports offset/limit for large files. Returns line-numbered output (1-based). \
Directories cannot be read. The file must be read before Write or Edit can \
modify it. Paths are Workdir-relative unless an absolute path is explicitly readable. \
modify it. \
When exactly one Workdir is attached, target_workdir may be omitted.";
const DEFAULT_LIMIT: usize = 2000;
@@ -24,7 +24,7 @@ pub(crate) struct ReadParams {
/// Worker-local alias of the Workdir attachment to use.
#[serde(default)]
pub target_workdir: Option<String>,
/// Workdir-relative path, or an absolute path covered by readable scope.
/// Text file to read; directories are not supported.
pub file_path: String,
#[serde(flatten)]
pub read: fs_operation::text::LineReadArgs,
+2 -2
View File
@@ -20,7 +20,7 @@ pub const MAX_IMAGE_BYTES: usize = 10 * 1024 * 1024;
const IMAGE_READ_CHUNK_BYTES: usize = 1024 * 1024;
const DESCRIPTION: &str = "Attach an image from the selected Workdir attachment to the next model request. \
The path must be logical and Workdir-relative. Supported formats: PNG, JPEG, GIF, and WebP. \
Supported formats: PNG, JPEG, GIF, and WebP. \
When exactly one Workdir is attached, target_workdir may be omitted.";
#[derive(Debug, Deserialize, schemars::JsonSchema)]
@@ -28,7 +28,7 @@ struct ViewImageParams {
/// Worker-local alias of the Workdir attachment to use.
#[serde(default)]
target_workdir: Option<String>,
/// Logical path relative to the bound Workdir root.
/// Image file to attach to the next model request.
path: String,
}
+1 -1
View File
@@ -20,7 +20,7 @@ pub(crate) struct WriteParams {
/// Worker-local alias of the Workdir attachment to use.
#[serde(default)]
pub target_workdir: Option<String>,
/// Logical path relative to the bound Workdir root.
/// File to create or overwrite.
pub file_path: String,
#[serde(flatten)]
pub write: fs_operation::text::WriteArgs,
+687 -2
View File
@@ -3,6 +3,11 @@
//! `ToolServerHandle::register_tool` / `flush_pending` are `pub(crate)` in
//! agen, so from here we exercise the factories directly — the same
//! code path that `flush_pending()` runs at production time.
//!
//! Scoped routing cases connect real LocalWorkdirSession, WorkdirToolBroker::scope,
//! attachment router and Tool factories without replacing their permission or path checks.
//! They use RAII temporary roots and existing image fixtures; command cases require the
//! same local shell as the Bash suite and run in the normal `tools --test integration` target.
use std::path::Path;
use std::sync::Arc;
@@ -80,6 +85,101 @@ fn setup_routed() -> (TempDir, TempDir, Arc<WorkdirSessionRouter>, Registry) {
(left, right, router, Registry::new(definitions))
}
const READ_SAME_FILE: &str = "IFS= read -r line < same.txt && printf '%s\\n' \"$line\"";
struct ScopedRegistry {
roots: Vec<TempDir>,
directories: Vec<std::path::PathBuf>,
brokers: Vec<WorkdirToolBroker>,
leases: Vec<workdir::WorkdirScopeLease>,
router: Arc<WorkdirSessionRouter>,
reg: Registry,
}
impl ScopedRegistry {
// Cwd and scope targets here are delegation declarations in provider coordinates.
async fn new(attachments: &[(&str, &str, bool)]) -> Self {
let router = Arc::new(WorkdirSessionRouter::new());
let mut roots = Vec::new();
let mut directories = Vec::new();
let mut brokers = Vec::new();
let mut leases = Vec::new();
for &(alias, cwd, command) in attachments {
let root = TempDir::new().unwrap();
let directory = root.path().join(cwd);
std::fs::create_dir_all(directory.join("subdir")).unwrap();
std::fs::write(directory.join("same.txt"), format!("{alias} original\n")).unwrap();
std::fs::write(
directory.join("subdir/same.txt"),
format!("{alias} nested needle\n"),
)
.unwrap();
std::fs::write(
directory.join("image.png"),
include_bytes!("fixtures/view-image/valid.png"),
)
.unwrap();
if cwd != "." {
// Decoys make accidental root-based resolution observable.
std::fs::write(root.path().join("same.txt"), "root decoy\n").unwrap();
std::fs::write(root.path().join("image.png"), "not an image").unwrap();
}
let source: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::new(
Scope::writable(root.path()).unwrap(),
root.path().to_path_buf(),
));
let broker = WorkdirToolBroker::new(source);
let lease = broker
.scope(WorkdirToolScope {
rules: vec![WorkdirToolScopeRule {
target: WorkdirPath::new(cwd).unwrap(),
permission: WorkdirToolScopePermission::Write,
recursive: true,
symlink_policy: Default::default(),
}],
cwd: WorkdirPath::new(cwd).unwrap(),
command,
})
.await
.unwrap();
router
.attach(
WorkdirAttachmentAlias::new(alias).unwrap(),
lease.tool_session(),
)
.unwrap();
roots.push(root);
directories.push(directory);
brokers.push(broker);
leases.push(lease);
}
let mut definitions = routed_builtin_tools(
router.clone(),
Tracker::new(),
tools::bash_output::BashOutputView::new(),
);
definitions.push(routed_view_image_tool(router.clone()));
Self {
roots,
directories,
brokers,
leases,
router,
reg: Registry::new(definitions),
}
}
async fn close(self) {
self.router.close_all().await.unwrap();
for lease in &self.leases {
lease.close().await.unwrap();
}
for broker in &self.brokers {
broker.close().await.unwrap();
}
}
}
fn valid_gif_with_len(target_len: usize) -> Vec<u8> {
let fixture = include_bytes!("fixtures/view-image/valid.gif");
assert_eq!(
@@ -156,6 +256,71 @@ fn routed_tool_schemas_share_optional_target_workdir() {
}
}
#[test]
fn routed_tool_help_does_not_override_session_path_coordinates() {
fn assert_no_root_coordinates(tool: &str, description: &str) {
let normalized = description
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
.to_ascii_lowercase();
for obsolete in ["workdir-relative", "workdir root", "root-relative"] {
assert!(
!normalized.contains(obsolete),
"{tool} help overrides session path coordinates: {description}"
);
}
}
fn check_schema_descriptions(tool: &str, schema: &serde_json::Value) {
match schema {
serde_json::Value::Object(object) => {
if let Some(description) = object.get("description").and_then(|v| v.as_str()) {
assert_no_root_coordinates(tool, description);
}
for value in object.values() {
check_schema_descriptions(tool, value);
}
}
serde_json::Value::Array(array) => {
for value in array {
check_schema_descriptions(tool, value);
}
}
_ => {}
}
}
let router = Arc::new(WorkdirSessionRouter::new());
let mut definitions = routed_builtin_tools(
router.clone(),
Tracker::new(),
tools::bash_output::BashOutputView::new(),
);
definitions.push(routed_view_image_tool(router));
for definition in definitions {
let (meta, _) = definition();
assert_no_root_coordinates(&meta.name, &meta.description);
// Referenced type definitions are model-visible too, not just properties.
check_schema_descriptions(&meta.name, &meta.input_schema);
let path_parameter = match meta.name.as_str() {
"Read" | "Write" | "Edit" => "file_path",
"Glob" | "Grep" | "ViewImage" => "path",
"Bash" => "cwd",
other => panic!("missing path parameter contract for {other}"),
};
let help = meta.input_schema["properties"][path_parameter]["description"]
.as_str()
.unwrap_or_default();
assert!(
!help.trim().is_empty(),
"{}.{path_parameter} must retain argument-specific help",
meta.name
);
}
}
#[tokio::test]
async fn routed_tools_require_alias_only_when_attachment_set_is_ambiguous() {
let (_left, _right, router, reg) = setup_routed();
@@ -284,6 +449,427 @@ async fn target_alias_routes_each_tool_and_isolates_read_before_edit() {
router.close_all().await.unwrap();
}
#[tokio::test]
async fn scoped_cwd_file_search_and_image_tools_share_reusable_paths() {
for cwd in [".", "work"] {
let fixture = ScopedRegistry::new(&[("checkout", cwd, false)]).await;
let read = fixture.reg.get("Read");
let original = call(&read, json!({ "file_path": "same.txt" })).await;
assert!(original.content.unwrap().contains("checkout original"));
call(
&fixture.reg.get("Write"),
json!({ "file_path": "same.txt", "content": "written marker\n" }),
)
.await;
call(
&fixture.reg.get("Edit"),
json!({ "file_path": "same.txt", "old_string": "written", "new_string": "edited" }),
)
.await;
let edited = call(&read, json!({ "file_path": "same.txt" })).await;
assert!(edited.content.unwrap().contains("edited marker"));
assert_eq!(
std::fs::read_to_string(fixture.directories[0].join("same.txt")).unwrap(),
"edited marker\n"
);
// Omitted search paths start at cwd, not at the underlying provider root.
for (name, input) in [
("Glob", json!({ "pattern": "same.txt" })),
(
"Grep",
json!({ "pattern": "edited marker", "output_mode": "files_with_matches" }),
),
] {
let result = call(&fixture.reg.get(name), input).await;
assert_eq!(
result.content.unwrap().trim(),
"same.txt",
"{name}, cwd={cwd}"
);
}
// A search rooted below cwd still returns cwd-relative reusable paths.
for (name, input) in [
("Glob", json!({ "pattern": "*.txt", "path": "subdir" })),
(
"Grep",
json!({ "pattern": "needle", "path": "subdir", "output_mode": "files_with_matches" }),
),
] {
let result = call(&fixture.reg.get(name), input).await;
let body = result.content.unwrap();
let paths: Vec<_> = body.lines().collect();
assert_eq!(paths, ["subdir/same.txt"], "{name}, cwd={cwd}");
let path = paths[0];
let found = call(&read, json!({ "file_path": path })).await;
assert!(found.content.unwrap().contains("checkout nested needle"));
call(
&fixture.reg.get("Edit"),
json!({ "file_path": path, "old_string": "needle", "new_string": format!("needle-{name}") }),
)
.await;
}
assert_eq!(
std::fs::read_to_string(fixture.directories[0].join("subdir/same.txt")).unwrap(),
"checkout nested needle-Grep-Glob\n"
);
let image = call(
&fixture.reg.get("ViewImage"),
json!({ "path": "image.png" }),
)
.await;
assert_eq!(image.attachments.len(), 1);
let agen::tool::Attachment::Image(image) = &image.attachments[0];
assert_eq!(image.mime_type(), "image/png");
assert_eq!(
image.data(),
include_bytes!("fixtures/view-image/valid.png")
);
if cwd != "." {
assert_eq!(
std::fs::read_to_string(fixture.roots[0].path().join("same.txt")).unwrap(),
"root decoy\n"
);
}
fixture.close().await;
}
}
#[tokio::test]
async fn scoped_bash_relative_cwd_and_cd_do_not_change_later_tool_cwd() {
for cwd in [".", "work"] {
let fixture = ScopedRegistry::new(&[("checkout", cwd, true)]).await;
for (command_cwd, command, expected) in [
(".", READ_SAME_FILE, "checkout original\n"),
("subdir", READ_SAME_FILE, "checkout nested needle\n"),
("./subdir//.", READ_SAME_FILE, "checkout nested needle\n"),
(
".",
"cd subdir && IFS= read -r line < same.txt && printf '%s\\n' \"$line\"",
"checkout nested needle\n",
),
(".", READ_SAME_FILE, "checkout original\n"),
] {
let output = call(
&fixture.reg.get("Bash"),
json!({ "cwd": command_cwd, "command": command, "timeout": 5 }),
)
.await;
let body: serde_json::Value = serde_json::from_str(&output.content.unwrap()).unwrap();
assert_eq!(body["output"]["terminal"]["exit_code"], 0, "{body}");
assert_eq!(body["stdout_preview"], expected, "cwd={cwd}, {command}");
let subsequent =
call(&fixture.reg.get("Read"), json!({ "file_path": "same.txt" })).await;
assert!(subsequent.content.unwrap().contains("checkout original"));
}
fixture.close().await;
}
}
#[tokio::test]
async fn scoped_aliases_with_different_cwds_keep_paths_and_observations_separate() {
let fixture =
ScopedRegistry::new(&[("left", "work", true), ("right", "other/deep", true)]).await;
let read = fixture.reg.get("Read");
let edit = fixture.reg.get("Edit");
call(
&read,
json!({ "target_workdir": "left", "file_path": "same.txt" }),
)
.await;
let unread = call_err(
&edit,
json!({ "target_workdir": "right", "file_path": "same.txt", "old_string": "original", "new_string": "wrong" }),
)
.await;
assert!(unread.to_string().contains("has not been read"), "{unread}");
call(
&read,
json!({ "target_workdir": "right", "file_path": "same.txt" }),
)
.await;
call(
&fixture.reg.get("Write"),
json!({ "target_workdir": "right", "file_path": "same.txt", "content": "right written\n" }),
)
.await;
call(
&edit,
json!({ "target_workdir": "right", "file_path": "same.txt", "old_string": "written", "new_string": "edited" }),
)
.await;
assert_eq!(
std::fs::read_to_string(fixture.directories[0].join("same.txt")).unwrap(),
"left original\n"
);
assert_eq!(
std::fs::read_to_string(fixture.directories[1].join("same.txt")).unwrap(),
"right edited\n"
);
// Different valid bytes make cross-alias image reads observable too.
let png: &[u8] = include_bytes!("fixtures/view-image/valid.png");
let gif: &[u8] = include_bytes!("fixtures/view-image/valid.gif");
std::fs::write(fixture.directories[1].join("image.png"), gif).unwrap();
for (alias, expected, image_bytes, mime) in [
("left", "left original", png, "image/png"),
("right", "right edited", gif, "image/gif"),
] {
let output = call(
&read,
json!({ "target_workdir": alias, "file_path": "same.txt" }),
)
.await;
assert!(output.content.unwrap().contains(expected));
for (name, input) in [
(
"Glob",
json!({ "target_workdir": alias, "pattern": "*.txt", "path": "subdir" }),
),
(
"Grep",
json!({ "target_workdir": alias, "pattern": alias, "path": "subdir", "output_mode": "files_with_matches" }),
),
] {
let output = call(&fixture.reg.get(name), input).await;
let path = output.content.unwrap();
let path = path.trim();
assert_eq!(path, "subdir/same.txt", "{alias}: {name}");
let found = call(&read, json!({ "target_workdir": alias, "file_path": path })).await;
assert!(
found
.content
.unwrap()
.contains(&format!("{alias} nested needle"))
);
}
let image = call(
&fixture.reg.get("ViewImage"),
json!({ "target_workdir": alias, "path": "image.png" }),
)
.await;
let agen::tool::Attachment::Image(image) = &image.attachments[0];
assert_eq!(image.mime_type(), mime);
assert_eq!(image.data(), image_bytes);
let output = call(
&fixture.reg.get("Bash"),
json!({ "target_workdir": alias, "cwd": ".", "command": READ_SAME_FILE, "timeout": 5 }),
)
.await;
let body: serde_json::Value = serde_json::from_str(&output.content.unwrap()).unwrap();
assert_eq!(body["output"]["terminal"]["exit_code"], 0, "{body}");
assert_eq!(body["stdout_preview"], format!("{expected}\n"));
}
fixture.close().await;
}
#[tokio::test]
async fn scoped_command_false_rejects_bash_without_fallback_to_another_alias() {
let fixture =
ScopedRegistry::new(&[("enabled", "work", true), ("disabled", "other", false)]).await;
for cwd in [".", "subdir"] {
let error = call_err(
&fixture.reg.get("Bash"),
json!({ "target_workdir": "disabled", "cwd": cwd, "command": "printf forbidden > blocked.txt", "timeout": 5 }),
)
.await;
assert!(
error.to_string().contains("does not support Command"),
"{error}"
);
}
// Write permission remains usable, but does not imply command permission.
call(
&fixture.reg.get("Write"),
json!({ "target_workdir": "disabled", "file_path": "created.txt", "content": "allowed write\n" }),
)
.await;
assert_eq!(
std::fs::read_to_string(fixture.directories[1].join("created.txt")).unwrap(),
"allowed write\n"
);
for (root, directory) in fixture.roots.iter().zip(&fixture.directories) {
for path in [
root.path().to_path_buf(),
directory.clone(),
directory.join("subdir"),
] {
assert!(!path.join("blocked.txt").exists());
}
}
fixture.close().await;
}
#[tokio::test]
async fn scoped_normalized_paths_share_read_before_mutation_history() {
let fixture = ScopedRegistry::new(&[("checkout", "work", false)]).await;
let read = call(
&fixture.reg.get("Read"),
json!({ "file_path": "./subdir//./same.txt" }),
)
.await;
assert!(read.content.unwrap().contains("checkout nested needle"));
call(
&fixture.reg.get("Write"),
json!({ "file_path": "subdir/same.txt", "content": "normalized write\n" }),
)
.await;
call(
&fixture.reg.get("Edit"),
json!({ "file_path": "./subdir/same.txt", "old_string": "write", "new_string": "edit" }),
)
.await;
assert_eq!(
std::fs::read_to_string(fixture.directories[0].join("subdir/same.txt")).unwrap(),
"normalized edit\n"
);
for (name, input) in [
("Glob", json!({ "pattern": "*.txt", "path": "./subdir//." })),
(
"Grep",
json!({ "pattern": "normalized edit", "path": "./subdir//.", "output_mode": "files_with_matches" }),
),
] {
let result = call(&fixture.reg.get(name), input).await;
assert_eq!(result.content.unwrap().trim(), "subdir/same.txt", "{name}");
}
let image = call(
&fixture.reg.get("ViewImage"),
json!({ "path": "././image.png" }),
)
.await;
let agen::tool::Attachment::Image(image) = &image.attachments[0];
assert_eq!(
image.data(),
include_bytes!("fixtures/view-image/valid.png")
);
fixture.close().await;
}
#[tokio::test]
async fn scoped_tools_reject_parent_traversal_and_paths_outside_the_grant() {
let fixture = ScopedRegistry::new(&[("checkout", "work", true)]).await;
let outside = fixture.roots[0].path().join("outside");
std::fs::create_dir(&outside).unwrap();
std::fs::write(outside.join("secret.txt"), "private needle\n").unwrap();
for invalid in [
"../outside/secret.txt",
"subdir/../../outside/secret.txt",
"subdir\\same.txt",
] {
for (name, input) in [
("Read", json!({ "file_path": invalid })),
(
"Write",
json!({ "file_path": invalid, "content": "forbidden" }),
),
(
"Edit",
json!({ "file_path": invalid, "old_string": "needle", "new_string": "forbidden" }),
),
("Glob", json!({ "path": invalid, "pattern": "*" })),
("Grep", json!({ "path": invalid, "pattern": "needle" })),
("ViewImage", json!({ "path": invalid })),
(
"Bash",
json!({ "cwd": invalid, "command": "printf forbidden > blocked.txt", "timeout": 5 }),
),
] {
let error = call_err(&fixture.reg.get(name), input).await;
assert!(
error.to_string().contains("path"),
"{name}: {invalid}: {error}"
);
}
}
// These paths are inside the provider's grant, but outside the child's grant.
// Absolute readable paths must not bypass the delegated scope after cwd rebasing.
for (name, input) in [
("Read", json!({ "file_path": outside.join("secret.txt") })),
("Grep", json!({ "path": outside, "pattern": "needle" })),
(
"Bash",
json!({ "cwd": outside, "command": "printf forbidden > blocked.txt", "timeout": 5 }),
),
] {
let error = call_err(&fixture.reg.get(name), input).await;
let message = error.to_string();
assert!(
matches!(error, ToolError::InvalidArgument(_)),
"{name}: {message}"
);
assert!(
message.contains("scope") || message.contains("path"),
"{name}: {message}"
);
}
assert_eq!(
std::fs::read_to_string(outside.join("secret.txt")).unwrap(),
"private needle\n"
);
assert!(!outside.join("blocked.txt").exists());
assert!(!fixture.directories[0].join("blocked.txt").exists());
assert!(!fixture.roots[0].path().join("blocked.txt").exists());
fixture.close().await;
}
#[cfg(unix)]
#[tokio::test]
async fn scoped_cwd_tools_reject_resolved_symlink_escape_after_a_read() {
use std::os::unix::fs::symlink;
let fixture = ScopedRegistry::new(&[("checkout", "work", true)]).await;
let outside = fixture.roots[0].path().join("outside");
std::fs::create_dir(&outside).unwrap();
// Equal bytes avoid disguising an authority failure as a stale-hash rejection.
std::fs::write(outside.join("secret.txt"), "checkout original\n").unwrap();
let png = include_bytes!("fixtures/view-image/valid.png");
std::fs::write(outside.join("image.png"), png).unwrap();
call(&fixture.reg.get("Read"), json!({ "file_path": "same.txt" })).await;
std::fs::remove_file(fixture.directories[0].join("same.txt")).unwrap();
symlink(
"../outside/secret.txt",
fixture.directories[0].join("same.txt"),
)
.unwrap();
symlink("../outside", fixture.directories[0].join("escape")).unwrap();
for (name, input) in [
("Read", json!({ "file_path": "same.txt" })),
(
"Write",
json!({ "file_path": "escape/created.txt", "content": "forbidden" }),
),
(
"Edit",
json!({ "file_path": "same.txt", "old_string": "original", "new_string": "forbidden" }),
),
("Glob", json!({ "path": "escape", "pattern": "*" })),
("Grep", json!({ "path": "escape", "pattern": "original" })),
("ViewImage", json!({ "path": "escape/image.png" })),
(
"Bash",
json!({ "cwd": "escape", "command": "printf forbidden > blocked.txt", "timeout": 5 }),
),
] {
let error = call_err(&fixture.reg.get(name), input).await;
let message = error.to_string();
assert!(
message.contains("scope") || message.contains("symlink"),
"{name}: {message}"
);
}
assert_eq!(
std::fs::read_to_string(outside.join("secret.txt")).unwrap(),
"checkout original\n"
);
assert_eq!(std::fs::read(outside.join("image.png")).unwrap(), png);
assert!(!outside.join("created.txt").exists());
assert!(!outside.join("blocked.txt").exists());
fixture.close().await;
}
#[tokio::test]
async fn selected_attachment_capability_is_checked_without_fallback() {
let left = TempDir::new().unwrap();
@@ -416,6 +1002,7 @@ struct ExternalValidatingSession {
truncate_first_binary_response: bool,
command_response_losses: std::sync::atomic::AtomicUsize,
lose_terminal_only: bool,
collection_override: Option<workdir::CommandOutputCollection>,
mutate_after_first_binary_response: Option<std::path::PathBuf>,
}
@@ -534,7 +1121,11 @@ impl workdir::WorkdirSession for ExternalValidatingSession {
&self,
request: workdir::CommandOutputRequest,
) -> Result<workdir::CommandOutput, workdir::WorkdirError> {
let output = self.inner.command_output(request).await?;
let mut output = self.inner.command_output(request).await?;
if let (Some(collection), Some(terminal)) = (self.collection_override, &mut output.terminal)
{
terminal.output_collection = collection;
}
if (!self.lose_terminal_only || output.terminal.is_some())
&& self
.command_response_losses
@@ -575,6 +1166,7 @@ async fn external_view_image_uses_a_protocol_bounded_request() {
truncate_first_binary_response: false,
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
lose_terminal_only: false,
collection_override: None,
mutate_after_first_binary_response: None,
});
let png = include_bytes!("fixtures/view-image/valid.png");
@@ -600,6 +1192,7 @@ async fn external_view_image_rejects_truncated_and_cross_version_chunks() {
truncate_first_binary_response: true,
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
lose_terminal_only: false,
collection_override: None,
mutate_after_first_binary_response: None,
});
let (_meta, tool) = view_image_tool(truncated)();
@@ -618,6 +1211,7 @@ async fn external_view_image_rejects_truncated_and_cross_version_chunks() {
truncate_first_binary_response: false,
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
lose_terminal_only: false,
collection_override: None,
mutate_after_first_binary_response: Some(image_path),
});
let (_meta, tool) = view_image_tool(changing)();
@@ -1104,7 +1698,20 @@ async fn bash_cancellation_returns_bounded_progress_as_terminal_output() {
let execution = tokio::spawn(async move {
executing.execute(r#"{"command":"printf 'before'; printf 'err-before' >&2; while :; do :; done","cwd":"."}"#, execution_context).await
});
await_command_output(&mut events).await;
// Cancellation must follow both writes, not merely the first stdout chunk.
tokio::time::timeout(std::time::Duration::from_secs(5), async {
let mut ready = [false; 2];
while !ready.iter().all(|seen| *seen) {
if let workdir::CommandEvent::Output { stream, .. } = events.recv().await.unwrap() {
ready[match stream {
workdir::CommandStream::Stdout => 0,
workdir::CommandStream::Stderr => 1,
}] = true;
}
}
})
.await
.expect("both streams written before cancellation");
bash.cancel_execution(&context).await.unwrap();
let ToolError::Cancelled(output) =
tokio::time::timeout(std::time::Duration::from_secs(5), execution)
@@ -1281,6 +1888,7 @@ async fn lost_page_and_terminal_responses_replay_without_restarting_or_double_ap
mutate_after_first_binary_response: None,
command_response_losses: std::sync::atomic::AtomicUsize::new(1),
lose_terminal_only,
collection_override: None,
});
let view = tools::bash_output::BashOutputView::new();
let definitions = core_builtin_tools(session.clone(), Tracker::new(), view.clone());
@@ -1315,3 +1923,80 @@ async fn lost_page_and_terminal_responses_replay_without_restarting_or_double_ap
session.close().await.unwrap();
}
}
// Provider metadata is the fixture here; actual read/drain injection and HTTP
// forwarding are tested by workdir's production-supervisor suite.
#[tokio::test]
async fn exit_zero_with_incomplete_output_keeps_partial_bytes_and_safe_failure_in_retained_evidence()
{
use workdir::{
CommandFailure, CommandFailureStage as Stage, CommandIoErrorKind, CommandOutputCollection,
};
for stage in [
Stage::StdoutRead,
Stage::StderrRead,
Stage::StdoutDrain,
Stage::StderrDrain,
Stage::OutputDelivery,
] {
let root = TempDir::new().unwrap();
let failure = CommandFailure {
stage,
reason: CommandIoErrorKind::PermissionDenied,
os_code: None,
};
let collection = CommandOutputCollection::Incomplete { failure };
let session: WorkdirSessionHandle = Arc::new(ExternalValidatingSession {
inner: LocalWorkdirSession::new(
Scope::writable(root.path()).unwrap(),
root.path().into(),
),
truncate_first_binary_response: false,
mutate_after_first_binary_response: None,
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
lose_terminal_only: false,
collection_override: Some(collection),
});
let view = tools::bash_output::BashOutputView::new();
let (_, bash) = tools::bash_tool(session.clone(), view.clone())();
let out = call(
&bash,
json!({"cwd":".","command":"printf partial; printf err >&2"}),
)
.await;
assert!(out.summary.ends_with("(output incomplete)"));
let body: serde_json::Value = serde_json::from_str(&out.content.unwrap()).unwrap();
let metadata = &body["output"];
assert_eq!(body["stdout_preview"], "partial");
assert_eq!(body["stderr_preview"], "err");
assert_eq!(body["preview_omitted"], false);
assert_eq!(metadata["terminal"]["exit_code"], 0);
assert_eq!(metadata["terminal"]["status"], "completed");
assert_eq!(metadata["terminal"]["transport_complete"], false);
assert_eq!(
metadata["terminal"]["output_collection"],
serde_json::to_value(collection).unwrap()
);
assert_eq!(metadata["storage_truncated"], false);
assert_eq!(metadata["transport_truncation"], serde_json::Value::Null);
assert_eq!(metadata["stdout_seen"], 7);
assert_eq!(metadata["stdout_delivered"], 7);
let id = metadata["id"].as_str().unwrap();
assert_eq!(
serde_json::to_value(view.metadata(id).unwrap()).unwrap(),
*metadata
);
assert_eq!(
view.read(id, workdir::CommandStream::Stdout, 0, 100)
.unwrap(),
b"partial"
);
assert_eq!(
view.read(id, workdir::CommandStream::Stderr, 0, 100)
.unwrap(),
b"err"
);
assert!(!metadata.to_string().contains(root.path().to_str().unwrap()));
session.close().await.unwrap();
}
}
+63 -31
View File
@@ -785,6 +785,12 @@ mod tests {
#[derive(Default)]
struct PendingRuntimeBackend {
operations: std::sync::Mutex<Vec<worker_runtime::execution::WorkerLifecycleOperationId>>,
owners: std::sync::Mutex<
std::collections::HashMap<
worker_runtime::execution::WorkerLifecycleOperationId,
worker_runtime::identity::WorkerRef,
>,
>,
}
impl worker_runtime::execution::WorkerExecutionBackend for PendingRuntimeBackend {
@@ -806,6 +812,21 @@ mod tests {
)
}
fn restore_recovery_disposition(
&self,
operation_id: worker_runtime::execution::WorkerLifecycleOperationId,
worker_ref: &worker_runtime::identity::WorkerRef,
) -> Option<worker_runtime::execution::RestoreRecoveryDisposition> {
use worker_runtime::execution::RestoreRecoveryDisposition as D;
Some(
if self.owners.lock().unwrap().get(&operation_id) == Some(worker_ref) {
D::Continue
} else {
D::Absent
},
)
}
fn restore_worker(
&self,
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
@@ -813,6 +834,10 @@ mod tests {
use worker_runtime::execution::*;
let mut operations = self.operations.lock().unwrap();
operations.push(request.operation_id);
self.owners
.lock()
.unwrap()
.insert(request.operation_id, request.worker_ref.clone());
if operations.len() == 1 {
WorkerExecutionSpawnResult::ReconciliationRequired {
result: WorkerExecutionResult::errored(
@@ -1119,39 +1144,45 @@ mod tests {
}
#[tokio::test]
async fn rejected_resume_does_not_observe_or_open_retained_state() {
let lifecycle = FakeBackendWorkerLifecycle {
restore_response: restore_response(BackendWorkerRestoreState::Rejected),
observation: session_observation(serde_json::json!({
"availability": "unavailable",
"reason": "retention_missing",
"message": "must not be opened"
})),
calls: std::sync::Mutex::new(Vec::new()),
};
let target = client::BackendRuntimeTarget::new(
"http://127.0.0.1:3000",
"workspace-a",
"runtime-a",
"worker-a",
);
async fn settled_nonaccepted_resume_does_not_retry_observe_or_open_retained_state() {
for state in [
BackendWorkerRestoreState::Rejected,
BackendWorkerRestoreState::RolledBack,
BackendWorkerRestoreState::Superseded,
] {
let lifecycle = FakeBackendWorkerLifecycle {
restore_response: restore_response(state),
observation: session_observation(serde_json::json!({
"availability": "unavailable",
"reason": "retention_missing",
"message": "must not be opened"
})),
calls: std::sync::Mutex::new(Vec::new()),
};
let target = client::BackendRuntimeTarget::new(
"http://127.0.0.1:3000",
"workspace-a",
"runtime-a",
"worker-a",
);
let error = prepare_selected_worker(
&lifecycle,
target,
BackendWorkerPickerIntent::Resume,
Some(client::BackendWorkerRestoreRequest {
expected_observation_token: "observed-worker-state".into(),
request_id: "resume-operation".into(),
}),
|_| Ok(false),
)
.await
.unwrap_err()
.to_string();
let error = prepare_selected_worker(
&lifecycle,
target,
BackendWorkerPickerIntent::Resume,
Some(client::BackendWorkerRestoreRequest {
expected_observation_token: "observed-worker-state".into(),
request_id: "resume-operation".into(),
}),
|_| panic!("settled nonaccepted restore must not offer recovery retry"),
)
.await
.unwrap_err()
.to_string();
assert!(error.contains("Rejected"));
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
assert!(error.contains(&format!("{state:?}")));
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
}
}
#[test]
@@ -1176,6 +1207,7 @@ mod tests {
for state in [
BackendWorkerRestoreState::Rejected,
BackendWorkerRestoreState::RolledBack,
BackendWorkerRestoreState::Superseded,
BackendWorkerRestoreState::ReconciliationRequired,
] {
let mut response = restore_response(state);
+1
View File
@@ -23,5 +23,6 @@ thiserror.workspace = true
tokio = { workspace = true, features = ["io-util", "process", "rt", "sync", "time"] }
[dev-dependencies]
axum.workspace = true
serde_json.workspace = true
tempfile.workspace = true
+4 -5
View File
@@ -6,7 +6,7 @@
//! hashes alone do not accept same-content inode replacements. Callers own Read line numbering
//! and per-attachment read history.
//!
//! Create paths are checkout-root-relative and must be strict descendants of the observed target
//! Create paths use the target's session coordinates and must be strict descendants of the observed target
//! directory. Existing destinations are never overwritten; results return the updated observation
//! of the retained target directory, not the created file. Read results also return the validator
//! of the state actually read, never a replacement from a later independent stat.
@@ -18,7 +18,8 @@
//!
//! Search delegates to shared List/Glob/Grep engines. Scope rules form unions within layers and
//! intersections across layers, provider authority, and output roots. Wrappers transform coordinates
//! once; returned paths and Grep content are relative to the final session's logical root.
//! once; returned paths and Grep content are relative to the composed output root (the caller's
//! cwd by default).
//! Paths are not reverse-parsed from rendered filenames; non-UTF-8 names are not lossily aliased.
//! Unsupported providers reject search rather than fall back to broader trait searches. Search is
//! live traversal, not an atomic snapshot of all descendants or exclusion of external editors.
@@ -556,15 +557,13 @@ mod tests {
));
}
#[tokio::test]
async fn checkout_hardlink_save_isolates_alias_and_rejects_symlinks_special_files() {
async fn checkout_hardlink_save_isolates_alias_and_rejects_symlinks() {
let root = tempfile::tempdir().unwrap();
std::fs::write(root.path().join("a"), "original").unwrap();
std::fs::hard_link(root.path().join("a"), root.path().join("alias")).unwrap();
std::os::unix::fs::symlink("a", root.path().join("link")).unwrap();
let _socket = std::os::unix::net::UnixListener::bind(root.path().join("socket")).unwrap();
let session = session(root.path());
assert!(session.checkout_observe(path("link")).await.is_err());
assert!(session.checkout_observe(path("socket")).await.is_err());
let read = checked_read(&session, session.checkout_observe(path("a")).await.unwrap()).await;
session
.checkout_execute(CheckoutRequest {
+68
View File
@@ -0,0 +1,68 @@
//! Model-facing session coordinates, separate from provider paths and authority.
use crate::{WorkdirPath, WorkdirSessionCapabilities, WorkdirToolScopeRule};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
/// A snapshot of the environment actually used by a session's operations.
///
/// Cwd is a logical Workdir-root coordinate. Scope targets use that root or
/// explicit absolute provider paths. Scope is an additional allow ceiling, not
/// a replacement for provider checks. Providers
/// may retain further restrictions (including deny rules); this projection
/// never authorizes an operation.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct WorkdirSessionEnvironment {
#[serde(deserialize_with = "deserialize_cwd")]
pub cwd: WorkdirPath,
pub scope: Option<Vec<WorkdirToolScopeRule>>,
pub capabilities: WorkdirSessionCapabilities,
pub restrictions: Vec<String>,
}
fn deserialize_cwd<'de, D: serde::Deserializer<'de>>(
deserializer: D,
) -> Result<WorkdirPath, D::Error> {
let path = String::deserialize(deserializer)?;
WorkdirPath::new(path).map_err(serde::de::Error::custom)
}
impl WorkdirSessionEnvironment {
pub fn root(capabilities: WorkdirSessionCapabilities) -> Self {
Self {
cwd: WorkdirPath::root(),
scope: None,
capabilities,
restrictions: Vec::new(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn wire_cwd_is_normalized_and_cannot_claim_an_absolute_or_parent_base() {
let mut wire = serde_json::to_value(WorkdirSessionEnvironment::root(
WorkdirSessionCapabilities::READ_ONLY,
))
.unwrap();
for invalid in [
"/absolute",
"../parent",
"nested/../parent",
"nested\\other",
] {
wire["cwd"] = invalid.into();
assert!(
serde_json::from_value::<WorkdirSessionEnvironment>(wire.clone()).is_err(),
"{invalid}"
);
}
wire["cwd"] = "./nested//child".into();
let decoded: WorkdirSessionEnvironment = serde_json::from_value(wire).unwrap();
assert_eq!(decoded.cwd.as_str(), "nested/child");
}
}
+175
View File
@@ -0,0 +1,175 @@
//! Real Local/scoped/read-only/router sessions across loopback HTTP. Only the
//! HTTP authentication boundary is a fixture; coordinate and scope checks are
//! production implementations. Runs in the normal http-client library suite.
#![cfg(all(feature = "http-client", target_os = "linux"))]
use crate::http::*;
use crate::*;
use axum::{
Json, Router,
extract::State,
routing::{delete, post},
};
use std::sync::Arc;
struct Server(tokio::task::JoinHandle<()>);
impl Drop for Server {
fn drop(&mut self) {
self.0.abort();
}
}
async fn open(State(session): State<WorkdirSessionHandle>) -> Json<OpenWorkdirSessionResponse> {
Json(OpenWorkdirSessionResponse {
session_id: WorkdirSessionId::new("environment-session").unwrap(),
workdir_id: session.workdir().id().clone(),
capabilities: session.capabilities(),
environment: session.environment(),
})
}
async fn operation(
State(session): State<WorkdirSessionHandle>,
Json(request): Json<WorkdirSessionOperationRequest>,
) -> Result<
Json<WorkdirSessionOperationResult>,
(axum::http::StatusCode, Json<WorkdirTransportError>),
> {
dispatch_workdir_session_operation(session.as_ref(), request.operation)
.await
.map(Json)
.map_err(|error| {
(
axum::http::StatusCode::FORBIDDEN,
Json(WorkdirTransportError::from_workdir_error(&error)),
)
})
}
#[tokio::test]
async fn remote_environment_matches_scoped_readonly_operations_without_rebasing_twice() {
let root = tempfile::tempdir().unwrap();
std::fs::create_dir(root.path().join("nested")).unwrap();
std::fs::write(root.path().join("nested/file.txt"), "nested marker").unwrap();
std::fs::write(root.path().join("file.txt"), "root decoy").unwrap();
let broker = WorkdirToolBroker::new(Arc::new(LocalWorkdirSession::new(
manifest::Scope::writable(root.path()).unwrap(),
root.path().to_path_buf(),
)));
let lease = broker
.scope(WorkdirToolScope {
cwd: WorkdirPath::new("nested").unwrap(),
rules: vec![WorkdirToolScopeRule {
target: WorkdirPath::new("nested").unwrap(),
permission: WorkdirToolScopePermission::Write,
recursive: true,
symlink_policy: Default::default(),
}],
command: true,
})
.await
.unwrap();
let session: WorkdirSessionHandle = Arc::new(ReadOnlyWorkdirSession::new(lease.tool_session()));
let expected = session.environment();
let app = Router::new()
.route("/v1/working-directories/{id}/sessions", post(open))
.route("/v1/workdir-sessions/{id}/operations", post(operation))
.route(
"/v1/workdir-sessions/{id}",
delete(|| async { axum::http::StatusCode::NO_CONTENT }),
)
.with_state(session.clone());
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let _server = Server(tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
}));
let remote: WorkdirSessionHandle = Arc::new(
RemoteWorkdirSession::open(
reqwest::Client::builder().no_proxy().build().unwrap(),
format!("http://{address}").parse().unwrap(),
"fixture",
session.workdir().id().clone(),
OpenWorkdirSessionRequest::default(),
)
.await
.unwrap(),
);
let router = WorkdirSessionRouter::new();
router
.attach(WorkdirAttachmentAlias::new("remote").unwrap(), remote)
.unwrap();
let selected = router.resolve(Some("remote")).unwrap().session;
assert_eq!(selected.environment(), expected);
assert_eq!(expected.cwd, WorkdirPath::new("nested").unwrap());
assert!(
!expected
.capabilities
.supports(WorkdirSessionCapability::Command)
);
assert!(
expected
.scope
.unwrap()
.iter()
.all(|rule| rule.permission == WorkdirToolScopePermission::Read)
);
let found = selected
.glob(GlobRequest {
path: WorkdirPath::root(),
pattern: "*.txt".into(),
limit: 10,
})
.await
.unwrap();
assert_eq!(found.paths, vec![WorkdirPath::new("file.txt").unwrap()]);
let path = found.paths[0].clone();
let read = selected
.read(ReadRequest {
path: path.clone(),
offset: 0,
limit: 10,
max_bytes: 100,
})
.await
.unwrap();
assert_eq!(read.path, path);
assert_eq!(read.bytes, b"nested marker");
assert!(
selected
.write(WriteRequest {
path,
content: b"denied".to_vec(),
expected_hash: Some(read.content_hash)
})
.await
.is_err()
);
assert!(
selected
.read(ReadRequest {
path: WorkdirPath::new_scoped(root.path().join("file.txt").to_str().unwrap())
.unwrap(),
offset: 0,
limit: 10,
max_bytes: 100
})
.await
.is_err()
);
assert!(
selected
.start_command(CommandRequest {
command: "touch escaped".into(),
cwd: WorkdirPath::root(),
timeout_secs: 5,
transport_byte_budget: 100,
tool_call_id: None
})
.await
.is_err()
);
assert!(!root.path().join("nested/escaped").exists());
router.close_all().await.unwrap();
lease.close().await.unwrap();
broker.close().await.unwrap();
}
+4
View File
@@ -924,6 +924,8 @@ mod tests {
chunks: vec![],
next_cursor: 0,
terminal: Some(crate::CommandTerminal {
process_failure: None,
output_collection: crate::CommandOutputCollection::Complete,
status: crate::CommandStatus::Completed,
exit_code: Some(0),
stdout_bytes: 0,
@@ -1082,6 +1084,8 @@ mod tests {
}],
next_cursor: 3,
terminal: Some(crate::CommandTerminal {
process_failure: None,
output_collection: crate::CommandOutputCollection::Complete,
status: crate::CommandStatus::Completed,
exit_code: Some(0),
stdout_bytes: 100 * 1024 * 1024,
+113 -9
View File
@@ -54,6 +54,7 @@ pub struct OpenWorkdirSessionResponse {
pub session_id: WorkdirSessionId,
pub workdir_id: WorkdirId,
pub capabilities: WorkdirSessionCapabilities,
pub environment: crate::WorkdirSessionEnvironment,
}
/// One provider-side Workdir operation.
@@ -119,6 +120,51 @@ pub enum WorkdirSessionOperationResult {
CommandCancel,
}
/// Audit persistence is independent of the provider result and never authorizes a resend.
/// Only server-generated correlation and closed classifications cross this boundary.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct WorkdirAuditFailure {
pub operation_id: String,
pub stage: WorkdirAuditStage,
pub operation_outcome: WorkdirAuditOperationOutcome,
pub reason: WorkdirAuditFailureReason,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum WorkdirAuditStage {
BeforeDispatch,
DispatchRejected,
ResponseReceived,
ResponseUnknown,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum WorkdirAuditOperationOutcome {
NotDispatched,
Succeeded,
Failed { code: WorkdirTransportErrorCode },
Unknown,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum WorkdirAuditFailureReason {
AuditStoreUnavailable,
}
/// A successful operation can still have failed audit persistence. Consumers must
/// surface that degradation separately, without discarding the result or resending.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
pub struct WorkdirSessionOperationResponse {
#[serde(flatten)]
pub result: WorkdirSessionOperationResult,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub audit_failures: Vec<WorkdirAuditFailure>,
}
/// Dispatch one typed operation against any provider-neutral Workdir session.
///
/// Runtime HTTP handlers, outbound External providers, and Backend brokers can
@@ -208,6 +254,7 @@ pub enum WorkdirTransportErrorCode {
NotFound,
Conflict,
OutcomeUnknown,
NotDispatched,
Unsupported,
InvalidRequest,
Denied,
@@ -231,6 +278,7 @@ impl WorkdirTransportErrorCode {
Self::NotFound => "not_found",
Self::Conflict => "conflict",
Self::OutcomeUnknown => "outcome_unknown",
Self::NotDispatched => "not_dispatched",
Self::Unsupported => "unsupported",
Self::InvalidRequest => "invalid_request",
Self::Denied => "denied",
@@ -261,7 +309,7 @@ impl WorkdirTransportErrorCode {
| Self::SymlinkTargetIsDirectory
| Self::IsDirectory
| Self::SymlinkDirectoryNotTraversed => 400,
Self::Unavailable => 503,
Self::Unavailable | Self::NotDispatched => 503,
Self::Io | Self::Internal | Self::OutcomeUnknown => 500,
Self::Transport => 502,
}
@@ -298,6 +346,9 @@ impl WorkdirTransportError {
Code::OutcomeUnknown,
"Workdir operation may have effects; inspect before retrying",
),
WorkdirError::NotDispatched => {
(Code::NotDispatched, "Workdir operation was not dispatched")
}
WorkdirError::NotFound(_) => (Code::NotFound, "Workdir path was not found"),
WorkdirError::Conflict(_) => (
Code::Conflict,
@@ -375,12 +426,14 @@ impl WorkdirTransportError {
use WorkdirTransportErrorCode as Code;
let error = match self.code {
Code::NotFound => WorkdirError::NotFound("<remote>".into()),
Code::Conflict => WorkdirError::Conflict(self.message),
Code::Conflict => WorkdirError::Conflict(
"The target file's content or existence changed since it was last observed; read the file again before retrying".into(),
),
Code::OutcomeUnknown => WorkdirError::OutcomeUnknown(
"Workdir operation may have effects; inspect before retrying".into(),
),
Code::Unsupported => WorkdirError::UnsupportedOperation(self.message),
Code::InvalidRequest => WorkdirError::InvalidArgument(self.message),
Code::Unsupported => WorkdirError::UnsupportedOperation("Workdir operation is not supported".into()),
Code::InvalidRequest => WorkdirError::InvalidArgument("Workdir operation request is invalid".into()),
Code::Denied => WorkdirError::Denied(crate::WorkdirDenial {
reason: self.denial_reason,
// Provider-authored text is not trusted diagnostic data.
@@ -409,12 +462,13 @@ impl WorkdirTransportError {
target: "<remote-target>".into(),
},
Code::UnknownCommand => WorkdirError::UnknownCommand("<remote>".to_string()),
Code::Unavailable => WorkdirError::Unavailable(self.message),
Code::Unavailable => WorkdirError::Unavailable("Workdir session is unavailable".into()),
Code::NotDispatched => WorkdirError::NotDispatched,
Code::Io => WorkdirError::Io {
path: "<remote>".into(),
source: std::io::Error::other(self.message),
source: std::io::Error::other("Workdir I/O operation failed"),
},
Code::Transport => WorkdirError::Transport(self.message),
Code::Transport => WorkdirError::Transport("Workdir transport failed".into()),
Code::Internal => WorkdirError::OperationFailed,
};
let error = if let Some(reason) = self.failure_reason {
@@ -500,6 +554,7 @@ mod client {
workdir: Workdir,
session_id: WorkdirSessionId,
capabilities: WorkdirSessionCapabilities,
environment: crate::WorkdirSessionEnvironment,
closed: AtomicBool,
}
@@ -549,7 +604,8 @@ mod client {
if opened.workdir_id.as_str() != workdir_id.as_str() {
return Err(WorkdirError::Unavailable(
"Runtime opened a session for a different Workdir".to_string(),
));
)
.with_failure_reason(crate::WorkdirFailureReason::SessionIdentityMismatch));
}
Ok(Self {
client,
@@ -558,6 +614,7 @@ mod client {
workdir: Workdir::new(opened.workdir_id.as_str()),
session_id: opened.session_id,
capabilities: opened.capabilities,
environment: opened.environment,
closed: AtomicBool::new(false),
})
}
@@ -608,6 +665,7 @@ mod client {
WorkdirError::Unavailable(format!(
"Runtime returned a mismatched Workdir operation result; expected {expected}"
))
.with_failure_reason(crate::WorkdirFailureReason::OperationResultMismatch)
}
}
@@ -621,6 +679,12 @@ mod client {
self.capabilities
}
fn environment(&self) -> crate::WorkdirSessionEnvironment {
let mut environment = self.environment.clone();
environment.capabilities = self.capabilities();
environment
}
async fn authorize_scope_path(
&self,
request: WorkdirScopeAuthorizationRequest,
@@ -770,7 +834,8 @@ mod client {
}
Ok(_) if mutation => Err(WorkdirError::OutcomeUnknown(
"mismatched response after checkout mutation".into(),
)),
)
.with_failure_reason(crate::WorkdirFailureReason::OperationResultMismatch)),
Ok(_) => Err(Self::mismatch("checkout_execute")),
Err(error)
if mutation
@@ -1153,6 +1218,9 @@ mod client {
workdir,
session_id: WorkdirSessionId::new("s").unwrap(),
capabilities: WorkdirSessionCapabilities::EMPTY,
environment: crate::WorkdirSessionEnvironment::root(
WorkdirSessionCapabilities::EMPTY,
),
closed: AtomicBool::new(false),
};
let stat = StatRequest {
@@ -1210,6 +1278,37 @@ pub use client::{
mod tests {
use super::*;
#[test]
fn operation_response_keeps_success_and_independent_audit_failure() {
let original =
WorkdirSessionOperationResult::CommandStart(crate::CommandHandle("command-a".into()));
let response = WorkdirSessionOperationResponse {
result: original.clone(),
audit_failures: vec![WorkdirAuditFailure {
operation_id: "op-00000000-0000-7000-8000-000000000001".into(),
stage: WorkdirAuditStage::ResponseReceived,
operation_outcome: WorkdirAuditOperationOutcome::Succeeded,
reason: WorkdirAuditFailureReason::AuditStoreUnavailable,
}],
};
let json = serde_json::to_value(&response).unwrap();
assert_eq!(json["operation"], "command_start");
assert_eq!(json["result"], "command-a");
assert_eq!(
json["audit_failures"][0]["operation_outcome"]["kind"],
"succeeded"
);
assert_eq!(
serde_json::from_value::<WorkdirSessionOperationResponse>(json).unwrap(),
response
);
let normal = serde_json::to_value(&original).unwrap();
let decoded: WorkdirSessionOperationResponse =
serde_json::from_value(normal.clone()).unwrap();
assert!(decoded.audit_failures.is_empty());
assert_eq!(serde_json::to_value(decoded).unwrap(), normal);
}
#[test]
fn transport_error_round_trip_keeps_public_classification() {
for (code, expected_status, expected_error) in [
@@ -1229,6 +1328,11 @@ mod tests {
409,
"The target file's content or existence changed since it was last observed",
),
(
WorkdirTransportErrorCode::NotDispatched,
503,
"not dispatched",
),
(WorkdirTransportErrorCode::Unsupported, 400, "unsupported"),
(WorkdirTransportErrorCode::Denied, 403, "denied"),
(
+137
View File
@@ -5,6 +5,8 @@ pub use checkout::{
CheckoutObservation, CheckoutOperation, CheckoutOutput, CheckoutRequest, CheckoutResult,
CheckoutSearchOperation, CheckoutSearchRequest, CheckoutSearchResult,
};
mod environment;
pub use environment::WorkdirSessionEnvironment;
pub mod external;
pub mod http;
mod local;
@@ -16,6 +18,8 @@ pub mod workspace;
#[cfg(test)]
mod checkout_search_tests;
#[cfg(test)]
mod environment_tests;
#[cfg(test)]
mod external_local_tests;
use std::path::{Path, PathBuf};
@@ -185,6 +189,9 @@ pub type WriteOutcome = WriteResult;
/// Worker-bound operation session for a persistent [`Workdir`].
///
/// Filesystem search and commands run on the host owning materialization.
/// Relative operation paths and reusable search results use the session's cwd;
/// root-coordinate providers are adapted once at attachment, not by each Tool.
/// Scope declarations retain their own root-coordinate authority contract.
/// Session closure is terminal but does not delete the Workdir or files.
/// Unsupported checkout/search implementations explicitly reject requests, with no ambient
/// filesystem fallback. Providers own identity validators; callers own read hash history
@@ -194,6 +201,13 @@ pub trait WorkdirSession: std::fmt::Debug + Send + Sync {
fn workdir(&self) -> &Workdir;
fn capabilities(&self) -> WorkdirSessionCapabilities;
/// Describe the coordinates used by this session, not a caller's Profile
/// or filesystem authority. Root-coordinate providers use the default;
/// coordinate and authority wrappers must project their effective view.
fn environment(&self) -> WorkdirSessionEnvironment {
WorkdirSessionEnvironment::root(self.capabilities())
}
/// Validate an attenuated filesystem rule at the provider boundary without
/// exposing the resolved host path. Providers that cannot resolve symbolic
/// links must reject resolved-policy checks rather than downgrade them.
@@ -408,12 +422,36 @@ impl From<&str> for WorkdirDenial {
pub enum WorkdirFailureReason {
SessionClosed,
ProviderUnavailable,
ProviderGrantExpired,
ProviderAdmissionClosed,
ProviderDispatchRejected,
ProviderResponseLost,
ProviderOperationTimeout,
CommandTerminalMissing,
AuditStoreUnavailable,
HttpTimeout,
HttpConnect,
HttpTransfer,
HttpResponseInvalid,
HttpResponseTooLarge,
OperationResultMismatch,
SessionIdentityMismatch,
SessionOpenTaskFailed,
SessionOpenFailed,
AuthorizationMissing,
AuthorizationStale,
AuthorizationUnavailable,
SigningIdentityUnavailable,
AuthorizationSignFailed,
RuntimeStopped,
RuntimeStateUnavailable,
RuntimeBackendUnavailable,
RuntimeOwnerMismatch,
RuntimeInvalidRequest,
WorkingDirectoryNotFound,
WorkingDirectoryNotActive,
WorkingDirectoryRecordInvalid,
WorkingDirectoryAccessUnavailable,
}
impl WorkdirFailureReason {
@@ -421,12 +459,107 @@ impl WorkdirFailureReason {
match self {
Self::SessionClosed => "session_closed",
Self::ProviderUnavailable => "provider_unavailable",
Self::ProviderGrantExpired => "provider_grant_expired",
Self::ProviderAdmissionClosed => "provider_admission_closed",
Self::ProviderDispatchRejected => "provider_dispatch_rejected",
Self::ProviderResponseLost => "provider_response_lost",
Self::ProviderOperationTimeout => "provider_operation_timeout",
Self::CommandTerminalMissing => "command_terminal_missing",
Self::AuditStoreUnavailable => "audit_store_unavailable",
Self::HttpTimeout => "http_timeout",
Self::HttpConnect => "http_connect",
Self::HttpTransfer => "http_transfer",
Self::HttpResponseInvalid => "http_response_invalid",
Self::HttpResponseTooLarge => "http_response_too_large",
Self::OperationResultMismatch => "operation_result_mismatch",
Self::SessionIdentityMismatch => "session_identity_mismatch",
Self::SessionOpenTaskFailed => "session_open_task_failed",
Self::SessionOpenFailed => "session_open_failed",
Self::AuthorizationMissing => "authorization_missing",
Self::AuthorizationStale => "authorization_stale",
Self::AuthorizationUnavailable => "authorization_unavailable",
Self::SigningIdentityUnavailable => "signing_identity_unavailable",
Self::AuthorizationSignFailed => "authorization_sign_failed",
Self::RuntimeStopped => "runtime_stopped",
Self::RuntimeStateUnavailable => "runtime_state_unavailable",
Self::RuntimeBackendUnavailable => "runtime_backend_unavailable",
Self::RuntimeOwnerMismatch => "runtime_owner_mismatch",
Self::RuntimeInvalidRequest => "runtime_invalid_request",
Self::WorkingDirectoryNotFound => "working_directory_not_found",
Self::WorkingDirectoryNotActive => "working_directory_not_active",
Self::WorkingDirectoryRecordInvalid => "working_directory_record_invalid",
Self::WorkingDirectoryAccessUnavailable => "working_directory_access_unavailable",
}
}
/// The observed failing boundary, not a dispatch or effects assertion.
pub const fn stage(self) -> WorkdirFailureStage {
use WorkdirFailureStage as Stage;
match self {
Self::SessionClosed => Stage::Session,
Self::ProviderUnavailable => Stage::Provider,
Self::ProviderGrantExpired | Self::ProviderAdmissionClosed => Stage::ProviderAdmission,
Self::ProviderDispatchRejected => Stage::ProviderDispatch,
Self::ProviderResponseLost | Self::ProviderOperationTimeout => Stage::ProviderResponse,
Self::AuditStoreUnavailable => Stage::Audit,
Self::CommandTerminalMissing => Stage::Command,
Self::HttpTimeout | Self::HttpConnect | Self::HttpTransfer => Stage::Http,
Self::HttpResponseInvalid
| Self::HttpResponseTooLarge
| Self::OperationResultMismatch => Stage::ResponseValidation,
Self::SessionIdentityMismatch
| Self::SessionOpenTaskFailed
| Self::SessionOpenFailed
| Self::WorkingDirectoryNotFound
| Self::WorkingDirectoryNotActive
| Self::WorkingDirectoryRecordInvalid
| Self::WorkingDirectoryAccessUnavailable => Stage::SessionOpen,
Self::AuthorizationMissing
| Self::AuthorizationStale
| Self::AuthorizationUnavailable
| Self::SigningIdentityUnavailable
| Self::AuthorizationSignFailed => Stage::AuthorizationPreparation,
Self::RuntimeStopped
| Self::RuntimeStateUnavailable
| Self::RuntimeBackendUnavailable
| Self::RuntimeOwnerMismatch
| Self::RuntimeInvalidRequest => Stage::Runtime,
}
}
}
/// Internal operational boundary derived only from a bounded reason.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum WorkdirFailureStage {
Session,
Provider,
ProviderAdmission,
ProviderDispatch,
ProviderResponse,
Audit,
Command,
Http,
ResponseValidation,
SessionOpen,
AuthorizationPreparation,
Runtime,
}
impl WorkdirFailureStage {
pub const fn as_str(self) -> &'static str {
match self {
Self::Session => "session",
Self::Provider => "provider",
Self::ProviderAdmission => "provider_admission",
Self::ProviderDispatch => "provider_dispatch",
Self::ProviderResponse => "provider_response",
Self::Audit => "audit",
Self::Command => "command",
Self::Http => "http",
Self::ResponseValidation => "response_validation",
Self::SessionOpen => "session_open",
Self::AuthorizationPreparation => "authorization_preparation",
Self::Runtime => "runtime",
}
}
}
@@ -466,6 +599,10 @@ pub enum WorkdirError {
#[error("Workdir session is unavailable: {0}")]
Unavailable(String),
/// The broker has proof that this operation never reached the provider.
#[error("Workdir operation was not dispatched")]
NotDispatched,
#[error("Workdir operation failed")]
OperationFailed,
+46
View File
@@ -1340,6 +1340,52 @@ impl WorkdirSession for LocalWorkdirSession {
self.inner.capabilities
}
fn environment(&self) -> crate::WorkdirSessionEnvironment {
let scope = self.inner.scope.snapshot();
let project = |rule: manifest::ScopeRule| -> Option<crate::WorkdirToolScopeRule> {
let target = if self.inner.root.starts_with(&rule.target) && rule.recursive {
WorkdirPath::root()
} else {
let path = rule
.target
.strip_prefix(&self.inner.root)
.unwrap_or(&rule.target);
WorkdirPath::new_scoped(path.to_str()?).ok()?
};
Some(crate::WorkdirToolScopeRule {
target,
permission: match rule.permission {
Permission::Read => WorkdirToolScopePermission::Read,
Permission::Write => WorkdirToolScopePermission::Write,
},
recursive: rule.recursive,
symlink_policy: rule.symlink_policy,
})
};
crate::WorkdirSessionEnvironment {
cwd: WorkdirPath::root(),
scope: Some(
scope
.allow_rules()
.into_iter()
.filter_map(project)
.collect(),
),
capabilities: self.capabilities(),
restrictions: scope
.deny_rules()
.into_iter()
.map(|rule| {
project(rule)
.map(|rule| format!("Deny: {rule:?}"))
.unwrap_or_else(|| {
"Unrepresentable provider deny rule remains enforced".into()
})
})
.collect(),
}
}
async fn authorize_scope_path(
&self,
request: WorkdirScopeAuthorizationRequest,
+172 -48
View File
@@ -3,6 +3,7 @@
//! A replay page survives a lost HTTP response until its next cursor is sent.
use super::*;
use crate::{CommandFailure, CommandFailureStage, CommandIoErrorKind, CommandOutputCollection};
use tokio::io::{AsyncRead, AsyncReadExt};
use tokio::sync::mpsc;
@@ -146,6 +147,33 @@ pub(super) fn start(
telemetry: CommandTelemetry,
environment: BTreeMap<String, String>,
resources: Vec<Arc<dyn WorkdirSessionResource>>,
) -> LocalCommand {
let execution_id = command_id.clone();
let execution_telemetry = telemetry.clone();
start_with(
command_id,
telemetry,
resources,
move |cancel, sender| async move {
execute(
cwd,
request,
&execution_id,
&execution_telemetry,
environment,
cancel,
&sender,
)
.await
},
)
}
fn start_with<F: std::future::Future<Output = crate::CommandTerminal> + Send + 'static>(
command_id: String,
telemetry: CommandTelemetry,
resources: Vec<Arc<dyn WorkdirSessionResource>>,
execution: impl FnOnce(watch::Receiver<bool>, mpsc::Sender<Message>) -> F + Send + 'static,
) -> LocalCommand {
let (sender, receiver) = mpsc::channel(QUEUE_CHUNKS);
let (cancel, cancel_rx) = watch::channel(false);
@@ -160,16 +188,7 @@ pub(super) fn start(
}));
let task = tokio::spawn(async move {
let _resources = resources;
let terminal = execute(
cwd,
request,
&command_id,
&telemetry,
environment,
cancel_rx,
&sender,
)
.await;
let terminal = execution(cancel_rx, sender.clone()).await;
telemetry.terminal(&command_id, terminal.status, terminal.exit_code);
// Closing the byte queue fences this already-published terminal, without
// requiring spare queue capacity or a receiver to release task resources.
@@ -192,10 +211,18 @@ struct ByteAccounting {
}
impl ByteAccounting {
fn terminal(&self, status: CommandStatus, exit_code: Option<i32>) -> crate::CommandTerminal {
fn terminal(
&self,
status: CommandStatus,
exit_code: Option<i32>,
process_failure: Option<CommandFailure>,
output_collection: CommandOutputCollection,
) -> crate::CommandTerminal {
crate::CommandTerminal {
status,
exit_code,
process_failure,
output_collection,
stdout_bytes: self.stdout,
stderr_bytes: self.stderr,
stdout_delivered_bytes: self.stdout_delivered,
@@ -215,7 +242,7 @@ async fn pump<R: AsyncRead + Unpin>(
command_id: &str,
mut remaining: u64,
accounting: &mut ByteAccounting,
) -> std::io::Result<()> {
) -> Result<(), CommandFailure> {
let mut stdout_done = false;
let mut stderr_done = false;
let mut stdout_decoder = CommandOutputDecoder::default();
@@ -228,7 +255,11 @@ async fn pump<R: AsyncRead + Unpin>(
// Before quota, reserve first to retain bounded pipe backpressure.
let permit = if remaining > 0 {
let Ok(permit) = sender.reserve().await else {
return Ok(());
return Err(CommandFailure {
stage: CommandFailureStage::OutputDelivery,
reason: CommandIoErrorKind::Unknown,
os_code: None,
});
};
Some(permit)
} else {
@@ -242,7 +273,15 @@ async fn pump<R: AsyncRead + Unpin>(
CommandStream::Stdout => &stdout_buffer,
CommandStream::Stderr => &stderr_buffer,
};
let count = result?;
let count = result.map_err(|error| {
CommandFailure::io(
match stream {
CommandStream::Stdout => CommandFailureStage::StdoutRead,
CommandStream::Stderr => CommandFailureStage::StderrRead,
},
&error,
)
})?;
let decoder = match stream {
CommandStream::Stdout => &mut stdout_decoder,
CommandStream::Stderr => &mut stderr_decoder,
@@ -288,7 +327,7 @@ async fn execute(
command_id: &str,
telemetry: &CommandTelemetry,
environment: BTreeMap<String, String>,
mut cancel: watch::Receiver<bool>,
cancel: watch::Receiver<bool>,
sender: &mpsc::Sender<Message>,
) -> crate::CommandTerminal {
telemetry.started(command_id, request.tool_call_id);
@@ -304,74 +343,159 @@ async fn execute(
.kill_on_drop(true);
#[cfg(unix)]
command.as_std_mut().process_group(0);
let mut accounting = ByteAccounting::default();
let Ok(mut child) = command.spawn() else {
return accounting.terminal(CommandStatus::Failed, None);
let mut child = match command.spawn() {
Ok(child) => child,
Err(error) => {
return ByteAccounting::default().terminal(
CommandStatus::Failed,
None,
Some(CommandFailure::io(CommandFailureStage::Spawn, &error)),
CommandOutputCollection::NotStarted,
);
}
};
let mut process_group = CommandProcessGroupGuard::new(child.id());
let stdout = child.stdout.take().expect("piped stdout");
let stderr = child.stderr.take().expect("piped stderr");
let (status, exit_code) = {
supervise(
child,
stdout,
stderr,
request.timeout_secs,
request.transport_byte_budget,
command_id,
telemetry,
cancel,
sender,
)
.await
}
// Substitute only syscalls and readers: injected faults run the production supervisor.
trait CommandProcess: Send {
fn id(&self) -> Option<u32>;
fn start_kill(&mut self);
fn wait(
&mut self,
) -> impl std::future::Future<Output = std::io::Result<std::process::ExitStatus>> + Send;
}
impl CommandProcess for tokio::process::Child {
fn id(&self) -> Option<u32> {
self.id()
}
fn start_kill(&mut self) {
let _ = self.start_kill();
}
async fn wait(&mut self) -> std::io::Result<std::process::ExitStatus> {
self.wait().await
}
}
fn process_result(
result: std::io::Result<std::process::ExitStatus>,
) -> (CommandStatus, Option<i32>, Option<CommandFailure>) {
match result {
Ok(exit) => (
if exit.success() {
CommandStatus::Completed
} else {
CommandStatus::Failed
},
exit.code(),
None,
),
Err(error) => (
CommandStatus::Failed,
None,
Some(CommandFailure::io(CommandFailureStage::Wait, &error)),
),
}
}
#[allow(clippy::too_many_arguments)]
async fn supervise(
mut child: impl CommandProcess,
stdout: impl AsyncRead + Unpin,
stderr: impl AsyncRead + Unpin,
timeout_secs: u64,
byte_budget: u64,
command_id: &str,
telemetry: &CommandTelemetry,
mut cancel: watch::Receiver<bool>,
sender: &mpsc::Sender<Message>,
) -> crate::CommandTerminal {
let mut process_group = CommandProcessGroupGuard::new(child.id());
let mut accounting = ByteAccounting::default();
let (status, exit_code, process_failure, output_collection) = {
let output = pump(
stdout,
stderr,
sender,
telemetry,
command_id,
request.transport_byte_budget,
byte_budget,
&mut accounting,
);
tokio::pin!(output);
let mut drained = false;
let timeout = tokio::time::sleep(Duration::from_secs(request.timeout_secs.max(1)));
let mut collected = None;
let timeout = tokio::time::sleep(Duration::from_secs(timeout_secs.max(1)));
tokio::pin!(timeout);
let terminal = loop {
tokio::select! {
result = &mut output, if !drained => {
drained = true;
if result.is_err() || sender.is_closed() {
result = &mut output, if collected.is_none() => {
let failed = result.is_err();
collected = Some(result);
if failed {
process_group.terminate();
let _ = child.start_kill();
let _ = child.wait().await;
break (CommandStatus::Failed, None);
child.start_kill();
break process_result(child.wait().await);
}
}
exit = child.wait() => {
process_group.terminate();
break match exit {
Ok(exit) => (if exit.success() { CommandStatus::Completed } else { CommandStatus::Failed }, exit.code()),
Err(_) => (CommandStatus::Failed, None),
};
break process_result(exit);
}
_ = &mut timeout => {
process_group.terminate();
let _ = child.start_kill();
let code = child.wait().await.ok().and_then(|exit| exit.code());
break (CommandStatus::TimedOut, code);
child.start_kill();
let (_, code, failure) = process_result(child.wait().await);
break (CommandStatus::TimedOut, code, failure);
}
_ = sender.closed() => {
process_group.terminate();
let _ = child.start_kill();
let _ = child.wait().await;
break (CommandStatus::Cancelled, None);
child.start_kill();
let (_, code, failure) = process_result(child.wait().await);
break (CommandStatus::Cancelled, code, failure);
}
changed = cancel.changed() => {
if changed.is_err() || *cancel.borrow() {
process_group.terminate();
let _ = child.start_kill();
let code = child.wait().await.ok().and_then(|exit| exit.code());
break (CommandStatus::Cancelled, code);
child.start_kill();
let (_, code, failure) = process_result(child.wait().await);
break (CommandStatus::Cancelled, code, failure);
}
}
}
};
// Even after kill, preserve the bounded pipe tail before publishing terminal.
if !drained {
let _ = output.await;
}
terminal
// Exit is not pipe EOF. Retain read failures first observed during drain.
let result = match collected {
Some(result) => result,
None => output.await.map_err(|mut failure| {
failure.stage = match failure.stage {
CommandFailureStage::StdoutRead => CommandFailureStage::StdoutDrain,
CommandFailureStage::StderrRead => CommandFailureStage::StderrDrain,
stage => stage,
};
failure
}),
};
let collection = match result {
Ok(()) => CommandOutputCollection::Complete,
Err(failure) => CommandOutputCollection::Incomplete { failure },
};
(terminal.0, terminal.1, terminal.2, collection)
};
accounting.terminal(status, exit_code)
accounting.terminal(status, exit_code, process_failure, output_collection)
}
#[cfg(test)]
@@ -520,3 +520,349 @@ async fn slow_receiver_backpressures_before_quota_but_full_queue_does_not_block_
assert_eq!(queued_bytes, QUEUE_CHUNKS * COMMAND_EVENT_CHUNK_BYTES);
}
}
#[derive(Clone, Default)]
struct Gate(Arc<std::sync::Mutex<(bool, Option<std::task::Waker>)>>);
impl Gate {
fn open(&self) {
let mut state = self.0.lock().unwrap();
state.0 = true;
if let Some(waker) = state.1.take() {
waker.wake();
}
}
fn poll(&self, cx: &mut std::task::Context<'_>) -> std::task::Poll<()> {
let mut state = self.0.lock().unwrap();
if state.0 {
std::task::Poll::Ready(())
} else {
state.1 = Some(cx.waker().clone());
std::task::Poll::Pending
}
}
}
struct FaultProcess {
ready: Gate,
exited: Gate,
wait_error: Option<std::io::ErrorKind>,
}
impl CommandProcess for FaultProcess {
fn id(&self) -> Option<u32> {
None
}
fn start_kill(&mut self) {
self.ready.open();
}
async fn wait(&mut self) -> std::io::Result<std::process::ExitStatus> {
std::future::poll_fn(|cx| self.ready.poll(cx)).await;
self.exited.open();
match self.wait_error {
Some(kind) => Err(std::io::Error::new(
kind,
"/private/path TOKEN=secret command-body",
)),
None => Ok(std::process::ExitStatus::default()),
}
}
}
struct FaultReader {
prefix: &'static [u8],
prefix_read: Gate,
fail_ready: Gate,
}
impl AsyncRead for FaultReader {
fn poll_read(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buffer: &mut tokio::io::ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
if !self.prefix.is_empty() {
let count = self.prefix.len().min(buffer.remaining());
buffer.put_slice(&self.prefix[..count]);
self.prefix = &self.prefix[count..];
self.prefix_read.open();
return std::task::Poll::Ready(Ok(()));
}
std::task::ready!(self.fail_ready.poll(cx));
std::task::Poll::Ready(Err(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"/private/path TOKEN=secret command-body",
)))
}
}
// Faults replace only OS wait/read boundaries, not collection or finalization.
#[tokio::test]
async fn read_and_post_exit_drain_faults_keep_prefix_exit_and_safe_original_reason() {
for stream in [CommandStream::Stdout, CommandStream::Stderr] {
for after_exit in [false, true] {
let ready = Gate::default();
let exited = Gate::default();
let fail_ready = if after_exit {
exited.clone()
} else {
let gate = Gate::default();
gate.open();
gate
};
let reader = FaultReader {
prefix: b"partial",
prefix_read: if after_exit {
ready.clone()
} else {
Gate::default()
},
fail_ready,
};
let process = FaultProcess {
ready,
exited,
wait_error: None,
};
let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS);
let (_cancel, cancel_rx) = watch::channel(false);
let telemetry = CommandTelemetry::new();
let terminal = tokio::time::timeout(Duration::from_secs(5), async {
match stream {
CommandStream::Stdout => {
supervise(
process,
reader,
&b""[..],
10,
100,
"fault",
&telemetry,
cancel_rx,
&sender,
)
.await
}
CommandStream::Stderr => {
supervise(
process,
&b""[..],
reader,
10,
100,
"fault",
&telemetry,
cancel_rx,
&sender,
)
.await
}
}
})
.await
.unwrap();
let Message::Bytes(chunk) = receiver.recv().await.unwrap();
assert_eq!(chunk.bytes, b"partial");
assert_eq!(chunk.stream, stream);
assert_eq!(chunk.sequence, 0);
assert_eq!(chunk.offset, 0);
assert!(receiver.try_recv().is_err());
assert_eq!(terminal.status, CommandStatus::Completed);
assert_eq!(terminal.exit_code, Some(0));
assert_eq!(terminal.process_failure, None);
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7);
assert_eq!(
terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes,
7
);
assert_eq!(terminal.truncation_reason, None);
assert_eq!(
terminal.output_collection,
CommandOutputCollection::Incomplete {
failure: CommandFailure {
stage: match (stream, after_exit) {
(CommandStream::Stdout, false) => CommandFailureStage::StdoutRead,
(CommandStream::Stdout, true) => CommandFailureStage::StdoutDrain,
(CommandStream::Stderr, false) => CommandFailureStage::StderrRead,
(CommandStream::Stderr, true) => CommandFailureStage::StderrDrain,
},
reason: CommandIoErrorKind::PermissionDenied,
os_code: None
},
}
);
let wire = serde_json::to_string(&terminal).unwrap();
for secret in ["/private", "TOKEN", "secret", "command-body"] {
assert!(!wire.contains(secret));
}
}
}
}
#[tokio::test]
async fn wait_fault_keeps_collected_bytes_and_does_not_invent_exit_information() {
let ready = Gate::default();
ready.open();
let process = FaultProcess {
ready,
exited: Gate::default(),
wait_error: Some(std::io::ErrorKind::InvalidData),
};
let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS);
let (_cancel, cancel_rx) = watch::channel(false);
let terminal = supervise(
process,
&b"out"[..],
&b"err"[..],
10,
100,
"wait-fault",
&CommandTelemetry::new(),
cancel_rx,
&sender,
)
.await;
assert_eq!(terminal.status, CommandStatus::Failed);
assert_eq!(terminal.exit_code, None);
assert_eq!(
terminal.process_failure,
Some(CommandFailure {
stage: CommandFailureStage::Wait,
reason: CommandIoErrorKind::InvalidData,
os_code: None
})
);
assert_eq!(
terminal.output_collection,
CommandOutputCollection::Complete
);
assert_eq!(terminal.stdout_bytes, 3);
assert_eq!(terminal.stderr_bytes, 3);
let mut chunks = Vec::new();
while let Ok(Message::Bytes(chunk)) = receiver.try_recv() {
chunks.push(chunk);
}
assert_eq!(stream_bytes(&chunks, CommandStream::Stdout), b"out");
assert_eq!(stream_bytes(&chunks, CommandStream::Stderr), b"err");
assert!(!serde_json::to_string(&terminal).unwrap().contains("secret"));
}
#[tokio::test]
async fn spawn_fault_retains_safe_reason_without_claiming_output_collection() {
let root = TempDir::new().unwrap();
let (sender, _receiver) = mpsc::channel(QUEUE_CHUNKS);
let (_cancel, cancel_rx) = watch::channel(false);
let terminal = execute(
root.path().join("missing-secret-path"),
Harness::request("secret-command", 10, 100),
"spawn-fault",
&CommandTelemetry::new(),
BTreeMap::new(),
cancel_rx,
&sender,
)
.await;
assert_eq!(
terminal.process_failure,
Some(CommandFailure {
stage: CommandFailureStage::Spawn,
reason: CommandIoErrorKind::NotFound,
os_code: Some(libc::ENOENT)
})
);
assert_eq!(
terminal.output_collection,
CommandOutputCollection::NotStarted
);
assert_eq!(terminal.exit_code, None);
assert_eq!(terminal.status, CommandStatus::Failed);
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 0);
assert!(!serde_json::to_string(&terminal).unwrap().contains("secret"));
}
#[cfg(feature = "http-client")]
mod http;
#[tokio::test]
async fn drain_failure_and_byte_budget_keep_independent_loss_diagnostics() {
for stream in [CommandStream::Stdout, CommandStream::Stderr] {
let ready = Gate::default();
let exited = Gate::default();
let process = FaultProcess {
ready: ready.clone(),
exited: exited.clone(),
wait_error: None,
};
let reader = FaultReader {
prefix: b"partial",
prefix_read: ready,
fail_ready: exited,
};
let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS);
let (_cancel, cancel_rx) = watch::channel(false);
let telemetry = CommandTelemetry::new();
let terminal = tokio::time::timeout(Duration::from_secs(5), async {
match stream {
CommandStream::Stdout => {
supervise(
process,
reader,
&b""[..],
10,
3,
"quota-fault",
&telemetry,
cancel_rx,
&sender,
)
.await
}
CommandStream::Stderr => {
supervise(
process,
&b""[..],
reader,
10,
3,
"quota-fault",
&telemetry,
cancel_rx,
&sender,
)
.await
}
}
})
.await
.unwrap();
let Message::Bytes(chunk) = receiver.recv().await.unwrap();
assert_eq!(chunk.stream, stream);
assert_eq!(chunk.bytes, b"par");
assert!(receiver.try_recv().is_err());
assert_eq!(terminal.exit_code, Some(0));
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7);
assert_eq!(
terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes,
3
);
assert_eq!(
terminal.truncation_reason,
Some(crate::CommandOutputTruncation::ByteBudget)
);
assert_eq!(
terminal.output_collection,
CommandOutputCollection::Incomplete {
failure: CommandFailure {
stage: match stream {
CommandStream::Stdout => CommandFailureStage::StdoutDrain,
CommandStream::Stderr => CommandFailureStage::StderrDrain,
},
reason: CommandIoErrorKind::PermissionDenied,
os_code: None,
},
}
);
assert!(terminal.metadata_valid());
}
}
@@ -0,0 +1,241 @@
//! Loopback integration of the real supervisor, byte queue/replay, local session,
//! HTTP dispatcher/client. OS wait/read are
//! deterministic faults; session open and start bind that injected process fixture.
//! Authorization is not asserted here (Runtime's signed HTTP suite owns it).
use super::*;
use crate::http::*;
use axum::{Json, Router, extract::State, routing::post};
struct Server(tokio::task::JoinHandle<()>);
impl Drop for Server {
fn drop(&mut self) {
self.0.abort();
}
}
async fn operation(
State(session): State<Arc<LocalWorkdirSession>>,
Json(request): Json<WorkdirSessionOperationRequest>,
) -> Json<WorkdirSessionOperationResult> {
if matches!(request.operation, WorkdirSessionOperation::CommandStart(_)) {
return Json(WorkdirSessionOperationResult::CommandStart(CommandHandle(
"injected".into(),
)));
}
Json(
dispatch_workdir_session_operation(session.as_ref(), request.operation)
.await
.unwrap(),
)
}
#[tokio::test]
async fn pipe_and_wait_faults_cross_real_http_with_prefix_and_terminal_replay() {
tokio::time::timeout(Duration::from_secs(20), async {
for (stream, after_exit, wait_fault) in [
(CommandStream::Stdout, false, false),
(CommandStream::Stderr, false, false),
(CommandStream::Stdout, true, false),
(CommandStream::Stderr, true, false),
(CommandStream::Stdout, false, true),
] {
let root = TempDir::new().unwrap();
let session = Arc::new(LocalWorkdirSession::new(
Scope::writable(root.path()).unwrap(),
root.path().into(),
));
let ready = Gate::default();
let exited = Gate::default();
if wait_fault {
ready.open();
}
let process = FaultProcess {
ready: ready.clone(),
exited: exited.clone(),
wait_error: wait_fault.then_some(std::io::ErrorKind::InvalidData),
};
let fail_ready = if after_exit {
exited
} else {
let gate = Gate::default();
gate.open();
gate
};
let reader = FaultReader {
prefix: b"partial",
prefix_read: if after_exit { ready } else { Gate::default() },
fail_ready,
};
let command = start_with(
"injected".into(),
CommandTelemetry::new(),
vec![],
move |cancel, sender| async move {
let telemetry = CommandTelemetry::new();
if wait_fault {
supervise(
process,
&b"partial"[..],
&b""[..],
10,
100,
"injected",
&telemetry,
cancel,
&sender,
)
.await
} else {
match stream {
CommandStream::Stdout => {
supervise(
process,
reader,
&b""[..],
10,
100,
"injected",
&telemetry,
cancel,
&sender,
)
.await
}
CommandStream::Stderr => {
supervise(
process,
&b""[..],
reader,
10,
100,
"injected",
&telemetry,
cancel,
&sender,
)
.await
}
}
}
},
);
session
.inner
.commands
.lock()
.await
.insert("injected".into(), command);
let app = Router::new()
.route(
"/v1/working-directories/{id}/sessions",
post(|| async {
Json(OpenWorkdirSessionResponse {
session_id: WorkdirSessionId::new("fault-session").unwrap(),
workdir_id: crate::Workdir::new("fault-workdir").id().clone(),
capabilities: crate::WorkdirSessionCapabilities::ALL,
environment: crate::WorkdirSessionEnvironment::root(
crate::WorkdirSessionCapabilities::ALL,
),
})
}),
)
.route("/v1/workdir-sessions/{id}/operations", post(operation))
.with_state(session.clone());
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
let _server = Server(tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
}));
let remote: crate::WorkdirSessionHandle = Arc::new(
RemoteWorkdirSession::open(
reqwest::Client::builder().no_proxy().build().unwrap(),
base.parse().unwrap(),
"fixture-token",
crate::Workdir::new("fault-workdir").id().clone(),
OpenWorkdirSessionRequest::default(),
)
.await
.unwrap(),
);
let handle = CommandHandle("injected".into());
let mut request = CommandOutputRequest {
handle,
cursor: 0,
limit: 1,
wait: true,
};
let mut bytes = Vec::new();
let terminal = loop {
let page = remote.command_output(request.clone()).await.unwrap();
assert_eq!(remote.command_output(request.clone()).await.unwrap(), page);
for chunk in &page.chunks {
assert_eq!(chunk.sequence, request.cursor);
assert_eq!(chunk.offset, request.cursor);
assert_eq!(chunk.stream, stream);
bytes.extend_from_slice(&chunk.bytes);
request.cursor += chunk.bytes.len() as u64;
}
assert_eq!(page.next_cursor, request.cursor);
if let Some(terminal) = page.terminal {
break terminal;
}
};
assert_eq!(bytes, b"partial");
assert_eq!(terminal.truncation_reason, None);
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7);
assert_eq!(
terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes,
7
);
if wait_fault {
assert_eq!(terminal.status, CommandStatus::Failed);
assert_eq!(terminal.exit_code, None);
assert_eq!(
terminal.process_failure,
Some(CommandFailure {
stage: CommandFailureStage::Wait,
reason: CommandIoErrorKind::InvalidData,
os_code: None
})
);
assert_eq!(
terminal.output_collection,
CommandOutputCollection::Complete
);
} else {
assert_eq!(terminal.status, CommandStatus::Completed);
assert_eq!(terminal.exit_code, Some(0));
let stage = match (stream, after_exit) {
(CommandStream::Stdout, false) => CommandFailureStage::StdoutRead,
(CommandStream::Stderr, false) => CommandFailureStage::StderrRead,
(CommandStream::Stdout, true) => CommandFailureStage::StdoutDrain,
(CommandStream::Stderr, true) => CommandFailureStage::StderrDrain,
};
assert_eq!(
terminal.output_collection,
CommandOutputCollection::Incomplete {
failure: CommandFailure {
stage,
reason: CommandIoErrorKind::PermissionDenied,
os_code: None
}
}
);
}
let wire = serde_json::to_string(&terminal).unwrap();
for secret in [
"/private",
"TOKEN",
"secret",
"command-body",
"fixture-token",
] {
assert!(!wire.contains(secret));
}
session.close().await.unwrap();
}
})
.await
.expect("bounded loopback fault projection");
}
+257 -10
View File
@@ -132,12 +132,151 @@ pub enum CommandOutputTruncation {
ByteBudget,
}
/// Raw totals include discarded pipe bytes. Delivered totals count only prefixes
/// admitted to the transport queue, not page reads or retries by a consumer.
/// Closed diagnostic vocabulary: never serialize OS messages, paths, or errno text.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum CommandIoErrorKind {
NotFound,
PermissionDenied,
ConnectionRefused,
ConnectionReset,
ConnectionAborted,
NotConnected,
BrokenPipe,
AlreadyExists,
WouldBlock,
InvalidInput,
InvalidData,
TimedOut,
WriteZero,
Interrupted,
UnexpectedEof,
OutOfMemory,
HostUnreachable,
NetworkUnreachable,
AddrInUse,
AddrNotAvailable,
NetworkDown,
NotADirectory,
IsADirectory,
DirectoryNotEmpty,
ReadOnlyFilesystem,
StaleNetworkFileHandle,
StorageFull,
NotSeekable,
QuotaExceeded,
FileTooLarge,
ResourceBusy,
ExecutableFileBusy,
Deadlock,
CrossesDevices,
TooManyLinks,
InvalidFilename,
ArgumentListTooLong,
Unsupported,
Other,
Unknown,
}
impl From<std::io::ErrorKind> for CommandIoErrorKind {
fn from(kind: std::io::ErrorKind) -> Self {
use std::io::ErrorKind as Io;
match kind {
Io::NotFound => Self::NotFound,
Io::PermissionDenied => Self::PermissionDenied,
Io::ConnectionRefused => Self::ConnectionRefused,
Io::ConnectionReset => Self::ConnectionReset,
Io::ConnectionAborted => Self::ConnectionAborted,
Io::NotConnected => Self::NotConnected,
Io::BrokenPipe => Self::BrokenPipe,
Io::AlreadyExists => Self::AlreadyExists,
Io::WouldBlock => Self::WouldBlock,
Io::InvalidInput => Self::InvalidInput,
Io::InvalidData => Self::InvalidData,
Io::TimedOut => Self::TimedOut,
Io::WriteZero => Self::WriteZero,
Io::Interrupted => Self::Interrupted,
Io::UnexpectedEof => Self::UnexpectedEof,
Io::OutOfMemory => Self::OutOfMemory,
Io::HostUnreachable => Self::HostUnreachable,
Io::NetworkUnreachable => Self::NetworkUnreachable,
Io::AddrInUse => Self::AddrInUse,
Io::AddrNotAvailable => Self::AddrNotAvailable,
Io::NetworkDown => Self::NetworkDown,
Io::NotADirectory => Self::NotADirectory,
Io::IsADirectory => Self::IsADirectory,
Io::DirectoryNotEmpty => Self::DirectoryNotEmpty,
Io::ReadOnlyFilesystem => Self::ReadOnlyFilesystem,
Io::StaleNetworkFileHandle => Self::StaleNetworkFileHandle,
Io::StorageFull => Self::StorageFull,
Io::NotSeekable => Self::NotSeekable,
Io::QuotaExceeded => Self::QuotaExceeded,
Io::FileTooLarge => Self::FileTooLarge,
Io::ResourceBusy => Self::ResourceBusy,
Io::ExecutableFileBusy => Self::ExecutableFileBusy,
Io::Deadlock => Self::Deadlock,
Io::CrossesDevices => Self::CrossesDevices,
Io::TooManyLinks => Self::TooManyLinks,
Io::InvalidFilename => Self::InvalidFilename,
Io::ArgumentListTooLong => Self::ArgumentListTooLong,
Io::Unsupported => Self::Unsupported,
Io::Other => Self::Other,
_ => Self::Unknown,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum CommandFailureStage {
Spawn,
Wait,
StdoutRead,
StderrRead,
StdoutDrain,
StderrDrain,
OutputDelivery,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct CommandFailure {
pub stage: CommandFailureStage,
pub reason: CommandIoErrorKind,
/// Observed numeric OS code, if supplied by the source. Never inferred from text.
pub os_code: Option<i32>,
}
impl CommandFailure {
pub(crate) fn io(stage: CommandFailureStage, error: &std::io::Error) -> Self {
Self {
stage,
reason: error.kind().into(),
os_code: error.raw_os_error(),
}
}
}
/// EOF collection is independent of exit status and the explicit byte budget.
/// Incomplete counts are lower bounds on bytes actually produced by the process.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)]
pub enum CommandOutputCollection {
NotStarted,
Complete,
Incomplete { failure: CommandFailure },
}
/// Raw totals include discarded *observed* pipe bytes, not an unknown failed tail.
/// Delivered totals count only prefixes admitted to the transport queue, not page
/// reads or retries. Neither terminal arrival nor exit zero implies EOF collection.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct CommandTerminal {
pub status: CommandStatus,
pub exit_code: Option<i32>,
pub process_failure: Option<CommandFailure>,
pub output_collection: CommandOutputCollection,
pub stdout_bytes: u64,
pub stderr_bytes: u64,
pub stdout_delivered_bytes: u64,
@@ -145,6 +284,54 @@ pub struct CommandTerminal {
pub truncation_reason: Option<CommandOutputTruncation>,
}
impl CommandTerminal {
/// Validate provider-authored metadata before publishing it as collected evidence.
pub fn metadata_valid(&self) -> bool {
let truncated = self.stdout_bytes > self.stdout_delivered_bytes
|| self.stderr_bytes > self.stderr_delivered_bytes;
if self.status == CommandStatus::Running
|| self.stdout_delivered_bytes > self.stdout_bytes
|| self.stderr_delivered_bytes > self.stderr_bytes
|| self.truncation_reason != truncated.then_some(CommandOutputTruncation::ByteBudget)
{
return false;
}
match self.process_failure {
Some(CommandFailure {
stage: CommandFailureStage::Spawn,
..
}) => {
self.status == CommandStatus::Failed
&& self.exit_code.is_none()
&& self.output_collection == CommandOutputCollection::NotStarted
&& self.stdout_bytes == 0
&& self.stderr_bytes == 0
}
Some(CommandFailure {
stage: CommandFailureStage::Wait,
..
}) => {
self.status != CommandStatus::Completed
&& self.exit_code.is_none()
&& self.collection_started()
}
Some(_) => false,
None => self.collection_started(),
}
}
fn collection_started(&self) -> bool {
match self.output_collection {
CommandOutputCollection::NotStarted => false,
CommandOutputCollection::Complete => true,
CommandOutputCollection::Incomplete { failure } => !matches!(
failure.stage,
CommandFailureStage::Spawn | CommandFailureStage::Wait
),
}
}
}
/// Sending next_cursor acknowledges this page. Only the current page may be
/// retried; acknowledged bytes expire. Terminal fences all captured bytes,
/// including on timeout/cancel. Consumers must pull while the process runs.
@@ -186,19 +373,13 @@ impl CommandOutput {
self.next_cursor <= MAX_COMMAND_TRANSPORT_BYTES
&& end.is_none_or(|end| end == self.next_cursor)
&& self.terminal.as_ref().is_none_or(|terminal| {
let truncated = terminal.stdout_bytes > terminal.stdout_delivered_bytes
|| terminal.stderr_bytes > terminal.stderr_delivered_bytes;
stream_ends[0].is_none_or(|end| end == terminal.stdout_delivered_bytes)
terminal.metadata_valid()
&& stream_ends[0].is_none_or(|end| end == terminal.stdout_delivered_bytes)
&& stream_ends[1].is_none_or(|end| end == terminal.stderr_delivered_bytes)
&& terminal.status != CommandStatus::Running
&& terminal.stdout_delivered_bytes <= terminal.stdout_bytes
&& terminal.stderr_delivered_bytes <= terminal.stderr_bytes
&& terminal
.stdout_delivered_bytes
.checked_add(terminal.stderr_delivered_bytes)
== Some(self.next_cursor)
&& terminal.truncation_reason
== truncated.then_some(CommandOutputTruncation::ByteBudget)
})
}
}
@@ -262,4 +443,70 @@ mod tests {
.contains(&serde_json::json!("transport_byte_budget"))
);
}
#[test]
fn failure_wire_is_closed_and_does_not_infer_os_codes_from_custom_messages() {
let error = std::io::Error::new(
std::io::ErrorKind::Other,
"/host/private TOKEN=secret EACCES command",
);
let failure = CommandFailure::io(CommandFailureStage::Wait, &error);
assert_eq!(
serde_json::to_value(failure).unwrap(),
serde_json::json!({"stage":"wait","reason":"other","os_code":null})
);
let os_error = std::io::Error::from_raw_os_error(libc::EIO);
let actual = CommandFailure::io(CommandFailureStage::StdoutDrain, &os_error);
assert_eq!(actual.os_code, Some(libc::EIO));
for invalid in [
serde_json::json!({"stage":"wait","reason":"/host/private TOKEN=secret"}),
serde_json::json!({"stage":"/host/private","reason":"other"}),
serde_json::json!({"stage":"wait","reason":"other","message":"secret"}),
] {
assert!(serde_json::from_value::<CommandFailure>(invalid).is_err());
}
}
#[test]
fn provider_terminal_rejects_inconsistent_failure_domains_and_unstarted_byte_claims() {
let wire = serde_json::json!({"status":"completed","exit_code":0,"process_failure":null,"output_collection":{"state":"incomplete","failure":{"stage":"stdout_drain","reason":"other","os_code":null}},"stdout_bytes":3,"stderr_bytes":0,"stdout_delivered_bytes":3,"stderr_delivered_bytes":0,"truncation_reason":null});
let terminal: CommandTerminal = serde_json::from_value(wire.clone()).unwrap();
assert!(terminal.metadata_valid());
for (field, value) in [
(
"output_collection",
serde_json::json!({"state":"not_started"}),
),
(
"output_collection",
serde_json::json!({"state":"incomplete","failure":{"stage":"wait","reason":"other"}}),
),
(
"process_failure",
serde_json::json!({"stage":"stdout_read","reason":"other"}),
),
(
"process_failure",
serde_json::json!({"stage":"spawn","reason":"not_found"}),
),
("truncation_reason", serde_json::json!("byte_budget")),
] {
let mut invalid = wire.clone();
invalid[field] = value;
assert!(
!serde_json::from_value::<CommandTerminal>(invalid)
.unwrap()
.metadata_valid()
);
}
let mut unclassified = wire.clone();
unclassified
.as_object_mut()
.unwrap()
.remove("output_collection");
assert!(serde_json::from_value::<CommandTerminal>(unclassified).is_err());
let mut raw = wire;
raw["message"] = serde_json::json!("/private TOKEN=secret");
assert!(serde_json::from_value::<CommandTerminal>(raw).is_err());
}
}
+236 -31
View File
@@ -225,16 +225,27 @@ impl Drop for ActiveOperation<'_> {
#[derive(Debug, Clone)]
pub struct RoutedWorkdirSession {
state: Arc<RoutedSessionState>,
view: Option<WorkdirSessionHandle>,
}
impl RoutedWorkdirSession {
fn inner(&self) -> &WorkdirSessionHandle {
self.view.as_ref().unwrap_or(&self.state.session)
}
}
#[async_trait]
impl WorkdirSession for RoutedWorkdirSession {
fn workdir(&self) -> &Workdir {
self.state.session.workdir()
self.inner().workdir()
}
fn capabilities(&self) -> WorkdirSessionCapabilities {
self.state.session.capabilities()
self.inner().capabilities()
}
fn environment(&self) -> crate::WorkdirSessionEnvironment {
self.inner().environment()
}
async fn authorize_scope_path(
@@ -242,7 +253,7 @@ impl WorkdirSession for RoutedWorkdirSession {
request: WorkdirScopeAuthorizationRequest,
) -> Result<(), WorkdirError> {
let _active = self.state.enter()?;
self.state.session.authorize_scope_path(request).await
self.inner().authorize_scope_path(request).await
}
async fn scope_rules_overlap(
@@ -250,7 +261,7 @@ impl WorkdirSession for RoutedWorkdirSession {
request: WorkdirScopeOverlapRequest,
) -> Result<bool, WorkdirError> {
let _active = self.state.enter()?;
self.state.session.scope_rules_overlap(request).await
self.inner().scope_rules_overlap(request).await
}
async fn checkout_search(
@@ -258,7 +269,7 @@ impl WorkdirSession for RoutedWorkdirSession {
request: crate::CheckoutSearchRequest,
) -> Result<crate::CheckoutSearchResult, WorkdirError> {
let _active = self.state.enter()?;
self.state.session.checkout_search(request).await
self.inner().checkout_search(request).await
}
async fn checkout_observe(
@@ -266,59 +277,59 @@ impl WorkdirSession for RoutedWorkdirSession {
path: crate::WorkdirPath,
) -> Result<crate::CheckoutObservation, WorkdirError> {
let _active = self.state.enter()?;
self.state.session.checkout_observe(path).await
self.inner().checkout_observe(path).await
}
async fn checkout_execute(
&self,
request: crate::CheckoutRequest,
) -> Result<crate::CheckoutResult, WorkdirError> {
let _active = self.state.enter()?;
self.state.session.checkout_execute(request).await
self.inner().checkout_execute(request).await
}
async fn stat(&self, request: StatRequest) -> Result<StatResult, WorkdirError> {
let _active = self.state.enter()?;
self.state.session.stat(request).await
self.inner().stat(request).await
}
async fn read(&self, request: ReadRequest) -> Result<ReadResult, WorkdirError> {
let _active = self.state.enter()?;
self.state.session.read(request).await
self.inner().read(request).await
}
async fn read_bytes(&self, request: ReadBytesRequest) -> Result<ReadBytesResult, WorkdirError> {
let _active = self.state.enter()?;
self.state.session.read_bytes(request).await
self.inner().read_bytes(request).await
}
async fn write(&self, request: WriteRequest) -> Result<WriteResult, WorkdirError> {
let _active = self.state.enter()?;
self.state.session.write(request).await
self.inner().write(request).await
}
async fn edit(&self, request: EditRequest) -> Result<EditResult, WorkdirError> {
let _active = self.state.enter()?;
self.state.session.edit(request).await
self.inner().edit(request).await
}
async fn list(&self, request: ListRequest) -> Result<ListResult, WorkdirError> {
let _active = self.state.enter()?;
self.state.session.list(request).await
self.inner().list(request).await
}
async fn glob(&self, request: GlobRequest) -> Result<GlobResult, WorkdirError> {
let _active = self.state.enter()?;
self.state.session.glob(request).await
self.inner().glob(request).await
}
async fn grep(&self, request: GrepRequest) -> Result<GrepResult, WorkdirError> {
let _active = self.state.enter()?;
self.state.session.grep(request).await
self.inner().grep(request).await
}
async fn start_command(&self, request: CommandRequest) -> Result<CommandHandle, WorkdirError> {
let _active = self.state.enter()?;
let handle = self.state.session.start_command(request).await?;
let handle = self.inner().start_command(request).await?;
self.state
.active_commands
.lock()
@@ -329,7 +340,7 @@ impl WorkdirSession for RoutedWorkdirSession {
async fn command_status(&self, handle: CommandHandle) -> Result<CommandStatus, WorkdirError> {
let _active = self.state.enter()?;
let status = self.state.session.command_status(handle.clone()).await?;
let status = self.inner().command_status(handle.clone()).await?;
self.state.mark_terminal(&handle, status);
Ok(status)
}
@@ -340,7 +351,7 @@ impl WorkdirSession for RoutedWorkdirSession {
) -> Result<CommandOutput, WorkdirError> {
let _active = self.state.enter()?;
let handle = request.handle.clone();
let output = self.state.session.command_output(request).await?;
let output = self.inner().command_output(request).await?;
if let Some(terminal) = &output.terminal {
self.state.mark_terminal(&handle, terminal.status);
}
@@ -349,7 +360,7 @@ impl WorkdirSession for RoutedWorkdirSession {
async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError> {
let _active = self.state.enter()?;
self.state.session.cancel_command(handle.clone()).await?;
self.inner().cancel_command(handle.clone()).await?;
self.state
.active_commands
.lock()
@@ -359,11 +370,11 @@ impl WorkdirSession for RoutedWorkdirSession {
}
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<CommandEvent>> {
self.state.session.subscribe_command_events()
self.inner().subscribe_command_events()
}
fn command_snapshot(&self) -> Vec<CommandSnapshot> {
let snapshots = self.state.session.command_snapshot();
let snapshots = self.inner().command_snapshot();
self.state.reconcile_terminal_commands(&snapshots);
snapshots
}
@@ -374,7 +385,7 @@ impl WorkdirSession for RoutedWorkdirSession {
"Workdir attachment has active operations or commands".to_string(),
));
}
self.state.session.close().await
self.inner().close().await
}
}
@@ -387,6 +398,7 @@ impl WorkdirSession for RoutedWorkdirSession {
pub struct WorkdirSessionRouter {
sessions: RwLock<BTreeMap<WorkdirAttachmentAlias, Arc<RoutedSessionState>>>,
next_generation: AtomicU64,
coordinate_brokers: RwLock<BTreeMap<WorkdirAttachmentAlias, (u64, crate::WorkdirToolBroker)>>,
}
impl WorkdirSessionRouter {
@@ -473,9 +485,9 @@ impl WorkdirSessionRouter {
}
};
Ok(ResolvedWorkdirSession {
session: self.routed(&selected.0, selected.1.clone()),
alias: selected.0,
generation: selected.1.generation,
session: Arc::new(RoutedWorkdirSession { state: selected.1 }),
})
}
@@ -483,13 +495,22 @@ impl WorkdirSessionRouter {
&self,
alias: WorkdirAttachmentAlias,
session: WorkdirSessionHandle,
) -> Result<WorkdirSessionHandle, WorkdirError> {
self.attach_with_cwd(alias, session, None)
}
fn attach_with_cwd(
&self,
alias: WorkdirAttachmentAlias,
session: WorkdirSessionHandle,
cwd: Option<crate::WorkdirPath>,
) -> Result<WorkdirSessionHandle, WorkdirError> {
let mut sessions = self.sessions.write().map_err(|_| {
WorkdirError::Unavailable("Workdir attachment router is poisoned".to_string())
WorkdirError::Unavailable("Workdir attachment router is poisoned".into())
})?;
if sessions.contains_key(&alias) {
return Err(WorkdirError::Conflict(format!(
"Workdir attachment alias `{alias}` already exists"
"Workdir attachment alias \u{0060}{alias}\u{0060} already exists"
)));
}
let state = Arc::new(RoutedSessionState {
@@ -499,8 +520,61 @@ impl WorkdirSessionRouter {
active_operations: AtomicUsize::new(0),
active_commands: Mutex::new(HashSet::new()),
});
let view = if let Some(cwd) = cwd {
let source = Arc::new(RoutedWorkdirSession {
state: state.clone(),
view: None,
});
let broker = crate::WorkdirToolBroker::at_cwd(source, cwd)?;
let view = broker.tool_session();
self.coordinate_brokers
.write()
.map_err(|_| {
WorkdirError::Unavailable("Workdir coordinate broker map is poisoned".into())
})?
.insert(alias.clone(), (state.generation, broker));
Some(view)
} else {
None
};
sessions.insert(alias, state.clone());
Ok(Arc::new(RoutedWorkdirSession { state }))
Ok(Arc::new(RoutedWorkdirSession { state, view }))
}
fn routed(
&self,
alias: &WorkdirAttachmentAlias,
state: Arc<RoutedSessionState>,
) -> Arc<RoutedWorkdirSession> {
let view = self
.coordinate_broker(alias, state.generation)
.map(|broker| broker.tool_session());
Arc::new(RoutedWorkdirSession { state, view })
}
pub(crate) fn coordinate_broker(
&self,
alias: &WorkdirAttachmentAlias,
generation: u64,
) -> Option<crate::WorkdirToolBroker> {
self.coordinate_brokers
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(alias)
.filter(|(current, _)| *current == generation)
.map(|(_, broker)| broker.clone())
}
/// Bind tool coordinates once while retaining root-coordinate delegation
/// and provider authorization. Both views share the route's detach fence.
pub fn attach_at_cwd(
&self,
alias: WorkdirAttachmentAlias,
source: WorkdirSessionHandle,
cwd: crate::WorkdirPath,
) -> Result<WorkdirSessionHandle, WorkdirError> {
let cwd = crate::WorkdirPath::new(cwd.as_str())?;
self.attach_with_cwd(alias, source, Some(cwd))
}
pub fn session(&self, alias: &WorkdirAttachmentAlias) -> Option<WorkdirSessionHandle> {
@@ -509,7 +583,7 @@ impl WorkdirSessionRouter {
.ok()?
.get(alias)
.cloned()
.map(|state| Arc::new(RoutedWorkdirSession { state }) as WorkdirSessionHandle)
.map(|state| self.routed(alias, state) as WorkdirSessionHandle)
}
/// Compatibility projection used only while filesystem tool schemas have no
@@ -521,10 +595,8 @@ impl WorkdirSessionRouter {
return None;
}
sessions
.values()
.next()
.cloned()
.map(|state| Arc::new(RoutedWorkdirSession { state }) as WorkdirSessionHandle)
.first_key_value()
.map(|(alias, state)| self.routed(alias, state.clone()) as WorkdirSessionHandle)
}
pub fn can_detach(&self, alias: &WorkdirAttachmentAlias) -> Result<(), WorkdirError> {
@@ -620,6 +692,10 @@ impl WorkdirSessionRouter {
.is_some_and(|current| Arc::ptr_eq(current, &state))
{
sessions.remove(alias);
self.coordinate_brokers
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(alias);
}
Ok(state.session.clone())
}
@@ -661,6 +737,10 @@ impl WorkdirSessionRouter {
.is_some_and(|current| Arc::ptr_eq(current, &state))
{
sessions.remove(&alias);
self.coordinate_brokers
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&alias);
}
}
Ok(())
@@ -684,6 +764,131 @@ mod tests {
))
}
#[tokio::test]
async fn cwd_views_share_root_delegation_leases_and_attachment_fences() {
use crate::{
WorkdirPath as P, WorkdirToolBrokerRouter, WorkdirToolScope,
WorkdirToolScopePermission as Permission, WorkdirToolScopeRule,
};
let root = TempDir::new().unwrap();
for directory in ["work", "other", "other/inside"] {
std::fs::create_dir_all(root.path().join(directory)).unwrap();
std::fs::write(root.path().join(directory).join("marker"), directory).unwrap();
}
let router = Arc::new(WorkdirSessionRouter::new());
let alias = WorkdirAttachmentAlias::new("source").unwrap();
let parent = router
.attach_at_cwd(alias.clone(), session("wd", &root), P::new("work").unwrap())
.unwrap();
assert_eq!(parent.environment().cwd, P::new("work").unwrap());
let request = |cwd: &str| WorkdirToolScope {
rules: vec![WorkdirToolScopeRule {
target: P::new(cwd).unwrap(),
permission: Permission::Write,
recursive: true,
symlink_policy: Default::default(),
}],
cwd: P::new(cwd).unwrap(),
command: false,
};
let brokers = WorkdirToolBrokerRouter::new(router.clone());
// Declarations stay root-relative, not work/other.
let child = brokers
.scope(vec![(alias.clone(), request("other"))])
.await
.unwrap();
let grandchild = child
.broker_router()
.scope(vec![(alias.clone(), request("other/inside"))])
.await
.unwrap();
for (session, cwd) in [
(parent.clone(), "work"),
(
child.session_router().resolve(None).unwrap().session,
"other",
),
(
grandchild.session_router().resolve(None).unwrap().session,
"other/inside",
),
] {
assert_eq!(session.environment().cwd, P::new(cwd).unwrap());
let result = session
.read(ReadRequest {
path: P::new("marker").unwrap(),
offset: 0,
limit: 10,
max_bytes: 100,
})
.await
.unwrap();
assert_eq!(result.bytes, cwd.as_bytes());
assert_eq!(result.path, P::new("marker").unwrap());
assert_eq!(
session
.stat(StatRequest {
path: P::new("marker").unwrap()
})
.await
.unwrap()
.path,
P::new("marker").unwrap()
);
}
// The model's tool view and delegation view share write-lease exclusion.
let absolute = P::new_scoped(root.path().join("other/marker").to_str().unwrap()).unwrap();
let denied = parent
.write(WriteRequest {
path: absolute,
content: b"forbidden".to_vec(),
expected_hash: None,
})
.await
.unwrap_err();
assert_eq!(
denied.denial_reason(),
Some(crate::WorkdirDenialReason::ChildWriteLeaseConflict)
);
let old_child = child.session_router().resolve(None).unwrap().session;
router.begin_detach(&alias).await.unwrap();
assert!(
old_child
.read(ReadRequest {
path: P::new("marker").unwrap(),
offset: 0,
limit: 10,
max_bytes: 100
})
.await
.is_err()
);
router.cancel_detach(&alias);
grandchild.close().await.unwrap();
child.close().await.unwrap();
router.detach(&alias).await.unwrap();
router
.attach_at_cwd(
alias,
session("replacement", &root),
P::new("other").unwrap(),
)
.unwrap();
assert_eq!(
router.resolve(None).unwrap().session.environment().cwd,
P::new("other").unwrap()
);
assert!(
old_child
.stat(StatRequest {
path: P::new("marker").unwrap()
})
.await
.is_err()
);
router.close_all().await.unwrap();
}
#[tokio::test]
async fn checkout_routes_use_active_operation_detach_fence_and_old_handles_stay_closed() {
let root = TempDir::new().unwrap();
+85 -8
View File
@@ -108,17 +108,34 @@ impl WorkdirToolBroker {
Self::with_capabilities(source, capabilities)
}
/// Set the tool coordinate base without changing provider authority.
/// The input is relative to the source session's coordinates; delegation
/// declarations retain their existing coordinate contract.
pub fn at_cwd(source: WorkdirSessionHandle, cwd: FsPath) -> Result<Self, WorkdirError> {
let cwd = FsPath::new(cwd.as_str())?;
let capabilities = source.capabilities();
Ok(Self::with_capabilities_and_cwd(source, capabilities, cwd))
}
/// Own a session behind an additional Backend-authored capability ceiling.
/// The ceiling can only attenuate provider capabilities; it can never add one.
pub fn with_capabilities(
source: WorkdirSessionHandle,
capabilities: WorkdirSessionCapabilities,
) -> Self {
Self::with_capabilities_and_cwd(source, capabilities, FsPath::root())
}
fn with_capabilities_and_cwd(
source: WorkdirSessionHandle,
capabilities: WorkdirSessionCapabilities,
cwd: FsPath,
) -> Self {
let capabilities = source.capabilities().intersection(capabilities);
let (command_events, _) = broadcast::channel(64);
let authority = Arc::new(ScopedWorkdirSession {
source,
cwd: FsPath::new("").expect("empty Workdir path is valid"),
cwd,
scope: None,
capabilities,
validity: SessionValidity::root(),
@@ -416,6 +433,9 @@ impl WorkdirToolBrokerRouter {
.sessions
.resolve(Some(alias.as_str()))
.map_err(|error| WorkdirError::InvalidArgument(error.to_string()))?;
if let Some(broker) = self.sessions.coordinate_broker(alias, selected.generation) {
return Ok(broker);
}
let mut brokers = self.brokers.lock().map_err(|_| {
WorkdirError::Unavailable("Workdir tool broker router is poisoned".to_string())
})?;
@@ -684,7 +704,7 @@ impl ScopedWorkdirSession {
}
fn resolve_path(&self, path: &FsPath) -> Result<FsPath, WorkdirError> {
if self.cwd.as_str().is_empty() {
if self.cwd.as_str().is_empty() || Path::new(path.as_str()).is_absolute() {
return Ok(path.clone());
}
let joined = Path::new(self.cwd.as_str()).join(path.as_str());
@@ -1085,6 +1105,43 @@ impl WorkdirSession for ScopedWorkdirSession {
self.capabilities
}
fn environment(&self) -> crate::WorkdirSessionEnvironment {
let mut environment = self.source.environment();
let source_cwd = environment.cwd.clone();
environment.cwd = FsPath::new(
Path::new(source_cwd.as_str())
.join(self.cwd.as_str())
.to_str()
.expect("validated UTF-8 paths"),
)
.expect("validated session cwd");
if let Some(scope) = &self.scope {
if let Some(inherited) = environment.scope.take() {
environment
.restrictions
.push(format!("Provider allow ceiling: {inherited:?}"));
}
environment.scope = Some(
scope
.iter()
.cloned()
.map(|mut rule| {
rule.target = FsPath::new(
Path::new(source_cwd.as_str())
.join(rule.target.as_str())
.to_str()
.expect("validated UTF-8 paths"),
)
.expect("validated scope target");
rule
})
.collect(),
);
}
environment.capabilities = self.capabilities();
environment
}
async fn authorize_scope_path(
&self,
mut request: WorkdirScopeAuthorizationRequest,
@@ -1273,8 +1330,11 @@ impl WorkdirSession for ScopedWorkdirSession {
.resolve_operation_path(&request.path, WorkdirToolScopePermission::Read)
.await?;
self.ensure_read(&path, WorkdirSessionCapability::Read)?;
let caller_path = request.path.clone();
request.path = path;
self.source.stat(request).await
let mut result = self.source.stat(request).await?;
result.path = caller_path;
Ok(result)
}
async fn read(&self, mut request: ReadRequest) -> Result<ReadResult, WorkdirError> {
@@ -1282,8 +1342,11 @@ impl WorkdirSession for ScopedWorkdirSession {
.resolve_operation_path(&request.path, WorkdirToolScopePermission::Read)
.await?;
self.ensure_read(&path, WorkdirSessionCapability::Read)?;
let caller_path = request.path.clone();
request.path = path;
self.source.read(request).await
let mut result = self.source.read(request).await?;
result.path = caller_path;
Ok(result)
}
async fn read_bytes(
@@ -1422,9 +1485,10 @@ impl WorkdirSession for ScopedWorkdirSession {
)));
}
let tool_call_id = request.tool_call_id.clone();
if self.scope.is_some() {
request.cwd = self.resolve_path(&request.cwd)?;
}
request.cwd = self
.resolve_operation_path(&request.cwd, WorkdirToolScopePermission::Read)
.await?;
self.ensure_path(&request.cwd, WorkdirToolScopePermission::Read)?;
if let Some(tool_call_id) = &tool_call_id {
self.starting_tool_calls
.lock()
@@ -1601,7 +1665,20 @@ impl WorkdirSession for ReadOnlyWorkdirSession {
}
fn capabilities(&self) -> WorkdirSessionCapabilities {
WorkdirSessionCapabilities::READ_ONLY
self.inner
.capabilities()
.intersection(WorkdirSessionCapabilities::READ_ONLY)
}
fn environment(&self) -> crate::WorkdirSessionEnvironment {
let mut environment = self.inner.environment();
environment.capabilities = self.capabilities();
if let Some(scope) = &mut environment.scope {
for rule in scope {
rule.permission = WorkdirToolScopePermission::Read;
}
}
environment
}
async fn authorize_scope_path(
@@ -83,3 +83,34 @@ fn provider_cannot_inject_free_text_failure_reason() {
.is_err()
);
}
#[test]
fn provider_authored_operational_messages_never_enter_error_display_or_debug() {
for code in [
Code::Conflict,
Code::OutcomeUnknown,
Code::Unsupported,
Code::InvalidRequest,
Code::Unavailable,
Code::Io,
Code::Transport,
Code::Internal,
] {
let wire = serde_json::json!({
"code": code, "message":"/host/private command-secret body-secret proof-secret key-secret Authorization: Bearer bearer-secret",
"failure_reason": "session_open_task_failed"
});
let decoded: WorkdirTransportError = serde_json::from_value(wire).unwrap();
let error = decoded.into_workdir_error();
for text in [
format!("{error}"),
format!("{error:?}"),
serde_json::to_string(&WorkdirTransportError::from_workdir_error(&error)).unwrap(),
] {
assert!(!text.contains("secret"), "{text}");
assert!(!text.contains("/host/private"), "{text}");
}
assert_eq!(error.failure_reason(), Some(Reason::SessionOpenTaskFailed));
assert_eq!(WorkdirTransportError::from_workdir_error(&error).code, code);
}
}
+117 -10
View File
@@ -14,6 +14,18 @@ use workdir::{
},
};
fn opened(workdir_id: &str) -> String {
serde_json::to_string(&workdir::http::OpenWorkdirSessionResponse {
session_id: workdir::http::WorkdirSessionId::new("s").unwrap(),
workdir_id: Workdir::new(workdir_id).id().clone(),
capabilities: workdir::WorkdirSessionCapabilities::ALL,
environment: workdir::WorkdirSessionEnvironment::root(
workdir::WorkdirSessionCapabilities::ALL,
),
})
.unwrap()
}
fn reply(listener: &TcpListener, status: &str, body: &str) {
listener.set_nonblocking(true).unwrap();
let deadline = Instant::now() + Duration::from_secs(5);
@@ -73,11 +85,7 @@ async fn remote_http_denials_forward_only_typed_reasons_and_old_missing_reason()
error["denial_reason"] = serde_json::to_value(reason).unwrap();
}
let server = thread::spawn(move || {
reply(
&listener,
"200 OK",
r#"{"session_id":"s","workdir_id":"w","capabilities":{"bits":63}}"#,
);
reply(&listener, "200 OK", &opened("w"));
reply(&listener, "403 Forbidden", &error.to_string());
});
let remote = RemoteWorkdirSession::open(
@@ -163,11 +171,7 @@ async fn remote_checkout_diagnostics_preserve_mutation_outcome_and_read_failures
let address = listener.local_addr().unwrap();
let error = serde_json::json!({"code":code, "message":"/secret/host provider-secret", "denial_reason":reason});
let server = thread::spawn(move || {
reply(
&listener,
"200 OK",
r#"{"session_id":"s","workdir_id":"w","capabilities":{"bits":63}}"#,
);
reply(&listener, "200 OK", &opened("w"));
reply(
&listener,
&format!("{} Fixture failure", code.http_status()),
@@ -216,3 +220,106 @@ async fn remote_checkout_diagnostics_preserve_mutation_outcome_and_read_failures
}
}
}
#[tokio::test]
async fn mismatched_http_results_keep_cause_and_mutation_uncertainty() {
use workdir::{
CheckoutOperation, CheckoutRequest, CommandHandle, CommandOutputRequest,
WorkdirFailureReason as Failure,
};
for case in ["variant", "bytes", "mutation", "identity"] {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = thread::spawn(move || {
reply(
&listener,
"200 OK",
&opened(if case == "identity" {
"wrong-secret"
} else {
"w"
}),
);
if case != "identity" {
let result = if case == "bytes" {
serde_json::json!({"operation":"command_output","result":{
"chunks": [], "next_cursor": 123, "terminal": null
}})
} else {
serde_json::to_value(
workdir::http::WorkdirSessionOperationResult::CommandCancel,
)
.unwrap()
};
reply(&listener, "200 OK", &result.to_string());
}
});
let opened = RemoteWorkdirSession::open(
reqwest::Client::builder()
.no_proxy()
.timeout(Duration::from_secs(5))
.build()
.unwrap(),
format!("http://{address}").parse().unwrap(),
"request-secret",
Workdir::new("w").id().clone(),
OpenWorkdirSessionRequest::default(),
)
.await;
let error = if case == "identity" {
opened.unwrap_err()
} else {
let remote = opened.unwrap();
match case {
"variant" => remote
.stat(StatRequest {
path: WorkdirPath::new("file-secret").unwrap(),
})
.await
.unwrap_err(),
"bytes" => remote
.command_output(CommandOutputRequest {
handle: CommandHandle("command-secret".into()),
cursor: 0,
limit: 16,
wait: false,
})
.await
.unwrap_err(),
"mutation" => remote
.checkout_execute(CheckoutRequest {
target: WorkdirPath::new("file-secret").unwrap(),
validator: vec![1],
operation: CheckoutOperation::Write {
content: b"body-secret".to_vec(),
expected_hash: [0; 32],
},
})
.await
.unwrap_err(),
_ => unreachable!(),
}
};
server.join().unwrap();
let expected = if case == "identity" {
Failure::SessionIdentityMismatch
} else {
Failure::OperationResultMismatch
};
assert_eq!(error.failure_reason(), Some(expected), "{case}: {error:?}");
let transport = WorkdirTransportError::from_workdir_error(&error);
assert_eq!(
transport.code,
if case == "mutation" {
Code::OutcomeUnknown
} else {
Code::Unavailable
}
);
let wire = serde_json::to_string(&transport).unwrap();
assert!(!wire.contains("secret"), "{wire}");
assert!(!format!("{error:?}").contains("secret"));
let forwarded = transport.into_workdir_error();
assert_eq!(forwarded.failure_reason(), Some(expected));
}
}
+5 -2
View File
@@ -21,15 +21,17 @@ default = ["ws-server", "fs-store"]
fs-store = ["dep:fs4"]
# Offline fixture tooling only; never enabled by the product defaults.
test-support = ["fs-store", "ws-server"]
# Explicit process-test synchronization; independent of offline fixture tooling.
restore-checkpoints = ["fs-store"]
http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"]
ws-server = ["http-server", "axum/ws", "tokio/sync"]
[dependencies]
agen.workspace = true
async-trait.workspace = true
base64.workspace = true
axum = { workspace = true, optional = true }
futures = { workspace = true, optional = true }
futures.workspace = true
decodal.workspace = true
flow = { path = "../flow" }
fs4 = { workspace = true, optional = true }
@@ -58,6 +60,7 @@ server-api = { path = "../server-api" }
workdir.workspace = true
[dev-dependencies]
tools.workspace = true
workdir = { workspace = true, features = ["http-client"] }
futures.workspace = true
agen.workspace = true
+27
View File
@@ -39,3 +39,30 @@ not stripped or rewritten. Portability/secret-marker checks do not replace
review of the supplied artificial conversation or its independently verified hashes.
The installer refuses any destination content other than initialized `auth.toml`;
interrupted publication is fail-closed, not an overwrite/recovery facility.
## Owned-process Restore checkpoints
The independent, nondefault `restore-checkpoints` feature exists only for explicit
process tests in the shared `tests/e2e` harness. Offline `test-support` does not enable
it. Normal binaries reject `--test-restore-checkpoint`; no environment variable or
HTTP endpoint activates this facility. Do not deploy a checkpoint-enabled binary
as a replacement for the default product.
A test-enabled Runtime accepts `--test-restore-checkpoint <config.json>` with a
bounded regular JSON file selecting a loopback socket, Worker ID, one stage and a
diagnostic nonce. The nonce is only a handshake label, never execution identity,
request authority or routing state. The one-shot stages are real prepared candidate
with `NotCommitted`, and durable `Committed` before activation/terminal receipt.
Before acknowledging, the existing bounded canonical reader validates the exact
Worker/operation/decision, syncs its file/directories and requires no saved receipt.
No pending JSON is fabricated and no lifecycle work is skipped.
The Runtime sends bounded newline-delimited JSON evidence, then waits at most eight
seconds for byte `R`. EOF, invalid reply, unavailable peer or deadline releases the
wait into the original operation. The Runtime-state mutex is not held across that
wait; existing per-Worker ownership still covers the actual candidate and cleanup.
The harness verifies the ack, kills/reaps only its owned child and inspects the
canonical aggregate offline before restarting the same roots/identity without the
checkpoint flag or another fixture install. These tests are a separate cfg/binary
lane from featureless product regression; they do not certify arbitrary OS failure,
external power loss or a real model call.
+253 -4
View File
@@ -215,6 +215,38 @@ impl WorkerExecutionResult {
}
}
// Private one-shot synchronization at real publication boundaries. Disconnect
// releases the waiter during unwinding; timeout bounds a forgotten test owner.
#[cfg(test)]
#[derive(Debug)]
pub(crate) struct RestoreTestGate<T> {
pub(crate) entered: std::sync::mpsc::Sender<T>,
pub(crate) release: std::sync::mpsc::Receiver<()>,
}
#[cfg(test)]
impl<T> RestoreTestGate<T> {
pub(crate) fn wait(self, evidence: T) {
let _ = self.entered.send(evidence);
assert!(
!matches!(
self.release
.recv_timeout(std::time::Duration::from_secs(30)),
Err(std::sync::mpsc::RecvTimeoutError::Timeout)
),
"publication test gate timed out"
);
}
}
#[cfg(all(test, feature = "ws-server"))]
#[derive(Debug)]
pub(crate) struct RestoreFlushFault {
pub(crate) remaining_successes: usize,
pub(crate) gate: RestoreTestGate<protocol::Event>,
pub(crate) restored: std::sync::mpsc::Sender<Vec<protocol::Event>>,
}
/// Runtime hooks available to an execution backend for one Worker.
#[cfg(feature = "ws-server")]
type WorkerObservationPublisher = Arc<
@@ -242,10 +274,15 @@ enum WorkerExecutionPublicationState {
#[derive(Clone)]
pub struct WorkerExecutionContext {
worker_ref: WorkerRef,
restore_progress: Option<Arc<dyn Fn(WorkerLifecycleOperationId) + Send + Sync>>,
restore_repair_authorizer:
Option<Arc<dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync>>,
#[cfg(feature = "ws-server")]
observation_publisher: WorkerObservationPublisher,
#[cfg(feature = "ws-server")]
publication_state: Arc<Mutex<WorkerExecutionPublicationState>>,
#[cfg(all(test, feature = "ws-server"))]
flush_fault: Arc<Mutex<Option<RestoreFlushFault>>>,
}
impl WorkerExecutionContext {
@@ -256,7 +293,11 @@ impl WorkerExecutionContext {
) -> Self {
Self {
worker_ref,
restore_progress: None,
restore_repair_authorizer: None,
observation_publisher,
#[cfg(test)]
flush_fault: Arc::default(),
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Active)),
}
}
@@ -268,7 +309,11 @@ impl WorkerExecutionContext {
) -> Self {
Self {
worker_ref,
restore_progress: None,
restore_repair_authorizer: None,
observation_publisher,
#[cfg(test)]
flush_fault: Arc::default(),
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Pending(
VecDeque::new(),
))),
@@ -277,12 +322,56 @@ impl WorkerExecutionContext {
#[cfg(not(feature = "ws-server"))]
pub(crate) fn new(worker_ref: WorkerRef) -> Self {
Self { worker_ref }
Self {
worker_ref,
restore_progress: None,
restore_repair_authorizer: None,
}
}
#[cfg(not(feature = "ws-server"))]
pub(crate) fn candidate(worker_ref: WorkerRef) -> Self {
Self { worker_ref }
Self {
worker_ref,
restore_progress: None,
restore_repair_authorizer: None,
}
}
pub(crate) fn with_restore_repair_authorizer(
mut self,
authorizer: Arc<
dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync,
>,
) -> Self {
self.restore_repair_authorizer = Some(authorizer);
self
}
pub(crate) fn authorize_restore_source_repair(
&self,
operation: WorkerLifecycleOperationId,
) -> Result<(), RuntimeError> {
match &self.restore_repair_authorizer {
Some(authorizer) => authorizer(operation),
// Direct factory/component users own their source repair separately;
// only Runtime contexts carry a Runtime aggregate authority callback.
None => Ok(()),
}
}
pub(crate) fn with_restore_progress(
mut self,
progress: Arc<dyn Fn(WorkerLifecycleOperationId) + Send + Sync>,
) -> Self {
self.restore_progress = Some(progress);
self
}
pub(crate) fn restore_progressed(&self, operation_id: WorkerLifecycleOperationId) {
if let Some(progress) = &self.restore_progress {
progress(operation_id);
}
}
pub fn worker_ref(&self) -> &WorkerRef {
@@ -335,20 +424,65 @@ impl WorkerExecutionContext {
let Some(event) = event else {
return Ok(());
};
if let Err(error) = (self.observation_publisher)(self.worker_ref.clone(), event.clone())
{
#[cfg(test)]
let fault = {
let mut fault = self.flush_fault.lock().unwrap();
if fault.as_ref().is_some_and(|f| f.remaining_successes == 0) {
fault.take()
} else {
if let Some(fault) = fault.as_mut() {
fault.remaining_successes -= 1;
}
None
}
};
#[cfg(test)]
let mut restored = None;
#[cfg(test)]
let publication = if let Some(fault) = fault {
restored = Some(fault.restored);
fault.gate.wait(event.clone());
Err(RuntimeError::InvalidRequest(
"injected buffered publication failure".into(),
))
} else {
(self.observation_publisher)(self.worker_ref.clone(), event.clone())
};
#[cfg(not(test))]
let publication = (self.observation_publisher)(self.worker_ref.clone(), event.clone());
if let Err(error) = publication {
let mut state = self
.publication_state
.lock()
.map_err(|_| RuntimeError::StatePoisoned)?;
if let WorkerExecutionPublicationState::Activating(events) = &mut *state {
events.push_front(event);
#[cfg(test)]
if let Some(restored) = restored {
let _ = restored.send(events.iter().cloned().collect());
}
}
return Err(error);
}
}
}
#[cfg(all(test, feature = "ws-server"))]
pub(crate) fn test_fail_flush(&self, fault: RestoreFlushFault) {
assert!(self.flush_fault.lock().unwrap().replace(fault).is_none());
}
#[cfg(all(test, feature = "ws-server"))]
pub(crate) fn test_buffered_events(&self) -> Vec<protocol::Event> {
match &*self.publication_state.lock().unwrap() {
WorkerExecutionPublicationState::Pending(events)
| WorkerExecutionPublicationState::Activating(events) => {
events.iter().cloned().collect()
}
_ => Vec::new(),
}
}
#[cfg(not(feature = "ws-server"))]
pub(crate) fn activate_candidate(&self) -> Result<(), RuntimeError> {
Ok(())
@@ -424,6 +558,7 @@ pub struct WorkerExecutionSpawnRequest {
/// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker.
#[derive(Clone, Debug)]
pub struct WorkerExecutionRestoreRequest {
pub source_repair_possible: bool,
pub operation_id: WorkerLifecycleOperationId,
pub worker_ref: WorkerRef,
pub request: crate::catalog::CreateWorkerRequest,
@@ -565,6 +700,19 @@ impl fmt::Debug for WorkerProtocolTransport {
}
}
/// Derived resource disposition for an existing restore owner. Not durable
/// authority: Runtime must reread its decision before choosing any action.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RestoreRecoveryDisposition {
/// No live or pending owner for this exact operation. Not cleanup evidence.
Absent,
InFlight,
Continue,
Discard,
/// A retained cleanup-only owner; never reconstruct or activate it.
Reclaim,
}
/// The connection selects this backend; WorkerRef identifies an execution,
/// never a capability or authorization token.
pub trait WorkerExecutionBackend: Send + Sync + 'static {
@@ -633,6 +781,33 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
Ok(())
}
/// Completed resource-owner evidence retained while Runtime settlement may
/// still need retry. This is not a new request or execution authorization.
fn completed_restore_failure(
&self,
_operation_id: WorkerLifecycleOperationId,
_worker_ref: &WorkerRef,
) -> Option<WorkerExecutionSpawnResult> {
None
}
/// Opt in to Runtime-owned autonomous restore continuation. Pending tasks
/// signal completion through the original execution context; callbacks must
/// not wait for Runtime locks. Independent of resource disposition support.
fn supports_restore_continuation(&self) -> bool {
false
}
/// Describe exact operation resource ownership for warm/cold recovery
/// dispatch. This does not opt in to callbacks or autonomous continuation.
fn restore_recovery_disposition(
&self,
_operation_id: WorkerLifecycleOperationId,
_worker_ref: &WorkerRef,
) -> Option<RestoreRecoveryDisposition> {
None
}
fn reconcile_restore(
&self,
request: WorkerExecutionRestoreRequest,
@@ -640,6 +815,42 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
self.restore_worker(request)
}
/// Reclaim an uncommitted operation after restart without preparing,
/// repairing, or activating a replacement candidate. Runtime must establish
/// the durable uncommitted decision before calling this seam. Unsupported
/// cleanup is uncertainty, never a successful rollback.
fn reclaim_uncommitted_restore(
&self,
_request: WorkerExecutionRestoreRequest,
) -> WorkerExecutionSpawnResult {
WorkerExecutionSpawnResult::ReconciliationRequired {
result: WorkerExecutionResult::unsupported(
WorkerExecutionOperation::Restore,
"backend cannot reclaim an uncommitted restore after restart",
),
worker_state: None,
workdir_attachments: Vec::new(),
}
}
/// Discard only the exact unpublished operation after Runtime has proved
/// its durable decision uncommitted. This is not Stop and must not affect a
/// connected execution. RolledBack requires confirmed cleanup and no repair.
fn discard_uncommitted_restore(
&self,
_operation_id: WorkerLifecycleOperationId,
_worker_ref: &WorkerRef,
) -> WorkerExecutionSpawnResult {
WorkerExecutionSpawnResult::ReconciliationRequired {
result: WorkerExecutionResult::unsupported(
WorkerExecutionOperation::Restore,
"backend cannot confirm unpublished restore cleanup",
),
worker_state: None,
workdir_attachments: Vec::new(),
}
}
fn activate_restored_worker(
&self,
_operation_id: WorkerLifecycleOperationId,
@@ -901,6 +1112,28 @@ impl WorkerExecutionBackendRef {
self.backend.preflight_restore(request)
}
pub(crate) fn completed_restore_failure(
&self,
operation_id: WorkerLifecycleOperationId,
worker_ref: &WorkerRef,
) -> Option<WorkerExecutionSpawnResult> {
self.backend
.completed_restore_failure(operation_id, worker_ref)
}
pub(crate) fn supports_restore_continuation(&self) -> bool {
self.backend.supports_restore_continuation()
}
pub(crate) fn restore_recovery_disposition(
&self,
operation_id: WorkerLifecycleOperationId,
worker_ref: &WorkerRef,
) -> Option<RestoreRecoveryDisposition> {
self.backend
.restore_recovery_disposition(operation_id, worker_ref)
}
pub(crate) fn reconcile_restore(
&self,
request: WorkerExecutionRestoreRequest,
@@ -908,6 +1141,22 @@ impl WorkerExecutionBackendRef {
self.backend.reconcile_restore(request)
}
pub(crate) fn reclaim_uncommitted_restore(
&self,
request: WorkerExecutionRestoreRequest,
) -> WorkerExecutionSpawnResult {
self.backend.reclaim_uncommitted_restore(request)
}
pub(crate) fn discard_uncommitted_restore(
&self,
operation_id: WorkerLifecycleOperationId,
worker_ref: &WorkerRef,
) -> WorkerExecutionSpawnResult {
self.backend
.discard_uncommitted_restore(operation_id, worker_ref)
}
pub(crate) fn activate_restored_worker(
&self,
operation_id: WorkerLifecycleOperationId,
File diff suppressed because it is too large Load Diff
+109 -2
View File
@@ -25,6 +25,7 @@ use crate::interaction::{WorkerInput, WorkerInteractionAck};
use crate::management::{RuntimeSummary, WorkerDeleteResult};
#[cfg(feature = "ws-server")]
use crate::observation::WorkerObservationCursor;
#[cfg(feature = "fs-store")]
use crate::retention::{
WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory,
};
@@ -869,6 +870,7 @@ async fn open_workdir_session(
session_id: session_id.clone(),
workdir_id: session.workdir().id().clone(),
capabilities: session.capabilities(),
environment: session.environment(),
};
state
.workdir_sessions
@@ -1567,6 +1569,7 @@ fn protocol_error_event(message: impl Into<String>) -> protocol::Event {
}
}
#[cfg(feature = "fs-store")]
#[allow(dead_code)]
async fn worker_retention_inventory(
State(state): State<RuntimeHttpState>,
@@ -1588,6 +1591,7 @@ async fn worker_retention_inventory(
.map_err(RuntimeHttpRestError::runtime)
}
#[cfg(feature = "fs-store")]
#[allow(dead_code)]
async fn execute_worker_retention(
State(state): State<RuntimeHttpState>,
@@ -2187,7 +2191,25 @@ impl RuntimeHttpWorkdirError {
}
fn runtime(error: RuntimeError) -> Self {
match error {
use workdir::WorkdirFailureReason as Reason;
let reason = match &error {
RuntimeError::WorkingDirectory(diagnostic) => match diagnostic.code.as_str() {
"working_directory_not_found" => Reason::WorkingDirectoryNotFound,
"working_directory_not_active" => Reason::WorkingDirectoryNotActive,
"working_directory_record_invalid" => Reason::WorkingDirectoryRecordInvalid,
"working_directory_repository_access_unavailable" => {
Reason::WorkingDirectoryAccessUnavailable
}
_ => Reason::SessionOpenFailed,
},
RuntimeError::RuntimeStopped => Reason::RuntimeStopped,
RuntimeError::StatePoisoned => Reason::RuntimeStateUnavailable,
RuntimeError::ExecutionBackendUnavailable { .. } => Reason::RuntimeBackendUnavailable,
RuntimeError::WorkspaceOwnerMismatch { .. } => Reason::RuntimeOwnerMismatch,
RuntimeError::InvalidRequest(_) => Reason::RuntimeInvalidRequest,
_ => Reason::SessionOpenFailed,
};
let mut response = match error {
RuntimeError::WorkingDirectory(diagnostic)
if diagnostic.code == "working_directory_not_found" =>
{
@@ -2200,7 +2222,9 @@ impl RuntimeHttpWorkdirError {
WorkdirTransportErrorCode::Unavailable,
"Workdir session is unavailable",
),
}
};
response.payload.failure_reason = Some(reason);
response
}
}
@@ -2222,6 +2246,7 @@ impl IntoResponse for RuntimeHttpWorkdirError {
status = self.status.as_u16(),
code = self.payload.code.as_str(),
failure_reason = reason.as_str(),
failure_stage = reason.stage().as_str(),
"Workdir provider operation failed"
);
}
@@ -2434,6 +2459,7 @@ pub enum RuntimeHttpServerError {
#[cfg(test)]
mod tests {
mod command_transport_tests;
mod runtime_workdir_failure_tests;
#[cfg(feature = "fs-store")]
mod session_material_tests;
@@ -2631,6 +2657,87 @@ mod tests {
.unwrap()
}
// Generated management routes and real Workspace capability middleware stay
// connected even when this process has no filesystem retention provider.
#[cfg(not(feature = "fs-store"))]
#[tokio::test]
async fn memory_retention_routes_preserve_auth_and_return_typed_unavailable() {
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
let trust = workspace_trust(&identity);
let runtime = Runtime::new_memory();
let app = runtime_http_router_with_workspace_auth(
runtime,
None,
workspace_test_auth(vec![trust.clone()]),
);
let worker_id = WorkerId::now_v7();
let body = runtime_api::WorkerRetentionExecutionRequest {
operation_id: "memory-retention".into(),
input_fingerprint: "input".into(),
archive_id: None,
workspace_id: "workspace-a".into(),
source_runtime_id: "runtime-test".into(),
worker_id,
source_created_at: "2026-01-01T00:00:00Z".into(),
removed_at: "2026-01-02T00:00:00Z".into(),
effective_profile: None,
retention_class: None,
policy_id: "policy".into(),
session_disposition: runtime_api::SessionDisposition::Purge,
diagnostics_disposition: runtime_api::DiagnosticsDisposition::Purge,
};
for (method, path, body) in [
(
"GET",
format!("/v1/workers/{worker_id}/retention/inventory"),
vec![],
),
(
"POST",
format!("/v1/workers/{worker_id}/retention/execute"),
serde_json::to_vec(&body).unwrap(),
),
] {
let unsigned = app
.clone()
.oneshot(
Request::builder()
.method(method)
.uri(&path)
.header(header::CONTENT_TYPE, "application/json")
.body(Body::from(body.clone()))
.unwrap(),
)
.await
.unwrap();
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
let claims = signed_claims(&trust, method, &path, &body);
let response =
signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
let error: RuntimeHttpErrorResponse = read_json(response).await;
assert_eq!(error.error.code, "worker_retention_unavailable");
}
for (mut value, expected_status, expected_code) in [
(body.clone(), StatusCode::NOT_FOUND, "worker_not_found"),
(body.clone(), StatusCode::BAD_REQUEST, "worker_id_mismatch"),
] {
if expected_status == StatusCode::NOT_FOUND {
value.workspace_id = "workspace-b".into();
} else {
value.worker_id = WorkerId::now_v7();
}
let path = format!("/v1/workers/{worker_id}/retention/execute");
let bytes = serde_json::to_vec(&value).unwrap();
let claims = signed_claims(&trust, "POST", &path, &bytes);
let response =
signed_request(app.clone(), &identity, &claims, bytes, "workspace-a").await;
assert_eq!(response.status(), expected_status);
let error: RuntimeHttpErrorResponse = read_json(response).await;
assert_eq!(error.error.code, expected_code);
}
}
#[tokio::test]
async fn ordinary_signed_ping_succeeds_without_handshake_and_rejects_replay() {
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
@@ -444,6 +444,31 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
material_pin(&self.state, request, true)
}
async fn session_recording_capture(
&self,
request: runtime_api::SessionRecordingCaptureRequest,
) -> Result<runtime_api::SessionRecordingCaptureAvailability, runtime_api::RuntimeApiError>
{
let scope = material_scope(&self.state, &request.workspace_id)?;
#[cfg(feature = "fs-store")]
{
Ok(self
.state
.runtime
.session_recording_capture_scoped(&scope, &request))
}
#[cfg(not(feature = "fs-store"))]
{
let _ = scope;
Ok(
runtime_api::SessionRecordingCaptureAvailability::Unavailable {
reason: runtime_api::SessionPublicUnavailableReason::StorageUnavailable,
message: "Committed Session storage unavailable".into(),
},
)
}
}
async fn session_material_catalog(
&self,
request: runtime_api::SessionMaterialCatalogRequest,
@@ -694,6 +719,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
server_api::WorkerRestoreState::RolledBack => {
runtime_api::WorkerRestoreState::RolledBack
}
server_api::WorkerRestoreState::Superseded => {
runtime_api::WorkerRestoreState::Superseded
}
server_api::WorkerRestoreState::ReconciliationRequired => {
runtime_api::WorkerRestoreState::ReconciliationRequired
}
@@ -744,6 +772,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
server_api::WorkerRestoreState::RolledBack => {
runtime_api::WorkerRestoreState::RolledBack
}
server_api::WorkerRestoreState::Superseded => {
runtime_api::WorkerRestoreState::Superseded
}
server_api::WorkerRestoreState::ReconciliationRequired => {
runtime_api::WorkerRestoreState::ReconciliationRequired
}
@@ -856,13 +887,25 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
)
})?;
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
let inventory = self
.state
.runtime
.worker_retention_inventory(&scope.workspace_id, &worker_ref)
.map_err(RuntimeHttpRestError::runtime)
.map_err(api_error)?;
response(inventory)
#[cfg(feature = "fs-store")]
{
let inventory = self
.state
.runtime
.worker_retention_inventory(&scope.workspace_id, &worker_ref)
.map_err(RuntimeHttpRestError::runtime)
.map_err(api_error)?;
response(inventory)
}
#[cfg(not(feature = "fs-store"))]
{
let _ = (scope, worker_ref);
Err(runtime_api::RuntimeApiError::new(
503,
"worker_retention_unavailable",
"Worker retention requires filesystem storage",
))
}
}
async fn execute_retention(
@@ -880,27 +923,36 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
)
})?;
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
let request: WorkerRetentionExecutionRequest = request(value)?;
if request.workspace_id != scope.workspace_id {
if value.workspace_id != scope.workspace_id {
return Err(runtime_api::RuntimeApiError::new(
StatusCode::NOT_FOUND.as_u16(),
"worker_not_found",
"worker does not belong to the authenticated Workspace",
));
}
if request.worker_id != worker_ref.worker_id {
if value.worker_id != worker_ref.worker_id {
return Err(runtime_api::RuntimeApiError::new(
StatusCode::BAD_REQUEST.as_u16(),
"worker_id_mismatch",
"Worker retention request worker_id does not match route worker_id",
));
}
let result = self
.state
.runtime
.execute_worker_retention(&request)
.map_err(RuntimeHttpRestError::runtime)
.map_err(api_error)?;
response(result)
#[cfg(feature = "fs-store")]
{
let request: WorkerRetentionExecutionRequest = request(value)?;
let result = self
.state
.runtime
.execute_worker_retention(&request)
.map_err(RuntimeHttpRestError::runtime)
.map_err(api_error)?;
response(result)
}
#[cfg(not(feature = "fs-store"))]
Err(runtime_api::RuntimeApiError::new(
503,
"worker_retention_unavailable",
"Worker retention requires filesystem storage",
))
}
}
@@ -3,6 +3,7 @@
//! pre-seeded session is a fixture; no Runtime Worker/materializer is claimed here.
use super::*;
use std::collections::BTreeMap;
use std::time::Duration;
use workdir::http::{
RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture,
@@ -70,14 +71,23 @@ impl Harness {
}
async fn new() -> Self {
let root = tempfile::tempdir().unwrap();
let session: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::materialized_bound(
Workdir::new("wd-transport"),
root.path().to_path_buf(),
root.path().to_path_buf(),
SharedScope::new(Scope::writable(root.path()).unwrap()),
WorkdirSessionCapabilities::ALL,
));
Self::with_environment(tempfile::tempdir().unwrap(), BTreeMap::new()).await
}
async fn with_environment(
root: tempfile::TempDir,
command_environment: BTreeMap<String, String>,
) -> Self {
let session: WorkdirSessionHandle =
Arc::new(LocalWorkdirSession::materialized_bound_with_environment(
Workdir::new("wd-transport"),
root.path().to_path_buf(),
root.path().to_path_buf(),
SharedScope::new(Scope::writable(root.path()).unwrap()),
WorkdirSessionCapabilities::ALL,
command_environment,
Vec::new(),
));
let identity = RuntimeIdentityMaterial::generate("transport-key").unwrap();
let trust = workspace_trust(&identity);
let state = RuntimeHttpState {
@@ -101,6 +111,9 @@ impl Harness {
session_id: WorkdirSessionId::new("session-transport").unwrap(),
workdir_id: Workdir::new("wd-transport").id().clone(),
capabilities: WorkdirSessionCapabilities::ALL,
environment: workdir::WorkdirSessionEnvironment::root(
WorkdirSessionCapabilities::ALL,
),
})
}),
)
@@ -222,6 +235,234 @@ async fn real_http_waiting_command_survives_long_poll_expiry() {
.expect("bounded long-poll continuation over HTTP");
}
#[tokio::test]
async fn real_http_spawn_failure_preserves_safe_cause_counts_and_terminal_replay() {
tokio::time::timeout(Duration::from_secs(10), async {
let root = tempfile::tempdir().unwrap();
let missing_path = root.path().join("private-spawn-path-must-not-exist");
assert!(!missing_path.exists());
let private_path = missing_path.to_str().unwrap().to_owned();
let environment_secret = "spawn-environment-secret-must-not-leak";
let command_secret = "spawn-command-secret-must-not-leak";
let tool_call_secret = "spawn-tool-call-secret-must-not-leak";
// This is an OS spawn failure in the production local provider, not a
// fabricated terminal: PATH contains no bash. The test process environment
// is untouched, so parallel command tests retain their normal PATH.
let harness = Harness::with_environment(
root,
BTreeMap::from([
("PATH".into(), private_path.clone()),
("PRIVATE_AUTH_TOKEN".into(), environment_secret.into()),
]),
)
.await;
let command = format!("printf '{command_secret}'; printf ran > must-not-run");
let handle = harness
.remote
.start_command(CommandRequest {
transport_byte_budget: 1024,
command: command.clone(),
timeout_secs: 5,
cwd: WorkdirPath::root(),
tool_call_id: Some(tool_call_secret.into()),
})
.await
.unwrap();
let request = CommandOutputRequest {
handle,
cursor: 0,
limit: 16,
wait: true,
};
let page = harness.remote.command_output(request.clone()).await.unwrap();
assert!(page.chunks.is_empty());
assert_eq!(page.next_cursor, 0);
let terminal = page.terminal.as_ref().expect("spawn failure is terminal");
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
assert_eq!(terminal.exit_code, None);
assert_eq!(
terminal.process_failure,
Some(workdir::CommandFailure {
stage: workdir::CommandFailureStage::Spawn,
reason: workdir::CommandIoErrorKind::NotFound,
os_code: Some(libc::ENOENT),
})
);
assert_eq!(
terminal.output_collection,
workdir::CommandOutputCollection::NotStarted
);
assert_eq!(terminal.stdout_bytes, 0);
assert_eq!(terminal.stderr_bytes, 0);
assert_eq!(terminal.stdout_delivered_bytes, 0);
assert_eq!(terminal.stderr_delivered_bytes, 0);
assert_eq!(terminal.truncation_reason, None);
// A zero-byte terminal page remains replayable at its unchanged cursor;
// neither a retry nor a smaller nonwaiting read changes its accounting.
assert_eq!(
harness.remote.command_output(request.clone()).await.unwrap(),
page
);
let replay_request = CommandOutputRequest {
wait: false,
limit: 1,
..request
};
assert_eq!(
harness
.remote
.command_output(replay_request.clone())
.await
.unwrap(),
page
);
let response = harness
.raw_operation(
serde_json::to_value(WorkdirSessionOperation::CommandOutput(
replay_request.clone(),
))
.unwrap(),
)
.await;
assert_eq!(response.status(), reqwest::StatusCode::OK);
let body = response.text().await.unwrap();
for secret in [
private_path.as_str(),
harness._root.path().to_str().unwrap(),
"private-spawn-path-must-not-exist",
"PRIVATE_AUTH_TOKEN",
environment_secret,
command_secret,
command.as_str(),
tool_call_secret,
] {
assert!(
!body.contains(secret),
"private input leaked into HTTP terminal"
);
}
let wire: serde_json::Value = serde_json::from_str(&body).unwrap();
assert_eq!(
wire,
serde_json::json!({
"operation": "command_output",
"result": {
"chunks": [],
"next_cursor": 0,
"terminal": {
"status": "failed",
"exit_code": null,
"process_failure": {"stage": "spawn", "reason": "not_found", "os_code": libc::ENOENT},
"output_collection": {"state": "not_started"},
"stdout_bytes": 0,
"stderr_bytes": 0,
"stdout_delivered_bytes": 0,
"stderr_delivered_bytes": 0,
"truncation_reason": null
}
}
})
);
assert_eq!(
harness.remote.command_output(replay_request).await.unwrap(),
page
);
// Exercise the same real spawn failure through Bash, not a terminal
// fixture. Opening another remote client reuses the pre-seeded provider
// session and still passes through the signed HTTP middleware.
let bash_remote: WorkdirSessionHandle = Arc::new(
RemoteWorkdirSession::open_with_authorization(
harness.client.clone(),
harness.operation_url.join("/").unwrap(),
harness.authorization.clone(),
Workdir::new("wd-transport").id().clone(),
OpenWorkdirSessionRequest::default(),
)
.await
.unwrap(),
);
let view = tools::bash_output::BashOutputView::new();
let (_, bash) = tools::bash_tool(bash_remote, view.clone())();
let input = serde_json::json!({
"command": command,
"cwd": ".",
"timeout": 5
})
.to_string();
let output = bash
.execute(
&input,
agen::tool::ToolExecutionContext::new(tool_call_secret, "http-spawn-attempt", 0),
)
.await
.expect("Bash returns a structured failed-command result");
let content = output.content.as_deref().expect("Bash ToolOutput body");
let tool_body: serde_json::Value = serde_json::from_str(content).unwrap();
assert_eq!(tool_body["stdout_preview"], "");
assert_eq!(tool_body["stderr_preview"], "");
assert_eq!(tool_body["preview_omitted"], false);
assert_eq!(
tool_body["output"]["terminal"],
serde_json::json!({
"status": "failed",
"exit_code": null,
"process_failure": {"stage": "spawn", "reason": "not_found", "os_code": libc::ENOENT},
"output_collection": {"state": "not_started"},
"transport_complete": false
})
);
let output_id = tool_body["output"]["id"].as_str().unwrap();
let retained = view.metadata(output_id).expect("retained spawn failure");
assert_eq!(retained.terminal.status, Some(workdir::CommandStatus::Failed));
assert_eq!(retained.terminal.exit_code, None);
assert_eq!(retained.terminal.process_failure, terminal.process_failure);
assert_eq!(
retained.terminal.output_collection,
Some(workdir::CommandOutputCollection::NotStarted)
);
assert!(!retained.terminal.transport_complete);
assert_eq!(retained.stdout_saved, 0);
assert_eq!(retained.stderr_saved, 0);
assert_eq!(retained.stdout_seen, 0);
assert_eq!(retained.stderr_seen, 0);
assert_eq!(retained.stdout_delivered, 0);
assert_eq!(retained.stderr_delivered, 0);
assert!(!retained.storage_truncated);
assert!(retained.retention_limit.is_none());
assert_eq!(retained.transport_truncation, None);
let retained_wire = serde_json::to_value(&retained).unwrap();
assert_eq!(retained_wire, tool_body["output"]);
for stream in [CommandStream::Stdout, CommandStream::Stderr] {
assert_eq!(view.read(output_id, stream, 0, 1), Some(Vec::new()));
}
for secret in [
private_path.as_str(),
harness._root.path().to_str().unwrap(),
"private-spawn-path-must-not-exist",
"PRIVATE_AUTH_TOKEN",
environment_secret,
command_secret,
command.as_str(),
tool_call_secret,
] {
assert!(
!content.contains(secret),
"private input leaked into Bash ToolOutput body"
);
}
assert!(!harness._root.path().join("must-not-run").exists());
harness.remote.close().await.unwrap();
assert!(harness.session.command_snapshot().is_empty());
assert_eq!(
serde_json::to_value(view.metadata(output_id).unwrap()).unwrap(),
retained_wire
);
})
.await
.expect("bounded real spawn failure over signed HTTP");
}
#[tokio::test]
async fn real_http_closed_session_retains_failure_reason() {
let harness = Harness::new().await;
@@ -291,6 +532,11 @@ async fn real_http_command_pages_preserve_binary_streams_replay_and_nonzero_exit
assert_eq!(stderr, b"stderr\xe6\x97\xa5");
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
assert_eq!(terminal.exit_code, Some(9));
assert_eq!(terminal.process_failure, None);
assert_eq!(
terminal.output_collection,
workdir::CommandOutputCollection::Complete
);
assert!(harness.remote.command_output(CommandOutputRequest { handle, cursor: 0, limit: 1, wait: false }).await.is_err());
harness.remote.close().await.unwrap();
assert!(harness.session.command_snapshot().is_empty());
@@ -435,6 +681,11 @@ async fn real_http_command_budget_bounds_unique_payload_and_reports_larger_seen_
);
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
assert_eq!(terminal.exit_code, Some(9));
assert_eq!(terminal.process_failure, None);
assert_eq!(
terminal.output_collection,
workdir::CommandOutputCollection::Complete
);
assert!(
harness
.remote
@@ -0,0 +1,528 @@
//! Session-open integration through the production signed router, Runtime owner/binding
//! checks, HTTP DTO and tracing boundary. Only the execution backend is replaced
//! for deterministic lookup/open faults and inert owner-binding Worker fixtures;
//! no handler, middleware or grant is replaced. Runs in the normal http-server lib
//! suite with isolated loopback sockets.
use super::*;
use crate::working_directory::WorkingDirectoryDiagnostic;
use std::io::{self, Write};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
const OPEN_PATH: &str = "/v1/working-directories/private-workdir-marker/sessions";
const PRIVATE_DIAGNOSTIC: &str = concat!(
"raw-runtime-error-marker /private/host-path-marker ",
"command=private-command-marker contents=private-file-marker ",
"proof=private-proof-marker Authorization: Bearer private-header-marker ",
"working_directory_not_found authorization_stale runtime_stopped",
);
const PRIVATE_MARKERS: &[&str] = &[
"raw-runtime-error-marker",
"/private/host-path-marker",
"private-command-marker",
"private-file-marker",
"private-proof-marker",
"private-header-marker",
"private-workdir-marker",
"private-owner-marker",
];
#[derive(Clone, Default)]
struct LogCapture(Arc<Mutex<Vec<u8>>>);
impl Write for LogCapture {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.0.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
impl LogCapture {
fn subscriber(&self) -> impl tracing::Subscriber + Send + Sync + 'static {
let writer = self.clone();
tracing_subscriber::fmt()
.with_ansi(false)
.without_time()
.with_max_level(tracing::Level::WARN)
.with_writer(move || writer.clone())
.finish()
}
fn take(&self) -> String {
String::from_utf8(std::mem::take(&mut *self.0.lock().unwrap())).unwrap()
}
}
fn assert_private_absent(text: &str) {
for marker in PRIVATE_MARKERS {
assert!(
!text.contains(marker),
"private input escaped error boundary"
);
}
}
fn assert_failure_log(log: &str, status: StatusCode, code: &str, reason: &str, stage: &str) {
let events: Vec<_> = log
.lines()
.filter(|line| line.contains("workdir_provider_failure"))
.collect();
assert_eq!(events.len(), 1, "one diagnostic per failed request: {log}");
let event = events[0];
for field in [
"WARN".to_string(),
"worker_runtime::workdir".to_string(),
format!("status={}", status.as_u16()),
format!("code=\"{code}\""),
format!("failure_reason=\"{reason}\""),
format!("failure_stage=\"{stage}\""),
] {
assert!(event.contains(&field), "missing {field}: {event}");
}
assert_private_absent(log);
}
fn assert_failure_body(body: &str, code: &str, message: &str, reason: &str) {
assert_private_absent(body);
// Exact safe transport vocabulary; raw diagnostics and stage are not wire fields.
assert_eq!(
serde_json::from_str::<serde_json::Value>(body).unwrap(),
serde_json::json!({"code": code, "message": message, "failure_reason": reason}),
);
}
struct Harness {
server: tokio::task::JoinHandle<()>,
client: reqwest::Client,
base: String,
identity: RuntimeIdentityMaterial,
trust: WorkspaceIssuerTrustRecord,
}
impl Drop for Harness {
fn drop(&mut self) {
self.server.abort();
}
}
impl Harness {
async fn new(runtime: Runtime) -> Self {
let identity = RuntimeIdentityMaterial::generate("runtime-failure-test-key").unwrap();
let trust = workspace_trust(&identity);
let app = runtime_http_router_with_workspace_auth(
runtime,
None,
workspace_test_auth(vec![trust.clone()]),
);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
let server = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
Self {
server,
client: reqwest::Client::builder()
.no_proxy()
.timeout(Duration::from_secs(5))
.build()
.unwrap(),
base,
identity,
trust,
}
}
async fn send(
&self,
claims: &WorkspaceCapabilityClaims,
body: Vec<u8>,
workspace: &str,
) -> (StatusCode, String, String) {
let token = issue_workspace_capability_token(&self.identity.signing_key().unwrap(), claims)
.unwrap();
let response = self
.client
.post(format!("{}{OPEN_PATH}", self.base))
.bearer_auth(&token)
.header(header::CONTENT_TYPE, "application/json")
.header(RUNTIME_WORKSPACE_SCOPE_HEADER, workspace)
.header("x-private-test-header", "private-header-marker")
.body(body)
.send()
.await
.unwrap();
let status = StatusCode::from_u16(response.status().as_u16()).unwrap();
let headers = format!("{:?}", response.headers());
let body = response.text().await.unwrap();
assert_private_absent(&headers);
for output in [&body, &headers] {
assert!(!output.contains(&token), "signed bearer token leaked");
assert!(
!output.contains(&self.identity.private_key),
"signing key leaked"
);
}
(status, body, token)
}
async fn open(&self) -> (StatusCode, String, String) {
let body = serde_json::to_vec(&OpenWorkdirSessionRequest::default()).unwrap();
let claims = signed_claims(&self.trust, "POST", OPEN_PATH, &body);
self.send(&claims, body, "workspace-a").await
}
}
#[derive(Clone, Copy)]
enum FailureAt {
Lookup,
Open,
}
struct FailingBackend {
at: FailureAt,
code: &'static str,
lookups: AtomicUsize,
opens: AtomicUsize,
}
impl FailingBackend {
fn runtime(at: FailureAt, code: &'static str) -> (Runtime, Arc<Self>) {
let backend = Arc::new(Self {
at,
code,
lookups: AtomicUsize::new(0),
opens: AtomicUsize::new(0),
});
let runtime =
Runtime::with_execution_backend(RuntimeOptions::default(), backend.clone()).unwrap();
(runtime, backend)
}
fn failure(&self) -> WorkingDirectoryDiagnostic {
WorkingDirectoryDiagnostic::rejected(self.code, PRIVATE_DIAGNOSTIC)
}
}
impl WorkerExecutionBackend for FailingBackend {
fn backend_id(&self) -> &str {
"runtime-failure-test"
}
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
AcceptingBackend.spawn_worker(request)
}
fn dispatch_input(&self, worker: &WorkerRef, input: WorkerInput) -> WorkerExecutionResult {
AcceptingBackend.dispatch_input(worker, input)
}
fn stop_worker(&self, worker: &WorkerRef) -> WorkerExecutionResult {
AcceptingBackend.stop_worker(worker)
}
fn working_directory(
&self,
id: &str,
) -> Result<WorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
self.lookups.fetch_add(1, Ordering::SeqCst);
match self.at {
FailureAt::Lookup => Err(self.failure()),
FailureAt::Open => Ok(WorkingDirectoryStatus {
summary: crate::catalog::WorkingDirectorySummary {
working_directory_id: id.into(),
display_name: None,
repository_id: "repository-test".into(),
creation_selector: None,
creation_ref: None,
creation_tree: None,
current_selector: None,
current_ref: None,
current_tree: None,
observed_at_epoch_seconds: None,
materializer_kind: crate::catalog::MaterializerKind::RuntimeGitClone,
cleanup_target: None,
status: crate::catalog::WorkingDirectoryStatusKind::Active,
cleanliness: None,
occupied_by: None,
},
}),
}
}
fn open_workdir_session(
&self,
_: &str,
) -> Result<WorkdirSessionHandle, WorkingDirectoryDiagnostic> {
self.opens.fetch_add(1, Ordering::SeqCst);
Err(self.failure())
}
}
// Current-thread execution keeps the server tasks inside the thread-local tracing
// capture across awaits, without replacing the process-global subscriber.
#[tokio::test(flavor = "current_thread")]
async fn signed_http_session_open_preserves_runtime_availability_and_owner_failures() {
let capture = LogCapture::default();
let _guard = tracing::subscriber::set_default(capture.subscriber());
tracing::callsite::rebuild_interest_cache();
tokio::time::timeout(Duration::from_secs(15), async {
let stopped = Runtime::new_memory();
stopped.stop_runtime().unwrap();
let (owned, backend) = FailingBackend::runtime(FailureAt::Lookup, "must_not_reach_backend");
let mut request = task_request("bind owner");
request.config_bundle = None;
request.memory_settings.as_mut().unwrap().workspace_id = "workspace-a".into();
owned
.create_worker_scoped(
&RuntimeWorkspaceScope::new("workspace-a", "private-owner-marker"),
request,
)
.unwrap();
for (runtime, status, code, message, reason) in [
(
stopped,
StatusCode::SERVICE_UNAVAILABLE,
"unavailable",
"Workdir session is unavailable",
"runtime_stopped",
),
(
Runtime::new_memory(),
StatusCode::SERVICE_UNAVAILABLE,
"unavailable",
"Workdir session is unavailable",
"runtime_backend_unavailable",
),
(
owned,
StatusCode::NOT_FOUND,
"not_found",
"Workdir session was not found",
"runtime_owner_mismatch",
),
] {
let harness = Harness::new(runtime).await;
capture.take();
let (actual, body, token) = harness.open().await;
assert_eq!(actual, status);
assert_failure_body(&body, code, message, reason);
let log = capture.take();
assert_failure_log(&log, status, code, reason, "runtime");
assert!(!log.contains(&token));
assert!(!log.contains(&harness.identity.private_key));
}
assert_eq!(backend.lookups.load(Ordering::SeqCst), 0);
assert_eq!(backend.opens.load(Ordering::SeqCst), 0);
})
.await
.expect("bounded Runtime failure HTTP requests");
}
#[tokio::test(flavor = "current_thread")]
async fn signed_http_session_open_allowlists_diagnostic_codes_without_guessing_unknown_causes() {
let capture = LogCapture::default();
let _guard = tracing::subscriber::set_default(capture.subscriber());
tracing::callsite::rebuild_interest_cache();
tokio::time::timeout(Duration::from_secs(20), async {
for (diagnostic, reason) in [
("working_directory_not_found", "working_directory_not_found"),
(
"working_directory_not_active",
"working_directory_not_active",
),
(
"working_directory_record_invalid",
"working_directory_record_invalid",
),
(
"working_directory_repository_access_unavailable",
"working_directory_access_unavailable",
),
// Plausible provider text/code is not evidence of a typed cause.
("authorization_stale", "session_open_failed"),
(
"working_directory_not_found_private-proof-marker",
"session_open_failed",
),
] {
for at in [FailureAt::Lookup, FailureAt::Open] {
let (runtime, backend) = FailingBackend::runtime(at, diagnostic);
let harness = Harness::new(runtime).await;
capture.take();
let (status, body, token) = harness.open().await;
let (expected, code, message) = if diagnostic == "working_directory_not_found" {
(
StatusCode::NOT_FOUND,
"not_found",
"Workdir session was not found",
)
} else {
(
StatusCode::SERVICE_UNAVAILABLE,
"unavailable",
"Workdir session is unavailable",
)
};
assert_eq!(status, expected, "{diagnostic}");
assert_failure_body(&body, code, message, reason);
let log = capture.take();
assert_failure_log(&log, status, code, reason, "session_open");
assert!(!log.contains(&token));
assert!(!log.contains(&harness.identity.private_key));
assert_eq!(backend.lookups.load(Ordering::SeqCst), 1, "no hidden retry");
assert_eq!(
backend.opens.load(Ordering::SeqCst),
usize::from(matches!(at, FailureAt::Open)),
);
}
}
})
.await
.expect("bounded diagnostic fault matrix");
}
#[tokio::test(flavor = "current_thread")]
async fn signed_http_session_open_rejects_bad_capability_scope_and_owner_binding_before_backend() {
let capture = LogCapture::default();
let _guard = tracing::subscriber::set_default(capture.subscriber());
tracing::callsite::rebuild_interest_cache();
tokio::time::timeout(Duration::from_secs(15), async {
let (runtime, backend) =
FailingBackend::runtime(FailureAt::Open, "working_directory_record_invalid");
let mut request = task_request("foreign workspace binding");
request.config_bundle = None;
request.memory_settings.as_mut().unwrap().workspace_id = "other-workspace".into();
let foreign_worker = runtime
.create_worker_scoped(
&RuntimeWorkspaceScope::new("other-workspace", "https://backend.test"),
request,
)
.unwrap()
.worker_ref;
let harness = Harness::new(runtime).await;
for case in ["permission", "body", "workspace", "owner_binding"] {
let body = if case == "owner_binding" {
serde_json::to_vec(&OpenWorkdirSessionRequest {
owner_worker_id: Some(foreign_worker.worker_id.to_string()),
})
.unwrap()
} else {
b"{}".to_vec()
};
let mut claims = signed_claims(&harness.trust, "POST", OPEN_PATH, &body);
if case == "permission" {
claims.operation = "workers:read".into();
}
if case == "workspace" {
claims.issuer_workspace_id = "other-workspace".into();
}
let sent = if case == "body" {
b"{ }".to_vec()
} else {
body
};
capture.take();
let (status, body, token) = harness.send(&claims, sent, "workspace-a").await;
assert_eq!(
status,
if case == "owner_binding" {
StatusCode::NOT_FOUND
} else {
StatusCode::UNAUTHORIZED
},
"{case}",
);
assert_private_absent(&body);
let log = capture.take();
assert_private_absent(&log);
assert!(!log.contains(&token));
assert!(!log.contains(&harness.identity.private_key));
if case == "owner_binding" {
assert_failure_body(
&body,
"not_found",
"Workdir session was not found",
"working_directory_not_found",
);
assert_failure_log(
&log,
status,
"not_found",
"working_directory_not_found",
"session_open",
);
} else {
let wire: serde_json::Value = serde_json::from_str(&body).unwrap();
assert_eq!(wire["error"]["code"], "unauthorized");
assert!(
!log.contains("workdir_provider_failure"),
"auth rejection is not a provider fault",
);
}
}
assert_eq!(backend.lookups.load(Ordering::SeqCst), 0);
assert_eq!(backend.opens.load(Ordering::SeqCst), 0);
})
.await
.expect("bounded authorization rejection requests");
}
#[tokio::test(flavor = "current_thread")]
async fn runtime_error_response_keeps_raw_details_private_and_unclassified_causes_generic() {
let capture = LogCapture::default();
let _guard = tracing::subscriber::set_default(capture.subscriber());
tracing::callsite::rebuild_interest_cache();
for (error, status, code, message, reason, stage) in [
(
RuntimeError::ExecutionBackendUnavailable {
message: PRIVATE_DIAGNOSTIC.into(),
},
StatusCode::SERVICE_UNAVAILABLE,
"unavailable",
"Workdir session is unavailable",
"runtime_backend_unavailable",
"runtime",
),
(
RuntimeError::StatePoisoned,
StatusCode::SERVICE_UNAVAILABLE,
"unavailable",
"Workdir session is unavailable",
"runtime_state_unavailable",
"runtime",
),
(
RuntimeError::InvalidRequest(PRIVATE_DIAGNOSTIC.into()),
StatusCode::BAD_REQUEST,
"invalid_request",
"Workdir operation request is invalid",
"runtime_invalid_request",
"runtime",
),
(
RuntimeError::StoreCorrupt {
operation: "session_open",
path: PathBuf::from("/private/host-path-marker"),
message: PRIVATE_DIAGNOSTIC.into(),
},
StatusCode::SERVICE_UNAVAILABLE,
"unavailable",
"Workdir session is unavailable",
"session_open_failed",
"session_open",
),
] {
capture.take();
let response = RuntimeHttpWorkdirError::runtime(error).into_response();
assert_eq!(response.status(), status);
let bytes = to_bytes(response.into_body(), 16384).await.unwrap();
let body = std::str::from_utf8(&bytes).unwrap();
assert_failure_body(body, code, message, reason);
assert_failure_log(&capture.take(), status, code, reason, stage);
}
}
@@ -150,6 +150,27 @@ impl Fixture {
limit: 1,
}
}
fn recording_request(&self) -> runtime_api::SessionRecordingCaptureRequest {
runtime_api::SessionRecordingCaptureRequest {
workspace_id: self.scope.workspace_id.clone(),
source: self.source(),
expected_session_id: self.session.to_string(),
start: Default::default(),
max_entries: 1000,
max_serialized_bytes: 128 * 1024,
}
}
fn capture(&self, source: Source) -> session_store::capture::RecordingCaptureSlice {
let mut request = self.recording_request();
request.source = source;
match self
.runtime
.session_recording_capture_scoped(&self.scope, &request)
{
runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => slice,
other => panic!("{other:?}"),
}
}
fn catalog(&self) -> runtime_api::SessionMaterialCatalogPage {
match self
.runtime
@@ -226,6 +247,9 @@ fn pending_session_proof_is_not_missing_material_and_pin_release_allows_empty_ar
.is_err(),
"retention intent is not source availability"
);
assert!(
matches!(f.runtime.session_recording_capture_scoped(&f.scope, &f.recording_request()), runtime_api::SessionRecordingCaptureAvailability::NotMaterialized { session_id } if session_id == f.session.to_string())
);
// A missing committed source is never the same fact as a pending pointer.
metadata
.write(
@@ -479,6 +503,7 @@ fn source_scope_session_and_active_pointer_mismatches_never_return_material() {
#[test]
fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source() {
let f = Fixture::new();
let frozen = f.capture(f.source());
let original: Vec<_> = f.reader(&f.source()).descriptors().cloned().collect();
let pin = SessionRetentionPin(uuid::Uuid::now_v7());
f.runtime
@@ -516,6 +541,7 @@ fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source()
let archived = Source::Archived {
archive_id: "canonical-archive".into(),
};
assert_eq!(f.capture(archived.clone()), frozen);
let reader = f.reader(&archived);
assert_eq!(reader.descriptors().cloned().collect::<Vec<_>>(), original);
let body = reader.read_content(&original[0], 0, 16384).unwrap();
@@ -951,11 +977,52 @@ async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router
.authorizer(SignedAuthorizer { identity, trust })
.build()
.unwrap();
let frozen = match client
.session_recording_capture(f.recording_request())
.await
.unwrap()
{
runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => slice,
other => panic!("{other:?}"),
};
frozen.capture.validate(Default::default()).unwrap();
assert!(frozen.next.is_none());
assert!(
serde_json::to_vec(&frozen).unwrap().len()
<= f.recording_request().max_serialized_bytes
);
let snapshot = serde_json::to_vec(&frozen).unwrap();
assert!(!String::from_utf8_lossy(&snapshot).contains("private-system-prompt"));
f.store
.append(
f.session,
f.segment,
&LogEntry::AnnotatedAssistantItem {
ts: 6,
entry: message("late-answer", LoggedRole::Assistant, "not in frozen input"),
},
)
.unwrap();
assert_eq!(serde_json::to_vec(&frozen).unwrap(), snapshot);
let page = match client.session_material_catalog(f.request()).await.unwrap() {
Catalog::Page { page } => page,
other => panic!("{other:?}"),
};
let descriptor = page.descriptors[0].clone();
assert_eq!(
frozen
.capture
.recording_text(0)
.unwrap()
.0
.identity
.entry_ref,
descriptor.entry_ref
);
assert_eq!(
frozen.capture.recording_text(0).unwrap().0.sha256,
descriptor.sha256
);
let read = client
.session_material_read(runtime_api::SessionMaterialReadRequest {
workspace_id: "workspace-a".into(),
@@ -1019,6 +1086,13 @@ async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router
);
}
assert!(!f.root().exists());
assert!(matches!(
client
.session_recording_capture(f.recording_request())
.await
.unwrap(),
runtime_api::SessionRecordingCaptureAvailability::Unavailable { .. }
));
assert!(matches!(
client.session_material_catalog(f.request()).await.unwrap(),
Catalog::Unavailable { .. }
@@ -1050,41 +1124,54 @@ async fn canonical_http_rejects_replay_tampered_body_and_workspace_mismatch() {
None,
workspace_test_auth(vec![trust.clone()]),
);
let path = "/v1/session-public/material/catalog";
let body = serde_json::to_vec(&f.request()).unwrap();
let claims = signed_claims(&trust, "POST", path, &body);
let first = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
assert_eq!(first.status(), StatusCode::OK);
let replay = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
assert_eq!(replay.status(), StatusCode::UNAUTHORIZED);
let claims = signed_claims(&trust, "POST", path, &body);
let changed = signed_request(
app.clone(),
&identity,
&claims,
b"{}".to_vec(),
"workspace-a",
)
.await;
assert_eq!(changed.status(), StatusCode::UNAUTHORIZED);
let mut wrong_scope = f.request();
wrong_scope.workspace_id = "other-workspace".into();
let wrong_body = serde_json::to_vec(&wrong_scope).unwrap();
let claims = signed_claims(&trust, "POST", path, &wrong_body);
let response = signed_request(app.clone(), &identity, &claims, wrong_body, "workspace-a").await;
assert_eq!(response.status(), StatusCode::FORBIDDEN);
let unsigned = app
.oneshot(
Request::builder()
.method("POST")
.uri(path)
.header(header::CONTENT_TYPE, "application/json")
.body(Body::from(body))
.unwrap(),
for (path, body) in [
(
"/v1/session-public/material/catalog",
serde_json::to_vec(&f.request()).unwrap(),
),
(
"/v1/session-public/recording-capture",
serde_json::to_vec(&f.recording_request()).unwrap(),
),
] {
let claims = signed_claims(&trust, "POST", path, &body);
let first =
signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
assert_eq!(first.status(), StatusCode::OK);
let replay =
signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
assert_eq!(replay.status(), StatusCode::UNAUTHORIZED);
let claims = signed_claims(&trust, "POST", path, &body);
let changed = signed_request(
app.clone(),
&identity,
&claims,
b"{}".to_vec(),
"workspace-a",
)
.await
.unwrap();
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
.await;
assert_eq!(changed.status(), StatusCode::UNAUTHORIZED);
let mut wrong_scope: serde_json::Value = serde_json::from_slice(&body).unwrap();
wrong_scope["workspace_id"] = "other-workspace".into();
let wrong_body = serde_json::to_vec(&wrong_scope).unwrap();
let claims = signed_claims(&trust, "POST", path, &wrong_body);
let response =
signed_request(app.clone(), &identity, &claims, wrong_body, "workspace-a").await;
assert_eq!(response.status(), StatusCode::FORBIDDEN);
let unsigned = app
.clone()
.oneshot(
Request::builder()
.method("POST")
.uri(path)
.header(header::CONTENT_TYPE, "application/json")
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
}
}
#[tokio::test]
@@ -1170,3 +1257,152 @@ async fn explicit_material_erasure_requires_delete_capability_and_matching_scope
);
assert!(f.root().exists());
}
#[test]
fn recording_capture_rejects_scope_session_cursor_and_budget_without_returning_bodies() {
use runtime_api::SessionRecordingCaptureAvailability as Capture;
let f = Fixture::new();
let request = f.recording_request();
let frozen = f.capture(f.source());
assert_eq!(frozen.capture.entries.len(), 3);
let mut request = request;
for invalid in 0..5 {
let mut bad = request.clone();
match invalid {
0 => bad.workspace_id = "other".into(),
1 => bad.expected_session_id = session_store::new_session_id().to_string(),
2 => bad.start.canonical_order = 100,
3 => bad.max_entries = 0,
4 => bad.max_serialized_bytes = 4 * 1024 * 1024 + 1,
_ => unreachable!(),
}
assert!(matches!(
f.runtime.session_recording_capture_scoped(&f.scope, &bad),
Capture::Unavailable { .. }
));
}
let text = frozen.capture.recording_text(1).unwrap().2;
request.start = session_store::capture::RecordingPosition {
canonical_order: 1,
byte_offset: text.find('🍎').unwrap() as u64 + 1,
};
assert!(matches!(
f.runtime
.session_recording_capture_scoped(&f.scope, &request),
Capture::Unavailable {
reason: Reason::InvalidCursor,
..
}
));
request.start = Default::default();
request.max_entries = 1;
let Capture::Captured { slice } = f
.runtime
.session_recording_capture_scoped(&f.scope, &request)
else {
panic!()
};
assert_eq!(slice.capture.entries.len(), 1);
assert_eq!(slice.next.unwrap().canonical_order, 1);
}
#[test]
fn recording_capture_compaction_preserves_original_descriptors_and_excludes_summary_and_new_tail() {
let f = Fixture::new();
let old = f.capture(f.source());
let compact = session_store::new_segment_id();
let mut summary = message(
"summary",
LoggedRole::System,
"derived summary is not original",
);
summary.metadata.input_metadata = Some(
session_store::HistoryInputMetadata::new(
session_store::HistoryInputUsePolicy {
recording_material: session_store::InputEvidenceUse::Exclude,
memory_evidence: session_store::InputEvidenceUse::Exclude,
observation: session_store::InputObservationUse::Hidden,
compaction: session_store::InputCompactionUse::OpeningSummary,
},
None,
None,
)
.unwrap(),
);
summary.metadata.derivation = Some(session_store::LoggedHistoryDerivation {
sources: vec![
LoggedSessionHistoryEntryId("user-1".into()),
LoggedSessionHistoryEntryId("answer-1".into()),
],
});
f.store
.create_segment(
f.session,
compact,
&[
LogEntry::AnnotatedSegmentStart {
ts: 6,
session_id: f.session,
system_prompt: Some("private".into()),
config: Default::default(),
history: vec![summary],
forked_from: None,
compacted_from: Some(session_store::SegmentOrigin {
segment_id: f.segment,
at_turn_index: 0,
}),
},
LogEntry::AnnotatedAssistantItem {
ts: 7,
entry: message(
"new-answer",
LoggedRole::Assistant,
"new original after compaction",
),
},
],
)
.unwrap();
WorkerAggregateStore::new(
f.root().parent().unwrap(),
format!("worker-runtime-{}", f.worker.worker_id),
)
.unwrap()
.set_active(
&format!("worker-runtime-{}", f.worker.worker_id),
Some(session_store::WorkerActiveSegmentRef::active_segment(
f.session, compact,
)),
None,
)
.unwrap();
let frozen = f.capture(f.source());
assert_eq!(frozen.capture.entries.len(), old.capture.entries.len() + 1);
for entry in &old.capture.entries {
assert_eq!(
frozen.capture.recording_text(entry.order).unwrap(),
old.capture.recording_text(entry.order).unwrap()
);
}
assert!(
!serde_json::to_string(&frozen)
.unwrap()
.contains("derived summary is not original")
);
let bytes = serde_json::to_vec(&frozen).unwrap();
f.store
.append(
f.session,
compact,
&LogEntry::AnnotatedAssistantItem {
ts: 8,
entry: message("tail", LoggedRole::Assistant, "next Job only"),
},
)
.unwrap();
assert_eq!(serde_json::to_vec(&frozen).unwrap(), bytes);
assert_eq!(
f.capture(f.source()).capture.entries.len(),
frozen.capture.entries.len() + 1
);
}
+5
View File
@@ -16,6 +16,11 @@ pub mod management;
pub mod observation;
pub mod profile_archive;
pub mod resource;
#[cfg(feature = "restore-checkpoints")]
pub mod restore_checkpoints;
#[cfg(test)]
#[path = "../../../tests/support/restore_environment.rs"]
mod restore_test_environment;
#[cfg(feature = "fs-store")]
pub mod retention;
mod runtime;
+72
View File
@@ -117,6 +117,12 @@ fn run() -> Result<(), ProcessError> {
let listener = tokio::net::TcpListener::bind(config.http.bind_addr).await?;
let local_addr = listener.local_addr()?;
let worker_runtime = build_runtime(&config)?;
#[cfg(feature = "restore-checkpoints")]
if let Some(checkpoint) = &config.test_restore_checkpoint {
worker_runtime
.install_test_restore_checkpoint(checkpoint.clone())
.map_err(ProcessError::Runtime)?;
}
eprintln!(
"yoi-runtime listening on {local_addr}; intended client is a trusted backend/proxy, not a browser"
);
@@ -391,6 +397,21 @@ where
let (flag, inline_value) = split_flag_value(arg)?;
match flag.as_str() {
"--help" | "-h" => return Ok(None),
#[cfg(feature = "restore-checkpoints")]
"--test-restore-checkpoint" => {
let path = take_value(&flag, inline_value, &mut args)?;
if config.test_restore_checkpoint.is_some() {
return Err(ProcessError::usage(
"--test-restore-checkpoint may only be specified once".into(),
));
}
config.test_restore_checkpoint = Some(
worker_runtime::restore_checkpoints::RestoreCheckpointConfig::from_path(
Path::new(&path),
)
.map_err(ProcessError::Runtime)?,
);
}
"--bind" => {
let value = take_value(&flag, inline_value, &mut args)?;
config.http.bind_addr = value.parse::<SocketAddr>().map_err(|error| {
@@ -465,6 +486,12 @@ where
}
apply_store_selection(&mut config);
#[cfg(feature = "restore-checkpoints")]
if config.test_restore_checkpoint.is_some() && config.no_store {
return Err(ProcessError::usage(
"--test-restore-checkpoint requires an fs Runtime store".into(),
));
}
Ok(Some(config))
}
@@ -521,6 +548,8 @@ struct ProcessConfig {
no_store: bool,
backend_resource_endpoint: Option<String>,
backend_resource_token: Option<String>,
#[cfg(feature = "restore-checkpoints")]
test_restore_checkpoint: Option<worker_runtime::restore_checkpoints::RestoreCheckpointConfig>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -542,6 +571,8 @@ impl ProcessConfig {
no_store: false,
backend_resource_endpoint: None,
backend_resource_token: None,
#[cfg(feature = "restore-checkpoints")]
test_restore_checkpoint: None,
})
}
@@ -1227,6 +1258,47 @@ Auth commands:
mod tests {
use super::*;
#[test]
fn checkpoint_cli_is_absent_by_default() {
let config = parse_args([] as [&str; 0]).unwrap().unwrap();
#[cfg(feature = "restore-checkpoints")]
assert!(config.test_restore_checkpoint.is_none());
#[cfg(not(feature = "restore-checkpoints"))]
{
let _ = config;
let error = parse_args(["--test-restore-checkpoint", "never-opened.json"]).unwrap_err();
assert!(error.to_string().contains("unknown argument"));
}
}
#[cfg(feature = "restore-checkpoints")]
#[test]
fn checkpoint_cli_requires_valid_explicit_file_and_fs_store() {
use worker_runtime::restore_checkpoints::{
RestoreCheckpointConfig, RestoreCheckpointStage,
};
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("checkpoint.json");
let checkpoint = RestoreCheckpointConfig {
endpoint: "127.0.0.1:1234".parse().unwrap(),
worker_id: protocol::WorkerId::now_v7(),
stage: RestoreCheckpointStage::CommittedPreterminal,
nonce: "cli-test".into(),
};
std::fs::write(&path, serde_json::to_vec(&checkpoint).unwrap()).unwrap();
let args = ["--test-restore-checkpoint", path.to_str().unwrap()];
let config = parse_args(args).unwrap().unwrap();
assert_eq!(config.test_restore_checkpoint, Some(checkpoint));
assert!(parse_args([args[0], args[1], "--no-store"]).is_err());
assert!(parse_args([args[0], args[1], args[0], args[1]]).is_err());
assert!(parse_args([args[0]]).is_err());
std::fs::write(&path, b"private-invalid-body").unwrap();
let error = parse_args(args).unwrap_err().to_string();
assert!(!error.contains("private-invalid-body"));
std::fs::remove_file(&path).unwrap();
assert!(parse_args(args).is_err());
}
#[test]
fn help_does_not_start_server() {
assert!(parse_args(["--help"]).unwrap().is_none());
+2
View File
@@ -1,6 +1,8 @@
#[cfg(feature = "http-server")]
use crate::auth::BACKEND_RESOURCE_FETCH_PERMISSION;
use crate::identity::WorkerId;
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef, sha256_hex};
#[cfg(feature = "http-server")]
use crate::workspace_request::RuntimeWorkspaceRequestClient;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
@@ -0,0 +1,144 @@
//! Explicit, ephemeral synchronization for owned Runtime process tests only.
//!
//! The nonce authenticates no Runtime action and is never a routing identity.
//! Socket failure, EOF and the fixed deadline only release the gate; the lifecycle
//! owner continues its original operation and remains responsible for cleanup.
use std::fs::OpenOptions;
use std::io::{Read, Write};
use std::net::{SocketAddr, TcpStream};
use std::path::Path;
use std::time::{Duration, Instant};
use protocol::WorkerId;
use serde::{Deserialize, Serialize};
use crate::error::RuntimeError;
use crate::execution::WorkerLifecycleOperationId;
pub const RESTORE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(8);
pub const MAX_RESTORE_CHECKPOINT_CONFIG_BYTES: usize = 4096;
pub const MAX_RESTORE_CHECKPOINT_ACK_BYTES: usize = 4096;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RestoreCheckpointStage {
PreparedNotCommitted,
CommittedPreterminal,
ActivatedPreterminal,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RestoreCheckpointCommit {
NotCommitted,
Committed,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RestoreCheckpointConfig {
pub endpoint: SocketAddr,
pub worker_id: WorkerId,
pub stage: RestoreCheckpointStage,
pub nonce: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RestoreCheckpointAck {
pub nonce: String,
pub stage: RestoreCheckpointStage,
pub runtime_id: String,
pub worker_id: WorkerId,
pub operation_id: WorkerLifecycleOperationId,
pub commit: RestoreCheckpointCommit,
pub receipt_absent: bool,
pub pid: u32,
}
fn invalid_config() -> RuntimeError {
// Do not include serde errors, paths or input contents in process diagnostics.
RuntimeError::InvalidRequest("invalid test restore checkpoint configuration".into())
}
impl RestoreCheckpointConfig {
pub fn validate(&self) -> Result<(), RuntimeError> {
if !self.endpoint.ip().is_loopback()
|| self.endpoint.port() == 0
|| self.nonce.is_empty()
|| self.nonce.len() > 256
|| self.nonce.chars().any(char::is_control)
{
return Err(invalid_config());
}
Ok(())
}
pub fn from_path(path: &Path) -> Result<Self, RuntimeError> {
let mut options = OpenOptions::new();
options.read(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
// A FIFO must not stall startup before its regular-file check.
options.custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW);
}
let file = options.open(path).map_err(|_| invalid_config())?;
let metadata = file.metadata().map_err(|_| invalid_config())?;
if !metadata.is_file() || metadata.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 {
return Err(invalid_config());
}
let mut bytes = Vec::new();
file.take(MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 + 1)
.read_to_end(&mut bytes)
.map_err(|_| invalid_config())?;
if bytes.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES {
return Err(invalid_config());
}
let config: Self = serde_json::from_slice(&bytes).map_err(|_| invalid_config())?;
config.validate()?;
Ok(config)
}
}
pub(crate) fn wait_for_release(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) {
let _ = exchange(config, ack);
}
fn remaining(deadline: Instant) -> std::io::Result<Duration> {
deadline
.checked_duration_since(Instant::now())
.filter(|remaining| !remaining.is_zero())
.ok_or_else(|| std::io::Error::from(std::io::ErrorKind::TimedOut))
}
fn exchange(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) -> std::io::Result<()> {
let deadline = Instant::now() + RESTORE_CHECKPOINT_TIMEOUT;
let mut bytes = serde_json::to_vec(ack)?;
bytes.push(b'\n');
if bytes.len() > MAX_RESTORE_CHECKPOINT_ACK_BYTES {
return Err(std::io::Error::from(std::io::ErrorKind::InvalidData));
}
let mut stream = TcpStream::connect_timeout(&config.endpoint, remaining(deadline)?)?;
// Each syscall shares one budget; partial writes cannot reset the deadline.
let mut pending = bytes.as_slice();
while !pending.is_empty() {
stream.set_write_timeout(Some(remaining(deadline)?))?;
let written = stream.write(pending)?;
if written == 0 {
return Err(std::io::Error::from(std::io::ErrorKind::WriteZero));
}
pending = &pending[written..];
}
stream.set_read_timeout(Some(remaining(deadline)?))?;
let mut release = [0];
stream.read_exact(&mut release)?;
if release != [b'R'] {
return Err(std::io::Error::from(std::io::ErrorKind::InvalidData));
}
Ok(())
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,139 @@
use super::*;
use std::io::{BufRead, BufReader};
use std::net::TcpListener;
use std::thread;
fn config(endpoint: SocketAddr) -> RestoreCheckpointConfig {
RestoreCheckpointConfig {
endpoint,
worker_id: WorkerId::now_v7(),
stage: RestoreCheckpointStage::PreparedNotCommitted,
nonce: "checkpoint-test-handshake".into(),
}
}
fn ack(config: &RestoreCheckpointConfig) -> RestoreCheckpointAck {
RestoreCheckpointAck {
nonce: config.nonce.clone(),
stage: config.stage,
runtime_id: "checkpoint-runtime".into(),
worker_id: config.worker_id,
operation_id: WorkerLifecycleOperationId::new(),
commit: RestoreCheckpointCommit::NotCommitted,
receipt_absent: true,
pid: std::process::id(),
}
}
#[test]
fn config_rejects_nonliteral_nonloopback_bad_stage_unknown_fields_and_invalid_worker() {
let valid = serde_json::to_value(config("127.0.0.1:1234".parse().unwrap())).unwrap();
for (field, value) in [
("endpoint", "localhost:1234"),
("endpoint", "192.0.2.1:1234"),
("endpoint", "0.0.0.0:1234"),
("endpoint", "127.0.0.1:0"),
("endpoint", "[::ffff:127.0.0.1]:1234"),
("worker_id", "not-a-worker-id"),
("stage", "after_terminal"),
("nonce", ""),
("nonce", "secret\nbody"),
("timeout", "unbounded"),
] {
let mut invalid = valid.clone();
invalid[field] = value.into();
let parsed = serde_json::from_value::<RestoreCheckpointConfig>(invalid);
assert!(
parsed.is_err() || parsed.unwrap().validate().is_err(),
"{field}"
);
}
for endpoint in ["127.0.0.1:1234", "[::1]:1234"] {
config(endpoint.parse().unwrap()).validate().unwrap();
}
}
#[test]
fn config_file_bounds_and_errors_do_not_disclose_contents() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("config.json");
assert!(RestoreCheckpointConfig::from_path(&path).is_err());
std::fs::write(&path, vec![b'x'; MAX_RESTORE_CHECKPOINT_CONFIG_BYTES + 1]).unwrap();
assert!(RestoreCheckpointConfig::from_path(&path).is_err());
std::fs::write(
&path,
br#"{"nonce":"private-config-body","stage":"secret"}"#,
)
.unwrap();
let error = RestoreCheckpointConfig::from_path(&path)
.unwrap_err()
.to_string();
assert!(!error.contains("private-config-body"));
assert!(!error.contains("secret"));
let mut valid = config("127.0.0.1:1234".parse().unwrap());
valid.nonce = "x".repeat(257);
assert!(valid.validate().is_err());
valid.nonce = "valid".into();
std::fs::write(&path, serde_json::to_vec(&valid).unwrap()).unwrap();
assert_eq!(RestoreCheckpointConfig::from_path(&path).unwrap(), valid);
assert!(RestoreCheckpointConfig::from_path(dir.path()).is_err());
}
#[test]
fn explicit_release_eof_and_invalid_release_close_the_owned_stream() {
for release in [Some(b'R'), None, Some(b'X')] {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let config = config(listener.local_addr().unwrap());
let expected = ack(&config);
let sent = expected.clone();
let task = thread::spawn(move || exchange(&config, &sent));
let (mut stream, _) = listener.accept().unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(2)))
.unwrap();
let mut line = String::new();
BufReader::new(&mut stream).read_line(&mut line).unwrap();
assert!(line.len() <= MAX_RESTORE_CHECKPOINT_ACK_BYTES);
assert_eq!(
serde_json::from_str::<RestoreCheckpointAck>(&line).unwrap(),
expected
);
if let Some(byte) = release {
stream.write_all(&[byte]).unwrap();
} else {
stream.shutdown(std::net::Shutdown::Write).unwrap();
}
assert_eq!(task.join().unwrap().is_ok(), release == Some(b'R'));
let mut byte = [0];
assert_eq!(stream.read(&mut byte).unwrap(), 0);
}
}
#[test]
fn absent_listener_resumes_without_waiting_or_retrying() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let config = config(listener.local_addr().unwrap());
drop(listener);
let start = Instant::now();
wait_for_release(&config, &ack(&config));
assert!(start.elapsed() < Duration::from_secs(2));
}
#[test]
fn silent_peer_hits_fixed_deadline_and_stream_is_dropped() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let config = config(listener.local_addr().unwrap());
let start = Instant::now();
let task = thread::spawn(move || exchange(&config, &ack(&config)));
let (mut stream, _) = listener.accept().unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(12)))
.unwrap();
let mut reader = BufReader::new(&mut stream);
let mut line = String::new();
reader.read_line(&mut line).unwrap();
assert!(task.join().unwrap().is_err());
assert!(start.elapsed() >= RESTORE_CHECKPOINT_TIMEOUT - Duration::from_millis(100));
assert!(start.elapsed() < RESTORE_CHECKPOINT_TIMEOUT + Duration::from_secs(3));
assert_eq!(reader.read(&mut [0]).unwrap(), 0);
}
@@ -3,7 +3,7 @@
//! only the empty segment envelope is built with current APIs.
use super::tests::{request, write_json};
use super::*;
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions, SCHEMA_VERSION};
use serde_json::{Value, json};
use session_store::{LogEntry, SegmentId, SessionId, Store, WorkerSessionStore};
use std::collections::BTreeMap;
@@ -226,7 +226,7 @@ fn completed_v4_archive(
#[test]
fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
for runtime_schema in [11, 12] {
for runtime_schema in [11, 12, SCHEMA_VERSION] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
@@ -329,7 +329,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
(storage.checksum_sha256, storage.bytes, storage.file_count),
(checksum, bytes, count)
);
if runtime_schema == 12 {
if runtime_schema == SCHEMA_VERSION {
for name in [
PathBuf::from("runtime.json"),
PathBuf::from(format!("workers/{worker}/worker.json")),
@@ -346,8 +346,8 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
);
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
assert_eq!(tree_bytes(&root), once, "retry rewrote migrated evidence");
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists());
// session-storage, not the immutable receipt's old checksum, attests
// current bytes. Tampering with those bytes must remain detectable.
let path = archive_session
@@ -360,7 +360,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
#[test]
fn corrupt_archive_does_not_block_startup_or_healthy_session_migration_and_repair_converges() {
for runtime_schema in [11, 12] {
for runtime_schema in [11, 12, SCHEMA_VERSION] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
@@ -440,8 +440,8 @@ fn unmigratable_archive_preserves_evidence_across_startup_and_reopen() {
] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (live_worker, live_session, live_segment) = runtime_fixture(&root, 12);
let (worker, _, segment) = stopped_worker(&root, 12);
let (live_worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION);
let (worker, _, segment) = stopped_worker(&root, SCHEMA_VERSION);
let provider = FsWorkerRetentionProvider::new(&root);
let path = provider
.worker_dir(worker)
@@ -542,8 +542,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() {
] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, _, _) = runtime_fixture(&root, 12);
completed_v4_archive(&root, 12);
let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION);
completed_v4_archive(&root, SCHEMA_VERSION);
let provider = FsWorkerRetentionProvider::new(&root);
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
let archive = provider.archive_dir("archive-a").unwrap();
@@ -575,8 +575,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() {
fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
runtime_fixture(&root, 12);
let (session, _, _, _) = completed_v4_archive(&root, 12);
runtime_fixture(&root, SCHEMA_VERSION);
let (session, _, _, _) = completed_v4_archive(&root, SCHEMA_VERSION);
let ledger = root.join("retention/session-sources");
fs::create_dir_all(&ledger).unwrap();
fs::write(ledger.join(format!("{session}.json")), b"{broken\n").unwrap();
@@ -591,7 +591,7 @@ fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
fn runtime_v5_probe_and_staging_skip_erased_originals() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, live_session, live_segment) = runtime_fixture(&root, 12);
let (worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION);
let erased = root
.join("orphaned-workers")
.join(WorkerId::now_v7().to_string());
@@ -643,7 +643,7 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
for corruption in ["broken-json", "future-schema"] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, 12);
let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION);
let session_root = root
.join("workers")
.join(worker.to_string())
@@ -674,17 +674,19 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
assert_eq!(tree_bytes(&root), before);
assert!(FsRuntimeStore::migrate(&options).is_err());
assert_eq!(tree_bytes(&root), before);
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists());
}
}
#[cfg(unix)]
#[test]
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, _, _) = runtime_fixture(&root, 12);
// The same TempDir owns the relative bind/connect reference. No cwd change
// or alias is needed, and the real live-socket migration refusal stays tested.
let temp = tempfile::tempdir_in(".").unwrap();
let root = Path::new(temp.path().file_name().unwrap()).join("runtime");
let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION);
let before = tree_bytes(&root);
let runs = root.join("workers").join(worker.to_string()).join("runs");
fs::create_dir_all(runs.join("1")).unwrap();
@@ -695,7 +697,7 @@ fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
drop(listener);
fs::remove_dir_all(runs).unwrap();
assert_eq!(tree_bytes(&root), before);
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
}
#[test]
@@ -703,7 +705,7 @@ fn current_runtime_upgrades_v5_role_metadata_through_staging_without_reserializi
{
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("runtime");
let (worker, session, segment) = runtime_fixture(&root, 12);
let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION);
let aggregate = root.join("workers").join(worker.to_string());
let path = aggregate
.join("session/segments")
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,112 @@
use super::*;
use crate::restore_checkpoints::{
RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage,
wait_for_release,
};
impl Runtime {
/// Install an explicitly selected, one-shot process-test gate after normal startup.
/// It is not persisted or consulted by admission, routing or recovery authority.
pub fn install_test_restore_checkpoint(
&self,
config: RestoreCheckpointConfig,
) -> Result<(), RuntimeError> {
config.validate()?;
let mut state = self.lock()?;
if !matches!(state.persistence, RuntimePersistence::Fs(_)) {
return Err(RuntimeError::InvalidRequest(
"test restore checkpoint requires an fs Runtime store".into(),
));
}
state.restore_checkpoint = Some(config);
Ok(())
}
// The lifecycle caller holds the per-Worker operation lock. No Runtime-state
// mutex is retained while waiting; the real backend candidate remains owned by
// the same operation throughout this synchronous call.
pub(super) fn test_restore_checkpoint(
&self,
worker_ref: &WorkerRef,
operation_id: WorkerLifecycleOperationId,
stage: RestoreCheckpointStage,
) {
let config = {
let Ok(mut state) = self.lock() else {
return;
};
let slot = &mut state.restore_checkpoint;
if !slot.as_ref().is_some_and(|config| {
config.worker_id == worker_ref.worker_id && config.stage == stage
}) {
return;
}
// Consume even if canonical verification or the handshake fails.
slot.take().expect("matched checkpoint")
};
let Ok(ack) = self.restore_checkpoint_ack(&config, worker_ref, operation_id) else {
return;
};
wait_for_release(&config, &ack);
}
pub(super) fn restore_checkpoint_ack(
&self,
config: &RestoreCheckpointConfig,
worker_ref: &WorkerRef,
operation_id: WorkerLifecycleOperationId,
) -> Result<RestoreCheckpointAck, RuntimeError> {
let invalid = || {
RuntimeError::InvalidRequest("test restore checkpoint authority did not match".into())
};
let state = self.lock()?;
let RuntimePersistence::Fs(store) = &state.persistence else {
return Err(invalid());
};
// The existing helper validates the exact bounded canonical aggregate and
// syncs that same file plus its owning directories. Never manufacture a
// journal marker or infer commit from an in-memory projection.
let record = store.read_durable_worker_record(worker_ref)?;
let PersistedWorkerExecutionState::ReconciliationRequired(
PersistedWorkerLifecycleOperation::Restore(pending),
) = &record.execution_state
else {
return Err(invalid());
};
let expected = match config.stage {
RestoreCheckpointStage::PreparedNotCommitted => RestoreCommitFact::NotCommitted,
RestoreCheckpointStage::CommittedPreterminal
| RestoreCheckpointStage::ActivatedPreterminal => RestoreCommitFact::Committed,
};
if config.worker_id != worker_ref.worker_id
|| pending.operation_id != operation_id
|| pending.commit != expected
{
return Err(invalid());
}
let guard = &record.restore_guard;
let id = guard.active_request_id.as_ref().ok_or_else(invalid)?;
let owner = guard.owners.get(id).ok_or_else(invalid)?;
if owner.operation_id != operation_id
|| owner.request.request_id != *id
|| owner.receipt.is_some()
{
return Err(invalid());
}
let runtime_id = state.runtime_identity.clone().ok_or_else(invalid)?;
Ok(RestoreCheckpointAck {
nonce: config.nonce.clone(),
stage: config.stage,
runtime_id,
worker_id: record.worker_id,
operation_id,
commit: match expected {
RestoreCommitFact::NotCommitted => RestoreCheckpointCommit::NotCommitted,
RestoreCommitFact::Committed => RestoreCheckpointCommit::Committed,
RestoreCommitFact::LegacyUnknown => unreachable!(),
},
receipt_absent: true,
pid: std::process::id(),
})
}
}
@@ -0,0 +1,304 @@
use super::*;
use crate::restore_checkpoints::{
RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage,
};
use std::io::{BufRead, BufReader, Write};
use std::net::TcpListener;
use std::time::Duration;
#[test]
fn checkpoint_stages_ack_canonical_decision_and_eof_resumes_same_operation_once() {
// Component scope: durable Runtime with a substituted execution backend.
// Real prepared Controllers and process death are owned by the E2E suite.
for (stage, release) in [
(RestoreCheckpointStage::PreparedNotCommitted, "release"),
(RestoreCheckpointStage::CommittedPreterminal, "eof"),
(RestoreCheckpointStage::CommittedPreterminal, "timeout"),
(RestoreCheckpointStage::ActivatedPreterminal, "release"),
] {
let dir = tempfile::tempdir().unwrap();
let backend = Arc::new(TestExecutionBackend::default());
let runtime = Runtime::with_fs_store_and_execution_backend(
FsRuntimeStoreOptions::new(dir.path().join("runtime"))
.with_runtime_id("checkpoint-runtime"),
backend.clone(),
)
.unwrap();
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let worker = runtime
.create_worker(task_request("checkpoint original operation"))
.unwrap();
runtime.stop_worker(&worker.worker_ref, None).unwrap();
let request = runtime.test_restore_request(&worker.worker_ref);
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
runtime
.install_test_restore_checkpoint(RestoreCheckpointConfig {
endpoint: listener.local_addr().unwrap(),
worker_id: worker.worker_id,
stage,
nonce: "runtime-stage-test".into(),
})
.unwrap();
let cloned = runtime.clone();
let worker_ref = worker.worker_ref.clone();
let sent_request = request.clone();
let (sent, received) = std::sync::mpsc::channel();
let task = std::thread::spawn(move || {
sent.send(cloned.restore_worker_operation(&worker_ref, sent_request))
.unwrap();
});
listener.set_nonblocking(true).unwrap();
let stream = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(async {
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
let (stream, _) = tokio::time::timeout(Duration::from_secs(5), listener.accept())
.await
.expect("checkpoint did not connect")
.unwrap();
stream.into_std().unwrap()
});
stream.set_nonblocking(false).unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(3)))
.unwrap();
let mut reader = BufReader::new(stream);
let mut line = String::new();
reader.read_line(&mut line).unwrap();
let ack: RestoreCheckpointAck = serde_json::from_str(&line).unwrap();
assert_eq!(ack.stage, stage);
assert_eq!(ack.nonce, "runtime-stage-test");
assert_eq!(ack.runtime_id, "checkpoint-runtime");
assert_eq!(ack.worker_id, worker.worker_id);
assert_eq!(ack.pid, std::process::id());
assert!(ack.receipt_absent);
assert_eq!(
ack.commit,
match stage {
RestoreCheckpointStage::PreparedNotCommitted =>
RestoreCheckpointCommit::NotCommitted,
RestoreCheckpointStage::CommittedPreterminal
| RestoreCheckpointStage::ActivatedPreterminal => RestoreCheckpointCommit::Committed,
}
);
assert_eq!(
backend.restore_operation_ids.lock().unwrap().as_slice(),
&[ack.operation_id]
);
if stage == RestoreCheckpointStage::ActivatedPreterminal {
assert_eq!(backend.activation_operation_ids.lock().unwrap().as_slice(), &[ack.operation_id]);
} else {
assert!(backend.activation_operation_ids.lock().unwrap().is_empty());
}
assert!(received.try_recv().is_err());
let canonical = runtime_store(&runtime)
.read_durable_worker_record(&worker.worker_ref)
.unwrap();
let id = canonical.restore_guard.active_request_id.as_ref().unwrap();
assert_eq!(id, &request.request_id);
assert!(canonical.restore_guard.owners[id].receipt.is_none());
// Release, EOF and timeout all resume the original lifecycle owner.
let start = std::time::Instant::now();
if release == "release" {
reader.get_mut().write_all(b"R").unwrap();
} else if release == "eof" {
reader
.get_mut()
.shutdown(std::net::Shutdown::Write)
.unwrap();
}
let result = received
.recv_timeout(Duration::from_secs(12))
.unwrap()
.unwrap();
assert!(start.elapsed() < Duration::from_secs(11));
task.join().unwrap();
assert_eq!(result.state, WorkerRestoreState::Accepted);
assert_eq!(
backend.activation_operation_ids.lock().unwrap().as_slice(),
&[ack.operation_id]
);
assert_eq!(
runtime
.restore_worker_operation(&worker.worker_ref, request)
.unwrap(),
result
);
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
assert!(runtime.lock().unwrap().restore_checkpoint.is_none());
runtime.stop_worker(&worker.worker_ref, None).unwrap();
let next = runtime.test_restore_request(&worker.worker_ref);
assert_eq!(
runtime
.restore_worker_operation(&worker.worker_ref, next)
.unwrap()
.state,
WorkerRestoreState::Accepted
);
assert_eq!(*backend.restore_count.lock().unwrap(), 2);
}
}
#[test]
fn checkpoint_disabled_mismatched_or_unreachable_does_not_change_restore_result() {
for mode in ["disabled", "mismatch", "unreachable"] {
let dir = tempfile::tempdir().unwrap();
let backend = Arc::new(TestExecutionBackend::default());
let runtime = Runtime::with_fs_store_and_execution_backend(
FsRuntimeStoreOptions::new(dir.path().join("runtime"))
.with_runtime_id("checkpoint-runtime"),
backend.clone(),
)
.unwrap();
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let worker = runtime
.create_worker(task_request("checkpoint absent"))
.unwrap();
runtime.stop_worker(&worker.worker_ref, None).unwrap();
let request = runtime.test_restore_request(&worker.worker_ref);
if mode != "disabled" {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let endpoint = listener.local_addr().unwrap();
drop(listener);
runtime
.install_test_restore_checkpoint(RestoreCheckpointConfig {
endpoint,
worker_id: if mode == "mismatch" {
WorkerId::now_v7()
} else {
worker.worker_id
},
stage: RestoreCheckpointStage::PreparedNotCommitted,
nonce: "absent-test".into(),
})
.unwrap();
}
let result = runtime
.restore_worker_operation(&worker.worker_ref, request.clone())
.unwrap();
assert_eq!(result.state, WorkerRestoreState::Accepted, "{mode}");
assert_eq!(
runtime
.restore_worker_operation(&worker.worker_ref, request)
.unwrap(),
result
);
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
assert_eq!(
runtime.lock().unwrap().restore_checkpoint.is_some(),
mode == "mismatch"
);
}
}
#[test]
fn checkpoint_verification_requires_canonical_worker_operation_commit_and_absent_receipt() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().join("runtime");
let backend = Arc::new(TestExecutionBackend::default());
let runtime = Runtime::with_fs_store_and_execution_backend(
FsRuntimeStoreOptions::new(&root).with_runtime_id("checkpoint-runtime"),
backend.clone(),
)
.unwrap();
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
runtime.store_config_bundle(test_bundle()).unwrap();
let worker = runtime
.create_worker(task_request("checkpoint validation"))
.unwrap();
runtime.stop_worker(&worker.worker_ref, None).unwrap();
let request = runtime.test_restore_request(&worker.worker_ref);
*backend.restore_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::Errored(
WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, "not yet connected"),
));
assert_eq!(
runtime
.restore_worker_operation(&worker.worker_ref, request.clone())
.unwrap()
.state,
WorkerRestoreState::ReconciliationRequired
);
let operation = backend.restore_operation_ids.lock().unwrap()[0];
let mut config = RestoreCheckpointConfig {
endpoint: "127.0.0.1:1234".parse().unwrap(),
worker_id: worker.worker_id,
stage: RestoreCheckpointStage::PreparedNotCommitted,
nonce: "verification-test".into(),
};
let path = root
.join("workers")
.join(worker.worker_id.to_string())
.join("worker.json");
let before = std::fs::read(&path).unwrap();
assert!(
runtime
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
.is_ok()
);
assert!(
runtime
.restore_checkpoint_ack(
&config,
&worker.worker_ref,
WorkerLifecycleOperationId::new()
)
.is_err()
);
config.worker_id = WorkerId::now_v7();
assert!(
runtime
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
.is_err()
);
config.worker_id = worker.worker_id;
config.stage = RestoreCheckpointStage::CommittedPreterminal;
assert!(
runtime
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
.is_err()
);
config.stage = RestoreCheckpointStage::ActivatedPreterminal;
assert!(runtime.restore_checkpoint_ack(&config, &worker.worker_ref, operation).is_err());
config.stage = RestoreCheckpointStage::PreparedNotCommitted;
runtime_store(&runtime).fail_next_worker_reconcile_sync();
assert!(
runtime
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
.is_err()
);
assert_eq!(
std::fs::read(&path).unwrap(),
before,
"verification must not write authority"
);
// In-memory equality must not hide an unreadable canonical aggregate.
std::fs::write(&path, b"unreadable canonical aggregate").unwrap();
assert!(
runtime
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
.is_err()
);
assert_eq!(
std::fs::read(&path).unwrap(),
b"unreadable canonical aggregate"
);
std::fs::write(&path, before).unwrap();
backend.restore_result.lock().unwrap().take();
assert_eq!(
runtime
.restore_worker_operation(&worker.worker_ref, request)
.unwrap()
.state,
WorkerRestoreState::Accepted
);
assert!(
runtime
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
.is_err(),
"terminal operation cannot ack a preterminal checkpoint"
);
}
@@ -0,0 +1,522 @@
//! Runtime-owned continuation of an admitted restore, independent of its caller.
//!
//! Completion callbacks only signal a condition variable. They never acquire a
//! Runtime/lifecycle lock and are never joined by this driver while holding one.
//! Each step revalidates the canonical owner under the existing Worker lock;
//! waiting/backoff and driver teardown happen outside it. A repeated obstruction
//! parks this owner with its reason, rather than spinning or inventing rollback.
use super::*;
use crate::execution::RestoreRecoveryDisposition;
use std::collections::HashMap;
use std::sync::{Condvar, Weak};
use std::thread::JoinHandle;
use std::time::Duration;
type Key = (WorkerRef, WorkerLifecycleOperationId);
#[derive(Debug, Default)]
pub(super) struct RestoreContinuations {
owners: Mutex<HashMap<Key, Arc<Continuation>>>,
#[cfg(test)]
paused: std::sync::atomic::AtomicBool,
#[cfg(test)]
pub(super) terminal_receipt_gate:
Mutex<Option<crate::execution::RestoreTestGate<WorkerLifecycleOperationId>>>,
}
#[derive(Debug, Default)]
struct Schedule {
signaled: bool,
cancelled: bool,
finished: bool,
blocked_reason: Option<String>,
parked: bool,
}
#[derive(Debug, Default)]
struct Continuation {
schedule: Mutex<Schedule>,
wake: Condvar,
task: Mutex<Option<JoinHandle<()>>>,
}
impl RestoreContinuations {
pub(super) fn signal(&self, worker_ref: &WorkerRef, operation: WorkerLifecycleOperationId) {
if let Some(owner) = self
.owners
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&(worker_ref.clone(), operation))
{
let mut schedule = owner
.schedule
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
schedule.signaled = true;
owner.wake.notify_all();
}
}
}
impl Drop for RestoreContinuations {
fn drop(&mut self) {
let owners = self
.owners
.get_mut()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for owner in owners.values() {
let mut schedule = owner
.schedule
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
schedule.cancelled = true;
owner.wake.notify_all();
}
for owner in owners.values() {
if let Some(task) = owner
.task
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
{
if task.thread().id() != std::thread::current().id() {
let _ = task.join();
}
}
}
}
}
#[cfg(test)]
pub(crate) struct ContinuationPause(Arc<RestoreContinuations>);
#[cfg(test)]
impl Drop for ContinuationPause {
fn drop(&mut self) {
self.0
.paused
.store(false, std::sync::atomic::Ordering::Release);
for owner in self.0.owners.lock().unwrap().values() {
owner.schedule.lock().unwrap().signaled = true;
owner.wake.notify_all();
}
}
}
enum Step {
Finished,
Waiting,
Contended,
Retry(String),
}
impl Runtime {
pub(super) fn restore_repair_authorizer(
&self,
worker_ref: WorkerRef,
) -> Arc<dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync> {
let inner = Arc::downgrade(&self.inner);
Arc::new(move |operation| {
let inner = inner.upgrade().ok_or(RuntimeError::RuntimeStopped)?;
// The factory caller owns the lifecycle lock. Never reacquire it
// here: it may be awaiting this exact factory task.
let mut state = inner.lock().map_err(|_| RuntimeError::StatePoisoned)?;
state.ensure_running()?;
state.refresh_uncertain_worker_write(&worker_ref)?;
let worker = state.worker(&worker_ref)?;
state.ensure_subject_not_reserved(&worker_ref, false)?;
if worker.pending_stop.is_some()
|| worker.pending_restore.is_none_or(|pending| {
pending.operation_id != operation
|| pending.commit == RestoreCommitFact::LegacyUnknown
})
{
return Err(RuntimeError::RestoreObservationConflict {
worker_id: worker_ref.worker_id,
});
}
if let Some(owner) = worker
.restore_guard
.active_request_id
.as_ref()
.and_then(|id| worker.restore_guard.owners.get(id))
{
owner.validate_workdir_preparation_authority(worker)?;
}
if worker
.pending_restore
.is_some_and(|pending| pending.source_repair_possible)
{
return Ok(());
}
let mut candidate = worker.clone();
candidate
.pending_restore
.as_mut()
.expect("validated restore owner")
.source_repair_possible = true;
// Grant before touching another canonical store. Unknown durability
// denies repair until authoritative reconciliation; no old-memory write.
state.persist_restore_worker_record(&candidate)?;
state.workers.insert(worker_ref.worker_id, candidate);
Ok(())
})
}
#[cfg(test)]
pub(crate) fn test_wait_restore_continuation_parked(
&self,
worker_ref: &WorkerRef,
operation: WorkerLifecycleOperationId,
timeout: Duration,
) -> String {
let owner = self
.restore_continuations
.owners
.lock()
.unwrap()
.get(&(worker_ref.clone(), operation))
.cloned()
.unwrap();
let (schedule, waited) = owner
.wake
.wait_timeout_while(owner.schedule.lock().unwrap(), timeout, |s| {
!s.parked && !s.finished
})
.unwrap();
assert!(
!waited.timed_out() && schedule.parked,
"continuation did not park: {:?}",
schedule.blocked_reason
);
schedule.blocked_reason.clone().expect("parked reason")
}
/// Gate autonomous steps when a component test observes a single fault
/// boundary. Dedicated continuation tests run without this gate.
#[cfg(test)]
pub(crate) fn test_pause_restore_continuation(&self) -> ContinuationPause {
assert!(
!self
.restore_continuations
.paused
.swap(true, std::sync::atomic::Ordering::AcqRel)
);
ContinuationPause(self.restore_continuations.clone())
}
#[cfg(test)]
pub(crate) fn test_wait_restore_continuation(
&self,
worker_ref: &WorkerRef,
operation: WorkerLifecycleOperationId,
timeout: Duration,
) {
let owner = self
.restore_continuations
.owners
.lock()
.unwrap()
.get(&(worker_ref.clone(), operation))
.cloned()
.expect("continuation owner");
let schedule = owner.schedule.lock().unwrap();
let (schedule, waited) = owner
.wake
.wait_timeout_while(schedule, timeout, |s| !s.finished && !s.cancelled)
.unwrap();
assert!(
!waited.timed_out() && schedule.finished,
"restore continuation did not settle: {:?}",
schedule.blocked_reason
);
}
pub(super) fn ensure_restore_continuation(
&self,
worker_ref: &WorkerRef,
) -> Result<(), RuntimeError> {
let operation = {
let state = self.lock()?;
let worker = state.worker(worker_ref)?;
worker
.pending_restore
.map(|pending| pending.operation_id)
.or_else(|| {
worker
.restore_guard
.active_request_id
.as_ref()
.and_then(|id| worker.restore_guard.owners.get(id))
.filter(|owner| owner.receipt.is_none())
.map(|owner| owner.operation_id)
})
};
if let Some(operation) = operation {
self.start_restore_continuation(worker_ref.clone(), operation)?;
}
Ok(())
}
pub(super) fn start_restore_continuation(
&self,
worker_ref: WorkerRef,
operation: WorkerLifecycleOperationId,
) -> Result<(), RuntimeError> {
{
let state = self.lock()?;
let Some(backend) = &state.execution_backend else {
return Ok(());
};
if !backend.supports_restore_continuation()
|| backend
.restore_recovery_disposition(operation, &worker_ref)
.is_none()
{
return Ok(());
}
}
let key = (worker_ref.clone(), operation);
let mut owners = self
.restore_continuations
.owners
.lock()
.map_err(|_| RuntimeError::StatePoisoned)?;
if owners.contains_key(&key) {
return Ok(());
}
let owner = Arc::new(Continuation::default());
let retained = owner.clone();
let state = Arc::downgrade(&self.inner);
let operations = Arc::downgrade(&self.worker_operations);
let continuations = Arc::downgrade(&self.restore_continuations);
let task = std::thread::Builder::new()
.name("restore-continuation".into())
.spawn(move || {
run(
state,
operations,
continuations,
retained,
worker_ref,
operation,
);
})
.map_err(|error| {
RuntimeError::InvalidRequest(format!(
"Restore continuation could not start: {error}"
))
})?;
*owner.task.lock().map_err(|_| RuntimeError::StatePoisoned)? = Some(task);
owners.insert(key, owner);
Ok(())
}
fn continue_restore_operation(
&self,
worker_ref: &WorkerRef,
operation: WorkerLifecycleOperationId,
) -> Result<Step, RuntimeError> {
#[cfg(test)]
if self
.restore_continuations
.paused
.load(std::sync::atomic::Ordering::Acquire)
{
return Ok(Step::Contended);
}
let lease = self.worker_operation_lock(worker_ref.worker_id)?;
// Another caller may be waiting for a backend task. Do not queue a join
// of that caller/driver while retaining the lock it needs to complete.
let _guard = match lease.try_lock() {
Ok(guard) => guard,
Err(std::sync::TryLockError::WouldBlock) => return Ok(Step::Contended),
Err(_) => return Err(RuntimeError::StatePoisoned),
};
let stop_is_owned = {
let mut state = self.lock()?;
state.ensure_running()?;
state.refresh_uncertain_worker_write(worker_ref)?;
let worker = state.worker(worker_ref)?;
worker.pending_stop.is_some()
&& worker
.pending_restore
.is_some_and(|pending| pending.operation_id == operation)
};
if stop_is_owned {
// Stop supersedes publication, not ownership of late resources.
// Retain this same continuation until its cleanup is confirmed.
self.reconcile_pending_stop_under_lock(worker_ref)?;
return Ok(Step::Finished);
}
let (backend, pending) = {
let mut state = self.lock()?;
state.ensure_running()?;
state.refresh_uncertain_worker_write(worker_ref)?;
let worker = state.worker(worker_ref)?;
if worker.pending_stop.is_some() {
return Ok(Step::Finished);
}
let owner = worker
.restore_guard
.active_request_id
.as_ref()
.and_then(|id| worker.restore_guard.owners.get(id));
if owner.is_some_and(|owner| owner.operation_id != operation || owner.receipt.is_some())
{
return Ok(Step::Finished);
}
let Some(pending) = worker.pending_restore else {
return Ok(Step::Finished);
};
if pending.operation_id != operation {
return Ok(Step::Finished);
}
if pending.commit == RestoreCommitFact::LegacyUnknown {
return Ok(Step::Retry(
"Saved Restore has no authoritative commit evidence".into(),
));
}
state.ensure_subject_not_reserved(worker_ref, false)?;
if let Some(owner) = owner {
owner.validate_workdir_preparation_authority(worker)?;
}
let Some(backend) = state.execution_backend.clone() else {
return Ok(Step::Finished);
};
(backend, pending)
};
match backend.restore_recovery_disposition(operation, worker_ref) {
None => return Ok(Step::Finished),
Some(RestoreRecoveryDisposition::InFlight) => return Ok(Step::Waiting),
Some(RestoreRecoveryDisposition::Discard)
if pending.commit == RestoreCommitFact::NotCommitted =>
{
match backend.discard_uncommitted_restore(operation, worker_ref) {
WorkerExecutionSpawnResult::RolledBack(result) => {
let receipt = RuntimeWorkerRestoreResult::failed(
WorkerRestoreState::RolledBack,
"worker_restore_rolled_back",
result.message_or_default(),
);
#[cfg(feature = "fs-store")]
self.lock()?
.record_restore_failure(worker_ref, &result, &receipt)?;
#[cfg(not(feature = "fs-store"))]
self.settle_restore_rollback(worker_ref, &receipt)?;
return Ok(Step::Finished);
}
result => {
return Ok(Step::Retry(format!(
"Restore cleanup remains unresolved: {result:?}"
)));
}
}
}
Some(_) => {}
}
let result = self.restore_admitted_worker_under_lock(worker_ref, pending.mode)?;
if backend.restore_recovery_disposition(operation, worker_ref)
== Some(RestoreRecoveryDisposition::InFlight)
{
return Ok(Step::Waiting);
}
if result.state == WorkerRestoreState::ReconciliationRequired {
Ok(Step::Retry(result.message.unwrap_or_else(|| {
"Restore reconciliation remains pending".into()
})))
} else {
Ok(Step::Finished)
}
}
}
fn run(
inner: Weak<Mutex<RuntimeState>>,
operations: Weak<Mutex<BTreeMap<WorkerId, Arc<Mutex<()>>>>>,
continuations: Weak<RestoreContinuations>,
owner: Arc<Continuation>,
worker_ref: WorkerRef,
operation: WorkerLifecycleOperationId,
) {
let mut repeated = 0u32;
let mut last_reason = None;
loop {
let (Some(inner), Some(worker_operations), Some(restore_continuations)) = (
inner.upgrade(),
operations.upgrade(),
continuations.upgrade(),
) else {
break;
};
owner
.schedule
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.parked = false;
let runtime = Runtime {
inner,
worker_operations,
restore_continuations,
};
let step = runtime.continue_restore_operation(&worker_ref, operation);
// No lifecycle lock or strong Runtime/registry reference survives waiting.
drop(runtime);
let mut schedule = owner
.schedule
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if schedule.cancelled {
break;
}
let contended = matches!(&step, Ok(Step::Contended));
let reason = match step {
Ok(Step::Finished) => {
schedule.finished = true;
owner.wake.notify_all();
break;
}
Ok(Step::Waiting) | Ok(Step::Contended) => None,
Ok(Step::Retry(reason)) => Some(reason),
Err(error) => Some(sanitize_worker_create_failure_message(&error.to_string())),
};
if let Some(reason) = &reason {
repeated = repeated.saturating_add(1);
last_reason = Some(reason.clone());
schedule.blocked_reason = Some(reason.clone());
}
// Completed failed attempts can have signaled while this step joined
// them. Do not let our own unchanged failure reset the retry budget.
if reason.is_some() {
schedule.signaled = false;
}
if (!contended && reason.is_none()) || repeated >= 4 {
if repeated >= 4 {
schedule.parked = true;
owner.wake.notify_all();
tracing::warn!(worker_id = %worker_ref.worker_id, operation_id = %operation, reason = ?last_reason,
"Restore continuation parked after bounded attempts; operation remains unresolved");
}
schedule = owner
.wake
.wait_while(schedule, |s| !s.signaled && !s.cancelled)
.unwrap_or_else(std::sync::PoisonError::into_inner);
} else {
let delay = Duration::from_millis(100 * (1u64 << repeated.min(4)));
schedule = owner
.wake
.wait_timeout_while(schedule, delay, |s| !s.signaled && !s.cancelled)
.unwrap_or_else(std::sync::PoisonError::into_inner)
.0;
}
if schedule.cancelled {
break;
}
if schedule.signaled {
schedule.signaled = false;
// A completion wakes this same owner, not a new request. Its next
// observed result, not the signal itself, determines progress.
}
}
}
@@ -55,6 +55,10 @@ impl WorkdirSession for RetainedWorkdirSession {
self.inner.capabilities()
}
fn environment(&self) -> workdir::WorkdirSessionEnvironment {
self.inner.environment()
}
async fn authorize_scope_path(
&self,
request: WorkdirScopeAuthorizationRequest,
+14 -1
View File
@@ -376,6 +376,12 @@ impl StoppedWorkerTemplate {
root: stage.clone(),
_owner_lock: Some(lock),
worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)),
#[cfg(test)]
worker_reconcile_sync_fault: Default::default(),
#[cfg(test)]
worker_reconcile_read_fault: Default::default(),
#[cfg(test)]
worker_reconcile_counts: Default::default(),
};
let result = (|| {
let worker = PersistedWorkerRecord {
@@ -463,8 +469,15 @@ impl StoppedWorkerTemplate {
}
fn validate_shape(&self) -> Result<(), RuntimeError> {
// Version 1 templates carry only settled Session material, never a
// lifecycle operation. Schema 12 -> 13 changes pending decision evidence,
// not that payload. Accept the fixed schema-12 artifact read-only and
// install through the current aggregate owner; never rewrite its bytes.
if self.template_version != TEMPLATE_VERSION
|| self.runtime_schema_version != SCHEMA_VERSION
|| !matches!(
self.runtime_schema_version,
RESTORE_ADMISSION_SCHEMA_VERSION | SCHEMA_VERSION
)
{
return Err(reject("unsupported template schema"));
}
@@ -563,6 +563,8 @@ fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() {
PersistedWorkerLifecycleOperation::Restore(PersistedWorkerRestoreOperation {
operation_id: WorkerLifecycleOperationId::new(),
mode: PersistedWorkerRestoreMode::Explicit,
commit: crate::runtime::RestoreCommitFact::NotCommitted,
source_repair_possible: false,
request: request(),
binding: None,
last_settled_status: WorkerStatus::Stopped,
@@ -632,6 +634,38 @@ fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() {
drop(held);
}
#[test]
fn schema_v12_settled_template_installs_current_aggregate_without_rewriting_session() {
let (_source, _, mut template) = source();
template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION;
let bytes = serde_json::to_vec(&template).unwrap();
let (_target, options) = destination(&template);
template.install(&options, request(), &resolved()).unwrap();
assert_eq!(serde_json::to_vec(&template).unwrap(), bytes);
let worker = options.root.join(WORKERS_DIR).join(worker_id().to_string());
assert_eq!(
fs::read(worker.join("session/session.json")).unwrap(),
template.session_manifest
);
assert_eq!(
fs::read(worker.join(format!("session/segments/{}.jsonl", template.segment_id))).unwrap(),
template.segment_jsonl
);
let aggregate: WorkerAggregateRecord =
read_bounded_json(&worker.join(WORKER_FILE), "read installed fixture").unwrap();
let saved = aggregate.validate(&worker.join(WORKER_FILE)).unwrap();
assert_eq!(saved.status, WorkerStatus::Stopped);
assert!(matches!(
saved.execution_state,
PersistedWorkerExecutionState::Available(_)
));
// Unsupported template schemas do not enter the versioned reader.
template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION - 1;
assert!(template.validate_shape().is_err());
template.runtime_schema_version = SCHEMA_VERSION + 1;
assert!(template.validate_shape().is_err());
}
#[test]
fn fresh_manifest_and_memory_settings_are_required() {
let (_source, _, template) = source();
File diff suppressed because it is too large Load Diff
@@ -66,13 +66,32 @@ impl DeferredWorkdirSession {
let open = self.open.clone();
let session = tokio::task::spawn_blocking(move || open())
.await
.map_err(|_| WorkdirError::Unavailable("Workdir session opener failed".into()))?
.map_err(WorkdirError::Unavailable)?;
.map_err(|_| {
WorkdirError::Unavailable("Workdir session opener failed".into())
.with_failure_reason(WorkdirFailureReason::SessionOpenTaskFailed)
})?
.map_err(|_| {
// The opener's string contract cannot prove a more specific cause.
WorkdirError::Unavailable("Workdir session could not be opened".into())
.with_failure_reason(WorkdirFailureReason::SessionOpenFailed)
})?;
if session.workdir().id() != self.workdir.id() {
session.close().await?;
return Err(WorkdirError::Unavailable(
"Workdir session identity mismatch".into(),
));
if let Err(error) = session.close().await {
// Retain a failed-close owner for explicit close; never use it.
state.closed = true;
state.session = Some(session);
tracing::warn!(
target: "worker_runtime::workdir",
event = "workdir_session_identity_cleanup_failed",
failure_reason = error.failure_reason().map(WorkdirFailureReason::as_str),
denial_reason = error.denial_reason().map(WorkdirDenialReason::as_str),
"Mismatched Workdir session cleanup failed"
);
}
return Err(
WorkdirError::Unavailable("Workdir session identity mismatch".into())
.with_failure_reason(WorkdirFailureReason::SessionIdentityMismatch),
);
}
if let Some(mut source) = session.subscribe_command_events() {
let events = self.events.clone();
@@ -105,6 +124,18 @@ macro_rules! impl_deferred_session {
self.capabilities
}
fn environment(&self) -> WorkdirSessionEnvironment {
let mut environment = self.state.try_lock().ok().and_then(|state| {
state.session.as_ref().map(|session| session.environment())
}).unwrap_or_else(|| {
let mut environment = WorkdirSessionEnvironment::root(self.capabilities);
environment.restrictions.push("Deferred provider: scope is checked on connection and every operation".into());
environment
});
environment.capabilities = self.capabilities;
environment
}
$(
async fn $method(&self, $request: $request_type) -> Result<$result, WorkdirError> {
self.session().await?.$method($request).await
@@ -162,6 +193,21 @@ mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
fn short_tempdir() -> tempfile::TempDir {
let target = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.parent()
.unwrap()
.join("target");
std::fs::create_dir_all(&target).unwrap();
tempfile::Builder::new()
.prefix("d")
.rand_bytes(3)
.tempdir_in(target)
.unwrap()
}
fn read_request() -> ReadRequest {
ReadRequest {
path: WorkdirPath::new("README.md").unwrap(),
@@ -194,9 +240,60 @@ mod tests {
assert_eq!(attempts.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn router_cleanup_of_unopened_session_never_connects_or_follows_reused_alias() {
let attempts = Arc::new(AtomicUsize::new(0));
let observed = attempts.clone();
let session = DeferredWorkdirSession::handle(
"old",
WorkdirSessionCapabilities::READ_ONLY,
move || {
observed.fetch_add(1, Ordering::SeqCst);
Err("offline".into())
},
);
let router = WorkdirSessionRouter::new();
let alias = WorkdirAttachmentAlias::new("w").unwrap();
let routed = router.attach(alias.clone(), session.clone()).unwrap();
assert!(routed.subscribe_command_events().is_some());
assert!(routed.command_snapshot().is_empty());
router.close_all().await.unwrap();
router.close_all().await.unwrap();
assert!(router.aliases().is_empty());
assert!(matches!(
session.read(read_request()).await,
Err(WorkdirError::SessionClosed)
));
assert_eq!(attempts.load(Ordering::SeqCst), 0);
let new_attempts = Arc::new(AtomicUsize::new(0));
let new_observed = new_attempts.clone();
let replacement = DeferredWorkdirSession::handle(
"new",
WorkdirSessionCapabilities::READ_ONLY,
move || {
new_observed.fetch_add(1, Ordering::SeqCst);
Err("offline".into())
},
);
let new_routed = router.attach(alias, replacement).unwrap();
// A retained old connection cannot open the replacement provider.
assert!(routed.read(read_request()).await.is_err());
assert_eq!(new_attempts.load(Ordering::SeqCst), 0);
assert!(matches!(
new_routed.read(read_request()).await,
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
));
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
// Even after a failed open, cleanup must not try connecting again.
router.close_all().await.unwrap();
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
assert_eq!(attempts.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn failed_open_retries_on_use_and_keeps_read_only_capabilities() {
let temp = tempfile::tempdir().unwrap();
let temp = short_tempdir();
std::fs::write(temp.path().join("README.md"), "restored").unwrap();
let root = temp.path().to_path_buf();
let attempts = Arc::new(AtomicUsize::new(0));
@@ -224,10 +321,16 @@ mod tests {
},
);
assert_eq!(attempts.load(Ordering::SeqCst), 0);
let failed = session.read(read_request()).await.unwrap_err();
assert!(matches!(
session.read(read_request()).await,
Err(WorkdirError::Unavailable(_))
failed.classification_source(),
WorkdirError::Unavailable(_)
));
assert_eq!(
failed.failure_reason(),
Some(WorkdirFailureReason::SessionOpenFailed)
);
assert_eq!(attempts.load(Ordering::SeqCst), 1);
assert_eq!(
session.read(read_request()).await.unwrap().bytes,
b"restored"
@@ -253,4 +356,126 @@ mod tests {
);
session.close().await.unwrap();
}
#[tokio::test]
async fn opener_task_and_open_errors_keep_distinct_safe_causes() {
for panic in [false, true] {
let attempts = Arc::new(AtomicUsize::new(0));
let observed = attempts.clone();
let session = DeferredWorkdirSession::handle(
"saved",
WorkdirSessionCapabilities::READ_ONLY,
move || {
observed.fetch_add(1, Ordering::SeqCst);
if panic {
panic!(
"/host/private bearer-secret command-secret proof-secret key-secret"
);
}
Err("/host/private bearer-secret command-secret proof-secret key-secret".into())
},
);
let error = session.read(read_request()).await.unwrap_err();
assert_eq!(attempts.load(Ordering::SeqCst), 1);
let reason = if panic {
WorkdirFailureReason::SessionOpenTaskFailed
} else {
WorkdirFailureReason::SessionOpenFailed
};
assert_eq!(error.failure_reason(), Some(reason));
assert_eq!(reason.stage().as_str(), "session_open");
let transport = workdir::http::WorkdirTransportError::from_workdir_error(&error);
assert_eq!(transport.code.http_status(), 503);
assert!(
!serde_json::to_string(&transport)
.unwrap()
.contains("secret")
);
assert!(!format!("{error:?}").contains("secret"));
session.close().await.unwrap();
assert_eq!(attempts.load(Ordering::SeqCst), 1);
}
}
#[derive(Debug)]
struct MismatchedSession {
workdir: Workdir,
closes: Arc<AtomicUsize>,
fail_close: bool,
}
macro_rules! mismatched_session {
($( $method:ident($request:ident: $request_type:ty) -> $result:ty; )*) => {
#[async_trait]
impl WorkdirSession for MismatchedSession {
fn workdir(&self) -> &Workdir { &self.workdir }
fn capabilities(&self) -> WorkdirSessionCapabilities { WorkdirSessionCapabilities::READ_ONLY }
$(async fn $method(&self, _: $request_type) -> Result<$result, WorkdirError> {
panic!("mismatched session must never be used");
})*
async fn close(&self) -> Result<(), WorkdirError> {
let attempt = self.closes.fetch_add(1, Ordering::SeqCst);
if self.fail_close && attempt == 0 {
Err(WorkdirError::Transport("/host/private key-secret".into())
.with_failure_reason(WorkdirFailureReason::HttpTransfer))
} else { Ok(()) }
}
}
};
}
mismatched_session! {
stat(request: StatRequest) -> StatResult;
read(request: ReadRequest) -> ReadResult;
write(request: WriteRequest) -> WriteResult;
edit(request: EditRequest) -> EditResult;
list(request: ListRequest) -> ListResult;
glob(request: GlobRequest) -> GlobResult;
grep(request: GrepRequest) -> GrepResult;
start_command(request: CommandRequest) -> CommandHandle;
command_status(handle: CommandHandle) -> CommandStatus;
command_output(request: CommandOutputRequest) -> CommandOutput;
cancel_command(handle: CommandHandle) -> ();
}
#[tokio::test]
async fn identity_mismatch_preserves_primary_cause_and_failed_close_owner() {
for fail_close in [false, true] {
let closes = Arc::new(AtomicUsize::new(0));
let raw: WorkdirSessionHandle = Arc::new(MismatchedSession {
workdir: Workdir::new("wrong-secret"),
closes: closes.clone(),
fail_close,
});
let attempts = Arc::new(AtomicUsize::new(0));
let observed = attempts.clone();
let session = DeferredWorkdirSession::handle(
"saved",
WorkdirSessionCapabilities::READ_ONLY,
move || {
observed.fetch_add(1, Ordering::SeqCst);
Ok(raw.clone())
},
);
let error = session.read(read_request()).await.unwrap_err();
assert_eq!(
error.failure_reason(),
Some(WorkdirFailureReason::SessionIdentityMismatch)
);
assert!(!format!("{error:?}").contains("secret"));
assert_eq!(closes.load(Ordering::SeqCst), 1);
if fail_close {
assert!(matches!(
session.read(read_request()).await,
Err(WorkdirError::SessionClosed)
));
assert_eq!(attempts.load(Ordering::SeqCst), 1);
}
session.close().await.unwrap();
assert_eq!(
closes.load(Ordering::SeqCst),
if fail_close { 2 } else { 1 }
);
assert_eq!(attempts.load(Ordering::SeqCst), 1);
}
}
}
@@ -0,0 +1,185 @@
//! Explicit, operation-owned repair of saved Session material. This is not
//! read-only candidate preparation: the canonical Session/attribution is the
//! receipt for replay after a lost response, and effects are reported as such.
use super::*;
impl ProfileRuntimeWorkerFactory {
pub(super) async fn recover_saved_restore_source(
&self,
request: &WorkerExecutionRestoreRequest,
) -> Result<bool, RuntimeRestoreFailure> {
let cause = |cause| RuntimeRestoreFailure::started(RuntimeRestoreStage::Recovery, cause);
let name = Self::runtime_worker_name_for_ref(&request.worker_ref);
let root = self
.worker_aggregate_dir(&request.worker_ref)
.map_err(|_| cause("restore recovery aggregate unavailable"))?;
let metadata_store = WorkerAggregateStore::open_read_only(&root, &name)
.map_err(|_| cause("restore recovery metadata open failed"))?;
let metadata = metadata_store
.read_by_name(&name)
.map_err(|_| cause("restore recovery metadata read failed"))?
.ok_or_else(|| cause("restore recovery metadata missing"))?;
let active = metadata
.active
.as_ref()
.ok_or_else(|| cause("restore recovery Session missing"))?;
let pending = active.segment_id.is_none();
let incomplete = request.request.subjektiv_attached
&& !matches!(
metadata.subjektiv_session_attribution,
Some(session_store::SubjektivSessionAttributionState::Confirmed { .. })
);
if !pending && !incomplete {
return Ok(false);
}
request
.context
.authorize_restore_source_repair(request.operation_id)
.map_err(|_| cause("canonical source repair authority is not durably confirmed"))?;
// The accepted Runtime operation already owns this Session. Its stable
// Session identity and sole committed seed, not a new restore request,
// are used to reconcile an interrupted first-head publication.
let session_store = WorkerSessionStore::new_with_retention(
root.join("session"),
self.session_retention()
.map_err(|_| cause("restore recovery retention unavailable"))?,
)
.map_err(|_| cause("restore recovery Session open failed"))?;
let (manifest, mut loader) = Self::manifest_for_restore(&metadata, &request.request)
.map_err(|_| cause("restore recovery saved manifest invalid"))?;
let workspace = RuntimeWorkspaceBackendRef::from_worker_request(
&request.request,
self.runtime_id.as_deref(),
)
.map_err(cause)?;
#[cfg(feature = "http-server")]
let request_client = request
.request
.workspace_api
.as_ref()
.and_then(|api| self.workspace_request_clients.get(&api.workspace_id));
let context = workspace.worker_context(
#[cfg(feature = "http-server")]
&request.worker_ref,
#[cfg(feature = "http-server")]
request.workspace_scope.as_ref(),
#[cfg(feature = "http-server")]
self.worker_mutation_identity.as_ref(),
#[cfg(feature = "http-server")]
request_client,
#[cfg(feature = "http-server")]
self.embedded_worker_mutation_dispatcher.as_ref(),
#[cfg(feature = "http-server")]
Some(self.prompt_projection_cache.clone()),
);
if pending && context.workspace_id().is_some() {
let bundle = request
.config_bundle
.as_ref()
.ok_or_else(|| cause("pending recovery launch material missing"))?;
let projection = self
.observe_bundle_prompt_projection(
bundle,
request
.request
.workspace_api
.as_ref()
.map(|api| api.workspace_id.as_str()),
)
.map_err(|_| cause("pending recovery Prompt projection invalid"))?
.ok_or_else(|| cause("pending recovery Prompt projection missing"))?;
loader = loader.with_effective_catalog(projection.projection.catalog.clone());
}
self.failed_restore_resources
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(
request.worker_ref.clone(),
FailedRestoreResources {
operation_id: request.operation_id,
sessions: Vec::new(),
candidate_workdir_cleanup: None,
allocation: None,
feature_tasks: None,
},
);
let store = CombinedStore::new(session_store, metadata_store);
let worker_result = Worker::restore_pending_from_worker_metadata_with_context(
&name,
manifest,
store,
loader,
context,
WorkerFilesystemAuthority::None,
)
.await;
let mut worker = worker_result.map_err(|error| {
self.retain_construction_failure(request, &error);
cause(restore_worker_cause(&error))
})?;
let tasks = worker.retain_startup_feature_cleanup();
let sessions = worker.workdir_sessions();
let allocation = worker.restore_allocation_cleanup_handle();
self.failed_restore_resources
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(
request.worker_ref.clone(),
FailedRestoreResources {
operation_id: request.operation_id,
sessions: vec![sessions.clone()],
candidate_workdir_cleanup: None,
allocation: allocation.clone(),
feature_tasks: Some(tasks.clone()),
},
);
let result = async {
if let Some(binding) = request.request.backend_job.as_ref() {
worker
.bind_backend_job(binding.clone())
.map_err(|_| cause("recovery Backend Job binding failed"))?;
}
worker
.finalize_subjektiv_session_attribution(
request.request.subjektiv_attached,
SubjektivSessionAttributionLifecycle::RestoredSession,
)
.map_err(|error| cause(restore_worker_cause(&error)))?;
if pending {
worker::WorkerController::recover_pending_head(&mut worker, &root)
.await
.map_err(|_| cause("pending Session head recovery failed"))?;
}
Ok::<_, RuntimeRestoreFailure>(())
}
.await;
// Never release allocation while the Worker guard is attached. Retained
// handles in the same operation remain available on every cleanup error.
drop(worker);
let cleanup = async {
tasks
.shutdown()
.await
.map_err(|_| cause("recovery Feature settlement failed"))?;
sessions
.close_all()
.await
.map_err(|_| cause("recovery Workdir close failed"))?;
if let Some(allocation) = allocation {
allocation
.release()
.map_err(|_| cause("recovery allocation release failed"))?;
}
Ok::<_, RuntimeRestoreFailure>(())
}
.await;
result?;
cleanup?;
self.failed_restore_resources
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&request.worker_ref);
Ok(true)
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,927 @@
// Included by the production Profile publication suite. Each fault is at a
// distinct partial-publication boundary; no factory result is substituted.
fn publication_gate<T>() -> (
mpsc::Sender<()>,
mpsc::Receiver<T>,
crate::execution::RestoreTestGate<T>,
) {
let (release, receiver) = mpsc::channel();
let (entered, evidence) = mpsc::channel();
(
release,
evidence,
crate::execution::RestoreTestGate {
entered,
release: receiver,
},
)
}
fn assert_single_publication(fixture: &Fixture, expected: usize) {
let faults = &fixture.backend.factory.restore_faults;
assert_eq!(
faults.executions.load(Ordering::SeqCst),
expected,
"construction reran"
);
assert_eq!(
faults.effects_calls.load(Ordering::SeqCst),
expected,
"restore effects reran"
);
assert_eq!(
faults.publication_attempts.load(Ordering::SeqCst),
expected,
"Controller start reran"
);
fixture.assert_quiet();
}
fn assert_real_attach(
fixture: &Fixture,
handle: &WorkerHandle,
) -> crate::execution::WorkerProtocolTransport {
let worker_ref = &fixture.request.worker_ref;
assert!(matches!(
fixture
.backend
.worker_session(crate::execution::WorkerSessionObservationRequest {
worker_ref: worker_ref.clone(),
}),
runtime_api::WorkerSessionAvailability::LiveProtocol
));
let transport = fixture
.backend
.clone()
.attach_worker_protocol(worker_ref)
.unwrap();
transport.validate().unwrap();
let paired = subscribe_worker_protocol_session(handle);
assert_eq!(
serde_json::to_value(&transport.snapshot).unwrap(),
serde_json::to_value(paired.snapshot_event).unwrap()
);
assert!(
matches!(&transport.snapshot, Event::Snapshot { session, greeting, state, in_flight, .. }
if session == &session_store::public_snapshot::project_current_session_snapshot(&fixture.entries())
&& !greeting.tools.is_empty() && state.state == protocol::WorkerState::Idle && in_flight.is_empty())
);
transport
}
fn observe_publication_markers(
fixture: &Fixture,
observations: tokio::sync::broadcast::Receiver<crate::observation::WorkerObservationEvent>,
markers: &'static [&'static str],
) -> tokio::sync::broadcast::Receiver<crate::observation::WorkerObservationEvent> {
fixture
.backend
.run_on_adapter_runtime(async move {
let mut observations = observations;
let mut counts = vec![0; markers.len()];
let mut snapshots = 0;
tokio::time::timeout(WAIT, async {
loop {
let observed = observations.recv().await.unwrap();
if matches!(observed.payload, Event::Snapshot { .. }) {
snapshots += 1;
}
for (index, marker) in markers.iter().enumerate() {
if is_marker(&observed.payload, marker) {
counts[index] += 1;
}
}
if counts.last() == Some(&1) {
break;
}
}
})
.await
.map_err(|_| format!("lost original subscription markers: {counts:?}"))?;
assert_eq!(
counts,
vec![1; markers.len()],
"lost or duplicated prefix on original receiver"
);
assert_eq!(snapshots, 1, "initial full snapshot was lost or replayed");
while let Ok(event) = observations.try_recv() {
assert!(
!markers
.iter()
.any(|marker| is_marker(&event.payload, marker)),
"duplicate after tail fence"
);
}
Ok(observations)
})
.unwrap()
}
fn partial_registration_recovers_same_controller(hub: bool) {
let mut fixture = Fixture::no_model();
Arc::get_mut(&mut fixture.backend)
.unwrap()
.spawn_restore_timeout = Duration::from_millis(250);
let runtime = stopped_runtime(&fixture);
let pause = runtime.test_pause_restore_continuation();
let worker_ref = fixture.request.worker_ref.clone();
let scope = fixture.request.workspace_scope.as_ref().unwrap();
let request = runtime.test_restore_request(&worker_ref);
let observations = runtime.subscribe_worker_observation().unwrap();
let (started_release, started_entered) = activation_gate(&fixture, true);
let first = runtime
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
.unwrap();
assert_eq!(
first.state,
server_api::WorkerRestoreState::ReconciliationRequired
);
let started = started_entered.recv_timeout(WAIT).unwrap();
let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started);
let old = started.handle.as_ref().unwrap().clone();
let canonical = fixture.canonical_files();
alert_marker(&old, "registration-original-paired-prefix");
let (release, entered, fault) = publication_gate();
if hub {
*fixture
.backend
.factory
.observation_hub
.registration_gate
.lock()
.unwrap() = Some(RestoreHubRegistrationGate {
gate: fault,
fail: true,
});
} else {
*fixture.backend.bridge_fault.lock().unwrap() = Some(fault);
}
started_release.release();
drop(pause);
let boundary_handle = entered.recv_timeout(WAIT).unwrap();
assert!(boundary_handle.same_controller(&old));
let durable = durable_operation(&runtime, &worker_ref).unwrap();
assert_eq!(durable.operation_id, started.operation_id);
assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed);
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
if hub {
let (_, shared, _) = fixture
.backend
.factory
.observation_hub
.get(&worker_ref)
.unwrap();
assert!(
Arc::ptr_eq(&shared, &old.shared_state),
"real hub insert preceded fault"
);
} else {
assert!(
fixture
.backend
.factory
.observation_hub
.get(&worker_ref)
.is_none()
);
}
let (operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref);
assert_eq!(operation, started.operation_id);
assert!(!uncertain);
assert!(
context
.test_buffered_events()
.iter()
.any(|event| matches!(event, Event::Snapshot { .. }))
);
alert_marker(&old, "registration-during-fault");
let pause = runtime.test_pause_restore_continuation();
release.send(()).unwrap();
// This lock also observes the end of the actual registration attempt, not
// merely the test's entered notification. Its retained scope owns cleanup.
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
assert!(old.same_controller(&execution.handle));
assert_eq!(execution.restore_operation_id, Some(operation));
assert_eq!(
execution.tasks.tasks.lock().unwrap().len(),
if hub { 2 } else { 1 }
);
let bridge = execution.tasks.protocol_bridge.lock().unwrap();
if hub {
assert!(bridge.is_none());
} else {
let bridge = bridge.as_ref().unwrap();
assert!(
!bridge.snapshot_pending,
"snapshot already registered, do not replay it"
);
let streams = bridge.streams.lock().unwrap();
assert!(
streams.is_some(),
"original paired streams survived failed task admission"
);
assert_eq!(
serde_json::to_value(&streams.as_ref().unwrap().snapshot_event).unwrap(),
serde_json::to_value(&started.snapshot).unwrap()
);
}
drop(bridge);
assert!(
!old.protocol_is_closed(),
"publication failure must not clean a committed Controller"
);
assert_single_publication(&fixture, 1);
drop(pause);
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
assert!(durable_operation(&runtime, &worker_ref).is_none());
let owner = restore_owner(&runtime, &worker_ref, &request.request_id);
assert_eq!(
owner["operation_id"],
serde_json::to_value(operation).unwrap()
);
assert_eq!(owner["receipt"]["state"], "accepted");
assert!(
fixture
.backend
.pending_restore(&worker_ref)
.unwrap()
.is_none()
);
assert!(!pending.task_in_flight.load(Ordering::Acquire));
assert!(pending.activated.lock().unwrap().is_none());
assert!(pending.result.lock().unwrap().is_none());
assert!(pending.task.try_lock().unwrap().is_none());
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
assert!(old.same_controller(&fixture.backend.workers.lock().unwrap()[&worker_ref].handle));
let transport = assert_real_attach(&fixture, &old);
assert_eq!(fixture.canonical_files(), canonical);
assert_single_publication(&fixture, 1);
alert_marker(&old, "registration-tail-fence");
let observations = observe_publication_markers(
&fixture,
observations,
&[
"registration-original-paired-prefix",
"registration-during-fault",
"registration-tail-fence",
],
);
assert_eq!(
runtime
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_single_publication(&fixture, 1);
// Retain the original transport/handle and scope through Stop/replacement.
runtime
.stop_worker_scoped(scope, &worker_ref, None)
.unwrap();
assert!(old.protocol_is_closed());
assert!(transport.validate().is_err());
let replacement_request = runtime.test_restore_request(&worker_ref);
assert_eq!(
runtime
.restore_worker_operation_scoped(scope, &worker_ref, replacement_request.clone())
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
let replacement = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
assert!(!old.same_controller(&replacement.handle));
assert_ne!(replacement.restore_operation_id, Some(operation));
assert!(
transport
.dispatch(Method::Pause {
command: protocol::WorkerCommandEnvelope::new(801)
})
.is_err()
);
assert!(
transport
.dispatch(Method::Shutdown {
command: protocol::WorkerCommandEnvelope::new(802)
})
.is_err()
);
for method in [
Method::Pause {
command: protocol::WorkerCommandEnvelope::new(803),
},
Method::Shutdown {
command: protocol::WorkerCommandEnvelope::new(804),
},
] {
let old_command = old.clone();
assert!(
fixture
.backend
.run_on_adapter_runtime(async move {
old_command
.send(method)
.await
.map_err(|error| error.to_string())
})
.is_err(),
"closed original Controller command must not route to replacement"
);
}
fixture
.backend
.factory
.deactivate_stopped_controller(&worker_ref, &old)
.unwrap();
let (_, shared, _) = fixture
.backend
.factory
.observation_hub
.get(&worker_ref)
.unwrap();
assert!(Arc::ptr_eq(&shared, &replacement.handle.shared_state));
alert_marker(&old, "registration-stopped-event-must-not-leak");
alert_marker(&replacement.handle, "registration-replacement-fence");
fixture
.backend
.run_on_adapter_runtime(async move {
let mut observations = observations;
tokio::time::timeout(WAIT, async {
loop {
let event = observations.recv().await.unwrap();
assert!(!is_marker(
&event.payload,
"registration-stopped-event-must-not-leak"
));
if is_marker(&event.payload, "registration-replacement-fence") {
break;
}
}
})
.await
.map_err(|_| "replacement observation fence missing".to_string())?;
drop(transport);
Ok(())
})
.unwrap();
assert_single_publication(&fixture, 2);
assert_eq!(
runtime
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
.unwrap()
.state,
server_api::WorkerRestoreState::Accepted
);
assert_eq!(restore_owner_count(&runtime, &worker_ref), 2);
let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
assert!(current.handle.same_controller(&replacement.handle));
assert_eq!(
current.restore_operation_id,
replacement.restore_operation_id
);
assert_eq!(
replacement.handle.shared_state.snapshot().state,
protocol::WorkerState::Idle
);
drop(current);
drop(execution);
drop(replacement);
drop(pending);
runtime
.stop_worker_scoped(scope, &worker_ref, None)
.unwrap();
fixture.assert_released();
}
fn restore_owner_count(runtime: &EmbeddedRuntime, worker_ref: &WorkerRef) -> usize {
let saved = runtime
.test_runtime_store()
.read_worker_record(worker_ref)
.unwrap();
serde_json::to_value(saved.restore_guard).unwrap()["owners"]
.as_object()
.unwrap()
.len()
}
#[test]
#[serial_test::serial(worker_allocation)]
fn bridge_partial_snapshot_registration_retains_original_streams_and_autonomously_recovers() {
partial_registration_recovers_same_controller(false);
}
#[test]
#[serial_test::serial(worker_allocation)]
fn hub_partial_insert_registration_retains_original_controller_and_autonomously_recovers() {
partial_registration_recovers_same_controller(true);
}
#[test]
#[serial_test::serial(worker_allocation)]
fn context_partial_flush_retains_only_undelivered_queue_and_recovers_without_repost() {
let mut fixture = Fixture::no_model();
Arc::get_mut(&mut fixture.backend)
.unwrap()
.spawn_restore_timeout = Duration::from_millis(250);
let runtime = stopped_runtime(&fixture);
let pause = runtime.test_pause_restore_continuation();
let worker_ref = fixture.request.worker_ref.clone();
let scope = fixture.request.workspace_scope.as_ref().unwrap();
let request = runtime.test_restore_request(&worker_ref);
let observations = runtime.subscribe_worker_observation().unwrap();
let (started_release, started_entered) = activation_gate(&fixture, true);
assert_eq!(
runtime
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
.unwrap()
.state,
server_api::WorkerRestoreState::ReconciliationRequired
);
let started = started_entered.recv_timeout(WAIT).unwrap();
let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started);
let old = started.handle.unwrap();
let (operation, context, _) = runtime.test_pending_restore_context(&worker_ref);
// Stop after the actual bridge and hub registration, without failing that
// boundary, so the flush begins with its real original full snapshot.
let (hub_release, hub_entered, hub_gate) = publication_gate();
*fixture
.backend
.factory
.observation_hub
.registration_gate
.lock()
.unwrap() = Some(RestoreHubRegistrationGate {
gate: hub_gate,
fail: false,
});
started_release.release();
drop(pause);
assert!(
hub_entered
.recv_timeout(WAIT)
.unwrap()
.same_controller(&old)
);
let buffered_prefix = context.test_buffered_events();
assert!(matches!(
buffered_prefix.first(),
Some(Event::Snapshot { .. })
));
let prefix_length = buffered_prefix.len();
let marker = |text: &str| Event::Error {
code: protocol::ErrorCode::Internal,
message: text.into(),
};
context
.publish_protocol_event(marker("flush-delivered"))
.unwrap();
context
.publish_protocol_event(marker("flush-failed"))
.unwrap();
context
.publish_protocol_event(marker("flush-pending"))
.unwrap();
let (release, entered, gate) = publication_gate();
let (restored, restored_queue) = mpsc::channel();
context.test_fail_flush(crate::execution::RestoreFlushFault {
remaining_successes: prefix_length + 1,
gate,
restored,
});
drop(hub_release);
assert!(
matches!(entered.recv_timeout(WAIT).unwrap(), Event::Error { message, .. } if message == "flush-failed")
);
let partial_execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
assert!(old.same_controller(&partial_execution.handle));
assert_eq!(partial_execution.restore_operation_id, Some(operation));
assert_eq!(partial_execution.tasks.tasks.lock().unwrap().len(), 2);
let (_, shared, _) = fixture
.backend
.factory
.observation_hub
.get(&worker_ref)
.unwrap();
assert!(Arc::ptr_eq(&shared, &old.shared_state));
let pause = runtime.test_pause_restore_continuation();
context
.publish_protocol_event(marker("flush-arrived-during-error"))
.unwrap();
release.send(()).unwrap();
let restored = restored_queue.recv_timeout(WAIT).unwrap();
let undelivered: Vec<_> = restored
.iter()
.filter_map(|event| match event {
Event::Error { message, .. } if message.starts_with("flush-") => Some(message.as_str()),
_ => None,
})
.collect();
assert_eq!(
undelivered,
[
"flush-failed",
"flush-pending",
"flush-arrived-during-error"
]
);
assert!(
!restored
.iter()
.any(|event| matches!(event, Event::Snapshot { .. })),
"delivered snapshot must not rejoin the failed suffix"
);
assert_eq!(
serde_json::to_value(context.test_buffered_events()).unwrap(),
serde_json::to_value(&restored).unwrap()
);
assert_eq!(
durable_operation(&runtime, &worker_ref)
.unwrap()
.operation_id,
operation
);
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
// The next terminal gate observes successful retry only after the same
// queued owner has repaired its failed callback and remaining suffix.
let (terminal_release, terminal_entered, gate) = publication_gate();
runtime.test_terminal_receipt_gate(gate);
drop(pause);
assert_eq!(terminal_entered.recv_timeout(WAIT).unwrap(), operation);
assert!(context.test_buffered_events().is_empty());
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
assert_eq!(
durable_operation(&runtime, &worker_ref).unwrap().commit,
crate::runtime::RestoreCommitFact::Committed
);
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
assert!(old.same_controller(&execution.handle));
assert_eq!(execution.restore_operation_id, Some(operation));
drop(terminal_release); // RAII disconnect, not a second request.
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
assert_eq!(
restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"],
"accepted"
);
assert_real_attach(&fixture, &old);
fixture
.backend
.run_on_adapter_runtime(async move {
let mut observations = observations;
let mut messages = Vec::new();
let mut snapshots = 0;
while let Ok(event) = observations.try_recv() {
match event.payload {
Event::Error { message, .. } if message.starts_with("flush-") => {
messages.push(message)
}
Event::Snapshot { .. } => snapshots += 1,
_ => {}
}
}
assert_eq!(
messages,
[
"flush-delivered",
"flush-failed",
"flush-pending",
"flush-arrived-during-error"
]
);
assert_eq!(snapshots, 1);
Ok(())
})
.unwrap();
assert_single_publication(&fixture, 1);
drop(partial_execution);
drop(execution);
drop(pending);
runtime
.stop_worker_scoped(scope, &worker_ref, None)
.unwrap();
fixture.assert_released();
}
#[derive(Clone, Copy, Debug)]
enum TerminalFault {
BeforeRename,
AfterRename,
SyncUnknown,
ReadUnknown,
}
fn published_terminal_receipt_fault(fault: TerminalFault) {
let mut fixture = Fixture::no_model();
// Reuse the existing ordinary non-network model boundary, not the product
// Codex profile. All fault/convergence assertions precede the sole input.
let script = MockClient::sequential(vec![MockResponse::Complete(simple_text_events())]);
let injected = script.clone();
let factory = fixture
.backend
.factory
.as_ref()
.clone()
.with_spawn_model_client_factory(move || Box::new(injected.clone()));
fixture.backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap());
let runtime = Arc::new(stopped_runtime(&fixture));
let pause = runtime.test_pause_restore_continuation();
let worker_ref = fixture.request.worker_ref.clone();
let scope = fixture.request.workspace_scope.clone().unwrap();
let request = runtime.test_restore_request(&worker_ref);
let observations = runtime.subscribe_worker_observation().unwrap();
let mut catalog = runtime
.subscribe_event_selector_scoped(
&scope,
protocol::subscription::EventSubscriptionSelector::RuntimeWorkers,
)
.unwrap();
let (release, entered, gate) = publication_gate();
runtime.test_terminal_receipt_gate(gate);
let restoring = {
let runtime = runtime.clone();
let worker_ref = worker_ref.clone();
let scope = scope.clone();
let request = request.clone();
std::thread::spawn(move || {
runtime.restore_worker_operation_scoped(&scope, &worker_ref, request)
})
};
let operation = entered.recv_timeout(WAIT).unwrap();
// Restore legitimately repairs derived metadata before publication. Receipt
// failures must not rerun those effects; compare from this actual boundary.
let canonical = fixture.canonical_files();
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
let old = execution.handle.clone();
assert_eq!(execution.restore_operation_id, Some(operation));
assert!(
fixture
.backend
.pending_restore(&worker_ref)
.unwrap()
.is_none()
);
assert_eq!(execution.tasks.tasks.lock().unwrap().len(), 2);
assert!(execution.tasks.protocol_bridge.lock().unwrap().is_none());
assert_eq!(
durable_operation(&runtime, &worker_ref)
.unwrap()
.operation_id,
operation
);
assert_eq!(
durable_operation(&runtime, &worker_ref).unwrap().commit,
crate::runtime::RestoreCommitFact::Committed
);
let owner = restore_owner(&runtime, &worker_ref, &request.request_id);
assert!(owner["receipt"].is_null());
let (same_operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref);
assert_eq!(same_operation, operation);
assert!(!uncertain);
assert!(context.test_buffered_events().is_empty());
let transport = assert_real_attach(&fixture, &old);
assert_single_publication(&fixture, 1);
alert_marker(&old, "terminal-before-receipt");
let store = runtime.test_runtime_store();
let reconcile_before = store.test_worker_reconcile_counts();
store.fail_next_worker_write(if matches!(fault, TerminalFault::BeforeRename) {
AtomicWriteFault::BeforeRename
} else {
AtomicWriteFault::AfterRename
});
match fault {
TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(),
TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(),
_ => {}
}
release.send(()).unwrap();
let first = restoring.join().unwrap().unwrap();
let reconcile_after = store.test_worker_reconcile_counts();
let reconciliation = (
reconcile_after.0 - reconcile_before.0,
reconcile_after.1 - reconcile_before.1,
);
assert_eq!(
reconciliation,
match fault {
TerminalFault::BeforeRename => (0, 0),
TerminalFault::AfterRename => (1, 4), // canonical file plus Worker/workers/root directories
TerminalFault::SyncUnknown => (1, 1),
TerminalFault::ReadUnknown => (1, 0),
},
"{fault:?}: actual canonical read/sync attempts"
);
assert_eq!(
first.state,
if matches!(fault, TerminalFault::AfterRename) {
server_api::WorkerRestoreState::Accepted
} else {
server_api::WorkerRestoreState::ReconciliationRequired
},
"{fault:?}: {first:?}"
);
assert_eq!(fixture.canonical_files(), canonical);
assert!(!old.protocol_is_closed());
transport.validate().unwrap();
assert_single_publication(&fixture, 1);
let disk_owner = restore_owner(&runtime, &worker_ref, &request.request_id);
assert_eq!(disk_owner["operation_id"], owner["operation_id"]);
match fault {
TerminalFault::BeforeRename => {
assert!(disk_owner["receipt"].is_null());
assert!(!runtime.test_pending_restore_context(&worker_ref).2);
}
TerminalFault::AfterRename => {
assert_eq!(disk_owner["receipt"]["state"], "accepted");
assert!(durable_operation(&runtime, &worker_ref).is_none());
}
TerminalFault::SyncUnknown | TerminalFault::ReadUnknown => {
assert_eq!(
disk_owner["receipt"]["state"], "accepted",
"visible rename alone is not durable proof"
);
let (same_operation, _, fenced) = runtime.test_pending_restore_context(&worker_ref);
assert_eq!(same_operation, operation);
assert!(
fenced,
"unknown canonical outcome must retain the memory owner/write fence"
);
// Keep durability unproved for one competing Stop as well: it must
// fail before any Shutdown, never manufacture a Stopped judgment.
match fault {
TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(),
TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(),
_ => unreachable!(),
}
assert!(matches!(
runtime.stop_worker_scoped(&scope, &worker_ref, None),
Err(crate::error::RuntimeError::StoreCommitOutcomeUnknown { .. })
));
assert!(!old.protocol_is_closed());
assert!(!execution.shutdown_requested.load(Ordering::Acquire));
assert!(runtime.test_pending_restore_context(&worker_ref).2);
assert_single_publication(&fixture, 1);
}
}
// Recovery is autonomous after the failed write / confirmed unknown fence.
// No POST, fresh request, or input is needed to make it converge.
drop(pause);
if !matches!(fault, TerminalFault::AfterRename) {
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
}
assert!(durable_operation(&runtime, &worker_ref).is_none());
assert_eq!(
restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"],
"accepted"
);
let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
assert!(old.same_controller(&current.handle));
assert_eq!(current.restore_operation_id, Some(operation));
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
assert_real_attach(&fixture, &old);
assert_single_publication(&fixture, 1);
// A canonical terminal reread may bypass the final redundant upsert. The
// original subscriber must still have the current state and observation token.
let fresh_catalog = runtime
.subscribe_event_selector_scoped(
&scope,
protocol::subscription::EventSubscriptionSelector::RuntimeWorkers,
)
.unwrap();
let protocol::subscription::SubscriptionSnapshot::Workers { workers } =
fresh_catalog.snapshot()
else {
panic!("Runtime catalog must return Workers");
};
assert_eq!(workers.len(), 1);
let mut last_applied = None;
use futures::FutureExt;
while let Some(update) = catalog.recv().now_or_never() {
match update.unwrap().payload {
protocol::subscription::SubscriptionEventPayload::WorkerUpserted { worker } => {
last_applied = Some(worker);
}
payload => panic!("unexpected catalog update: {payload:?}"),
}
}
assert!(
serde_json::to_value(last_applied.expect("restore must publish its catalog state"))
.unwrap()
== serde_json::to_value(&workers[0]).unwrap(),
"original catalog subscriber diverged after {fault:?}"
);
drop(fresh_catalog);
drop(catalog);
alert_marker(&old, "terminal-after-recovery");
let _observations = observe_publication_markers(
&fixture,
observations,
&["terminal-before-receipt", "terminal-after-recovery"],
);
let accepted = runtime
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
.unwrap();
assert_eq!(accepted.state, server_api::WorkerRestoreState::Accepted);
assert_eq!(
runtime
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
.unwrap(),
accepted
);
assert_eq!(restore_owner_count(&runtime, &worker_ref), 1);
let mut stale = request;
stale.request_id = "terminal-fresh-stale-intent".into();
assert!(matches!(
runtime.restore_worker_operation_scoped(&scope, &worker_ref, stale),
Err(crate::error::RuntimeError::RestoreObservationConflict { .. })
));
assert_single_publication(&fixture, 1);
assert_eq!(
script.call_count.load(Ordering::SeqCst),
0,
"receipt recovery cannot trigger input or a model call"
);
// This transport was paired before the failed receipt write. Exercise its
// normal log-entry receiver after autonomous settlement, not a new attach.
transport
.dispatch(Method::submit_text(
"terminal-ordinary-input",
"terminal-ordinary-input",
))
.unwrap();
fixture
.backend
.run_on_adapter_runtime(async move {
let mut transport = transport;
let (mut admitted, mut committed, mut text, mut ended) = (false, false, false, false);
tokio::time::timeout(WAIT, async {
while !(admitted && committed && text && ended) {
match transport.events.recv().await.unwrap() {
Event::SubmissionAccepted {
submission_request_id,
..
} if submission_request_id == "terminal-ordinary-input" => admitted = true,
Event::UserMessage { segments, .. } => {
assert_eq!(
segments,
vec![protocol::Segment::text("terminal-ordinary-input")]
);
assert!(!committed, "paired log receiver duplicated input");
committed = true;
}
Event::TextDone { text: output } => {
assert_eq!(output, "hello from worker");
text = true;
}
Event::RunEnd { .. } => ended = true,
Event::SubmissionRejected { .. } | Event::Error { .. } => {
panic!("ordinary input failed after receipt recovery")
}
_ => {}
}
}
})
.await
.map_err(|_| {
"ordinary input failed to settle on original paired transport".to_string()
})?;
drop(transport);
Ok(())
})
.unwrap();
assert_eq!(script.call_count.load(Ordering::SeqCst), 1);
assert_eq!(fixture.entries().iter().filter(|entry| matches!(entry,
LogEntry::AnnotatedUserInput { segments, .. } if segments == &vec![protocol::Segment::text("terminal-ordinary-input")]
)).count(), 1);
assert_single_publication(&fixture, 1);
drop(execution);
drop(current);
runtime
.stop_worker_scoped(&scope, &worker_ref, None)
.unwrap();
assert!(old.protocol_is_closed());
fixture.assert_released();
}
#[test]
#[serial_test::serial(worker_allocation)]
fn published_controller_terminal_receipt_before_rename_autonomously_settles_without_reconstruction()
{
published_terminal_receipt_fault(TerminalFault::BeforeRename);
}
#[test]
#[serial_test::serial(worker_allocation)]
fn published_controller_terminal_receipt_after_rename_is_canonically_reread_and_synced() {
published_terminal_receipt_fault(TerminalFault::AfterRename);
}
#[test]
#[serial_test::serial(worker_allocation)]
fn published_controller_terminal_receipt_sync_unknown_retains_owner_and_write_fence() {
published_terminal_receipt_fault(TerminalFault::SyncUnknown);
}
#[test]
#[serial_test::serial(worker_allocation)]
fn published_controller_terminal_receipt_read_unknown_retains_owner_and_write_fence() {
published_terminal_receipt_fault(TerminalFault::ReadUnknown);
}
+48 -7
View File
@@ -1,23 +1,32 @@
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
#[cfg(feature = "http-server")]
use std::time::Duration;
use std::time::{SystemTime, UNIX_EPOCH};
#[cfg(feature = "http-server")]
use crate::workspace_request::{
RuntimeWorkspaceRequest, RuntimeWorkspaceRequestClient, RuntimeWorkspaceRequestError,
};
use worker::WorkspaceResponse;
#[cfg(feature = "http-server")]
use worker::{
WorkspaceBinaryRequest, WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError,
WorkspacePromptCatalogResolution, WorkspacePromptProjection, WorkspaceRequest,
WorkspaceRequestMethod, WorkspaceResponse, WorkspaceServerOperation,
WorkspaceRequestMethod, WorkspaceServerOperation,
};
use crate::auth::{
RuntimeAuthError, RuntimeIdentityMaterial, RuntimeWorkerMutationSourceSigner,
WORKER_REMOVE_PERMISSION, WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION,
WorkerMutationActorKind, WorkerMutationOperation, WorkerMutationSourceClaims, new_token_id,
WORKER_REMOVE_PERMISSION, WorkerMutationActorKind, WorkerMutationOperation,
WorkerMutationSourceClaims, new_token_id,
};
#[cfg(feature = "http-server")]
use crate::auth::{WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION};
use crate::runtime::RuntimeWorkspaceScope;
#[cfg(feature = "http-server")]
use crate::worker_backend::WorkspacePromptProjectionCache;
#[cfg(feature = "http-server")]
fn ticket_assignment_role_segment(role: server_api::TicketAssignmentRole) -> &'static str {
match role {
server_api::TicketAssignmentRole::Orchestrator => "orchestrator",
@@ -144,6 +153,7 @@ pub trait EmbeddedWorkerMutationDispatcher: Send + Sync {
#[derive(Clone)]
enum RuntimeWorkerMutationTransport {
#[cfg(feature = "http-server")]
Remote {
request_client: RuntimeWorkspaceRequestClient,
},
@@ -161,6 +171,7 @@ pub struct RuntimeWorkerMutationForwarder {
}
impl RuntimeWorkerMutationForwarder {
#[cfg(feature = "http-server")]
pub fn remote(
identity: &RuntimeIdentityMaterial,
scope: RuntimeWorkspaceScope,
@@ -206,6 +217,7 @@ impl RuntimeWorkerMutationForwarder {
target_worker_id,
)?;
match (&self.transport, proof) {
#[cfg(feature = "http-server")]
(
RuntimeWorkerMutationTransport::Remote { request_client },
RuntimeOwnedWorkerMutationProof::Remote(token),
@@ -231,6 +243,7 @@ impl RuntimeWorkerMutationForwarder {
}
}
#[cfg(feature = "http-server")]
struct RemoteWorkerRemoveHttpRequest {
request_client: RuntimeWorkspaceRequestClient,
source_worker_id: String,
@@ -240,6 +253,7 @@ struct RemoteWorkerRemoveHttpRequest {
reason: String,
}
#[cfg(feature = "http-server")]
fn execute_remote_worker_remove_http(
request: RemoteWorkerRemoveHttpRequest,
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
@@ -263,6 +277,7 @@ fn execute_remote_worker_remove_http(
execute_remote_worker_remove_http_blocking(request)
}
#[cfg(feature = "http-server")]
fn execute_remote_worker_remove_http_blocking(
request: RemoteWorkerRemoveHttpRequest,
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
@@ -315,6 +330,7 @@ fn execute_remote_worker_remove_http_blocking(
}
}
#[cfg(feature = "http-server")]
#[derive(Clone)]
pub struct RuntimeOwnedWorkspaceClient {
workspace_id: String,
@@ -325,6 +341,7 @@ pub struct RuntimeOwnedWorkspaceClient {
prompt_projection_cache: Option<Arc<WorkspacePromptProjectionCache>>,
}
#[cfg(feature = "http-server")]
impl RuntimeOwnedWorkspaceClient {
pub fn new(
workspace_id: impl Into<String>,
@@ -495,6 +512,7 @@ impl RuntimeOwnedWorkspaceClient {
}
}
#[cfg(feature = "http-server")]
impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
@@ -510,6 +528,7 @@ impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
}
}
#[cfg(feature = "http-server")]
fn generated_workspace_response<T: serde::Serialize>(
response: Result<T, server_api::client_support::ClientError<server_api::RepositoryApiError>>,
) -> Result<WorkspaceResponse, WorkspaceClientError> {
@@ -530,6 +549,7 @@ fn generated_workspace_response<T: serde::Serialize>(
}
}
#[cfg(feature = "http-server")]
impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
fn workspace_id(&self) -> Option<&str> {
Some(self.request_client.workspace_id())
@@ -929,12 +949,17 @@ fn unix_now_seconds() -> u64 {
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "http-server")]
use crate::auth::{
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WorkerMutationSourceExpectation,
decode_runtime_request_source_claims, decode_worker_mutation_source_claims,
request_body_digest, verify_worker_mutation_source_proof,
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, decode_runtime_request_source_claims,
request_body_digest,
};
use crate::auth::{
WorkerMutationSourceExpectation, decode_worker_mutation_source_claims,
verify_worker_mutation_source_proof,
};
#[cfg(feature = "http-server")]
fn binary_loopback(response: Vec<u8>) -> (String, std::thread::JoinHandle<(String, Vec<u8>)>) {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
@@ -972,6 +997,7 @@ mod tests {
(base_url, server)
}
#[cfg(feature = "http-server")]
#[test]
fn generated_ticket_assignment_transport_signs_exact_authoring_and_bounded_read_subjects() {
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
@@ -1104,6 +1130,7 @@ mod tests {
}
}
#[cfg(feature = "http-server")]
#[test]
fn binary_roundtrip_signs_exact_query_method_worker_and_raw_body() {
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
@@ -1194,6 +1221,7 @@ mod tests {
}
}
#[cfg(feature = "http-server")]
#[test]
fn binary_responses_enforce_declared_and_streamed_limits_with_fixed_errors() {
for response in [
@@ -1213,6 +1241,7 @@ mod tests {
}
}
#[cfg(feature = "http-server")]
#[test]
fn binary_redirect_returns_status_without_forwarding_authentication() {
let (base_url, server) = binary_loopback(b"HTTP/1.1 307 Temporary Redirect\r\nLocation: http://127.0.0.1:1/private\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec());
@@ -1231,6 +1260,7 @@ mod tests {
assert!(result.body.is_empty());
}
#[cfg(feature = "http-server")]
#[test]
fn binary_invalid_or_normalized_targets_fail_with_fixed_errors_before_http() {
let client = RuntimeOwnedWorkspaceClient::new(
@@ -1277,6 +1307,7 @@ mod tests {
);
}
#[cfg(feature = "http-server")]
#[test]
fn current_prompt_projection_uses_the_shared_runtime_cache_without_http() {
let cache = Arc::new(WorkspacePromptProjectionCache::default());
@@ -1312,6 +1343,7 @@ mod tests {
assert!(Arc::ptr_eq(&projection.catalog, &second.catalog));
}
#[cfg(feature = "http-server")]
#[test]
fn prompt_projection_cache_miss_fetches_content_identity() {
use std::io::{Read, Write};
@@ -1356,6 +1388,7 @@ mod tests {
assert_eq!(fetched.projection.source_digest, "source-3");
}
#[cfg(feature = "http-server")]
#[test]
fn concurrent_prompt_projection_miss_fetches_once_and_shares_catalog() {
use std::io::{Read, Write};
@@ -1427,6 +1460,7 @@ mod tests {
);
}
#[cfg(feature = "http-server")]
#[test]
fn current_prompt_projection_rejects_cross_workspace_response() {
use std::io::{Read, Write};
@@ -1473,6 +1507,7 @@ mod tests {
assert!(error.to_string().contains("scope mismatch"));
}
#[cfg(feature = "http-server")]
#[test]
fn ordinary_workspace_forwarding_stamps_runtime_identity_and_signs_path_and_query() {
use std::io::{Read, Write};
@@ -1531,6 +1566,7 @@ mod tests {
);
}
#[cfg(feature = "http-server")]
#[test]
fn workspace_worker_discovery_signs_dedicated_permission_and_encoded_query() {
use std::io::{Read, Write};
@@ -1652,6 +1688,7 @@ mod tests {
);
}
#[cfg(feature = "http-server")]
#[tokio::test(flavor = "multi_thread")]
async fn remote_forwarder_is_safe_in_async_runtime_and_stamps_signed_proof() {
use std::io::{Read, Write};
@@ -1815,6 +1852,7 @@ mod tests {
assert_eq!(reason, "retire obsolete Worker");
}
#[cfg(feature = "http-server")]
#[test]
fn runtime_owned_workspace_client_returns_successful_response_without_request_timeout() {
// Effective deadline clearing is exercised with a short inherited deadline in
@@ -1838,6 +1876,7 @@ mod tests {
server.join().unwrap();
}
#[cfg(feature = "http-server")]
#[test]
fn runtime_owned_workspace_client_honors_per_request_timeouts() {
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
@@ -1857,6 +1896,7 @@ mod tests {
server.join().unwrap();
}
#[cfg(feature = "http-server")]
#[test]
fn runtime_owned_workspace_client_reports_request_timeouts() {
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
@@ -1878,6 +1918,7 @@ mod tests {
server.join().unwrap();
}
#[cfg(feature = "http-server")]
fn delayed_workspace_response(delay: Duration) -> (String, std::thread::JoinHandle<()>) {
use std::io::{Read, Write};
use std::net::TcpListener;
+16 -12
View File
@@ -3676,8 +3676,13 @@ mod tests {
#[test]
fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() {
crate::restore_test_environment::require_restore_test_environment();
let repo = create_clean_repo();
let runtime_root = tempfile::tempdir().unwrap();
// ssh-agent and the broker receive relative socket paths in this owned
// fixture tree, so the test does not depend on checkout/TMPDIR length.
// Neither this process nor its peers change cwd.
let runtime_root = tempfile::tempdir_in(".").unwrap();
let runtime_path = Path::new(runtime_root.path().file_name().unwrap());
let key_root = tempfile::tempdir().unwrap();
let key_path = key_root.path().join("id_ed25519");
let fallback_key_path = key_root.path().join("id_ed25519_fallback");
@@ -3689,7 +3694,7 @@ mod tests {
.unwrap();
assert!(status.success());
}
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
let materializer = RuntimeGitMaterializer::new(runtime_path);
let mut request = request(repo.path());
request.materialization = Some(crate::catalog::RepositoryMaterializationContext {
workspace_id: "workspace-1".to_string(),
@@ -3731,7 +3736,7 @@ mod tests {
kind: server_api::RepositorySourceKind::Ssh,
uri: "ssh://git@example.test/repo.git".to_string(),
};
let command_access = RepositoryCommandAccess::prepare(runtime_root.path(), &ssh_operation)
let command_access = RepositoryCommandAccess::prepare(runtime_path, &ssh_operation)
.unwrap()
.unwrap();
assert!(!command_access.root.join("identity").exists());
@@ -3873,18 +3878,17 @@ mod tests {
.unwrap();
// Restore preflight must validate without consuming this one-shot SSH
// delivery or creating agent/session resources needed by actual bind.
let record_before =
fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap();
let record_before = fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap();
materializer
.preflight_bind_working_directory(&id, None)
.unwrap();
assert_eq!(
fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(),
fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap(),
record_before
);
assert_eq!(materializer.list_working_directories().unwrap().len(), 1);
assert_eq!(
fs::read_dir(runtime_root.path().join(".repository-agents"))
fs::read_dir(runtime_path.join(".repository-agents"))
.map(|entries| entries.count())
.unwrap_or_default(),
0
@@ -3898,7 +3902,7 @@ mod tests {
assert!(ssh_policy.contains("__repository-ssh"));
assert!(!ssh_policy.contains("SSH_AUTH_SOCK"));
assert_eq!(
fs::read_dir(runtime_root.path().join(".repository-agents"))
fs::read_dir(runtime_path.join(".repository-agents"))
.map(|entries| entries.count())
.unwrap_or_default(),
1
@@ -3916,7 +3920,7 @@ mod tests {
"consumed delivery expiry must not revoke a live Workdir session"
);
assert_eq!(
fs::read_dir(runtime_root.path().join(".repository-agents"))
fs::read_dir(runtime_path.join(".repository-agents"))
.map(|entries| entries.count())
.unwrap_or_default(),
1,
@@ -3925,7 +3929,7 @@ mod tests {
drop(binding);
assert!(!ssh_command.exists());
assert_eq!(
fs::read_dir(runtime_root.path().join(".repository-agents"))
fs::read_dir(runtime_path.join(".repository-agents"))
.map(|entries| entries.count())
.unwrap_or_default(),
0
@@ -3971,7 +3975,7 @@ mod tests {
],
&read_only_command_policy,
));
let broker_socket = fs::read_dir(runtime_root.path().join(REPOSITORY_ACCESS_DIR))
let broker_socket = fs::read_dir(runtime_path.join(REPOSITORY_ACCESS_DIR))
.unwrap()
.filter_map(Result::ok)
.map(|entry| entry.path().join("b.sock"))
@@ -4104,7 +4108,7 @@ mod tests {
.code,
"working_directory_repository_access_expired"
);
let restored = RuntimeGitMaterializer::new(runtime_root.path());
let restored = RuntimeGitMaterializer::new(runtime_path);
assert_eq!(
restored.bind_working_directory(&id, None).unwrap_err().code,
"working_directory_remote_repository_access_required"

Some files were not shown because too many files have changed in this diff Show More