Compare commits
85
Commits
aa0fb2f787
...
dogfooding
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e7351c16b0 | ||
|
|
00d96b7163 | ||
|
|
2b4bf4b7bd | ||
|
|
e7ebc92f83 | ||
|
|
181b0d637e | ||
|
|
b0733e53b2 | ||
|
|
618880d38d | ||
|
|
79b76c8789 | ||
|
|
897c025b85 | ||
|
|
5b12c3e085 | ||
|
|
623c31615d | ||
|
|
af719e387d | ||
|
|
dddb21bd70 | ||
|
|
e8d4e964cb | ||
|
|
eb24cb04dc | ||
|
|
e84057f402 | ||
|
|
15c3006d09 | ||
|
|
b3209c2012 | ||
|
|
6f5a424661 | ||
|
|
8d47d2670b | ||
|
|
b19de45a82 | ||
|
|
1e88893ef9 | ||
|
|
1fd2e09971 | ||
|
|
f85d831104 | ||
|
|
2b24db5a15 | ||
|
|
e1945d977c | ||
|
|
a10875d68e | ||
|
|
f35dcef4d0 | ||
|
|
f5ebae1134 | ||
|
|
6be4e8388b | ||
|
|
9e3f87ac52 | ||
|
|
ceb01f6eaa | ||
|
|
1b13563e19 | ||
|
|
848a1e9a5b | ||
|
|
a77453338c | ||
|
|
d12bc2ea0b | ||
|
|
a1df3742b1 | ||
|
|
d8818b776b | ||
|
|
40c36ccb1d | ||
|
|
80c151d084 | ||
|
|
c6a8759e7b | ||
|
|
f17b5212c7 | ||
|
|
85f70d4bc1 | ||
|
|
b284934e34 | ||
|
|
2d33321fab | ||
|
|
0975471986 | ||
|
|
214d6e21dd | ||
|
|
fca5c23345 | ||
|
|
f9b1500ba2 | ||
|
|
931bf3b27d | ||
|
|
6bb671be21 | ||
|
|
b8f2fa479c | ||
|
|
1639be9841 | ||
|
|
907392cce8 | ||
|
|
6776973e4c | ||
|
|
73a3e674c9 | ||
|
|
994e75233b | ||
|
|
8b6e3070dd | ||
|
|
b860c67ea2 | ||
|
|
533477c2e7 | ||
|
|
5aa2dc579a | ||
|
|
280b3d2a25 | ||
|
|
ec74f4cdc4 | ||
|
|
5af9dc8e4a | ||
|
|
ca6d4b4b72 | ||
|
|
cccecbe7ca | ||
|
|
7a64c38493 | ||
|
|
b1361d86e9 | ||
|
|
6181426abd | ||
|
|
cd331b93c6 | ||
|
|
60eabea19a | ||
|
|
7394ec6784 | ||
|
|
28b7cc414b | ||
|
|
86d5cf9403 | ||
|
|
6f0a23e5cc | ||
|
|
2f4a46bbdb | ||
|
|
4872cf2aac | ||
|
|
734d4b44ae | ||
|
|
6810f51f46 | ||
|
|
a1748b552c | ||
|
|
62b2d27b4a | ||
|
|
62e2b44305 | ||
|
|
b5c105545c | ||
|
|
61c20b0995 | ||
|
|
b7a22146a0 |
Generated
+13
-2
@@ -3990,6 +3990,7 @@ dependencies = [
|
|||||||
"protocol",
|
"protocol",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"session-store",
|
||||||
"tokio",
|
"tokio",
|
||||||
"workdir",
|
"workdir",
|
||||||
]
|
]
|
||||||
@@ -4783,12 +4784,14 @@ version = "0.1.0"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"chrono",
|
"chrono",
|
||||||
"feature-storage",
|
"feature-storage",
|
||||||
|
"job",
|
||||||
"manifest",
|
"manifest",
|
||||||
"memory",
|
"memory",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"server-api",
|
"server-api",
|
||||||
|
"session-store",
|
||||||
"sha2 0.11.0",
|
"sha2 0.11.0",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
@@ -6497,6 +6500,7 @@ name = "workdir"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
|
"axum",
|
||||||
"fs-operation",
|
"fs-operation",
|
||||||
"libc",
|
"libc",
|
||||||
"manifest",
|
"manifest",
|
||||||
@@ -6520,7 +6524,6 @@ dependencies = [
|
|||||||
"axum",
|
"axum",
|
||||||
"base64 0.22.1",
|
"base64 0.22.1",
|
||||||
"chrono",
|
"chrono",
|
||||||
"clap",
|
|
||||||
"client",
|
"client",
|
||||||
"config-source",
|
"config-source",
|
||||||
"dotenv",
|
"dotenv",
|
||||||
@@ -6558,6 +6561,7 @@ dependencies = [
|
|||||||
"toml",
|
"toml",
|
||||||
"tools",
|
"tools",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
"tracing-subscriber",
|
||||||
"tungstenite 0.28.0",
|
"tungstenite 0.28.0",
|
||||||
"uuid",
|
"uuid",
|
||||||
"wip-client",
|
"wip-client",
|
||||||
@@ -6597,6 +6601,7 @@ dependencies = [
|
|||||||
"tokio",
|
"tokio",
|
||||||
"tokio-tungstenite 0.29.0",
|
"tokio-tungstenite 0.29.0",
|
||||||
"toml",
|
"toml",
|
||||||
|
"tools",
|
||||||
"tower",
|
"tower",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
@@ -6684,6 +6689,7 @@ dependencies = [
|
|||||||
"futures",
|
"futures",
|
||||||
"manifest",
|
"manifest",
|
||||||
"project-record",
|
"project-record",
|
||||||
|
"protocol",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
@@ -6699,7 +6705,6 @@ dependencies = [
|
|||||||
"tui",
|
"tui",
|
||||||
"uuid",
|
"uuid",
|
||||||
"workdir",
|
"workdir",
|
||||||
"worker",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -6707,15 +6712,21 @@ name = "yoi-e2e"
|
|||||||
version = "0.0.0"
|
version = "0.0.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"chrono",
|
"chrono",
|
||||||
|
"config-source",
|
||||||
"libc",
|
"libc",
|
||||||
|
"manifest",
|
||||||
|
"protocol",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"server-api",
|
"server-api",
|
||||||
|
"session-store",
|
||||||
"sha2 0.11.0",
|
"sha2 0.11.0",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
|
"tungstenite 0.29.0",
|
||||||
"webauthn-authenticator-rs",
|
"webauthn-authenticator-rs",
|
||||||
"webauthn-rs-proto",
|
"webauthn-rs-proto",
|
||||||
|
"worker-runtime",
|
||||||
"yoi-workspace-server",
|
"yoi-workspace-server",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -100,21 +100,6 @@ pub fn worker_allocation_path() -> Option<PathBuf> {
|
|||||||
worker_allocation_path_from_runtime_dir(runtime_dir())
|
worker_allocation_path_from_runtime_dir(runtime_dir())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `<runtime_dir>/<worker_name>/` — per-Worker runtime directory.
|
|
||||||
pub fn worker_runtime_dir(worker_name: &str) -> Option<PathBuf> {
|
|
||||||
worker_runtime_dir_from_runtime_dir(runtime_dir(), worker_name)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `<runtime_dir>/<worker_name>/sock` — Worker's Unix socket path.
|
|
||||||
///
|
|
||||||
/// The `worker` crate's `RuntimeDir::socket_path()` creates the socket inside the
|
|
||||||
/// Worker process. This function **predicts** the path for external callers that
|
|
||||||
/// know the Worker name, such as the TUI attach flow. Both should return the same
|
|
||||||
/// path.
|
|
||||||
pub fn worker_socket_path(worker_name: &str) -> Option<PathBuf> {
|
|
||||||
worker_socket_path_from_runtime_dir(runtime_dir(), worker_name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- internals --------------------------------------------------------------
|
// ---- internals --------------------------------------------------------------
|
||||||
|
|
||||||
fn resolve_config_dir_from_parts(
|
fn resolve_config_dir_from_parts(
|
||||||
@@ -204,20 +189,6 @@ fn worker_allocation_path_from_runtime_dir(runtime_dir: Option<PathBuf>) -> Opti
|
|||||||
Some(runtime_dir?.join("workers.json"))
|
Some(runtime_dir?.join("workers.json"))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn worker_runtime_dir_from_runtime_dir(
|
|
||||||
runtime_dir: Option<PathBuf>,
|
|
||||||
worker_name: &str,
|
|
||||||
) -> Option<PathBuf> {
|
|
||||||
Some(runtime_dir?.join(worker_name))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn worker_socket_path_from_runtime_dir(
|
|
||||||
runtime_dir: Option<PathBuf>,
|
|
||||||
worker_name: &str,
|
|
||||||
) -> Option<PathBuf> {
|
|
||||||
Some(worker_runtime_dir_from_runtime_dir(runtime_dir, worker_name)?.join("sock"))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Treats empty environment values as unset. `std::env::var` distinguishes
|
/// Treats empty environment values as unset. `std::env::var` distinguishes
|
||||||
/// `Ok("")` from `Err(NotPresent)`, but path resolution treats both as unset
|
/// `Ok("")` from `Err(NotPresent)`, but path resolution treats both as unset
|
||||||
/// for intuitive behavior.
|
/// for intuitive behavior.
|
||||||
@@ -455,16 +426,8 @@ mod tests {
|
|||||||
PathBuf::from("/sand/sessions")
|
PathBuf::from("/sand/sessions")
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
worker_allocation_path_from_runtime_dir(runtime_dir.clone()).unwrap(),
|
worker_allocation_path_from_runtime_dir(runtime_dir).unwrap(),
|
||||||
PathBuf::from("/sand/run/workers.json")
|
PathBuf::from("/sand/run/workers.json")
|
||||||
);
|
);
|
||||||
assert_eq!(
|
|
||||||
worker_runtime_dir_from_runtime_dir(runtime_dir.clone(), "foo").unwrap(),
|
|
||||||
PathBuf::from("/sand/run/foo")
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
worker_socket_path_from_runtime_dir(runtime_dir, "foo").unwrap(),
|
|
||||||
PathBuf::from("/sand/run/foo/sock")
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+184
-83
@@ -617,7 +617,8 @@ impl fmt::Display for McpRequestStage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A running initialized stdio MCP client.
|
/// An owned stdio MCP process, initialized by `connect` or explicitly after
|
||||||
|
/// `spawn_uninitialized` when startup cleanup must retain it before any await.
|
||||||
pub struct McpStdioClient {
|
pub struct McpStdioClient {
|
||||||
server_name: String,
|
server_name: String,
|
||||||
limits: McpStdioLimits,
|
limits: McpStdioLimits,
|
||||||
@@ -636,6 +637,9 @@ pub struct McpStdioClient {
|
|||||||
|
|
||||||
impl McpStdioClient {
|
impl McpStdioClient {
|
||||||
/// Spawn, initialize, negotiate capabilities, and send notifications/initialized.
|
/// Spawn, initialize, negotiate capabilities, and send notifications/initialized.
|
||||||
|
/// Initialization errors attempt shutdown, but this convenience API cannot
|
||||||
|
/// return an unreaped client on failure/cancellation. Its error is not proof
|
||||||
|
/// of cleanup; use `spawn_uninitialized` with retained ownership when needed.
|
||||||
pub async fn connect(
|
pub async fn connect(
|
||||||
spec: McpStdioServerSpec,
|
spec: McpStdioServerSpec,
|
||||||
limits: McpStdioLimits,
|
limits: McpStdioLimits,
|
||||||
@@ -667,6 +671,21 @@ impl McpStdioClient {
|
|||||||
async fn spawn(
|
async fn spawn(
|
||||||
spec: McpStdioServerSpec,
|
spec: McpStdioServerSpec,
|
||||||
limits: McpStdioLimits,
|
limits: McpStdioLimits,
|
||||||
|
) -> Result<Self, McpClientError> {
|
||||||
|
Self::spawn_uninitialized(spec, limits)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn without awaiting the handshake, so a cleanup owner can retain the
|
||||||
|
/// client before initialization can fail or be cancelled. Call
|
||||||
|
/// `initialize_with_timeout` before discovery; on error/cancellation, shut
|
||||||
|
/// down this same client rather than spawning a replacement.
|
||||||
|
///
|
||||||
|
/// Once the process spawns, this returns its owner even if an expected pipe
|
||||||
|
/// is missing. Initialization then fails, but cleanup can still reap the
|
||||||
|
/// actual child rather than relying on kill-on-drop for a post-spawn error.
|
||||||
|
pub fn spawn_uninitialized(
|
||||||
|
spec: McpStdioServerSpec,
|
||||||
|
limits: McpStdioLimits,
|
||||||
) -> Result<Self, McpClientError> {
|
) -> Result<Self, McpClientError> {
|
||||||
let redactor = spec.redactor();
|
let redactor = spec.redactor();
|
||||||
let mut command = Command::new(&spec.command);
|
let mut command = Command::new(&spec.command);
|
||||||
@@ -688,29 +707,23 @@ impl McpStdioClient {
|
|||||||
McpErrorKind::Io(redactor.redact(&err.to_string())),
|
McpErrorKind::Io(redactor.redact(&err.to_string())),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
let stdin = child.stdin.take().ok_or_else(|| {
|
let stdin = child.stdin.take();
|
||||||
McpClientError::new(
|
let stdout = child.stdout.take();
|
||||||
&spec.name,
|
let stderr = child.stderr.take();
|
||||||
McpPhase::Spawn,
|
let missing_pipe = if stdin.is_none() {
|
||||||
McpErrorKind::Protocol("child stdin was not piped".into()),
|
Some("stdin")
|
||||||
)
|
} else if stdout.is_none() {
|
||||||
})?;
|
Some("stdout")
|
||||||
let stdout = child.stdout.take().ok_or_else(|| {
|
} else if stderr.is_none() {
|
||||||
McpClientError::new(
|
Some("stderr")
|
||||||
&spec.name,
|
} else {
|
||||||
McpPhase::Spawn,
|
None
|
||||||
McpErrorKind::Protocol("child stdout was not piped".into()),
|
};
|
||||||
)
|
|
||||||
})?;
|
|
||||||
let stderr = child.stderr.take().ok_or_else(|| {
|
|
||||||
McpClientError::new(
|
|
||||||
&spec.name,
|
|
||||||
McpPhase::Spawn,
|
|
||||||
McpErrorKind::Protocol("child stderr was not piped".into()),
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let stdin: SharedStdin = Arc::new(Mutex::new(Some(Box::new(stdin))));
|
let stdin: SharedStdin =
|
||||||
|
Arc::new(Mutex::new(stdin.map(|stdin| {
|
||||||
|
Box::new(stdin) as Box<dyn AsyncWrite + Send + Unpin>
|
||||||
|
})));
|
||||||
let diagnostics = Arc::new(Mutex::new(BoundedDiagnostics::new(
|
let diagnostics = Arc::new(Mutex::new(BoundedDiagnostics::new(
|
||||||
spec.name.clone(),
|
spec.name.clone(),
|
||||||
limits.max_diagnostic_lines,
|
limits.max_diagnostic_lines,
|
||||||
@@ -718,16 +731,32 @@ impl McpStdioClient {
|
|||||||
)));
|
)));
|
||||||
let list_changes = Arc::new(Mutex::new(BoundedListChanged::new(spec.name.clone())));
|
let list_changes = Arc::new(Mutex::new(BoundedListChanged::new(spec.name.clone())));
|
||||||
let (tx, rx) = mpsc::channel(16);
|
let (tx, rx) = mpsc::channel(16);
|
||||||
let reader_task = spawn_stdout_reader(
|
// Do not return an error that discards an already spawned child. Queue
|
||||||
spec.name.clone(),
|
// setup failure before reader events so initialization cannot succeed.
|
||||||
stdout,
|
if let Some(pipe) = missing_pipe {
|
||||||
stdin.clone(),
|
tx.try_send(ReaderEvent::Error(McpClientError::new(
|
||||||
tx,
|
&spec.name,
|
||||||
limits.clone(),
|
McpPhase::Spawn,
|
||||||
redactor.clone(),
|
McpErrorKind::Protocol(format!("child {pipe} was not piped")),
|
||||||
list_changes.clone(),
|
)))
|
||||||
);
|
.expect("new response channel has capacity");
|
||||||
let stderr_task = spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone());
|
}
|
||||||
|
let reader_task = match stdout {
|
||||||
|
Some(stdout) => spawn_stdout_reader(
|
||||||
|
spec.name.clone(),
|
||||||
|
stdout,
|
||||||
|
stdin.clone(),
|
||||||
|
tx,
|
||||||
|
limits.clone(),
|
||||||
|
redactor.clone(),
|
||||||
|
list_changes.clone(),
|
||||||
|
),
|
||||||
|
None => tokio::spawn(async {}),
|
||||||
|
};
|
||||||
|
let stderr_task = match stderr {
|
||||||
|
Some(stderr) => spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone()),
|
||||||
|
None => tokio::spawn(async {}),
|
||||||
|
};
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
server_name: spec.name,
|
server_name: spec.name,
|
||||||
@@ -746,6 +775,25 @@ impl McpStdioClient {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Perform the bounded handshake on an already retained client. Failure or
|
||||||
|
/// cancellation does not drop or shut down the client: its owner must do so.
|
||||||
|
/// Do not retry initialization on a partially initialized connection.
|
||||||
|
pub async fn initialize_with_timeout(&mut self) -> Result<(), McpClientError> {
|
||||||
|
let started = Instant::now();
|
||||||
|
match timeout(self.limits.startup_timeout, self.initialize()).await {
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(_) => Err(McpClientError::new(
|
||||||
|
&self.server_name,
|
||||||
|
McpPhase::Initialize,
|
||||||
|
McpErrorKind::Timeout {
|
||||||
|
operation: "startup".to_string(),
|
||||||
|
elapsed: started.elapsed(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.with_diagnostics(self.snapshot_diagnostics().await)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn initialize(&mut self) -> Result<(), McpClientError> {
|
async fn initialize(&mut self) -> Result<(), McpClientError> {
|
||||||
let result: InitializeResult = self
|
let result: InitializeResult = self
|
||||||
.request(
|
.request(
|
||||||
@@ -1080,6 +1128,8 @@ impl McpStdioClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Close stdin and wait for process exit, falling back to terminate and kill.
|
/// Close stdin and wait for process exit, falling back to terminate and kill.
|
||||||
|
/// Errors and cancellation retain the child for another attempt on this
|
||||||
|
/// client. Only confirmed reaping releases the process handle.
|
||||||
pub async fn shutdown(&mut self) -> Result<ShutdownReport, McpClientError> {
|
pub async fn shutdown(&mut self) -> Result<ShutdownReport, McpClientError> {
|
||||||
self.shutdown_started = true;
|
self.shutdown_started = true;
|
||||||
let deadline = Instant::now() + self.limits.shutdown_timeout;
|
let deadline = Instant::now() + self.limits.shutdown_timeout;
|
||||||
@@ -1091,64 +1141,55 @@ impl McpStdioClient {
|
|||||||
.await
|
.await
|
||||||
.is_ok();
|
.is_ok();
|
||||||
|
|
||||||
let mut child = match self.child.take() {
|
// Keep the actual process handle in the client across every await and
|
||||||
Some(child) => child,
|
// error. Taking it before wait loses retry/reaping authority on cancel.
|
||||||
None => return Ok(ShutdownReport::already_finished()),
|
if self.child.is_none() {
|
||||||
};
|
return Ok(ShutdownReport::already_finished());
|
||||||
|
}
|
||||||
if !closed {
|
if !closed {
|
||||||
self.reader_task.abort();
|
self.reader_task.abort();
|
||||||
return self.terminate_then_kill(child).await;
|
return self.terminate_then_kill().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let child = self.child.as_mut().expect("checked child");
|
||||||
if let Ok(Some(status)) = child.try_wait() {
|
if let Ok(Some(status)) = child.try_wait() {
|
||||||
self.reader_task.abort();
|
return Ok(self.finish_shutdown(status, false, false));
|
||||||
self.stderr_task.abort();
|
|
||||||
return Ok(ShutdownReport {
|
|
||||||
exit_status: Some(status),
|
|
||||||
terminated: false,
|
|
||||||
killed: false,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
match timeout_at(deadline, child.wait()).await {
|
match timeout_at(deadline, child.wait()).await {
|
||||||
Ok(Ok(status)) => {
|
Ok(Ok(status)) => Ok(self.finish_shutdown(status, false, false)),
|
||||||
self.reader_task.abort();
|
|
||||||
self.stderr_task.abort();
|
|
||||||
Ok(ShutdownReport {
|
|
||||||
exit_status: Some(status),
|
|
||||||
terminated: false,
|
|
||||||
killed: false,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
Ok(Err(err)) => Err(McpClientError::new(
|
Ok(Err(err)) => Err(McpClientError::new(
|
||||||
&self.server_name,
|
&self.server_name,
|
||||||
McpPhase::Shutdown,
|
McpPhase::Shutdown,
|
||||||
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
||||||
)
|
)
|
||||||
.with_diagnostics(self.snapshot_diagnostics().await)),
|
.with_diagnostics(self.snapshot_diagnostics().await)),
|
||||||
Err(_) => self.terminate_then_kill(child).await,
|
Err(_) => self.terminate_then_kill().await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn terminate_then_kill(
|
fn finish_shutdown(
|
||||||
&mut self,
|
&mut self,
|
||||||
mut child: Child,
|
status: ExitStatus,
|
||||||
) -> Result<ShutdownReport, McpClientError> {
|
terminated: bool,
|
||||||
let mut terminated = false;
|
killed: bool,
|
||||||
let mut killed = false;
|
) -> ShutdownReport {
|
||||||
if send_terminate(&mut child).is_ok() {
|
// No await separates confirmed reaping from releasing the handle.
|
||||||
terminated = true;
|
drop(self.child.take());
|
||||||
|
self.reader_task.abort();
|
||||||
|
self.stderr_task.abort();
|
||||||
|
ShutdownReport {
|
||||||
|
exit_status: Some(status),
|
||||||
|
terminated,
|
||||||
|
killed,
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn terminate_then_kill(&mut self) -> Result<ShutdownReport, McpClientError> {
|
||||||
|
let child = self.child.as_mut().expect("shutdown retains child");
|
||||||
|
let terminated = send_terminate(child).is_ok();
|
||||||
match timeout(self.limits.kill_timeout, child.wait()).await {
|
match timeout(self.limits.kill_timeout, child.wait()).await {
|
||||||
Ok(Ok(status)) => {
|
Ok(Ok(status)) => Ok(self.finish_shutdown(status, terminated, false)),
|
||||||
self.reader_task.abort();
|
|
||||||
self.stderr_task.abort();
|
|
||||||
Ok(ShutdownReport {
|
|
||||||
exit_status: Some(status),
|
|
||||||
terminated,
|
|
||||||
killed,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
Ok(Err(err)) => Err(McpClientError::new(
|
Ok(Err(err)) => Err(McpClientError::new(
|
||||||
&self.server_name,
|
&self.server_name,
|
||||||
McpPhase::Shutdown,
|
McpPhase::Shutdown,
|
||||||
@@ -1163,7 +1204,6 @@ impl McpStdioClient {
|
|||||||
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
killed = true;
|
|
||||||
let status = timeout(self.limits.kill_timeout, child.wait())
|
let status = timeout(self.limits.kill_timeout, child.wait())
|
||||||
.await
|
.await
|
||||||
.map_err(|_| {
|
.map_err(|_| {
|
||||||
@@ -1183,13 +1223,7 @@ impl McpStdioClient {
|
|||||||
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
self.reader_task.abort();
|
Ok(self.finish_shutdown(status, terminated, true))
|
||||||
self.stderr_task.abort();
|
|
||||||
Ok(ShutdownReport {
|
|
||||||
exit_status: Some(status),
|
|
||||||
terminated,
|
|
||||||
killed,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1197,10 +1231,10 @@ impl McpStdioClient {
|
|||||||
|
|
||||||
impl Drop for McpStdioClient {
|
impl Drop for McpStdioClient {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if !self.shutdown_started {
|
// An attempted shutdown is not proof of exit. Errors and cancellation
|
||||||
if let Some(child) = &mut self.child {
|
// leave the child here so Drop must still make a best-effort kill.
|
||||||
let _ = child.start_kill();
|
if let Some(child) = &mut self.child {
|
||||||
}
|
let _ = child.start_kill();
|
||||||
}
|
}
|
||||||
self.reader_task.abort();
|
self.reader_task.abort();
|
||||||
self.stderr_task.abort();
|
self.stderr_task.abort();
|
||||||
@@ -1784,3 +1818,70 @@ fn send_terminate(child: &mut Child) -> Result<(), ()> {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod deadline_tests;
|
mod deadline_tests;
|
||||||
|
|
||||||
|
#[cfg(all(test, unix))]
|
||||||
|
mod shutdown_retention_tests {
|
||||||
|
//! The real client and OS process are used; only the external server is a
|
||||||
|
//! local shell that acknowledges initialization and ignores TERM/EOF.
|
||||||
|
use super::*;
|
||||||
|
use std::future::{Future, poll_fn};
|
||||||
|
use std::task::Poll;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancelled_graceful_or_terminate_wait_retries_reaping_the_same_child() {
|
||||||
|
for cancel_in_fallback in [false, true] {
|
||||||
|
let script = r#"trap '' TERM
|
||||||
|
read init || exit 1
|
||||||
|
printf '%s\n' '{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-11-25","capabilities":{},"serverInfo":{"name":"retained","version":"1"}}}'
|
||||||
|
while :; do :; done
|
||||||
|
"#;
|
||||||
|
let mut client = McpStdioClient::connect(
|
||||||
|
McpStdioServerSpec::new("retained", "/bin/sh").args(["-c", script]),
|
||||||
|
McpStdioLimits {
|
||||||
|
shutdown_timeout: if cancel_in_fallback {
|
||||||
|
Duration::ZERO
|
||||||
|
} else {
|
||||||
|
Duration::from_secs(30)
|
||||||
|
},
|
||||||
|
kill_timeout: Duration::from_secs(30),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("initialize after installing TERM handler");
|
||||||
|
let pid = client.child.as_ref().unwrap().id();
|
||||||
|
let mut shutdown = Box::pin(client.shutdown());
|
||||||
|
poll_fn(|cx| {
|
||||||
|
assert!(shutdown.as_mut().poll(cx).is_pending());
|
||||||
|
Poll::Ready(())
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
drop(shutdown);
|
||||||
|
assert_eq!(client.child.as_ref().and_then(Child::id), pid);
|
||||||
|
assert!(client.shutdown_started);
|
||||||
|
|
||||||
|
client.limits.shutdown_timeout = Duration::from_millis(20);
|
||||||
|
client.limits.kill_timeout = Duration::from_millis(50);
|
||||||
|
let report = timeout(Duration::from_secs(2), client.shutdown())
|
||||||
|
.await
|
||||||
|
.expect("retry bounded")
|
||||||
|
.expect("retry reaps original child");
|
||||||
|
assert!(
|
||||||
|
report.exit_status.is_some(),
|
||||||
|
"not an already-finished receipt"
|
||||||
|
);
|
||||||
|
assert!(report.killed, "TERM was ignored");
|
||||||
|
assert!(client.child.is_none());
|
||||||
|
let mut status = 0;
|
||||||
|
assert_eq!(
|
||||||
|
unsafe { libc::waitpid(pid.unwrap() as libc::pid_t, &mut status, libc::WNOHANG) },
|
||||||
|
-1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::io::Error::last_os_error().raw_os_error(),
|
||||||
|
Some(libc::ECHILD),
|
||||||
|
"shutdown, not a later observer, reaped the original process"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -210,12 +210,7 @@ impl MergeRequestThreadEvent {
|
|||||||
}
|
}
|
||||||
fn bound_bodies(&mut self) {
|
fn bound_bodies(&mut self) {
|
||||||
match self {
|
match self {
|
||||||
Self::Review(value) => {
|
Self::Review(_) => {}
|
||||||
truncate_body(&mut value.body);
|
|
||||||
for finding in &mut value.findings {
|
|
||||||
truncate_body(&mut finding.body);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Self::ReviewRevoked(value) => truncate_body(&mut value.reason),
|
Self::ReviewRevoked(value) => truncate_body(&mut value.reason),
|
||||||
Self::ReviewCancelled(value) => truncate_body(&mut value.reason),
|
Self::ReviewCancelled(value) => truncate_body(&mut value.reason),
|
||||||
Self::Comment(value) => truncate_body(&mut value.body),
|
Self::Comment(value) => truncate_body(&mut value.body),
|
||||||
@@ -939,10 +934,6 @@ impl MergeRequestStore {
|
|||||||
&self,
|
&self,
|
||||||
i: SubmitMergeRequestReview,
|
i: SubmitMergeRequestReview,
|
||||||
) -> Result<ReviewEvent, MergeRequestError> {
|
) -> Result<ReviewEvent, MergeRequestError> {
|
||||||
bounded_body("review body", &i.body)?;
|
|
||||||
for finding in &i.findings {
|
|
||||||
bounded_body("review finding", &finding.body)?;
|
|
||||||
}
|
|
||||||
let mut c = self.lock()?;
|
let mut c = self.lock()?;
|
||||||
let t = c.transaction()?;
|
let t = c.transaction()?;
|
||||||
let g: Option<(String, String, String, String, String, String)> = t
|
let g: Option<(String, String, String, String, String, String)> = t
|
||||||
@@ -1747,7 +1738,23 @@ fn ticket_event(
|
|||||||
) -> Result<(), MergeRequestError> {
|
) -> Result<(), MergeRequestError> {
|
||||||
let ticket = &mr.ticket_ids[0];
|
let ticket = &mr.ticket_ids[0];
|
||||||
let n:i64=t.query_row("SELECT COALESCE(MAX(event_index),-1)+1 FROM typed_ticket_events WHERE workspace_id=?1 AND ticket_id=?2",params![mr.workspace_id,ticket],|r|r.get(0))?;
|
let n:i64=t.query_row("SELECT COALESCE(MAX(event_index),-1)+1 FROM typed_ticket_events WHERE workspace_id=?1 AND ticket_id=?2",params![mr.workspace_id,ticket],|r|r.get(0))?;
|
||||||
t.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,author,at,from_state,to_state,heading,body)VALUES(?1,?2,?3,'comment',?4,?5,NULL,NULL,'Merge Request integrated',?6)",params![mr.workspace_id,ticket,n,format!("worker:{}:{}",e.merged_by.runtime_id,e.merged_by.worker_id),e.created_at.to_rfc3339(),format!("Merge Request `{}` integrated approved source ref `{}`; Ticket remains in progress until guarded Ticket completion.",mr.merge_request_id,e.approved_source_ref)])?;
|
t.execute(
|
||||||
|
"INSERT INTO typed_ticket_events(
|
||||||
|
workspace_id,ticket_id,event_index,kind,author,at,from_state,to_state,heading,body,event_ref
|
||||||
|
) VALUES(?1,?2,?3,'comment',?4,?5,NULL,NULL,'Merge Request integrated',?6,?7)",
|
||||||
|
params![
|
||||||
|
mr.workspace_id,
|
||||||
|
ticket,
|
||||||
|
n,
|
||||||
|
format!("worker:{}:{}", e.merged_by.runtime_id, e.merged_by.worker_id),
|
||||||
|
e.created_at.to_rfc3339(),
|
||||||
|
format!(
|
||||||
|
"Merge Request `{}` integrated approved source ref `{}`; Ticket remains in progress until guarded Ticket completion.",
|
||||||
|
mr.merge_request_id, e.approved_source_ref
|
||||||
|
),
|
||||||
|
format!("{ticket}:{n}"),
|
||||||
|
],
|
||||||
|
)?;
|
||||||
for (k, v) in [
|
for (k, v) in [
|
||||||
("implementation_assignment_id", a),
|
("implementation_assignment_id", a),
|
||||||
("merge_request_id", &mr.merge_request_id),
|
("merge_request_id", &mr.merge_request_id),
|
||||||
|
|||||||
@@ -53,8 +53,8 @@ fn fixture() -> (tempfile::TempDir, MergeRequestStore) {
|
|||||||
let d = tempfile::tempdir().unwrap();
|
let d = tempfile::tempdir().unwrap();
|
||||||
let p = d.path().join("db");
|
let p = d.path().join("db");
|
||||||
let c = Connection::open(&p).unwrap();
|
let c = Connection::open(&p).unwrap();
|
||||||
c.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY);CREATE TABLE repositories(workspace_id TEXT,repository_id TEXT,PRIMARY KEY(workspace_id,repository_id));CREATE TABLE ticket_current_worker_assignments(workspace_id TEXT,ticket_id TEXT,assignment_id TEXT,runtime_id TEXT,worker_id TEXT,updated_at TEXT,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_tickets(workspace_id TEXT,ticket_id TEXT,workflow_state TEXT,workflow_state_explicit INTEGER,updated_at TEXT,title TEXT NOT NULL,body TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_ticket_targets(workspace_id TEXT,ticket_id TEXT,ordinal INTEGER,repository_key TEXT,ref_selector TEXT,access TEXT);CREATE TABLE typed_ticket_events(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,kind TEXT,author TEXT,at TEXT,from_state TEXT,to_state TEXT,heading TEXT,body TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index));CREATE TABLE typed_ticket_event_attributes(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,key TEXT,value TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index,key));INSERT INTO workspaces VALUES('W');INSERT INTO repositories VALUES('W','R');INSERT INTO repositories VALUES('W','R2');INSERT INTO ticket_current_worker_assignments VALUES('W','T','A','runtime','coder','t');INSERT INTO typed_tickets VALUES('W','T','inprogress',1,'t','Title','Requirements');CREATE VIEW ticket_active_worker_assignments AS SELECT current.* FROM ticket_current_worker_assignments current JOIN typed_tickets ticket ON ticket.workspace_id=current.workspace_id AND ticket.ticket_id=current.ticket_id WHERE ticket.workflow_state NOT IN ('done','closed');").unwrap();
|
c.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY);CREATE TABLE repositories(workspace_id TEXT,repository_id TEXT,PRIMARY KEY(workspace_id,repository_id));CREATE TABLE ticket_current_worker_assignments(workspace_id TEXT,ticket_id TEXT,assignment_id TEXT,runtime_id TEXT,worker_id TEXT,updated_at TEXT,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_tickets(workspace_id TEXT,ticket_id TEXT,workflow_state TEXT,workflow_state_explicit INTEGER,updated_at TEXT,title TEXT NOT NULL,body TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_ticket_targets(workspace_id TEXT,ticket_id TEXT,ordinal INTEGER,repository_key TEXT,ref_selector TEXT,access TEXT);CREATE TABLE typed_ticket_events(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,kind TEXT,author TEXT,at TEXT,from_state TEXT,to_state TEXT,heading TEXT,body TEXT,event_ref TEXT NOT NULL DEFAULT '',PRIMARY KEY(workspace_id,ticket_id,event_index),UNIQUE(workspace_id,ticket_id,event_ref));CREATE TABLE typed_ticket_event_attributes(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,key TEXT,value TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index,key));INSERT INTO workspaces VALUES('W');INSERT INTO repositories VALUES('W','R');INSERT INTO repositories VALUES('W','R2');INSERT INTO ticket_current_worker_assignments VALUES('W','T','A','runtime','coder','t');INSERT INTO typed_tickets VALUES('W','T','inprogress',1,'t','Title','Requirements');CREATE VIEW ticket_active_worker_assignments AS SELECT current.* FROM ticket_current_worker_assignments current JOIN typed_tickets ticket ON ticket.workspace_id=current.workspace_id AND ticket.ticket_id=current.ticket_id WHERE ticket.workflow_state NOT IN ('done','closed');").unwrap();
|
||||||
c.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body) VALUES('W','T',-1,'comment','Requirements')",[]).unwrap();
|
c.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body,event_ref) VALUES('W','T',-1,'comment','Requirements','T:-1')",[]).unwrap();
|
||||||
drop(c);
|
drop(c);
|
||||||
let a = Assignments(Arc::new(Mutex::new(CurrentAssignment {
|
let a = Assignments(Arc::new(Mutex::new(CurrentAssignment {
|
||||||
assignment_id: "A".into(),
|
assignment_id: "A".into(),
|
||||||
@@ -376,6 +376,49 @@ fn approve_for(
|
|||||||
fn approve(s: &MergeRequestStore, subject: &str, token: &str) -> ReviewEvent {
|
fn approve(s: &MergeRequestStore, subject: &str, token: &str) -> ReviewEvent {
|
||||||
approve_for(s, "MR", "R", subject, token)
|
approve_for(s, "MR", "R", subject, token)
|
||||||
}
|
}
|
||||||
|
#[test]
|
||||||
|
fn review_submission_preserves_long_body_and_findings_on_reload() {
|
||||||
|
for (body, finding_body) in [
|
||||||
|
("b".repeat(16 * 1024 + 1), "finding".into()),
|
||||||
|
("review".into(), "指摘".repeat(4096)),
|
||||||
|
] {
|
||||||
|
let (_dir, store) = fixture();
|
||||||
|
open(&store);
|
||||||
|
request(&store, "source", "token");
|
||||||
|
let findings = vec![ReviewFinding {
|
||||||
|
severity: FindingSeverity::Major,
|
||||||
|
code: None,
|
||||||
|
path: None,
|
||||||
|
line: None,
|
||||||
|
body: finding_body,
|
||||||
|
}];
|
||||||
|
let submitted = store
|
||||||
|
.submit_review(SubmitMergeRequestReview {
|
||||||
|
merge_request_id: "MR".into(),
|
||||||
|
ticket_id: "T".into(),
|
||||||
|
current_subject_ref: "source".into(),
|
||||||
|
capability_token: "token".into(),
|
||||||
|
decision: ReviewDecision::RequestChanges,
|
||||||
|
body: body.clone(),
|
||||||
|
findings: findings.clone(),
|
||||||
|
now: at(4),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(submitted.body, body);
|
||||||
|
assert_eq!(submitted.findings, findings);
|
||||||
|
let reloaded = store.get_by_id("W", "MR").unwrap();
|
||||||
|
let saved = reloaded
|
||||||
|
.thread
|
||||||
|
.iter()
|
||||||
|
.find_map(|event| match event {
|
||||||
|
MergeRequestThreadEvent::Review(review) => Some(review),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(saved, &submitted);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn review_submission_authorization_rejects_invalid_grants_before_side_effects() {
|
fn review_submission_authorization_rejects_invalid_grants_before_side_effects() {
|
||||||
let (_d, store) = fixture();
|
let (_d, store) = fixture();
|
||||||
@@ -457,6 +500,18 @@ fn integration_preserves_selectors_ticket_state_and_assignment_and_replays_its_m
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(replayed, merged);
|
assert_eq!(replayed, merged);
|
||||||
let connection = Connection::open(d.path().join("db")).unwrap();
|
let connection = Connection::open(d.path().join("db")).unwrap();
|
||||||
|
let ticket_refs = connection
|
||||||
|
.prepare("SELECT event_ref FROM typed_ticket_events WHERE event_index >= 0")
|
||||||
|
.unwrap()
|
||||||
|
.query_map([], |row| row.get::<_, String>(0))
|
||||||
|
.unwrap()
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
ticket_refs,
|
||||||
|
vec!["T:0"],
|
||||||
|
"replay must not append another post"
|
||||||
|
);
|
||||||
let (state, active): (String, bool) = connection.query_row(
|
let (state, active): (String, bool) = connection.query_row(
|
||||||
"SELECT workflow_state, EXISTS(SELECT 1 FROM ticket_active_worker_assignments WHERE workspace_id='W' AND ticket_id='T') FROM typed_tickets WHERE workspace_id='W' AND ticket_id='T'",
|
"SELECT workflow_state, EXISTS(SELECT 1 FROM ticket_active_worker_assignments WHERE workspace_id='W' AND ticket_id='T') FROM typed_tickets WHERE workspace_id='W' AND ticket_id='T'",
|
||||||
[],
|
[],
|
||||||
@@ -477,6 +532,87 @@ fn integration_preserves_selectors_ticket_state_and_assignment_and_replays_its_m
|
|||||||
assert!(!json.contains(banned), "{banned} in {json}")
|
assert!(!json.contains(banned), "{banned} in {json}")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
#[test]
|
||||||
|
fn integration_post_failure_rolls_back_and_same_operation_recovers_with_legacy_empty_ref() {
|
||||||
|
let (dir, store) = fixture();
|
||||||
|
open(&store);
|
||||||
|
let review = approve(&store, "source", "approved");
|
||||||
|
request(&store, "source", "pending");
|
||||||
|
let completion = CompleteMergeRequest {
|
||||||
|
merge_request_id: "MR".into(),
|
||||||
|
ticket_id: "T".into(),
|
||||||
|
operation_id: "recover-integration".into(),
|
||||||
|
approval_event_id: review.event_id,
|
||||||
|
current_subject_ref: "source".into(),
|
||||||
|
target_ref_before: "before".into(),
|
||||||
|
target_ref_after: "after".into(),
|
||||||
|
strategy: MergeStrategy::FastForward,
|
||||||
|
resolution: ConflictResolution::None,
|
||||||
|
auth: auth(),
|
||||||
|
now: at(5),
|
||||||
|
};
|
||||||
|
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||||
|
// Retain a comment written before integration populated event_ref.
|
||||||
|
connection.execute_batch(
|
||||||
|
"INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body,event_ref)
|
||||||
|
VALUES('W','T',0,'comment','Legacy integration','');
|
||||||
|
CREATE TRIGGER reject_integration_post BEFORE INSERT ON typed_ticket_events
|
||||||
|
WHEN NEW.heading='Merge Request integrated'
|
||||||
|
BEGIN SELECT RAISE(ABORT,'injected integration post failure'); END;",
|
||||||
|
).unwrap();
|
||||||
|
let before = store.get_by_id("W", "MR").unwrap();
|
||||||
|
let error = store.complete(completion.clone()).unwrap_err();
|
||||||
|
assert!(
|
||||||
|
error
|
||||||
|
.to_string()
|
||||||
|
.contains("injected integration post failure")
|
||||||
|
);
|
||||||
|
let failed = store.get_by_id("W", "MR").unwrap();
|
||||||
|
assert_eq!(failed.state, MergeRequestState::Open);
|
||||||
|
assert_eq!(failed.thread, before.thread);
|
||||||
|
assert!(store.authorize_review_submission("MR", "pending").is_ok());
|
||||||
|
assert_eq!(
|
||||||
|
connection
|
||||||
|
.query_row(
|
||||||
|
"SELECT COUNT(*) FROM typed_ticket_event_attributes",
|
||||||
|
[],
|
||||||
|
|row| row.get::<_, i64>(0),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
|
||||||
|
connection
|
||||||
|
.execute_batch("DROP TRIGGER reject_integration_post")
|
||||||
|
.unwrap();
|
||||||
|
let merged = store.complete(completion.clone()).unwrap();
|
||||||
|
assert_eq!(store.complete(completion).unwrap(), merged);
|
||||||
|
assert_eq!(
|
||||||
|
store.get_by_id("W", "MR").unwrap().state,
|
||||||
|
MergeRequestState::Merged
|
||||||
|
);
|
||||||
|
assert!(store.authorize_review_submission("MR", "pending").is_err());
|
||||||
|
let posts = connection
|
||||||
|
.prepare("SELECT event_ref,body FROM typed_ticket_events WHERE event_index >= 0 ORDER BY event_index")
|
||||||
|
.unwrap()
|
||||||
|
.query_map([], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)))
|
||||||
|
.unwrap()
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
posts.len(),
|
||||||
|
2,
|
||||||
|
"retry and replay must append exactly one post"
|
||||||
|
);
|
||||||
|
assert_eq!(posts[0], (String::new(), "Legacy integration".into()));
|
||||||
|
assert_eq!(posts[1].0, "T:1");
|
||||||
|
assert!(posts[1].1.contains("`MR`"));
|
||||||
|
assert_eq!(connection.query_row(
|
||||||
|
"SELECT value FROM typed_ticket_event_attributes WHERE event_index=1 AND key='operation_id'",
|
||||||
|
[], |row| row.get::<_, String>(0),
|
||||||
|
).unwrap(), "recover-integration");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn source_move_cancels_submission_and_old_approval_is_reusable_when_source_returns() {
|
fn source_move_cancels_submission_and_old_approval_is_reusable_when_source_returns() {
|
||||||
let (_d, s) = fixture();
|
let (_d, s) = fixture();
|
||||||
@@ -955,7 +1091,7 @@ fn ticket_rescope_requires_fresh_review_before_merge_and_remains_recoverable() {
|
|||||||
let connection = Connection::open(dir.path().join("db")).unwrap();
|
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||||
connection
|
connection
|
||||||
.execute_batch(
|
.execute_batch(
|
||||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL)",
|
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL,'T:1')",
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
connection
|
connection
|
||||||
@@ -1082,7 +1218,7 @@ fn ticket_rescope_after_integration_accepts_fresh_requirement_attestation() {
|
|||||||
let connection = Connection::open(dir.path().join("db")).unwrap();
|
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||||
connection
|
connection
|
||||||
.execute_batch(
|
.execute_batch(
|
||||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL)",
|
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL,'T:1')",
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
connection
|
connection
|
||||||
@@ -1307,6 +1443,24 @@ fn partial_and_full_integration_retain_ticket_state_and_active_assignment() {
|
|||||||
assert_eq!(second_request.merged_result().unwrap(), &second_merge);
|
assert_eq!(second_request.merged_result().unwrap(), &second_merge);
|
||||||
assert_eq!(first_request.integration_approval().unwrap(), &first);
|
assert_eq!(first_request.integration_approval().unwrap(), &first);
|
||||||
assert_eq!(second_request.integration_approval().unwrap(), &second);
|
assert_eq!(second_request.integration_approval().unwrap(), &second);
|
||||||
|
let posts = connection
|
||||||
|
.prepare(
|
||||||
|
"SELECT event.event_ref, attribute.value
|
||||||
|
FROM typed_ticket_events event
|
||||||
|
JOIN typed_ticket_event_attributes attribute USING(workspace_id,ticket_id,event_index)
|
||||||
|
WHERE attribute.key='merge_request_id' ORDER BY event.event_index",
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.query_map([], |row| {
|
||||||
|
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
posts,
|
||||||
|
vec![("T:0".into(), "MR".into()), ("T:1".into(), "MR-2".into())]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1324,7 +1478,7 @@ fn historical_ticket_completion_payload_remains_readable_and_unchanged_on_migrat
|
|||||||
}"#;
|
}"#;
|
||||||
connection.execute_batch(
|
connection.execute_batch(
|
||||||
"UPDATE typed_tickets SET workflow_state='done' WHERE workspace_id='W' AND ticket_id='T';
|
"UPDATE typed_tickets SET workflow_state='done' WHERE workspace_id='W' AND ticket_id='T';
|
||||||
INSERT INTO typed_ticket_events VALUES('W','T',0,'state_changed','worker:runtime:coder','2026-07-26T12:00:07Z','inprogress','done','Ticket requirements completed','historical body');"
|
INSERT INTO typed_ticket_events VALUES('W','T',0,'state_changed','worker:runtime:coder','2026-07-26T12:00:07Z','inprogress','done','Ticket requirements completed','historical body','T:0');"
|
||||||
).unwrap();
|
).unwrap();
|
||||||
connection
|
connection
|
||||||
.execute(
|
.execute(
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ pub enum AuthenticatedInputSource {
|
|||||||
/// Assigned whenever a serialized tracked method crosses an untrusted
|
/// Assigned whenever a serialized tracked method crosses an untrusted
|
||||||
/// protocol boundary. Receivers must handle it exactly like public input.
|
/// protocol boundary. Receivers must handle it exactly like public input.
|
||||||
UntrustedWire,
|
UntrustedWire,
|
||||||
/// Stamped only by the operator-owned standalone in-process transport.
|
/// Stamped only by the operator-owned Standalone Host after local admission.
|
||||||
/// This is not a Backend Account or authority accepted from wire fields.
|
/// This is not a Backend Account or authority accepted from wire fields.
|
||||||
LocalOperator,
|
LocalOperator,
|
||||||
Account {
|
Account {
|
||||||
@@ -399,15 +399,12 @@ pub enum Method {
|
|||||||
|
|
||||||
/// Typed lifecycle events sent from a child Worker to its parent.
|
/// Typed lifecycle events sent from a child Worker to its parent.
|
||||||
///
|
///
|
||||||
/// Delivered as `Method::WorkerEvent` over the parent's Unix socket. The
|
/// Delivered in-process to the parent Controller's protocol intake. Agent-visible variants
|
||||||
/// parent Controller always applies variant-specific side effects
|
/// are queued into the notification buffer. Reports are advisory: receivers must check the
|
||||||
/// (registry / worker-allocation updates). Agent-visible variants are also
|
/// actual child session before treating its work as complete.
|
||||||
/// queued into the notification buffer; control-plane-only variants are
|
|
||||||
/// not injected into the parent's LLM context.
|
|
||||||
///
|
///
|
||||||
/// Transport is fire-and-forget; receivers must tolerate out-of-order
|
/// Retained legacy control-only variants can be decoded from saved history, but do not
|
||||||
/// delivery (e.g. `TurnEnded` arriving after `ShutDown` for the same
|
/// register live children, delegate scope, or establish a socket callback.
|
||||||
/// child Worker).
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
||||||
@@ -429,16 +426,8 @@ pub enum WorkerEvent {
|
|||||||
/// Child has stopped (controller loop is exiting).
|
/// Child has stopped (controller loop is exiting).
|
||||||
ShutDown { worker_name: String },
|
ShutDown { worker_name: String },
|
||||||
|
|
||||||
/// Child SubWorker sub-delegated scope to a grandchild SubWorker via `SubWorkerSpawn`.
|
/// Legacy process-child scope report retained for saved Session decoding only.
|
||||||
///
|
/// Live Controllers ignore it; only the Host-owned Internal registry grants child scope.
|
||||||
/// Control-plane only: receivers apply registry side effects and
|
|
||||||
/// propagate upward, but do not expose this as an agent notification.
|
|
||||||
///
|
|
||||||
/// The parent uses this to add the grandchild to its own
|
|
||||||
/// `spawned_workers.json` so it can manage the grandchild directly
|
|
||||||
/// even if the intermediate child dies. The parent then re-fires
|
|
||||||
/// this event upward (if it has a parent of its own) to maintain
|
|
||||||
/// the chain to root.
|
|
||||||
ScopeSubDelegated {
|
ScopeSubDelegated {
|
||||||
/// Sub-delegating Worker (= the sender itself).
|
/// Sub-delegating Worker (= the sender itself).
|
||||||
parent_worker: String,
|
parent_worker: String,
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ license.workspace = true
|
|||||||
api-macros = { workspace = true, features = ["reqwest", "axum"] }
|
api-macros = { workspace = true, features = ["reqwest", "axum"] }
|
||||||
manifest.workspace = true
|
manifest.workspace = true
|
||||||
protocol.workspace = true
|
protocol.workspace = true
|
||||||
|
session-store.workspace = true
|
||||||
serde = { workspace = true, features = ["derive"] }
|
serde = { workspace = true, features = ["derive"] }
|
||||||
workdir.workspace = true
|
workdir.workspace = true
|
||||||
|
|
||||||
|
|||||||
@@ -86,6 +86,8 @@ pub enum WorkerRestoreState {
|
|||||||
Accepted,
|
Accepted,
|
||||||
Rejected,
|
Rejected,
|
||||||
RolledBack,
|
RolledBack,
|
||||||
|
/// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.
|
||||||
|
Superseded,
|
||||||
ReconciliationRequired,
|
ReconciliationRequired,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -836,6 +838,34 @@ pub struct SessionMaterialPinResponse {
|
|||||||
pub pin_id: String,
|
pub pin_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// One body-bearing immutable slice. The byte budget includes slice metadata,
|
||||||
|
/// but not the enclosing availability enum or the caller's durable Job envelope.
|
||||||
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct SessionRecordingCaptureRequest {
|
||||||
|
pub workspace_id: String,
|
||||||
|
pub source: SessionPublicSource,
|
||||||
|
pub expected_session_id: String,
|
||||||
|
pub start: session_store::capture::RecordingPosition,
|
||||||
|
pub max_entries: usize,
|
||||||
|
pub max_serialized_bytes: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
|
#[serde(tag = "availability", rename_all = "snake_case")]
|
||||||
|
pub enum SessionRecordingCaptureAvailability {
|
||||||
|
NotMaterialized {
|
||||||
|
session_id: String,
|
||||||
|
},
|
||||||
|
Captured {
|
||||||
|
slice: session_store::capture::RecordingCaptureSlice,
|
||||||
|
},
|
||||||
|
Unavailable {
|
||||||
|
reason: SessionPublicUnavailableReason,
|
||||||
|
message: String,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct SessionMaterialCatalogRequest {
|
pub struct SessionMaterialCatalogRequest {
|
||||||
@@ -1246,6 +1276,16 @@ pub trait RuntimeApi {
|
|||||||
#[body] request: SessionMaterialPinRequest,
|
#[body] request: SessionMaterialPinRequest,
|
||||||
) -> Result<SessionMaterialPinResponse, RuntimeApiError>;
|
) -> Result<SessionMaterialPinResponse, RuntimeApiError>;
|
||||||
|
|
||||||
|
#[post(
|
||||||
|
"/v1/session-public/recording-capture",
|
||||||
|
status = 200,
|
||||||
|
error_status = 400
|
||||||
|
)]
|
||||||
|
async fn session_recording_capture(
|
||||||
|
&self,
|
||||||
|
#[body] request: SessionRecordingCaptureRequest,
|
||||||
|
) -> Result<SessionRecordingCaptureAvailability, RuntimeApiError>;
|
||||||
|
|
||||||
#[post(
|
#[post(
|
||||||
"/v1/session-public/material/catalog",
|
"/v1/session-public/material/catalog",
|
||||||
status = 200,
|
status = 200,
|
||||||
@@ -1540,13 +1580,24 @@ mod tests {
|
|||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(request.preparation.is_none());
|
assert!(request.preparation.is_none());
|
||||||
for state in [
|
for (state, wire) in [
|
||||||
"accepted",
|
(WorkerRestoreState::Accepted, "accepted"),
|
||||||
"rejected",
|
(WorkerRestoreState::Rejected, "rejected"),
|
||||||
"rolled_back",
|
(WorkerRestoreState::RolledBack, "rolled_back"),
|
||||||
"reconciliation_required",
|
(WorkerRestoreState::Superseded, "superseded"),
|
||||||
|
(
|
||||||
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"reconciliation_required",
|
||||||
|
),
|
||||||
] {
|
] {
|
||||||
assert!(serde_json::from_value::<WorkerRestoreState>(serde_json::json!(state)).is_ok());
|
assert_eq!(
|
||||||
|
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(wire)).unwrap(),
|
||||||
|
state
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(state).unwrap(),
|
||||||
|
serde_json::json!(wire)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
assert!(
|
assert!(
|
||||||
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(
|
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(
|
||||||
@@ -1730,6 +1781,13 @@ mod tests {
|
|||||||
Err(test_error(501))
|
Err(test_error(501))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn session_recording_capture(
|
||||||
|
&self,
|
||||||
|
_request: SessionRecordingCaptureRequest,
|
||||||
|
) -> Result<SessionRecordingCaptureAvailability, RuntimeApiError> {
|
||||||
|
Err(test_error(501))
|
||||||
|
}
|
||||||
|
|
||||||
async fn session_material_catalog(
|
async fn session_material_catalog(
|
||||||
&self,
|
&self,
|
||||||
_request: SessionMaterialCatalogRequest,
|
_request: SessionMaterialCatalogRequest,
|
||||||
|
|||||||
+139
-11
@@ -125,6 +125,8 @@ pub struct WorkdirOperationApiError {
|
|||||||
pub message: String,
|
pub message: String,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub diagnostics: Option<Vec<Diagnostic>>,
|
pub diagnostics: Option<Vec<Diagnostic>>,
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub audit_failures: Vec<workdir::http::WorkdirAuditFailure>,
|
||||||
#[serde(skip, default = "default_repository_error_status")]
|
#[serde(skip, default = "default_repository_error_status")]
|
||||||
status: u16,
|
status: u16,
|
||||||
/// Server-local metadata retained by generated error response extensions, never the wire body.
|
/// Server-local metadata retained by generated error response extensions, never the wire body.
|
||||||
@@ -156,6 +158,7 @@ impl WorkdirOperationApiError {
|
|||||||
code: None,
|
code: None,
|
||||||
message: error.message,
|
message: error.message,
|
||||||
diagnostics: Some(error.diagnostics),
|
diagnostics: Some(error.diagnostics),
|
||||||
|
audit_failures: Vec::new(),
|
||||||
status: error.status,
|
status: error.status,
|
||||||
workdir_log: None,
|
workdir_log: None,
|
||||||
denial_reason: None,
|
denial_reason: None,
|
||||||
@@ -170,6 +173,7 @@ impl WorkdirOperationApiError {
|
|||||||
code: Some(error.code),
|
code: Some(error.code),
|
||||||
message: error.message,
|
message: error.message,
|
||||||
diagnostics: None,
|
diagnostics: None,
|
||||||
|
audit_failures: Vec::new(),
|
||||||
status,
|
status,
|
||||||
workdir_log: None,
|
workdir_log: None,
|
||||||
denial_reason: error.denial_reason,
|
denial_reason: error.denial_reason,
|
||||||
@@ -225,6 +229,10 @@ impl_openapi_schema!(
|
|||||||
PasskeyLoginOptionsResponse,
|
PasskeyLoginOptionsResponse,
|
||||||
PasskeyLoginCompleteRequest,
|
PasskeyLoginCompleteRequest,
|
||||||
PasskeyAdditionOptionsRequest,
|
PasskeyAdditionOptionsRequest,
|
||||||
|
PasskeySummary,
|
||||||
|
PasskeyListResponse,
|
||||||
|
PasskeyAdditionResultRequest,
|
||||||
|
PasskeyAdditionResultResponse,
|
||||||
PasskeyRegistrationOptionsRequest,
|
PasskeyRegistrationOptionsRequest,
|
||||||
PasskeyRegistrationOptionsResponse,
|
PasskeyRegistrationOptionsResponse,
|
||||||
PasskeyRegistrationCompleteRequest,
|
PasskeyRegistrationCompleteRequest,
|
||||||
@@ -548,8 +556,9 @@ pub struct WorkspaceWorkerSessionHistoryResponse {
|
|||||||
/// comparisons do not replace admission checks under the Runtime operation lock; conflicts do
|
/// comparisons do not replace admission checks under the Runtime operation lock; conflicts do
|
||||||
/// not trigger automatic token replacement.
|
/// not trigger automatic token replacement.
|
||||||
/// [`WorkerRestoreState`] Accepted means Restore committed, not that the subsequent run succeeded.
|
/// [`WorkerRestoreState`] Accepted means Restore committed, not that the subsequent run succeeded.
|
||||||
/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects, and
|
/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects,
|
||||||
/// ReconciliationRequired needing settled-state checks. Response loss does not imply rollback.
|
/// Superseded by a later Stop, and ReconciliationRequired needing settled-state checks.
|
||||||
|
/// Response loss does not imply rollback.
|
||||||
#[api(reqwest, axum, openapi)]
|
#[api(reqwest, axum, openapi)]
|
||||||
pub trait ServerApi {
|
pub trait ServerApi {
|
||||||
#[get("/health", status = 200, error_status = 400)]
|
#[get("/health", status = 200, error_status = 400)]
|
||||||
@@ -648,6 +657,33 @@ pub trait ServerApi {
|
|||||||
#[body] request: PasskeyRegistrationCompleteRequest,
|
#[body] request: PasskeyRegistrationCompleteRequest,
|
||||||
) -> Result<server_api_responses::AuthPasskeyRegistrationComplete, RepositoryApiError>;
|
) -> Result<server_api_responses::AuthPasskeyRegistrationComplete, RepositoryApiError>;
|
||||||
|
|
||||||
|
#[get(
|
||||||
|
"/api/auth/passkeys",
|
||||||
|
status = 200,
|
||||||
|
error_status = 400,
|
||||||
|
additional_error_statuses = [403, 500],
|
||||||
|
bearer_auth = true,
|
||||||
|
browser_auth = true
|
||||||
|
)]
|
||||||
|
async fn auth_passkey_list(
|
||||||
|
&self,
|
||||||
|
#[extension] context: ServerRequestContext,
|
||||||
|
) -> Result<PasskeyListResponse, RepositoryApiError>;
|
||||||
|
|
||||||
|
#[post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
status = 200,
|
||||||
|
error_status = 400,
|
||||||
|
additional_error_statuses = [403, 500],
|
||||||
|
bearer_auth = true,
|
||||||
|
browser_auth = true
|
||||||
|
)]
|
||||||
|
async fn auth_passkey_addition_result(
|
||||||
|
&self,
|
||||||
|
#[extension] context: ServerRequestContext,
|
||||||
|
#[body] request: PasskeyAdditionResultRequest,
|
||||||
|
) -> Result<PasskeyAdditionResultResponse, RepositoryApiError>;
|
||||||
|
|
||||||
#[post(
|
#[post(
|
||||||
"/api/auth/passkeys/addition/options",
|
"/api/auth/passkeys/addition/options",
|
||||||
status = 200,
|
status = 200,
|
||||||
@@ -4115,6 +4151,8 @@ impl RequestActor {
|
|||||||
pub struct WhoamiResponse {
|
pub struct WhoamiResponse {
|
||||||
pub actor: Option<RequestActor>,
|
pub actor: Option<RequestActor>,
|
||||||
pub session: BrowserSessionState,
|
pub session: BrowserSessionState,
|
||||||
|
/// Non-secret comparison evidence, never an authentication credential.
|
||||||
|
pub authentication_reference: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Browser navigation evidence, not an authorization grant or a generic 401 classification.
|
/// Browser navigation evidence, not an authorization grant or a generic 401 classification.
|
||||||
@@ -4142,6 +4180,8 @@ pub struct AuthBootstrapUserRequest {
|
|||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct AuthUserResponse {
|
pub struct AuthUserResponse {
|
||||||
pub user: AuthenticatedUser,
|
pub user: AuthenticatedUser,
|
||||||
|
/// Comparison evidence for the exact newly issued session, not a credential.
|
||||||
|
pub authentication_reference: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Starts new-account registration only; an existing handle is never interpreted as key addition.
|
/// Starts new-account registration only; an existing handle is never interpreted as key addition.
|
||||||
@@ -4158,16 +4198,64 @@ pub struct PasskeyRegistrationOptionsRequest {
|
|||||||
pub browser_origin: Option<String>,
|
pub browser_origin: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Adds a credential to the current authenticated actor; target identity is never public input.
|
/// Adds to the current actor. Expected identity/binding fields only fence stale UI;
|
||||||
|
/// they never select an owner or authorize an addition.
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct PasskeyAdditionOptionsRequest {
|
pub struct PasskeyAdditionOptionsRequest {
|
||||||
|
pub expected_user_id: String,
|
||||||
|
pub expected_account_id: String,
|
||||||
|
pub expected_authentication_reference: String,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
|
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
|
||||||
pub browser_origin: Option<String>,
|
pub browser_origin: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Safe account metadata. Neither the WebAuthn credential ID nor key material is exposed.
|
||||||
|
/// Normal-login verification is observed verifier success, not inferred discoverability.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct PasskeySummary {
|
||||||
|
pub key_reference: String,
|
||||||
|
pub created_at: String,
|
||||||
|
pub last_used_at: Option<String>,
|
||||||
|
pub normal_login_verified_at: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct PasskeyListResponse {
|
||||||
|
pub user: AuthenticatedUser,
|
||||||
|
pub keys: Vec<PasskeySummary>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct PasskeyAdditionResultRequest {
|
||||||
|
pub challenge_id: String,
|
||||||
|
pub expected_user_id: String,
|
||||||
|
pub expected_account_id: String,
|
||||||
|
/// Original options authentication reference; the current same-owner session may differ.
|
||||||
|
pub expected_authentication_reference: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unknown is not proof that registration failed: completion may still be in flight.
|
||||||
|
/// Saved proves the credential commit, not session issuance or current session health.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)]
|
||||||
|
pub enum PasskeyAdditionResultResponse {
|
||||||
|
Unknown {},
|
||||||
|
Saved {
|
||||||
|
user: AuthenticatedUser,
|
||||||
|
key: PasskeySummary,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
@@ -6976,6 +7064,8 @@ pub struct WorkingDirectoryCleanupTarget {
|
|||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct WorkingDirectoryRemovalRequest {
|
pub struct WorkingDirectoryRemovalRequest {
|
||||||
|
/// Stable for an exact resend; a new explicit request uses a new identity.
|
||||||
|
pub operation_id: String,
|
||||||
pub reason: String,
|
pub reason: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -7510,7 +7600,13 @@ pub struct CurrentWorkerWorkdirAttachmentResponse {
|
|||||||
pub alias: String,
|
pub alias: String,
|
||||||
pub working_directory_id: String,
|
pub working_directory_id: String,
|
||||||
pub capabilities: workdir::WorkdirSessionCapabilities,
|
pub capabilities: workdir::WorkdirSessionCapabilities,
|
||||||
|
/// Present for attachment responses, absent after detach.
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub environment: Option<workdir::WorkdirSessionEnvironment>,
|
||||||
pub attached: bool,
|
pub attached: bool,
|
||||||
|
/// The Workdir was already attached to this Worker; alias is its existing name.
|
||||||
|
#[serde(default)]
|
||||||
|
pub already_attached: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
@@ -7522,7 +7618,9 @@ pub struct CurrentWorkerWorkdirOperationRequest {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
#[serde(transparent)]
|
#[serde(transparent)]
|
||||||
pub struct CurrentWorkerWorkdirOperationResponse(pub workdir::http::WorkdirSessionOperationResult);
|
pub struct CurrentWorkerWorkdirOperationResponse(
|
||||||
|
pub workdir::http::WorkdirSessionOperationResponse,
|
||||||
|
);
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
@@ -10570,8 +10668,8 @@ pub enum WorkerOperationState {
|
|||||||
/// Cross-layer outcome of an explicit Worker restore operation.
|
/// Cross-layer outcome of an explicit Worker restore operation.
|
||||||
///
|
///
|
||||||
/// `Rejected` is reserved for read-only preflight failures. Once live restore
|
/// `Rejected` is reserved for read-only preflight failures. Once live restore
|
||||||
/// work starts, failure is either a confirmed `RolledBack` operation or a
|
/// work starts, distinguish confirmed `RolledBack`, unresolved
|
||||||
/// `ReconciliationRequired` result whose commit state must be reread/retried.
|
/// `ReconciliationRequired`, and a later Stop's `Superseded` outcome.
|
||||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
@@ -10579,6 +10677,8 @@ pub enum WorkerRestoreState {
|
|||||||
Accepted,
|
Accepted,
|
||||||
Rejected,
|
Rejected,
|
||||||
RolledBack,
|
RolledBack,
|
||||||
|
/// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.
|
||||||
|
Superseded,
|
||||||
ReconciliationRequired,
|
ReconciliationRequired,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -11642,6 +11742,10 @@ pub fn auth_api_typescript() -> String {
|
|||||||
AuthBootstrapUserRequest::decl(&config),
|
AuthBootstrapUserRequest::decl(&config),
|
||||||
AuthUserResponse::decl(&config),
|
AuthUserResponse::decl(&config),
|
||||||
PasskeyAdditionOptionsRequest::decl(&config),
|
PasskeyAdditionOptionsRequest::decl(&config),
|
||||||
|
PasskeySummary::decl(&config),
|
||||||
|
PasskeyListResponse::decl(&config),
|
||||||
|
PasskeyAdditionResultRequest::decl(&config),
|
||||||
|
PasskeyAdditionResultResponse::decl(&config),
|
||||||
PasskeyRegistrationOptionsRequest::decl(&config),
|
PasskeyRegistrationOptionsRequest::decl(&config),
|
||||||
PasskeyRegistrationOptionsResponse::decl(&config),
|
PasskeyRegistrationOptionsResponse::decl(&config),
|
||||||
PasskeyRegistrationCompleteRequest::decl(&config),
|
PasskeyRegistrationCompleteRequest::decl(&config),
|
||||||
@@ -12672,6 +12776,7 @@ mod tests {
|
|||||||
(WorkerRestoreState::Accepted, "accepted"),
|
(WorkerRestoreState::Accepted, "accepted"),
|
||||||
(WorkerRestoreState::Rejected, "rejected"),
|
(WorkerRestoreState::Rejected, "rejected"),
|
||||||
(WorkerRestoreState::RolledBack, "rolled_back"),
|
(WorkerRestoreState::RolledBack, "rolled_back"),
|
||||||
|
(WorkerRestoreState::Superseded, "superseded"),
|
||||||
(
|
(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"reconciliation_required",
|
"reconciliation_required",
|
||||||
@@ -13280,10 +13385,13 @@ mod tests {
|
|||||||
alias: "checkout".to_string(),
|
alias: "checkout".to_string(),
|
||||||
working_directory_id: "workdir".to_string(),
|
working_directory_id: "workdir".to_string(),
|
||||||
capabilities: workdir::WorkdirSessionCapabilities::EMPTY,
|
capabilities: workdir::WorkdirSessionCapabilities::EMPTY,
|
||||||
|
environment: None,
|
||||||
attached: false,
|
attached: false,
|
||||||
|
already_attached: false,
|
||||||
};
|
};
|
||||||
let value = serde_json::to_value(&response).unwrap();
|
let value = serde_json::to_value(&response).unwrap();
|
||||||
assert_eq!(value.as_object().unwrap().len(), 5);
|
assert_eq!(value.as_object().unwrap().len(), 6);
|
||||||
|
assert_eq!(value["already_attached"], false);
|
||||||
assert!(value.get("connection_id").is_none());
|
assert!(value.get("connection_id").is_none());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
serde_json::from_value::<CurrentWorkerWorkdirAttachmentResponse>(value).unwrap(),
|
serde_json::from_value::<CurrentWorkerWorkdirAttachmentResponse>(value).unwrap(),
|
||||||
@@ -14069,7 +14177,8 @@ mod tests {
|
|||||||
"display_name": "Hare",
|
"display_name": "Hare",
|
||||||
"auth_method": "browser_session"
|
"auth_method": "browser_session"
|
||||||
},
|
},
|
||||||
"session": { "kind": "authenticated", "expires_at": "2026-10-20T00:00:00Z" }
|
"session": { "kind": "authenticated", "expires_at": "2026-10-20T00:00:00Z" },
|
||||||
|
"authentication_reference": "synthetic-session-reference"
|
||||||
});
|
});
|
||||||
let decoded = serde_json::from_value::<WhoamiResponse>(whoami.clone())
|
let decoded = serde_json::from_value::<WhoamiResponse>(whoami.clone())
|
||||||
.expect("server whoami fixture should match shared DTO");
|
.expect("server whoami fixture should match shared DTO");
|
||||||
@@ -14091,7 +14200,8 @@ mod tests {
|
|||||||
"account_id": "account-1",
|
"account_id": "account-1",
|
||||||
"handle": "hare",
|
"handle": "hare",
|
||||||
"display_name": "Hare"
|
"display_name": "Hare"
|
||||||
}
|
},
|
||||||
|
"authentication_reference": "synthetic-session-reference"
|
||||||
});
|
});
|
||||||
let decoded = serde_json::from_value::<AuthUserResponse>(auth_user.clone())
|
let decoded = serde_json::from_value::<AuthUserResponse>(auth_user.clone())
|
||||||
.expect("server auth-user fixture should match shared DTO");
|
.expect("server auth-user fixture should match shared DTO");
|
||||||
@@ -14264,7 +14374,7 @@ mod tests {
|
|||||||
serde_json::json!({"kind":"expired"}),
|
serde_json::json!({"kind":"expired"}),
|
||||||
serde_json::json!({"kind":"authenticated", "expires_at":null}),
|
serde_json::json!({"kind":"authenticated", "expires_at":null}),
|
||||||
] {
|
] {
|
||||||
let value = serde_json::json!({"actor":null, "session":session});
|
let value = serde_json::json!({"actor":null, "session":session, "authentication_reference": null});
|
||||||
let response: WhoamiResponse = serde_json::from_value(value.clone()).unwrap();
|
let response: WhoamiResponse = serde_json::from_value(value.clone()).unwrap();
|
||||||
assert_eq!(serde_json::to_value(response).unwrap(), value);
|
assert_eq!(serde_json::to_value(response).unwrap(), value);
|
||||||
}
|
}
|
||||||
@@ -14802,10 +14912,28 @@ mod openapi_artifact_tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
assert!(
|
assert!(
|
||||||
serde_json::from_value::<PasskeyAdditionOptionsRequest>(serde_json::json!({})).is_ok()
|
serde_json::from_value::<PasskeyAdditionOptionsRequest>(serde_json::json!({})).is_err()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn passkey_addition_requires_expected_owner_and_binding() {
|
||||||
|
let expected = serde_json::json!({
|
||||||
|
"expected_user_id": "user-a", "expected_account_id": "account-a",
|
||||||
|
"expected_authentication_reference": "reference"
|
||||||
|
});
|
||||||
|
assert!(serde_json::from_value::<PasskeyAdditionOptionsRequest>(expected.clone()).is_ok());
|
||||||
|
for field in [
|
||||||
|
"expected_user_id",
|
||||||
|
"expected_account_id",
|
||||||
|
"expected_authentication_reference",
|
||||||
|
] {
|
||||||
|
let mut missing = expected.clone();
|
||||||
|
missing.as_object_mut().unwrap().remove(field);
|
||||||
|
assert!(serde_json::from_value::<PasskeyAdditionOptionsRequest>(missing).is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn auth_workspace_catalog_and_typed_header_contract_is_strict() {
|
fn auth_workspace_catalog_and_typed_header_contract_is_strict() {
|
||||||
let document = canonical_openapi_document().expect("canonical OpenAPI contract must build");
|
let document = canonical_openapi_document().expect("canonical OpenAPI contract must build");
|
||||||
|
|||||||
@@ -11,5 +11,7 @@ checks or Runtime catalog lifecycle management.
|
|||||||
- [`worker_session_store`]: storage boundary for a Session bound to a Worker
|
- [`worker_session_store`]: storage boundary for a Session bound to a Worker
|
||||||
- [`worker_metadata`]: current Worker state and active Segment switching
|
- [`worker_metadata`]: current Worker state and active Segment switching
|
||||||
- [`system_item`]: records of system input presented to the model
|
- [`system_item`]: records of system input presented to the model
|
||||||
|
- [`capture`]: Worker-independent immutable public values and bounded body transport;
|
||||||
|
authorization, source I/O, and delivery lifecycle stay with the Host
|
||||||
- [`public_snapshot`] / [`public_index`]: public projections of stored history
|
- [`public_snapshot`] / [`public_index`]: public projections of stored history
|
||||||
- [`event_trace`]: diagnostic records independent of history replay
|
- [`event_trace`]: diagnostic records independent of history replay
|
||||||
|
|||||||
@@ -0,0 +1,967 @@
|
|||||||
|
//! Exploration domain for immutable Session captures, with no Workspace/Memory mutation capability.
|
||||||
|
//!
|
||||||
|
//! A capture is an ordered projection of committed entries; exploration excludes reasoning and
|
||||||
|
//! system prompts, including Host compaction summaries and other derived context. Full inherited
|
||||||
|
//! summary input belongs to compaction, not public observation or Memory evidence. Overview,
|
||||||
|
//! search, bounded reads, and evidence conversion share `SessionEntryRef`.
|
||||||
|
//! `SessionEntryRef` is a contextual UI reference. `ScopedSessionEntryRef` additionally binds
|
||||||
|
//! the Session and exact immutable capture, and is required for portable checked operations.
|
||||||
|
//! Appends preserve existing entry identities. Sparse real user/assistant anchors and intervening
|
||||||
|
//! entry counts index long histories. Search with an empty filter also provides bounded range lists.
|
||||||
|
//!
|
||||||
|
//! Host-fixed snapshot exploration, Memory candidate evidence validation, and current Worker
|
||||||
|
//! observation share only this domain; they do not enable or modify each other's Features.
|
||||||
|
//! The provider owns authorization to recapture a current Worker. A capture itself proves neither
|
||||||
|
//! completion nor approval and grants no business-mutation authority.
|
||||||
|
//!
|
||||||
|
//! Recording Hosts use the common committed replay through the guarded APIs below.
|
||||||
|
//! [`recording_catalog_worker_guarded`] and [`recording_catalog_fs_guarded`] expose
|
||||||
|
//! descriptors only to an admission callback under the caller's existing publication/
|
||||||
|
//! erasure guard and persisted active selector, without allocating a full body DTO.
|
||||||
|
//! [`capture_recording_slice_worker_guarded`] and [`capture_recording_slice_fs_guarded`]
|
||||||
|
//! separately construct bounded body-bearing inputs. Admission descriptors are not
|
||||||
|
//! sufficient Job inputs; saved recording DTOs are read/sliced without live storage.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
LoggedSessionHistoryMetadata as SessionHistoryMetadata,
|
||||||
|
LoggedSessionHistoryOrigin as HistoryOrigin,
|
||||||
|
};
|
||||||
|
mod body;
|
||||||
|
mod recording;
|
||||||
|
use agen::{HistoryEntry, Item, Role};
|
||||||
|
pub use body::*;
|
||||||
|
use protocol::{
|
||||||
|
SessionContentPart, SessionEntryProvenance, SessionMessageRole, SessionSnapshot,
|
||||||
|
SessionSnapshotEntryData,
|
||||||
|
};
|
||||||
|
pub use recording::*;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
const DEFAULT_SEARCH_LIMIT: usize = 20;
|
||||||
|
const MAX_SEARCH_LIMIT: usize = 50;
|
||||||
|
const DEFAULT_READ_MAX_ITEMS: usize = 40;
|
||||||
|
const MAX_READ_MAX_ITEMS: usize = 80;
|
||||||
|
const DEFAULT_READ_MAX_BYTES: usize = 32 * 1024;
|
||||||
|
const OVERVIEW_ANCHOR_STRIDE: usize = 8;
|
||||||
|
|
||||||
|
/// Append-stable contextual UI reference derived from a logical entry identity.
|
||||||
|
/// Portable consumers must bind it with `SessionCapture::scoped_ref` before use.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||||
|
#[serde(transparent)]
|
||||||
|
pub struct SessionEntryRef(String);
|
||||||
|
|
||||||
|
impl SessionEntryRef {
|
||||||
|
pub fn from_history_entry_id(entry_id: &crate::LoggedSessionHistoryEntryId) -> Self {
|
||||||
|
Self(format!("E{}", entry_id.0))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn parse(value: &str) -> Option<Self> {
|
||||||
|
let suffix = value.strip_prefix('E')?;
|
||||||
|
if suffix.is_empty()
|
||||||
|
// Legacy Host boundary identities contain both Session and Segment UUIDs.
|
||||||
|
|| suffix.len() > 256
|
||||||
|
|| !suffix
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(Self(value.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn as_str(&self) -> &str {
|
||||||
|
&self.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for SessionEntryRef {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
formatter.write_str(&self.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum ReferenceKind {
|
||||||
|
User,
|
||||||
|
Assistant,
|
||||||
|
System,
|
||||||
|
Tool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ReferenceKind {
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::User => "user",
|
||||||
|
Self::Assistant => "assistant",
|
||||||
|
Self::System => "system",
|
||||||
|
Self::Tool => "tool",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn parse(value: &str) -> Option<Self> {
|
||||||
|
match value {
|
||||||
|
"user" => Some(Self::User),
|
||||||
|
"assistant" | "agent" => Some(Self::Assistant),
|
||||||
|
"system" => Some(Self::System),
|
||||||
|
"tool" => Some(Self::Tool),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum ToolPart {
|
||||||
|
Input,
|
||||||
|
Output,
|
||||||
|
Both,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ToolPart {
|
||||||
|
pub fn parse(value: &str) -> Option<Self> {
|
||||||
|
match value {
|
||||||
|
"input" => Some(Self::Input),
|
||||||
|
"output" => Some(Self::Output),
|
||||||
|
"both" => Some(Self::Both),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Input => "input",
|
||||||
|
Self::Output => "output",
|
||||||
|
Self::Both => "both",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn matches(self, actual: ToolPart) -> bool {
|
||||||
|
matches!(self, ToolPart::Both) || self == actual
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct OverviewItem {
|
||||||
|
pub id: SessionEntryRef,
|
||||||
|
pub origin: SessionEntryProvenance,
|
||||||
|
pub entry_range: [u64; 2],
|
||||||
|
pub kind: ReferenceKind,
|
||||||
|
pub label: String,
|
||||||
|
pub text: String,
|
||||||
|
pub intervening_entries: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ReferenceEntry {
|
||||||
|
pub attachment_count: usize,
|
||||||
|
use_policy: Option<protocol::SessionEntryUsePolicy>,
|
||||||
|
pub id: SessionEntryRef,
|
||||||
|
pub origin: SessionEntryProvenance,
|
||||||
|
pub entry_range: [u64; 2],
|
||||||
|
pub kind: ReferenceKind,
|
||||||
|
pub tool_part: Option<ToolPart>,
|
||||||
|
pub tool_name: Option<String>,
|
||||||
|
pub label: String,
|
||||||
|
pub summary: String,
|
||||||
|
search_text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub struct SearchOptions {
|
||||||
|
pub query: String,
|
||||||
|
pub kind: Option<ReferenceKind>,
|
||||||
|
pub tool_part: Option<ToolPart>,
|
||||||
|
pub tool_name: Option<String>,
|
||||||
|
pub limit: Option<usize>,
|
||||||
|
pub min_entry_index: Option<u64>,
|
||||||
|
pub from: Option<SessionEntryRef>,
|
||||||
|
pub through: Option<SessionEntryRef>,
|
||||||
|
pub offset: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct SearchHit {
|
||||||
|
pub id: SessionEntryRef,
|
||||||
|
pub origin: SessionEntryProvenance,
|
||||||
|
pub kind: ReferenceKind,
|
||||||
|
pub tool_part: Option<ToolPart>,
|
||||||
|
pub tool_name: Option<String>,
|
||||||
|
pub entry_range: [u64; 2],
|
||||||
|
pub label: String,
|
||||||
|
pub summary: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub enum ReadSelector<'a> {
|
||||||
|
Id(&'a str),
|
||||||
|
EntryRange([u64; 2]),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum ReadDetail {
|
||||||
|
Compact,
|
||||||
|
Full,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ReadOptions {
|
||||||
|
pub include_tools: bool,
|
||||||
|
pub tool_part: ToolPart,
|
||||||
|
pub detail: ReadDetail,
|
||||||
|
pub max_items: usize,
|
||||||
|
pub max_bytes: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ReadOptions {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
include_tools: true,
|
||||||
|
tool_part: ToolPart::Both,
|
||||||
|
detail: ReadDetail::Compact,
|
||||||
|
max_items: DEFAULT_READ_MAX_ITEMS,
|
||||||
|
max_bytes: DEFAULT_READ_MAX_BYTES,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ReadEntry {
|
||||||
|
pub id: SessionEntryRef,
|
||||||
|
pub origin: SessionEntryProvenance,
|
||||||
|
pub kind: ReferenceKind,
|
||||||
|
pub tool_part: Option<ToolPart>,
|
||||||
|
pub tool_name: Option<String>,
|
||||||
|
pub entry_range: [u64; 2],
|
||||||
|
pub label: String,
|
||||||
|
pub text: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ReadResult {
|
||||||
|
pub entries: Vec<ReadEntry>,
|
||||||
|
pub truncated: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct SessionEntryEvidence {
|
||||||
|
pub segment_id: String,
|
||||||
|
pub entry_ref: SessionEntryRef,
|
||||||
|
pub origin: SessionEntryProvenance,
|
||||||
|
/// Inclusive original-source positions in `segment_id`, not view/UI order.
|
||||||
|
pub entry_range: [u64; 2],
|
||||||
|
pub kind: ReferenceKind,
|
||||||
|
pub tool_part: Option<ToolPart>,
|
||||||
|
pub label: String,
|
||||||
|
pub summary: String,
|
||||||
|
pub excerpt: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct CapturedHistoryEntry {
|
||||||
|
use_policy: Option<protocol::SessionEntryUsePolicy>,
|
||||||
|
item: Item,
|
||||||
|
entry_id: crate::LoggedSessionHistoryEntryId,
|
||||||
|
origin: SessionEntryProvenance,
|
||||||
|
dto: CaptureEntryDto,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct SessionCapture {
|
||||||
|
session_id: String,
|
||||||
|
segment_id: String,
|
||||||
|
capture_id: String,
|
||||||
|
view_entry_count: u64,
|
||||||
|
entries: Arc<Vec<CapturedHistoryEntry>>,
|
||||||
|
overview: Vec<OverviewItem>,
|
||||||
|
index: Vec<ReferenceEntry>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SessionCapture {
|
||||||
|
pub fn from_session_snapshot(
|
||||||
|
session_id: impl Into<String>,
|
||||||
|
segment_id: impl Into<String>,
|
||||||
|
snapshot: SessionSnapshot,
|
||||||
|
) -> Self {
|
||||||
|
let segment_id = segment_id.into();
|
||||||
|
let view_entry_count = snapshot.entries.len() as u64;
|
||||||
|
let entries = snapshot
|
||||||
|
.entries
|
||||||
|
.into_iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter_map(|(index, entry)| {
|
||||||
|
let policy = CaptureUsePolicy {
|
||||||
|
recording_material: false,
|
||||||
|
memory_evidence: entry.use_policy.is_none_or(|p| p.memory_evidence),
|
||||||
|
observation_readable: entry.use_policy.is_none_or(|p| p.observation_readable),
|
||||||
|
};
|
||||||
|
let explicit = entry.use_policy.is_some();
|
||||||
|
let item = match entry.data {
|
||||||
|
SessionSnapshotEntryData::UserInput { segments } => {
|
||||||
|
Item::user_message(protocol::Segment::flatten_to_text(&segments))
|
||||||
|
}
|
||||||
|
SessionSnapshotEntryData::Message { role, content } => Item::Message {
|
||||||
|
id: None,
|
||||||
|
status: None,
|
||||||
|
role: match role {
|
||||||
|
SessionMessageRole::User => Role::User,
|
||||||
|
SessionMessageRole::Assistant => Role::Assistant,
|
||||||
|
},
|
||||||
|
content: content
|
||||||
|
.into_iter()
|
||||||
|
.map(|part| match part {
|
||||||
|
SessionContentPart::Text { text } => {
|
||||||
|
agen::ContentPart::Text { text }
|
||||||
|
}
|
||||||
|
SessionContentPart::Refusal { refusal } => {
|
||||||
|
agen::ContentPart::Refusal { refusal }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
},
|
||||||
|
SessionSnapshotEntryData::ToolCall {
|
||||||
|
call_id,
|
||||||
|
name,
|
||||||
|
arguments,
|
||||||
|
} => Item::tool_call(call_id, name, arguments),
|
||||||
|
SessionSnapshotEntryData::ToolResult {
|
||||||
|
call_id,
|
||||||
|
summary,
|
||||||
|
content,
|
||||||
|
is_error,
|
||||||
|
..
|
||||||
|
} => Item::tool_result_item(call_id, summary, content, is_error),
|
||||||
|
SessionSnapshotEntryData::SystemItem { content, .. } if explicit => {
|
||||||
|
Item::system_message(content)
|
||||||
|
}
|
||||||
|
_ => return None,
|
||||||
|
};
|
||||||
|
let body = CaptureBody::project(&item, &entry.provenance, explicit)?;
|
||||||
|
if !policy.any() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(CaptureEntryDto {
|
||||||
|
entry_id: entry.entry_id,
|
||||||
|
order: index as u64,
|
||||||
|
source: CaptureSource {
|
||||||
|
segment_id: segment_id.clone(),
|
||||||
|
entry_index: index as u64,
|
||||||
|
provenance: entry.provenance,
|
||||||
|
history_origin: None,
|
||||||
|
derived_from: entry.derived_from,
|
||||||
|
},
|
||||||
|
policy,
|
||||||
|
body,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
Self::from_projected(session_id.into(), segment_id, view_entry_count, entries)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn new(
|
||||||
|
session_id: impl Into<String>,
|
||||||
|
segment_id: impl Into<String>,
|
||||||
|
items: Vec<Item>,
|
||||||
|
) -> Self {
|
||||||
|
let segment_id = segment_id.into();
|
||||||
|
let view_entry_count = items.len() as u64;
|
||||||
|
let entries = items
|
||||||
|
.into_iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter_map(|(index, item)| {
|
||||||
|
let origin = SessionEntryProvenance::LegacyUnknown;
|
||||||
|
let body = CaptureBody::project(&item, &origin, false)?;
|
||||||
|
Some(CaptureEntryDto {
|
||||||
|
entry_id: format!("{index:08}"),
|
||||||
|
order: index as u64,
|
||||||
|
source: CaptureSource {
|
||||||
|
segment_id: segment_id.clone(),
|
||||||
|
entry_index: index as u64,
|
||||||
|
provenance: origin,
|
||||||
|
history_origin: None,
|
||||||
|
derived_from: Vec::new(),
|
||||||
|
},
|
||||||
|
policy: CaptureUsePolicy::PUBLIC,
|
||||||
|
body,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
Self::from_projected(session_id.into(), segment_id, view_entry_count, entries)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Project a complete Host-fixed history whose source positions start at zero.
|
||||||
|
/// Select a suffix with `select_order_range` on the resulting value instead
|
||||||
|
/// of passing a rebased history vector: annotations do not carry source positions.
|
||||||
|
pub fn from_complete_history(
|
||||||
|
session_id: impl Into<String>,
|
||||||
|
segment_id: impl Into<String>,
|
||||||
|
entries: Vec<HistoryEntry<SessionHistoryMetadata>>,
|
||||||
|
) -> Self {
|
||||||
|
let segment_id = segment_id.into();
|
||||||
|
let view_entry_count = entries.len() as u64;
|
||||||
|
let entries = entries
|
||||||
|
.into_iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter_map(|(index, entry)| {
|
||||||
|
let origin = public_provenance(&entry.annotation.origin);
|
||||||
|
let mut item = entry.item;
|
||||||
|
let mut explicit = false;
|
||||||
|
let policy = if let Some(input) = entry.annotation.input_metadata {
|
||||||
|
explicit = true;
|
||||||
|
if matches!(
|
||||||
|
item,
|
||||||
|
Item::Message {
|
||||||
|
role: Role::System,
|
||||||
|
..
|
||||||
|
}
|
||||||
|
) {
|
||||||
|
// Only the explicit public projection crosses this boundary.
|
||||||
|
let public = input.public_item()?;
|
||||||
|
item = Item::system_message(public.body.clone());
|
||||||
|
} else if input.feature_payload().is_some() || input.public_item().is_some() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
CaptureUsePolicy::from(input.usage())
|
||||||
|
} else {
|
||||||
|
if matches!(entry.annotation.origin, HistoryOrigin::Provided { .. }) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
CaptureUsePolicy::PUBLIC
|
||||||
|
};
|
||||||
|
if !policy.any() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let body = CaptureBody::project(&item, &origin, explicit)?;
|
||||||
|
Some(CaptureEntryDto {
|
||||||
|
entry_id: entry.annotation.entry_id.0,
|
||||||
|
order: index as u64,
|
||||||
|
source: CaptureSource {
|
||||||
|
segment_id: segment_id.clone(),
|
||||||
|
entry_index: index as u64,
|
||||||
|
provenance: origin,
|
||||||
|
history_origin: Some(entry.annotation.origin),
|
||||||
|
derived_from: entry
|
||||||
|
.annotation
|
||||||
|
.derivation
|
||||||
|
.map(|d| d.sources.into_iter().map(|id| id.0).collect())
|
||||||
|
.unwrap_or_default(),
|
||||||
|
},
|
||||||
|
policy,
|
||||||
|
body,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
Self::from_projected(session_id.into(), segment_id, view_entry_count, entries)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_projected(
|
||||||
|
session_id: String,
|
||||||
|
segment_id: String,
|
||||||
|
view_entry_count: u64,
|
||||||
|
dtos: Vec<CaptureEntryDto>,
|
||||||
|
) -> Self {
|
||||||
|
let dto = CaptureDto {
|
||||||
|
version: CAPTURE_VERSION,
|
||||||
|
session_id: session_id.clone(),
|
||||||
|
segment_id: segment_id.clone(),
|
||||||
|
view_entry_count,
|
||||||
|
entries: dtos,
|
||||||
|
};
|
||||||
|
let capture_id = dto.fingerprint();
|
||||||
|
let entries: Vec<_> = dto
|
||||||
|
.entries
|
||||||
|
.into_iter()
|
||||||
|
.map(|dto| CapturedHistoryEntry {
|
||||||
|
use_policy: Some(protocol::SessionEntryUsePolicy {
|
||||||
|
observation_readable: dto.policy.observation_readable,
|
||||||
|
memory_evidence: dto.policy.memory_evidence,
|
||||||
|
}),
|
||||||
|
item: dto.body.to_item(),
|
||||||
|
entry_id: crate::LoggedSessionHistoryEntryId(dto.entry_id.clone()),
|
||||||
|
origin: dto.source.provenance.clone(),
|
||||||
|
dto,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let mut overview = Vec::new();
|
||||||
|
let mut index = Vec::new();
|
||||||
|
|
||||||
|
for entry in &entries {
|
||||||
|
let item = &entry.item;
|
||||||
|
let entry_range = [entry.dto.order; 2];
|
||||||
|
let attachment_count = match entry.dto.body {
|
||||||
|
CaptureBody::ToolResult {
|
||||||
|
attachment_count, ..
|
||||||
|
} => attachment_count,
|
||||||
|
_ => 0,
|
||||||
|
};
|
||||||
|
match item {
|
||||||
|
Item::Message { role, content, .. } => {
|
||||||
|
let Some(kind) = message_reference_kind(&entry.origin, role, entry.use_policy)
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let text = content
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.as_text())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("");
|
||||||
|
let label = format!("{} message", kind.as_str());
|
||||||
|
let summary = truncate_chars(&text, 240);
|
||||||
|
let id = SessionEntryRef::from_history_entry_id(&entry.entry_id);
|
||||||
|
index.push(ReferenceEntry {
|
||||||
|
use_policy: entry.use_policy,
|
||||||
|
attachment_count,
|
||||||
|
id: id.clone(),
|
||||||
|
origin: entry.origin.clone(),
|
||||||
|
entry_range,
|
||||||
|
kind,
|
||||||
|
tool_part: None,
|
||||||
|
tool_name: None,
|
||||||
|
label: label.clone(),
|
||||||
|
summary: summary.clone(),
|
||||||
|
search_text: text.clone(),
|
||||||
|
});
|
||||||
|
if entry.use_policy.is_none_or(|p| p.observation_readable)
|
||||||
|
&& matches!(kind, ReferenceKind::User | ReferenceKind::Assistant)
|
||||||
|
{
|
||||||
|
overview.push(OverviewItem {
|
||||||
|
id: id.clone(),
|
||||||
|
origin: entry.origin.clone(),
|
||||||
|
entry_range,
|
||||||
|
kind,
|
||||||
|
label,
|
||||||
|
text,
|
||||||
|
intervening_entries: 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Item::ToolCall {
|
||||||
|
name, arguments, ..
|
||||||
|
} => {
|
||||||
|
let text = format!("{name}\n{arguments}");
|
||||||
|
index.push(ReferenceEntry {
|
||||||
|
use_policy: entry.use_policy,
|
||||||
|
attachment_count,
|
||||||
|
id: SessionEntryRef::from_history_entry_id(&entry.entry_id),
|
||||||
|
origin: entry.origin.clone(),
|
||||||
|
entry_range,
|
||||||
|
kind: ReferenceKind::Tool,
|
||||||
|
tool_part: Some(ToolPart::Input),
|
||||||
|
tool_name: Some(name.clone()),
|
||||||
|
label: format!("tool input: {name}"),
|
||||||
|
summary: format!("Tool call {name}"),
|
||||||
|
search_text: text,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Item::ToolResult {
|
||||||
|
summary,
|
||||||
|
content,
|
||||||
|
attachments: _,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
let attachment_marker = if attachment_count == 0 {
|
||||||
|
String::new()
|
||||||
|
} else {
|
||||||
|
format!("\n[{} image attachment(s)]", attachment_count)
|
||||||
|
};
|
||||||
|
let text = format!(
|
||||||
|
"{summary}\n{}{attachment_marker}",
|
||||||
|
content.as_deref().unwrap_or_default(),
|
||||||
|
);
|
||||||
|
index.push(ReferenceEntry {
|
||||||
|
use_policy: entry.use_policy,
|
||||||
|
attachment_count,
|
||||||
|
id: SessionEntryRef::from_history_entry_id(&entry.entry_id),
|
||||||
|
origin: entry.origin.clone(),
|
||||||
|
entry_range,
|
||||||
|
kind: ReferenceKind::Tool,
|
||||||
|
tool_part: Some(ToolPart::Output),
|
||||||
|
tool_name: None,
|
||||||
|
label: "tool output".to_string(),
|
||||||
|
summary: truncate_chars(summary, 240),
|
||||||
|
search_text: text,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Item::AssistantResponseBoundary { .. } | Item::Reasoning { .. } => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if overview.len() > 2 {
|
||||||
|
let last = overview.len() - 1;
|
||||||
|
overview = overview
|
||||||
|
.into_iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter_map(|(index, entry)| {
|
||||||
|
(index == 0 || index == last || index % OVERVIEW_ANCHOR_STRIDE == 0)
|
||||||
|
.then_some(entry)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
}
|
||||||
|
|
||||||
|
for overview_index in 0..overview.len().saturating_sub(1) {
|
||||||
|
let current_entry = overview[overview_index].entry_range[0];
|
||||||
|
let next_entry = overview[overview_index + 1].entry_range[0];
|
||||||
|
overview[overview_index].intervening_entries = index
|
||||||
|
.iter()
|
||||||
|
.filter(|entry| {
|
||||||
|
let entry_index = entry.entry_range[0];
|
||||||
|
entry_index > current_entry && entry_index < next_entry
|
||||||
|
})
|
||||||
|
.count();
|
||||||
|
}
|
||||||
|
|
||||||
|
Self {
|
||||||
|
session_id,
|
||||||
|
segment_id,
|
||||||
|
capture_id,
|
||||||
|
view_entry_count,
|
||||||
|
entries: Arc::new(entries),
|
||||||
|
overview,
|
||||||
|
index,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Public source entries are immutable; callers retain their own business decisions.
|
||||||
|
pub fn entries(&self) -> impl ExactSizeIterator<Item = &CaptureEntryDto> {
|
||||||
|
self.entries.iter().map(|entry| &entry.dto)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn overview(&self) -> &[OverviewItem] {
|
||||||
|
&self.overview
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Capture-local UI position, not the original source position used by evidence.
|
||||||
|
pub fn view_order_for_ref(&self, reference: &SessionEntryRef) -> Option<u64> {
|
||||||
|
self.index
|
||||||
|
.iter()
|
||||||
|
.find(|entry| entry.id == *reference)
|
||||||
|
.map(|entry| entry.entry_range[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn search(&self, options: &SearchOptions) -> Vec<SearchHit> {
|
||||||
|
if self.validate_search(options).is_err() {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
let query = options.query.trim().to_lowercase();
|
||||||
|
let limit = options
|
||||||
|
.limit
|
||||||
|
.unwrap_or(DEFAULT_SEARCH_LIMIT)
|
||||||
|
.clamp(1, MAX_SEARCH_LIMIT);
|
||||||
|
let tool_name = options.tool_name.as_deref();
|
||||||
|
let min_entry_index = options
|
||||||
|
.from
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|reference| self.view_order_for_ref(reference))
|
||||||
|
.unwrap_or_else(|| options.min_entry_index.unwrap_or(0));
|
||||||
|
let max_entry_index = options
|
||||||
|
.through
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|reference| self.view_order_for_ref(reference))
|
||||||
|
.unwrap_or(u64::MAX);
|
||||||
|
let mut skipped = 0usize;
|
||||||
|
let mut hits = Vec::new();
|
||||||
|
|
||||||
|
for entry in &self.index {
|
||||||
|
if entry.use_policy.is_some_and(|p| !p.observation_readable) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if entry.entry_range[0] < min_entry_index || entry.entry_range[0] > max_entry_index {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Some(kind) = options.kind {
|
||||||
|
if entry.kind != kind {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(part) = options.tool_part {
|
||||||
|
if entry.kind != ReferenceKind::Tool {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(actual) = entry.tool_part else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if !part.matches(actual) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(name) = tool_name {
|
||||||
|
if entry.tool_name.as_deref() != Some(name) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !query.is_empty() && !entry.search_text.to_lowercase().contains(&query) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if skipped < options.offset {
|
||||||
|
skipped += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
hits.push(SearchHit {
|
||||||
|
id: entry.id.clone(),
|
||||||
|
origin: entry.origin.clone(),
|
||||||
|
kind: entry.kind,
|
||||||
|
tool_part: entry.tool_part,
|
||||||
|
tool_name: entry.tool_name.clone(),
|
||||||
|
entry_range: entry.entry_range,
|
||||||
|
label: entry.label.clone(),
|
||||||
|
summary: entry.summary.clone(),
|
||||||
|
});
|
||||||
|
if hits.len() >= limit {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
hits
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn read(&self, selector: ReadSelector<'_>, options: ReadOptions) -> ReadResult {
|
||||||
|
let max_items = options.max_items.clamp(1, MAX_READ_MAX_ITEMS);
|
||||||
|
let max_bytes = options.max_bytes.max(1);
|
||||||
|
let mut entries = Vec::new();
|
||||||
|
let mut bytes = 0usize;
|
||||||
|
let mut truncated = false;
|
||||||
|
|
||||||
|
let selected: Vec<&ReferenceEntry> = match selector {
|
||||||
|
ReadSelector::Id(id) => self
|
||||||
|
.index
|
||||||
|
.iter()
|
||||||
|
.filter(|entry| entry.id.as_str() == id)
|
||||||
|
.collect(),
|
||||||
|
ReadSelector::EntryRange([start, end]) => self
|
||||||
|
.index
|
||||||
|
.iter()
|
||||||
|
.filter(|entry| entry.entry_range[0] >= start && entry.entry_range[0] <= end)
|
||||||
|
.collect(),
|
||||||
|
};
|
||||||
|
|
||||||
|
for entry in selected {
|
||||||
|
if entry.use_policy.is_some_and(|p| !p.observation_readable) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if entries.len() >= max_items {
|
||||||
|
truncated = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if entry.kind == ReferenceKind::Tool {
|
||||||
|
if !options.include_tools {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Some(actual) = entry.tool_part {
|
||||||
|
if !options.tool_part.matches(actual) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let Some(item) = self
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.find(|captured| captured.dto.order == entry.entry_range[0])
|
||||||
|
.map(|entry| &entry.item)
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let remaining = max_bytes.saturating_sub(bytes);
|
||||||
|
if remaining == 0 {
|
||||||
|
truncated = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let rendered = render_item(item, entry, options.detail, usize::MAX);
|
||||||
|
truncated |= rendered.len() > remaining;
|
||||||
|
let text = truncate_bytes(&rendered, remaining);
|
||||||
|
bytes = bytes.saturating_add(text.len());
|
||||||
|
entries.push(ReadEntry {
|
||||||
|
id: entry.id.clone(),
|
||||||
|
origin: entry.origin.clone(),
|
||||||
|
kind: entry.kind,
|
||||||
|
tool_part: entry.tool_part,
|
||||||
|
tool_name: entry.tool_name.clone(),
|
||||||
|
entry_range: entry.entry_range,
|
||||||
|
label: entry.label.clone(),
|
||||||
|
text,
|
||||||
|
});
|
||||||
|
if truncated {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ReadResult { entries, truncated }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolves the committed tool-call input for a provider call id. This is
|
||||||
|
/// used by explicit-memory receipts only after the containing run is durable,
|
||||||
|
/// so a model-authored claim can cite its own committed tool invocation
|
||||||
|
/// without inventing a SessionEntryRef.
|
||||||
|
pub fn evidence_for_tool_call(&self, call_id: &str) -> Option<SessionEntryEvidence> {
|
||||||
|
let captured = self.entries.iter().find(|entry| {
|
||||||
|
matches!(
|
||||||
|
&entry.item,
|
||||||
|
Item::ToolCall {
|
||||||
|
call_id: stored_call_id,
|
||||||
|
..
|
||||||
|
} if stored_call_id == call_id
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let reference = self.index.iter().find(|entry| {
|
||||||
|
entry.entry_range[0] == captured.dto.order
|
||||||
|
&& entry.kind == ReferenceKind::Tool
|
||||||
|
&& entry.tool_part == Some(ToolPart::Input)
|
||||||
|
})?;
|
||||||
|
self.evidence_for(reference.id.as_str())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn evidence_for(&self, id: &str) -> Option<SessionEntryEvidence> {
|
||||||
|
let entry = self.index.iter().find(|entry| entry.id.as_str() == id)?;
|
||||||
|
if entry.use_policy.is_some_and(|p| !p.memory_evidence) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let captured = self
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.find(|captured| captured.dto.order == entry.entry_range[0])?;
|
||||||
|
let excerpt = render_item(&captured.item, entry, ReadDetail::Compact, 2 * 1024);
|
||||||
|
Some(SessionEntryEvidence {
|
||||||
|
segment_id: captured.dto.source.segment_id.clone(),
|
||||||
|
entry_ref: entry.id.clone(),
|
||||||
|
origin: entry.origin.clone(),
|
||||||
|
entry_range: [captured.dto.source.entry_index; 2],
|
||||||
|
kind: entry.kind,
|
||||||
|
tool_part: entry.tool_part,
|
||||||
|
label: entry.label.clone(),
|
||||||
|
summary: entry.summary.clone(),
|
||||||
|
excerpt,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn public_provenance(origin: &HistoryOrigin) -> SessionEntryProvenance {
|
||||||
|
match origin {
|
||||||
|
HistoryOrigin::HumanInput { .. } | HistoryOrigin::LocalHumanInput => {
|
||||||
|
SessionEntryProvenance::HumanInput
|
||||||
|
}
|
||||||
|
HistoryOrigin::WorkerInput { .. } => SessionEntryProvenance::WorkerInput,
|
||||||
|
HistoryOrigin::FlowInstruction { .. } => SessionEntryProvenance::FlowInstruction,
|
||||||
|
HistoryOrigin::BackendInstruction { .. } | HistoryOrigin::Provided { .. } => {
|
||||||
|
SessionEntryProvenance::BackendInstruction
|
||||||
|
}
|
||||||
|
HistoryOrigin::ModelOutput { .. } => SessionEntryProvenance::ModelOutput,
|
||||||
|
HistoryOrigin::ToolOutput { .. } => SessionEntryProvenance::ToolOutput,
|
||||||
|
HistoryOrigin::CompactionSummary
|
||||||
|
| HistoryOrigin::DerivedSummary
|
||||||
|
| HistoryOrigin::JournalContext => SessionEntryProvenance::DerivedSummary,
|
||||||
|
HistoryOrigin::LegacyUnknown => SessionEntryProvenance::LegacyUnknown,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn message_reference_kind(
|
||||||
|
origin: &SessionEntryProvenance,
|
||||||
|
provider_role: &Role,
|
||||||
|
use_policy: Option<protocol::SessionEntryUsePolicy>,
|
||||||
|
) -> Option<ReferenceKind> {
|
||||||
|
if use_policy.is_some() {
|
||||||
|
return match provider_role {
|
||||||
|
Role::System => Some(ReferenceKind::System),
|
||||||
|
Role::User => Some(ReferenceKind::User),
|
||||||
|
Role::Assistant => Some(ReferenceKind::Assistant),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
match origin {
|
||||||
|
SessionEntryProvenance::HumanInput | SessionEntryProvenance::WorkerInput => {
|
||||||
|
Some(ReferenceKind::User)
|
||||||
|
}
|
||||||
|
SessionEntryProvenance::ModelOutput => Some(ReferenceKind::Assistant),
|
||||||
|
SessionEntryProvenance::ToolOutput => Some(ReferenceKind::Tool),
|
||||||
|
SessionEntryProvenance::LegacyUnknown => match provider_role {
|
||||||
|
Role::User => Some(ReferenceKind::User),
|
||||||
|
Role::Assistant => Some(ReferenceKind::Assistant),
|
||||||
|
Role::System => None,
|
||||||
|
},
|
||||||
|
// Flow/backend/system content remains out of the observation surface
|
||||||
|
// even when represented with a provider user/system role.
|
||||||
|
SessionEntryProvenance::FlowInstruction
|
||||||
|
| SessionEntryProvenance::BackendInstruction
|
||||||
|
| SessionEntryProvenance::DerivedSummary => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn render_item(
|
||||||
|
item: &Item,
|
||||||
|
entry: &ReferenceEntry,
|
||||||
|
detail: ReadDetail,
|
||||||
|
max_bytes: usize,
|
||||||
|
) -> String {
|
||||||
|
let text = match item {
|
||||||
|
Item::Message { role, content, .. } => {
|
||||||
|
let text = content
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.as_text())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("");
|
||||||
|
format!("[{} {:?}] {text}", entry.id, role)
|
||||||
|
}
|
||||||
|
Item::ToolCall {
|
||||||
|
name, arguments, ..
|
||||||
|
} => match detail {
|
||||||
|
ReadDetail::Compact => format!("[{} ToolInput {name}] (arguments omitted)", entry.id),
|
||||||
|
ReadDetail::Full => format!("[{} ToolInput {name}]\narguments: {arguments}", entry.id),
|
||||||
|
},
|
||||||
|
Item::ToolResult {
|
||||||
|
summary,
|
||||||
|
content,
|
||||||
|
attachments: _,
|
||||||
|
is_error,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
let attachment_line = if entry.attachment_count == 0 {
|
||||||
|
String::new()
|
||||||
|
} else {
|
||||||
|
format!("\nattachments: {} image(s)", entry.attachment_count)
|
||||||
|
};
|
||||||
|
match detail {
|
||||||
|
ReadDetail::Compact => format!(
|
||||||
|
"[{} ToolOutput{}]\nsummary: {summary}\ncontent: (omitted){attachment_line}",
|
||||||
|
entry.id,
|
||||||
|
if *is_error { " error" } else { "" },
|
||||||
|
),
|
||||||
|
ReadDetail::Full => format!(
|
||||||
|
"[{} ToolOutput{}]\nsummary: {summary}\ncontent: {}{attachment_line}",
|
||||||
|
entry.id,
|
||||||
|
if *is_error { " error" } else { "" },
|
||||||
|
content.as_deref().unwrap_or_default(),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Item::AssistantResponseBoundary { .. } => {
|
||||||
|
format!("[{} Assistant response boundary omitted]", entry.id)
|
||||||
|
}
|
||||||
|
Item::Reasoning { .. } => format!("[{} Reasoning omitted]", entry.id),
|
||||||
|
};
|
||||||
|
truncate_bytes(&text, max_bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn truncate_chars(text: &str, max_chars: usize) -> String {
|
||||||
|
if max_chars == 0 {
|
||||||
|
return "… [truncated]".to_string();
|
||||||
|
}
|
||||||
|
if text.chars().count() <= max_chars {
|
||||||
|
return text.to_string();
|
||||||
|
}
|
||||||
|
let mut out = text.chars().take(max_chars).collect::<String>();
|
||||||
|
out.push_str("… [truncated]");
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
fn truncate_bytes(text: &str, max_bytes: usize) -> String {
|
||||||
|
if text.len() <= max_bytes {
|
||||||
|
return text.to_owned();
|
||||||
|
}
|
||||||
|
let marker = "… [truncated]";
|
||||||
|
let mut end = max_bytes.saturating_sub(marker.len());
|
||||||
|
while !text.is_char_boundary(end) {
|
||||||
|
end -= 1;
|
||||||
|
}
|
||||||
|
let mut out = text[..end].to_owned();
|
||||||
|
if max_bytes >= marker.len() {
|
||||||
|
out.push_str(marker);
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,425 @@
|
|||||||
|
//! Host capture and pure canonical slicing. No Job state or live-reader fallback lives here.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::committed_replay::CommittedPublicReplay;
|
||||||
|
use crate::public_material::{
|
||||||
|
PublicMaterialDescriptor, PublicMaterialError, PublicMaterialLimits, PublicMaterialUse,
|
||||||
|
};
|
||||||
|
use crate::retention::SessionRetentionStore;
|
||||||
|
use crate::{SegmentId, SessionId};
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
/// Continuation in canonical Journal order, not an observation/Memory view cursor.
|
||||||
|
/// A Host must reconcile the descriptor at this position with its saved checkpoint.
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct RecordingPosition {
|
||||||
|
pub canonical_order: u64,
|
||||||
|
pub byte_offset: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct RecordingCaptureSlice {
|
||||||
|
pub capture: CaptureDto,
|
||||||
|
pub next: Option<RecordingPosition>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Inspect recording admission descriptors without constructing a body-bearing DTO.
|
||||||
|
///
|
||||||
|
/// The Host must authorize the source and hold the **same** Session publication/erasure
|
||||||
|
/// guard from before reading its persisted active pointer until this function returns
|
||||||
|
/// (including the callback). Pass that pointer's Segment, not a filename/mtime-derived
|
||||||
|
/// selector. This function verifies the guard and stored Session identities but does
|
||||||
|
/// not authorize the caller or establish that the supplied guard fences this root.
|
||||||
|
/// No lock is recursively acquired, no active pointer is reread, and no replay handle
|
||||||
|
/// or body-read capability is exposed to `inspect`.
|
||||||
|
///
|
||||||
|
/// The common committed recording replay fixes per-file ends before decoding. It
|
||||||
|
/// retains only descriptors/locations, decoding one record at a time and streaming
|
||||||
|
/// canonical bodies into their digests. `replay_limits` bound record allocation,
|
||||||
|
/// aggregate storage extent, entries and Segments; there is no full-capture text/JSON
|
||||||
|
/// allocation or `CaptureLimits` ceiling. The callback receives only descriptors in
|
||||||
|
/// canonical order and may return a Host-specific mapping (including a `Result`). It
|
||||||
|
/// runs once on success and not at all on replay/identity/resource errors.
|
||||||
|
///
|
||||||
|
/// This catalog is for Host admission, **not** a descriptor-only Job input. Construct
|
||||||
|
/// the Job's bounded body input with the guarded slice API at its own authorized
|
||||||
|
/// boundary; after saving it, execute only that immutable value. Neither descriptors
|
||||||
|
/// nor a completed callback authorize delayed adoption after erasure.
|
||||||
|
///
|
||||||
|
/// ```no_run
|
||||||
|
/// # use std::path::Path;
|
||||||
|
/// # use session_store::{SessionId, SegmentId};
|
||||||
|
/// # use session_store::retention::SessionRetentionGuard;
|
||||||
|
/// # use session_store::public_material::PublicMaterialError;
|
||||||
|
/// # fn admit(root: &Path, session: SessionId, active: SegmentId,
|
||||||
|
/// # guard: &SessionRetentionGuard) -> Result<Vec<u64>, PublicMaterialError> {
|
||||||
|
/// // `active` was resolved from persisted Host metadata while holding `guard`.
|
||||||
|
/// session_store::capture::recording_catalog_worker_guarded(
|
||||||
|
/// root, session, active, Default::default(), guard,
|
||||||
|
/// |descriptors| descriptors.iter().map(|d| d.total_bytes).collect(),
|
||||||
|
/// )
|
||||||
|
/// # }
|
||||||
|
/// ```
|
||||||
|
pub fn recording_catalog_worker_guarded<T>(
|
||||||
|
session_root: &Path,
|
||||||
|
expected_session: SessionId,
|
||||||
|
active_segment: SegmentId,
|
||||||
|
replay_limits: PublicMaterialLimits,
|
||||||
|
guard: &crate::retention::SessionRetentionGuard,
|
||||||
|
inspect: impl FnOnce(&[PublicMaterialDescriptor]) -> T,
|
||||||
|
) -> Result<T, PublicMaterialError> {
|
||||||
|
if guard.session_id() != expected_session {
|
||||||
|
return Err(PublicMaterialError::Inconsistent);
|
||||||
|
}
|
||||||
|
let replay = CommittedPublicReplay::open_worker_guarded(
|
||||||
|
session_root,
|
||||||
|
active_segment,
|
||||||
|
replay_limits,
|
||||||
|
None,
|
||||||
|
PublicMaterialUse::Recording,
|
||||||
|
)?;
|
||||||
|
if replay.session_id() != expected_session {
|
||||||
|
return Err(PublicMaterialError::Inconsistent);
|
||||||
|
}
|
||||||
|
Ok(inspect(&replay.into_recording_descriptors()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// FsStore-layout counterpart of [`recording_catalog_worker_guarded`].
|
||||||
|
///
|
||||||
|
/// `sessions_root` is the directory containing per-Session directories. Invoke this
|
||||||
|
/// inside the Host's `with_recording_boundary` callback, passing **its existing guard
|
||||||
|
/// and persisted active Segment**. That same external retention control store must
|
||||||
|
/// fence publishers, pointer changes, archive moves and erasure. Do not reacquire the
|
||||||
|
/// guard or use a separately opened legacy public reader inside the boundary.
|
||||||
|
/// All descriptor-only, allocation and callback guarantees of the Worker variant
|
||||||
|
/// apply; this function does not construct an unbounded transient `CaptureDto`.
|
||||||
|
pub fn recording_catalog_fs_guarded<T>(
|
||||||
|
sessions_root: &Path,
|
||||||
|
session: SessionId,
|
||||||
|
active_segment: SegmentId,
|
||||||
|
replay_limits: PublicMaterialLimits,
|
||||||
|
guard: &crate::retention::SessionRetentionGuard,
|
||||||
|
inspect: impl FnOnce(&[PublicMaterialDescriptor]) -> T,
|
||||||
|
) -> Result<T, PublicMaterialError> {
|
||||||
|
if guard.session_id() != session {
|
||||||
|
return Err(PublicMaterialError::Inconsistent);
|
||||||
|
}
|
||||||
|
let replay = CommittedPublicReplay::open_fs_guarded(
|
||||||
|
sessions_root,
|
||||||
|
session,
|
||||||
|
active_segment,
|
||||||
|
replay_limits,
|
||||||
|
)?;
|
||||||
|
Ok(inspect(&replay.into_recording_descriptors()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Capture the selected original lineage, including typed Host run boundaries.
|
||||||
|
/// The caller must hold the Session publication/erasure guard while selecting the
|
||||||
|
/// active pointer and through this call. Runtime publishers use this same lock
|
||||||
|
/// for appends and Segment creation. No lock is recursively acquired here.
|
||||||
|
pub fn capture_recording_worker_guarded(
|
||||||
|
session_root: &Path,
|
||||||
|
expected_session: SessionId,
|
||||||
|
active_segment: SegmentId,
|
||||||
|
replay_limits: PublicMaterialLimits,
|
||||||
|
capture_limits: CaptureLimits,
|
||||||
|
guard: &crate::retention::SessionRetentionGuard,
|
||||||
|
) -> Result<CaptureDto, PublicMaterialError> {
|
||||||
|
if guard.session_id() != expected_session {
|
||||||
|
return Err(PublicMaterialError::Inconsistent);
|
||||||
|
}
|
||||||
|
let replay = CommittedPublicReplay::open_worker_guarded(
|
||||||
|
session_root,
|
||||||
|
active_segment,
|
||||||
|
replay_limits,
|
||||||
|
None,
|
||||||
|
PublicMaterialUse::Recording,
|
||||||
|
)?;
|
||||||
|
if replay.session_id() != expected_session {
|
||||||
|
return Err(PublicMaterialError::Inconsistent);
|
||||||
|
}
|
||||||
|
replay.into_recording_dto(capture_limits)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Standalone convenience entry; Hosts resolving a mutable active pointer should
|
||||||
|
/// instead hold the fence while resolving it and call the guarded entry.
|
||||||
|
pub fn capture_recording_worker(
|
||||||
|
session_root: &Path,
|
||||||
|
expected_session: SessionId,
|
||||||
|
active_segment: SegmentId,
|
||||||
|
retention: &SessionRetentionStore,
|
||||||
|
replay_limits: PublicMaterialLimits,
|
||||||
|
capture_limits: CaptureLimits,
|
||||||
|
) -> Result<CaptureDto, PublicMaterialError> {
|
||||||
|
let guard = retention
|
||||||
|
.lock_live(expected_session)
|
||||||
|
.map_err(|error| match error {
|
||||||
|
crate::StoreError::SessionErased(_) => PublicMaterialError::Erased,
|
||||||
|
_ => PublicMaterialError::Unavailable,
|
||||||
|
})?;
|
||||||
|
capture_recording_worker_guarded(
|
||||||
|
session_root,
|
||||||
|
expected_session,
|
||||||
|
active_segment,
|
||||||
|
replay_limits,
|
||||||
|
capture_limits,
|
||||||
|
&guard,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bounded construction for one Job input: descriptor replay is bounded separately;
|
||||||
|
/// only at most the requested body budget is materialized from the fixed files.
|
||||||
|
pub fn capture_recording_slice_worker_guarded(
|
||||||
|
session_root: &Path,
|
||||||
|
expected_session: SessionId,
|
||||||
|
active_segment: SegmentId,
|
||||||
|
replay_limits: PublicMaterialLimits,
|
||||||
|
start: RecordingPosition,
|
||||||
|
max_entries: usize,
|
||||||
|
max_serialized_bytes: usize,
|
||||||
|
guard: &crate::retention::SessionRetentionGuard,
|
||||||
|
) -> Result<RecordingCaptureSlice, PublicMaterialError> {
|
||||||
|
if guard.session_id() != expected_session {
|
||||||
|
return Err(PublicMaterialError::Inconsistent);
|
||||||
|
}
|
||||||
|
let replay = CommittedPublicReplay::open_worker_guarded(
|
||||||
|
session_root,
|
||||||
|
active_segment,
|
||||||
|
replay_limits,
|
||||||
|
None,
|
||||||
|
PublicMaterialUse::Recording,
|
||||||
|
)?;
|
||||||
|
if replay.session_id() != expected_session {
|
||||||
|
return Err(PublicMaterialError::Inconsistent);
|
||||||
|
}
|
||||||
|
replay.recording_slice(start, max_entries, max_serialized_bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The standalone FsStore layout uses the same committed replay and fixed values.
|
||||||
|
pub fn capture_recording_fs_guarded(
|
||||||
|
sessions_root: &Path,
|
||||||
|
session: SessionId,
|
||||||
|
active_segment: SegmentId,
|
||||||
|
replay_limits: PublicMaterialLimits,
|
||||||
|
capture_limits: CaptureLimits,
|
||||||
|
guard: &crate::retention::SessionRetentionGuard,
|
||||||
|
) -> Result<CaptureDto, PublicMaterialError> {
|
||||||
|
if guard.session_id() != session {
|
||||||
|
return Err(PublicMaterialError::Inconsistent);
|
||||||
|
}
|
||||||
|
CommittedPublicReplay::open_fs_guarded(sessions_root, session, active_segment, replay_limits)?
|
||||||
|
.into_recording_dto(capture_limits)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bounded standalone capture; the caller holds its FsStore retention fence.
|
||||||
|
pub fn capture_recording_slice_fs_guarded(
|
||||||
|
sessions_root: &Path,
|
||||||
|
session: SessionId,
|
||||||
|
active_segment: SegmentId,
|
||||||
|
replay_limits: PublicMaterialLimits,
|
||||||
|
start: RecordingPosition,
|
||||||
|
max_entries: usize,
|
||||||
|
max_serialized_bytes: usize,
|
||||||
|
guard: &crate::retention::SessionRetentionGuard,
|
||||||
|
) -> Result<RecordingCaptureSlice, PublicMaterialError> {
|
||||||
|
if guard.session_id() != session {
|
||||||
|
return Err(PublicMaterialError::Inconsistent);
|
||||||
|
}
|
||||||
|
CommittedPublicReplay::open_fs_guarded(sessions_root, session, active_segment, replay_limits)?
|
||||||
|
.recording_slice(start, max_entries, max_serialized_bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CaptureDto {
|
||||||
|
/// Select the oldest canonical prefix fitting both budgets, including JSON
|
||||||
|
/// escaping, metadata and continuation. Only immutable owned bytes are read.
|
||||||
|
/// A later Job can recapture at `next`; never combine live pages into this value.
|
||||||
|
pub fn recording_slice(
|
||||||
|
&self,
|
||||||
|
start: RecordingPosition,
|
||||||
|
max_entries: usize,
|
||||||
|
max_serialized_bytes: usize,
|
||||||
|
) -> Result<RecordingCaptureSlice, CaptureError> {
|
||||||
|
if max_entries == 0 {
|
||||||
|
return Err(CaptureError::BudgetTooSmall);
|
||||||
|
}
|
||||||
|
if start.canonical_order > self.view_entry_count {
|
||||||
|
return Err(CaptureError::InvalidRange);
|
||||||
|
}
|
||||||
|
let mut result = RecordingCaptureSlice {
|
||||||
|
capture: CaptureDto {
|
||||||
|
version: self.version,
|
||||||
|
session_id: self.session_id.clone(),
|
||||||
|
segment_id: self.segment_id.clone(),
|
||||||
|
view_entry_count: self.view_entry_count,
|
||||||
|
entries: Vec::new(),
|
||||||
|
},
|
||||||
|
next: (start.canonical_order < self.view_entry_count).then_some(start),
|
||||||
|
};
|
||||||
|
if body::json_size(&result, max_serialized_bytes).is_err() {
|
||||||
|
return Err(CaptureError::BudgetTooSmall);
|
||||||
|
}
|
||||||
|
if start.canonical_order == self.view_entry_count {
|
||||||
|
if start.byte_offset != 0 {
|
||||||
|
return Err(CaptureError::InvalidRange);
|
||||||
|
}
|
||||||
|
result.capture.validate(CaptureLimits {
|
||||||
|
max_bytes: max_serialized_bytes,
|
||||||
|
max_text_bytes: max_serialized_bytes,
|
||||||
|
max_entries,
|
||||||
|
..Default::default()
|
||||||
|
})?;
|
||||||
|
return Ok(result);
|
||||||
|
}
|
||||||
|
let first = self
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.position(|entry| entry.order == start.canonical_order)
|
||||||
|
.ok_or(CaptureError::InvalidRange)?;
|
||||||
|
let mut expected_order = start.canonical_order;
|
||||||
|
for entry in self.entries.iter().skip(first).take(max_entries) {
|
||||||
|
if entry.order != expected_order {
|
||||||
|
return Err(CaptureError::InvalidRange);
|
||||||
|
}
|
||||||
|
let CaptureBody::CanonicalPublic {
|
||||||
|
descriptor,
|
||||||
|
byte_offset,
|
||||||
|
text,
|
||||||
|
boundary,
|
||||||
|
} = &entry.body
|
||||||
|
else {
|
||||||
|
return Err(CaptureError::Invalid("not a recording capture"));
|
||||||
|
};
|
||||||
|
if !entry.policy.recording_material {
|
||||||
|
return Err(CaptureError::PolicyDenied);
|
||||||
|
}
|
||||||
|
let offset = if entry.order == start.canonical_order {
|
||||||
|
start.byte_offset
|
||||||
|
} else {
|
||||||
|
*byte_offset
|
||||||
|
};
|
||||||
|
let local = offset
|
||||||
|
.checked_sub(*byte_offset)
|
||||||
|
.and_then(|o| usize::try_from(o).ok())
|
||||||
|
.ok_or(CaptureError::InvalidRange)?;
|
||||||
|
if local > text.len()
|
||||||
|
|| !text.is_char_boundary(local)
|
||||||
|
|| offset >= descriptor.total_bytes
|
||||||
|
{
|
||||||
|
return Err(CaptureError::InvalidRange);
|
||||||
|
}
|
||||||
|
let item = CaptureEntryDto {
|
||||||
|
entry_id: entry.entry_id.clone(),
|
||||||
|
order: entry.order,
|
||||||
|
source: entry.source.clone(),
|
||||||
|
policy: entry.policy,
|
||||||
|
body: CaptureBody::CanonicalPublic {
|
||||||
|
descriptor: descriptor.clone(),
|
||||||
|
byte_offset: offset,
|
||||||
|
text: String::new(),
|
||||||
|
boundary: *boundary,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
result.capture.entries.push(item);
|
||||||
|
// At most budget bytes are considered before allocating a trial fragment.
|
||||||
|
let mut low = 0;
|
||||||
|
let mut high = text.len().saturating_sub(local).min(max_serialized_bytes);
|
||||||
|
let prior_next = result.next;
|
||||||
|
while low < high {
|
||||||
|
let mid = low + (high - low).div_ceil(2);
|
||||||
|
let mut end = local + mid;
|
||||||
|
while !text.is_char_boundary(end) {
|
||||||
|
end -= 1;
|
||||||
|
}
|
||||||
|
if let CaptureBody::CanonicalPublic { text: fragment, .. } =
|
||||||
|
&mut result.capture.entries.last_mut().unwrap().body
|
||||||
|
{
|
||||||
|
*fragment = text[local..end].to_owned();
|
||||||
|
}
|
||||||
|
let next_offset = offset + (end - local) as u64;
|
||||||
|
result.next = if next_offset < descriptor.total_bytes {
|
||||||
|
Some(RecordingPosition {
|
||||||
|
canonical_order: entry.order,
|
||||||
|
byte_offset: next_offset,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
(entry.order + 1 < self.view_entry_count).then_some(RecordingPosition {
|
||||||
|
canonical_order: entry.order + 1,
|
||||||
|
byte_offset: 0,
|
||||||
|
})
|
||||||
|
};
|
||||||
|
if body::json_size(&result, max_serialized_bytes).is_ok() {
|
||||||
|
low = mid;
|
||||||
|
} else {
|
||||||
|
high = mid - 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut end = local + low;
|
||||||
|
while !text.is_char_boundary(end) {
|
||||||
|
end -= 1;
|
||||||
|
}
|
||||||
|
if end == local {
|
||||||
|
result.capture.entries.pop();
|
||||||
|
result.next = prior_next;
|
||||||
|
if result.capture.entries.is_empty() {
|
||||||
|
return Err(CaptureError::BudgetTooSmall);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if let CaptureBody::CanonicalPublic { text: fragment, .. } =
|
||||||
|
&mut result.capture.entries.last_mut().unwrap().body
|
||||||
|
{
|
||||||
|
*fragment = text[local..end].to_owned();
|
||||||
|
}
|
||||||
|
let next_offset = offset + (end - local) as u64;
|
||||||
|
result.next = if next_offset < descriptor.total_bytes {
|
||||||
|
Some(RecordingPosition {
|
||||||
|
canonical_order: entry.order,
|
||||||
|
byte_offset: next_offset,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
(entry.order + 1 < self.view_entry_count).then_some(RecordingPosition {
|
||||||
|
canonical_order: entry.order + 1,
|
||||||
|
byte_offset: 0,
|
||||||
|
})
|
||||||
|
};
|
||||||
|
if next_offset < descriptor.total_bytes {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
expected_order = entry.order + 1;
|
||||||
|
}
|
||||||
|
body::json_size(&result, max_serialized_bytes).map_err(|_| CaptureError::BudgetTooSmall)?;
|
||||||
|
result.capture.validate(CaptureLimits {
|
||||||
|
max_bytes: max_serialized_bytes,
|
||||||
|
max_text_bytes: max_serialized_bytes,
|
||||||
|
max_entries,
|
||||||
|
..Default::default()
|
||||||
|
})?;
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve canonical text already in the value. No original storage is consulted.
|
||||||
|
pub fn recording_text(
|
||||||
|
&self,
|
||||||
|
order: u64,
|
||||||
|
) -> Result<(&crate::public_material::PublicMaterialDescriptor, u64, &str), CaptureError> {
|
||||||
|
let entry = self
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.find(|entry| entry.order == order)
|
||||||
|
.ok_or(CaptureError::UnknownReference)?;
|
||||||
|
if !entry.policy.recording_material {
|
||||||
|
return Err(CaptureError::PolicyDenied);
|
||||||
|
}
|
||||||
|
match &entry.body {
|
||||||
|
CaptureBody::CanonicalPublic {
|
||||||
|
descriptor,
|
||||||
|
byte_offset,
|
||||||
|
text,
|
||||||
|
..
|
||||||
|
} => Ok((descriptor, *byte_offset, text)),
|
||||||
|
_ => Err(CaptureError::Invalid("not canonical recording text")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -38,6 +38,8 @@ pub(crate) fn read_without_atime_bounded(path: &Path, max_bytes: u64) -> io::Res
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub mod capture;
|
||||||
|
mod committed_replay;
|
||||||
pub mod event_trace;
|
pub mod event_trace;
|
||||||
pub mod fs_store;
|
pub mod fs_store;
|
||||||
pub mod history;
|
pub mod history;
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -46,6 +46,13 @@ pub struct SessionRetentionStore {
|
|||||||
/// Do not recursively acquire the same Session lock inside a guarded callback.
|
/// Do not recursively acquire the same Session lock inside a guarded callback.
|
||||||
pub struct SessionRetentionGuard {
|
pub struct SessionRetentionGuard {
|
||||||
_file: File,
|
_file: File,
|
||||||
|
session_id: SessionId,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SessionRetentionGuard {
|
||||||
|
pub fn session_id(&self) -> SessionId {
|
||||||
|
self.session_id
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SessionRetentionStore {
|
impl SessionRetentionStore {
|
||||||
@@ -71,7 +78,10 @@ impl SessionRetentionStore {
|
|||||||
}
|
}
|
||||||
let file = options.open(self.root.join(format!("{session}.lock")))?;
|
let file = options.open(self.root.join(format!("{session}.lock")))?;
|
||||||
FileExt::lock_exclusive(&file)?;
|
FileExt::lock_exclusive(&file)?;
|
||||||
Ok(SessionRetentionGuard { _file: file })
|
Ok(SessionRetentionGuard {
|
||||||
|
_file: file,
|
||||||
|
session_id: session,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn read(&self, session: SessionId) -> Result<RetentionState, StoreError> {
|
fn read(&self, session: SessionId) -> Result<RetentionState, StoreError> {
|
||||||
|
|||||||
@@ -194,6 +194,21 @@ impl WorkerSessionStore {
|
|||||||
Ok(store)
|
Ok(store)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Open a current retained Session without creating or migrating anything.
|
||||||
|
/// The live constraint is checked here and again by every later publication;
|
||||||
|
/// attaching the control store itself does not acquire publication authority.
|
||||||
|
pub fn open_existing_with_retention(
|
||||||
|
root: impl Into<PathBuf>,
|
||||||
|
retention: crate::retention::SessionRetentionStore,
|
||||||
|
) -> Result<Self, StoreError> {
|
||||||
|
let mut store = Self::open_read_only(root)?;
|
||||||
|
if let Some(session) = *store.session_id.lock().unwrap() {
|
||||||
|
retention.ensure_live(session)?;
|
||||||
|
}
|
||||||
|
store.retention = Some(retention);
|
||||||
|
Ok(store)
|
||||||
|
}
|
||||||
|
|
||||||
fn lock_publication(
|
fn lock_publication(
|
||||||
&self,
|
&self,
|
||||||
session: SessionId,
|
session: SessionId,
|
||||||
|
|||||||
@@ -0,0 +1,711 @@
|
|||||||
|
//! Public consumer boundary: real capture adapters, checked DTO codec and pure readers.
|
||||||
|
//! No Worker, persistence, live Session provider, or substituted capture implementation.
|
||||||
|
|
||||||
|
use agen::{HistoryEntry, Item};
|
||||||
|
use protocol::{
|
||||||
|
SessionEntryProvenance, SessionSnapshot, SessionSnapshotEntry, SessionSnapshotEntryData,
|
||||||
|
};
|
||||||
|
use session_store::capture::*;
|
||||||
|
use session_store::{
|
||||||
|
FeatureInputPayload, HistoryInputMetadata, HistoryInputUsePolicy, InputCompactionUse,
|
||||||
|
InputEvidenceUse, InputObservationUse, LoggedHistoryDerivation, LoggedSessionHistoryEntryId,
|
||||||
|
LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, PublicSystemItem,
|
||||||
|
PublicSystemPresentation,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn history(
|
||||||
|
id: &str,
|
||||||
|
item: Item,
|
||||||
|
input_metadata: Option<HistoryInputMetadata>,
|
||||||
|
) -> HistoryEntry<LoggedSessionHistoryMetadata> {
|
||||||
|
HistoryEntry::new(
|
||||||
|
item,
|
||||||
|
LoggedSessionHistoryMetadata {
|
||||||
|
entry_id: LoggedSessionHistoryEntryId(id.into()),
|
||||||
|
origin: LoggedSessionHistoryOrigin::LocalHumanInput,
|
||||||
|
derivation: Some(LoggedHistoryDerivation {
|
||||||
|
sources: vec![LoggedSessionHistoryEntryId("earlier-entry".into())],
|
||||||
|
}),
|
||||||
|
input_metadata,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn usage(recording: bool, memory: bool, observation: bool) -> HistoryInputUsePolicy {
|
||||||
|
HistoryInputUsePolicy {
|
||||||
|
recording_material: if recording {
|
||||||
|
InputEvidenceUse::Include
|
||||||
|
} else {
|
||||||
|
InputEvidenceUse::Exclude
|
||||||
|
},
|
||||||
|
memory_evidence: if memory {
|
||||||
|
InputEvidenceUse::Include
|
||||||
|
} else {
|
||||||
|
InputEvidenceUse::Exclude
|
||||||
|
},
|
||||||
|
observation: if observation {
|
||||||
|
InputObservationUse::Readable
|
||||||
|
} else {
|
||||||
|
InputObservationUse::Hidden
|
||||||
|
},
|
||||||
|
compaction: InputCompactionUse::Retain,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reference(capture: &SessionCapture, id: &str) -> ScopedSessionEntryRef {
|
||||||
|
capture
|
||||||
|
.scoped_ref(&SessionEntryRef::parse(id).unwrap())
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn public_consumer_roundtrip_retains_body_original_source_order_origin_and_independent_policy() {
|
||||||
|
let entries = vec![
|
||||||
|
history(
|
||||||
|
"recording",
|
||||||
|
Item::user_message("record-only"),
|
||||||
|
Some(HistoryInputMetadata::new(usage(true, false, false), None, None).unwrap()),
|
||||||
|
),
|
||||||
|
history(
|
||||||
|
"memory",
|
||||||
|
Item::user_message("memory-only"),
|
||||||
|
Some(HistoryInputMetadata::new(usage(false, true, false), None, None).unwrap()),
|
||||||
|
),
|
||||||
|
history(
|
||||||
|
"observation",
|
||||||
|
Item::user_message("observation-only"),
|
||||||
|
Some(HistoryInputMetadata::new(usage(false, false, true), None, None).unwrap()),
|
||||||
|
),
|
||||||
|
];
|
||||||
|
let capture = SessionCapture::from_complete_history("synthetic-session", "selected", entries);
|
||||||
|
let mut dto = capture.to_dto();
|
||||||
|
// The checked boundary also accepts entries retained from different original segments.
|
||||||
|
dto.entries[0].source.segment_id = "original-a".into();
|
||||||
|
dto.entries[0].source.entry_index = 80;
|
||||||
|
dto.entries[1].source.segment_id = "original-b".into();
|
||||||
|
dto.entries[1].source.entry_index = 3;
|
||||||
|
let capture = SessionCapture::try_from_dto(dto.clone(), CaptureLimits::default()).unwrap();
|
||||||
|
let encoded = capture.encode(CaptureLimits::default()).unwrap();
|
||||||
|
let restored = SessionCapture::decode(&encoded, CaptureLimits::default()).unwrap();
|
||||||
|
assert_eq!(restored.to_dto(), dto);
|
||||||
|
assert_eq!(restored.capture_id(), capture.capture_id());
|
||||||
|
for (id, allowed) in [
|
||||||
|
("Erecording", CapturePurpose::Recording),
|
||||||
|
("Ememory", CapturePurpose::MemoryEvidence),
|
||||||
|
("Eobservation", CapturePurpose::Observation),
|
||||||
|
] {
|
||||||
|
let reference = reference(&capture, id);
|
||||||
|
assert_eq!(
|
||||||
|
restored.resolve_ref(&reference, allowed).unwrap(),
|
||||||
|
capture.resolve_ref(&reference, allowed).unwrap()
|
||||||
|
);
|
||||||
|
for purpose in [
|
||||||
|
CapturePurpose::Recording,
|
||||||
|
CapturePurpose::MemoryEvidence,
|
||||||
|
CapturePurpose::Observation,
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
restored.resolve_ref(&reference, purpose).is_ok(),
|
||||||
|
purpose == allowed
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(restored.overview().len(), 1);
|
||||||
|
assert_eq!(restored.search(&SearchOptions::default()).len(), 1);
|
||||||
|
assert!(restored.evidence_for("Erecording").is_none());
|
||||||
|
let evidence = restored.evidence_for("Ememory").unwrap();
|
||||||
|
assert_eq!(evidence.segment_id, "original-b");
|
||||||
|
assert_eq!(evidence.entry_range, [3, 3]);
|
||||||
|
assert!(evidence.excerpt.contains("memory-only"));
|
||||||
|
let recorded = restored
|
||||||
|
.slice(CapturePurpose::Recording, None, 10, 16_000)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(recorded.slice().items.len(), 1);
|
||||||
|
assert_eq!(recorded.slice().items[0].text, "record-only");
|
||||||
|
for (purpose, expected) in [
|
||||||
|
(CapturePurpose::MemoryEvidence, "memory-only"),
|
||||||
|
(CapturePurpose::Observation, "observation-only"),
|
||||||
|
] {
|
||||||
|
let slice = restored.slice(purpose, None, 10, 16_000).unwrap();
|
||||||
|
assert_eq!(slice.slice().items.len(), 1);
|
||||||
|
assert_eq!(slice.slice().items[0].text, expected);
|
||||||
|
assert!(
|
||||||
|
restored
|
||||||
|
.read_raw(
|
||||||
|
&reference(&restored, "Erecording"),
|
||||||
|
purpose,
|
||||||
|
RawField::MessagePart(0),
|
||||||
|
0,
|
||||||
|
100
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn common_data_drops_reasoning_raw_system_and_private_payload_even_when_unindexed() {
|
||||||
|
let payload = FeatureInputPayload::encode(
|
||||||
|
"capture-test",
|
||||||
|
1,
|
||||||
|
&serde_json::json!({"secret": "PRIVATE-PAYLOAD"}),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let public = PublicSystemItem {
|
||||||
|
kind: "notice".into(),
|
||||||
|
body: "public notice".into(),
|
||||||
|
presentation: PublicSystemPresentation::Ignored,
|
||||||
|
};
|
||||||
|
let system =
|
||||||
|
HistoryInputMetadata::new(usage(true, true, true), Some(payload.clone()), Some(public))
|
||||||
|
.unwrap();
|
||||||
|
let private = HistoryInputMetadata::new(usage(true, true, true), Some(payload), None).unwrap();
|
||||||
|
let capture = SessionCapture::from_complete_history(
|
||||||
|
"s",
|
||||||
|
"segment",
|
||||||
|
vec![
|
||||||
|
history("raw", Item::system_message("RAW-SYSTEM-PROMPT"), None),
|
||||||
|
history("reasoning", Item::reasoning("HIDDEN-REASONING"), None),
|
||||||
|
history(
|
||||||
|
"private",
|
||||||
|
Item::system_message("PRIVATE-SYSTEM-BODY"),
|
||||||
|
Some(private),
|
||||||
|
),
|
||||||
|
history(
|
||||||
|
"public",
|
||||||
|
Item::system_message("MODEL-SYSTEM-BODY"),
|
||||||
|
Some(system),
|
||||||
|
),
|
||||||
|
history("ordinary", Item::user_message("ordinary"), None),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
let encoded = String::from_utf8(capture.encode(CaptureLimits::default()).unwrap()).unwrap();
|
||||||
|
let retained_debug = format!("{capture:?}");
|
||||||
|
for secret in [
|
||||||
|
"RAW-SYSTEM-PROMPT",
|
||||||
|
"HIDDEN-REASONING",
|
||||||
|
"PRIVATE-SYSTEM-BODY",
|
||||||
|
"MODEL-SYSTEM-BODY",
|
||||||
|
"PRIVATE-PAYLOAD",
|
||||||
|
] {
|
||||||
|
assert!(!encoded.contains(secret));
|
||||||
|
assert!(!retained_debug.contains(secret));
|
||||||
|
}
|
||||||
|
assert_eq!(capture.to_dto().entries.len(), 2);
|
||||||
|
assert!(encoded.contains("public notice"));
|
||||||
|
// Filtered entries do not renumber the public source positions.
|
||||||
|
assert_eq!(capture.to_dto().entries[0].order, 3);
|
||||||
|
assert_eq!(
|
||||||
|
capture.view_order_for_ref(&SessionEntryRef::parse("Eordinary").unwrap()),
|
||||||
|
Some(4)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn protocol_projection_does_not_invent_recording_permission() {
|
||||||
|
let capture = SessionCapture::from_session_snapshot(
|
||||||
|
"s",
|
||||||
|
"seg",
|
||||||
|
SessionSnapshot {
|
||||||
|
pending_submissions: Default::default(),
|
||||||
|
entries: vec![SessionSnapshotEntry {
|
||||||
|
entry_id: "projected".into(),
|
||||||
|
timestamp: 0,
|
||||||
|
provenance: SessionEntryProvenance::HumanInput,
|
||||||
|
derived_from: vec!["source".into()],
|
||||||
|
use_policy: Some(protocol::SessionEntryUsePolicy {
|
||||||
|
observation_readable: true,
|
||||||
|
memory_evidence: false,
|
||||||
|
}),
|
||||||
|
data: SessionSnapshotEntryData::Message {
|
||||||
|
role: protocol::SessionMessageRole::User,
|
||||||
|
content: vec![protocol::SessionContentPart::Text {
|
||||||
|
text: "projected".into(),
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let reference = reference(&capture, "Eprojected");
|
||||||
|
assert!(matches!(
|
||||||
|
capture.resolve_ref(&reference, CapturePurpose::Recording),
|
||||||
|
Err(CaptureError::PolicyDenied)
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
capture
|
||||||
|
.resolve_ref(&reference, CapturePurpose::Observation)
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
assert!(capture.evidence_for("Eprojected").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn malformed_dtos_and_oversized_representations_are_rejected() {
|
||||||
|
let capture = SessionCapture::new("s", "seg", vec![Item::user_message("body")]);
|
||||||
|
let original = serde_json::to_value(capture.to_dto()).unwrap();
|
||||||
|
let mut invalids = Vec::new();
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["version"] = 999.into();
|
||||||
|
invalids.push(value);
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["unexpected"] = true.into();
|
||||||
|
invalids.push(value);
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["entries"][0]["policy"]["recording_material"] = "include".into();
|
||||||
|
invalids.push(value);
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["entries"][0]["policy"]
|
||||||
|
.as_object_mut()
|
||||||
|
.unwrap()
|
||||||
|
.remove("memory_evidence");
|
||||||
|
invalids.push(value);
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["entries"][0]["body"] = serde_json::json!({"kind":"reasoning","text":"secret"});
|
||||||
|
invalids.push(value);
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["entries"][0]["body"]["feature_payload"] = serde_json::json!({"private":true});
|
||||||
|
invalids.push(value);
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["entries"][0]["body"]["role"] = "system".into();
|
||||||
|
invalids.push(value);
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["entries"][0]["source"]["unknown"] = true.into();
|
||||||
|
invalids.push(value);
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["entries"][0]["source"]["history_origin"] =
|
||||||
|
serde_json::json!({"kind":"legacy_unknown","unknown":true});
|
||||||
|
invalids.push(value);
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["entries"][0]["entry_id"] = "bad/id".into();
|
||||||
|
invalids.push(value);
|
||||||
|
let mut value = original.clone();
|
||||||
|
value["entries"]
|
||||||
|
.as_array_mut()
|
||||||
|
.unwrap()
|
||||||
|
.push(original["entries"][0].clone());
|
||||||
|
invalids.push(value);
|
||||||
|
for invalid in invalids {
|
||||||
|
let bytes = serde_json::to_vec(&invalid).unwrap();
|
||||||
|
assert!(
|
||||||
|
SessionCapture::decode(&bytes, CaptureLimits::default()).is_err(),
|
||||||
|
"accepted {invalid}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let bytes = capture.encode(CaptureLimits::default()).unwrap();
|
||||||
|
assert!(
|
||||||
|
SessionCapture::decode(
|
||||||
|
&bytes,
|
||||||
|
CaptureLimits {
|
||||||
|
max_bytes: bytes.len() - 1,
|
||||||
|
..CaptureLimits::default()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
capture
|
||||||
|
.encode(CaptureLimits {
|
||||||
|
max_entries: 0,
|
||||||
|
..CaptureLimits::default()
|
||||||
|
})
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
capture
|
||||||
|
.encode(CaptureLimits {
|
||||||
|
max_text_bytes: 3,
|
||||||
|
..CaptureLimits::default()
|
||||||
|
})
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let mut reversed = capture.to_dto();
|
||||||
|
reversed.view_entry_count = 2;
|
||||||
|
let mut second = reversed.entries[0].clone();
|
||||||
|
second.entry_id = "second".into();
|
||||||
|
second.source.entry_index = 4;
|
||||||
|
second.order = 0;
|
||||||
|
reversed.entries[0].order = 1;
|
||||||
|
reversed.entries.push(second);
|
||||||
|
assert!(SessionCapture::try_from_dto(reversed, CaptureLimits::default()).is_err());
|
||||||
|
let mut duplicate_source = capture.to_dto();
|
||||||
|
duplicate_source.view_entry_count = 2;
|
||||||
|
let mut second = duplicate_source.entries[0].clone();
|
||||||
|
second.entry_id = "distinct-id".into();
|
||||||
|
second.order = 1;
|
||||||
|
duplicate_source.entries.push(second);
|
||||||
|
assert!(matches!(
|
||||||
|
SessionCapture::try_from_dto(duplicate_source, CaptureLimits::default()),
|
||||||
|
Err(CaptureError::Invalid("duplicate source position"))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn raw_utf8_tool_input_and_result_are_lossless_without_preview_markers() {
|
||||||
|
let arguments = "\"quoted\"\\path\n日本語🙂".repeat(300);
|
||||||
|
let result = "result\n\t\\\"λ🙂".repeat(400);
|
||||||
|
let capture = SessionCapture::new(
|
||||||
|
"s",
|
||||||
|
"seg",
|
||||||
|
vec![
|
||||||
|
Item::tool_call("call", "tool", arguments.clone()),
|
||||||
|
Item::tool_result_item("call", "summary", Some(result.clone()), false),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
for (id, field, expected) in [
|
||||||
|
("E00000000", RawField::ToolArguments, arguments),
|
||||||
|
("E00000001", RawField::ToolContent, result),
|
||||||
|
] {
|
||||||
|
let reference = reference(&capture, id);
|
||||||
|
let mut offset = 0;
|
||||||
|
let mut actual = String::new();
|
||||||
|
loop {
|
||||||
|
let part = capture
|
||||||
|
.read_raw(&reference, CapturePurpose::Observation, field, offset, 17)
|
||||||
|
.unwrap();
|
||||||
|
assert!(part.text.len() <= 17);
|
||||||
|
assert_eq!(part.offset, offset);
|
||||||
|
actual.push_str(&part.text);
|
||||||
|
match part.next_offset {
|
||||||
|
Some(next) => {
|
||||||
|
assert!(next > offset);
|
||||||
|
offset = next;
|
||||||
|
}
|
||||||
|
None => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(actual, expected);
|
||||||
|
}
|
||||||
|
let unicode = SessionCapture::new("s", "seg", vec![Item::tool_call("call", "tool", "🙂")]);
|
||||||
|
let reference = reference(&unicode, "E00000000");
|
||||||
|
assert!(matches!(
|
||||||
|
unicode.read_raw(
|
||||||
|
&reference,
|
||||||
|
CapturePurpose::Observation,
|
||||||
|
RawField::ToolArguments,
|
||||||
|
1,
|
||||||
|
8
|
||||||
|
),
|
||||||
|
Err(CaptureError::InvalidRange)
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
unicode.read_raw(
|
||||||
|
&reference,
|
||||||
|
CapturePurpose::Observation,
|
||||||
|
RawField::ToolArguments,
|
||||||
|
0,
|
||||||
|
3
|
||||||
|
),
|
||||||
|
Err(CaptureError::BudgetTooSmall)
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
unicode.read_raw(
|
||||||
|
&reference,
|
||||||
|
CapturePurpose::Observation,
|
||||||
|
RawField::ToolArguments,
|
||||||
|
5,
|
||||||
|
8
|
||||||
|
),
|
||||||
|
Err(CaptureError::InvalidRange)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn escaped_json_slice_budget_includes_metadata_and_bound_continuation() {
|
||||||
|
let body = "\"\\\n\t🙂日本語".repeat(500);
|
||||||
|
let capture = SessionCapture::new(
|
||||||
|
"synthetic-session",
|
||||||
|
"segment",
|
||||||
|
vec![Item::tool_call("call", "tool", body.clone())],
|
||||||
|
);
|
||||||
|
let other_session = SessionCapture::new(
|
||||||
|
"other-session",
|
||||||
|
"segment",
|
||||||
|
vec![Item::tool_call("call", "tool", body.clone())],
|
||||||
|
);
|
||||||
|
let changed = SessionCapture::new(
|
||||||
|
"synthetic-session",
|
||||||
|
"segment",
|
||||||
|
vec![Item::tool_call("call", "tool", "changed")],
|
||||||
|
);
|
||||||
|
let mut cursor = None;
|
||||||
|
let mut actual = String::new();
|
||||||
|
let budget = 1400;
|
||||||
|
loop {
|
||||||
|
let part = capture
|
||||||
|
.slice(CapturePurpose::Recording, cursor.as_ref(), 1, budget)
|
||||||
|
.unwrap();
|
||||||
|
assert!(part.encoded_len() <= budget);
|
||||||
|
assert_eq!(part.encoded_len(), part.as_json().len());
|
||||||
|
assert_eq!(serde_json::to_vec(part.slice()).unwrap(), part.as_json());
|
||||||
|
assert_eq!(part.slice().items.len(), 1);
|
||||||
|
let item = &part.slice().items[0];
|
||||||
|
assert_eq!(item.offset, actual.len());
|
||||||
|
actual.push_str(&item.text);
|
||||||
|
cursor = part.slice().next_cursor.clone();
|
||||||
|
if let Some(cursor) = &cursor {
|
||||||
|
assert!(matches!(
|
||||||
|
other_session.slice(CapturePurpose::Recording, Some(cursor), 1, budget),
|
||||||
|
Err(CaptureError::ForeignReference)
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
changed.slice(CapturePurpose::Recording, Some(cursor), 1, budget),
|
||||||
|
Err(CaptureError::ForeignReference)
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
capture.slice(CapturePurpose::Observation, Some(cursor), 1, budget),
|
||||||
|
Err(CaptureError::ForeignReference)
|
||||||
|
));
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(actual, body);
|
||||||
|
assert!(matches!(
|
||||||
|
capture.slice(CapturePurpose::Recording, None, 1, 8),
|
||||||
|
Err(CaptureError::BudgetTooSmall)
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
capture.slice(CapturePurpose::Recording, None, 0, budget),
|
||||||
|
Err(CaptureError::BudgetTooSmall)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn snapshot_immutability_and_scoped_operations_reject_foreign_or_unknown_refs_and_bad_ranges() {
|
||||||
|
let mut source = vec![
|
||||||
|
Item::user_message("before"),
|
||||||
|
Item::assistant_message("answer"),
|
||||||
|
];
|
||||||
|
let capture = SessionCapture::new("s", "seg", source.clone());
|
||||||
|
let fixed_bytes = capture.encode(CaptureLimits::default()).unwrap();
|
||||||
|
let first = reference(&capture, "E00000000");
|
||||||
|
let second = reference(&capture, "E00000001");
|
||||||
|
source.push(Item::user_message("appended"));
|
||||||
|
source[0] = Item::user_message("compacted");
|
||||||
|
source.clear();
|
||||||
|
assert_eq!(
|
||||||
|
capture.encode(CaptureLimits::default()).unwrap(),
|
||||||
|
fixed_bytes
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
capture.clone().encode(CaptureLimits::default()).unwrap(),
|
||||||
|
fixed_bytes
|
||||||
|
);
|
||||||
|
let foreign = SessionCapture::new(
|
||||||
|
"different",
|
||||||
|
"seg",
|
||||||
|
vec![
|
||||||
|
Item::user_message("before"),
|
||||||
|
Item::assistant_message("answer"),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
assert!(matches!(
|
||||||
|
foreign.resolve_ref(&first, CapturePurpose::Observation),
|
||||||
|
Err(CaptureError::ForeignReference)
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
capture.checked_range(&second, &first, CapturePurpose::Observation),
|
||||||
|
Err(CaptureError::InvalidRange)
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
capture
|
||||||
|
.checked_range(&first, &second, CapturePurpose::Observation)
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
let unknown = SessionEntryRef::parse("E99999999").unwrap();
|
||||||
|
assert_eq!(capture.view_order_for_ref(&unknown), None);
|
||||||
|
assert!(matches!(
|
||||||
|
capture.scoped_ref(&unknown),
|
||||||
|
Err(CaptureError::UnknownReference)
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
capture
|
||||||
|
.search_checked(&SearchOptions {
|
||||||
|
from: Some(unknown.clone()),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
capture
|
||||||
|
.search(&SearchOptions {
|
||||||
|
through: Some(unknown),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
capture
|
||||||
|
.read_checked(ReadSelector::EntryRange([1, 0]), ReadOptions::default())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
capture
|
||||||
|
.read_checked(ReadSelector::EntryRange([0, 99]), ReadOptions::default())
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn slices_preserve_empty_and_long_result_fields_under_item_and_metadata_budgets() {
|
||||||
|
let summary = "summary\n\"🙂".repeat(100);
|
||||||
|
let content = "content\t\\日本語".repeat(200);
|
||||||
|
let capture = SessionCapture::new(
|
||||||
|
"s",
|
||||||
|
"seg",
|
||||||
|
vec![
|
||||||
|
Item::tool_result_item("first", summary.clone(), Some(content.clone()), true),
|
||||||
|
Item::tool_result_item("empty", "", Some(String::new()), false),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
let mut cursor = None;
|
||||||
|
let mut fields: Vec<(String, RawField, String)> = Vec::new();
|
||||||
|
loop {
|
||||||
|
let part = capture
|
||||||
|
.slice(CapturePurpose::Recording, cursor.as_ref(), 2, 1500)
|
||||||
|
.unwrap();
|
||||||
|
assert!(part.slice().items.len() <= 2);
|
||||||
|
assert!(serde_json::to_vec(part.slice()).unwrap().len() <= 1500);
|
||||||
|
for item in &part.slice().items {
|
||||||
|
let id = item.reference.entry_ref.to_string();
|
||||||
|
if let Some((last_id, last_field, text)) = fields.last_mut() {
|
||||||
|
if *last_id == id && *last_field == item.field {
|
||||||
|
assert_eq!(text.len(), item.offset);
|
||||||
|
text.push_str(&item.text);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(item.offset, 0);
|
||||||
|
fields.push((id, item.field, item.text.clone()));
|
||||||
|
}
|
||||||
|
cursor = part.slice().next_cursor.clone();
|
||||||
|
if cursor.is_none() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
fields,
|
||||||
|
vec![
|
||||||
|
("E00000000".into(), RawField::ToolSummary, summary),
|
||||||
|
("E00000000".into(), RawField::ToolContent, content),
|
||||||
|
("E00000001".into(), RawField::ToolSummary, String::new()),
|
||||||
|
("E00000001".into(), RawField::ToolContent, String::new()),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
let metadata_heavy = SessionCapture::new(
|
||||||
|
"s",
|
||||||
|
"seg",
|
||||||
|
vec![Item::tool_call("call", "name".repeat(200), "x")],
|
||||||
|
);
|
||||||
|
assert!(matches!(
|
||||||
|
metadata_heavy.slice(CapturePurpose::Recording, None, 1, 700),
|
||||||
|
Err(CaptureError::BudgetTooSmall)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn evidence_uses_original_segment_positions_after_roundtrip_and_view_selection() {
|
||||||
|
let full = SessionCapture::new(
|
||||||
|
"s",
|
||||||
|
"selected",
|
||||||
|
vec![
|
||||||
|
Item::user_message("first"),
|
||||||
|
Item::assistant_message("second"),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
let mut dto = full.to_dto();
|
||||||
|
dto.entries[0].source.segment_id = "original-a".into();
|
||||||
|
dto.entries[0].source.entry_index = 80;
|
||||||
|
dto.entries[1].source.segment_id = "original-b".into();
|
||||||
|
dto.entries[1].source.entry_index = 3;
|
||||||
|
let encoded = dto.encode(CaptureLimits::default()).unwrap();
|
||||||
|
let restored = SessionCapture::decode(&encoded, CaptureLimits::default()).unwrap();
|
||||||
|
for (id, segment, index, order) in [
|
||||||
|
("E00000000", "original-a", 80, 0),
|
||||||
|
("E00000001", "original-b", 3, 1),
|
||||||
|
] {
|
||||||
|
let evidence = restored.evidence_for(id).unwrap();
|
||||||
|
assert_eq!(evidence.segment_id, segment);
|
||||||
|
assert_eq!(evidence.entry_range, [index; 2]);
|
||||||
|
let read = restored.read(ReadSelector::Id(id), ReadOptions::default());
|
||||||
|
assert_eq!(read.entries[0].entry_range, [order; 2]);
|
||||||
|
}
|
||||||
|
let selected = restored.select_order_range(1..2).unwrap();
|
||||||
|
assert_eq!(selected.overview()[0].entry_range, [0, 0]);
|
||||||
|
let evidence = selected.evidence_for("E00000001").unwrap();
|
||||||
|
assert_eq!(evidence.segment_id, "original-b");
|
||||||
|
assert_eq!(evidence.entry_range, [3, 3]);
|
||||||
|
let roundtrip = SessionCapture::decode(
|
||||||
|
&selected.encode(CaptureLimits::default()).unwrap(),
|
||||||
|
CaptureLimits::default(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(roundtrip.to_dto(), selected.to_dto());
|
||||||
|
assert_eq!(
|
||||||
|
roundtrip.evidence_for("E00000001").unwrap().entry_range,
|
||||||
|
[3, 3]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fixed_history_selection_preserves_original_coordinates_and_checks_private_position_bounds() {
|
||||||
|
let history = vec![
|
||||||
|
history("old", Item::user_message("old"), None),
|
||||||
|
history("hidden", Item::reasoning("PRIVATE"), None),
|
||||||
|
history("selected", Item::user_message("selected"), None),
|
||||||
|
history("private-tail", Item::system_message("PRIVATE"), None),
|
||||||
|
];
|
||||||
|
let full = SessionCapture::from_complete_history("s", "seg", history);
|
||||||
|
let suffix = full.select_order_range(1..4).unwrap();
|
||||||
|
let original = full
|
||||||
|
.entries()
|
||||||
|
.find(|entry| entry.entry_id == "selected")
|
||||||
|
.unwrap();
|
||||||
|
let selected = suffix.entries().next().unwrap();
|
||||||
|
assert_eq!(selected.entry_id, original.entry_id);
|
||||||
|
assert_eq!(selected.source, original.source);
|
||||||
|
assert_eq!(selected.body, original.body);
|
||||||
|
assert_eq!(selected.policy, original.policy);
|
||||||
|
assert!(suffix.evidence_for("Eold").is_none());
|
||||||
|
assert_eq!(suffix.search(&SearchOptions::default()).len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
suffix.view_order_for_ref(&SessionEntryRef::parse("Eselected").unwrap()),
|
||||||
|
Some(1)
|
||||||
|
);
|
||||||
|
assert_eq!(selected.source.entry_index, 2);
|
||||||
|
assert_eq!(selected.order, 1);
|
||||||
|
assert_eq!(suffix.to_dto().view_entry_count, 3);
|
||||||
|
assert_eq!(
|
||||||
|
suffix.evidence_for("Eselected").unwrap().entry_range,
|
||||||
|
[2, 2]
|
||||||
|
);
|
||||||
|
let nested = suffix.select_order_range(1..2).unwrap();
|
||||||
|
assert_eq!(nested.entries().next().unwrap().order, 0);
|
||||||
|
assert_eq!(nested.entries().next().unwrap().source, original.source);
|
||||||
|
let restored = SessionCapture::decode(
|
||||||
|
&suffix.encode(CaptureLimits::default()).unwrap(),
|
||||||
|
CaptureLimits::default(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(restored.to_dto(), suffix.to_dto());
|
||||||
|
assert!(full.select_order_range(4..5).is_err());
|
||||||
|
let reversed = std::ops::Range { start: 3, end: 2 };
|
||||||
|
assert!(full.select_order_range(reversed).is_err());
|
||||||
|
let hidden_only = full.select_order_range(3..4).unwrap();
|
||||||
|
assert_eq!(hidden_only.entries().len(), 0);
|
||||||
|
assert_eq!(hidden_only.to_dto().view_entry_count, 1);
|
||||||
|
let mut invalid = full.to_dto();
|
||||||
|
invalid.view_entry_count = 2; // the selected entry's view order is outside this extent
|
||||||
|
assert!(SessionCapture::try_from_dto(invalid, CaptureLimits::default()).is_err());
|
||||||
|
assert_eq!(
|
||||||
|
full.retain_entries(|entry| entry.entry_id == "selected")
|
||||||
|
.entries()
|
||||||
|
.next()
|
||||||
|
.unwrap()
|
||||||
|
.source,
|
||||||
|
original.source
|
||||||
|
);
|
||||||
|
assert_eq!(full.to_dto().view_entry_count, 4);
|
||||||
|
}
|
||||||
@@ -23,7 +23,7 @@ serde.workspace = true
|
|||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
session-store.workspace = true
|
session-store.workspace = true
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
tokio = { workspace = true, features = ["rt", "sync", "time"] }
|
tokio = { workspace = true, features = ["rt", "sync", "time", "net", "io-util", "io-std", "signal"] }
|
||||||
uuid = { workspace = true, features = ["v7"] }
|
uuid = { workspace = true, features = ["v7"] }
|
||||||
worker.workspace = true
|
worker.workspace = true
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ A Host that starts existing Workers in the same process. Assembles interactive W
|
|||||||
|
|
||||||
## Reference index
|
## Reference index
|
||||||
|
|
||||||
- [`host`] — bootstrap and confirmed shutdown.
|
- [`host`] — bootstrap and confirmed shutdown, with or without a TUI.
|
||||||
|
- [`unix`] — optional operator transport; disconnect does not release execution ownership.
|
||||||
- [`launch`] — Profile, cwd, and Subject validation before launch.
|
- [`launch`] — Profile, cwd, and Subject validation before launch.
|
||||||
- [`store`] — Worker records and leases.
|
- [`store`] — Worker records and leases.
|
||||||
- [`jobs`] — Job execution and independent result and cleanup lifetimes.
|
- [`jobs`] — Job execution and independent result and cleanup lifetimes.
|
||||||
|
|||||||
@@ -24,7 +24,6 @@ use session_store::{
|
|||||||
};
|
};
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
use worker::bootstrap::{WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout};
|
use worker::bootstrap::{WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout};
|
||||||
use worker::controller::WorkerControllerTransport;
|
|
||||||
use worker::ipc::protocol_session::{
|
use worker::ipc::protocol_session::{
|
||||||
WorkerProtocolSessionStreams, dispatch_worker_protocol_method, live_log_entry_event,
|
WorkerProtocolSessionStreams, dispatch_worker_protocol_method, live_log_entry_event,
|
||||||
subscribe_worker_protocol_session,
|
subscribe_worker_protocol_session,
|
||||||
@@ -222,7 +221,6 @@ impl StandaloneHost {
|
|||||||
workspace_context,
|
workspace_context,
|
||||||
filesystem_authority,
|
filesystem_authority,
|
||||||
WorkerBootstrapLayout::Direct { runtime_base },
|
WorkerBootstrapLayout::Direct { runtime_base },
|
||||||
WorkerControllerTransport::InProcess,
|
|
||||||
);
|
);
|
||||||
if let Some(subject) = &subject {
|
if let Some(subject) = &subject {
|
||||||
bootstrap = bootstrap.with_subjektiv_host(subject.host.clone());
|
bootstrap = bootstrap.with_subjektiv_host(subject.host.clone());
|
||||||
@@ -430,7 +428,6 @@ impl StandaloneHost {
|
|||||||
workspace_context,
|
workspace_context,
|
||||||
filesystem_authority,
|
filesystem_authority,
|
||||||
WorkerBootstrapLayout::Direct { runtime_base },
|
WorkerBootstrapLayout::Direct { runtime_base },
|
||||||
WorkerControllerTransport::InProcess,
|
|
||||||
);
|
);
|
||||||
if let Some(subject) = &subject {
|
if let Some(subject) = &subject {
|
||||||
bootstrap = bootstrap.with_subjektiv_host(subject.host.clone());
|
bootstrap = bootstrap.with_subjektiv_host(subject.host.clone());
|
||||||
@@ -520,10 +517,7 @@ impl StandaloneHost {
|
|||||||
/// Working events, committed session entries, alert snapshots, and the
|
/// Working events, committed session entries, alert snapshots, and the
|
||||||
/// initial history snapshot are merged behind the client boundary.
|
/// initial history snapshot are merged behind the client boundary.
|
||||||
pub fn connect(&self) -> Client<InProcessSocket> {
|
pub fn connect(&self) -> Client<InProcessSocket> {
|
||||||
let streams = subscribe_worker_protocol_session(&self.handle);
|
Client::new(connect_protocol(self.handle.clone()))
|
||||||
let (socket, peer) = InProcessSocket::pair();
|
|
||||||
tokio::spawn(run_protocol_session(self.handle.clone(), streams, peer));
|
|
||||||
Client::new(socket)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Logical Subject disconnection, not Worker shutdown or completion of its represented work.
|
/// Logical Subject disconnection, not Worker shutdown or completion of its represented work.
|
||||||
@@ -565,6 +559,35 @@ impl StandaloneHost {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Run without a TUI until an admitted protocol Shutdown, controller exit, or process signal.
|
||||||
|
/// Client EOF only disconnects that client. All exits confirm the same production shutdown
|
||||||
|
/// and lease release as the in-process UI; an unconfirmed shutdown retains the lease.
|
||||||
|
pub async fn serve_unix(
|
||||||
|
self,
|
||||||
|
endpoint: crate::unix::UnixEndpoint,
|
||||||
|
) -> Result<(), crate::HeadlessError> {
|
||||||
|
let mut adapter = crate::unix::UnixAdapter::start(endpoint, self.handle.clone());
|
||||||
|
let signal = async {
|
||||||
|
let mut terminate =
|
||||||
|
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
|
||||||
|
tokio::select! {
|
||||||
|
result = tokio::signal::ctrl_c() => result,
|
||||||
|
_ = terminate.recv() => Ok(()),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let result = tokio::select! {
|
||||||
|
_ = self.handle.protocol_closed() => Ok(()),
|
||||||
|
result = signal => result,
|
||||||
|
result = adapter.ended() => result,
|
||||||
|
};
|
||||||
|
// Reclaim the public entry and every connection before releasing execution ownership.
|
||||||
|
let transport = adapter.shutdown().await;
|
||||||
|
self.shutdown().await?;
|
||||||
|
result?;
|
||||||
|
transport?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub fn with_shutdown_timeout(mut self, shutdown_timeout: Duration) -> Self {
|
pub fn with_shutdown_timeout(mut self, shutdown_timeout: Duration) -> Self {
|
||||||
self.shutdown_timeout = shutdown_timeout;
|
self.shutdown_timeout = shutdown_timeout;
|
||||||
self
|
self
|
||||||
@@ -647,9 +670,8 @@ impl Drop for StandaloneHost {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The in-process channel is owned by the local operator's Host, not an
|
// Both in-process views and credential-checked Unix clients are admitted by this operator's
|
||||||
// arbitrary socket or Backend account. decode_method discards wire provenance;
|
// Host. decode_method discards wire provenance; only this boundary stamps local human input.
|
||||||
// only this boundary can stamp accepted local human input.
|
|
||||||
fn authorize_local_input(method: protocol::Method) -> protocol::Method {
|
fn authorize_local_input(method: protocol::Method) -> protocol::Method {
|
||||||
use protocol::{AuthenticatedInputSource, Method};
|
use protocol::{AuthenticatedInputSource, Method};
|
||||||
match method {
|
match method {
|
||||||
@@ -692,6 +714,13 @@ fn authorize_local_input(method: protocol::Method) -> protocol::Method {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn connect_protocol(handle: worker::WorkerHandle) -> InProcessSocket {
|
||||||
|
let streams = subscribe_worker_protocol_session(&handle);
|
||||||
|
let (socket, peer) = InProcessSocket::pair();
|
||||||
|
tokio::spawn(run_protocol_session(handle, streams, peer));
|
||||||
|
socket
|
||||||
|
}
|
||||||
|
|
||||||
async fn run_protocol_session(
|
async fn run_protocol_session(
|
||||||
handle: worker::WorkerHandle,
|
handle: worker::WorkerHandle,
|
||||||
streams: WorkerProtocolSessionStreams,
|
streams: WorkerProtocolSessionStreams,
|
||||||
|
|||||||
@@ -25,7 +25,46 @@ use thiserror::Error;
|
|||||||
#[path = "job_store/legacy_migrations.rs"]
|
#[path = "job_store/legacy_migrations.rs"]
|
||||||
mod legacy_migrations;
|
mod legacy_migrations;
|
||||||
|
|
||||||
const SCHEMA_VERSION: i64 = 4;
|
const SCHEMA_VERSION: i64 = 5;
|
||||||
|
|
||||||
|
/// Availability of the immutable input body, independent of execution outcome.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum JobInputDisposition {
|
||||||
|
Available,
|
||||||
|
Reclaimed,
|
||||||
|
Erased,
|
||||||
|
Invalidated,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl JobInputDisposition {
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Available => "available",
|
||||||
|
Self::Reclaimed => "reclaimed",
|
||||||
|
Self::Erased => "erased",
|
||||||
|
Self::Invalidated => "invalidated",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Host-authorized retirement; erasure and legacy invalidation do not change outcomes.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum JobInputRetirementReason {
|
||||||
|
Completed,
|
||||||
|
ExplicitErasure,
|
||||||
|
LegacyInvalidation,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl JobInputRetirementReason {
|
||||||
|
pub fn disposition(self) -> JobInputDisposition {
|
||||||
|
match self {
|
||||||
|
Self::Completed => JobInputDisposition::Reclaimed,
|
||||||
|
Self::ExplicitErasure => JobInputDisposition::Erased,
|
||||||
|
Self::LegacyInvalidation => JobInputDisposition::Invalidated,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
@@ -67,7 +106,11 @@ impl JobAttemptState {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
pub struct JobSnapshot {
|
pub struct JobSnapshot {
|
||||||
|
/// Receipt metadata. After retirement `input` is null; use the verified loader to execute.
|
||||||
pub request: JobRequest,
|
pub request: JobRequest,
|
||||||
|
pub input_digest: String,
|
||||||
|
pub intent_fingerprint: String,
|
||||||
|
pub input_disposition: JobInputDisposition,
|
||||||
/// Durable Host-issued domain delegation, not model input or Profile identity.
|
/// Durable Host-issued domain delegation, not model input or Profile identity.
|
||||||
pub domain_grant: Option<Value>,
|
pub domain_grant: Option<Value>,
|
||||||
pub state: JobState,
|
pub state: JobState,
|
||||||
@@ -77,14 +120,27 @@ pub struct JobSnapshot {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl JobSnapshot {
|
impl JobSnapshot {
|
||||||
|
/// Load only the original available body, checking both content and immutable intent.
|
||||||
|
pub fn load_executable_request(&self) -> Result<&JobRequest, JobStoreError> {
|
||||||
|
if self.input_disposition != JobInputDisposition::Available {
|
||||||
|
return Err(JobStoreError::InvalidTransition("Job input is retired"));
|
||||||
|
}
|
||||||
|
self.request.validate()?;
|
||||||
|
if self.request.input_digest()? != self.input_digest
|
||||||
|
|| self.request.fingerprint()? != self.intent_fingerprint
|
||||||
|
{
|
||||||
|
return Err(JobStoreError::Corrupt(
|
||||||
|
"immutable input or intent fingerprint mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(&self.request)
|
||||||
|
}
|
||||||
|
|
||||||
/// Shared Feature-facing read contract; no Workspace or execution identity leaks.
|
/// Shared Feature-facing read contract; no Workspace or execution identity leaks.
|
||||||
pub fn outcome(&self) -> job::JobOutcome {
|
pub fn outcome(&self) -> job::JobOutcome {
|
||||||
job::JobOutcome {
|
job::JobOutcome {
|
||||||
job_id: self.request.job_id.clone(),
|
job_id: self.request.job_id.clone(),
|
||||||
input_digest: self
|
input_digest: self.input_digest.clone(),
|
||||||
.request
|
|
||||||
.input_digest()
|
|
||||||
.expect("validated immutable Job input"),
|
|
||||||
attempt_id: self.attempt.attempt_id.clone(),
|
attempt_id: self.attempt.attempt_id.clone(),
|
||||||
attempt: self.attempt.number,
|
attempt: self.attempt.number,
|
||||||
state: match self.state {
|
state: match self.state {
|
||||||
@@ -204,9 +260,23 @@ impl JobStore {
|
|||||||
tx.execute_batch(RECONCILIATION_SCHEMA)?;
|
tx.execute_batch(RECONCILIATION_SCHEMA)?;
|
||||||
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
|
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
|
||||||
}
|
}
|
||||||
SCHEMA_VERSION => {}
|
4 | SCHEMA_VERSION => {}
|
||||||
other => return Err(JobStoreError::UnsupportedSchema(other)),
|
other => return Err(JobStoreError::UnsupportedSchema(other)),
|
||||||
}
|
}
|
||||||
|
if version < 5 {
|
||||||
|
tx.execute_batch(INPUT_RETENTION_SCHEMA)?;
|
||||||
|
let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?;
|
||||||
|
if has_legacy_archive {
|
||||||
|
tx.execute_batch("DROP TRIGGER legacy_job_archive_immutable;
|
||||||
|
CREATE TRIGGER legacy_job_archive_immutable BEFORE UPDATE OF job_id,fingerprint,attempts_json ON legacy_job_archive
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'immutable legacy Job archive'); END;
|
||||||
|
CREATE TRIGGER legacy_job_archive_input_retirement BEFORE UPDATE OF request_json ON legacy_job_archive
|
||||||
|
WHEN NOT EXISTS (SELECT 1 FROM job_intents WHERE job_id=OLD.job_id AND input_disposition<>'available')
|
||||||
|
OR (json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') AND json_type(NEW.request_json,'$.input')='null') IS NOT TRUE
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'immutable legacy Job input'); END;")?;
|
||||||
|
}
|
||||||
|
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
|
||||||
|
}
|
||||||
// An accepted result is the only evidence of success. Reserved attempts
|
// An accepted result is the only evidence of success. Reserved attempts
|
||||||
// were never started and remain pending; cancelled attempts never resume.
|
// were never started and remain pending; cancelled attempts never resume.
|
||||||
tx.execute_batch(
|
tx.execute_batch(
|
||||||
@@ -249,10 +319,7 @@ impl JobStore {
|
|||||||
[&request.job_id],
|
[&request.job_id],
|
||||||
|row| row.get(0),
|
|row| row.get(0),
|
||||||
)?;
|
)?;
|
||||||
if stored != fingerprint
|
if stored != fingerprint || snapshot.domain_grant != grant {
|
||||||
|| snapshot.request != request
|
|
||||||
|| snapshot.domain_grant != grant
|
|
||||||
{
|
|
||||||
return Err(JobStoreError::IntentConflict(request.job_id));
|
return Err(JobStoreError::IntentConflict(request.job_id));
|
||||||
}
|
}
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
@@ -334,6 +401,7 @@ impl JobStore {
|
|||||||
.conn
|
.conn
|
||||||
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
let previous = require_snapshot(&tx, job_id)?;
|
let previous = require_snapshot(&tx, job_id)?;
|
||||||
|
previous.load_executable_request()?;
|
||||||
if previous.state != JobState::Failed || previous.attempt.state != JobAttemptState::Failed {
|
if previous.state != JobState::Failed || previous.attempt.state != JobAttemptState::Failed {
|
||||||
return Err(JobStoreError::InvalidTransition(
|
return Err(JobStoreError::InvalidTransition(
|
||||||
"retry requires known failure",
|
"retry requires known failure",
|
||||||
@@ -354,6 +422,81 @@ impl JobStore {
|
|||||||
Ok(snapshot)
|
Ok(snapshot)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Historical snapshots for one domain Session, including terminal and retired Jobs.
|
||||||
|
pub(crate) fn list_session_snapshots(
|
||||||
|
&self,
|
||||||
|
session_id: &str,
|
||||||
|
after: Option<&str>,
|
||||||
|
limit: usize,
|
||||||
|
) -> Result<Vec<JobSnapshot>, JobStoreError> {
|
||||||
|
let mut stmt = self.conn.prepare("SELECT i.job_id FROM job_intents i JOIN job_domain_grants g ON g.job_id=i.job_id WHERE json_extract(g.grant_json,'$.session_id')=?1 AND (?2 IS NULL OR i.job_id>?2) ORDER BY i.job_id LIMIT ?3")?;
|
||||||
|
let ids = stmt
|
||||||
|
.query_map(
|
||||||
|
params![session_id, after, limit.clamp(1, 100) as i64],
|
||||||
|
|r| r.get::<_, String>(0),
|
||||||
|
)?
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||||
|
ids.into_iter()
|
||||||
|
.map(|id| require_snapshot(&self.conn, &id))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The Host must fence domain mutation and delivery before erasure/invalidation.
|
||||||
|
/// This atomically prevents dispatch/retry/result acceptance, without rewriting outcome.
|
||||||
|
pub(crate) fn reclaim_input(
|
||||||
|
&mut self,
|
||||||
|
job_id: &str,
|
||||||
|
expected_input_digest: &str,
|
||||||
|
reason: JobInputRetirementReason,
|
||||||
|
) -> Result<JobSnapshot, JobStoreError> {
|
||||||
|
let tx = self
|
||||||
|
.conn
|
||||||
|
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
|
let snapshot = require_snapshot(&tx, job_id)?;
|
||||||
|
if snapshot.input_digest != expected_input_digest {
|
||||||
|
return Err(JobStoreError::DigestMismatch);
|
||||||
|
}
|
||||||
|
let desired = reason.disposition();
|
||||||
|
if snapshot.input_disposition == desired {
|
||||||
|
tx.commit()?;
|
||||||
|
reclaim_retired_input_storage(&self.conn)?;
|
||||||
|
return Ok(snapshot);
|
||||||
|
}
|
||||||
|
if reason == JobInputRetirementReason::Completed
|
||||||
|
&& (snapshot.state != JobState::Completed
|
||||||
|
|| snapshot.attempt.state != JobAttemptState::Completed
|
||||||
|
|| snapshot.result.is_none())
|
||||||
|
{
|
||||||
|
return Err(JobStoreError::InvalidTransition(
|
||||||
|
"reclaim requires known completed result",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if snapshot.input_disposition != JobInputDisposition::Available
|
||||||
|
&& reason != JobInputRetirementReason::ExplicitErasure
|
||||||
|
{
|
||||||
|
return Err(JobStoreError::InvalidTransition(
|
||||||
|
"retired input cannot be restored or reclassified",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if reason == JobInputRetirementReason::Completed {
|
||||||
|
snapshot.load_executable_request()?;
|
||||||
|
}
|
||||||
|
let mut metadata = snapshot.request.clone();
|
||||||
|
metadata.input = Value::Null;
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE job_intents SET request_json=?2, input_disposition=?3 WHERE job_id=?1",
|
||||||
|
params![job_id, serde_json::to_string(&metadata)?, desired.as_str()],
|
||||||
|
)?;
|
||||||
|
let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?;
|
||||||
|
if has_legacy_archive {
|
||||||
|
tx.execute("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input',json('null')) WHERE job_id=?1", [job_id])?;
|
||||||
|
}
|
||||||
|
let snapshot = require_snapshot(&tx, job_id)?;
|
||||||
|
tx.commit()?;
|
||||||
|
reclaim_retired_input_storage(&self.conn)?;
|
||||||
|
Ok(snapshot)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn get(&self, job_id: &str) -> Result<JobSnapshot, JobStoreError> {
|
pub(crate) fn get(&self, job_id: &str) -> Result<JobSnapshot, JobStoreError> {
|
||||||
require_snapshot(&self.conn, job_id)
|
require_snapshot(&self.conn, job_id)
|
||||||
}
|
}
|
||||||
@@ -383,7 +526,7 @@ impl JobStore {
|
|||||||
let mut statement = self.conn.prepare(
|
let mut statement = self.conn.prepare(
|
||||||
"SELECT i.job_id FROM job_intents i JOIN job_attempts a
|
"SELECT i.job_id FROM job_intents i JOIN job_attempts a
|
||||||
ON a.job_id = i.job_id AND a.number = i.current_attempt
|
ON a.job_id = i.job_id AND a.number = i.current_attempt
|
||||||
WHERE i.state = 'pending' AND a.state = 'reserved' ORDER BY i.rowid",
|
WHERE i.state = 'pending' AND i.input_disposition='available' AND a.state = 'reserved' ORDER BY i.rowid",
|
||||||
)?;
|
)?;
|
||||||
let ids = statement
|
let ids = statement
|
||||||
.query_map([], |row| row.get::<_, String>(0))?
|
.query_map([], |row| row.get::<_, String>(0))?
|
||||||
@@ -406,6 +549,7 @@ impl JobStore {
|
|||||||
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
let snapshot = require_snapshot(&tx, job_id)?;
|
let snapshot = require_snapshot(&tx, job_id)?;
|
||||||
check_attempt(&snapshot, attempt_id)?;
|
check_attempt(&snapshot, attempt_id)?;
|
||||||
|
snapshot.load_executable_request()?;
|
||||||
if snapshot.state != JobState::Pending
|
if snapshot.state != JobState::Pending
|
||||||
|| snapshot.attempt.state != JobAttemptState::Reserved
|
|| snapshot.attempt.state != JobAttemptState::Reserved
|
||||||
{
|
{
|
||||||
@@ -447,7 +591,7 @@ impl JobStore {
|
|||||||
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
let snapshot = require_snapshot(&tx, &bound.job_id)?;
|
let snapshot = require_snapshot(&tx, &bound.job_id)?;
|
||||||
check_attempt(&snapshot, &bound.attempt_id)?;
|
check_attempt(&snapshot, &bound.attempt_id)?;
|
||||||
if snapshot.request.input_digest()? != bound.input_digest {
|
if snapshot.input_digest != bound.input_digest {
|
||||||
return Err(JobStoreError::DigestMismatch);
|
return Err(JobStoreError::DigestMismatch);
|
||||||
}
|
}
|
||||||
let (digest, encoded) =
|
let (digest, encoded) =
|
||||||
@@ -466,6 +610,7 @@ impl JobStore {
|
|||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
return Ok(snapshot);
|
return Ok(snapshot);
|
||||||
}
|
}
|
||||||
|
snapshot.load_executable_request()?;
|
||||||
if snapshot.state != JobState::Pending
|
if snapshot.state != JobState::Pending
|
||||||
|| snapshot.attempt.state != JobAttemptState::Dispatched
|
|| snapshot.attempt.state != JobAttemptState::Dispatched
|
||||||
{
|
{
|
||||||
@@ -649,7 +794,7 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
|||||||
let row = conn
|
let row = conn
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT i.request_json, i.state, i.acknowledged, a.attempt_id, a.number,
|
"SELECT i.request_json, i.state, i.acknowledged, a.attempt_id, a.number,
|
||||||
a.state, a.failure, a.result_json FROM job_intents i JOIN job_attempts a
|
a.state, a.failure, a.result_json, a.input_digest, i.fingerprint, i.input_disposition FROM job_intents i JOIN job_attempts a
|
||||||
ON a.job_id = i.job_id AND a.number = i.current_attempt WHERE i.job_id = ?1",
|
ON a.job_id = i.job_id AND a.number = i.current_attempt WHERE i.job_id = ?1",
|
||||||
[job_id],
|
[job_id],
|
||||||
|row| {
|
|row| {
|
||||||
@@ -662,12 +807,27 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
|||||||
row.get::<_, String>(5)?,
|
row.get::<_, String>(5)?,
|
||||||
row.get::<_, Option<String>>(6)?,
|
row.get::<_, Option<String>>(6)?,
|
||||||
row.get::<_, Option<String>>(7)?,
|
row.get::<_, Option<String>>(7)?,
|
||||||
|
row.get::<_, String>(8)?,
|
||||||
|
row.get::<_, String>(9)?,
|
||||||
|
row.get::<_, String>(10)?,
|
||||||
))
|
))
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
row.map(
|
row.map(
|
||||||
|(request, state, acknowledged, attempt_id, number, attempt_state, failure, result)| {
|
|(
|
||||||
|
request,
|
||||||
|
state,
|
||||||
|
acknowledged,
|
||||||
|
attempt_id,
|
||||||
|
number,
|
||||||
|
attempt_state,
|
||||||
|
failure,
|
||||||
|
result,
|
||||||
|
input_digest,
|
||||||
|
intent_fingerprint,
|
||||||
|
disposition,
|
||||||
|
)| {
|
||||||
let state = match state.as_str() {
|
let state = match state.as_str() {
|
||||||
"pending" => JobState::Pending,
|
"pending" => JobState::Pending,
|
||||||
"completed" => JobState::Completed,
|
"completed" => JobState::Completed,
|
||||||
@@ -693,8 +853,17 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
|||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let request_value: Value = serde_json::from_str(&request)?;
|
||||||
|
if request_value.get("input").is_none() {
|
||||||
|
return Err(JobStoreError::Corrupt(
|
||||||
|
"stored Job input field is missing".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
Ok(JobSnapshot {
|
Ok(JobSnapshot {
|
||||||
request: serde_json::from_str(&request)?,
|
request: serde_json::from_value(request_value)?,
|
||||||
|
input_digest,
|
||||||
|
intent_fingerprint,
|
||||||
|
input_disposition: serde_json::from_value(Value::String(disposition))?,
|
||||||
domain_grant: conn
|
domain_grant: conn
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT grant_json FROM job_domain_grants WHERE job_id = ?1",
|
"SELECT grant_json FROM job_domain_grants WHERE job_id = ?1",
|
||||||
@@ -719,6 +888,37 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
|||||||
.transpose()
|
.transpose()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Retirement is already durable before this maintenance step. Failure must be
|
||||||
|
// retried with the same Job/digest/reason; it must never restore input. VACUUM
|
||||||
|
// removes historical free-page copies, including stores written before secure_delete.
|
||||||
|
fn reclaim_retired_input_storage(conn: &Connection) -> Result<(), JobStoreError> {
|
||||||
|
conn.execute_batch("PRAGMA secure_delete=ON; VACUUM;")?;
|
||||||
|
let busy: i64 = conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| r.get(0))?;
|
||||||
|
if busy != 0 {
|
||||||
|
return Err(JobStoreError::InvalidTransition(
|
||||||
|
"input retired; SQLite body cleanup blocked by a reader, retry retirement",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
const INPUT_RETENTION_SCHEMA: &str = "
|
||||||
|
ALTER TABLE job_intents ADD COLUMN input_disposition TEXT NOT NULL DEFAULT 'available'
|
||||||
|
CHECK (input_disposition IN ('available','reclaimed','erased','invalidated'));
|
||||||
|
DROP TRIGGER job_intent_immutable;
|
||||||
|
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF
|
||||||
|
job_id, fingerprint, serialization_key, max_concurrent ON job_intents
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
|
||||||
|
CREATE TRIGGER job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON job_intents
|
||||||
|
WHEN (
|
||||||
|
((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated'))
|
||||||
|
OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased'))
|
||||||
|
AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input')
|
||||||
|
AND json_type(NEW.request_json,'$.input') = 'null'
|
||||||
|
) IS NOT TRUE
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'immutable Job input retirement'); END;
|
||||||
|
";
|
||||||
|
|
||||||
const RECONCILIATION_SCHEMA: &str = "
|
const RECONCILIATION_SCHEMA: &str = "
|
||||||
DROP INDEX job_serialization_singleflight;
|
DROP INDEX job_serialization_singleflight;
|
||||||
CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key)
|
CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key)
|
||||||
@@ -885,7 +1085,11 @@ mod tests {
|
|||||||
Some("unconfirmed"),
|
Some("unconfirmed"),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
store.conn.execute_batch("DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap();
|
store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable;
|
||||||
|
ALTER TABLE job_intents DROP COLUMN input_disposition;
|
||||||
|
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
|
||||||
|
DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap();
|
||||||
drop(store);
|
drop(store);
|
||||||
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
|
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
|
||||||
assert_eq!(reopened.get("first").unwrap(), unknown);
|
assert_eq!(reopened.get("first").unwrap(), unknown);
|
||||||
@@ -911,10 +1115,7 @@ mod tests {
|
|||||||
JobResultSubmission {
|
JobResultSubmission {
|
||||||
job_id: snapshot.request.job_id.clone(),
|
job_id: snapshot.request.job_id.clone(),
|
||||||
attempt_id: snapshot.attempt.attempt_id.clone(),
|
attempt_id: snapshot.attempt.attempt_id.clone(),
|
||||||
input_digest: snapshot
|
input_digest: snapshot.input_digest.clone(),
|
||||||
.request
|
|
||||||
.input_digest()
|
|
||||||
.expect("validated immutable Job input"),
|
|
||||||
result,
|
result,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -926,6 +1127,314 @@ mod tests {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn input_reclaim_reopens_as_receipt_and_exact_replay_never_rehydrates() {
|
||||||
|
let (dir, mut store) = fixture();
|
||||||
|
let original = request("retained");
|
||||||
|
let grant = Some(json!({"session_id":"session-a", "consumer":"fixture"}));
|
||||||
|
let reserved = store
|
||||||
|
.reserve_granted(original.clone(), grant.clone())
|
||||||
|
.unwrap();
|
||||||
|
let running = store
|
||||||
|
.dispatch("retained", &reserved.attempt.attempt_id, 8)
|
||||||
|
.unwrap();
|
||||||
|
let result = submission(&running, json!({"saved":true}));
|
||||||
|
let completed = store.accept(&result).unwrap();
|
||||||
|
store.ack("retained").unwrap();
|
||||||
|
let retired = store
|
||||||
|
.reclaim_input(
|
||||||
|
"retained",
|
||||||
|
&completed.input_digest,
|
||||||
|
JobInputRetirementReason::Completed,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(retired.input_disposition, JobInputDisposition::Reclaimed);
|
||||||
|
assert!(retired.request.input.is_null());
|
||||||
|
assert!(retired.load_executable_request().is_err());
|
||||||
|
assert_eq!(
|
||||||
|
retired.outcome().input_digest,
|
||||||
|
original.input_digest().unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(retired.result, completed.result);
|
||||||
|
drop(store);
|
||||||
|
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
reopened
|
||||||
|
.reserve_granted(original.clone(), grant.clone())
|
||||||
|
.unwrap(),
|
||||||
|
retired
|
||||||
|
);
|
||||||
|
assert_eq!(reopened.accept(&result).unwrap(), retired);
|
||||||
|
assert_eq!(
|
||||||
|
reopened
|
||||||
|
.list_session_snapshots("session-a", None, 1)
|
||||||
|
.unwrap(),
|
||||||
|
vec![retired.clone()]
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
reopened
|
||||||
|
.list_session_snapshots("session-b", None, 100)
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
reopened
|
||||||
|
.list_session_snapshots("session-a", Some("retained"), 100)
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
let mut changed = original.clone();
|
||||||
|
changed.input = json!({"changed":true});
|
||||||
|
assert!(matches!(
|
||||||
|
reopened.reserve_granted(changed, grant),
|
||||||
|
Err(JobStoreError::IntentConflict(_))
|
||||||
|
));
|
||||||
|
assert!(reopened.retry("retained").is_err());
|
||||||
|
assert!(
|
||||||
|
reopened
|
||||||
|
.dispatch("retained", &retired.attempt.attempt_id, 8)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(reopened.conn.execute("UPDATE job_intents SET request_json=?1,input_disposition='available' WHERE job_id='retained'", [serde_json::to_string(&original).unwrap()]).is_err());
|
||||||
|
let erased = reopened
|
||||||
|
.reclaim_input(
|
||||||
|
"retained",
|
||||||
|
&retired.input_digest,
|
||||||
|
JobInputRetirementReason::ExplicitErasure,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
|
||||||
|
assert_eq!(erased.state, JobState::Completed);
|
||||||
|
assert!(erased.acknowledged);
|
||||||
|
assert_eq!(
|
||||||
|
reopened
|
||||||
|
.reserve_granted(original, erased.domain_grant.clone())
|
||||||
|
.unwrap(),
|
||||||
|
erased
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unfinished_unknown_and_failed_inputs_are_kept_until_explicit_invalidation() {
|
||||||
|
let (_dir, mut store) = fixture();
|
||||||
|
for (id, state) in [
|
||||||
|
("pending", None),
|
||||||
|
("unknown", Some(JobAttemptState::Unknown)),
|
||||||
|
("failed", Some(JobAttemptState::Failed)),
|
||||||
|
] {
|
||||||
|
let original = request(id);
|
||||||
|
let snapshot = store.reserve(original.clone()).unwrap();
|
||||||
|
if let Some(state) = state {
|
||||||
|
store.dispatch(id, &snapshot.attempt.attempt_id, 8).unwrap();
|
||||||
|
store
|
||||||
|
.finish(
|
||||||
|
id,
|
||||||
|
&snapshot.attempt.attempt_id,
|
||||||
|
state,
|
||||||
|
Some("not complete"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let before = store.get(id).unwrap();
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.reclaim_input(
|
||||||
|
id,
|
||||||
|
&before.input_digest,
|
||||||
|
JobInputRetirementReason::Completed
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(store.get(id).unwrap(), before);
|
||||||
|
assert_eq!(before.load_executable_request().unwrap(), &original);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.reclaim_input(id, "wrong", JobInputRetirementReason::ExplicitErasure)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let retired = store
|
||||||
|
.reclaim_input(
|
||||||
|
id,
|
||||||
|
&before.input_digest,
|
||||||
|
JobInputRetirementReason::LegacyInvalidation,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(retired.state, before.state);
|
||||||
|
assert_eq!(retired.attempt, before.attempt);
|
||||||
|
assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated);
|
||||||
|
assert!(store.retry(id).is_err());
|
||||||
|
assert!(store.dispatch(id, &before.attempt.attempt_id, 8).is_err());
|
||||||
|
assert_eq!(store.reserve(original).unwrap(), retired);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn erased_dispatched_input_refuses_delayed_result_without_rewriting_unknown() {
|
||||||
|
let (dir, mut store) = fixture();
|
||||||
|
let running = dispatched(&mut store, request("erase-running"));
|
||||||
|
let result = submission(&running, json!({"late":true}));
|
||||||
|
let erased = store
|
||||||
|
.reclaim_input(
|
||||||
|
"erase-running",
|
||||||
|
&running.input_digest,
|
||||||
|
JobInputRetirementReason::ExplicitErasure,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(store.accept(&result).is_err());
|
||||||
|
assert_eq!(erased.state, JobState::Pending);
|
||||||
|
drop(store);
|
||||||
|
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
|
||||||
|
let recovered = reopened.get("erase-running").unwrap();
|
||||||
|
assert_eq!(recovered.state, JobState::Unknown);
|
||||||
|
assert_eq!(recovered.input_disposition, JobInputDisposition::Erased);
|
||||||
|
assert_eq!(recovered.input_digest, running.input_digest);
|
||||||
|
assert!(reopened.accept(&result).is_err());
|
||||||
|
assert!(reopened.retry("erase-running").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn available_input_loader_refuses_tamper_missing_body_and_envelope_changes() {
|
||||||
|
for mutation in [
|
||||||
|
"json_set(request_json,'$.input',json('{\"tampered\":true}'))",
|
||||||
|
"json_remove(request_json,'$.input')",
|
||||||
|
"json_set(request_json,'$.instruction','changed')",
|
||||||
|
] {
|
||||||
|
let (_dir, mut store) = fixture();
|
||||||
|
store.reserve(request("tamper")).unwrap();
|
||||||
|
// Simulate external corruption bypassing SQLite's normal write guard.
|
||||||
|
store
|
||||||
|
.conn
|
||||||
|
.execute_batch("DROP TRIGGER job_input_retirement")
|
||||||
|
.unwrap();
|
||||||
|
store
|
||||||
|
.conn
|
||||||
|
.execute_batch(&format!("UPDATE job_intents SET request_json={mutation}"))
|
||||||
|
.unwrap();
|
||||||
|
match store.get("tamper") {
|
||||||
|
Ok(snapshot) => {
|
||||||
|
assert!(snapshot.load_executable_request().is_err());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.dispatch("tamper", &snapshot.attempt.attempt_id, 8)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(JobStoreError::Json(_) | JobStoreError::Corrupt(_)) => {}
|
||||||
|
other => panic!("unexpected corrupt input read: {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn input_retirement_cleanup_failure_is_durable_and_exact_retry_finishes_without_rehydration() {
|
||||||
|
let (dir, mut store) = fixture();
|
||||||
|
let mut original = request("physical-erasure");
|
||||||
|
let private_body = "private-input-retirement-original-bytes".repeat(1000);
|
||||||
|
original.input = json!({"body":private_body});
|
||||||
|
let reserved = store.reserve(original.clone()).unwrap();
|
||||||
|
// A reader retaining the old SQLite snapshot prevents truncation of the old WAL.
|
||||||
|
let reader = Connection::open(dir.path().join("jobs.sqlite3")).unwrap();
|
||||||
|
reader
|
||||||
|
.execute_batch("BEGIN; SELECT request_json FROM job_intents;")
|
||||||
|
.unwrap();
|
||||||
|
store.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.reclaim_input(
|
||||||
|
"physical-erasure",
|
||||||
|
&reserved.input_digest,
|
||||||
|
JobInputRetirementReason::ExplicitErasure
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let retired = store.get("physical-erasure").unwrap();
|
||||||
|
assert_eq!(retired.input_disposition, JobInputDisposition::Erased);
|
||||||
|
assert!(retired.load_executable_request().is_err());
|
||||||
|
reader.execute_batch("ROLLBACK").unwrap();
|
||||||
|
drop(reader);
|
||||||
|
let replay = store
|
||||||
|
.reclaim_input(
|
||||||
|
"physical-erasure",
|
||||||
|
&reserved.input_digest,
|
||||||
|
JobInputRetirementReason::ExplicitErasure,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(replay, retired);
|
||||||
|
assert_eq!(store.reserve(original).unwrap(), retired);
|
||||||
|
let needle = b"private-input-retirement-original-bytes";
|
||||||
|
for entry in std::fs::read_dir(dir.path()).unwrap() {
|
||||||
|
let path = entry.unwrap().path();
|
||||||
|
if path.is_file() {
|
||||||
|
let bytes = std::fs::read(&path).unwrap();
|
||||||
|
assert!(
|
||||||
|
!bytes.windows(needle.len()).any(|bytes| bytes == needle),
|
||||||
|
"retired body remained in {}",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn missing_original_null_input_is_not_treated_as_available_null() {
|
||||||
|
let (_dir, mut store) = fixture();
|
||||||
|
let mut original = request("null-original");
|
||||||
|
original.input = Value::Null;
|
||||||
|
let reserved = store.reserve(original).unwrap();
|
||||||
|
assert!(reserved.load_executable_request().is_ok());
|
||||||
|
store.conn.execute_batch("DROP TRIGGER job_input_retirement; UPDATE job_intents SET request_json=json_remove(request_json,'$.input')").unwrap();
|
||||||
|
assert!(store.get("null-original").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn input_retirement_migration_preserves_receipts_and_rolls_back_failed_ddl() {
|
||||||
|
let (dir, mut store) = fixture();
|
||||||
|
let original = store.reserve(request("upgrade")).unwrap();
|
||||||
|
store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable;
|
||||||
|
ALTER TABLE job_intents DROP COLUMN input_disposition;
|
||||||
|
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
|
||||||
|
PRAGMA user_version=4;
|
||||||
|
CREATE TRIGGER job_input_retirement BEFORE UPDATE ON job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;").unwrap();
|
||||||
|
drop(store);
|
||||||
|
let path = dir.path().join("jobs.sqlite3");
|
||||||
|
assert!(JobStore::open(&path).is_err());
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
let version: i64 = conn
|
||||||
|
.pragma_query_value(None, "user_version", |r| r.get(0))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(version, 4);
|
||||||
|
let present: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM pragma_table_info('job_intents') WHERE name='input_disposition')",[],|r|r.get(0)).unwrap();
|
||||||
|
assert!(!present);
|
||||||
|
conn.execute_batch("DROP TRIGGER job_input_retirement")
|
||||||
|
.unwrap();
|
||||||
|
drop(conn);
|
||||||
|
let mut reopened = JobStore::open(&path).unwrap();
|
||||||
|
assert_eq!(reopened.get("upgrade").unwrap(), original);
|
||||||
|
assert_eq!(
|
||||||
|
reopened
|
||||||
|
.get("upgrade")
|
||||||
|
.unwrap()
|
||||||
|
.load_executable_request()
|
||||||
|
.unwrap(),
|
||||||
|
&request("upgrade")
|
||||||
|
);
|
||||||
|
let retired = reopened
|
||||||
|
.reclaim_input(
|
||||||
|
"upgrade",
|
||||||
|
&original.input_digest,
|
||||||
|
JobInputRetirementReason::ExplicitErasure,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(retired.state, JobState::Pending);
|
||||||
|
assert!(
|
||||||
|
reopened
|
||||||
|
.conn
|
||||||
|
.execute_batch("UPDATE job_intents SET input_disposition='available'")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn database_configuration_schema_and_identity_constraints() {
|
fn database_configuration_schema_and_identity_constraints() {
|
||||||
let (_dir, mut store) = fixture();
|
let (_dir, mut store) = fixture();
|
||||||
|
|||||||
@@ -143,6 +143,42 @@ mod tests {
|
|||||||
(original, receipt, grant)
|
(original, receipt, grant)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_archive_input_erasure_preserves_fingerprints_and_attempt_receipts() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("jobs.sqlite3");
|
||||||
|
seed(&path, 2);
|
||||||
|
let mut store = JobStore::open(&path).unwrap();
|
||||||
|
let before: (String, String) = store
|
||||||
|
.conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'",
|
||||||
|
[],
|
||||||
|
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let snapshot = store.get("done").unwrap();
|
||||||
|
store
|
||||||
|
.reclaim_input(
|
||||||
|
"done",
|
||||||
|
&snapshot.input_digest,
|
||||||
|
JobInputRetirementReason::ExplicitErasure,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let (body, fingerprint, attempts): (String,String,String) = store.conn.query_row("SELECT request_json,fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
|
||||||
|
assert!(serde_json::from_str::<Value>(&body).unwrap()["input"].is_null());
|
||||||
|
assert_eq!((fingerprint, attempts), before);
|
||||||
|
assert!(store.conn.execute_batch("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input','restored') WHERE job_id='done'").is_err());
|
||||||
|
drop(store);
|
||||||
|
let mut reopened = JobStore::open(&path).unwrap();
|
||||||
|
let receipt = reopened
|
||||||
|
.reserve_granted(request("done"), snapshot.domain_grant)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(receipt.state, JobState::Completed);
|
||||||
|
assert_eq!(receipt.input_disposition, JobInputDisposition::Erased);
|
||||||
|
assert!(receipt.load_executable_request().is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn legacy_versions_preserve_intent_receipts_attempts_grants_and_replay() {
|
fn legacy_versions_preserve_intent_receipts_attempts_grants_and_replay() {
|
||||||
for version in [1, 2] {
|
for version in [1, 2] {
|
||||||
|
|||||||
+113
-48
@@ -26,7 +26,10 @@ use job::{JobRequest, JobResultSubmission};
|
|||||||
use tokio::sync::watch;
|
use tokio::sync::watch;
|
||||||
|
|
||||||
use crate::job_store::JobStore;
|
use crate::job_store::JobStore;
|
||||||
pub use crate::job_store::{JobAttempt, JobAttemptState, JobSnapshot, JobState, JobStoreError};
|
pub use crate::job_store::{
|
||||||
|
JobAttempt, JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot,
|
||||||
|
JobState, JobStoreError,
|
||||||
|
};
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub enum JobServiceError {
|
pub enum JobServiceError {
|
||||||
@@ -40,6 +43,8 @@ pub enum JobServiceError {
|
|||||||
Capacity,
|
Capacity,
|
||||||
#[error("standalone Job cleanup failed: {0}")]
|
#[error("standalone Job cleanup failed: {0}")]
|
||||||
Cleanup(String),
|
Cleanup(String),
|
||||||
|
#[error("standalone Job shutdown is incomplete; execution resources are still owned")]
|
||||||
|
ShutdownIncomplete,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) trait JobAttemptFence: Send + Sync {
|
pub(crate) trait JobAttemptFence: Send + Sync {
|
||||||
@@ -173,6 +178,36 @@ impl StandaloneJobs {
|
|||||||
Ok(self.with_store(|store| store.reserve(request))?)
|
Ok(self.with_store(|store| store.reserve(request))?)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Enumerate historical domain snapshots for a Session, including retired input receipts.
|
||||||
|
pub fn list_session_snapshots(
|
||||||
|
&self,
|
||||||
|
session_id: &str,
|
||||||
|
after: Option<&str>,
|
||||||
|
limit: usize,
|
||||||
|
) -> Result<Vec<JobSnapshot>, JobServiceError> {
|
||||||
|
self.with_store(|store| store.list_session_snapshots(session_id, after, limit))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retire body only. The caller fences domain effects/delivered snapshots before erasure.
|
||||||
|
/// Normal reclamation also requires that execution cleanup has finished.
|
||||||
|
pub fn reclaim_input(
|
||||||
|
&self,
|
||||||
|
job_id: &str,
|
||||||
|
expected_input_digest: &str,
|
||||||
|
reason: JobInputRetirementReason,
|
||||||
|
) -> Result<JobSnapshot, JobServiceError> {
|
||||||
|
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||||
|
if reason == JobInputRetirementReason::Completed
|
||||||
|
&& executions
|
||||||
|
.jobs
|
||||||
|
.get(job_id)
|
||||||
|
.is_some_and(|execution| !execution.task.is_finished())
|
||||||
|
{
|
||||||
|
return Err(JobServiceError::Capacity);
|
||||||
|
}
|
||||||
|
self.with_store(|store| store.reclaim_input(job_id, expected_input_digest, reason))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn get(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
pub fn get(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||||
Ok(self.with_store(|store| store.get(job_id))?)
|
Ok(self.with_store(|store| store.get(job_id))?)
|
||||||
}
|
}
|
||||||
@@ -274,6 +309,12 @@ impl StandaloneJobs {
|
|||||||
job_id: &str,
|
job_id: &str,
|
||||||
client: Option<Box<dyn LlmClient>>,
|
client: Option<Box<dyn LlmClient>>,
|
||||||
) -> Result<(), JobServiceError> {
|
) -> Result<(), JobServiceError> {
|
||||||
|
// Preparation owns copies of the immutable input and domain adapter too. Exclude
|
||||||
|
// owner shutdown before creating them, not only before publishing the task.
|
||||||
|
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||||
|
if executions.closed {
|
||||||
|
return Err(JobServiceError::Closed);
|
||||||
|
}
|
||||||
let client = client.or_else(|| {
|
let client = client.or_else(|| {
|
||||||
self.model_client
|
self.model_client
|
||||||
.lock()
|
.lock()
|
||||||
@@ -282,16 +323,14 @@ impl StandaloneJobs {
|
|||||||
.map(|c| c.clone_boxed())
|
.map(|c| c.clone_boxed())
|
||||||
});
|
});
|
||||||
let snapshot = self.get(job_id)?;
|
let snapshot = self.get(job_id)?;
|
||||||
|
snapshot.load_executable_request()?;
|
||||||
let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds));
|
let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds));
|
||||||
let deadline = tokio::time::Instant::now() + timeout;
|
let deadline = tokio::time::Instant::now() + timeout;
|
||||||
let fence: Arc<dyn JobAttemptFence> = Arc::new(BoundAttemptFence {
|
let fence: Arc<dyn JobAttemptFence> = Arc::new(BoundAttemptFence {
|
||||||
store: self.store.clone(),
|
store: self.store.clone(),
|
||||||
job_id: job_id.into(),
|
job_id: job_id.into(),
|
||||||
attempt_id: snapshot.attempt.attempt_id.clone(),
|
attempt_id: snapshot.attempt.attempt_id.clone(),
|
||||||
input_digest: snapshot
|
input_digest: snapshot.input_digest.clone(),
|
||||||
.request
|
|
||||||
.input_digest()
|
|
||||||
.map_err(JobStoreError::from)?,
|
|
||||||
deadline,
|
deadline,
|
||||||
});
|
});
|
||||||
let grant = match (
|
let grant = match (
|
||||||
@@ -311,10 +350,6 @@ impl StandaloneJobs {
|
|||||||
let prepared =
|
let prepared =
|
||||||
worker::job::prepare_job_with_grant(&snapshot.request, &self.cwd, client, grant)
|
worker::job::prepare_job_with_grant(&snapshot.request, &self.cwd, client, grant)
|
||||||
.map_err(|error| JobServiceError::Configuration(error.to_string()))?;
|
.map_err(|error| JobServiceError::Configuration(error.to_string()))?;
|
||||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
|
||||||
if executions.closed {
|
|
||||||
return Err(JobServiceError::Closed);
|
|
||||||
}
|
|
||||||
executions.jobs.retain(|_, run| {
|
executions.jobs.retain(|_, run| {
|
||||||
!(run.task.is_finished() && matches!(run.complete.borrow().as_ref(), Some(Ok(()))))
|
!(run.task.is_finished() && matches!(run.complete.borrow().as_ref(), Some(Ok(()))))
|
||||||
});
|
});
|
||||||
@@ -367,10 +402,7 @@ impl StandaloneJobs {
|
|||||||
store: store.clone(),
|
store: store.clone(),
|
||||||
job_id: id.clone(),
|
job_id: id.clone(),
|
||||||
attempt_id: attempt_id.clone(),
|
attempt_id: attempt_id.clone(),
|
||||||
input_digest: snapshot
|
input_digest: snapshot.input_digest.clone(),
|
||||||
.request
|
|
||||||
.input_digest()
|
|
||||||
.map_err(JobStoreError::from)?,
|
|
||||||
deadline,
|
deadline,
|
||||||
});
|
});
|
||||||
let (cancel, mut cancellation) = watch::channel(false);
|
let (cancel, mut cancellation) = watch::channel(false);
|
||||||
@@ -446,23 +478,25 @@ impl StandaloneJobs {
|
|||||||
.clone()
|
.clone()
|
||||||
.expect("completion");
|
.expect("completion");
|
||||||
result.map_err(JobServiceError::Cleanup)?;
|
result.map_err(JobServiceError::Cleanup)?;
|
||||||
let task = {
|
// Completion is published immediately before the task returns. Do not remove
|
||||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
// its ownership record until the runtime confirms that return (and future drop).
|
||||||
if executions
|
while self
|
||||||
.jobs
|
.executions
|
||||||
.get(job_id)
|
.lock()
|
||||||
.is_some_and(|run| run.attempt_id == attempt)
|
.expect("Job executions poisoned")
|
||||||
{
|
.jobs
|
||||||
executions.jobs.remove(job_id)
|
.get(job_id)
|
||||||
} else {
|
.is_some_and(|run| run.attempt_id == attempt && !run.task.is_finished())
|
||||||
None
|
{
|
||||||
}
|
tokio::task::yield_now().await;
|
||||||
};
|
}
|
||||||
if let Some(execution) = task {
|
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||||
execution
|
if executions
|
||||||
.task
|
.jobs
|
||||||
.await
|
.get(job_id)
|
||||||
.map_err(|error| JobServiceError::Cleanup(error.to_string()))??;
|
.is_some_and(|run| run.attempt_id == attempt)
|
||||||
|
{
|
||||||
|
executions.jobs.remove(job_id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
self.get(job_id)
|
self.get(job_id)
|
||||||
@@ -476,33 +510,63 @@ impl StandaloneJobs {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> {
|
/// Initiate owner shutdown without blocking a synchronous caller. Success confirms all
|
||||||
|
/// local executions have returned, including request/history and Feature cleanup. An
|
||||||
|
/// incomplete/error result retains the service resources and must retain the Host lease.
|
||||||
|
pub(crate) fn try_shutdown(&self) -> Result<(), JobServiceError> {
|
||||||
self.close();
|
self.close();
|
||||||
let tasks = {
|
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
for execution in executions.jobs.values() {
|
||||||
std::mem::take(&mut executions.jobs)
|
if !execution.task.is_finished() {
|
||||||
};
|
return Err(JobServiceError::ShutdownIncomplete);
|
||||||
let mut error = None;
|
}
|
||||||
for execution in tasks.into_values() {
|
match execution.complete.borrow().as_ref() {
|
||||||
match execution.task.await {
|
Some(Ok(())) => {}
|
||||||
Ok(Ok(())) => {}
|
Some(Err(error)) => return Err(JobServiceError::Cleanup(error.clone())),
|
||||||
Ok(Err(failure)) => {
|
None => {
|
||||||
error = Some(failure);
|
return Err(JobServiceError::Cleanup(
|
||||||
}
|
"Internal Worker ended without cleanup confirmation".into(),
|
||||||
Err(failure) => {
|
));
|
||||||
error = Some(JobServiceError::Cleanup(failure.to_string()));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Close SQLite before the Host releases its lease. Cloned handles may
|
executions.jobs.clear();
|
||||||
// outlive the Host but cannot mutate or retain the live connection.
|
// Retained external handles share these slots; they cannot retain a live store,
|
||||||
|
// provider or model client after successful owner shutdown.
|
||||||
self.store.lock().expect("Job store poisoned").take();
|
self.store.lock().expect("Job store poisoned").take();
|
||||||
self.domain.lock().expect("Job domain poisoned").take();
|
self.domain.lock().expect("Job domain poisoned").take();
|
||||||
self.model_client
|
self.model_client
|
||||||
.lock()
|
.lock()
|
||||||
.expect("Job model transport poisoned")
|
.expect("Job model transport poisoned")
|
||||||
.take();
|
.take();
|
||||||
error.map_or(Ok(()), Err)
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> {
|
||||||
|
self.close();
|
||||||
|
let completions: Vec<_> = self
|
||||||
|
.executions
|
||||||
|
.lock()
|
||||||
|
.expect("Job executions poisoned")
|
||||||
|
.jobs
|
||||||
|
.values()
|
||||||
|
.map(|execution| execution.complete.clone())
|
||||||
|
.collect();
|
||||||
|
// Never take execution ownership out of the shared service across an await: a
|
||||||
|
// cancelled shutdown future or a concurrent close must still see the barrier.
|
||||||
|
for mut completion in completions {
|
||||||
|
completion
|
||||||
|
.wait_for(Option::is_some)
|
||||||
|
.await
|
||||||
|
.map_err(|error| JobServiceError::Cleanup(error.to_string()))?;
|
||||||
|
}
|
||||||
|
loop {
|
||||||
|
match self.try_shutdown() {
|
||||||
|
// Only the task's final return follows its completion notification.
|
||||||
|
Err(JobServiceError::ShutdownIncomplete) => tokio::task::yield_now().await,
|
||||||
|
result => return result,
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -574,7 +638,8 @@ impl JobAttemptFence for BoundAttemptFence {
|
|||||||
if snapshot.state != JobState::Pending
|
if snapshot.state != JobState::Pending
|
||||||
|| snapshot.attempt.state != JobAttemptState::Dispatched
|
|| snapshot.attempt.state != JobAttemptState::Dispatched
|
||||||
|| snapshot.attempt.attempt_id != self.attempt_id
|
|| snapshot.attempt.attempt_id != self.attempt_id
|
||||||
|| snapshot.request.input_digest().map_err(|e| e.to_string())? != self.input_digest
|
|| snapshot.input_digest != self.input_digest
|
||||||
|
|| snapshot.input_disposition != JobInputDisposition::Available
|
||||||
{
|
{
|
||||||
return Err("Job domain grant is not bound to the current live attempt".into());
|
return Err("Job domain grant is not bound to the current live attempt".into());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -468,3 +468,104 @@ async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_han
|
|||||||
assert!(matches!(operation, Err(JobServiceError::Closed)));
|
assert!(matches!(operation, Err(JobServiceError::Closed)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancelled_shutdown_future_keeps_execution_barrier_and_unknown_input() {
|
||||||
|
// Actual generic runner + SQLite; only the provider stream is scripted/blocked.
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let path = temp.path().join("jobs.sqlite3");
|
||||||
|
let jobs = StandaloneJobs::open(&path, temp.path().into()).unwrap();
|
||||||
|
let reserved = jobs.request(request("cancel-shutdown-wait")).unwrap();
|
||||||
|
let model = DomainTestClient {
|
||||||
|
stall: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
jobs.start_with_model_client(&reserved.request.job_id, model.clone())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
await_model(&model).await;
|
||||||
|
{
|
||||||
|
let shutdown = jobs.shutdown();
|
||||||
|
tokio::pin!(shutdown);
|
||||||
|
assert!(futures::poll!(shutdown.as_mut()).is_pending());
|
||||||
|
// Drop the caller's waiter while cancellation/cleanup is still pending.
|
||||||
|
}
|
||||||
|
assert!(matches!(
|
||||||
|
jobs.try_shutdown(),
|
||||||
|
Err(JobServiceError::ShutdownIncomplete)
|
||||||
|
));
|
||||||
|
assert!(jobs.store.lock().unwrap().is_some());
|
||||||
|
assert!(matches!(
|
||||||
|
jobs.request(request("after-shutdown")),
|
||||||
|
Err(JobServiceError::Closed)
|
||||||
|
));
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), jobs.shutdown())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(jobs.store.lock().unwrap().is_none());
|
||||||
|
let reopened = StandaloneJobs::open(&path, temp.path().into()).unwrap();
|
||||||
|
let unknown = reopened.get(&reserved.request.job_id).unwrap();
|
||||||
|
assert_eq!(unknown.attempt.state, JobAttemptState::Unknown);
|
||||||
|
assert_eq!(unknown.input_disposition, JobInputDisposition::Available);
|
||||||
|
assert_eq!(unknown.request, reserved.request);
|
||||||
|
assert_eq!(unknown.input_digest, reserved.input_digest);
|
||||||
|
assert!(reopened.retry(&unknown.request.job_id).is_err());
|
||||||
|
reopened.shutdown().await.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn input_retirement_service_fences_live_attempt_and_preserves_session_receipts() {
|
||||||
|
// Production service + SQLite + attempt/result fences, without a model or Worker fixture.
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let path = temp.path().join("jobs.sqlite3");
|
||||||
|
let jobs = StandaloneJobs::open(&path, temp.path().to_path_buf()).unwrap();
|
||||||
|
let original = request("input-retirement");
|
||||||
|
let grant = json!({"session_id":"session-a"});
|
||||||
|
let reserved = jobs
|
||||||
|
.request_granted(original.clone(), grant.clone())
|
||||||
|
.unwrap();
|
||||||
|
jobs.with_store(|store| store.dispatch("input-retirement", &reserved.attempt.attempt_id, 8))
|
||||||
|
.unwrap();
|
||||||
|
let fence = BoundAttemptFence {
|
||||||
|
store: jobs.store.clone(),
|
||||||
|
job_id: "input-retirement".into(),
|
||||||
|
attempt_id: reserved.attempt.attempt_id.clone(),
|
||||||
|
input_digest: reserved.input_digest.clone(),
|
||||||
|
deadline: tokio::time::Instant::now() + Duration::from_secs(10),
|
||||||
|
};
|
||||||
|
assert!(fence.ensure_live().is_ok());
|
||||||
|
assert!(
|
||||||
|
jobs.reclaim_input(
|
||||||
|
"input-retirement",
|
||||||
|
&reserved.input_digest,
|
||||||
|
JobInputRetirementReason::Completed
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let erased = jobs
|
||||||
|
.reclaim_input(
|
||||||
|
"input-retirement",
|
||||||
|
&reserved.input_digest,
|
||||||
|
JobInputRetirementReason::ExplicitErasure,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(fence.ensure_live().is_err());
|
||||||
|
assert!(
|
||||||
|
sink(&jobs, "input-retirement", &reserved.attempt.attempt_id)
|
||||||
|
.submit(result("input-retirement", &reserved.attempt.attempt_id))
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(jobs.start("input-retirement").await.is_err());
|
||||||
|
assert_eq!(
|
||||||
|
jobs.list_session_snapshots("session-a", None, 100).unwrap(),
|
||||||
|
vec![erased.clone()]
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
jobs.list_session_snapshots("session-b", None, 100)
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
assert_eq!(jobs.request_granted(original, grant).unwrap(), erased);
|
||||||
|
jobs.shutdown().await.unwrap();
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,6 +7,44 @@ pub mod launch;
|
|||||||
pub mod store;
|
pub mod store;
|
||||||
pub mod subjektiv;
|
pub mod subjektiv;
|
||||||
mod subjektiv_sessions;
|
mod subjektiv_sessions;
|
||||||
|
pub mod unix;
|
||||||
|
|
||||||
|
pub use unix::{UnixEndpoint, relay_unix_client, run_unix_client};
|
||||||
|
|
||||||
|
/// Restore reads the saved cwd and Profile, not a newly resolved launch recipe.
|
||||||
|
pub enum HeadlessLaunch {
|
||||||
|
Fresh(ResolvedStandaloneLaunch),
|
||||||
|
Restore {
|
||||||
|
state_dir: std::path::PathBuf,
|
||||||
|
worker_id: WorkerId,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub enum HeadlessError {
|
||||||
|
#[error(transparent)]
|
||||||
|
Transport(#[from] std::io::Error),
|
||||||
|
#[error(transparent)]
|
||||||
|
Startup(#[from] StandaloneStartupError),
|
||||||
|
#[error(transparent)]
|
||||||
|
Shutdown(#[from] StandaloneShutdownError),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reserve the optional public transport before creating or restoring execution state.
|
||||||
|
pub async fn run_headless(
|
||||||
|
launch: HeadlessLaunch,
|
||||||
|
endpoint: std::path::PathBuf,
|
||||||
|
) -> Result<(), HeadlessError> {
|
||||||
|
let endpoint = UnixEndpoint::bind(endpoint)?;
|
||||||
|
let host = match launch {
|
||||||
|
HeadlessLaunch::Fresh(launch) => StandaloneHost::start(launch).await?,
|
||||||
|
HeadlessLaunch::Restore {
|
||||||
|
state_dir,
|
||||||
|
worker_id,
|
||||||
|
} => StandaloneHost::restore(state_dir, worker_id).await?,
|
||||||
|
};
|
||||||
|
host.serve_unix(endpoint).await
|
||||||
|
}
|
||||||
|
|
||||||
pub use host::{
|
pub use host::{
|
||||||
StandaloneHost, StandaloneShutdownError, StandaloneStartupError,
|
StandaloneHost, StandaloneShutdownError, StandaloneStartupError,
|
||||||
|
|||||||
@@ -54,6 +54,8 @@ pub enum SubjectError {
|
|||||||
Journal(#[from] subjektiv::journal::JournalError),
|
Journal(#[from] subjektiv::journal::JournalError),
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
Storage(#[from] feature_storage::FeatureStorageError),
|
Storage(#[from] feature_storage::FeatureStorageError),
|
||||||
|
#[error("local Subject Job shutdown: {0}")]
|
||||||
|
JobShutdown(#[from] crate::jobs::JobServiceError),
|
||||||
#[error("local Subject storage metadata: {0}")]
|
#[error("local Subject storage metadata: {0}")]
|
||||||
Json(#[from] serde_json::Error),
|
Json(#[from] serde_json::Error),
|
||||||
#[error("local Subject storage permission or identity is invalid: {0}")]
|
#[error("local Subject storage permission or identity is invalid: {0}")]
|
||||||
@@ -338,8 +340,16 @@ impl SubjectConnection {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub fn close(&mut self) -> Result<(), SubjectError> {
|
pub fn close(&mut self) -> Result<(), SubjectError> {
|
||||||
self.host.jobs.lock().expect("Subject Jobs poisoned").take();
|
|
||||||
self.host.live.store(false, Ordering::Release);
|
self.host.live.store(false, Ordering::Release);
|
||||||
|
{
|
||||||
|
let mut jobs = self.host.jobs.lock().expect("Subject Jobs poisoned");
|
||||||
|
if let Some(service) = jobs.as_ref() {
|
||||||
|
// Signal cancellation, but retain the handle and exclusive lease until
|
||||||
|
// the generic owner confirms local request/history/resource teardown.
|
||||||
|
service.try_shutdown()?;
|
||||||
|
}
|
||||||
|
jobs.take();
|
||||||
|
}
|
||||||
self.host.catalog.manager.shutdown()?;
|
self.host.catalog.manager.shutdown()?;
|
||||||
if let Some(file) = &self.lease {
|
if let Some(file) = &self.lease {
|
||||||
file.unlock()?;
|
file.unlock()?;
|
||||||
|
|||||||
@@ -5,9 +5,12 @@
|
|||||||
//! recovery authority; shutdown does not delete either original Sessions or pending requests.
|
//! recovery authority; shutdown does not delete either original Sessions or pending requests.
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::jobs::{JobAttemptState, JobSnapshot, JobState};
|
use crate::jobs::{
|
||||||
|
JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot, JobState,
|
||||||
|
};
|
||||||
|
use session_store::capture::{CaptureDto, CaptureLimits, RecordingPosition};
|
||||||
use subjektiv::api::JobAttemptBinding;
|
use subjektiv::api::JobAttemptBinding;
|
||||||
use subjektiv::journal::recording::JournalMaterial;
|
use subjektiv::journal::recording::JournalInput;
|
||||||
use subjektiv::journal::transport::*;
|
use subjektiv::journal::transport::*;
|
||||||
use subjektiv::journal::*;
|
use subjektiv::journal::*;
|
||||||
use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeature};
|
use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeature};
|
||||||
@@ -16,7 +19,6 @@ use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeat
|
|||||||
mod tests;
|
mod tests;
|
||||||
|
|
||||||
pub(super) const PURPOSE: &str = "subjektiv_journal";
|
pub(super) const PURPOSE: &str = "subjektiv_journal";
|
||||||
const INPUT_BUDGET: usize = 48 * 1024;
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
@@ -27,6 +29,7 @@ struct LocalJournalGrant {
|
|||||||
session_id: String,
|
session_id: String,
|
||||||
journal_id: String,
|
journal_id: String,
|
||||||
ranges: Vec<JournalSourceRange>,
|
ranges: Vec<JournalSourceRange>,
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
origins: Vec<SourceOrigin>,
|
origins: Vec<SourceOrigin>,
|
||||||
}
|
}
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
@@ -35,11 +38,6 @@ struct SourceOrigin {
|
|||||||
pointer: JournalSourcePointer,
|
pointer: JournalSourcePointer,
|
||||||
origin: session_store::public_material::PublicMaterialOrigin,
|
origin: session_store::public_material::PublicMaterialOrigin,
|
||||||
}
|
}
|
||||||
impl LocalJournalGrant {
|
|
||||||
fn input(&self) -> serde_json::Value {
|
|
||||||
serde_json::json!({"selected_ranges": self.ranges, "origins": self.origins})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl StandaloneSubjects {
|
impl StandaloneSubjects {
|
||||||
/// Explicit operator erasure, separate from ordinary Worker removal. Requires the
|
/// Explicit operator erasure, separate from ordinary Worker removal. Requires the
|
||||||
@@ -84,32 +82,29 @@ impl StandaloneSubjects {
|
|||||||
self.state_root.clone(),
|
self.state_root.clone(),
|
||||||
)
|
)
|
||||||
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
|
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
|
||||||
for snapshot in jobs
|
let mut after = None;
|
||||||
.pending()
|
loop {
|
||||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
let page = jobs
|
||||||
.into_iter()
|
.list_session_snapshots(session_id, after.as_deref(), 100)
|
||||||
.chain(
|
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||||
jobs.unknown_granted()
|
if page.is_empty() {
|
||||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?,
|
break;
|
||||||
)
|
}
|
||||||
{
|
for snapshot in &page {
|
||||||
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
|
|
||||||
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
|
|
||||||
}) {
|
|
||||||
jobs.cancel(&snapshot.request.job_id)
|
jobs.cancel(&snapshot.request.job_id)
|
||||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||||
|
jobs.reclaim_input(
|
||||||
|
&snapshot.request.job_id,
|
||||||
|
&snapshot.input_digest,
|
||||||
|
JobInputRetirementReason::ExplicitErasure,
|
||||||
|
)
|
||||||
|
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||||
|
if snapshot.state == JobState::Completed && !snapshot.acknowledged {
|
||||||
|
jobs.acknowledge(&snapshot.request.job_id)
|
||||||
|
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
after = page.last().map(|job| job.request.job_id.clone());
|
||||||
for snapshot in jobs
|
|
||||||
.unacknowledged_results()
|
|
||||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
|
||||||
{
|
|
||||||
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
|
|
||||||
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
|
|
||||||
}) {
|
|
||||||
jobs.acknowledge(&snapshot.request.job_id)
|
|
||||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
session_store::FsStore::new_with_retention(
|
session_store::FsStore::new_with_retention(
|
||||||
path,
|
path,
|
||||||
@@ -590,17 +585,28 @@ impl LocalSubjectHost {
|
|||||||
self.ensure_live()?;
|
self.ensure_live()?;
|
||||||
let jobs = self.jobs()?;
|
let jobs = self.jobs()?;
|
||||||
for saved in jobs.unacknowledged_results().map_err(service_error)? {
|
for saved in jobs.unacknowledged_results().map_err(service_error)? {
|
||||||
if self.journal_grant(&saved).is_ok() {
|
if self.journal_grant_scope(&saved).is_ok() {
|
||||||
self.complete_journal_job(&jobs, &saved)?;
|
self.complete_journal_job(&jobs, &saved)?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for pending in jobs.pending().map_err(service_error)? {
|
for pending in jobs.pending().map_err(service_error)? {
|
||||||
if self.journal_grant(&pending).is_ok() {
|
if pending.request.purpose != PURPOSE {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
self.journal_grant_scope(&pending)?;
|
||||||
|
if pending.request.input.get("capture").is_some() {
|
||||||
|
self.journal_grant(&pending)?;
|
||||||
return Ok(Some((jobs, pending.request.job_id)));
|
return Ok(Some((jobs, pending.request.job_id)));
|
||||||
}
|
}
|
||||||
|
// Old descriptor jobs never acquire new execution authority. Their immutable
|
||||||
|
// identity/input is retired, not upgraded in place; saved checkpoint wins.
|
||||||
|
let replacement = self.replace_legacy_journal_job(&jobs, &pending)?;
|
||||||
|
if let Some(id) = replacement {
|
||||||
|
return Ok(Some((jobs, id)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for unknown in jobs.unknown_granted().map_err(service_error)? {
|
for unknown in jobs.unknown_granted().map_err(service_error)? {
|
||||||
let Ok(grant) = self.journal_grant(&unknown) else {
|
let Ok(grant) = self.journal_grant_scope(&unknown) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let context = authority(self, &grant.session_id)?;
|
let context = authority(self, &grant.session_id)?;
|
||||||
@@ -685,32 +691,96 @@ impl LocalSubjectHost {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let materials = self.select_journal_materials(&context, &journal.id)?;
|
let mut retained = None;
|
||||||
if materials.is_empty() {
|
// Saved body snapshots always win over descriptor migration. In particular,
|
||||||
|
// a failed continuation must not recapture its original text even if an
|
||||||
|
// invalidated legacy grant still bounds a larger unprocessed suffix.
|
||||||
|
for legacy_pass in [false, true] {
|
||||||
|
let mut after_job = None;
|
||||||
|
loop {
|
||||||
|
let page = jobs
|
||||||
|
.list_session_snapshots(
|
||||||
|
&journal.source_session_id,
|
||||||
|
after_job.as_deref(),
|
||||||
|
100,
|
||||||
|
)
|
||||||
|
.map_err(service_error)?;
|
||||||
|
if page.is_empty() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
for saved in &page {
|
||||||
|
if saved.request.purpose != PURPOSE {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let legacy = saved.input_disposition
|
||||||
|
== JobInputDisposition::Invalidated
|
||||||
|
|| (saved.input_disposition == JobInputDisposition::Available
|
||||||
|
&& saved.request.input.get("capture").is_none());
|
||||||
|
if legacy != legacy_pass
|
||||||
|
|| (!legacy
|
||||||
|
&& saved.input_disposition != JobInputDisposition::Available)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let saved_grant = self.journal_grant_scope(saved)?;
|
||||||
|
if saved_grant.journal_id != journal.id {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if legacy {
|
||||||
|
if let Some(id) = self.replace_legacy_journal_job(&jobs, saved)? {
|
||||||
|
return Ok(Some((jobs, id)));
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
self.journal_grant(saved)?;
|
||||||
|
let original = JournalInput::decode(&saved.request.input)
|
||||||
|
.map_err(journal_error)?;
|
||||||
|
let continuation = self.build_journal_input(
|
||||||
|
&context,
|
||||||
|
&journal.id,
|
||||||
|
Some(&original.capture),
|
||||||
|
)?;
|
||||||
|
if !continuation.selected_ranges.is_empty() {
|
||||||
|
retained = Some(continuation);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if saved.state == JobState::Completed && saved.acknowledged {
|
||||||
|
// Recover the acknowledgement-before-reclamation gap without
|
||||||
|
// turning a retained receipt into an executable empty input.
|
||||||
|
jobs.reclaim_input(
|
||||||
|
&saved.request.job_id,
|
||||||
|
&saved.input_digest,
|
||||||
|
JobInputRetirementReason::Completed,
|
||||||
|
)
|
||||||
|
.map_err(service_error)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if retained.is_some() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
after_job = page.last().map(|job| job.request.job_id.clone());
|
||||||
|
}
|
||||||
|
if retained.is_some() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let input = match retained {
|
||||||
|
Some(input) => input,
|
||||||
|
None => self.build_journal_input(&context, &journal.id, None)?,
|
||||||
|
};
|
||||||
|
|
||||||
|
if input.selected_ranges.is_empty() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let ranges = materials
|
|
||||||
.into_iter()
|
|
||||||
.map(|material| material.range)
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let origins = self.journal_origins(&ranges)?;
|
|
||||||
let grant = LocalJournalGrant {
|
let grant = LocalJournalGrant {
|
||||||
writer_binding_id: Some(self.writer.binding_id.clone()),
|
writer_binding_id: Some(self.writer.binding_id.clone()),
|
||||||
binding: self.binding.clone(),
|
binding: self.binding.clone(),
|
||||||
session_id: journal.source_session_id,
|
session_id: journal.source_session_id,
|
||||||
journal_id: journal.id.clone(),
|
journal_id: journal.id.clone(),
|
||||||
ranges,
|
ranges: input.selected_ranges.clone(),
|
||||||
origins,
|
origins: vec![],
|
||||||
};
|
|
||||||
let request = job::JobRequest {
|
|
||||||
job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()),
|
|
||||||
purpose: PURPOSE.into(), input_ref: format!("subjektiv://{}/journal/{}", self.binding.subject_id, journal.id),
|
|
||||||
input: grant.input(),
|
|
||||||
instruction: "ReadJournalState first. Input descriptors are the frozen canonical public ranges authorized for this Job, not already read material. Read every selected source byte with bounded ReadJournalText before recording; then read initial/current position and open text in bounded pages. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(),
|
|
||||||
profile: "builtin:subjektiv-journal-recording".into(),
|
|
||||||
serialization_key: Some(format!("subjektiv-journal:{}", journal.id)),
|
|
||||||
limits: job::JobLimits { timeout_seconds: 600, ..Default::default() },
|
|
||||||
};
|
};
|
||||||
|
let request = self.journal_request(&journal.id, &input)?;
|
||||||
let saved = jobs
|
let saved = jobs
|
||||||
.request_granted(
|
.request_granted(
|
||||||
request,
|
request,
|
||||||
@@ -727,6 +797,138 @@ impl LocalSubjectHost {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn replace_legacy_journal_job(
|
||||||
|
&self,
|
||||||
|
jobs: &crate::jobs::StandaloneJobs,
|
||||||
|
old: &JobSnapshot,
|
||||||
|
) -> Result<Option<String>, SubjektivHostError> {
|
||||||
|
let grant = self.journal_grant_scope(old)?;
|
||||||
|
if old.input_disposition == JobInputDisposition::Available {
|
||||||
|
old.load_executable_request()
|
||||||
|
.map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?;
|
||||||
|
if old.request.input
|
||||||
|
!= serde_json::json!({"selected_ranges":grant.ranges,"origins":grant.origins})
|
||||||
|
{
|
||||||
|
return Err(SubjektivHostError::InvalidArgument(
|
||||||
|
"invalid legacy Journal descriptor input".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
} else if old.input_disposition != JobInputDisposition::Invalidated {
|
||||||
|
return Err(SubjektivHostError::InvalidArgument(
|
||||||
|
"legacy Journal input is unavailable".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let context = authority(self, &grant.session_id)?;
|
||||||
|
if let Some(active) = self
|
||||||
|
.catalog
|
||||||
|
.store
|
||||||
|
.active_journal_generation(&context, &grant.journal_id)
|
||||||
|
.map_err(journal_error)?
|
||||||
|
{
|
||||||
|
if active.job.job_id != old.request.job_id {
|
||||||
|
return Err(SubjektivHostError::InvalidArgument(
|
||||||
|
"another generation owns legacy migration".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
self.catalog
|
||||||
|
.store
|
||||||
|
.close_journal_generation(&context, &active.id, &active.job)
|
||||||
|
.map_err(journal_error)?;
|
||||||
|
}
|
||||||
|
jobs.cancel(&old.request.job_id).map_err(service_error)?;
|
||||||
|
let current = self
|
||||||
|
.catalog
|
||||||
|
.store
|
||||||
|
.read_journal(&context, &grant.journal_id)
|
||||||
|
.map_err(journal_error)?;
|
||||||
|
let end = grant
|
||||||
|
.ranges
|
||||||
|
.last()
|
||||||
|
.ok_or_else(|| SubjektivHostError::InvalidArgument("empty legacy selection".into()))?;
|
||||||
|
let end_position = if end.bytes.end == end.source.total_bytes {
|
||||||
|
(end.source.order + 1, 0)
|
||||||
|
} else {
|
||||||
|
(end.source.order, end.bytes.end)
|
||||||
|
};
|
||||||
|
let replacement = if (current.process_order, current.process_byte) >= end_position {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
let mut capture = self.recording_capture(&grant.session_id)?;
|
||||||
|
capture
|
||||||
|
.entries
|
||||||
|
.retain(|entry| grant.ranges.iter().any(|r| r.source.order == entry.order));
|
||||||
|
for entry in &mut capture.entries {
|
||||||
|
let range = grant
|
||||||
|
.ranges
|
||||||
|
.iter()
|
||||||
|
.find(|r| r.source.order == entry.order)
|
||||||
|
.unwrap();
|
||||||
|
let session_store::capture::CaptureBody::CanonicalPublic {
|
||||||
|
descriptor,
|
||||||
|
byte_offset,
|
||||||
|
text,
|
||||||
|
..
|
||||||
|
} = &mut entry.body
|
||||||
|
else {
|
||||||
|
unreachable!()
|
||||||
|
};
|
||||||
|
if source_descriptor(&grant.session_id, descriptor) != range.source
|
||||||
|
|| !grant.origins.iter().any(|origin| {
|
||||||
|
origin.pointer == range.source.pointer && origin.origin == descriptor.origin
|
||||||
|
})
|
||||||
|
|| !text.is_char_boundary(range.bytes.start as usize)
|
||||||
|
|| !text.is_char_boundary(range.bytes.end as usize)
|
||||||
|
{
|
||||||
|
return Err(SubjektivHostError::InvalidArgument(
|
||||||
|
"legacy original source identity/range changed".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
*text = text[range.bytes.start as usize..range.bytes.end as usize].to_owned();
|
||||||
|
*byte_offset = range.bytes.start;
|
||||||
|
}
|
||||||
|
let input = self.build_journal_input(&context, &grant.journal_id, Some(&capture))?;
|
||||||
|
let request = self.journal_request(&grant.journal_id, &input)?;
|
||||||
|
let next_grant = LocalJournalGrant {
|
||||||
|
ranges: input.selected_ranges.clone(),
|
||||||
|
origins: vec![],
|
||||||
|
..grant
|
||||||
|
};
|
||||||
|
Some(
|
||||||
|
jobs.request_granted(
|
||||||
|
request,
|
||||||
|
serde_json::to_value(next_grant)
|
||||||
|
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?,
|
||||||
|
)
|
||||||
|
.map_err(service_error)?
|
||||||
|
.request
|
||||||
|
.job_id,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
jobs.reclaim_input(
|
||||||
|
&old.request.job_id,
|
||||||
|
&old.input_digest,
|
||||||
|
JobInputRetirementReason::LegacyInvalidation,
|
||||||
|
)
|
||||||
|
.map_err(service_error)?;
|
||||||
|
Ok(replacement)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn journal_request(
|
||||||
|
&self,
|
||||||
|
journal_id: &str,
|
||||||
|
input: &JournalInput,
|
||||||
|
) -> Result<job::JobRequest, SubjektivHostError> {
|
||||||
|
Ok(job::JobRequest {
|
||||||
|
job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()), purpose: PURPOSE.into(),
|
||||||
|
input_ref: format!("subjektiv://{}/journal/{journal_id}", self.binding.subject_id),
|
||||||
|
input: serde_json::to_value(input).map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?,
|
||||||
|
instruction: "Input contains the frozen canonical public source bodies, provenance and byte ranges. Use only this saved input, never re-fetch Source. ReadJournalState first and read initial/current Journal position/open text in bounded pages if needed. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(),
|
||||||
|
profile: "builtin:subjektiv-journal-recording".into(),
|
||||||
|
serialization_key: Some(format!("subjektiv-journal:{journal_id}")),
|
||||||
|
limits: job::JobLimits { timeout_seconds: 600, ..Default::default() },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn pin_journal_session(&self, session: &str) -> Result<(), SubjektivHostError> {
|
fn pin_journal_session(&self, session: &str) -> Result<(), SubjektivHostError> {
|
||||||
let path = crate::subjektiv_sessions::authorized_sessions_path(
|
let path = crate::subjektiv_sessions::authorized_sessions_path(
|
||||||
&self.catalog.state_root,
|
&self.catalog.state_root,
|
||||||
@@ -869,11 +1071,16 @@ impl LocalSubjectHost {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn material_reader(
|
fn with_recording_boundary<T>(
|
||||||
&self,
|
&self,
|
||||||
session: &str,
|
session: &str,
|
||||||
) -> Result<session_store::public_material::SessionPublicMaterialReader, SubjektivHostError>
|
read: impl FnOnce(
|
||||||
{
|
&Path,
|
||||||
|
session_store::SessionId,
|
||||||
|
session_store::SegmentId,
|
||||||
|
&session_store::retention::SessionRetentionGuard,
|
||||||
|
) -> Result<T, SubjektivHostError>,
|
||||||
|
) -> Result<T, SubjektivHostError> {
|
||||||
let sessions = crate::subjektiv_sessions::authorized_sessions_path(
|
let sessions = crate::subjektiv_sessions::authorized_sessions_path(
|
||||||
&self.catalog.state_root,
|
&self.catalog.state_root,
|
||||||
&self.binding.scope_id,
|
&self.binding.scope_id,
|
||||||
@@ -906,6 +1113,11 @@ impl LocalSubjectHost {
|
|||||||
.ok_or_else(|| SubjektivHostError::InvalidArgument("invalid Session storage".into()))?
|
.ok_or_else(|| SubjektivHostError::InvalidArgument("invalid Session storage".into()))?
|
||||||
.join("worker")
|
.join("worker")
|
||||||
.join(&record.storage_key);
|
.join(&record.storage_key);
|
||||||
|
let session_id = session
|
||||||
|
.parse()
|
||||||
|
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?;
|
||||||
|
let retention = crate::store::session_retention(&self.catalog.state_root);
|
||||||
|
let guard = retention.lock_live(session_id).map_err(source_error)?;
|
||||||
// This is the persisted active selector, not the newest filename/mtime or a model claim.
|
// This is the persisted active selector, not the newest filename/mtime or a model claim.
|
||||||
let metadata =
|
let metadata =
|
||||||
session_store::WorkerAggregateStore::open_read_only(metadata_root, &record.storage_key)
|
session_store::WorkerAggregateStore::open_read_only(metadata_root, &record.storage_key)
|
||||||
@@ -933,154 +1145,203 @@ impl LocalSubjectHost {
|
|||||||
"no attested active lineage for the retained Session".into(),
|
"no attested active lineage for the retained Session".into(),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention(
|
read(&sessions, session_id, active, &guard)
|
||||||
&sessions,
|
}
|
||||||
session
|
|
||||||
.parse()
|
fn recording_capture(&self, session: &str) -> Result<CaptureDto, SubjektivHostError> {
|
||||||
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
|
self.with_recording_boundary(session, |sessions, session_id, active, guard| {
|
||||||
active,
|
session_store::capture::capture_recording_fs_guarded(
|
||||||
session_store::public_material::PublicMaterialLimits {
|
sessions,
|
||||||
max_segments: 256,
|
session_id,
|
||||||
max_entries: 100_000,
|
active,
|
||||||
max_record_bytes: 64 * 1024 * 1024,
|
session_store::public_material::PublicMaterialLimits::default(),
|
||||||
},
|
CaptureLimits {
|
||||||
crate::store::session_retention(&self.catalog.state_root),
|
max_bytes: 128 * 1024 * 1024,
|
||||||
)
|
max_entries: 100_000,
|
||||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
|
max_text_bytes: 64 * 1024 * 1024,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
guard,
|
||||||
|
)
|
||||||
|
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn journal_sources(
|
fn journal_sources(
|
||||||
&self,
|
&self,
|
||||||
session: &str,
|
session: &str,
|
||||||
) -> Result<Vec<JournalSourceRange>, SubjektivHostError> {
|
) -> Result<Vec<JournalSourceRange>, SubjektivHostError> {
|
||||||
let reader = self.material_reader(session)?;
|
self.with_recording_boundary(session, |sessions, session_id, active, guard| {
|
||||||
Ok(reader
|
session_store::capture::recording_catalog_fs_guarded(
|
||||||
.descriptors()
|
sessions,
|
||||||
.map(|descriptor| JournalSourceRange {
|
session_id,
|
||||||
source: source_descriptor(session, descriptor),
|
active,
|
||||||
bytes: JournalByteRange {
|
session_store::public_material::PublicMaterialLimits::default(),
|
||||||
start: 0,
|
guard,
|
||||||
end: descriptor.total_bytes,
|
|descriptors| {
|
||||||
|
descriptors
|
||||||
|
.iter()
|
||||||
|
.map(|descriptor| JournalSourceRange {
|
||||||
|
source: source_descriptor(session, descriptor),
|
||||||
|
bytes: JournalByteRange {
|
||||||
|
start: 0,
|
||||||
|
end: descriptor.total_bytes,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
},
|
},
|
||||||
})
|
)
|
||||||
.collect())
|
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
|
||||||
}
|
|
||||||
|
|
||||||
fn source_reader(
|
|
||||||
&self,
|
|
||||||
source: &JournalSource,
|
|
||||||
) -> Result<session_store::public_material::SessionPublicMaterialReader, SubjektivHostError>
|
|
||||||
{
|
|
||||||
let sessions = crate::subjektiv_sessions::authorized_sessions_path(
|
|
||||||
&self.catalog.state_root,
|
|
||||||
&self.binding.scope_id,
|
|
||||||
&self.binding.subject_id,
|
|
||||||
&self.catalog.store,
|
|
||||||
&source.session_id,
|
|
||||||
)?;
|
|
||||||
// Frozen, already admitted original pointers do not follow the Worker's later active
|
|
||||||
// Session/Segment. This reads that original lineage and still checks descriptor identity,
|
|
||||||
// canonical order and digest below; the shared service enforces the frozen byte range.
|
|
||||||
session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention(
|
|
||||||
&sessions,
|
|
||||||
source
|
|
||||||
.session_id
|
|
||||||
.parse()
|
|
||||||
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
|
|
||||||
source.pointer.segment_id,
|
|
||||||
session_store::public_material::PublicMaterialLimits {
|
|
||||||
max_segments: 256,
|
|
||||||
max_entries: 100_000,
|
|
||||||
max_record_bytes: 64 * 1024 * 1024,
|
|
||||||
},
|
|
||||||
crate::store::session_retention(&self.catalog.state_root),
|
|
||||||
)
|
|
||||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn journal_origins(
|
|
||||||
&self,
|
|
||||||
ranges: &[JournalSourceRange],
|
|
||||||
) -> Result<Vec<SourceOrigin>, SubjektivHostError> {
|
|
||||||
if ranges.is_empty() {
|
|
||||||
return Err(SubjektivHostError::InvalidArgument(
|
|
||||||
"empty material selection".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
ranges
|
|
||||||
.iter()
|
|
||||||
.map(|range| {
|
|
||||||
let reader = self.source_reader(&range.source)?;
|
|
||||||
reader
|
|
||||||
.descriptors()
|
|
||||||
.find(|descriptor| {
|
|
||||||
source_descriptor(&range.source.session_id, descriptor) == range.source
|
|
||||||
})
|
|
||||||
.map(|descriptor| SourceOrigin {
|
|
||||||
pointer: range.source.pointer.clone(),
|
|
||||||
origin: descriptor.origin.clone(),
|
|
||||||
})
|
|
||||||
.ok_or_else(|| {
|
|
||||||
SubjektivHostError::InvalidArgument(
|
|
||||||
"source provenance/identity changed".into(),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn journal_source_window(
|
|
||||||
&self,
|
|
||||||
source: &JournalSource,
|
|
||||||
offset: u64,
|
|
||||||
max_bytes: usize,
|
|
||||||
) -> Result<String, SubjektivHostError> {
|
|
||||||
let reader = self.source_reader(source)?;
|
|
||||||
let descriptor = reader.descriptors().find(|descriptor| source_descriptor(&source.session_id, descriptor) == *source)
|
|
||||||
.ok_or_else(|| SubjektivHostError::InvalidArgument("canonical source identity/order/digest changed or is outside the authorized Session".into()))?;
|
|
||||||
reader
|
|
||||||
.read_content(descriptor, offset, max_bytes)
|
|
||||||
.map(|window| window.content)
|
|
||||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn journal_source_material(
|
|
||||||
&self,
|
|
||||||
source: &JournalSource,
|
|
||||||
start: u64,
|
|
||||||
limit: usize,
|
|
||||||
) -> JournalResult<JournalMaterial> {
|
|
||||||
let text = self
|
|
||||||
.journal_source_window(source, start, limit.max(4))
|
|
||||||
.map_err(|error| JournalError::Conflict(error.to_string()))?;
|
|
||||||
Ok(JournalMaterial {
|
|
||||||
range: JournalSourceRange {
|
|
||||||
source: source.clone(),
|
|
||||||
bytes: JournalByteRange {
|
|
||||||
start,
|
|
||||||
end: start + text.len() as u64,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
text,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn select_journal_materials(
|
fn build_journal_input(
|
||||||
&self,
|
&self,
|
||||||
context: &JournalHostContext,
|
context: &JournalHostContext,
|
||||||
id: &str,
|
id: &str,
|
||||||
) -> Result<Vec<JournalMaterial>, SubjektivHostError> {
|
saved: Option<&CaptureDto>,
|
||||||
subjektiv::journal::recording::select_materials_bounded(
|
) -> Result<JournalInput, SubjektivHostError> {
|
||||||
&self.catalog.store,
|
let session = self
|
||||||
context,
|
.catalog
|
||||||
id,
|
.store
|
||||||
INPUT_BUDGET,
|
.read_journal(context, id)
|
||||||
|source, start, limit| self.journal_source_material(source, start, limit),
|
.map_err(journal_error)?
|
||||||
)
|
.source_session_id;
|
||||||
|
JournalInput::build(&self.catalog.store, context, id, |order, offset, budget| {
|
||||||
|
let capture = match saved {
|
||||||
|
Some(capture) => capture.clone(),
|
||||||
|
None => {
|
||||||
|
let capture = self
|
||||||
|
.with_recording_boundary(&session, |sessions, session_id, active, guard| {
|
||||||
|
session_store::capture::capture_recording_slice_fs_guarded(
|
||||||
|
sessions,
|
||||||
|
session_id,
|
||||||
|
active,
|
||||||
|
session_store::public_material::PublicMaterialLimits::default(),
|
||||||
|
RecordingPosition {
|
||||||
|
canonical_order: order,
|
||||||
|
byte_offset: offset,
|
||||||
|
},
|
||||||
|
MAX_JOURNAL_RANGES,
|
||||||
|
budget,
|
||||||
|
guard,
|
||||||
|
)
|
||||||
|
.map(|slice| slice.capture)
|
||||||
|
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
|
||||||
|
})
|
||||||
|
.map_err(|e| JournalError::Conflict(e.to_string()))?;
|
||||||
|
// Admission and capture are separate saves. An append between them
|
||||||
|
// may be in replay but has not yet been accepted for this Journal.
|
||||||
|
return self.accepted_capture_prefix(context, id, capture);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let end = capture
|
||||||
|
.entries
|
||||||
|
.last()
|
||||||
|
.map(|entry| {
|
||||||
|
capture
|
||||||
|
.recording_text(entry.order)
|
||||||
|
.map(|(descriptor, start, text)| {
|
||||||
|
if start + text.len() as u64 == descriptor.total_bytes {
|
||||||
|
RecordingPosition {
|
||||||
|
canonical_order: entry.order + 1,
|
||||||
|
byte_offset: 0,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
RecordingPosition {
|
||||||
|
canonical_order: entry.order,
|
||||||
|
byte_offset: start + text.len() as u64,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.transpose()
|
||||||
|
.map_err(|e| JournalError::Conflict(e.to_string()))?;
|
||||||
|
if end.is_none_or(|end| (order, offset) >= (end.canonical_order, end.byte_offset)) {
|
||||||
|
return Ok(CaptureDto {
|
||||||
|
entries: vec![],
|
||||||
|
..capture
|
||||||
|
});
|
||||||
|
}
|
||||||
|
capture
|
||||||
|
.recording_slice(
|
||||||
|
RecordingPosition {
|
||||||
|
canonical_order: order,
|
||||||
|
byte_offset: offset,
|
||||||
|
},
|
||||||
|
MAX_JOURNAL_RANGES,
|
||||||
|
budget,
|
||||||
|
)
|
||||||
|
.map(|slice| slice.capture)
|
||||||
|
.map_err(|e| JournalError::Conflict(e.to_string()))
|
||||||
|
})
|
||||||
.map_err(journal_error)
|
.map_err(journal_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn journal_grant(
|
fn accepted_capture_prefix(
|
||||||
|
&self,
|
||||||
|
context: &JournalHostContext,
|
||||||
|
id: &str,
|
||||||
|
mut capture: CaptureDto,
|
||||||
|
) -> JournalResult<CaptureDto> {
|
||||||
|
let mut selected = Vec::new();
|
||||||
|
for mut entry in capture.entries {
|
||||||
|
let status = match self
|
||||||
|
.catalog
|
||||||
|
.store
|
||||||
|
.read_journal_source(context, id, entry.order)
|
||||||
|
{
|
||||||
|
Ok(status) => status,
|
||||||
|
Err(JournalError::NotFound(_)) => break,
|
||||||
|
Err(error) => return Err(error),
|
||||||
|
};
|
||||||
|
let session_store::capture::CaptureBody::CanonicalPublic {
|
||||||
|
descriptor,
|
||||||
|
byte_offset,
|
||||||
|
text,
|
||||||
|
..
|
||||||
|
} = &mut entry.body
|
||||||
|
else {
|
||||||
|
return Err(JournalError::InvalidInput("non-recording capture".into()));
|
||||||
|
};
|
||||||
|
if source_descriptor(&capture.session_id, descriptor) != status.source {
|
||||||
|
return Err(JournalError::Conflict(
|
||||||
|
"captured source differs from admitted identity".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let Some(accepted) = status
|
||||||
|
.accepted
|
||||||
|
.iter()
|
||||||
|
.find(|range| range.start <= *byte_offset && *byte_offset < range.end)
|
||||||
|
else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
let end = status
|
||||||
|
.unavailable
|
||||||
|
.iter()
|
||||||
|
.filter(|range| range.end > *byte_offset)
|
||||||
|
.fold(accepted.end, |end, range| {
|
||||||
|
end.min(range.start.max(*byte_offset))
|
||||||
|
});
|
||||||
|
let mut length = text.len().min(end.saturating_sub(*byte_offset) as usize);
|
||||||
|
while !text.is_char_boundary(length) {
|
||||||
|
length -= 1;
|
||||||
|
}
|
||||||
|
if length == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
text.truncate(length);
|
||||||
|
let partial = *byte_offset + (text.len() as u64) < descriptor.total_bytes;
|
||||||
|
selected.push(entry);
|
||||||
|
if partial {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
capture.entries = selected;
|
||||||
|
Ok(capture)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn journal_grant_scope(
|
||||||
&self,
|
&self,
|
||||||
snapshot: &JobSnapshot,
|
snapshot: &JobSnapshot,
|
||||||
) -> Result<LocalJournalGrant, SubjektivHostError> {
|
) -> Result<LocalJournalGrant, SubjektivHostError> {
|
||||||
@@ -1097,7 +1358,6 @@ impl LocalSubjectHost {
|
|||||||
|| snapshot.request.purpose != PURPOSE
|
|| snapshot.request.purpose != PURPOSE
|
||||||
|| snapshot.request.serialization_key.as_deref()
|
|| snapshot.request.serialization_key.as_deref()
|
||||||
!= Some(&format!("subjektiv-journal:{}", grant.journal_id))
|
!= Some(&format!("subjektiv-journal:{}", grant.journal_id))
|
||||||
|| snapshot.request.input != grant.input()
|
|
||||||
{
|
{
|
||||||
return Err(SubjektivHostError::InvalidArgument(
|
return Err(SubjektivHostError::InvalidArgument(
|
||||||
"Job input/grant does not match this Journal capability".into(),
|
"Job input/grant does not match this Journal capability".into(),
|
||||||
@@ -1135,12 +1395,33 @@ impl LocalSubjectHost {
|
|||||||
Ok(grant)
|
Ok(grant)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn journal_grant(
|
||||||
|
&self,
|
||||||
|
snapshot: &JobSnapshot,
|
||||||
|
) -> Result<LocalJournalGrant, SubjektivHostError> {
|
||||||
|
snapshot
|
||||||
|
.load_executable_request()
|
||||||
|
.map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?;
|
||||||
|
let grant = self.journal_grant_scope(snapshot)?;
|
||||||
|
let input = JournalInput::decode(&snapshot.request.input).map_err(journal_error)?;
|
||||||
|
if input.subject_id != self.binding.subject_id
|
||||||
|
|| input.journal_id != grant.journal_id
|
||||||
|
|| input.capture.session_id != grant.session_id
|
||||||
|
|| input.selected_ranges != grant.ranges
|
||||||
|
{
|
||||||
|
return Err(SubjektivHostError::InvalidArgument(
|
||||||
|
"saved Journal body identity/ranges differ from grant".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(grant)
|
||||||
|
}
|
||||||
|
|
||||||
fn complete_journal_job(
|
fn complete_journal_job(
|
||||||
&self,
|
&self,
|
||||||
jobs: &crate::jobs::StandaloneJobs,
|
jobs: &crate::jobs::StandaloneJobs,
|
||||||
snapshot: &JobSnapshot,
|
snapshot: &JobSnapshot,
|
||||||
) -> Result<(), SubjektivHostError> {
|
) -> Result<(), SubjektivHostError> {
|
||||||
let grant = self.journal_grant(snapshot)?;
|
let grant = self.journal_grant_scope(snapshot)?;
|
||||||
let context = authority(self, &grant.session_id)?;
|
let context = authority(self, &grant.session_id)?;
|
||||||
let completion: JournalJobCompletion =
|
let completion: JournalJobCompletion =
|
||||||
serde_json::from_value(snapshot.result.clone().ok_or_else(|| {
|
serde_json::from_value(snapshot.result.clone().ok_or_else(|| {
|
||||||
@@ -1174,6 +1455,12 @@ impl LocalSubjectHost {
|
|||||||
}
|
}
|
||||||
jobs.acknowledge(&snapshot.request.job_id)
|
jobs.acknowledge(&snapshot.request.job_id)
|
||||||
.map_err(service_error)?;
|
.map_err(service_error)?;
|
||||||
|
jobs.reclaim_input(
|
||||||
|
&snapshot.request.job_id,
|
||||||
|
&snapshot.input_digest,
|
||||||
|
JobInputRetirementReason::Completed,
|
||||||
|
)
|
||||||
|
.map_err(service_error)?;
|
||||||
if let Err(error) = self.prune_retained_workers() {
|
if let Err(error) = self.prune_retained_workers() {
|
||||||
// Pruning is independent of a successful checkpoint/result. Keep the originals
|
// Pruning is independent of a successful checkpoint/result. Keep the originals
|
||||||
// and continue draining other saved prefixes; a retention error is not model failure.
|
// and continue draining other saved prefixes; a retention error is not model failure.
|
||||||
@@ -1192,47 +1479,18 @@ pub(super) fn grant(
|
|||||||
.journal_grant(snapshot)
|
.journal_grant(snapshot)
|
||||||
.map_err(|error| error.to_string())?;
|
.map_err(|error| error.to_string())?;
|
||||||
let context = authority(&host, &grant.session_id).map_err(|error| error.to_string())?;
|
let context = authority(&host, &grant.session_id).map_err(|error| error.to_string())?;
|
||||||
// Re-fetch selected identities before opening mutation authority. Only descriptors are
|
let input = JournalInput::decode(&snapshot.request.input).map_err(|e| e.to_string())?;
|
||||||
// retained in Job storage; the model must fetch actual source bytes with bounded tools.
|
let input_digest = snapshot.input_digest.clone();
|
||||||
for range in &grant.ranges {
|
|
||||||
let length = usize::try_from(
|
|
||||||
range
|
|
||||||
.bytes
|
|
||||||
.end
|
|
||||||
.checked_sub(range.bytes.start)
|
|
||||||
.ok_or("invalid source range")?,
|
|
||||||
)
|
|
||||||
.map_err(|_| "source range overflow")?;
|
|
||||||
if length > INPUT_BUDGET {
|
|
||||||
return Err("source range exceeds frozen input budget".into());
|
|
||||||
}
|
|
||||||
let text = host
|
|
||||||
.journal_source_window(&range.source, range.bytes.start, length.max(4))
|
|
||||||
.map_err(|error| error.to_string())?;
|
|
||||||
if text.len() != length {
|
|
||||||
return Err("frozen Journal source byte range changed".into());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if host
|
|
||||||
.journal_origins(&grant.ranges)
|
|
||||||
.map_err(|error| error.to_string())?
|
|
||||||
!= grant.origins
|
|
||||||
{
|
|
||||||
return Err(
|
|
||||||
"frozen Journal provenance does not match the authorized original Session".into(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
let generation = host
|
let generation = host
|
||||||
.catalog
|
.catalog
|
||||||
.store
|
.store
|
||||||
.prepare_journal_generation(
|
.prepare_journal_input(
|
||||||
&context,
|
&context,
|
||||||
&grant.journal_id,
|
&input,
|
||||||
JobAttemptBinding {
|
JobAttemptBinding {
|
||||||
job_id: snapshot.request.job_id.clone(),
|
job_id: snapshot.request.job_id.clone(),
|
||||||
attempt_id: snapshot.attempt.attempt_id.clone(),
|
attempt_id: snapshot.attempt.attempt_id.clone(),
|
||||||
},
|
},
|
||||||
grant.ranges.clone(),
|
|
||||||
)
|
)
|
||||||
.map_err(|error| error.to_string())?;
|
.map_err(|error| error.to_string())?;
|
||||||
let job_context = JournalJobContext::validated(
|
let job_context = JournalJobContext::validated(
|
||||||
@@ -1242,12 +1500,17 @@ pub(super) fn grant(
|
|||||||
generation.job.clone(),
|
generation.job.clone(),
|
||||||
)
|
)
|
||||||
.map_err(|error| error.to_string())?;
|
.map_err(|error| error.to_string())?;
|
||||||
|
input
|
||||||
|
.validate_generation(&generation)
|
||||||
|
.map_err(|e| e.to_string())?;
|
||||||
let adapter = Arc::new(LocalJournalHost {
|
let adapter = Arc::new(LocalJournalHost {
|
||||||
host,
|
host,
|
||||||
grant,
|
grant,
|
||||||
context,
|
context,
|
||||||
job_context,
|
job_context,
|
||||||
|
#[cfg(test)]
|
||||||
generation,
|
generation,
|
||||||
|
input_digest,
|
||||||
fence,
|
fence,
|
||||||
});
|
});
|
||||||
Ok(worker::job::JobFeatureGrant {
|
Ok(worker::job::JobFeatureGrant {
|
||||||
@@ -1263,7 +1526,9 @@ struct LocalJournalHost {
|
|||||||
grant: LocalJournalGrant,
|
grant: LocalJournalGrant,
|
||||||
context: JournalHostContext,
|
context: JournalHostContext,
|
||||||
job_context: JournalJobContext,
|
job_context: JournalJobContext,
|
||||||
|
#[cfg(test)]
|
||||||
generation: JournalGeneration,
|
generation: JournalGeneration,
|
||||||
|
input_digest: String,
|
||||||
fence: Arc<dyn crate::jobs::JobAttemptFence>,
|
fence: Arc<dyn crate::jobs::JobAttemptFence>,
|
||||||
}
|
}
|
||||||
impl LocalJournalHost {
|
impl LocalJournalHost {
|
||||||
@@ -1296,18 +1561,20 @@ impl JournalJobHost for LocalJournalHost {
|
|||||||
request: JournalJobRequest,
|
request: JournalJobRequest,
|
||||||
) -> Result<JournalJobResponse, SubjektivHostError> {
|
) -> Result<JournalJobResponse, SubjektivHostError> {
|
||||||
let context = self.live()?;
|
let context = self.live()?;
|
||||||
subjektiv::journal::service::execute_bounded(
|
// The generic runner has supplied this exact immutable input before it can
|
||||||
|
// invoke this bound tool. Preparation alone is not evidence of delivery.
|
||||||
|
subjektiv::journal::service::attest_input_delivery(
|
||||||
&self.host.catalog.store,
|
&self.host.catalog.store,
|
||||||
&context,
|
&context,
|
||||||
|
&self.input_digest,
|
||||||
|
&self.grant.ranges,
|
||||||
|
)
|
||||||
|
.map_err(journal_error)?;
|
||||||
|
subjektiv::journal::service::execute_snapshot(
|
||||||
|
&self.host.catalog.store,
|
||||||
|
&context,
|
||||||
|
&self.input_digest,
|
||||||
request,
|
request,
|
||||||
|source, start, limit| {
|
|
||||||
if source.session_id != self.generation.initial_journal.source_session_id {
|
|
||||||
return Err(JournalError::PermissionDenied(
|
|
||||||
"foreign original Session".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
self.host.journal_source_material(source, start, limit)
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
.map_err(journal_error)
|
.map_err(journal_error)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
//! Integration of standalone attribution, original FsStore material, shared Journal storage
|
//! Integration of standalone attribution, original FsStore material, shared Journal storage
|
||||||
//! and the local Job adapter. Only the outer model/execution lease is replaced in these cases.
|
//! and the local Job adapter. Only the outer model/execution lease is replaced in these cases.
|
||||||
//! The generic Job runner is exercised separately by the standalone Jobs integration suite.
|
//! The scripted model also exercises saved direct input through the actual generic Job runner.
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::store::{StaleLeasePolicy, StandaloneWorkerAllocation, StandaloneWorkerStore};
|
use crate::store::{StaleLeasePolicy, StandaloneWorkerAllocation, StandaloneWorkerStore};
|
||||||
@@ -144,8 +144,8 @@ impl Fixture {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
.contains(sentinel)
|
.contains(sentinel)
|
||||||
);
|
);
|
||||||
// Include SQLite/WAL/journal files: immutable request/grant storage must never hold
|
// Include SQLite/WAL/journal files: erasure must scrub retired original bodies,
|
||||||
// original text, even in freed pages left behind by explicit source erasure.
|
// including any freed pages or journal left by the immutable request store.
|
||||||
for entry in fs::read_dir(self.connection.jobs_path.parent().unwrap()).unwrap() {
|
for entry in fs::read_dir(self.connection.jobs_path.parent().unwrap()).unwrap() {
|
||||||
let entry = entry.unwrap();
|
let entry = entry.unwrap();
|
||||||
if entry.file_type().unwrap().is_file() {
|
if entry.file_type().unwrap().is_file() {
|
||||||
@@ -194,9 +194,9 @@ impl Fixture {
|
|||||||
context,
|
context,
|
||||||
job_context,
|
job_context,
|
||||||
generation,
|
generation,
|
||||||
|
input_digest: snapshot.input_digest.clone(),
|
||||||
fence: fence.clone(),
|
fence: fence.clone(),
|
||||||
};
|
};
|
||||||
adapter.read_all_sources();
|
|
||||||
(adapter, fence, snapshot)
|
(adapter, fence, snapshot)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -220,37 +220,6 @@ fn message(id: &str, text: &str) -> session_store::LoggedHistoryEntry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
impl LocalJournalHost {
|
impl LocalJournalHost {
|
||||||
fn read_all_sources(&self) {
|
|
||||||
for range in &self.generation.selected_ranges {
|
|
||||||
let mut offset = range.bytes.start as usize;
|
|
||||||
let mut content_token = None;
|
|
||||||
loop {
|
|
||||||
let response = self
|
|
||||||
.execute(JournalJobRequest::ReadText {
|
|
||||||
read: JournalTextRead {
|
|
||||||
target: JournalTextTarget::Source {
|
|
||||||
pointer: range.source.pointer.clone(),
|
|
||||||
},
|
|
||||||
offset,
|
|
||||||
limit: MAX_JOURNAL_READ_BYTES,
|
|
||||||
content_token: content_token.clone(),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
let JournalJobResponse::Text(text) = response else {
|
|
||||||
panic!()
|
|
||||||
};
|
|
||||||
match text.next_byte {
|
|
||||||
Some(next) => {
|
|
||||||
assert!(next > offset);
|
|
||||||
offset = next;
|
|
||||||
content_token = Some(text.content_token);
|
|
||||||
}
|
|
||||||
None => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fn state(&self) -> Result<JournalJobState, SubjektivHostError> {
|
fn state(&self) -> Result<JournalJobState, SubjektivHostError> {
|
||||||
match self.execute(JournalJobRequest::ReadState)? {
|
match self.execute(JournalJobRequest::ReadState)? {
|
||||||
JournalJobResponse::State(state) => Ok(state),
|
JournalJobResponse::State(state) => Ok(state),
|
||||||
@@ -362,8 +331,8 @@ fn duplicate_admission_reserves_one_job_and_excludes_private_prompt() {
|
|||||||
assert_eq!(jobs.pending().unwrap().len(), 1);
|
assert_eq!(jobs.pending().unwrap().len(), 1);
|
||||||
let input = jobs.get(&id).unwrap().request.input.to_string();
|
let input = jobs.get(&id).unwrap().request.input.to_string();
|
||||||
assert!(
|
assert!(
|
||||||
!input.contains("short request"),
|
input.contains("short request"),
|
||||||
"Job storage retains descriptors, never a second copy of original text"
|
"Job storage supplies the canonical body directly"
|
||||||
);
|
);
|
||||||
assert!(!input.contains("PRIVATE PROMPT"));
|
assert!(!input.contains("PRIVATE PROMPT"));
|
||||||
let foreign = SubjektivHostContext {
|
let foreign = SubjektivHostContext {
|
||||||
@@ -498,14 +467,16 @@ fn generation_input_stays_frozen_and_next_prefix_includes_last_input() {
|
|||||||
&adapter.generation.job,
|
&adapter.generation.job,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let materials = fixture
|
let next = fixture
|
||||||
.host()
|
.host()
|
||||||
.select_journal_materials(&adapter.context, &adapter.grant.journal_id)
|
.build_journal_input(&adapter.context, &adapter.grant.journal_id, None)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(materials[0].range.source.pointer.entry_ref, "Elast");
|
assert_eq!(next.selected_ranges[0].source.pointer.entry_ref, "Elast");
|
||||||
assert!(
|
assert!(
|
||||||
materials[0]
|
next.capture
|
||||||
.text
|
.recording_text(next.selected_ranges[0].source.order)
|
||||||
|
.unwrap()
|
||||||
|
.2
|
||||||
.contains("last input while generation runs")
|
.contains("last input while generation runs")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -547,7 +518,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() {
|
|||||||
let (adapter, _, _) = fixture.adapter();
|
let (adapter, _, _) = fixture.adapter();
|
||||||
let range = &adapter.grant.ranges[0];
|
let range = &adapter.grant.ranges[0];
|
||||||
assert!(range.bytes.end < range.source.total_bytes);
|
assert!(range.bytes.end < range.source.total_bytes);
|
||||||
assert!(range.bytes.end - range.bytes.start < INPUT_BUDGET as u64);
|
assert!(range.bytes.end - range.bytes.start < job::MAX_JOB_INPUT_BYTES as u64);
|
||||||
let snapshot = fixture
|
let snapshot = fixture
|
||||||
.host()
|
.host()
|
||||||
.jobs()
|
.jobs()
|
||||||
@@ -557,7 +528,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() {
|
|||||||
.pop()
|
.pop()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(serde_json::to_vec(&snapshot.request.input).unwrap().len() < 128 * 1024);
|
assert!(serde_json::to_vec(&snapshot.request.input).unwrap().len() < 128 * 1024);
|
||||||
assert!(!snapshot.request.input.to_string().contains("漢字"));
|
assert!(snapshot.request.input.to_string().contains("漢字"));
|
||||||
assert!(adapter.finish().is_err());
|
assert!(adapter.finish().is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -628,43 +599,35 @@ impl JournalScript {
|
|||||||
)
|
)
|
||||||
} else if let Some(state) = result("state") {
|
} else if let Some(state) = result("state") {
|
||||||
let state = &state["value"];
|
let state = &state["value"];
|
||||||
for (index, range) in state["selected_ranges"]
|
let input: serde_json::Value = request
|
||||||
.as_array()
|
.items
|
||||||
.unwrap()
|
|
||||||
.iter()
|
.iter()
|
||||||
.enumerate()
|
.filter_map(Item::as_text)
|
||||||
{
|
.find_map(|text| {
|
||||||
let mut offset = range["bytes"]["start"].as_u64().unwrap();
|
text.lines()
|
||||||
let mut token = Value::Null;
|
.find_map(|line| line.strip_prefix("input_json: "))
|
||||||
loop {
|
})
|
||||||
let id = format!("source-{index}-{offset}");
|
.map(|json| serde_json::from_str(json).unwrap())
|
||||||
if let Some(page) = result(&id) {
|
.expect("direct saved Journal input");
|
||||||
let page = &page["value"];
|
let input = JournalInput::decode(&input).unwrap();
|
||||||
let key = format!("{}:{id}", state["generation_id"].as_str().unwrap());
|
assert_eq!(
|
||||||
let text = page["text"].as_str().unwrap().to_owned();
|
serde_json::to_value(&input.selected_ranges).unwrap(),
|
||||||
if let Some(previous) =
|
state["selected_ranges"]
|
||||||
self.source_reads.lock().unwrap().insert(key, text.clone())
|
);
|
||||||
{
|
for entry in &input.capture.entries {
|
||||||
assert_eq!(
|
let (_, offset, text) = input.capture.recording_text(entry.order).unwrap();
|
||||||
previous, text,
|
let key = format!(
|
||||||
"frozen source read changed within a generation"
|
"{}:{}:{offset}",
|
||||||
);
|
state["generation_id"].as_str().unwrap(),
|
||||||
}
|
entry.order
|
||||||
if let Some(next) = page["next_byte"].as_u64() {
|
);
|
||||||
assert!(next > offset);
|
if let Some(previous) = self
|
||||||
offset = next;
|
.source_reads
|
||||||
token = page["content_token"].clone();
|
.lock()
|
||||||
} else {
|
.unwrap()
|
||||||
break;
|
.insert(key, text.to_owned())
|
||||||
}
|
{
|
||||||
} else {
|
assert_eq!(previous, text, "saved source changed within generation");
|
||||||
let input = json!({"target":{"target":"source","pointer":range["source"]["pointer"]},"offset":offset,"limit":MAX_JOURNAL_READ_BYTES,"content_token":token});
|
|
||||||
return vec![
|
|
||||||
Event::tool_use_start(0, &id, "ReadJournalText"),
|
|
||||||
Event::tool_input_delta(0, input.to_string()),
|
|
||||||
Event::tool_use_stop(0),
|
|
||||||
];
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let (name, action) = if state["current_open"].is_null() {
|
let (name, action) = if state["current_open"].is_null() {
|
||||||
@@ -1166,6 +1129,11 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
|
|||||||
.items;
|
.items;
|
||||||
assert_eq!(entries.len(), 1);
|
assert_eq!(entries.len(), 1);
|
||||||
let saved_entry = entries[0].id.clone();
|
let saved_entry = entries[0].id.clone();
|
||||||
|
let original_input = jobs.get(&id).unwrap().request.input;
|
||||||
|
fixture.append(
|
||||||
|
"newer-than-snapshot",
|
||||||
|
"NEW INPUT MUST NOT MIX INTO RESUMED SNAPSHOT",
|
||||||
|
);
|
||||||
jobs.shutdown().await.unwrap();
|
jobs.shutdown().await.unwrap();
|
||||||
fixture.connection.close().unwrap();
|
fixture.connection.close().unwrap();
|
||||||
let mut reopened = StandaloneSubjects::open_existing(&state)
|
let mut reopened = StandaloneSubjects::open_existing(&state)
|
||||||
@@ -1176,8 +1144,29 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
reopened.host.bind_jobs(jobs.clone()).unwrap();
|
reopened.host.bind_jobs(jobs.clone()).unwrap();
|
||||||
assert_eq!(jobs.get(&id).unwrap().state, JobState::Unknown);
|
assert_eq!(jobs.get(&id).unwrap().state, JobState::Unknown);
|
||||||
// Domain reconciliation inspects the now-fenced prior attempt and preserves its receipts.
|
// Deny every original-source read while preparing and running the replacement.
|
||||||
|
let source_path = fixture
|
||||||
|
.sessions
|
||||||
|
.root_dir()
|
||||||
|
.join(&fixture.context.session_id);
|
||||||
|
let denied = fixture
|
||||||
|
._temp
|
||||||
|
.path()
|
||||||
|
.join("temporarily-unavailable-original");
|
||||||
|
fs::rename(&source_path, &denied).unwrap();
|
||||||
let (_, replacement) = reopened.host.next_journal_job().unwrap().unwrap();
|
let (_, replacement) = reopened.host.next_journal_job().unwrap().unwrap();
|
||||||
|
let replacement_input = jobs.get(&replacement).unwrap().request.input;
|
||||||
|
assert_eq!(replacement_input["capture"], original_input["capture"]);
|
||||||
|
assert!(
|
||||||
|
!replacement_input
|
||||||
|
.to_string()
|
||||||
|
.contains("NEW INPUT MUST NOT MIX")
|
||||||
|
);
|
||||||
|
assert_eq!(jobs.get(&id).unwrap().request.input, original_input);
|
||||||
|
assert_eq!(
|
||||||
|
jobs.get(&id).unwrap().input_disposition,
|
||||||
|
JobInputDisposition::Available
|
||||||
|
);
|
||||||
assert_ne!(
|
assert_ne!(
|
||||||
replacement, id,
|
replacement, id,
|
||||||
"unknown external execution is not blindly replayed"
|
"unknown external execution is not blindly replayed"
|
||||||
@@ -1205,8 +1194,26 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
|
|||||||
.items;
|
.items;
|
||||||
assert_eq!(entries.len(), 1);
|
assert_eq!(entries.len(), 1);
|
||||||
assert_eq!(entries[0].id, saved_entry);
|
assert_eq!(entries[0].id, saved_entry);
|
||||||
|
fs::rename(&denied, &source_path).unwrap();
|
||||||
|
let journal = reopened
|
||||||
|
.host
|
||||||
|
.catalog
|
||||||
|
.store
|
||||||
|
.read_journal(&context, &journal_id)
|
||||||
|
.unwrap();
|
||||||
|
let original = JournalInput::decode(&original_input).unwrap();
|
||||||
|
let end = original.selected_ranges.last().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
(journal.process_order, journal.process_byte),
|
||||||
|
if end.bytes.end == end.source.total_bytes {
|
||||||
|
(end.source.order + 1, 0)
|
||||||
|
} else {
|
||||||
|
(end.source.order, end.bytes.end)
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert!(reopened.host.admit_journal(&fixture.context).unwrap());
|
||||||
assert!(
|
assert!(
|
||||||
reopened
|
!reopened
|
||||||
.host
|
.host
|
||||||
.session_fully_reflected(&fixture.context.session_id)
|
.session_fully_reflected(&fixture.context.session_id)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
@@ -1302,9 +1309,9 @@ fn missing_original_material_blocks_worker_removal_without_erasing_state() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn frozen_original_remains_readable_after_worker_switches_its_active_session() {
|
fn frozen_input_survives_active_session_change_and_source_tool_remains_refused() {
|
||||||
let fixture = Fixture::new("old Session original");
|
let fixture = Fixture::new("old Session original");
|
||||||
let (adapter, _, _) = fixture.adapter();
|
let (adapter, _, snapshot) = fixture.adapter();
|
||||||
let workers = StandaloneWorkerStore::open_existing(&fixture.host().catalog.state_root).unwrap();
|
let workers = StandaloneWorkerStore::open_existing(&fixture.host().catalog.state_root).unwrap();
|
||||||
let worker = fixture._allocation.as_ref().unwrap().worker_id();
|
let worker = fixture._allocation.as_ref().unwrap().worker_id();
|
||||||
let record = workers.load(worker).unwrap();
|
let record = workers.load(worker).unwrap();
|
||||||
@@ -1318,22 +1325,30 @@ fn frozen_original_remains_readable_after_worker_switches_its_active_session() {
|
|||||||
)),
|
)),
|
||||||
))
|
))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let response = adapter
|
assert!(
|
||||||
.execute(JournalJobRequest::ReadText {
|
adapter
|
||||||
read: JournalTextRead {
|
.execute(JournalJobRequest::ReadText {
|
||||||
target: JournalTextTarget::Source {
|
read: JournalTextRead {
|
||||||
pointer: adapter.generation.selected_ranges[0].source.pointer.clone(),
|
target: JournalTextTarget::Source {
|
||||||
},
|
pointer: adapter.generation.selected_ranges[0].source.pointer.clone()
|
||||||
offset: 0,
|
},
|
||||||
limit: MAX_JOURNAL_READ_BYTES,
|
offset: 0,
|
||||||
content_token: None,
|
limit: MAX_JOURNAL_READ_BYTES,
|
||||||
},
|
content_token: None
|
||||||
})
|
}
|
||||||
.unwrap();
|
})
|
||||||
let JournalJobResponse::Text(page) = response else {
|
.is_err(),
|
||||||
panic!()
|
"original Source lookup is explicitly unavailable"
|
||||||
};
|
);
|
||||||
assert!(page.text.contains("old Session original"));
|
let input = JournalInput::decode(&snapshot.request.input).unwrap();
|
||||||
|
assert!(
|
||||||
|
input
|
||||||
|
.capture
|
||||||
|
.recording_text(0)
|
||||||
|
.unwrap()
|
||||||
|
.2
|
||||||
|
.contains("old Session original")
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
fixture
|
fixture
|
||||||
.host()
|
.host()
|
||||||
@@ -1512,14 +1527,18 @@ fn relocated_erasure_fences_old_and_retained_readers_publishers_and_late_pins()
|
|||||||
const SENTINEL: &str = "original-plaintext-sentinel-must-never-enter-immutable-job-storage";
|
const SENTINEL: &str = "original-plaintext-sentinel-must-never-enter-immutable-job-storage";
|
||||||
let mut fixture = Fixture::new(SENTINEL);
|
let mut fixture = Fixture::new(SENTINEL);
|
||||||
let (_, _, snapshot) = fixture.adapter();
|
let (_, _, snapshot) = fixture.adapter();
|
||||||
fixture.assert_job_storage_excludes(&snapshot, SENTINEL);
|
assert!(snapshot.request.input.to_string().contains(SENTINEL));
|
||||||
let state = fixture.host().catalog.state_root.clone();
|
let state = fixture.host().catalog.state_root.clone();
|
||||||
let binding = fixture.connection.binding.clone();
|
let binding = fixture.connection.binding.clone();
|
||||||
let session: session_store::SessionId = fixture.context.session_id.parse().unwrap();
|
let session: session_store::SessionId = fixture.context.session_id.parse().unwrap();
|
||||||
let old_reader = fixture
|
let old_reader = SessionPublicMaterialReader::open_fs_with_retention(
|
||||||
.host()
|
fixture.sessions.root_dir(),
|
||||||
.material_reader(&fixture.context.session_id)
|
session,
|
||||||
.unwrap();
|
fixture.segment,
|
||||||
|
PublicMaterialLimits::default(),
|
||||||
|
crate::store::session_retention(&state),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let descriptor = old_reader.descriptors().next().unwrap().clone();
|
let descriptor = old_reader.descriptors().next().unwrap().clone();
|
||||||
let (worker, workers) = fixture.stop_worker();
|
let (worker, workers) = fixture.stop_worker();
|
||||||
assert!(workers.delete(worker).is_err());
|
assert!(workers.delete(worker).is_err());
|
||||||
@@ -1759,14 +1778,12 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job(
|
|||||||
let snapshot = jobs.get(&next).unwrap();
|
let snapshot = jobs.get(&next).unwrap();
|
||||||
let grant = fixture.host().journal_grant(&snapshot).unwrap();
|
let grant = fixture.host().journal_grant(&snapshot).unwrap();
|
||||||
assert_eq!(grant.ranges.len(), 1);
|
assert_eq!(grant.ranges.len(), 1);
|
||||||
assert_eq!(
|
let input = JournalInput::decode(&snapshot.request.input).unwrap();
|
||||||
grant.origins[0].origin,
|
let (descriptor, _, content) = input
|
||||||
PublicMaterialOrigin::HostRunBoundary
|
.capture
|
||||||
);
|
.recording_text(grant.ranges[0].source.order)
|
||||||
let content = fixture
|
|
||||||
.host()
|
|
||||||
.journal_source_window(&grant.ranges[0].source, 0, 4096)
|
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
assert_eq!(descriptor.origin, PublicMaterialOrigin::HostRunBoundary);
|
||||||
let value: serde_json::Value = serde_json::from_str(&content).unwrap();
|
let value: serde_json::Value = serde_json::from_str(&content).unwrap();
|
||||||
assert_eq!(value["kind"], "run_boundary");
|
assert_eq!(value["kind"], "run_boundary");
|
||||||
assert_eq!(value["boundary"]["kind"], "errored");
|
assert_eq!(value["boundary"]["kind"], "errored");
|
||||||
@@ -1791,3 +1808,621 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job(
|
|||||||
assert!(fixture.host().next_journal_job().unwrap().is_none());
|
assert!(fixture.host().next_journal_job().unwrap().is_none());
|
||||||
jobs.shutdown().await.unwrap();
|
jobs.shutdown().await.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Original FsStore -> bounded common capture -> persisted Job -> actual generic runner ->
|
||||||
|
/// sequential Journal tools. Only the LLM transport is scripted. Source files are deliberately
|
||||||
|
/// unavailable after persistence, and input bodies are inspected in the provider request.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn saved_body_jobs_process_over_a_hundred_entries_and_utf8_without_live_source_reads() {
|
||||||
|
let fixture = Fixture::new("oldest original");
|
||||||
|
for index in 0..115 {
|
||||||
|
fixture.append(
|
||||||
|
&format!("entry-{index}"),
|
||||||
|
&format!("public original {index}"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
fixture.append("long-utf8", &"漢字🙂\n\"".repeat(24_000));
|
||||||
|
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||||
|
let expected = fixture
|
||||||
|
.host()
|
||||||
|
.recording_capture(&fixture.context.session_id)
|
||||||
|
.unwrap();
|
||||||
|
let source_path = fixture
|
||||||
|
.sessions
|
||||||
|
.root_dir()
|
||||||
|
.join(&fixture.context.session_id);
|
||||||
|
let denied = fixture._temp.path().join("denied-original-source");
|
||||||
|
let mut seen: std::collections::BTreeMap<u64, String> = Default::default();
|
||||||
|
let mut jobs_processed = 0;
|
||||||
|
loop {
|
||||||
|
let Some((jobs, id)) = fixture.host().next_journal_job().unwrap() else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
let saved = jobs.get(&id).unwrap();
|
||||||
|
let input = JournalInput::decode(&saved.request.input).unwrap();
|
||||||
|
assert!(
|
||||||
|
serde_json::to_vec(&saved.request.input).unwrap().len() <= job::MAX_JOB_INPUT_BYTES
|
||||||
|
);
|
||||||
|
assert!(input.capture.entries.len() <= MAX_JOURNAL_RANGES);
|
||||||
|
for entry in &input.capture.entries {
|
||||||
|
let (_, offset, text) = input.capture.recording_text(entry.order).unwrap();
|
||||||
|
let previous = seen.entry(entry.order).or_default();
|
||||||
|
assert_eq!(
|
||||||
|
previous.len() as u64,
|
||||||
|
offset,
|
||||||
|
"no gap, preview or duplicate prefix"
|
||||||
|
);
|
||||||
|
previous.push_str(text);
|
||||||
|
}
|
||||||
|
// Deny all catalogs/reads, including grant preparation. The model still receives
|
||||||
|
// exactly the persisted body, and the Host must not follow the active pointer.
|
||||||
|
fs::rename(&source_path, &denied).unwrap();
|
||||||
|
let model = JournalScript::default();
|
||||||
|
jobs.start_with_model_client(&id, model.clone())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let result = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(result.state, JobState::Completed);
|
||||||
|
assert!(
|
||||||
|
!model.source_reads.lock().unwrap().is_empty(),
|
||||||
|
"provider saw saved source bodies"
|
||||||
|
);
|
||||||
|
fixture.host().complete_journal_job(&jobs, &result).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
jobs.get(&id).unwrap().input_disposition,
|
||||||
|
JobInputDisposition::Reclaimed
|
||||||
|
);
|
||||||
|
fs::rename(&denied, &source_path).unwrap();
|
||||||
|
jobs_processed += 1;
|
||||||
|
assert!(jobs_processed < 20, "bounded prefix drain must progress");
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
jobs_processed > 2,
|
||||||
|
"entry count and long text both require continuation"
|
||||||
|
);
|
||||||
|
assert_eq!(seen.len(), expected.entries.len());
|
||||||
|
for entry in &expected.entries {
|
||||||
|
assert_eq!(
|
||||||
|
seen[&entry.order],
|
||||||
|
expected.recording_text(entry.order).unwrap().2
|
||||||
|
);
|
||||||
|
}
|
||||||
|
fixture.host().jobs().unwrap().shutdown().await.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful_records() {
|
||||||
|
let fixture = Fixture::new("original descriptor-only input");
|
||||||
|
let (adapter, _, modern) = fixture.adapter();
|
||||||
|
let jobs = fixture.host().jobs().unwrap();
|
||||||
|
fixture
|
||||||
|
.host()
|
||||||
|
.catalog
|
||||||
|
.store
|
||||||
|
.close_journal_generation(
|
||||||
|
&adapter.context,
|
||||||
|
&adapter.generation.id,
|
||||||
|
&adapter.generation.job,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
jobs.cancel(&modern.request.job_id).unwrap();
|
||||||
|
let input = JournalInput::decode(&modern.request.input).unwrap();
|
||||||
|
let origins: Vec<_> = input
|
||||||
|
.capture
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.map(|entry| {
|
||||||
|
let (descriptor, _, _) = input.capture.recording_text(entry.order).unwrap();
|
||||||
|
SourceOrigin {
|
||||||
|
pointer: source_descriptor(&fixture.context.session_id, descriptor).pointer,
|
||||||
|
origin: descriptor.origin.clone(),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let mut old_grant = adapter.grant.clone();
|
||||||
|
old_grant.origins = origins;
|
||||||
|
old_grant.writer_binding_id = None;
|
||||||
|
let mut request = modern.request.clone();
|
||||||
|
request.job_id = format!("legacy-descriptor:{}", uuid::Uuid::now_v7());
|
||||||
|
request.input =
|
||||||
|
serde_json::json!({"selected_ranges":old_grant.ranges,"origins":old_grant.origins});
|
||||||
|
let old = jobs
|
||||||
|
.request_granted(request, serde_json::to_value(&old_grant).unwrap())
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
fixture.host().journal_grant(&old).is_err(),
|
||||||
|
"descriptor jobs must not execute as snapshots"
|
||||||
|
);
|
||||||
|
let generation = fixture
|
||||||
|
.host()
|
||||||
|
.catalog
|
||||||
|
.store
|
||||||
|
.prepare_journal_generation(
|
||||||
|
&adapter.context,
|
||||||
|
&old_grant.journal_id,
|
||||||
|
JobAttemptBinding {
|
||||||
|
job_id: old.request.job_id.clone(),
|
||||||
|
attempt_id: old.attempt.attempt_id.clone(),
|
||||||
|
},
|
||||||
|
old_grant.ranges.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let context = JournalJobContext::validated(
|
||||||
|
&fixture.host().catalog.store,
|
||||||
|
&adapter.context,
|
||||||
|
&generation.id,
|
||||||
|
generation.job.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let saved = fixture
|
||||||
|
.host()
|
||||||
|
.catalog
|
||||||
|
.store
|
||||||
|
.execute_journal(
|
||||||
|
&context,
|
||||||
|
JournalOperation {
|
||||||
|
request_id: "legacy-record-success-response-lost".into(),
|
||||||
|
expected_state: generation.initial_journal.state_token.clone(),
|
||||||
|
action: JournalAction::Create {
|
||||||
|
title: "already saved".into(),
|
||||||
|
body: "legacy successful operation".into(),
|
||||||
|
ongoing: "unfinished".into(),
|
||||||
|
state: JournalEntryState::Open,
|
||||||
|
source_end: None,
|
||||||
|
correction_refs: vec![],
|
||||||
|
},
|
||||||
|
covered_ranges: generation.selected_ranges.clone(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fixture.append(
|
||||||
|
"new-not-selected",
|
||||||
|
"LATEST SOURCE MUST WAIT UNTIL OLD INPUT FINISHES",
|
||||||
|
);
|
||||||
|
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||||
|
let (_, next) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||||
|
assert_ne!(next, old.request.job_id);
|
||||||
|
let replacement = jobs.get(&next).unwrap();
|
||||||
|
let replacement_input = JournalInput::decode(&replacement.request.input).unwrap();
|
||||||
|
assert!(
|
||||||
|
!replacement
|
||||||
|
.request
|
||||||
|
.input
|
||||||
|
.to_string()
|
||||||
|
.contains("LATEST SOURCE MUST WAIT")
|
||||||
|
);
|
||||||
|
assert_eq!(replacement_input.selected_ranges, old_grant.ranges);
|
||||||
|
let retired = jobs.get(&old.request.job_id).unwrap();
|
||||||
|
assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated);
|
||||||
|
assert_eq!(retired.input_digest, old.input_digest);
|
||||||
|
assert_eq!(retired.intent_fingerprint, old.intent_fingerprint);
|
||||||
|
assert!(retired.request.input.is_null());
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.host()
|
||||||
|
.catalog
|
||||||
|
.store
|
||||||
|
.execute_journal(
|
||||||
|
&context,
|
||||||
|
JournalOperation {
|
||||||
|
request_id: "legacy-delayed".into(),
|
||||||
|
expected_state: saved.state_token,
|
||||||
|
action: JournalAction::ConfirmUnchanged,
|
||||||
|
covered_ranges: old_grant.ranges,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
jobs.start_with_model_client(&next, JournalScript::default())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&next))
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(completed.state, JobState::Completed);
|
||||||
|
fixture
|
||||||
|
.host()
|
||||||
|
.complete_journal_job(&jobs, &completed)
|
||||||
|
.unwrap();
|
||||||
|
let entries = fixture
|
||||||
|
.host()
|
||||||
|
.catalog
|
||||||
|
.store
|
||||||
|
.list_journal_entries(&adapter.context, &adapter.grant.journal_id, None, 32)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(entries.items.len(), 1);
|
||||||
|
assert_eq!(entries.items[0].id, saved.entry_id.unwrap());
|
||||||
|
jobs.shutdown().await.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn explicit_erase_retires_every_historical_body_across_job_history_pages() {
|
||||||
|
const SENTINEL: &str = "historical-input-erasure-sentinel-781";
|
||||||
|
let mut fixture = Fixture::new("ordinary original");
|
||||||
|
let (adapter, _, first) = fixture.adapter();
|
||||||
|
let jobs = fixture.host().jobs().unwrap();
|
||||||
|
for index in 0..105 {
|
||||||
|
let mut request = first.request.clone();
|
||||||
|
request.job_id = format!("historical-{index:03}");
|
||||||
|
request.purpose = "historical-other-domain-job".into();
|
||||||
|
request.serialization_key = None;
|
||||||
|
request.input = serde_json::json!({"body":SENTINEL, "index":index});
|
||||||
|
let saved = jobs
|
||||||
|
.request_granted(
|
||||||
|
request,
|
||||||
|
serde_json::json!({"session_id":fixture.context.session_id}),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
jobs.cancel(&saved.request.job_id).unwrap();
|
||||||
|
}
|
||||||
|
let state = fixture.host().catalog.state_root.clone();
|
||||||
|
let subject = fixture.connection.binding.subject_id.clone();
|
||||||
|
fixture.connection.close().unwrap();
|
||||||
|
StandaloneSubjects::open_existing(&state)
|
||||||
|
.unwrap()
|
||||||
|
.erase_session_completely(&subject, &fixture.context.session_id, "erase-all-history")
|
||||||
|
.unwrap();
|
||||||
|
let jobs = crate::jobs::StandaloneJobs::open(
|
||||||
|
&fixture.connection.jobs_path,
|
||||||
|
fixture._temp.path().into(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let mut after = None;
|
||||||
|
let mut count = 0;
|
||||||
|
loop {
|
||||||
|
let page = jobs
|
||||||
|
.list_session_snapshots(&fixture.context.session_id, after.as_deref(), 100)
|
||||||
|
.unwrap();
|
||||||
|
if page.is_empty() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
for saved in &page {
|
||||||
|
assert_eq!(saved.input_disposition, JobInputDisposition::Erased);
|
||||||
|
assert!(saved.request.input.is_null());
|
||||||
|
assert!(saved.load_executable_request().is_err());
|
||||||
|
}
|
||||||
|
count += page.len();
|
||||||
|
after = page.last().map(|job| job.request.job_id.clone());
|
||||||
|
}
|
||||||
|
assert_eq!(count, 106);
|
||||||
|
assert!(
|
||||||
|
adapter.execute(JournalJobRequest::ReadState).is_err(),
|
||||||
|
"prepared local adapter is invalidated"
|
||||||
|
);
|
||||||
|
fixture.assert_job_storage_excludes(&jobs.get(&first.request.job_id).unwrap(), SENTINEL);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn saved_snapshot_missing_tampered_foreign_or_changed_initial_state_is_not_recaptured() {
|
||||||
|
let fixture = Fixture::new("saved immutable original");
|
||||||
|
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||||
|
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||||
|
let saved = jobs.get(&id).unwrap();
|
||||||
|
for changed in [serde_json::Value::Null, serde_json::json!({"capture":{}})] {
|
||||||
|
let mut malformed = saved.clone();
|
||||||
|
malformed.request.input = changed;
|
||||||
|
assert!(fixture.host().journal_grant(&malformed).is_err());
|
||||||
|
}
|
||||||
|
let original = JournalInput::decode(&saved.request.input).unwrap();
|
||||||
|
for field in ["subject_id", "journal_id", "initial_state_token"] {
|
||||||
|
let mut changed = saved.request.clone();
|
||||||
|
changed.job_id = format!("changed-{field}");
|
||||||
|
changed.serialization_key = None;
|
||||||
|
changed.input[field] = serde_json::json!("foreign-or-altered");
|
||||||
|
// Even a separately reserved immutable Job may not adopt the original capability.
|
||||||
|
let changed = jobs
|
||||||
|
.request_granted(changed, saved.domain_grant.clone().unwrap())
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
grant(
|
||||||
|
fixture.host().clone(),
|
||||||
|
&changed,
|
||||||
|
Arc::new(TestFence(AtomicBool::new(true)))
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let mut generation = fixture
|
||||||
|
.host()
|
||||||
|
.catalog
|
||||||
|
.store
|
||||||
|
.prepare_journal_generation(
|
||||||
|
&authority(fixture.host(), &fixture.context.session_id).unwrap(),
|
||||||
|
&original.journal_id,
|
||||||
|
JobAttemptBinding {
|
||||||
|
job_id: id,
|
||||||
|
attempt_id: saved.attempt.attempt_id.clone(),
|
||||||
|
},
|
||||||
|
original.selected_ranges.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
generation.initial_journal.state_token = "changed-after-snapshot".into();
|
||||||
|
assert!(original.validate_generation(&generation).is_err());
|
||||||
|
assert_eq!(
|
||||||
|
saved.request.input,
|
||||||
|
jobs.get(&saved.request.job_id).unwrap().request.input
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn append_between_admission_and_body_capture_does_not_publish_unaccepted_input() {
|
||||||
|
let fixture = Fixture::new("accepted oldest prefix");
|
||||||
|
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||||
|
fixture.append("not-yet-admitted", "newly saved but not accepted");
|
||||||
|
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||||
|
let input = jobs.get(&id).unwrap().request.input;
|
||||||
|
assert!(input.to_string().contains("accepted oldest prefix"));
|
||||||
|
assert!(!input.to_string().contains("newly saved but not accepted"));
|
||||||
|
jobs.start_with_model_client(&id, JournalScript::default())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(completed.state, JobState::Completed);
|
||||||
|
fixture
|
||||||
|
.host()
|
||||||
|
.complete_journal_job(&jobs, &completed)
|
||||||
|
.unwrap();
|
||||||
|
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||||
|
let (_, next) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||||
|
let remaining = jobs.get(&next).unwrap().request.input;
|
||||||
|
assert!(
|
||||||
|
remaining
|
||||||
|
.to_string()
|
||||||
|
.contains("newly saved but not accepted")
|
||||||
|
);
|
||||||
|
assert!(!remaining.to_string().contains("accepted oldest prefix"));
|
||||||
|
jobs.shutdown().await.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Generic Job preparation must not attest delivery. The saved body is supplied by the
|
||||||
|
/// runner before a bound Journal tool may execute; no model-provided digest is trusted.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn actual_bound_tool_attests_delivery_but_prepared_blocked_job_does_not() {
|
||||||
|
let fixture = Fixture::new("body delivered through generic runner");
|
||||||
|
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||||
|
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||||
|
let saved = jobs.get(&id).unwrap();
|
||||||
|
let model = JournalScript {
|
||||||
|
gate_first: true,
|
||||||
|
pause_after_record: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
jobs.start_with_model_client(&id, model.clone())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let host_context = authority(fixture.host(), &fixture.context.session_id).unwrap();
|
||||||
|
let journal_id = fixture
|
||||||
|
.host()
|
||||||
|
.catalog
|
||||||
|
.store
|
||||||
|
.journal_for_session(&host_context)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let generation = fixture
|
||||||
|
.host()
|
||||||
|
.catalog
|
||||||
|
.store
|
||||||
|
.active_journal_generation(&host_context, &journal_id)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
let context = JournalJobContext::validated(
|
||||||
|
&fixture.host().catalog.store,
|
||||||
|
&host_context,
|
||||||
|
&generation.id,
|
||||||
|
generation.job.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let undelivered = subjektiv::journal::service::execute_snapshot(
|
||||||
|
&fixture.host().catalog.store,
|
||||||
|
&context,
|
||||||
|
&saved.input_digest,
|
||||||
|
JournalJobRequest::ReadState,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
undelivered.to_string().contains("delivery"),
|
||||||
|
"prepared grant must not satisfy delivery gate: {undelivered}"
|
||||||
|
);
|
||||||
|
let before = fixture
|
||||||
|
.host()
|
||||||
|
.catalog
|
||||||
|
.store
|
||||||
|
.read_journal(&host_context, &journal_id)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!((before.process_order, before.process_byte), (0, 0));
|
||||||
|
model.release.notify_one();
|
||||||
|
tokio::time::timeout(
|
||||||
|
std::time::Duration::from_secs(20),
|
||||||
|
model.record_saved.notified(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
subjektiv::journal::service::execute_snapshot(
|
||||||
|
&fixture.host().catalog.store,
|
||||||
|
&context,
|
||||||
|
&saved.input_digest,
|
||||||
|
JournalJobRequest::ReadState
|
||||||
|
)
|
||||||
|
.is_ok(),
|
||||||
|
"actual bound tool execution attests exact saved input"
|
||||||
|
);
|
||||||
|
model.release.notify_one();
|
||||||
|
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(completed.state, JobState::Completed);
|
||||||
|
fixture
|
||||||
|
.host()
|
||||||
|
.complete_journal_job(&jobs, &completed)
|
||||||
|
.unwrap();
|
||||||
|
jobs.shutdown().await.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
struct BlockedJournalRequest {
|
||||||
|
entered: Arc<tokio::sync::Notify>,
|
||||||
|
dropped: Arc<tokio::sync::Notify>,
|
||||||
|
request_dropped: Arc<AtomicBool>,
|
||||||
|
}
|
||||||
|
struct OwnedBlockedJournalRequest {
|
||||||
|
// A local request copy, including the saved input, must be dropped before erasure
|
||||||
|
// can acquire the lease. This does not claim remote/provider bytes are retractable.
|
||||||
|
_request: agen::llm_client::Request,
|
||||||
|
model: BlockedJournalRequest,
|
||||||
|
}
|
||||||
|
impl Drop for OwnedBlockedJournalRequest {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.model.request_dropped.store(true, Ordering::Release);
|
||||||
|
self.model.dropped.notify_one();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl agen::llm_client::LlmClient for BlockedJournalRequest {
|
||||||
|
async fn stream(
|
||||||
|
&self,
|
||||||
|
request: agen::llm_client::Request,
|
||||||
|
) -> Result<
|
||||||
|
std::pin::Pin<
|
||||||
|
Box<
|
||||||
|
dyn futures::Stream<
|
||||||
|
Item = Result<
|
||||||
|
agen::llm_client::event::Event,
|
||||||
|
agen::llm_client::error::ClientError,
|
||||||
|
>,
|
||||||
|
> + Send,
|
||||||
|
>,
|
||||||
|
>,
|
||||||
|
agen::llm_client::error::ClientError,
|
||||||
|
> {
|
||||||
|
let owned = OwnedBlockedJournalRequest {
|
||||||
|
_request: request,
|
||||||
|
model: self.clone(),
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
owned
|
||||||
|
._request
|
||||||
|
.items
|
||||||
|
.iter()
|
||||||
|
.filter_map(agen::llm_client::types::Item::as_text)
|
||||||
|
.any(|text| text.contains("local blocked source body"))
|
||||||
|
);
|
||||||
|
self.entered.notify_one();
|
||||||
|
std::future::pending::<()>().await;
|
||||||
|
drop(owned);
|
||||||
|
unreachable!()
|
||||||
|
}
|
||||||
|
fn clone_boxed(&self) -> Box<dyn agen::llm_client::LlmClient> {
|
||||||
|
Box::new(self.clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Real generic Jobs and a retained public handle, with only the provider transport blocked.
|
||||||
|
/// Connection close must initiate cancellation and preserve the exclusive lease until local
|
||||||
|
/// request/history ownership is gone (or return an explicit incomplete barrier).
|
||||||
|
#[tokio::test]
|
||||||
|
async fn close_cancels_retained_local_job_before_erase_can_acquire_exclusive_lease() {
|
||||||
|
let mut fixture = Fixture::new("local blocked source body");
|
||||||
|
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||||
|
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||||
|
let retained = jobs.clone();
|
||||||
|
let model = BlockedJournalRequest::default();
|
||||||
|
retained
|
||||||
|
.start_with_model_client(&id, model.clone())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let state = fixture.host().catalog.state_root.clone();
|
||||||
|
let subject = fixture.connection.binding.subject_id.clone();
|
||||||
|
let close_error = fixture.connection.close().unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
close_error,
|
||||||
|
SubjectError::JobShutdown(crate::jobs::JobServiceError::ShutdownIncomplete)
|
||||||
|
));
|
||||||
|
assert!(fixture.connection.lease.is_some());
|
||||||
|
assert!(fixture.host().jobs.lock().unwrap().is_some());
|
||||||
|
assert!(matches!(
|
||||||
|
retained.start(&id).await,
|
||||||
|
Err(crate::jobs::JobServiceError::Closed)
|
||||||
|
));
|
||||||
|
let locally_dropped_before_erase = model.request_dropped.load(Ordering::Acquire);
|
||||||
|
let early_erasure = StandaloneSubjects::open_existing(&state)
|
||||||
|
.unwrap()
|
||||||
|
.erase_session_completely(
|
||||||
|
&subject,
|
||||||
|
&fixture.context.session_id,
|
||||||
|
"erase-blocked-local-run",
|
||||||
|
);
|
||||||
|
let early_lease_was_retained = matches!(early_erasure, Err(SubjectError::Active(_)));
|
||||||
|
let teardown_from_close = model.request_dropped.load(Ordering::Acquire)
|
||||||
|
|| tokio::time::timeout(std::time::Duration::from_secs(5), model.dropped.notified())
|
||||||
|
.await
|
||||||
|
.is_ok();
|
||||||
|
// Always tear down before asserting the observed violation, so a failed regression
|
||||||
|
// cannot leak the blocked model future into another test.
|
||||||
|
let _ = retained.shutdown().await;
|
||||||
|
assert!(model.request_dropped.load(Ordering::Acquire));
|
||||||
|
fixture.connection.close().unwrap();
|
||||||
|
assert!(fixture.connection.lease.is_none());
|
||||||
|
assert!(fixture.host().jobs.lock().unwrap().is_none());
|
||||||
|
// Cancellation is not completion/erasure. Reopening retains the same unknown
|
||||||
|
// attempt's immutable body until the explicit erasure transaction retires it.
|
||||||
|
let reopened = crate::jobs::StandaloneJobs::open(
|
||||||
|
&fixture.connection.jobs_path,
|
||||||
|
fixture._temp.path().into(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let unknown = reopened.get(&id).unwrap();
|
||||||
|
assert_eq!(unknown.attempt.state, crate::jobs::JobAttemptState::Unknown);
|
||||||
|
assert_eq!(unknown.input_disposition, JobInputDisposition::Available);
|
||||||
|
assert!(
|
||||||
|
unknown
|
||||||
|
.request
|
||||||
|
.input
|
||||||
|
.to_string()
|
||||||
|
.contains("local blocked source body")
|
||||||
|
);
|
||||||
|
reopened.shutdown().await.unwrap();
|
||||||
|
StandaloneSubjects::open_existing(&state)
|
||||||
|
.unwrap()
|
||||||
|
.erase_session_completely(
|
||||||
|
&subject,
|
||||||
|
&fixture.context.session_id,
|
||||||
|
"erase-blocked-local-run",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let erased_store = crate::jobs::StandaloneJobs::open(
|
||||||
|
&fixture.connection.jobs_path,
|
||||||
|
fixture._temp.path().into(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let erased = erased_store.get(&id).unwrap();
|
||||||
|
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
|
||||||
|
fixture.assert_job_storage_excludes(&erased, "local blocked source body");
|
||||||
|
erased_store.shutdown().await.unwrap();
|
||||||
|
assert!(
|
||||||
|
locally_dropped_before_erase || early_lease_was_retained,
|
||||||
|
"close released exclusive lease while a retained generic Job still owned local input/history"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
teardown_from_close,
|
||||||
|
"connection close did not initiate cancellation of the externally retained local Job"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
retained.get(&id).is_err(),
|
||||||
|
"retained handles are closed after teardown"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -127,7 +127,6 @@ async fn local_host_compaction_freezes_pending_journal_through_request_and_durab
|
|||||||
worker::WorkerWorkspaceContext::no_workspace(),
|
worker::WorkerWorkspaceContext::no_workspace(),
|
||||||
worker::WorkerFilesystemAuthority::None,
|
worker::WorkerFilesystemAuthority::None,
|
||||||
worker::WorkerBootstrapLayout::Direct { runtime_base: fixture._temp.path().join("runtime") },
|
worker::WorkerBootstrapLayout::Direct { runtime_base: fixture._temp.path().join("runtime") },
|
||||||
worker::WorkerControllerTransport::InProcess,
|
|
||||||
).with_model_client(model.clone()).with_subjektiv_host(fixture.host().clone()).prepare_restored("journal-test").await.unwrap();
|
).with_model_client(model.clone()).with_subjektiv_host(fixture.host().clone()).prepare_restored("journal-test").await.unwrap();
|
||||||
assert_eq!(prepared.worker().session_id().to_string(), fixture.context.session_id);
|
assert_eq!(prepared.worker().session_id().to_string(), fixture.context.session_id);
|
||||||
let source_segment = prepared.worker().segment_id();
|
let source_segment = prepared.worker().segment_id();
|
||||||
|
|||||||
@@ -0,0 +1,296 @@
|
|||||||
|
//! Optional operator transport owned by the Standalone Host, never by a Worker.
|
||||||
|
//!
|
||||||
|
//! The endpoint's existing parent must be private to the current Unix user. Binding never
|
||||||
|
//! removes an existing entry, including stale sockets. Recovery is an explicit operator action.
|
||||||
|
//! Peers must have that same uid; admitted methods still cross the normal local protocol
|
||||||
|
//! authorization boundary. Disconnect only drops a view, not the Host or its execution lease.
|
||||||
|
|
||||||
|
use std::io;
|
||||||
|
use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use client::transport::Socket;
|
||||||
|
use protocol::stream::{JsonLineWriter, decode_event, decode_method};
|
||||||
|
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||||
|
use tokio::net::{UnixListener, UnixStream};
|
||||||
|
use tokio::sync::oneshot;
|
||||||
|
use tokio::task::{JoinHandle, JoinSet};
|
||||||
|
|
||||||
|
/// A bound endpoint reservation. Drop removes only the exact socket created by this bind.
|
||||||
|
/// It can be reserved before starting a Host, so bind failures do not allocate Workers.
|
||||||
|
pub struct UnixEndpoint {
|
||||||
|
listener: UnixListener,
|
||||||
|
path: PathBuf,
|
||||||
|
device: u64,
|
||||||
|
inode: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl UnixEndpoint {
|
||||||
|
pub fn bind(path: impl AsRef<Path>) -> io::Result<Self> {
|
||||||
|
let path = path.as_ref();
|
||||||
|
let name = path.file_name().ok_or_else(|| {
|
||||||
|
io::Error::new(io::ErrorKind::InvalidInput, "endpoint requires a filename")
|
||||||
|
})?;
|
||||||
|
let parent = std::fs::canonicalize(
|
||||||
|
path.parent()
|
||||||
|
.filter(|p| !p.as_os_str().is_empty())
|
||||||
|
.unwrap_or(Path::new(".")),
|
||||||
|
)?;
|
||||||
|
let metadata = std::fs::metadata(&parent)?;
|
||||||
|
// SAFETY: geteuid has no preconditions and does not mutate process identity.
|
||||||
|
let uid = unsafe { libc::geteuid() };
|
||||||
|
if !metadata.is_dir() || metadata.uid() != uid || metadata.mode() & 0o077 != 0 {
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::PermissionDenied,
|
||||||
|
"endpoint parent must be a current-user-owned private directory (mode 0700)",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let path = parent.join(name);
|
||||||
|
let listener = UnixListener::bind(&path)?;
|
||||||
|
let metadata = std::fs::symlink_metadata(&path)?;
|
||||||
|
let endpoint = Self {
|
||||||
|
listener,
|
||||||
|
path,
|
||||||
|
device: metadata.dev(),
|
||||||
|
inode: metadata.ino(),
|
||||||
|
};
|
||||||
|
std::fs::set_permissions(&endpoint.path, std::fs::Permissions::from_mode(0o600))?;
|
||||||
|
Ok(endpoint)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn path(&self) -> &Path {
|
||||||
|
&self.path
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for UnixEndpoint {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if let Ok(current) = std::fs::symlink_metadata(&self.path)
|
||||||
|
&& current.file_type().is_socket()
|
||||||
|
&& current.dev() == self.device
|
||||||
|
&& current.ino() == self.inode
|
||||||
|
{
|
||||||
|
let _ = std::fs::remove_file(&self.path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct UnixAdapter {
|
||||||
|
task: Option<JoinHandle<io::Result<()>>>,
|
||||||
|
stop: Option<oneshot::Sender<()>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl UnixAdapter {
|
||||||
|
pub(crate) fn start(endpoint: UnixEndpoint, handle: worker::WorkerHandle) -> Self {
|
||||||
|
let (stop, mut stopped) = oneshot::channel();
|
||||||
|
let task = tokio::spawn(async move {
|
||||||
|
let mut connections = JoinSet::new();
|
||||||
|
let result = loop {
|
||||||
|
tokio::select! {
|
||||||
|
_ = &mut stopped => break Ok(()),
|
||||||
|
accepted = endpoint.listener.accept(), if connections.len() < 64 => {
|
||||||
|
match accepted {
|
||||||
|
Ok((stream, _)) => {
|
||||||
|
// The filesystem gate protects connection establishment; credentials
|
||||||
|
// also fence open connections to the Host's operator identity.
|
||||||
|
let allowed = stream.peer_cred().is_ok_and(|cred| {
|
||||||
|
// SAFETY: geteuid has no preconditions.
|
||||||
|
cred.uid() == unsafe { libc::geteuid() }
|
||||||
|
});
|
||||||
|
if allowed {
|
||||||
|
let socket = crate::host::connect_protocol(handle.clone());
|
||||||
|
connections.spawn(bridge(stream, socket));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => break Err(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = connections.join_next(), if !connections.is_empty() => {}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
connections.shutdown().await;
|
||||||
|
// Keep the bound endpoint alive until every connection has been reclaimed.
|
||||||
|
drop(endpoint);
|
||||||
|
result
|
||||||
|
});
|
||||||
|
Self {
|
||||||
|
task: Some(task),
|
||||||
|
stop: Some(stop),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn ended(&mut self) -> io::Result<()> {
|
||||||
|
let result = self.task.as_mut().expect("adapter task").await;
|
||||||
|
self.task.take();
|
||||||
|
result.map_err(io::Error::other)?
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn shutdown(mut self) -> io::Result<()> {
|
||||||
|
if let Some(stop) = self.stop.take() {
|
||||||
|
let _ = stop.send(());
|
||||||
|
}
|
||||||
|
if self.task.is_some() {
|
||||||
|
self.ended().await?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for UnixAdapter {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if let Some(stop) = self.stop.take() {
|
||||||
|
let _ = stop.send(());
|
||||||
|
}
|
||||||
|
// Sending stop lets the task join its connection set and drop its endpoint.
|
||||||
|
// A Host dropped without confirmed shutdown keeps the production execution lease.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn bridge(
|
||||||
|
stream: UnixStream,
|
||||||
|
mut socket: client::transport::in_process::Socket,
|
||||||
|
) -> io::Result<()> {
|
||||||
|
let (reader, writer) = stream.into_split();
|
||||||
|
let mut reader = BufReader::new(reader).lines();
|
||||||
|
let mut writer = JsonLineWriter::new(writer);
|
||||||
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
line = reader.next_line() => {
|
||||||
|
let Some(message) = line? else { return Ok(()); };
|
||||||
|
if message.trim().is_empty() { continue; }
|
||||||
|
socket.send(message).await.map_err(io::Error::other)?;
|
||||||
|
}
|
||||||
|
event = socket.next() => {
|
||||||
|
let Some(message) = event.map_err(io::Error::other)? else { return Ok(()); };
|
||||||
|
let event = decode_event(&message).map_err(io::Error::other)?;
|
||||||
|
writer.write_event(&event).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Connect a JSONL operator client. EOF disconnects without issuing Shutdown.
|
||||||
|
/// Input is decoded as Worker Methods and output contains Worker Events only.
|
||||||
|
pub async fn run_unix_client(endpoint: PathBuf) -> io::Result<()> {
|
||||||
|
relay_unix_client(endpoint, tokio::io::stdin(), tokio::io::stdout()).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The CLI's stream relay, also usable by embedders that own their input/output streams.
|
||||||
|
pub async fn relay_unix_client<R, W>(endpoint: PathBuf, input: R, output: W) -> io::Result<()>
|
||||||
|
where
|
||||||
|
R: tokio::io::AsyncRead + Unpin,
|
||||||
|
W: tokio::io::AsyncWrite + Unpin,
|
||||||
|
{
|
||||||
|
let stream = UnixStream::connect(endpoint).await?;
|
||||||
|
let (reader, writer) = stream.into_split();
|
||||||
|
let mut input = BufReader::new(input).lines();
|
||||||
|
let mut to_host = JsonLineWriter::new(writer);
|
||||||
|
let mut from_host = BufReader::new(reader).lines();
|
||||||
|
let mut output = JsonLineWriter::new(output);
|
||||||
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
line = input.next_line() => {
|
||||||
|
let Some(line) = line? else { return Ok(()); };
|
||||||
|
if line.trim().is_empty() { continue; }
|
||||||
|
let method = decode_method(&line).map_err(io::Error::other)?;
|
||||||
|
to_host.write(&method).await?;
|
||||||
|
}
|
||||||
|
line = from_host.next_line() => {
|
||||||
|
let Some(line) = line? else { return Ok(()); };
|
||||||
|
let event = decode_event(&line).map_err(io::Error::other)?;
|
||||||
|
output.write_event(&event).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn endpoint_requires_private_parent_and_is_owner_only() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let path = directory.path().join("host.sock");
|
||||||
|
std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o755)).unwrap();
|
||||||
|
let error = UnixEndpoint::bind(&path)
|
||||||
|
.err()
|
||||||
|
.expect("public parent rejected");
|
||||||
|
assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
|
||||||
|
assert!(!path.exists());
|
||||||
|
std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
let endpoint = UnixEndpoint::bind(&path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::metadata(&path).unwrap().permissions().mode() & 0o777,
|
||||||
|
0o600
|
||||||
|
);
|
||||||
|
let client = UnixStream::connect(&path).await.unwrap();
|
||||||
|
let (server, _) = endpoint.listener.accept().await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
client.peer_cred().unwrap().uid(),
|
||||||
|
server.peer_cred().unwrap().uid()
|
||||||
|
);
|
||||||
|
drop(endpoint);
|
||||||
|
assert!(!path.exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bind_failure_precedes_worker_state_and_startup_failure_releases_only_its_endpoint() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
let path = directory.path().join("host.sock");
|
||||||
|
let owner = UnixEndpoint::bind(&path).unwrap();
|
||||||
|
let state_dir = directory.path().join("must-not-be-opened");
|
||||||
|
let error = crate::run_headless(
|
||||||
|
crate::HeadlessLaunch::Restore {
|
||||||
|
state_dir: state_dir.clone(),
|
||||||
|
worker_id: protocol::WorkerId::now_v7(),
|
||||||
|
},
|
||||||
|
path.clone(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(error, crate::HeadlessError::Transport(_)));
|
||||||
|
assert!(!state_dir.exists());
|
||||||
|
assert!(path.exists());
|
||||||
|
drop(owner);
|
||||||
|
let error = crate::run_headless(
|
||||||
|
crate::HeadlessLaunch::Restore {
|
||||||
|
state_dir,
|
||||||
|
worker_id: protocol::WorkerId::now_v7(),
|
||||||
|
},
|
||||||
|
path.clone(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(error, crate::HeadlessError::Startup(_)));
|
||||||
|
assert!(
|
||||||
|
!path.exists(),
|
||||||
|
"a failed Host startup must release its own bind"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bind_failure_and_old_endpoint_drop_preserve_another_owner() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
let path = directory.path().join("host.sock");
|
||||||
|
let first = UnixEndpoint::bind(&path).unwrap();
|
||||||
|
let first_inode = std::fs::metadata(&path).unwrap().ino();
|
||||||
|
assert_eq!(
|
||||||
|
UnixEndpoint::bind(&path).err().unwrap().kind(),
|
||||||
|
io::ErrorKind::AddrInUse
|
||||||
|
);
|
||||||
|
assert_eq!(std::fs::metadata(&path).unwrap().ino(), first_inode);
|
||||||
|
std::fs::rename(&path, directory.path().join("retained.sock")).unwrap();
|
||||||
|
let second = UnixEndpoint::bind(&path).unwrap();
|
||||||
|
let second_inode = std::fs::metadata(&path).unwrap().ino();
|
||||||
|
assert_ne!(first_inode, second_inode);
|
||||||
|
drop(first);
|
||||||
|
assert_eq!(std::fs::metadata(&path).unwrap().ino(), second_inode);
|
||||||
|
let _client = UnixStream::connect(&path).await.unwrap();
|
||||||
|
let _connection = second.listener.accept().await.unwrap();
|
||||||
|
drop(second);
|
||||||
|
assert!(!path.exists());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -608,6 +608,239 @@ async fn standalone_metadata_fails_closed_on_incomplete_or_newer_records() -> Te
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn headless_unix_client_uses_production_history_lease_and_local_authority() -> TestResult {
|
||||||
|
use protocol::stream::{JsonLineReader, encode_method};
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
use tokio::io::AsyncWriteExt;
|
||||||
|
|
||||||
|
let temp = tempfile::tempdir()?;
|
||||||
|
// Shorten only the owned endpoint's components, preserving the harness TMPDIR
|
||||||
|
// even when it is checkout-local. No process-global cwd or external /tmp fallback.
|
||||||
|
let socket_dir = tempfile::Builder::new()
|
||||||
|
.prefix("")
|
||||||
|
.rand_bytes(4)
|
||||||
|
.tempdir()?;
|
||||||
|
std::fs::set_permissions(socket_dir.path(), std::fs::Permissions::from_mode(0o700))?;
|
||||||
|
let endpoint_path = socket_dir.path().join("s");
|
||||||
|
let endpoint = standalone::UnixEndpoint::bind(&endpoint_path)?;
|
||||||
|
let state_dir = temp.path().join("state");
|
||||||
|
let launch = StandaloneLaunchConfig::new(
|
||||||
|
temp.path(),
|
||||||
|
&state_dir,
|
||||||
|
manifest::ProfileSelector::Default,
|
||||||
|
"headless-operator",
|
||||||
|
)
|
||||||
|
.resolve()?;
|
||||||
|
let model = ScriptedClient::new(vec![vec![
|
||||||
|
LlmEvent::text_block_start(0),
|
||||||
|
LlmEvent::text_delta(0, "headless production response"),
|
||||||
|
LlmEvent::text_block_stop(0, Some(StopReason::EndTurn)),
|
||||||
|
]]);
|
||||||
|
let inspection = model.clone();
|
||||||
|
let host = StandaloneHost::start_with_model_client(launch, model).await?;
|
||||||
|
let worker_id = host.worker_id();
|
||||||
|
let session_id = host.record().active_session_id;
|
||||||
|
let profile = host.record().manifest.clone();
|
||||||
|
let serve = tokio::spawn(host.serve_unix(endpoint));
|
||||||
|
|
||||||
|
// Exercise the real external client's stdin/stdout relay over the actual Unix adapter.
|
||||||
|
let (mut input, relay_input) = tokio::io::duplex(64 * 1024);
|
||||||
|
let (relay_output, output) = tokio::io::duplex(64 * 1024);
|
||||||
|
let relay = tokio::spawn(standalone::relay_unix_client(
|
||||||
|
endpoint_path.clone(),
|
||||||
|
relay_input,
|
||||||
|
relay_output,
|
||||||
|
));
|
||||||
|
let mut events = JsonLineReader::new(output);
|
||||||
|
let snapshot = tokio::time::timeout(Duration::from_secs(10), events.next::<Event>())
|
||||||
|
.await??
|
||||||
|
.expect("snapshot");
|
||||||
|
assert!(matches!(snapshot, Event::Snapshot { .. }));
|
||||||
|
let request = Method::SubmitTracked {
|
||||||
|
submission_request_id: protocol::new_submission_request_id(),
|
||||||
|
input: vec![protocol::Segment::text("headless production input")],
|
||||||
|
source: protocol::AuthenticatedInputSource::Backend {
|
||||||
|
operation_id: "forged-backend-source".into(),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let wire = encode_method(&request)?;
|
||||||
|
println!("headless input: {wire}");
|
||||||
|
input.write_all(wire.as_bytes()).await?;
|
||||||
|
input.write_all(b"\n").await?;
|
||||||
|
let mut saw_user = false;
|
||||||
|
let mut saw_response = false;
|
||||||
|
let mut saw_state = false;
|
||||||
|
tokio::time::timeout(Duration::from_secs(20), async {
|
||||||
|
loop {
|
||||||
|
let event = events.next::<Event>().await?.expect("live event");
|
||||||
|
match &event {
|
||||||
|
Event::UserMessage { segments, .. } => {
|
||||||
|
saw_user |= format!("{segments:?}").contains("headless production input");
|
||||||
|
}
|
||||||
|
Event::TextDelta { text } => {
|
||||||
|
saw_response |= text == "headless production response";
|
||||||
|
}
|
||||||
|
Event::WorkerState { .. } => saw_state = true,
|
||||||
|
Event::RunEnd { .. } => break,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
println!(
|
||||||
|
"headless output: {}",
|
||||||
|
protocol::stream::encode_event(&event)?
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok::<_, Box<dyn std::error::Error>>(())
|
||||||
|
})
|
||||||
|
.await??;
|
||||||
|
assert!(saw_user && saw_response && saw_state);
|
||||||
|
assert_eq!(inspection.requests().len(), 1);
|
||||||
|
assert!(matches!(
|
||||||
|
StandaloneHost::restore_with_model_client(
|
||||||
|
state_dir.clone(),
|
||||||
|
worker_id,
|
||||||
|
ScriptedClient::new(vec![])
|
||||||
|
)
|
||||||
|
.await,
|
||||||
|
Err(StandaloneStartupError::WorkerActive)
|
||||||
|
));
|
||||||
|
|
||||||
|
// EOF is a disconnect, not a shutdown; another view can reconnect to the same history.
|
||||||
|
drop(input);
|
||||||
|
tokio::time::timeout(Duration::from_secs(10), relay).await???;
|
||||||
|
assert!(!serve.is_finished());
|
||||||
|
let mut malformed = tokio::net::UnixStream::connect(&endpoint_path).await?;
|
||||||
|
malformed.write_all(b"not-json\n").await?;
|
||||||
|
let mut discarded = Vec::new();
|
||||||
|
use tokio::io::AsyncReadExt;
|
||||||
|
tokio::time::timeout(
|
||||||
|
Duration::from_secs(10),
|
||||||
|
malformed.read_to_end(&mut discarded),
|
||||||
|
)
|
||||||
|
.await??;
|
||||||
|
assert!(
|
||||||
|
!serve.is_finished(),
|
||||||
|
"invalid client input must not stop the Host"
|
||||||
|
);
|
||||||
|
let mut socket_client =
|
||||||
|
Client::new(client::transport::unix_socket::Socket::connect(&endpoint_path).await?);
|
||||||
|
let snapshot = tokio::time::timeout(Duration::from_secs(10), async {
|
||||||
|
loop {
|
||||||
|
let event = socket_client.next_event().await?.expect("reconnect event");
|
||||||
|
if matches!(event, Event::Snapshot { .. }) {
|
||||||
|
break Ok::<_, Box<dyn std::error::Error>>(event);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await??;
|
||||||
|
let snapshot_text = format!("{snapshot:?}");
|
||||||
|
assert!(
|
||||||
|
snapshot_text.contains("headless production input"),
|
||||||
|
"{snapshot_text}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
snapshot_text.contains("headless production response"),
|
||||||
|
"{snapshot_text}"
|
||||||
|
);
|
||||||
|
socket_client
|
||||||
|
.send(&Method::Shutdown {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(0),
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
tokio::time::timeout(Duration::from_secs(10), async {
|
||||||
|
loop {
|
||||||
|
if let Some(Event::CommandAcknowledged { acknowledgement }) =
|
||||||
|
socket_client.next_event().await?
|
||||||
|
{
|
||||||
|
assert_eq!(
|
||||||
|
acknowledgement.disposition,
|
||||||
|
protocol::WorkerCommandDisposition::StaleCommandId
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok::<_, Box<dyn std::error::Error>>(())
|
||||||
|
})
|
||||||
|
.await??;
|
||||||
|
assert!(
|
||||||
|
!serve.is_finished(),
|
||||||
|
"stale Shutdown must not bypass Controller admission"
|
||||||
|
);
|
||||||
|
socket_client
|
||||||
|
.send(&Method::Shutdown {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(100),
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
tokio::time::timeout(Duration::from_secs(15), serve).await???;
|
||||||
|
assert!(!endpoint_path.exists());
|
||||||
|
let store = StandaloneWorkerStore::open(&state_dir)?;
|
||||||
|
let saved = store.load(worker_id)?;
|
||||||
|
assert_eq!(saved.status, StandaloneWorkerStatus::Stopped);
|
||||||
|
use session_store::Store;
|
||||||
|
let history = session_store::FsStore::new(store.sessions_dir(worker_id))?.read_all(
|
||||||
|
saved.active_session_id,
|
||||||
|
saved.active_segment_id.expect("active segment"),
|
||||||
|
)?;
|
||||||
|
let origins = history
|
||||||
|
.iter()
|
||||||
|
.filter_map(|entry| match entry {
|
||||||
|
session_store::LogEntry::AnnotatedUserInput { history, .. } => Some(history),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.flatten()
|
||||||
|
.map(|entry| &entry.metadata.origin)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert!(origins.contains(&&session_store::LoggedSessionHistoryOrigin::LocalHumanInput));
|
||||||
|
assert!(!format!("{origins:?}").contains("forged-backend-source"));
|
||||||
|
|
||||||
|
// This is the same restoration path called by the TUI, not a headless catalog/import.
|
||||||
|
let restored = StandaloneHost::restore_with_model_client(
|
||||||
|
state_dir.clone(),
|
||||||
|
worker_id,
|
||||||
|
ScriptedClient::new(vec![]),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
assert_eq!(restored.worker_id(), worker_id);
|
||||||
|
assert_eq!(restored.record().active_session_id, session_id);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&restored.record().manifest)?,
|
||||||
|
serde_json::to_value(&profile)?
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
restored.record().cwd.canonical_path,
|
||||||
|
temp.path().canonicalize()?
|
||||||
|
);
|
||||||
|
let mut in_process = restored.connect();
|
||||||
|
let snapshot = tokio::time::timeout(Duration::from_secs(10), async {
|
||||||
|
loop {
|
||||||
|
let event = in_process
|
||||||
|
.next_event()
|
||||||
|
.await?
|
||||||
|
.expect("in-process restored event");
|
||||||
|
if matches!(event, Event::Snapshot { .. }) {
|
||||||
|
break Ok::<_, Box<dyn std::error::Error>>(event);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await??;
|
||||||
|
assert!(format!("{snapshot:?}").contains("headless production response"));
|
||||||
|
assert!(
|
||||||
|
!endpoint_path.exists(),
|
||||||
|
"in-process restore must not recreate the adapter"
|
||||||
|
);
|
||||||
|
let runtime = state_dir
|
||||||
|
.join(worker_id.to_string())
|
||||||
|
.join("runtime")
|
||||||
|
.join(format!("standalone-{worker_id}"));
|
||||||
|
assert!(!runtime.join("sock").exists());
|
||||||
|
assert!(!runtime.join("pid").exists());
|
||||||
|
restored.shutdown().await?;
|
||||||
|
println!(
|
||||||
|
"verdict: input/state/response/disconnect/explicit shutdown, shared production restore and concurrent lease exclusion passed for {worker_id}"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn wait_for_run_end(client: &mut Client<InProcessSocket>) -> TestResult {
|
async fn wait_for_run_end(client: &mut Client<InProcessSocket>) -> TestResult {
|
||||||
tokio::time::timeout(Duration::from_secs(10), async {
|
tokio::time::timeout(Duration::from_secs(10), async {
|
||||||
loop {
|
loop {
|
||||||
|
|||||||
@@ -479,38 +479,28 @@ fn journal_response(request: &Request) -> Vec<LlmEvent> {
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
for (index, range) in state["selected_ranges"]
|
let input: Value = request
|
||||||
.as_array()
|
.items
|
||||||
.unwrap()
|
|
||||||
.iter()
|
.iter()
|
||||||
.enumerate()
|
.filter_map(Item::as_text)
|
||||||
{
|
.find_map(|text| {
|
||||||
let mut offset = range["bytes"]["start"].as_u64().unwrap();
|
text.lines()
|
||||||
let mut token = Value::Null;
|
.find_map(|line| line.strip_prefix("input_json: "))
|
||||||
loop {
|
})
|
||||||
let id = format!("journal-source-{index}-{offset}");
|
.map(|json| serde_json::from_str(json).unwrap())
|
||||||
match result(&id) {
|
.expect("saved direct Journal input");
|
||||||
Some(page) => match page["value"]["next_byte"].as_u64() {
|
let input = subjektiv::journal::recording::JournalInput::decode(&input).unwrap();
|
||||||
Some(next) => {
|
assert_eq!(
|
||||||
assert!(next > offset);
|
serde_json::to_value(&input.selected_ranges).unwrap(),
|
||||||
offset = next;
|
state["selected_ranges"]
|
||||||
token = page["value"]["content_token"].clone();
|
);
|
||||||
}
|
assert!(
|
||||||
None => break,
|
input
|
||||||
},
|
.capture
|
||||||
None => {
|
.entries
|
||||||
return tool(
|
.iter()
|
||||||
&id,
|
.all(|entry| input.capture.recording_text(entry.order).is_ok())
|
||||||
"ReadJournalText",
|
);
|
||||||
json!({
|
|
||||||
"target":{"target":"source","pointer":range["source"]["pointer"]},
|
|
||||||
"offset":offset,"limit":subjektiv::journal::MAX_JOURNAL_READ_BYTES,"content_token":token
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let (name, action) = if state["current_open"].is_null() {
|
let (name, action) = if state["current_open"].is_null() {
|
||||||
(
|
(
|
||||||
"CreateJournalEntry",
|
"CreateJournalEntry",
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ serde = { workspace = true, features = ["derive"] }
|
|||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
sha2.workspace = true
|
sha2.workspace = true
|
||||||
server-api.workspace = true
|
server-api.workspace = true
|
||||||
|
session-store.workspace = true
|
||||||
|
job.workspace = true
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
uuid = { workspace = true, features = ["v7"] }
|
uuid = { workspace = true, features = ["v7"] }
|
||||||
|
|
||||||
|
|||||||
@@ -194,6 +194,29 @@ impl SubjektivStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// End a body after the Host has confirmed its Runtime removal. This is not a
|
||||||
|
/// completed Journal handoff: source coverage, positions and saved receipts stay
|
||||||
|
/// unchanged. Revocation and closing generations are atomic, so no retained Job
|
||||||
|
/// can write after deletion even while Backend Job cleanup is being retried.
|
||||||
|
pub fn release_removed_journal_body(&self, host: &JournalHostContext) -> JournalResult<()> {
|
||||||
|
self.database.try_transaction(|tx| {
|
||||||
|
super::store::check_session_host(self, tx, host)?;
|
||||||
|
let attr = host.attribution.as_ref().expect("checked Session authority");
|
||||||
|
let binding: Option<String> = tx.query_row(
|
||||||
|
"SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 ORDER BY rowid DESC LIMIT 1",
|
||||||
|
params![host.subject_id, attr.runtime_id, attr.worker_id], |r| r.get(0),
|
||||||
|
).optional()?;
|
||||||
|
let binding = binding.unwrap_or_else(|| crate::issued_id("journal-removed-body"));
|
||||||
|
tx.execute("INSERT OR IGNORE INTO journal_writer_bindings(binding_id,subject_id,runtime_id,worker_id,active) VALUES(?1,?2,?3,?4,0)",params![binding,host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||||
|
tx.execute("UPDATE journal_writer_bindings SET active=0 WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||||
|
// Bind legacy attributed Sessions to an inactive lifetime, never to the
|
||||||
|
// successor. Retained unfinished history is not an active writer.
|
||||||
|
tx.execute("INSERT OR IGNORE INTO journal_session_bindings(session_id,binding_id) SELECT session_id,?4 FROM subject_session_attributions WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id,binding])?;
|
||||||
|
tx.execute("UPDATE journal_generations SET active=0 WHERE journal_id IN (SELECT j.id FROM journals j JOIN subject_session_attributions a ON a.session_id=j.source_session_id WHERE a.subject_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3)",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original
|
/// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original
|
||||||
/// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs.
|
/// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs.
|
||||||
/// This transaction rejects retained writers, pending source bytes and active generations.
|
/// This transaction rejects retained writers, pending source bytes and active generations.
|
||||||
|
|||||||
@@ -5,8 +5,221 @@
|
|||||||
//! original canonical identity while streaming and return only the requested bounded fragment.
|
//! original canonical identity while streaming and return only the requested bounded fragment.
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use session_store::capture::{CaptureDto, CaptureLimits};
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
/// A Job owns only this bounded original-body prefix. The initial content condition binds
|
||||||
|
/// the separate generation snapshot (whose potentially large Journal text remains paged).
|
||||||
|
/// Neither possessing these bytes nor delivering them is a processing checkpoint.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct JournalInput {
|
||||||
|
pub version: u32,
|
||||||
|
pub subject_id: String,
|
||||||
|
pub journal_id: String,
|
||||||
|
pub initial_state_token: String,
|
||||||
|
pub initial_open_id: Option<String>,
|
||||||
|
pub previous_fixed_end: Option<JournalSourcePointer>,
|
||||||
|
pub selected_ranges: Vec<JournalSourceRange>,
|
||||||
|
pub capture: CaptureDto,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl JournalInput {
|
||||||
|
/// Capture callback runs once for this immutable Job, never once per page. Fitting uses
|
||||||
|
/// only that captured value and measures the complete serialized input, including the
|
||||||
|
/// initial Journal condition and ranges duplicated beside the capture's descriptors.
|
||||||
|
pub fn build(
|
||||||
|
store: &SubjektivStore,
|
||||||
|
host: &JournalHostContext,
|
||||||
|
journal_id: &str,
|
||||||
|
capture_once: impl FnOnce(u64, u64, usize) -> JournalResult<CaptureDto>,
|
||||||
|
) -> JournalResult<Self> {
|
||||||
|
let (journal, open, previous_fixed_end) = store.database.try_with_connection(|tx| {
|
||||||
|
super::store::check_host(store, tx, host, true)?;
|
||||||
|
Ok::<_, JournalError>((
|
||||||
|
super::store::journal(tx, host, journal_id, true)?,
|
||||||
|
super::store::open(tx, journal_id)?,
|
||||||
|
super::store::previous_fixed(tx, journal_id)?.map(|p| p.0),
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
let mut input = Self {
|
||||||
|
version: 1,
|
||||||
|
subject_id: journal.subject_id.clone(),
|
||||||
|
journal_id: journal_id.into(),
|
||||||
|
initial_state_token: journal.state_token,
|
||||||
|
initial_open_id: open.map(|e| e.id),
|
||||||
|
previous_fixed_end,
|
||||||
|
selected_ranges: Vec::new(),
|
||||||
|
capture: CaptureDto {
|
||||||
|
version: 1,
|
||||||
|
session_id: journal.source_session_id,
|
||||||
|
segment_id: "pending".into(),
|
||||||
|
view_entry_count: 0,
|
||||||
|
entries: Vec::new(),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let envelope = serde_json::to_vec(&input)?.len();
|
||||||
|
// Capture once at a fixed storage boundary. Any adjustment for the Journal envelope
|
||||||
|
// below operates on this value only, never by making another live capture request.
|
||||||
|
let budget = job::MAX_JOB_INPUT_BYTES.saturating_sub(envelope);
|
||||||
|
input.capture = capture_once(journal.process_order, journal.process_byte, budget)?;
|
||||||
|
input.selected_ranges = capture_ranges(&input.capture)?;
|
||||||
|
if input.selected_ranges.is_empty() {
|
||||||
|
return Ok(input);
|
||||||
|
}
|
||||||
|
input.fit()?;
|
||||||
|
let value = serde_json::to_value(&input)?;
|
||||||
|
Self::decode(&value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fit(&mut self) -> JournalResult<()> {
|
||||||
|
let original = self.capture.clone();
|
||||||
|
let first = self
|
||||||
|
.selected_ranges
|
||||||
|
.first()
|
||||||
|
.ok_or_else(|| invalid("empty recording selection"))?;
|
||||||
|
let start = session_store::capture::RecordingPosition {
|
||||||
|
canonical_order: first.source.order,
|
||||||
|
byte_offset: first.bytes.start,
|
||||||
|
};
|
||||||
|
let mut budget = serde_json::to_vec(&original)?.len();
|
||||||
|
loop {
|
||||||
|
let bytes = serde_json::to_vec(self)?.len();
|
||||||
|
if bytes <= job::MAX_JOB_INPUT_BYTES {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
budget = budget
|
||||||
|
.checked_sub(bytes - job::MAX_JOB_INPUT_BYTES)
|
||||||
|
.ok_or_else(|| invalid("Journal envelope cannot fit source bytes"))?;
|
||||||
|
self.capture = original
|
||||||
|
.recording_slice(start, MAX_JOURNAL_RANGES, budget)
|
||||||
|
.map_err(|e| invalid(&e.to_string()))?
|
||||||
|
.capture;
|
||||||
|
self.selected_ranges = capture_ranges(&self.capture)?;
|
||||||
|
if self.selected_ranges.is_empty() {
|
||||||
|
return Err(invalid("Journal envelope cannot fit one source scalar"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn decode(value: &serde_json::Value) -> JournalResult<Self> {
|
||||||
|
if serde_json::to_vec(value)?.len() > job::MAX_JOB_INPUT_BYTES {
|
||||||
|
return Err(invalid(
|
||||||
|
"Journal input exceeds whole serialized Job input limit",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let input: Self = serde_json::from_value(value.clone())?;
|
||||||
|
input
|
||||||
|
.capture
|
||||||
|
.validate(CaptureLimits {
|
||||||
|
max_bytes: job::MAX_JOB_INPUT_BYTES,
|
||||||
|
max_entries: MAX_JOURNAL_RANGES,
|
||||||
|
max_text_bytes: job::MAX_JOB_INPUT_BYTES,
|
||||||
|
..CaptureLimits::default()
|
||||||
|
})
|
||||||
|
.map_err(|e| invalid(&e.to_string()))?;
|
||||||
|
if input.version != 1
|
||||||
|
|| input.subject_id.is_empty()
|
||||||
|
|| input.journal_id.is_empty()
|
||||||
|
|| input.initial_state_token.is_empty()
|
||||||
|
|| input.selected_ranges.is_empty()
|
||||||
|
|| input.selected_ranges.len() > MAX_JOURNAL_RANGES
|
||||||
|
|| input
|
||||||
|
.selected_ranges
|
||||||
|
.iter()
|
||||||
|
.any(|r| r.source.session_id != input.capture.session_id)
|
||||||
|
{
|
||||||
|
return Err(invalid("invalid Journal input identity or selection"));
|
||||||
|
}
|
||||||
|
if capture_ranges(&input.capture)? != input.selected_ranges {
|
||||||
|
return Err(conflict(
|
||||||
|
"Journal body snapshot does not match selected canonical ranges",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(input)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn validate_generation(&self, generation: &JournalGeneration) -> JournalResult<()> {
|
||||||
|
if generation.journal_id != self.journal_id
|
||||||
|
|| generation.initial_journal.subject_id != self.subject_id
|
||||||
|
|| generation.initial_journal.source_session_id != self.capture.session_id
|
||||||
|
|| generation.initial_journal.state_token != self.initial_state_token
|
||||||
|
|| generation.initial_open.as_ref().map(|e| &e.id) != self.initial_open_id.as_ref()
|
||||||
|
|| generation.previous_fixed_end != self.previous_fixed_end
|
||||||
|
|| generation.selected_ranges != self.selected_ranges
|
||||||
|
{
|
||||||
|
return Err(conflict(
|
||||||
|
"saved Journal input differs from its initial generation snapshot",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Converts the shared recording projection into Journal's coverage language without
|
||||||
|
/// interpreting model text or constructing a second capture representation.
|
||||||
|
pub fn capture_ranges(capture: &CaptureDto) -> JournalResult<Vec<JournalSourceRange>> {
|
||||||
|
use session_store::capture::CaptureBody;
|
||||||
|
let mut ranges = Vec::new();
|
||||||
|
for entry in &capture.entries {
|
||||||
|
let CaptureBody::CanonicalPublic {
|
||||||
|
descriptor,
|
||||||
|
byte_offset,
|
||||||
|
text,
|
||||||
|
..
|
||||||
|
} = &entry.body
|
||||||
|
else {
|
||||||
|
return Err(invalid(
|
||||||
|
"Journal requires canonical original recording capture",
|
||||||
|
));
|
||||||
|
};
|
||||||
|
if !entry.policy.recording_material
|
||||||
|
|| text.is_empty()
|
||||||
|
|| descriptor.identity.segment_id.to_string() != entry.source.segment_id
|
||||||
|
|| descriptor.identity.entry_ref != format!("E{}", entry.entry_id)
|
||||||
|
|| descriptor.canonical_order != entry.order
|
||||||
|
|| byte_offset
|
||||||
|
.checked_add(text.len() as u64)
|
||||||
|
.is_none_or(|end| end > descriptor.total_bytes)
|
||||||
|
{
|
||||||
|
return Err(conflict(
|
||||||
|
"recording body identity, policy or range mismatch",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let source = JournalSource {
|
||||||
|
session_id: capture.session_id.clone(),
|
||||||
|
pointer: JournalSourcePointer {
|
||||||
|
segment_id: descriptor.identity.segment_id,
|
||||||
|
entry_ref: descriptor.identity.entry_ref.clone(),
|
||||||
|
},
|
||||||
|
order: descriptor.canonical_order,
|
||||||
|
content_sha256: descriptor.sha256.clone(),
|
||||||
|
total_bytes: descriptor.total_bytes,
|
||||||
|
};
|
||||||
|
if *byte_offset == 0 && text.len() as u64 == descriptor.total_bytes {
|
||||||
|
verify_canonical_text(&source, text)?;
|
||||||
|
}
|
||||||
|
ranges.push(JournalSourceRange {
|
||||||
|
source,
|
||||||
|
bytes: JournalByteRange {
|
||||||
|
start: *byte_offset,
|
||||||
|
end: *byte_offset + text.len() as u64,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
for pair in ranges.windows(2) {
|
||||||
|
if pair[0].source.order.checked_add(1) != Some(pair[1].source.order)
|
||||||
|
|| pair[0].bytes.end != pair[0].source.total_bytes
|
||||||
|
|| pair[1].bytes.start != 0
|
||||||
|
{
|
||||||
|
return Err(conflict(
|
||||||
|
"recording capture is not an oldest contiguous prefix",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(ranges)
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct JournalMaterial {
|
pub struct JournalMaterial {
|
||||||
|
|||||||
@@ -66,8 +66,8 @@ fn text_page(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Adapter for Host captures bounded by their storage contract. Production source reads should
|
/// Legacy adapter for Host captures bounded by their storage contract.
|
||||||
/// use [`execute_bounded`] to avoid allocating an entire source for one small read.
|
/// Production Hosts use [`attest_input_delivery`] and [`execute_snapshot`] instead.
|
||||||
pub fn execute(
|
pub fn execute(
|
||||||
store: &SubjektivStore,
|
store: &SubjektivStore,
|
||||||
context: &JournalJobContext,
|
context: &JournalJobContext,
|
||||||
@@ -101,23 +101,149 @@ pub fn execute(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reads are delivered through this service before mutation authority is enabled. The durable
|
pub(crate) fn migrate_input_delivery(
|
||||||
/// delivery ledger stores byte ranges only: Jobs may retain immutable descriptors without
|
tx: &rusqlite::Transaction<'_>,
|
||||||
/// accumulating a permanent second copy of the original Session in their request ledgers.
|
) -> feature_storage::Result<()> {
|
||||||
/// The Host verifies original provenance/digest/length and returns a bounded UTF-8 fragment.
|
tx.execute_batch(
|
||||||
|
"CREATE TABLE journal_generation_input_deliveries (
|
||||||
|
generation_id TEXT PRIMARY KEY NOT NULL REFERENCES journal_generations(id) ON DELETE CASCADE,
|
||||||
|
input_digest TEXT NOT NULL CHECK(length(input_digest)=71 AND substr(input_digest,1,7)='sha256:' AND substr(input_digest,8) NOT GLOB '*[^0-9a-f]*')
|
||||||
|
);
|
||||||
|
CREATE TRIGGER journal_generation_input_delivery_no_update
|
||||||
|
BEFORE UPDATE ON journal_generation_input_deliveries
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'Journal input delivery bindings are immutable'); END;",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_input_digest(input_digest: &str) -> JournalResult<()> {
|
||||||
|
// Preserve JobRequest::input_digest() verbatim, including its algorithm prefix.
|
||||||
|
if input_digest.strip_prefix("sha256:").is_none_or(|hex| {
|
||||||
|
hex.len() != 64
|
||||||
|
|| !hex
|
||||||
|
.bytes()
|
||||||
|
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||||
|
}) {
|
||||||
|
return Err(JournalError::InvalidInput(
|
||||||
|
"immutable Job input digest must be sha256: followed by 64 lowercase hex digits".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Trusted Host-only delivery evidence, never a model tool or a claim of understanding.
|
||||||
|
/// Before calling, the Host must validate the saved, immutable body-bearing input against the
|
||||||
|
/// bound Job's input digest, its Session/provenance/ranges, and the actual dispatch plan that
|
||||||
|
/// supplies that exact input. The service cannot verify bytes it does not own. This transaction
|
||||||
|
/// stores only the digest bound to the generation's immutable selection; it neither copies the
|
||||||
|
/// body nor changes records, position, reflection or checkpoint. Exact replay is allowed only
|
||||||
|
/// while the same generation and writer binding remain active; changed digests/ranges conflict.
|
||||||
|
pub fn attest_input_delivery(
|
||||||
|
store: &SubjektivStore,
|
||||||
|
context: &JournalJobContext,
|
||||||
|
input_digest: &str,
|
||||||
|
selected_ranges: &[JournalSourceRange],
|
||||||
|
) -> JournalResult<()> {
|
||||||
|
validate_input_digest(input_digest)?;
|
||||||
|
store.database.try_transaction(|tx| {
|
||||||
|
super::lifecycle::reject_erased(tx, &context.host)?;
|
||||||
|
let generation = super::store::job(store, tx, context, true)?;
|
||||||
|
if generation.selected_ranges != selected_ranges {
|
||||||
|
return Err(JournalError::Conflict(
|
||||||
|
"immutable Job input selection differs from generation".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let saved: Option<String> = tx.query_row(
|
||||||
|
"SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1",
|
||||||
|
[&generation.id], |row| row.get(0),
|
||||||
|
).optional()?;
|
||||||
|
if let Some(saved) = saved {
|
||||||
|
if saved != input_digest {
|
||||||
|
return Err(JournalError::Conflict(
|
||||||
|
"generation is bound to a different immutable Job input digest".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO journal_generation_input_deliveries(generation_id,input_digest) VALUES(?1,?2)",
|
||||||
|
params![generation.id, input_digest],
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_input_delivery(
|
||||||
|
tx: &rusqlite::Connection,
|
||||||
|
context: &JournalJobContext,
|
||||||
|
input_digest: &str,
|
||||||
|
) -> JournalResult<()> {
|
||||||
|
validate_input_digest(input_digest)?;
|
||||||
|
let saved: Option<String> = tx
|
||||||
|
.query_row(
|
||||||
|
"SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1",
|
||||||
|
[&context.generation_id],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.optional()?;
|
||||||
|
if saved.as_deref() != Some(input_digest) {
|
||||||
|
return Err(JournalError::Conflict(
|
||||||
|
"trusted Host delivery of this immutable Job input is required".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Production sequential execution over Journal state and the Host-attested immutable input.
|
||||||
|
/// The digest is taken from the bound saved Job by the Host, not from model arguments. Every
|
||||||
|
/// request requires the matching delivery binding and an active generation/writer. Source text
|
||||||
|
/// is already in immutable input: Source targets fail explicitly, with no Session reader or
|
||||||
|
/// fallback. Receipt-backed record, position and checkpoint constraints remain unchanged.
|
||||||
|
pub fn execute_snapshot(
|
||||||
|
store: &SubjektivStore,
|
||||||
|
context: &JournalJobContext,
|
||||||
|
input_digest: &str,
|
||||||
|
request: JournalJobRequest,
|
||||||
|
) -> JournalResult<JournalJobResponse> {
|
||||||
|
execute_inner(store, context, request, Some(input_digest), |_, _, _| {
|
||||||
|
Err(JournalError::PermissionDenied(
|
||||||
|
"Source reads are unavailable for immutable Journal input".into(),
|
||||||
|
))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Legacy/test source-read adapter. Production Hosts must use [`execute_snapshot`].
|
||||||
|
/// This ledger attests bounded Source responses only and cannot satisfy the input-digest gate.
|
||||||
pub fn execute_bounded(
|
pub fn execute_bounded(
|
||||||
store: &SubjektivStore,
|
store: &SubjektivStore,
|
||||||
context: &JournalJobContext,
|
context: &JournalJobContext,
|
||||||
request: JournalJobRequest,
|
request: JournalJobRequest,
|
||||||
|
read_source: impl FnMut(&JournalSource, u64, usize) -> JournalResult<recording::JournalMaterial>,
|
||||||
|
) -> JournalResult<JournalJobResponse> {
|
||||||
|
execute_inner(store, context, request, None, read_source)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn execute_inner(
|
||||||
|
store: &SubjektivStore,
|
||||||
|
context: &JournalJobContext,
|
||||||
|
request: JournalJobRequest,
|
||||||
|
input_digest: Option<&str>,
|
||||||
mut read_source: impl FnMut(&JournalSource, u64, usize) -> JournalResult<recording::JournalMaterial>,
|
mut read_source: impl FnMut(&JournalSource, u64, usize) -> JournalResult<recording::JournalMaterial>,
|
||||||
) -> JournalResult<JournalJobResponse> {
|
) -> JournalResult<JournalJobResponse> {
|
||||||
let generation = store
|
let generation = store.database.try_with_connection(|tx| {
|
||||||
.database
|
super::lifecycle::reject_erased(tx, &context.host)?;
|
||||||
.try_with_connection(|tx| super::store::job(store, tx, context, true))?;
|
let generation = super::store::job(store, tx, context, true)?;
|
||||||
|
if let Some(input_digest) = input_digest {
|
||||||
|
require_input_delivery(tx, context, input_digest)?;
|
||||||
|
}
|
||||||
|
Ok::<_, JournalError>(generation)
|
||||||
|
})?;
|
||||||
let mut delivered = None;
|
let mut delivered = None;
|
||||||
let response = match request {
|
let response = match request {
|
||||||
JournalJobRequest::Execute { operation } => {
|
JournalJobRequest::Execute { operation } => {
|
||||||
require_delivery(store, context, &generation.selected_ranges)?;
|
if input_digest.is_none() {
|
||||||
|
require_delivery(store, context, &generation.selected_ranges)?;
|
||||||
|
}
|
||||||
let receipt = if matches!(operation.action, JournalAction::Checkpoint { .. }) {
|
let receipt = if matches!(operation.action, JournalAction::Checkpoint { .. }) {
|
||||||
crate::api::checkpoint_journal(store, context, operation)?
|
crate::api::checkpoint_journal(store, context, operation)?
|
||||||
} else {
|
} else {
|
||||||
@@ -197,6 +323,11 @@ pub fn execute_bounded(
|
|||||||
JournalJobRequest::ReadText { read } => {
|
JournalJobRequest::ReadText { read } => {
|
||||||
let text = match &read.target {
|
let text = match &read.target {
|
||||||
JournalTextTarget::Source { pointer } => {
|
JournalTextTarget::Source { pointer } => {
|
||||||
|
if input_digest.is_some() {
|
||||||
|
return Err(JournalError::PermissionDenied(
|
||||||
|
"Source target is unavailable: raw source bodies are in immutable Job input".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let range = generation
|
let range = generation
|
||||||
.selected_ranges
|
.selected_ranges
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
@@ -202,7 +202,7 @@ pub(super) fn open(tx: &Connection, journal_id: &str) -> JournalResult<Option<Jo
|
|||||||
.optional()?;
|
.optional()?;
|
||||||
id.map(|id| entry(tx, journal_id, &id)).transpose()
|
id.map(|id| entry(tx, journal_id, &id)).transpose()
|
||||||
}
|
}
|
||||||
fn previous_fixed(
|
pub(super) fn previous_fixed(
|
||||||
tx: &Connection,
|
tx: &Connection,
|
||||||
journal_id: &str,
|
journal_id: &str,
|
||||||
) -> JournalResult<Option<(JournalSourcePointer, u64)>> {
|
) -> JournalResult<Option<(JournalSourcePointer, u64)>> {
|
||||||
@@ -569,6 +569,60 @@ impl SubjektivStore {
|
|||||||
Ok(snapshot)
|
Ok(snapshot)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
/// Rebind a saved body input to a new attempt without resampling its initial state.
|
||||||
|
/// The first binding freezes the full initial Journal snapshot; retries reuse it and
|
||||||
|
/// retain operation receipts. Different Jobs cannot borrow that snapshot as authority.
|
||||||
|
pub fn prepare_journal_input(
|
||||||
|
&self,
|
||||||
|
host: &JournalHostContext,
|
||||||
|
input: &super::recording::JournalInput,
|
||||||
|
binding: JobAttemptBinding,
|
||||||
|
) -> JournalResult<JournalGeneration> {
|
||||||
|
label(&binding.job_id)?;
|
||||||
|
label(&binding.attempt_id)?;
|
||||||
|
self.database.try_transaction(|tx| {
|
||||||
|
check_host(self, tx, host, false)?;
|
||||||
|
check_session_host(self, tx, host)?;
|
||||||
|
let current = journal(tx, host, &input.journal_id, true)?;
|
||||||
|
let existing: Option<String> = tx.query_row(
|
||||||
|
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 AND attempt_id=?3",
|
||||||
|
params![input.journal_id, binding.job_id, binding.attempt_id], |r| r.get(0)).optional()?;
|
||||||
|
if let Some(raw) = existing {
|
||||||
|
let g = serde_json::from_str(&raw)?;
|
||||||
|
input.validate_generation(&g)?;
|
||||||
|
return Ok(g);
|
||||||
|
}
|
||||||
|
// Retained receipt lookup does not reacquire writer authority. Only a new
|
||||||
|
// generation requires the current active Subject/Session writer lifetime.
|
||||||
|
check_host(self, tx, host, true)?;
|
||||||
|
if tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_generations WHERE journal_id=?1 AND active=1)", [&input.journal_id], |r| r.get::<_, bool>(0))? {
|
||||||
|
return Err(conflict("another active generation owns this Journal"));
|
||||||
|
}
|
||||||
|
let prior: Option<String> = tx.query_row(
|
||||||
|
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 ORDER BY rowid LIMIT 1",
|
||||||
|
params![input.journal_id, binding.job_id], |r| r.get(0)).optional()?;
|
||||||
|
let mut g = if let Some(raw) = prior {
|
||||||
|
serde_json::from_str::<JournalGeneration>(&raw)?
|
||||||
|
} else {
|
||||||
|
JournalGeneration { id: String::new(), journal_id: input.journal_id.clone(), job: binding.clone(),
|
||||||
|
selected_ranges: input.selected_ranges.clone(), initial_journal: current.clone(),
|
||||||
|
initial_open: open(tx, &input.journal_id)?, previous_fixed_end: previous_fixed(tx, &input.journal_id)?.map(|p| p.0) }
|
||||||
|
};
|
||||||
|
input.validate_generation(&g)?;
|
||||||
|
check_ranges(tx, ¤t, &input.selected_ranges, true)?;
|
||||||
|
let end = prefix(&g.initial_journal, &input.selected_ranges)?;
|
||||||
|
let cursor = (current.process_order, current.process_byte);
|
||||||
|
if cursor < (g.initial_journal.process_order, g.initial_journal.process_byte) || cursor > end {
|
||||||
|
return Err(conflict("saved input no longer owns the current processing cursor"));
|
||||||
|
}
|
||||||
|
g.id = issued_id("journal-generation");
|
||||||
|
g.job = binding;
|
||||||
|
tx.execute("INSERT INTO journal_generations(id,journal_id,job_id,attempt_id,request_json,snapshot_json,active) VALUES(?1,?2,?3,?4,?5,?6,1)",
|
||||||
|
params![g.id,g.journal_id,g.job.job_id,g.job.attempt_id,serde_json::to_string(&g.selected_ranges)?,serde_json::to_string(&g)?])?;
|
||||||
|
Ok(g)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn validate_journal_job(&self, ctx: &JournalJobContext) -> JournalResult<()> {
|
pub(super) fn validate_journal_job(&self, ctx: &JournalJobContext) -> JournalResult<()> {
|
||||||
self.database.try_with_connection(|tx| {
|
self.database.try_with_connection(|tx| {
|
||||||
job(self, tx, ctx, true)?;
|
job(self, tx, ctx, true)?;
|
||||||
@@ -597,6 +651,32 @@ impl SubjektivStore {
|
|||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
/// Read the exact retained Job/attempt snapshot without granting writer
|
||||||
|
/// authority or creating a generation. Used for outcome recovery after removal.
|
||||||
|
pub fn retained_journal_generation(
|
||||||
|
&self,
|
||||||
|
host: &JournalHostContext,
|
||||||
|
id: &str,
|
||||||
|
binding: &JobAttemptBinding,
|
||||||
|
) -> JournalResult<Option<JournalGeneration>> {
|
||||||
|
self.database.try_with_connection(|tx| {
|
||||||
|
check_host(self, tx, host, false)?;
|
||||||
|
check_session_host(self, tx, host)?;
|
||||||
|
journal(tx, host, id, true)?;
|
||||||
|
let raw: Option<String> = tx.query_row(
|
||||||
|
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 AND attempt_id=?3",
|
||||||
|
params![id, binding.job_id, binding.attempt_id], |r| r.get(0),
|
||||||
|
).optional()?;
|
||||||
|
raw.map(|raw| {
|
||||||
|
let generation: JournalGeneration = serde_json::from_str(&raw)?;
|
||||||
|
if generation.journal_id != id || &generation.job != binding {
|
||||||
|
return Err(conflict("retained generation identity mismatch"));
|
||||||
|
}
|
||||||
|
Ok(generation)
|
||||||
|
}).transpose()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn read_journal_generation(
|
pub fn read_journal_generation(
|
||||||
&self,
|
&self,
|
||||||
host: &JournalHostContext,
|
host: &JournalHostContext,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,10 @@
|
|||||||
//! Bounded, Host-bound Journal recording and sequential Job transport.
|
//! Bounded, Host-bound Journal recording and sequential Job transport.
|
||||||
//!
|
//!
|
||||||
//! Subject, Journal and generation authority never come from these model inputs.
|
//! Subject, Journal and generation authority never come from these model inputs.
|
||||||
//! Hosts validate live execution and public source bytes before selection; reading a
|
//! Hosts validate live execution and public source bytes before selection, save body-bearing
|
||||||
//! preview must not select the enclosing entry. Operation retries retain the entire
|
//! immutable Job input, and attest its exact digest and dispatch plan through the service.
|
||||||
|
//! Model requests cannot attest input delivery; Source targets are rejected by snapshot execution.
|
||||||
|
//! Initial/current Journal reads remain separate from input delivery. Operation retries retain the entire
|
||||||
//! JournalOperation, including its original expected_state and covered_ranges.
|
//! JournalOperation, including its original expected_state and covered_ranges.
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
@@ -50,8 +52,8 @@ pub struct JournalJobState {
|
|||||||
pub current_journal: JournalMetadata,
|
pub current_journal: JournalMetadata,
|
||||||
pub current_open: Option<JournalEntrySummary>,
|
pub current_open: Option<JournalEntrySummary>,
|
||||||
pub previous_fixed_end: Option<JournalSourcePointer>,
|
pub previous_fixed_end: Option<JournalSourcePointer>,
|
||||||
/// Frozen permission to fetch this prefix, not evidence of delivery or reflection.
|
/// Frozen source selection delivered in immutable Job input, not evidence of reflection.
|
||||||
/// The sequential service gates mutations on its durable Source-read delivery ledger.
|
/// Production execution requires the trusted Host's matching input-digest attestation.
|
||||||
pub selected_ranges: Vec<JournalSourceRange>,
|
pub selected_ranges: Vec<JournalSourceRange>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -83,7 +85,8 @@ pub enum JournalTextTarget {
|
|||||||
OpenOngoing {
|
OpenOngoing {
|
||||||
snapshot: JournalSnapshot,
|
snapshot: JournalSnapshot,
|
||||||
},
|
},
|
||||||
/// The Host restricts this pointer to this Job's frozen public source input.
|
/// Legacy source-read adapter only. Production snapshot execution rejects this target;
|
||||||
|
/// raw source bodies must already be present in the immutable Job input.
|
||||||
Source {
|
Source {
|
||||||
pointer: JournalSourcePointer,
|
pointer: JournalSourcePointer,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -639,6 +639,11 @@ static MIGRATIONS: &[FeatureMigration] = &[
|
|||||||
"add explicit Subject permission sources and immutable child execution grants",
|
"add explicit Subject permission sources and immutable child execution grants",
|
||||||
permissions::migrate,
|
permissions::migrate,
|
||||||
),
|
),
|
||||||
|
FeatureMigration::new(
|
||||||
|
13,
|
||||||
|
"bind Journal generation delivery to immutable Job input digest",
|
||||||
|
journal::service::migrate_input_delivery,
|
||||||
|
),
|
||||||
];
|
];
|
||||||
|
|
||||||
pub const REGISTRATION: FeatureRegistration =
|
pub const REGISTRATION: FeatureRegistration =
|
||||||
|
|||||||
+25
-76
@@ -19,6 +19,8 @@ use workdir::{
|
|||||||
WorkdirSessionRouter,
|
WorkdirSessionRouter,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
mod cleanup;
|
||||||
|
|
||||||
const DEFAULT_TIMEOUT_SECS: u64 = 120;
|
const DEFAULT_TIMEOUT_SECS: u64 = 120;
|
||||||
const MAX_TIMEOUT_SECS: u64 = 600;
|
const MAX_TIMEOUT_SECS: u64 = 600;
|
||||||
const INLINE_BYTE_BUDGET: usize = 12 * 1024;
|
const INLINE_BYTE_BUDGET: usize = 12 * 1024;
|
||||||
@@ -28,7 +30,9 @@ struct BashParams {
|
|||||||
/// Worker-local alias of the Workdir attachment to use.
|
/// Worker-local alias of the Workdir attachment to use.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
target_workdir: Option<String>,
|
target_workdir: Option<String>,
|
||||||
/// Workdir-relative directory in which to execute the command.
|
/// Directory in which to execute the command.
|
||||||
|
// Keep provider path-coordinate documentation out of the model-facing schema.
|
||||||
|
#[schemars(with = "String")]
|
||||||
cwd: WorkdirPath,
|
cwd: WorkdirPath,
|
||||||
command: String,
|
command: String,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -72,80 +76,19 @@ impl Drop for CommandGuard {
|
|||||||
let Some(handle) = self.handle.take() else {
|
let Some(handle) = self.handle.take() else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
let workdir = self.session.clone();
|
let cleanup = cleanup::Cleanup {
|
||||||
let state = Arc::clone(&self.state);
|
session: self.session.clone(),
|
||||||
let execution_id = self.execution_id.clone();
|
state: Arc::clone(&self.state),
|
||||||
let execution_nonce = self.execution_nonce;
|
execution_id: self.execution_id.clone(),
|
||||||
let mut retained = self.retained.take();
|
execution_nonce: self.execution_nonce,
|
||||||
let cursor = self.cursor;
|
handle,
|
||||||
// A dropped provider future is not terminal confirmation. Keep the live
|
cursor: self.cursor,
|
||||||
// execution registered until cleanup has both requested cancellation and
|
retained: self
|
||||||
// observed terminal command output, so cancellation/session teardown
|
.retained
|
||||||
// cannot race with an apparently empty registry.
|
.take()
|
||||||
tokio::spawn(async move {
|
.expect("started command owns output writer"),
|
||||||
let result = tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
};
|
||||||
let _ = workdir.cancel_command(handle.clone()).await;
|
tokio::spawn(cleanup.run());
|
||||||
let mut cursor = cursor;
|
|
||||||
loop {
|
|
||||||
let page = command_page(
|
|
||||||
&workdir,
|
|
||||||
CommandOutputRequest {
|
|
||||||
handle: handle.clone(),
|
|
||||||
cursor,
|
|
||||||
limit: workdir::MAX_COMMAND_READ_BYTES,
|
|
||||||
wait: true,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
if let Some(writer) = &mut retained {
|
|
||||||
let mut next = cursor;
|
|
||||||
for chunk in &page.chunks {
|
|
||||||
if chunk.sequence != next {
|
|
||||||
return Err(workdir::WorkdirError::Unavailable(
|
|
||||||
"command cleanup sequence gap".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
writer
|
|
||||||
.append_chunk(chunk)
|
|
||||||
.map_err(workdir::WorkdirError::Unavailable)?;
|
|
||||||
next = next.checked_add(chunk.bytes.len() as u64).ok_or_else(|| {
|
|
||||||
workdir::WorkdirError::Unavailable(
|
|
||||||
"command cleanup cursor overflow".into(),
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
}
|
|
||||||
if page.next_cursor != next {
|
|
||||||
return Err(workdir::WorkdirError::Unavailable(
|
|
||||||
"command cleanup cursor gap".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cursor = page.next_cursor;
|
|
||||||
if let Some(terminal) = page.terminal {
|
|
||||||
if let Some(writer) = &mut retained {
|
|
||||||
writer
|
|
||||||
.finish_transport(&terminal)
|
|
||||||
.map_err(workdir::WorkdirError::Unavailable)?;
|
|
||||||
}
|
|
||||||
return Ok::<(), workdir::WorkdirError>(());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
// Bounded cleanup can end with an unknown provider outcome. The writer then
|
|
||||||
// terminalizes partial evidence on drop; retaining a dead collector in `active`
|
|
||||||
// would leak registrations and falsely suggest it could still collect output.
|
|
||||||
let _ = result;
|
|
||||||
let mut state = state.lock().unwrap();
|
|
||||||
if state
|
|
||||||
.active
|
|
||||||
.get(&execution_id)
|
|
||||||
.is_some_and(|active| active.execution_nonce == execution_nonce)
|
|
||||||
{
|
|
||||||
state.active.remove(&execution_id);
|
|
||||||
state.cancellation_requested.remove(&execution_id);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -367,7 +310,7 @@ impl BashTool {
|
|||||||
guard.handle = None;
|
guard.handle = None;
|
||||||
|
|
||||||
let timed_out = terminal.status == workdir::CommandStatus::TimedOut;
|
let timed_out = terminal.status == workdir::CommandStatus::TimedOut;
|
||||||
let summary =
|
let mut summary =
|
||||||
if cancellation_requested || terminal.status == workdir::CommandStatus::Cancelled {
|
if cancellation_requested || terminal.status == workdir::CommandStatus::Cancelled {
|
||||||
format!("$ {cmd_summary} (cancelled)")
|
format!("$ {cmd_summary} (cancelled)")
|
||||||
} else if timed_out {
|
} else if timed_out {
|
||||||
@@ -379,6 +322,12 @@ impl BashTool {
|
|||||||
None => format!("$ {cmd_summary} (terminated)"),
|
None => format!("$ {cmd_summary} (terminated)"),
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
if matches!(
|
||||||
|
terminal.output_collection,
|
||||||
|
workdir::CommandOutputCollection::Incomplete { .. }
|
||||||
|
) {
|
||||||
|
summary.push_str(" (output incomplete)");
|
||||||
|
}
|
||||||
Ok(BashResult {
|
Ok(BashResult {
|
||||||
summary,
|
summary,
|
||||||
stdout_preview: String::from_utf8_lossy(&stdout_preview).into_owned(),
|
stdout_preview: String::from_utf8_lossy(&stdout_preview).into_owned(),
|
||||||
|
|||||||
@@ -0,0 +1,121 @@
|
|||||||
|
//! Finite Drop cleanup. A cancel acknowledgement is not termination evidence; only a
|
||||||
|
//! validated terminal fence confirms the process result. No provider text is logged.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::bash_output::{
|
||||||
|
CleanupCancel, CleanupDiagnostics, CleanupDrain, CleanupFailure, CleanupProviderFailure,
|
||||||
|
CleanupRegistration,
|
||||||
|
};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
const CLEANUP_TIMEOUT: Duration = Duration::from_secs(10);
|
||||||
|
// Reserve time for independent terminal observation even if cancel never responds.
|
||||||
|
const CANCEL_TIMEOUT: Duration = Duration::from_secs(2);
|
||||||
|
|
||||||
|
pub(super) struct Cleanup {
|
||||||
|
pub session: WorkdirSessionHandle,
|
||||||
|
pub state: Arc<Mutex<BashExecutionState>>,
|
||||||
|
pub execution_id: String,
|
||||||
|
pub execution_nonce: u64,
|
||||||
|
pub handle: CommandHandle,
|
||||||
|
pub cursor: u64,
|
||||||
|
pub retained: BashOutputWriter,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Cleanup {
|
||||||
|
pub async fn run(mut self) {
|
||||||
|
let deadline = tokio::time::Instant::now() + CLEANUP_TIMEOUT;
|
||||||
|
let cancel = match tokio::time::timeout(
|
||||||
|
CANCEL_TIMEOUT,
|
||||||
|
self.session.cancel_command(self.handle.clone()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(Ok(())) => CleanupCancel::Acknowledged,
|
||||||
|
Ok(Err(error)) => CleanupCancel::Failed {
|
||||||
|
reason: CleanupProviderFailure::from(&error),
|
||||||
|
},
|
||||||
|
Err(_) => CleanupCancel::DeadlineExceeded,
|
||||||
|
};
|
||||||
|
let drain = match tokio::time::timeout_at(deadline, self.drain()).await {
|
||||||
|
Ok(Ok(())) => CleanupDrain::TerminalConfirmed,
|
||||||
|
Ok(Err(reason)) => CleanupDrain::Failed { reason },
|
||||||
|
Err(_) => CleanupDrain::DeadlineExceeded,
|
||||||
|
};
|
||||||
|
let deadline_expired = matches!(drain, CleanupDrain::DeadlineExceeded);
|
||||||
|
// Finalize evidence before exposing an empty registry. Keep the lock through
|
||||||
|
// publication so an older cleanup can never remove a newer generation.
|
||||||
|
let mut state = self.state.lock().unwrap();
|
||||||
|
let registration = if state
|
||||||
|
.active
|
||||||
|
.get(&self.execution_id)
|
||||||
|
.is_some_and(|active| active.execution_nonce == self.execution_nonce)
|
||||||
|
{
|
||||||
|
state.active.remove(&self.execution_id);
|
||||||
|
state.cancellation_requested.remove(&self.execution_id);
|
||||||
|
CleanupRegistration::Released
|
||||||
|
} else {
|
||||||
|
CleanupRegistration::Superseded
|
||||||
|
};
|
||||||
|
let diagnostics = CleanupDiagnostics {
|
||||||
|
execution_nonce: self.execution_nonce,
|
||||||
|
cancel,
|
||||||
|
drain,
|
||||||
|
deadline_expired,
|
||||||
|
registration,
|
||||||
|
};
|
||||||
|
tracing::info!(
|
||||||
|
output_id = %self.retained.id(),
|
||||||
|
cleanup = ?diagnostics,
|
||||||
|
"Bash Drop cleanup finished; cancellation and terminal observation are independent"
|
||||||
|
);
|
||||||
|
self.retained.finish_cleanup(diagnostics);
|
||||||
|
drop(self.retained);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn drain(&mut self) -> Result<(), CleanupFailure> {
|
||||||
|
loop {
|
||||||
|
let page = command_page(
|
||||||
|
&self.session,
|
||||||
|
CommandOutputRequest {
|
||||||
|
handle: self.handle.clone(),
|
||||||
|
cursor: self.cursor,
|
||||||
|
limit: workdir::MAX_COMMAND_READ_BYTES,
|
||||||
|
wait: true,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|error| CleanupFailure::Provider {
|
||||||
|
reason: CleanupProviderFailure::from(&error),
|
||||||
|
})?;
|
||||||
|
let mut next = self.cursor;
|
||||||
|
for chunk in &page.chunks {
|
||||||
|
if chunk.sequence != next {
|
||||||
|
return Err(CleanupFailure::SequenceGap);
|
||||||
|
}
|
||||||
|
next = next
|
||||||
|
.checked_add(chunk.bytes.len() as u64)
|
||||||
|
.filter(|next| *next <= crate::bash_output::COMMAND_BYTES as u64)
|
||||||
|
.ok_or(CleanupFailure::DeliveryBudgetExceeded)?;
|
||||||
|
self.retained
|
||||||
|
.append_chunk(chunk)
|
||||||
|
.map_err(|_| CleanupFailure::RetainedChunkInvalid)?;
|
||||||
|
}
|
||||||
|
if page.next_cursor != next {
|
||||||
|
return Err(CleanupFailure::CursorGap);
|
||||||
|
}
|
||||||
|
self.cursor = next;
|
||||||
|
if let Some(terminal) = page.terminal {
|
||||||
|
self.retained
|
||||||
|
.finish_transport(&terminal)
|
||||||
|
.map_err(|_| CleanupFailure::TerminalMetadataInvalid)?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
// An immediately-ready provider must not starve the cleanup deadline.
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,397 @@
|
|||||||
|
//! Component tests enter through the production CommandGuard Drop, cleanup, page
|
||||||
|
//! validation and retained store. Only the Workdir provider is replaced; paused time
|
||||||
|
//! and a cancel barrier make faults/deadlines deterministic without OS timing.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
use workdir::*;
|
||||||
|
|
||||||
|
const PRIVATE: &str = "SECRET command /private/host/path output password=poison";
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug)]
|
||||||
|
enum Fault {
|
||||||
|
Cancel,
|
||||||
|
CancelHang,
|
||||||
|
Drain,
|
||||||
|
TerminalTimeout,
|
||||||
|
Sequence,
|
||||||
|
Cursor,
|
||||||
|
RetainedChunk,
|
||||||
|
TerminalMetadata,
|
||||||
|
CollectionIncomplete,
|
||||||
|
Complete,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct Provider {
|
||||||
|
local: LocalWorkdirSession,
|
||||||
|
fault: Fault,
|
||||||
|
cancelling: tokio::sync::Notify,
|
||||||
|
allow_drain: AtomicBool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl WorkdirSession for Provider {
|
||||||
|
fn workdir(&self) -> &Workdir {
|
||||||
|
self.local.workdir()
|
||||||
|
}
|
||||||
|
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
||||||
|
WorkdirSessionCapabilities::ALL
|
||||||
|
}
|
||||||
|
async fn stat(&self, _: StatRequest) -> Result<StatResult, WorkdirError> {
|
||||||
|
Err(WorkdirError::OperationFailed)
|
||||||
|
}
|
||||||
|
async fn read(&self, _: ReadRequest) -> Result<ReadResult, WorkdirError> {
|
||||||
|
Err(WorkdirError::OperationFailed)
|
||||||
|
}
|
||||||
|
async fn write(&self, _: WriteRequest) -> Result<WriteResult, WorkdirError> {
|
||||||
|
Err(WorkdirError::OperationFailed)
|
||||||
|
}
|
||||||
|
async fn edit(&self, _: EditRequest) -> Result<EditResult, WorkdirError> {
|
||||||
|
Err(WorkdirError::OperationFailed)
|
||||||
|
}
|
||||||
|
async fn list(&self, _: ListRequest) -> Result<ListResult, WorkdirError> {
|
||||||
|
Err(WorkdirError::OperationFailed)
|
||||||
|
}
|
||||||
|
async fn glob(&self, _: GlobRequest) -> Result<GlobResult, WorkdirError> {
|
||||||
|
Err(WorkdirError::OperationFailed)
|
||||||
|
}
|
||||||
|
async fn grep(&self, _: GrepRequest) -> Result<GrepResult, WorkdirError> {
|
||||||
|
Err(WorkdirError::OperationFailed)
|
||||||
|
}
|
||||||
|
async fn start_command(&self, _: CommandRequest) -> Result<CommandHandle, WorkdirError> {
|
||||||
|
Err(WorkdirError::OperationFailed)
|
||||||
|
}
|
||||||
|
async fn command_status(&self, _: CommandHandle) -> Result<CommandStatus, WorkdirError> {
|
||||||
|
Err(WorkdirError::OperationFailed)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn command_output(
|
||||||
|
&self,
|
||||||
|
request: CommandOutputRequest,
|
||||||
|
) -> Result<CommandOutput, WorkdirError> {
|
||||||
|
assert_eq!(request.cursor, 3);
|
||||||
|
while !self.allow_drain.load(Ordering::Acquire) {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
match self.fault {
|
||||||
|
Fault::Drain => {
|
||||||
|
return Err(WorkdirError::Transport(PRIVATE.into())
|
||||||
|
.with_failure_reason(WorkdirFailureReason::HttpTransfer));
|
||||||
|
}
|
||||||
|
Fault::TerminalTimeout => return std::future::pending().await,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
let mut chunk = CommandByteChunk {
|
||||||
|
sequence: 3,
|
||||||
|
offset: 3,
|
||||||
|
stream: CommandStream::Stdout,
|
||||||
|
bytes: b"tail".to_vec(),
|
||||||
|
};
|
||||||
|
if matches!(self.fault, Fault::Sequence) {
|
||||||
|
chunk.sequence = 4;
|
||||||
|
}
|
||||||
|
if matches!(self.fault, Fault::RetainedChunk) {
|
||||||
|
chunk.offset = 42;
|
||||||
|
}
|
||||||
|
let mut terminal = terminal();
|
||||||
|
if matches!(self.fault, Fault::TerminalMetadata) {
|
||||||
|
terminal.stdout_delivered_bytes = 6;
|
||||||
|
}
|
||||||
|
if matches!(self.fault, Fault::CollectionIncomplete) {
|
||||||
|
terminal.output_collection = CommandOutputCollection::Incomplete {
|
||||||
|
failure: CommandFailure {
|
||||||
|
stage: CommandFailureStage::StdoutDrain,
|
||||||
|
reason: CommandIoErrorKind::Other,
|
||||||
|
os_code: None,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
Ok(CommandOutput {
|
||||||
|
chunks: vec![chunk],
|
||||||
|
next_cursor: if matches!(self.fault, Fault::Cursor) {
|
||||||
|
8
|
||||||
|
} else {
|
||||||
|
7
|
||||||
|
},
|
||||||
|
terminal: Some(terminal),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cancel_command(&self, _: CommandHandle) -> Result<(), WorkdirError> {
|
||||||
|
self.cancelling.notify_one();
|
||||||
|
match self.fault {
|
||||||
|
Fault::Cancel => Err(WorkdirError::Unavailable(PRIVATE.into())),
|
||||||
|
Fault::CancelHang => std::future::pending().await,
|
||||||
|
_ => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async fn close(&self) -> Result<(), WorkdirError> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn terminal() -> CommandTerminal {
|
||||||
|
CommandTerminal {
|
||||||
|
status: CommandStatus::Completed,
|
||||||
|
exit_code: Some(0),
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: CommandOutputCollection::Complete,
|
||||||
|
stdout_bytes: 7,
|
||||||
|
stderr_bytes: 0,
|
||||||
|
stdout_delivered_bytes: 7,
|
||||||
|
stderr_delivered_bytes: 0,
|
||||||
|
truncation_reason: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Fixture {
|
||||||
|
_root: tempfile::TempDir,
|
||||||
|
provider: Arc<Provider>,
|
||||||
|
state: Arc<Mutex<BashExecutionState>>,
|
||||||
|
view: BashOutputView,
|
||||||
|
id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Fixture {
|
||||||
|
fn start(fault: Fault) -> (Self, CommandGuard) {
|
||||||
|
let base =
|
||||||
|
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/cleanup-fixtures");
|
||||||
|
std::fs::create_dir_all(&base).unwrap();
|
||||||
|
let root = tempfile::tempdir_in(base).unwrap();
|
||||||
|
let provider = Arc::new(Provider {
|
||||||
|
local: LocalWorkdirSession::new(
|
||||||
|
manifest::Scope::writable(root.path()).unwrap(),
|
||||||
|
root.path().into(),
|
||||||
|
),
|
||||||
|
fault,
|
||||||
|
cancelling: Default::default(),
|
||||||
|
allow_drain: AtomicBool::new(false),
|
||||||
|
});
|
||||||
|
let view = BashOutputView::new();
|
||||||
|
let mut retained = view.begin().unwrap();
|
||||||
|
retained.append(CommandStream::Stdout, b"pre");
|
||||||
|
let id = retained.id();
|
||||||
|
let state = Arc::new(Mutex::new(BashExecutionState::default()));
|
||||||
|
let handle = CommandHandle(PRIVATE.into());
|
||||||
|
state.lock().unwrap().active.insert(
|
||||||
|
PRIVATE.into(),
|
||||||
|
ActiveCommand {
|
||||||
|
call_id: PRIVATE.into(),
|
||||||
|
execution_nonce: 1,
|
||||||
|
session: provider.clone(),
|
||||||
|
handle: handle.clone(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
state
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.cancellation_requested
|
||||||
|
.insert(PRIVATE.into());
|
||||||
|
let guard = CommandGuard {
|
||||||
|
session: provider.clone(),
|
||||||
|
state: state.clone(),
|
||||||
|
execution_id: PRIVATE.into(),
|
||||||
|
execution_nonce: 1,
|
||||||
|
handle: Some(handle),
|
||||||
|
cursor: 3,
|
||||||
|
retained: Some(retained),
|
||||||
|
};
|
||||||
|
(
|
||||||
|
Self {
|
||||||
|
_root: root,
|
||||||
|
provider,
|
||||||
|
state,
|
||||||
|
view,
|
||||||
|
id,
|
||||||
|
},
|
||||||
|
guard,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn finish(&self) -> BashOutputMetadata {
|
||||||
|
self.provider.allow_drain.store(true, Ordering::Release);
|
||||||
|
// The provider's read retry backoff and cleanup deadlines are virtual time.
|
||||||
|
for _ in 0..4 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
tokio::time::advance(Duration::from_secs(3)).await;
|
||||||
|
}
|
||||||
|
for _ in 0..1000 {
|
||||||
|
if let Some(metadata) = self.view.metadata(&self.id) {
|
||||||
|
return metadata;
|
||||||
|
}
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
panic!("finite cleanup did not publish partial evidence");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_evidence(&self, m: &BashOutputMetadata, terminal: bool, bytes: &[u8]) {
|
||||||
|
assert_eq!(
|
||||||
|
self.view
|
||||||
|
.read(&self.id, CommandStream::Stdout, 0, 64)
|
||||||
|
.unwrap(),
|
||||||
|
bytes
|
||||||
|
);
|
||||||
|
assert_eq!(m.stdout_saved, bytes.len());
|
||||||
|
assert_eq!(m.stdout_delivered, bytes.len() as u64);
|
||||||
|
assert_eq!(m.stdout_seen, bytes.len() as u64);
|
||||||
|
assert_eq!(
|
||||||
|
m.terminal.status,
|
||||||
|
terminal.then_some(CommandStatus::Completed)
|
||||||
|
);
|
||||||
|
assert_eq!(m.terminal.exit_code, terminal.then_some(0));
|
||||||
|
if !terminal {
|
||||||
|
assert!(m.terminal.output_collection.is_none());
|
||||||
|
assert!(m.terminal.process_failure.is_none());
|
||||||
|
assert!(!m.terminal.transport_complete);
|
||||||
|
}
|
||||||
|
let diagnostics = m.cleanup.as_ref().unwrap();
|
||||||
|
assert_eq!(diagnostics.execution_nonce, 1);
|
||||||
|
let json = serde_json::to_string(diagnostics).unwrap();
|
||||||
|
let debug = format!("{diagnostics:?}");
|
||||||
|
for forbidden in [
|
||||||
|
PRIVATE, "SECRET", "/private", "password", "poison", "tail", "pre",
|
||||||
|
] {
|
||||||
|
assert!(!json.contains(forbidden), "diagnostics leaked data: {json}");
|
||||||
|
assert!(
|
||||||
|
!debug.contains(forbidden),
|
||||||
|
"tracing payload leaked data: {debug}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn cancel_failure_does_not_preclude_confirmed_terminal() {
|
||||||
|
let (f, guard) = Fixture::start(Fault::Cancel);
|
||||||
|
drop(guard);
|
||||||
|
f.provider.cancelling.notified().await;
|
||||||
|
assert!(f.state.lock().unwrap().active.contains_key(PRIVATE));
|
||||||
|
let m = f.finish().await;
|
||||||
|
f.assert_evidence(&m, true, b"pretail");
|
||||||
|
assert!(m.terminal.transport_complete);
|
||||||
|
assert!(matches!(
|
||||||
|
m.cleanup.as_ref().unwrap().cancel,
|
||||||
|
CleanupCancel::Failed { .. }
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
m.cleanup.as_ref().unwrap().drain,
|
||||||
|
CleanupDrain::TerminalConfirmed
|
||||||
|
));
|
||||||
|
assert!(!m.cleanup.as_ref().unwrap().deadline_expired);
|
||||||
|
assert!(f.state.lock().unwrap().active.is_empty());
|
||||||
|
assert!(f.state.lock().unwrap().cancellation_requested.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn acknowledged_cancel_drain_faults_keep_distinct_reasons_and_partial_bytes() {
|
||||||
|
for (fault, reason, bytes) in [
|
||||||
|
(Fault::Drain, "provider", b"pre".as_slice()),
|
||||||
|
(Fault::Sequence, "sequence_gap", b"pre"),
|
||||||
|
(Fault::Cursor, "cursor_gap", b"pretail"),
|
||||||
|
(Fault::RetainedChunk, "retained_chunk_invalid", b"pre"),
|
||||||
|
(
|
||||||
|
Fault::TerminalMetadata,
|
||||||
|
"terminal_metadata_invalid",
|
||||||
|
b"pretail",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let (f, guard) = Fixture::start(fault);
|
||||||
|
drop(guard);
|
||||||
|
f.provider.cancelling.notified().await;
|
||||||
|
let m = f.finish().await;
|
||||||
|
f.assert_evidence(&m, false, bytes);
|
||||||
|
let diagnostic = serde_json::to_value(m.cleanup.as_ref().unwrap()).unwrap();
|
||||||
|
assert_eq!(diagnostic["cancel"]["outcome"], "acknowledged");
|
||||||
|
assert_eq!(diagnostic["drain"]["outcome"], "failed");
|
||||||
|
assert_eq!(diagnostic["drain"]["reason"]["kind"], reason);
|
||||||
|
assert_eq!(diagnostic["registration"], "released");
|
||||||
|
assert_eq!(diagnostic["deadline_expired"], false);
|
||||||
|
assert!(f.state.lock().unwrap().active.is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn terminal_deadline_never_claims_stop_or_leaves_active_collector() {
|
||||||
|
let (f, guard) = Fixture::start(Fault::TerminalTimeout);
|
||||||
|
drop(guard);
|
||||||
|
f.provider.cancelling.notified().await;
|
||||||
|
let m = f.finish().await;
|
||||||
|
f.assert_evidence(&m, false, b"pre");
|
||||||
|
assert!(matches!(
|
||||||
|
m.cleanup.as_ref().unwrap().cancel,
|
||||||
|
CleanupCancel::Acknowledged
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
m.cleanup.as_ref().unwrap().drain,
|
||||||
|
CleanupDrain::DeadlineExceeded
|
||||||
|
));
|
||||||
|
assert!(m.cleanup.as_ref().unwrap().deadline_expired);
|
||||||
|
assert!(f.state.lock().unwrap().active.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn cancel_deadline_still_attempts_independent_terminal_observation() {
|
||||||
|
let (f, guard) = Fixture::start(Fault::CancelHang);
|
||||||
|
drop(guard);
|
||||||
|
f.provider.cancelling.notified().await;
|
||||||
|
let m = f.finish().await;
|
||||||
|
f.assert_evidence(&m, true, b"pretail");
|
||||||
|
assert!(matches!(
|
||||||
|
m.cleanup.as_ref().unwrap().cancel,
|
||||||
|
CleanupCancel::DeadlineExceeded
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
m.cleanup.as_ref().unwrap().drain,
|
||||||
|
CleanupDrain::TerminalConfirmed
|
||||||
|
));
|
||||||
|
assert!(!m.cleanup.as_ref().unwrap().deadline_expired);
|
||||||
|
assert!(f.state.lock().unwrap().active.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn old_cleanup_preserves_new_execution_and_cancellation_registration() {
|
||||||
|
let (f, guard) = Fixture::start(Fault::Complete);
|
||||||
|
drop(guard);
|
||||||
|
f.provider.cancelling.notified().await;
|
||||||
|
f.state
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.active
|
||||||
|
.get_mut(PRIVATE)
|
||||||
|
.unwrap()
|
||||||
|
.execution_nonce = 2;
|
||||||
|
let m = f.finish().await;
|
||||||
|
f.assert_evidence(&m, true, b"pretail");
|
||||||
|
assert!(matches!(
|
||||||
|
m.cleanup.as_ref().unwrap().registration,
|
||||||
|
CleanupRegistration::Superseded
|
||||||
|
));
|
||||||
|
assert_eq!(f.state.lock().unwrap().active[PRIVATE].execution_nonce, 2);
|
||||||
|
assert!(
|
||||||
|
f.state
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.cancellation_requested
|
||||||
|
.contains(PRIVATE)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn terminal_confirmation_is_independent_of_output_collection_completeness() {
|
||||||
|
let (f, guard) = Fixture::start(Fault::CollectionIncomplete);
|
||||||
|
drop(guard);
|
||||||
|
f.provider.cancelling.notified().await;
|
||||||
|
let m = f.finish().await;
|
||||||
|
f.assert_evidence(&m, true, b"pretail");
|
||||||
|
assert!(!m.terminal.transport_complete);
|
||||||
|
assert!(matches!(
|
||||||
|
m.terminal.output_collection,
|
||||||
|
Some(CommandOutputCollection::Incomplete { .. })
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
m.cleanup.as_ref().unwrap().drain,
|
||||||
|
CleanupDrain::TerminalConfirmed
|
||||||
|
));
|
||||||
|
assert!(f.state.lock().unwrap().active.is_empty());
|
||||||
|
}
|
||||||
+130
-10
@@ -19,7 +19,7 @@ use std::time::{Duration, Instant};
|
|||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use workdir::{CommandStatus, CommandStream};
|
use workdir::{CommandStatus, CommandStream};
|
||||||
|
|
||||||
const COMMAND_BYTES: usize = 4 * 1024 * 1024;
|
pub(crate) const COMMAND_BYTES: usize = 4 * 1024 * 1024;
|
||||||
const STORE_BYTES: usize = 32 * 1024 * 1024;
|
const STORE_BYTES: usize = 32 * 1024 * 1024;
|
||||||
const STORE_COUNT: usize = 64;
|
const STORE_COUNT: usize = 64;
|
||||||
const ORDER_COUNT: usize = 4096;
|
const ORDER_COUNT: usize = 4096;
|
||||||
@@ -55,6 +55,7 @@ struct Record {
|
|||||||
transport_counts: Option<(u64, u64)>,
|
transport_counts: Option<(u64, u64)>,
|
||||||
transport_truncation: Option<workdir::CommandOutputTruncation>,
|
transport_truncation: Option<workdir::CommandOutputTruncation>,
|
||||||
terminal: Option<OutputTerminal>,
|
terminal: Option<OutputTerminal>,
|
||||||
|
cleanup: Option<CleanupDiagnostics>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Serialize)]
|
#[derive(Clone, Debug, Serialize)]
|
||||||
@@ -68,8 +69,10 @@ pub struct OutputRange {
|
|||||||
pub struct OutputTerminal {
|
pub struct OutputTerminal {
|
||||||
pub status: Option<CommandStatus>,
|
pub status: Option<CommandStatus>,
|
||||||
pub exit_code: Option<i32>,
|
pub exit_code: Option<i32>,
|
||||||
/// False on a forcibly dropped collector or failed transport: received bytes are retained,
|
pub process_failure: Option<workdir::CommandFailure>,
|
||||||
/// but they are not a claim about the unobserved process tail.
|
pub output_collection: Option<workdir::CommandOutputCollection>,
|
||||||
|
/// True only for confirmed EOF collection (quota is reported separately). A terminal
|
||||||
|
/// fence or exit zero alone does not confirm the unobserved process tail.
|
||||||
pub transport_complete: bool,
|
pub transport_complete: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -93,10 +96,72 @@ pub struct BashOutputMetadata {
|
|||||||
pub transport_truncation: Option<workdir::CommandOutputTruncation>,
|
pub transport_truncation: Option<workdir::CommandOutputTruncation>,
|
||||||
pub storage_truncated: bool,
|
pub storage_truncated: bool,
|
||||||
pub terminal: OutputTerminal,
|
pub terminal: OutputTerminal,
|
||||||
|
pub cleanup: Option<CleanupDiagnostics>,
|
||||||
pub text_encoding: &'static str,
|
pub text_encoding: &'static str,
|
||||||
pub ordering: &'static str,
|
pub ordering: &'static str,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Drop cleanup observations, not assertions that cancellation stopped a process.
|
||||||
|
/// Correlate with the enclosing output UUID and this host-assigned execution generation;
|
||||||
|
/// caller identifiers, provider handles, commands and error text are deliberately excluded.
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
pub struct CleanupDiagnostics {
|
||||||
|
pub execution_nonce: u64,
|
||||||
|
pub cancel: CleanupCancel,
|
||||||
|
pub drain: CleanupDrain,
|
||||||
|
pub deadline_expired: bool,
|
||||||
|
pub registration: CleanupRegistration,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
#[serde(tag = "outcome", rename_all = "snake_case")]
|
||||||
|
pub enum CleanupCancel {
|
||||||
|
Acknowledged,
|
||||||
|
Failed { reason: CleanupProviderFailure },
|
||||||
|
DeadlineExceeded,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
#[serde(tag = "outcome", rename_all = "snake_case")]
|
||||||
|
pub enum CleanupDrain {
|
||||||
|
TerminalConfirmed,
|
||||||
|
Failed { reason: CleanupFailure },
|
||||||
|
DeadlineExceeded,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||||
|
pub enum CleanupFailure {
|
||||||
|
Provider { reason: CleanupProviderFailure },
|
||||||
|
SequenceGap,
|
||||||
|
CursorGap,
|
||||||
|
DeliveryBudgetExceeded,
|
||||||
|
RetainedChunkInvalid,
|
||||||
|
TerminalMetadataInvalid,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
pub struct CleanupProviderFailure {
|
||||||
|
pub failure_reason: Option<workdir::WorkdirFailureReason>,
|
||||||
|
pub denial_reason: Option<workdir::WorkdirDenialReason>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&workdir::WorkdirError> for CleanupProviderFailure {
|
||||||
|
fn from(error: &workdir::WorkdirError) -> Self {
|
||||||
|
Self {
|
||||||
|
failure_reason: error.failure_reason(),
|
||||||
|
denial_reason: error.denial_reason(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Serialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum CleanupRegistration {
|
||||||
|
Released,
|
||||||
|
Superseded,
|
||||||
|
}
|
||||||
|
|
||||||
/// A bounded reservation acquired before command start. Drop terminalizes partial evidence.
|
/// A bounded reservation acquired before command start. Drop terminalizes partial evidence.
|
||||||
pub struct BashOutputWriter {
|
pub struct BashOutputWriter {
|
||||||
record: Arc<Mutex<Record>>,
|
record: Arc<Mutex<Record>>,
|
||||||
@@ -212,6 +277,7 @@ impl BashOutputView {
|
|||||||
transport_counts: None,
|
transport_counts: None,
|
||||||
transport_truncation: None,
|
transport_truncation: None,
|
||||||
terminal: None,
|
terminal: None,
|
||||||
|
cleanup: None,
|
||||||
}));
|
}));
|
||||||
store.records.push_back(record.clone());
|
store.records.push_back(record.clone());
|
||||||
store.reserved += COMMAND_BYTES;
|
store.reserved += COMMAND_BYTES;
|
||||||
@@ -311,6 +377,7 @@ impl Record {
|
|||||||
transport_truncation: self.transport_truncation.clone(),
|
transport_truncation: self.transport_truncation.clone(),
|
||||||
storage_truncated: self.truncated || self.transport_truncation.is_some(),
|
storage_truncated: self.truncated || self.transport_truncation.is_some(),
|
||||||
terminal: self.terminal.clone().expect("finalized record"),
|
terminal: self.terminal.clone().expect("finalized record"),
|
||||||
|
cleanup: self.cleanup.clone(),
|
||||||
text_encoding: "utf8_lossy",
|
text_encoding: "utf8_lossy",
|
||||||
ordering: "provider_read_observation",
|
ordering: "provider_read_observation",
|
||||||
}
|
}
|
||||||
@@ -340,6 +407,14 @@ async fn reap(weak: Weak<Mutex<Store>>, mut shutdown: tokio::sync::watch::Receiv
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl BashOutputWriter {
|
impl BashOutputWriter {
|
||||||
|
pub(crate) fn id(&self) -> String {
|
||||||
|
self.record.lock().unwrap().id.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn finish_cleanup(&mut self, diagnostics: CleanupDiagnostics) {
|
||||||
|
self.record.lock().unwrap().cleanup = Some(diagnostics);
|
||||||
|
}
|
||||||
|
|
||||||
pub fn append_chunk(&mut self, chunk: &workdir::CommandByteChunk) -> Result<(), String> {
|
pub fn append_chunk(&mut self, chunk: &workdir::CommandByteChunk) -> Result<(), String> {
|
||||||
let expected = {
|
let expected = {
|
||||||
let r = self.record.lock().unwrap();
|
let r = self.record.lock().unwrap();
|
||||||
@@ -405,12 +480,9 @@ impl BashOutputWriter {
|
|||||||
terminal: &workdir::CommandTerminal,
|
terminal: &workdir::CommandTerminal,
|
||||||
) -> Result<BashOutputMetadata, String> {
|
) -> Result<BashOutputMetadata, String> {
|
||||||
let mut r = self.record.lock().unwrap();
|
let mut r = self.record.lock().unwrap();
|
||||||
if terminal.stdout_delivered_bytes != r.stdout_seen
|
if !terminal.metadata_valid()
|
||||||
|
|| terminal.stdout_delivered_bytes != r.stdout_seen
|
||||||
|| terminal.stderr_delivered_bytes != r.stderr_seen
|
|| terminal.stderr_delivered_bytes != r.stderr_seen
|
||||||
|| terminal.stdout_bytes < r.stdout_seen
|
|
||||||
|| terminal.stderr_bytes < r.stderr_seen
|
|
||||||
|| ((terminal.stdout_bytes != r.stdout_seen || terminal.stderr_bytes != r.stderr_seen)
|
|
||||||
!= terminal.truncation_reason.is_some())
|
|
||||||
{
|
{
|
||||||
return Err("command terminal counts do not fence delivered bytes".into());
|
return Err("command terminal counts do not fence delivered bytes".into());
|
||||||
}
|
}
|
||||||
@@ -419,7 +491,12 @@ impl BashOutputWriter {
|
|||||||
r.terminal = Some(OutputTerminal {
|
r.terminal = Some(OutputTerminal {
|
||||||
status: Some(terminal.status),
|
status: Some(terminal.status),
|
||||||
exit_code: terminal.exit_code,
|
exit_code: terminal.exit_code,
|
||||||
transport_complete: true,
|
transport_complete: matches!(
|
||||||
|
terminal.output_collection,
|
||||||
|
workdir::CommandOutputCollection::Complete
|
||||||
|
),
|
||||||
|
process_failure: terminal.process_failure,
|
||||||
|
output_collection: Some(terminal.output_collection),
|
||||||
});
|
});
|
||||||
Ok(r.metadata())
|
Ok(r.metadata())
|
||||||
}
|
}
|
||||||
@@ -439,6 +516,8 @@ impl Drop for BashOutputWriter {
|
|||||||
status: None,
|
status: None,
|
||||||
exit_code: None,
|
exit_code: None,
|
||||||
transport_complete: false,
|
transport_complete: false,
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: None,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -456,6 +535,8 @@ mod tests {
|
|||||||
status: Some(CommandStatus::Completed),
|
status: Some(CommandStatus::Completed),
|
||||||
exit_code: Some(0),
|
exit_code: Some(0),
|
||||||
transport_complete: true,
|
transport_complete: true,
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: Some(workdir::CommandOutputCollection::Complete),
|
||||||
})
|
})
|
||||||
.id
|
.id
|
||||||
}
|
}
|
||||||
@@ -514,6 +595,8 @@ mod tests {
|
|||||||
status: Some(CommandStatus::Completed),
|
status: Some(CommandStatus::Completed),
|
||||||
exit_code: Some(0),
|
exit_code: Some(0),
|
||||||
transport_complete: true,
|
transport_complete: true,
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: Some(workdir::CommandOutputCollection::Complete),
|
||||||
});
|
});
|
||||||
assert!(root.metadata(&metadata.id).is_some());
|
assert!(root.metadata(&metadata.id).is_some());
|
||||||
let _replacement = sibling.begin().unwrap();
|
let _replacement = sibling.begin().unwrap();
|
||||||
@@ -582,7 +665,7 @@ mod tests {
|
|||||||
let root = BashOutputView::new();
|
let root = BashOutputView::new();
|
||||||
let mut writer = root.begin().unwrap();
|
let mut writer = root.begin().unwrap();
|
||||||
writer.append(CommandStream::Stdout, b"abc");
|
writer.append(CommandStream::Stdout, b"abc");
|
||||||
let mut terminal = serde_json::json!({"status":"completed","exit_code":0,"stdout_bytes":9,"stderr_bytes":0,"stdout_delivered_bytes":4,"stderr_delivered_bytes":0,"truncation_reason":"byte_budget"});
|
let mut terminal = serde_json::json!({"status":"completed","exit_code":0,"process_failure":null,"output_collection":{"state":"complete"},"stdout_bytes":9,"stderr_bytes":0,"stdout_delivered_bytes":4,"stderr_delivered_bytes":0,"truncation_reason":"byte_budget"});
|
||||||
assert!(
|
assert!(
|
||||||
writer
|
writer
|
||||||
.finish_transport(&serde_json::from_value(terminal.clone()).unwrap())
|
.finish_transport(&serde_json::from_value(terminal.clone()).unwrap())
|
||||||
@@ -602,4 +685,41 @@ mod tests {
|
|||||||
Some(workdir::CommandOutputTruncation::ByteBudget)
|
Some(workdir::CommandOutputTruncation::ByteBudget)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn incomplete_collection_with_quota_retains_both_causes_without_claiming_complete_output() {
|
||||||
|
let root = BashOutputView::new();
|
||||||
|
let mut writer = root.begin().unwrap();
|
||||||
|
writer.append(CommandStream::Stdout, b"abc");
|
||||||
|
let terminal: workdir::CommandTerminal = serde_json::from_value(serde_json::json!({
|
||||||
|
"status":"completed", "exit_code":0, "process_failure":null,
|
||||||
|
"output_collection":{"state":"incomplete","failure":{"stage":"stdout_drain","reason":"other","os_code":null}},
|
||||||
|
"stdout_bytes":9, "stderr_bytes":0, "stdout_delivered_bytes":3, "stderr_delivered_bytes":0,
|
||||||
|
"truncation_reason":"byte_budget"
|
||||||
|
})).unwrap();
|
||||||
|
let metadata = writer.finish_transport(&terminal).unwrap();
|
||||||
|
assert!(!metadata.terminal.transport_complete);
|
||||||
|
assert_eq!(metadata.terminal.exit_code, Some(0));
|
||||||
|
assert_eq!(
|
||||||
|
metadata.terminal.output_collection,
|
||||||
|
Some(terminal.output_collection)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
metadata.transport_truncation,
|
||||||
|
Some(workdir::CommandOutputTruncation::ByteBudget)
|
||||||
|
);
|
||||||
|
assert_eq!(metadata.stdout_seen, 9);
|
||||||
|
assert_eq!(metadata.stdout_saved, 3);
|
||||||
|
assert!(metadata.storage_truncated);
|
||||||
|
assert!(metadata.retention_limit.is_none());
|
||||||
|
assert_eq!(
|
||||||
|
root.read(&metadata.id, CommandStream::Stdout, 0, 100)
|
||||||
|
.unwrap(),
|
||||||
|
b"abc"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(root.metadata(&metadata.id).unwrap()).unwrap(),
|
||||||
|
serde_json::to_value(metadata).unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ pub(crate) struct EditParams {
|
|||||||
/// Worker-local alias of the Workdir attachment to use.
|
/// Worker-local alias of the Workdir attachment to use.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub target_workdir: Option<String>,
|
pub target_workdir: Option<String>,
|
||||||
/// Logical path relative to the bound Workdir root.
|
/// Existing file in which to replace text.
|
||||||
pub file_path: String,
|
pub file_path: String,
|
||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
pub edit: fs_operation::text::EditArgs,
|
pub edit: fs_operation::text::EditArgs,
|
||||||
|
|||||||
@@ -167,6 +167,7 @@ pub(crate) fn checked_error(error: workdir::WorkdirError) -> ToolError {
|
|||||||
| WorkdirError::IsDirectory { .. }
|
| WorkdirError::IsDirectory { .. }
|
||||||
| WorkdirError::InvalidGlob { .. }
|
| WorkdirError::InvalidGlob { .. }
|
||||||
| WorkdirError::InvalidRegex { .. } => "checkout_invalid",
|
| WorkdirError::InvalidRegex { .. } => "checkout_invalid",
|
||||||
|
WorkdirError::NotDispatched => "checkout_not_dispatched",
|
||||||
WorkdirError::Unavailable { .. } | WorkdirError::SessionClosed => "checkout_unavailable",
|
WorkdirError::Unavailable { .. } | WorkdirError::SessionClosed => "checkout_unavailable",
|
||||||
// IO/transport failures may arrive after effects have begun. Do not infer
|
// IO/transport failures may arrive after effects have begun. Do not infer
|
||||||
// no-effects from their text or strip explicit OutcomeUnknown evidence.
|
// no-effects from their text or strip explicit OutcomeUnknown evidence.
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ pub(crate) struct GlobParams {
|
|||||||
pub target_workdir: Option<String>,
|
pub target_workdir: Option<String>,
|
||||||
/// Glob pattern, for example `**/*.rs` or `src/**/test_*.py`.
|
/// Glob pattern, for example `**/*.rs` or `src/**/test_*.py`.
|
||||||
pub pattern: String,
|
pub pattern: String,
|
||||||
/// Logical Workdir-relative directory. Defaults to the Workdir root.
|
/// Directory to search. Defaults to `.`.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub path: Option<String>,
|
pub path: Option<String>,
|
||||||
}
|
}
|
||||||
@@ -106,7 +106,7 @@ pub(crate) fn routed_glob_tool(router: Arc<WorkdirSessionRouter>) -> ToolDefinit
|
|||||||
Arc::new(move || {
|
Arc::new(move || {
|
||||||
let schema = schemars::schema_for!(GlobParams);
|
let schema = schemars::schema_for!(GlobParams);
|
||||||
let meta = ToolMeta::new("Glob")
|
let meta = ToolMeta::new("Glob")
|
||||||
.description("Find files matching a glob pattern inside the selected Workdir. Results are sorted and capped at 1000 entries. Paths are Workdir-relative.")
|
.description("Find files matching a glob pattern inside the selected Workdir. Results are sorted and capped at 1000 entries.")
|
||||||
.input_schema(serde_json::to_value(schema).expect("Glob schema serialization"));
|
.input_schema(serde_json::to_value(schema).expect("Glob schema serialization"));
|
||||||
let tool: Arc<dyn Tool> = Arc::new(GlobTool {
|
let tool: Arc<dyn Tool> = Arc::new(GlobTool {
|
||||||
router: router.clone(),
|
router: router.clone(),
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ pub(crate) struct GrepParams {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
target_workdir: Option<String>,
|
target_workdir: Option<String>,
|
||||||
pattern: String,
|
pattern: String,
|
||||||
/// Workdir-relative path, or an absolute path covered by readable scope. Defaults to the Workdir root.
|
/// File or directory to search. Defaults to `.`.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub path: Option<String>,
|
pub path: Option<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -160,7 +160,7 @@ pub(crate) fn routed_grep_tool(router: Arc<WorkdirSessionRouter>) -> ToolDefinit
|
|||||||
Arc::new(move || {
|
Arc::new(move || {
|
||||||
let schema = schemars::schema_for!(GrepParams);
|
let schema = schemars::schema_for!(GrepParams);
|
||||||
let meta = ToolMeta::new("Grep")
|
let meta = ToolMeta::new("Grep")
|
||||||
.description("Search a file or directory in the selected Workdir with a regex. Content results group lines by file; `>` marks matching lines and unmarked lines are context. Directory traversal executes inside the WorkdirSession provider. Results are bounded and Workdir-relative.")
|
.description("Search a file or directory in the selected Workdir with a regex. Content results group lines by file; `>` marks matching lines and unmarked lines are context. Directory traversal executes inside the WorkdirSession provider. Results are bounded.")
|
||||||
.input_schema(serde_json::to_value(schema).expect("Grep schema serialization"));
|
.input_schema(serde_json::to_value(schema).expect("Grep schema serialization"));
|
||||||
let tool: Arc<dyn Tool> = Arc::new(GrepTool {
|
let tool: Arc<dyn Tool> = Arc::new(GrepTool {
|
||||||
router: router.clone(),
|
router: router.clone(),
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ use workdir::{WorkdirPath, WorkdirSessionHandle, WorkdirSessionRouter};
|
|||||||
const DESCRIPTION: &str = "Read a text file from a Workdir attachment selected by its Worker-local alias. \
|
const DESCRIPTION: &str = "Read a text file from a Workdir attachment selected by its Worker-local alias. \
|
||||||
Supports offset/limit for large files. Returns line-numbered output (1-based). \
|
Supports offset/limit for large files. Returns line-numbered output (1-based). \
|
||||||
Directories cannot be read. The file must be read before Write or Edit can \
|
Directories cannot be read. The file must be read before Write or Edit can \
|
||||||
modify it. Paths are Workdir-relative unless an absolute path is explicitly readable. \
|
modify it. \
|
||||||
When exactly one Workdir is attached, target_workdir may be omitted.";
|
When exactly one Workdir is attached, target_workdir may be omitted.";
|
||||||
|
|
||||||
const DEFAULT_LIMIT: usize = 2000;
|
const DEFAULT_LIMIT: usize = 2000;
|
||||||
@@ -24,7 +24,7 @@ pub(crate) struct ReadParams {
|
|||||||
/// Worker-local alias of the Workdir attachment to use.
|
/// Worker-local alias of the Workdir attachment to use.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub target_workdir: Option<String>,
|
pub target_workdir: Option<String>,
|
||||||
/// Workdir-relative path, or an absolute path covered by readable scope.
|
/// Text file to read; directories are not supported.
|
||||||
pub file_path: String,
|
pub file_path: String,
|
||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
pub read: fs_operation::text::LineReadArgs,
|
pub read: fs_operation::text::LineReadArgs,
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ pub const MAX_IMAGE_BYTES: usize = 10 * 1024 * 1024;
|
|||||||
const IMAGE_READ_CHUNK_BYTES: usize = 1024 * 1024;
|
const IMAGE_READ_CHUNK_BYTES: usize = 1024 * 1024;
|
||||||
|
|
||||||
const DESCRIPTION: &str = "Attach an image from the selected Workdir attachment to the next model request. \
|
const DESCRIPTION: &str = "Attach an image from the selected Workdir attachment to the next model request. \
|
||||||
The path must be logical and Workdir-relative. Supported formats: PNG, JPEG, GIF, and WebP. \
|
Supported formats: PNG, JPEG, GIF, and WebP. \
|
||||||
When exactly one Workdir is attached, target_workdir may be omitted.";
|
When exactly one Workdir is attached, target_workdir may be omitted.";
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||||
@@ -28,7 +28,7 @@ struct ViewImageParams {
|
|||||||
/// Worker-local alias of the Workdir attachment to use.
|
/// Worker-local alias of the Workdir attachment to use.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
target_workdir: Option<String>,
|
target_workdir: Option<String>,
|
||||||
/// Logical path relative to the bound Workdir root.
|
/// Image file to attach to the next model request.
|
||||||
path: String,
|
path: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ pub(crate) struct WriteParams {
|
|||||||
/// Worker-local alias of the Workdir attachment to use.
|
/// Worker-local alias of the Workdir attachment to use.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub target_workdir: Option<String>,
|
pub target_workdir: Option<String>,
|
||||||
/// Logical path relative to the bound Workdir root.
|
/// File to create or overwrite.
|
||||||
pub file_path: String,
|
pub file_path: String,
|
||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
pub write: fs_operation::text::WriteArgs,
|
pub write: fs_operation::text::WriteArgs,
|
||||||
|
|||||||
@@ -3,6 +3,11 @@
|
|||||||
//! `ToolServerHandle::register_tool` / `flush_pending` are `pub(crate)` in
|
//! `ToolServerHandle::register_tool` / `flush_pending` are `pub(crate)` in
|
||||||
//! agen, so from here we exercise the factories directly — the same
|
//! agen, so from here we exercise the factories directly — the same
|
||||||
//! code path that `flush_pending()` runs at production time.
|
//! code path that `flush_pending()` runs at production time.
|
||||||
|
//!
|
||||||
|
//! Scoped routing cases connect real LocalWorkdirSession, WorkdirToolBroker::scope,
|
||||||
|
//! attachment router and Tool factories without replacing their permission or path checks.
|
||||||
|
//! They use RAII temporary roots and existing image fixtures; command cases require the
|
||||||
|
//! same local shell as the Bash suite and run in the normal `tools --test integration` target.
|
||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -80,6 +85,101 @@ fn setup_routed() -> (TempDir, TempDir, Arc<WorkdirSessionRouter>, Registry) {
|
|||||||
(left, right, router, Registry::new(definitions))
|
(left, right, router, Registry::new(definitions))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const READ_SAME_FILE: &str = "IFS= read -r line < same.txt && printf '%s\\n' \"$line\"";
|
||||||
|
|
||||||
|
struct ScopedRegistry {
|
||||||
|
roots: Vec<TempDir>,
|
||||||
|
directories: Vec<std::path::PathBuf>,
|
||||||
|
brokers: Vec<WorkdirToolBroker>,
|
||||||
|
leases: Vec<workdir::WorkdirScopeLease>,
|
||||||
|
router: Arc<WorkdirSessionRouter>,
|
||||||
|
reg: Registry,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScopedRegistry {
|
||||||
|
// Cwd and scope targets here are delegation declarations in provider coordinates.
|
||||||
|
async fn new(attachments: &[(&str, &str, bool)]) -> Self {
|
||||||
|
let router = Arc::new(WorkdirSessionRouter::new());
|
||||||
|
let mut roots = Vec::new();
|
||||||
|
let mut directories = Vec::new();
|
||||||
|
let mut brokers = Vec::new();
|
||||||
|
let mut leases = Vec::new();
|
||||||
|
for &(alias, cwd, command) in attachments {
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
let directory = root.path().join(cwd);
|
||||||
|
std::fs::create_dir_all(directory.join("subdir")).unwrap();
|
||||||
|
std::fs::write(directory.join("same.txt"), format!("{alias} original\n")).unwrap();
|
||||||
|
std::fs::write(
|
||||||
|
directory.join("subdir/same.txt"),
|
||||||
|
format!("{alias} nested needle\n"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
std::fs::write(
|
||||||
|
directory.join("image.png"),
|
||||||
|
include_bytes!("fixtures/view-image/valid.png"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
if cwd != "." {
|
||||||
|
// Decoys make accidental root-based resolution observable.
|
||||||
|
std::fs::write(root.path().join("same.txt"), "root decoy\n").unwrap();
|
||||||
|
std::fs::write(root.path().join("image.png"), "not an image").unwrap();
|
||||||
|
}
|
||||||
|
let source: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::new(
|
||||||
|
Scope::writable(root.path()).unwrap(),
|
||||||
|
root.path().to_path_buf(),
|
||||||
|
));
|
||||||
|
let broker = WorkdirToolBroker::new(source);
|
||||||
|
let lease = broker
|
||||||
|
.scope(WorkdirToolScope {
|
||||||
|
rules: vec![WorkdirToolScopeRule {
|
||||||
|
target: WorkdirPath::new(cwd).unwrap(),
|
||||||
|
permission: WorkdirToolScopePermission::Write,
|
||||||
|
recursive: true,
|
||||||
|
symlink_policy: Default::default(),
|
||||||
|
}],
|
||||||
|
cwd: WorkdirPath::new(cwd).unwrap(),
|
||||||
|
command,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
router
|
||||||
|
.attach(
|
||||||
|
WorkdirAttachmentAlias::new(alias).unwrap(),
|
||||||
|
lease.tool_session(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
roots.push(root);
|
||||||
|
directories.push(directory);
|
||||||
|
brokers.push(broker);
|
||||||
|
leases.push(lease);
|
||||||
|
}
|
||||||
|
let mut definitions = routed_builtin_tools(
|
||||||
|
router.clone(),
|
||||||
|
Tracker::new(),
|
||||||
|
tools::bash_output::BashOutputView::new(),
|
||||||
|
);
|
||||||
|
definitions.push(routed_view_image_tool(router.clone()));
|
||||||
|
Self {
|
||||||
|
roots,
|
||||||
|
directories,
|
||||||
|
brokers,
|
||||||
|
leases,
|
||||||
|
router,
|
||||||
|
reg: Registry::new(definitions),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn close(self) {
|
||||||
|
self.router.close_all().await.unwrap();
|
||||||
|
for lease in &self.leases {
|
||||||
|
lease.close().await.unwrap();
|
||||||
|
}
|
||||||
|
for broker in &self.brokers {
|
||||||
|
broker.close().await.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn valid_gif_with_len(target_len: usize) -> Vec<u8> {
|
fn valid_gif_with_len(target_len: usize) -> Vec<u8> {
|
||||||
let fixture = include_bytes!("fixtures/view-image/valid.gif");
|
let fixture = include_bytes!("fixtures/view-image/valid.gif");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -156,6 +256,71 @@ fn routed_tool_schemas_share_optional_target_workdir() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn routed_tool_help_does_not_override_session_path_coordinates() {
|
||||||
|
fn assert_no_root_coordinates(tool: &str, description: &str) {
|
||||||
|
let normalized = description
|
||||||
|
.split_whitespace()
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" ")
|
||||||
|
.to_ascii_lowercase();
|
||||||
|
for obsolete in ["workdir-relative", "workdir root", "root-relative"] {
|
||||||
|
assert!(
|
||||||
|
!normalized.contains(obsolete),
|
||||||
|
"{tool} help overrides session path coordinates: {description}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_schema_descriptions(tool: &str, schema: &serde_json::Value) {
|
||||||
|
match schema {
|
||||||
|
serde_json::Value::Object(object) => {
|
||||||
|
if let Some(description) = object.get("description").and_then(|v| v.as_str()) {
|
||||||
|
assert_no_root_coordinates(tool, description);
|
||||||
|
}
|
||||||
|
for value in object.values() {
|
||||||
|
check_schema_descriptions(tool, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
serde_json::Value::Array(array) => {
|
||||||
|
for value in array {
|
||||||
|
check_schema_descriptions(tool, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let router = Arc::new(WorkdirSessionRouter::new());
|
||||||
|
let mut definitions = routed_builtin_tools(
|
||||||
|
router.clone(),
|
||||||
|
Tracker::new(),
|
||||||
|
tools::bash_output::BashOutputView::new(),
|
||||||
|
);
|
||||||
|
definitions.push(routed_view_image_tool(router));
|
||||||
|
for definition in definitions {
|
||||||
|
let (meta, _) = definition();
|
||||||
|
assert_no_root_coordinates(&meta.name, &meta.description);
|
||||||
|
// Referenced type definitions are model-visible too, not just properties.
|
||||||
|
check_schema_descriptions(&meta.name, &meta.input_schema);
|
||||||
|
|
||||||
|
let path_parameter = match meta.name.as_str() {
|
||||||
|
"Read" | "Write" | "Edit" => "file_path",
|
||||||
|
"Glob" | "Grep" | "ViewImage" => "path",
|
||||||
|
"Bash" => "cwd",
|
||||||
|
other => panic!("missing path parameter contract for {other}"),
|
||||||
|
};
|
||||||
|
let help = meta.input_schema["properties"][path_parameter]["description"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap_or_default();
|
||||||
|
assert!(
|
||||||
|
!help.trim().is_empty(),
|
||||||
|
"{}.{path_parameter} must retain argument-specific help",
|
||||||
|
meta.name
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn routed_tools_require_alias_only_when_attachment_set_is_ambiguous() {
|
async fn routed_tools_require_alias_only_when_attachment_set_is_ambiguous() {
|
||||||
let (_left, _right, router, reg) = setup_routed();
|
let (_left, _right, router, reg) = setup_routed();
|
||||||
@@ -284,6 +449,427 @@ async fn target_alias_routes_each_tool_and_isolates_read_before_edit() {
|
|||||||
router.close_all().await.unwrap();
|
router.close_all().await.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn scoped_cwd_file_search_and_image_tools_share_reusable_paths() {
|
||||||
|
for cwd in [".", "work"] {
|
||||||
|
let fixture = ScopedRegistry::new(&[("checkout", cwd, false)]).await;
|
||||||
|
let read = fixture.reg.get("Read");
|
||||||
|
let original = call(&read, json!({ "file_path": "same.txt" })).await;
|
||||||
|
assert!(original.content.unwrap().contains("checkout original"));
|
||||||
|
call(
|
||||||
|
&fixture.reg.get("Write"),
|
||||||
|
json!({ "file_path": "same.txt", "content": "written marker\n" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
call(
|
||||||
|
&fixture.reg.get("Edit"),
|
||||||
|
json!({ "file_path": "same.txt", "old_string": "written", "new_string": "edited" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let edited = call(&read, json!({ "file_path": "same.txt" })).await;
|
||||||
|
assert!(edited.content.unwrap().contains("edited marker"));
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(fixture.directories[0].join("same.txt")).unwrap(),
|
||||||
|
"edited marker\n"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Omitted search paths start at cwd, not at the underlying provider root.
|
||||||
|
for (name, input) in [
|
||||||
|
("Glob", json!({ "pattern": "same.txt" })),
|
||||||
|
(
|
||||||
|
"Grep",
|
||||||
|
json!({ "pattern": "edited marker", "output_mode": "files_with_matches" }),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let result = call(&fixture.reg.get(name), input).await;
|
||||||
|
assert_eq!(
|
||||||
|
result.content.unwrap().trim(),
|
||||||
|
"same.txt",
|
||||||
|
"{name}, cwd={cwd}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A search rooted below cwd still returns cwd-relative reusable paths.
|
||||||
|
for (name, input) in [
|
||||||
|
("Glob", json!({ "pattern": "*.txt", "path": "subdir" })),
|
||||||
|
(
|
||||||
|
"Grep",
|
||||||
|
json!({ "pattern": "needle", "path": "subdir", "output_mode": "files_with_matches" }),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let result = call(&fixture.reg.get(name), input).await;
|
||||||
|
let body = result.content.unwrap();
|
||||||
|
let paths: Vec<_> = body.lines().collect();
|
||||||
|
assert_eq!(paths, ["subdir/same.txt"], "{name}, cwd={cwd}");
|
||||||
|
let path = paths[0];
|
||||||
|
let found = call(&read, json!({ "file_path": path })).await;
|
||||||
|
assert!(found.content.unwrap().contains("checkout nested needle"));
|
||||||
|
call(
|
||||||
|
&fixture.reg.get("Edit"),
|
||||||
|
json!({ "file_path": path, "old_string": "needle", "new_string": format!("needle-{name}") }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(fixture.directories[0].join("subdir/same.txt")).unwrap(),
|
||||||
|
"checkout nested needle-Grep-Glob\n"
|
||||||
|
);
|
||||||
|
let image = call(
|
||||||
|
&fixture.reg.get("ViewImage"),
|
||||||
|
json!({ "path": "image.png" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(image.attachments.len(), 1);
|
||||||
|
let agen::tool::Attachment::Image(image) = &image.attachments[0];
|
||||||
|
assert_eq!(image.mime_type(), "image/png");
|
||||||
|
assert_eq!(
|
||||||
|
image.data(),
|
||||||
|
include_bytes!("fixtures/view-image/valid.png")
|
||||||
|
);
|
||||||
|
if cwd != "." {
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(fixture.roots[0].path().join("same.txt")).unwrap(),
|
||||||
|
"root decoy\n"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
fixture.close().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn scoped_bash_relative_cwd_and_cd_do_not_change_later_tool_cwd() {
|
||||||
|
for cwd in [".", "work"] {
|
||||||
|
let fixture = ScopedRegistry::new(&[("checkout", cwd, true)]).await;
|
||||||
|
for (command_cwd, command, expected) in [
|
||||||
|
(".", READ_SAME_FILE, "checkout original\n"),
|
||||||
|
("subdir", READ_SAME_FILE, "checkout nested needle\n"),
|
||||||
|
("./subdir//.", READ_SAME_FILE, "checkout nested needle\n"),
|
||||||
|
(
|
||||||
|
".",
|
||||||
|
"cd subdir && IFS= read -r line < same.txt && printf '%s\\n' \"$line\"",
|
||||||
|
"checkout nested needle\n",
|
||||||
|
),
|
||||||
|
(".", READ_SAME_FILE, "checkout original\n"),
|
||||||
|
] {
|
||||||
|
let output = call(
|
||||||
|
&fixture.reg.get("Bash"),
|
||||||
|
json!({ "cwd": command_cwd, "command": command, "timeout": 5 }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let body: serde_json::Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
||||||
|
assert_eq!(body["output"]["terminal"]["exit_code"], 0, "{body}");
|
||||||
|
assert_eq!(body["stdout_preview"], expected, "cwd={cwd}, {command}");
|
||||||
|
let subsequent =
|
||||||
|
call(&fixture.reg.get("Read"), json!({ "file_path": "same.txt" })).await;
|
||||||
|
assert!(subsequent.content.unwrap().contains("checkout original"));
|
||||||
|
}
|
||||||
|
fixture.close().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn scoped_aliases_with_different_cwds_keep_paths_and_observations_separate() {
|
||||||
|
let fixture =
|
||||||
|
ScopedRegistry::new(&[("left", "work", true), ("right", "other/deep", true)]).await;
|
||||||
|
let read = fixture.reg.get("Read");
|
||||||
|
let edit = fixture.reg.get("Edit");
|
||||||
|
call(
|
||||||
|
&read,
|
||||||
|
json!({ "target_workdir": "left", "file_path": "same.txt" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let unread = call_err(
|
||||||
|
&edit,
|
||||||
|
json!({ "target_workdir": "right", "file_path": "same.txt", "old_string": "original", "new_string": "wrong" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(unread.to_string().contains("has not been read"), "{unread}");
|
||||||
|
call(
|
||||||
|
&read,
|
||||||
|
json!({ "target_workdir": "right", "file_path": "same.txt" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
call(
|
||||||
|
&fixture.reg.get("Write"),
|
||||||
|
json!({ "target_workdir": "right", "file_path": "same.txt", "content": "right written\n" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
call(
|
||||||
|
&edit,
|
||||||
|
json!({ "target_workdir": "right", "file_path": "same.txt", "old_string": "written", "new_string": "edited" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(fixture.directories[0].join("same.txt")).unwrap(),
|
||||||
|
"left original\n"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(fixture.directories[1].join("same.txt")).unwrap(),
|
||||||
|
"right edited\n"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Different valid bytes make cross-alias image reads observable too.
|
||||||
|
let png: &[u8] = include_bytes!("fixtures/view-image/valid.png");
|
||||||
|
let gif: &[u8] = include_bytes!("fixtures/view-image/valid.gif");
|
||||||
|
std::fs::write(fixture.directories[1].join("image.png"), gif).unwrap();
|
||||||
|
for (alias, expected, image_bytes, mime) in [
|
||||||
|
("left", "left original", png, "image/png"),
|
||||||
|
("right", "right edited", gif, "image/gif"),
|
||||||
|
] {
|
||||||
|
let output = call(
|
||||||
|
&read,
|
||||||
|
json!({ "target_workdir": alias, "file_path": "same.txt" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(output.content.unwrap().contains(expected));
|
||||||
|
for (name, input) in [
|
||||||
|
(
|
||||||
|
"Glob",
|
||||||
|
json!({ "target_workdir": alias, "pattern": "*.txt", "path": "subdir" }),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Grep",
|
||||||
|
json!({ "target_workdir": alias, "pattern": alias, "path": "subdir", "output_mode": "files_with_matches" }),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let output = call(&fixture.reg.get(name), input).await;
|
||||||
|
let path = output.content.unwrap();
|
||||||
|
let path = path.trim();
|
||||||
|
assert_eq!(path, "subdir/same.txt", "{alias}: {name}");
|
||||||
|
let found = call(&read, json!({ "target_workdir": alias, "file_path": path })).await;
|
||||||
|
assert!(
|
||||||
|
found
|
||||||
|
.content
|
||||||
|
.unwrap()
|
||||||
|
.contains(&format!("{alias} nested needle"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let image = call(
|
||||||
|
&fixture.reg.get("ViewImage"),
|
||||||
|
json!({ "target_workdir": alias, "path": "image.png" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let agen::tool::Attachment::Image(image) = &image.attachments[0];
|
||||||
|
assert_eq!(image.mime_type(), mime);
|
||||||
|
assert_eq!(image.data(), image_bytes);
|
||||||
|
let output = call(
|
||||||
|
&fixture.reg.get("Bash"),
|
||||||
|
json!({ "target_workdir": alias, "cwd": ".", "command": READ_SAME_FILE, "timeout": 5 }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let body: serde_json::Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
||||||
|
assert_eq!(body["output"]["terminal"]["exit_code"], 0, "{body}");
|
||||||
|
assert_eq!(body["stdout_preview"], format!("{expected}\n"));
|
||||||
|
}
|
||||||
|
fixture.close().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn scoped_command_false_rejects_bash_without_fallback_to_another_alias() {
|
||||||
|
let fixture =
|
||||||
|
ScopedRegistry::new(&[("enabled", "work", true), ("disabled", "other", false)]).await;
|
||||||
|
for cwd in [".", "subdir"] {
|
||||||
|
let error = call_err(
|
||||||
|
&fixture.reg.get("Bash"),
|
||||||
|
json!({ "target_workdir": "disabled", "cwd": cwd, "command": "printf forbidden > blocked.txt", "timeout": 5 }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(
|
||||||
|
error.to_string().contains("does not support Command"),
|
||||||
|
"{error}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Write permission remains usable, but does not imply command permission.
|
||||||
|
call(
|
||||||
|
&fixture.reg.get("Write"),
|
||||||
|
json!({ "target_workdir": "disabled", "file_path": "created.txt", "content": "allowed write\n" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(fixture.directories[1].join("created.txt")).unwrap(),
|
||||||
|
"allowed write\n"
|
||||||
|
);
|
||||||
|
for (root, directory) in fixture.roots.iter().zip(&fixture.directories) {
|
||||||
|
for path in [
|
||||||
|
root.path().to_path_buf(),
|
||||||
|
directory.clone(),
|
||||||
|
directory.join("subdir"),
|
||||||
|
] {
|
||||||
|
assert!(!path.join("blocked.txt").exists());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fixture.close().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn scoped_normalized_paths_share_read_before_mutation_history() {
|
||||||
|
let fixture = ScopedRegistry::new(&[("checkout", "work", false)]).await;
|
||||||
|
let read = call(
|
||||||
|
&fixture.reg.get("Read"),
|
||||||
|
json!({ "file_path": "./subdir//./same.txt" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(read.content.unwrap().contains("checkout nested needle"));
|
||||||
|
call(
|
||||||
|
&fixture.reg.get("Write"),
|
||||||
|
json!({ "file_path": "subdir/same.txt", "content": "normalized write\n" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
call(
|
||||||
|
&fixture.reg.get("Edit"),
|
||||||
|
json!({ "file_path": "./subdir/same.txt", "old_string": "write", "new_string": "edit" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(fixture.directories[0].join("subdir/same.txt")).unwrap(),
|
||||||
|
"normalized edit\n"
|
||||||
|
);
|
||||||
|
for (name, input) in [
|
||||||
|
("Glob", json!({ "pattern": "*.txt", "path": "./subdir//." })),
|
||||||
|
(
|
||||||
|
"Grep",
|
||||||
|
json!({ "pattern": "normalized edit", "path": "./subdir//.", "output_mode": "files_with_matches" }),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let result = call(&fixture.reg.get(name), input).await;
|
||||||
|
assert_eq!(result.content.unwrap().trim(), "subdir/same.txt", "{name}");
|
||||||
|
}
|
||||||
|
let image = call(
|
||||||
|
&fixture.reg.get("ViewImage"),
|
||||||
|
json!({ "path": "././image.png" }),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let agen::tool::Attachment::Image(image) = &image.attachments[0];
|
||||||
|
assert_eq!(
|
||||||
|
image.data(),
|
||||||
|
include_bytes!("fixtures/view-image/valid.png")
|
||||||
|
);
|
||||||
|
fixture.close().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn scoped_tools_reject_parent_traversal_and_paths_outside_the_grant() {
|
||||||
|
let fixture = ScopedRegistry::new(&[("checkout", "work", true)]).await;
|
||||||
|
let outside = fixture.roots[0].path().join("outside");
|
||||||
|
std::fs::create_dir(&outside).unwrap();
|
||||||
|
std::fs::write(outside.join("secret.txt"), "private needle\n").unwrap();
|
||||||
|
for invalid in [
|
||||||
|
"../outside/secret.txt",
|
||||||
|
"subdir/../../outside/secret.txt",
|
||||||
|
"subdir\\same.txt",
|
||||||
|
] {
|
||||||
|
for (name, input) in [
|
||||||
|
("Read", json!({ "file_path": invalid })),
|
||||||
|
(
|
||||||
|
"Write",
|
||||||
|
json!({ "file_path": invalid, "content": "forbidden" }),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Edit",
|
||||||
|
json!({ "file_path": invalid, "old_string": "needle", "new_string": "forbidden" }),
|
||||||
|
),
|
||||||
|
("Glob", json!({ "path": invalid, "pattern": "*" })),
|
||||||
|
("Grep", json!({ "path": invalid, "pattern": "needle" })),
|
||||||
|
("ViewImage", json!({ "path": invalid })),
|
||||||
|
(
|
||||||
|
"Bash",
|
||||||
|
json!({ "cwd": invalid, "command": "printf forbidden > blocked.txt", "timeout": 5 }),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let error = call_err(&fixture.reg.get(name), input).await;
|
||||||
|
assert!(
|
||||||
|
error.to_string().contains("path"),
|
||||||
|
"{name}: {invalid}: {error}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// These paths are inside the provider's grant, but outside the child's grant.
|
||||||
|
// Absolute readable paths must not bypass the delegated scope after cwd rebasing.
|
||||||
|
for (name, input) in [
|
||||||
|
("Read", json!({ "file_path": outside.join("secret.txt") })),
|
||||||
|
("Grep", json!({ "path": outside, "pattern": "needle" })),
|
||||||
|
(
|
||||||
|
"Bash",
|
||||||
|
json!({ "cwd": outside, "command": "printf forbidden > blocked.txt", "timeout": 5 }),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let error = call_err(&fixture.reg.get(name), input).await;
|
||||||
|
let message = error.to_string();
|
||||||
|
assert!(
|
||||||
|
matches!(error, ToolError::InvalidArgument(_)),
|
||||||
|
"{name}: {message}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
message.contains("scope") || message.contains("path"),
|
||||||
|
"{name}: {message}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(outside.join("secret.txt")).unwrap(),
|
||||||
|
"private needle\n"
|
||||||
|
);
|
||||||
|
assert!(!outside.join("blocked.txt").exists());
|
||||||
|
assert!(!fixture.directories[0].join("blocked.txt").exists());
|
||||||
|
assert!(!fixture.roots[0].path().join("blocked.txt").exists());
|
||||||
|
fixture.close().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn scoped_cwd_tools_reject_resolved_symlink_escape_after_a_read() {
|
||||||
|
use std::os::unix::fs::symlink;
|
||||||
|
|
||||||
|
let fixture = ScopedRegistry::new(&[("checkout", "work", true)]).await;
|
||||||
|
let outside = fixture.roots[0].path().join("outside");
|
||||||
|
std::fs::create_dir(&outside).unwrap();
|
||||||
|
// Equal bytes avoid disguising an authority failure as a stale-hash rejection.
|
||||||
|
std::fs::write(outside.join("secret.txt"), "checkout original\n").unwrap();
|
||||||
|
let png = include_bytes!("fixtures/view-image/valid.png");
|
||||||
|
std::fs::write(outside.join("image.png"), png).unwrap();
|
||||||
|
call(&fixture.reg.get("Read"), json!({ "file_path": "same.txt" })).await;
|
||||||
|
std::fs::remove_file(fixture.directories[0].join("same.txt")).unwrap();
|
||||||
|
symlink(
|
||||||
|
"../outside/secret.txt",
|
||||||
|
fixture.directories[0].join("same.txt"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
symlink("../outside", fixture.directories[0].join("escape")).unwrap();
|
||||||
|
|
||||||
|
for (name, input) in [
|
||||||
|
("Read", json!({ "file_path": "same.txt" })),
|
||||||
|
(
|
||||||
|
"Write",
|
||||||
|
json!({ "file_path": "escape/created.txt", "content": "forbidden" }),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Edit",
|
||||||
|
json!({ "file_path": "same.txt", "old_string": "original", "new_string": "forbidden" }),
|
||||||
|
),
|
||||||
|
("Glob", json!({ "path": "escape", "pattern": "*" })),
|
||||||
|
("Grep", json!({ "path": "escape", "pattern": "original" })),
|
||||||
|
("ViewImage", json!({ "path": "escape/image.png" })),
|
||||||
|
(
|
||||||
|
"Bash",
|
||||||
|
json!({ "cwd": "escape", "command": "printf forbidden > blocked.txt", "timeout": 5 }),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let error = call_err(&fixture.reg.get(name), input).await;
|
||||||
|
let message = error.to_string();
|
||||||
|
assert!(
|
||||||
|
message.contains("scope") || message.contains("symlink"),
|
||||||
|
"{name}: {message}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(outside.join("secret.txt")).unwrap(),
|
||||||
|
"checkout original\n"
|
||||||
|
);
|
||||||
|
assert_eq!(std::fs::read(outside.join("image.png")).unwrap(), png);
|
||||||
|
assert!(!outside.join("created.txt").exists());
|
||||||
|
assert!(!outside.join("blocked.txt").exists());
|
||||||
|
fixture.close().await;
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn selected_attachment_capability_is_checked_without_fallback() {
|
async fn selected_attachment_capability_is_checked_without_fallback() {
|
||||||
let left = TempDir::new().unwrap();
|
let left = TempDir::new().unwrap();
|
||||||
@@ -416,6 +1002,7 @@ struct ExternalValidatingSession {
|
|||||||
truncate_first_binary_response: bool,
|
truncate_first_binary_response: bool,
|
||||||
command_response_losses: std::sync::atomic::AtomicUsize,
|
command_response_losses: std::sync::atomic::AtomicUsize,
|
||||||
lose_terminal_only: bool,
|
lose_terminal_only: bool,
|
||||||
|
collection_override: Option<workdir::CommandOutputCollection>,
|
||||||
mutate_after_first_binary_response: Option<std::path::PathBuf>,
|
mutate_after_first_binary_response: Option<std::path::PathBuf>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -534,7 +1121,11 @@ impl workdir::WorkdirSession for ExternalValidatingSession {
|
|||||||
&self,
|
&self,
|
||||||
request: workdir::CommandOutputRequest,
|
request: workdir::CommandOutputRequest,
|
||||||
) -> Result<workdir::CommandOutput, workdir::WorkdirError> {
|
) -> Result<workdir::CommandOutput, workdir::WorkdirError> {
|
||||||
let output = self.inner.command_output(request).await?;
|
let mut output = self.inner.command_output(request).await?;
|
||||||
|
if let (Some(collection), Some(terminal)) = (self.collection_override, &mut output.terminal)
|
||||||
|
{
|
||||||
|
terminal.output_collection = collection;
|
||||||
|
}
|
||||||
if (!self.lose_terminal_only || output.terminal.is_some())
|
if (!self.lose_terminal_only || output.terminal.is_some())
|
||||||
&& self
|
&& self
|
||||||
.command_response_losses
|
.command_response_losses
|
||||||
@@ -575,6 +1166,7 @@ async fn external_view_image_uses_a_protocol_bounded_request() {
|
|||||||
truncate_first_binary_response: false,
|
truncate_first_binary_response: false,
|
||||||
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
||||||
lose_terminal_only: false,
|
lose_terminal_only: false,
|
||||||
|
collection_override: None,
|
||||||
mutate_after_first_binary_response: None,
|
mutate_after_first_binary_response: None,
|
||||||
});
|
});
|
||||||
let png = include_bytes!("fixtures/view-image/valid.png");
|
let png = include_bytes!("fixtures/view-image/valid.png");
|
||||||
@@ -600,6 +1192,7 @@ async fn external_view_image_rejects_truncated_and_cross_version_chunks() {
|
|||||||
truncate_first_binary_response: true,
|
truncate_first_binary_response: true,
|
||||||
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
||||||
lose_terminal_only: false,
|
lose_terminal_only: false,
|
||||||
|
collection_override: None,
|
||||||
mutate_after_first_binary_response: None,
|
mutate_after_first_binary_response: None,
|
||||||
});
|
});
|
||||||
let (_meta, tool) = view_image_tool(truncated)();
|
let (_meta, tool) = view_image_tool(truncated)();
|
||||||
@@ -618,6 +1211,7 @@ async fn external_view_image_rejects_truncated_and_cross_version_chunks() {
|
|||||||
truncate_first_binary_response: false,
|
truncate_first_binary_response: false,
|
||||||
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
||||||
lose_terminal_only: false,
|
lose_terminal_only: false,
|
||||||
|
collection_override: None,
|
||||||
mutate_after_first_binary_response: Some(image_path),
|
mutate_after_first_binary_response: Some(image_path),
|
||||||
});
|
});
|
||||||
let (_meta, tool) = view_image_tool(changing)();
|
let (_meta, tool) = view_image_tool(changing)();
|
||||||
@@ -1104,7 +1698,20 @@ async fn bash_cancellation_returns_bounded_progress_as_terminal_output() {
|
|||||||
let execution = tokio::spawn(async move {
|
let execution = tokio::spawn(async move {
|
||||||
executing.execute(r#"{"command":"printf 'before'; printf 'err-before' >&2; while :; do :; done","cwd":"."}"#, execution_context).await
|
executing.execute(r#"{"command":"printf 'before'; printf 'err-before' >&2; while :; do :; done","cwd":"."}"#, execution_context).await
|
||||||
});
|
});
|
||||||
await_command_output(&mut events).await;
|
// Cancellation must follow both writes, not merely the first stdout chunk.
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||||
|
let mut ready = [false; 2];
|
||||||
|
while !ready.iter().all(|seen| *seen) {
|
||||||
|
if let workdir::CommandEvent::Output { stream, .. } = events.recv().await.unwrap() {
|
||||||
|
ready[match stream {
|
||||||
|
workdir::CommandStream::Stdout => 0,
|
||||||
|
workdir::CommandStream::Stderr => 1,
|
||||||
|
}] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("both streams written before cancellation");
|
||||||
bash.cancel_execution(&context).await.unwrap();
|
bash.cancel_execution(&context).await.unwrap();
|
||||||
let ToolError::Cancelled(output) =
|
let ToolError::Cancelled(output) =
|
||||||
tokio::time::timeout(std::time::Duration::from_secs(5), execution)
|
tokio::time::timeout(std::time::Duration::from_secs(5), execution)
|
||||||
@@ -1281,6 +1888,7 @@ async fn lost_page_and_terminal_responses_replay_without_restarting_or_double_ap
|
|||||||
mutate_after_first_binary_response: None,
|
mutate_after_first_binary_response: None,
|
||||||
command_response_losses: std::sync::atomic::AtomicUsize::new(1),
|
command_response_losses: std::sync::atomic::AtomicUsize::new(1),
|
||||||
lose_terminal_only,
|
lose_terminal_only,
|
||||||
|
collection_override: None,
|
||||||
});
|
});
|
||||||
let view = tools::bash_output::BashOutputView::new();
|
let view = tools::bash_output::BashOutputView::new();
|
||||||
let definitions = core_builtin_tools(session.clone(), Tracker::new(), view.clone());
|
let definitions = core_builtin_tools(session.clone(), Tracker::new(), view.clone());
|
||||||
@@ -1315,3 +1923,80 @@ async fn lost_page_and_terminal_responses_replay_without_restarting_or_double_ap
|
|||||||
session.close().await.unwrap();
|
session.close().await.unwrap();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Provider metadata is the fixture here; actual read/drain injection and HTTP
|
||||||
|
// forwarding are tested by workdir's production-supervisor suite.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn exit_zero_with_incomplete_output_keeps_partial_bytes_and_safe_failure_in_retained_evidence()
|
||||||
|
{
|
||||||
|
use workdir::{
|
||||||
|
CommandFailure, CommandFailureStage as Stage, CommandIoErrorKind, CommandOutputCollection,
|
||||||
|
};
|
||||||
|
for stage in [
|
||||||
|
Stage::StdoutRead,
|
||||||
|
Stage::StderrRead,
|
||||||
|
Stage::StdoutDrain,
|
||||||
|
Stage::StderrDrain,
|
||||||
|
Stage::OutputDelivery,
|
||||||
|
] {
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
let failure = CommandFailure {
|
||||||
|
stage,
|
||||||
|
reason: CommandIoErrorKind::PermissionDenied,
|
||||||
|
os_code: None,
|
||||||
|
};
|
||||||
|
let collection = CommandOutputCollection::Incomplete { failure };
|
||||||
|
let session: WorkdirSessionHandle = Arc::new(ExternalValidatingSession {
|
||||||
|
inner: LocalWorkdirSession::new(
|
||||||
|
Scope::writable(root.path()).unwrap(),
|
||||||
|
root.path().into(),
|
||||||
|
),
|
||||||
|
truncate_first_binary_response: false,
|
||||||
|
mutate_after_first_binary_response: None,
|
||||||
|
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
||||||
|
lose_terminal_only: false,
|
||||||
|
collection_override: Some(collection),
|
||||||
|
});
|
||||||
|
let view = tools::bash_output::BashOutputView::new();
|
||||||
|
let (_, bash) = tools::bash_tool(session.clone(), view.clone())();
|
||||||
|
let out = call(
|
||||||
|
&bash,
|
||||||
|
json!({"cwd":".","command":"printf partial; printf err >&2"}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(out.summary.ends_with("(output incomplete)"));
|
||||||
|
let body: serde_json::Value = serde_json::from_str(&out.content.unwrap()).unwrap();
|
||||||
|
let metadata = &body["output"];
|
||||||
|
assert_eq!(body["stdout_preview"], "partial");
|
||||||
|
assert_eq!(body["stderr_preview"], "err");
|
||||||
|
assert_eq!(body["preview_omitted"], false);
|
||||||
|
assert_eq!(metadata["terminal"]["exit_code"], 0);
|
||||||
|
assert_eq!(metadata["terminal"]["status"], "completed");
|
||||||
|
assert_eq!(metadata["terminal"]["transport_complete"], false);
|
||||||
|
assert_eq!(
|
||||||
|
metadata["terminal"]["output_collection"],
|
||||||
|
serde_json::to_value(collection).unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(metadata["storage_truncated"], false);
|
||||||
|
assert_eq!(metadata["transport_truncation"], serde_json::Value::Null);
|
||||||
|
assert_eq!(metadata["stdout_seen"], 7);
|
||||||
|
assert_eq!(metadata["stdout_delivered"], 7);
|
||||||
|
let id = metadata["id"].as_str().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(view.metadata(id).unwrap()).unwrap(),
|
||||||
|
*metadata
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
view.read(id, workdir::CommandStream::Stdout, 0, 100)
|
||||||
|
.unwrap(),
|
||||||
|
b"partial"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
view.read(id, workdir::CommandStream::Stderr, 0, 100)
|
||||||
|
.unwrap(),
|
||||||
|
b"err"
|
||||||
|
);
|
||||||
|
assert!(!metadata.to_string().contains(root.path().to_str().unwrap()));
|
||||||
|
session.close().await.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -785,6 +785,12 @@ mod tests {
|
|||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct PendingRuntimeBackend {
|
struct PendingRuntimeBackend {
|
||||||
operations: std::sync::Mutex<Vec<worker_runtime::execution::WorkerLifecycleOperationId>>,
|
operations: std::sync::Mutex<Vec<worker_runtime::execution::WorkerLifecycleOperationId>>,
|
||||||
|
owners: std::sync::Mutex<
|
||||||
|
std::collections::HashMap<
|
||||||
|
worker_runtime::execution::WorkerLifecycleOperationId,
|
||||||
|
worker_runtime::identity::WorkerRef,
|
||||||
|
>,
|
||||||
|
>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl worker_runtime::execution::WorkerExecutionBackend for PendingRuntimeBackend {
|
impl worker_runtime::execution::WorkerExecutionBackend for PendingRuntimeBackend {
|
||||||
@@ -806,6 +812,21 @@ mod tests {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn restore_recovery_disposition(
|
||||||
|
&self,
|
||||||
|
operation_id: worker_runtime::execution::WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
) -> Option<worker_runtime::execution::RestoreRecoveryDisposition> {
|
||||||
|
use worker_runtime::execution::RestoreRecoveryDisposition as D;
|
||||||
|
Some(
|
||||||
|
if self.owners.lock().unwrap().get(&operation_id) == Some(worker_ref) {
|
||||||
|
D::Continue
|
||||||
|
} else {
|
||||||
|
D::Absent
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn restore_worker(
|
fn restore_worker(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
@@ -813,6 +834,10 @@ mod tests {
|
|||||||
use worker_runtime::execution::*;
|
use worker_runtime::execution::*;
|
||||||
let mut operations = self.operations.lock().unwrap();
|
let mut operations = self.operations.lock().unwrap();
|
||||||
operations.push(request.operation_id);
|
operations.push(request.operation_id);
|
||||||
|
self.owners
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.insert(request.operation_id, request.worker_ref.clone());
|
||||||
if operations.len() == 1 {
|
if operations.len() == 1 {
|
||||||
WorkerExecutionSpawnResult::ReconciliationRequired {
|
WorkerExecutionSpawnResult::ReconciliationRequired {
|
||||||
result: WorkerExecutionResult::errored(
|
result: WorkerExecutionResult::errored(
|
||||||
@@ -1119,39 +1144,45 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn rejected_resume_does_not_observe_or_open_retained_state() {
|
async fn settled_nonaccepted_resume_does_not_retry_observe_or_open_retained_state() {
|
||||||
let lifecycle = FakeBackendWorkerLifecycle {
|
for state in [
|
||||||
restore_response: restore_response(BackendWorkerRestoreState::Rejected),
|
BackendWorkerRestoreState::Rejected,
|
||||||
observation: session_observation(serde_json::json!({
|
BackendWorkerRestoreState::RolledBack,
|
||||||
"availability": "unavailable",
|
BackendWorkerRestoreState::Superseded,
|
||||||
"reason": "retention_missing",
|
] {
|
||||||
"message": "must not be opened"
|
let lifecycle = FakeBackendWorkerLifecycle {
|
||||||
})),
|
restore_response: restore_response(state),
|
||||||
calls: std::sync::Mutex::new(Vec::new()),
|
observation: session_observation(serde_json::json!({
|
||||||
};
|
"availability": "unavailable",
|
||||||
let target = client::BackendRuntimeTarget::new(
|
"reason": "retention_missing",
|
||||||
"http://127.0.0.1:3000",
|
"message": "must not be opened"
|
||||||
"workspace-a",
|
})),
|
||||||
"runtime-a",
|
calls: std::sync::Mutex::new(Vec::new()),
|
||||||
"worker-a",
|
};
|
||||||
);
|
let target = client::BackendRuntimeTarget::new(
|
||||||
|
"http://127.0.0.1:3000",
|
||||||
|
"workspace-a",
|
||||||
|
"runtime-a",
|
||||||
|
"worker-a",
|
||||||
|
);
|
||||||
|
|
||||||
let error = prepare_selected_worker(
|
let error = prepare_selected_worker(
|
||||||
&lifecycle,
|
&lifecycle,
|
||||||
target,
|
target,
|
||||||
BackendWorkerPickerIntent::Resume,
|
BackendWorkerPickerIntent::Resume,
|
||||||
Some(client::BackendWorkerRestoreRequest {
|
Some(client::BackendWorkerRestoreRequest {
|
||||||
expected_observation_token: "observed-worker-state".into(),
|
expected_observation_token: "observed-worker-state".into(),
|
||||||
request_id: "resume-operation".into(),
|
request_id: "resume-operation".into(),
|
||||||
}),
|
}),
|
||||||
|_| Ok(false),
|
|_| panic!("settled nonaccepted restore must not offer recovery retry"),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.unwrap_err()
|
.unwrap_err()
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
assert!(error.contains("Rejected"));
|
assert!(error.contains(&format!("{state:?}")));
|
||||||
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
|
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1176,6 +1207,7 @@ mod tests {
|
|||||||
for state in [
|
for state in [
|
||||||
BackendWorkerRestoreState::Rejected,
|
BackendWorkerRestoreState::Rejected,
|
||||||
BackendWorkerRestoreState::RolledBack,
|
BackendWorkerRestoreState::RolledBack,
|
||||||
|
BackendWorkerRestoreState::Superseded,
|
||||||
BackendWorkerRestoreState::ReconciliationRequired,
|
BackendWorkerRestoreState::ReconciliationRequired,
|
||||||
] {
|
] {
|
||||||
let mut response = restore_response(state);
|
let mut response = restore_response(state);
|
||||||
|
|||||||
@@ -23,5 +23,6 @@ thiserror.workspace = true
|
|||||||
tokio = { workspace = true, features = ["io-util", "process", "rt", "sync", "time"] }
|
tokio = { workspace = true, features = ["io-util", "process", "rt", "sync", "time"] }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
axum.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
tempfile.workspace = true
|
tempfile.workspace = true
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
//! hashes alone do not accept same-content inode replacements. Callers own Read line numbering
|
//! hashes alone do not accept same-content inode replacements. Callers own Read line numbering
|
||||||
//! and per-attachment read history.
|
//! and per-attachment read history.
|
||||||
//!
|
//!
|
||||||
//! Create paths are checkout-root-relative and must be strict descendants of the observed target
|
//! Create paths use the target's session coordinates and must be strict descendants of the observed target
|
||||||
//! directory. Existing destinations are never overwritten; results return the updated observation
|
//! directory. Existing destinations are never overwritten; results return the updated observation
|
||||||
//! of the retained target directory, not the created file. Read results also return the validator
|
//! of the retained target directory, not the created file. Read results also return the validator
|
||||||
//! of the state actually read, never a replacement from a later independent stat.
|
//! of the state actually read, never a replacement from a later independent stat.
|
||||||
@@ -18,7 +18,8 @@
|
|||||||
//!
|
//!
|
||||||
//! Search delegates to shared List/Glob/Grep engines. Scope rules form unions within layers and
|
//! Search delegates to shared List/Glob/Grep engines. Scope rules form unions within layers and
|
||||||
//! intersections across layers, provider authority, and output roots. Wrappers transform coordinates
|
//! intersections across layers, provider authority, and output roots. Wrappers transform coordinates
|
||||||
//! once; returned paths and Grep content are relative to the final session's logical root.
|
//! once; returned paths and Grep content are relative to the composed output root (the caller's
|
||||||
|
//! cwd by default).
|
||||||
//! Paths are not reverse-parsed from rendered filenames; non-UTF-8 names are not lossily aliased.
|
//! Paths are not reverse-parsed from rendered filenames; non-UTF-8 names are not lossily aliased.
|
||||||
//! Unsupported providers reject search rather than fall back to broader trait searches. Search is
|
//! Unsupported providers reject search rather than fall back to broader trait searches. Search is
|
||||||
//! live traversal, not an atomic snapshot of all descendants or exclusion of external editors.
|
//! live traversal, not an atomic snapshot of all descendants or exclusion of external editors.
|
||||||
@@ -556,15 +557,13 @@ mod tests {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn checkout_hardlink_save_isolates_alias_and_rejects_symlinks_special_files() {
|
async fn checkout_hardlink_save_isolates_alias_and_rejects_symlinks() {
|
||||||
let root = tempfile::tempdir().unwrap();
|
let root = tempfile::tempdir().unwrap();
|
||||||
std::fs::write(root.path().join("a"), "original").unwrap();
|
std::fs::write(root.path().join("a"), "original").unwrap();
|
||||||
std::fs::hard_link(root.path().join("a"), root.path().join("alias")).unwrap();
|
std::fs::hard_link(root.path().join("a"), root.path().join("alias")).unwrap();
|
||||||
std::os::unix::fs::symlink("a", root.path().join("link")).unwrap();
|
std::os::unix::fs::symlink("a", root.path().join("link")).unwrap();
|
||||||
let _socket = std::os::unix::net::UnixListener::bind(root.path().join("socket")).unwrap();
|
|
||||||
let session = session(root.path());
|
let session = session(root.path());
|
||||||
assert!(session.checkout_observe(path("link")).await.is_err());
|
assert!(session.checkout_observe(path("link")).await.is_err());
|
||||||
assert!(session.checkout_observe(path("socket")).await.is_err());
|
|
||||||
let read = checked_read(&session, session.checkout_observe(path("a")).await.unwrap()).await;
|
let read = checked_read(&session, session.checkout_observe(path("a")).await.unwrap()).await;
|
||||||
session
|
session
|
||||||
.checkout_execute(CheckoutRequest {
|
.checkout_execute(CheckoutRequest {
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
//! Model-facing session coordinates, separate from provider paths and authority.
|
||||||
|
|
||||||
|
use crate::{WorkdirPath, WorkdirSessionCapabilities, WorkdirToolScopeRule};
|
||||||
|
use schemars::JsonSchema;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// A snapshot of the environment actually used by a session's operations.
|
||||||
|
///
|
||||||
|
/// Cwd is a logical Workdir-root coordinate. Scope targets use that root or
|
||||||
|
/// explicit absolute provider paths. Scope is an additional allow ceiling, not
|
||||||
|
/// a replacement for provider checks. Providers
|
||||||
|
/// may retain further restrictions (including deny rules); this projection
|
||||||
|
/// never authorizes an operation.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct WorkdirSessionEnvironment {
|
||||||
|
#[serde(deserialize_with = "deserialize_cwd")]
|
||||||
|
pub cwd: WorkdirPath,
|
||||||
|
pub scope: Option<Vec<WorkdirToolScopeRule>>,
|
||||||
|
pub capabilities: WorkdirSessionCapabilities,
|
||||||
|
pub restrictions: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn deserialize_cwd<'de, D: serde::Deserializer<'de>>(
|
||||||
|
deserializer: D,
|
||||||
|
) -> Result<WorkdirPath, D::Error> {
|
||||||
|
let path = String::deserialize(deserializer)?;
|
||||||
|
WorkdirPath::new(path).map_err(serde::de::Error::custom)
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkdirSessionEnvironment {
|
||||||
|
pub fn root(capabilities: WorkdirSessionCapabilities) -> Self {
|
||||||
|
Self {
|
||||||
|
cwd: WorkdirPath::root(),
|
||||||
|
scope: None,
|
||||||
|
capabilities,
|
||||||
|
restrictions: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn wire_cwd_is_normalized_and_cannot_claim_an_absolute_or_parent_base() {
|
||||||
|
let mut wire = serde_json::to_value(WorkdirSessionEnvironment::root(
|
||||||
|
WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
for invalid in [
|
||||||
|
"/absolute",
|
||||||
|
"../parent",
|
||||||
|
"nested/../parent",
|
||||||
|
"nested\\other",
|
||||||
|
] {
|
||||||
|
wire["cwd"] = invalid.into();
|
||||||
|
assert!(
|
||||||
|
serde_json::from_value::<WorkdirSessionEnvironment>(wire.clone()).is_err(),
|
||||||
|
"{invalid}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
wire["cwd"] = "./nested//child".into();
|
||||||
|
let decoded: WorkdirSessionEnvironment = serde_json::from_value(wire).unwrap();
|
||||||
|
assert_eq!(decoded.cwd.as_str(), "nested/child");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,175 @@
|
|||||||
|
//! Real Local/scoped/read-only/router sessions across loopback HTTP. Only the
|
||||||
|
//! HTTP authentication boundary is a fixture; coordinate and scope checks are
|
||||||
|
//! production implementations. Runs in the normal http-client library suite.
|
||||||
|
#![cfg(all(feature = "http-client", target_os = "linux"))]
|
||||||
|
|
||||||
|
use crate::http::*;
|
||||||
|
use crate::*;
|
||||||
|
use axum::{
|
||||||
|
Json, Router,
|
||||||
|
extract::State,
|
||||||
|
routing::{delete, post},
|
||||||
|
};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
struct Server(tokio::task::JoinHandle<()>);
|
||||||
|
impl Drop for Server {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn open(State(session): State<WorkdirSessionHandle>) -> Json<OpenWorkdirSessionResponse> {
|
||||||
|
Json(OpenWorkdirSessionResponse {
|
||||||
|
session_id: WorkdirSessionId::new("environment-session").unwrap(),
|
||||||
|
workdir_id: session.workdir().id().clone(),
|
||||||
|
capabilities: session.capabilities(),
|
||||||
|
environment: session.environment(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
async fn operation(
|
||||||
|
State(session): State<WorkdirSessionHandle>,
|
||||||
|
Json(request): Json<WorkdirSessionOperationRequest>,
|
||||||
|
) -> Result<
|
||||||
|
Json<WorkdirSessionOperationResult>,
|
||||||
|
(axum::http::StatusCode, Json<WorkdirTransportError>),
|
||||||
|
> {
|
||||||
|
dispatch_workdir_session_operation(session.as_ref(), request.operation)
|
||||||
|
.await
|
||||||
|
.map(Json)
|
||||||
|
.map_err(|error| {
|
||||||
|
(
|
||||||
|
axum::http::StatusCode::FORBIDDEN,
|
||||||
|
Json(WorkdirTransportError::from_workdir_error(&error)),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn remote_environment_matches_scoped_readonly_operations_without_rebasing_twice() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
std::fs::create_dir(root.path().join("nested")).unwrap();
|
||||||
|
std::fs::write(root.path().join("nested/file.txt"), "nested marker").unwrap();
|
||||||
|
std::fs::write(root.path().join("file.txt"), "root decoy").unwrap();
|
||||||
|
let broker = WorkdirToolBroker::new(Arc::new(LocalWorkdirSession::new(
|
||||||
|
manifest::Scope::writable(root.path()).unwrap(),
|
||||||
|
root.path().to_path_buf(),
|
||||||
|
)));
|
||||||
|
let lease = broker
|
||||||
|
.scope(WorkdirToolScope {
|
||||||
|
cwd: WorkdirPath::new("nested").unwrap(),
|
||||||
|
rules: vec![WorkdirToolScopeRule {
|
||||||
|
target: WorkdirPath::new("nested").unwrap(),
|
||||||
|
permission: WorkdirToolScopePermission::Write,
|
||||||
|
recursive: true,
|
||||||
|
symlink_policy: Default::default(),
|
||||||
|
}],
|
||||||
|
command: true,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let session: WorkdirSessionHandle = Arc::new(ReadOnlyWorkdirSession::new(lease.tool_session()));
|
||||||
|
let expected = session.environment();
|
||||||
|
let app = Router::new()
|
||||||
|
.route("/v1/working-directories/{id}/sessions", post(open))
|
||||||
|
.route("/v1/workdir-sessions/{id}/operations", post(operation))
|
||||||
|
.route(
|
||||||
|
"/v1/workdir-sessions/{id}",
|
||||||
|
delete(|| async { axum::http::StatusCode::NO_CONTENT }),
|
||||||
|
)
|
||||||
|
.with_state(session.clone());
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
let _server = Server(tokio::spawn(async move {
|
||||||
|
axum::serve(listener, app).await.unwrap();
|
||||||
|
}));
|
||||||
|
let remote: WorkdirSessionHandle = Arc::new(
|
||||||
|
RemoteWorkdirSession::open(
|
||||||
|
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||||
|
format!("http://{address}").parse().unwrap(),
|
||||||
|
"fixture",
|
||||||
|
session.workdir().id().clone(),
|
||||||
|
OpenWorkdirSessionRequest::default(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let router = WorkdirSessionRouter::new();
|
||||||
|
router
|
||||||
|
.attach(WorkdirAttachmentAlias::new("remote").unwrap(), remote)
|
||||||
|
.unwrap();
|
||||||
|
let selected = router.resolve(Some("remote")).unwrap().session;
|
||||||
|
assert_eq!(selected.environment(), expected);
|
||||||
|
assert_eq!(expected.cwd, WorkdirPath::new("nested").unwrap());
|
||||||
|
assert!(
|
||||||
|
!expected
|
||||||
|
.capabilities
|
||||||
|
.supports(WorkdirSessionCapability::Command)
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
expected
|
||||||
|
.scope
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.all(|rule| rule.permission == WorkdirToolScopePermission::Read)
|
||||||
|
);
|
||||||
|
let found = selected
|
||||||
|
.glob(GlobRequest {
|
||||||
|
path: WorkdirPath::root(),
|
||||||
|
pattern: "*.txt".into(),
|
||||||
|
limit: 10,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(found.paths, vec![WorkdirPath::new("file.txt").unwrap()]);
|
||||||
|
let path = found.paths[0].clone();
|
||||||
|
let read = selected
|
||||||
|
.read(ReadRequest {
|
||||||
|
path: path.clone(),
|
||||||
|
offset: 0,
|
||||||
|
limit: 10,
|
||||||
|
max_bytes: 100,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(read.path, path);
|
||||||
|
assert_eq!(read.bytes, b"nested marker");
|
||||||
|
assert!(
|
||||||
|
selected
|
||||||
|
.write(WriteRequest {
|
||||||
|
path,
|
||||||
|
content: b"denied".to_vec(),
|
||||||
|
expected_hash: Some(read.content_hash)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
selected
|
||||||
|
.read(ReadRequest {
|
||||||
|
path: WorkdirPath::new_scoped(root.path().join("file.txt").to_str().unwrap())
|
||||||
|
.unwrap(),
|
||||||
|
offset: 0,
|
||||||
|
limit: 10,
|
||||||
|
max_bytes: 100
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
selected
|
||||||
|
.start_command(CommandRequest {
|
||||||
|
command: "touch escaped".into(),
|
||||||
|
cwd: WorkdirPath::root(),
|
||||||
|
timeout_secs: 5,
|
||||||
|
transport_byte_budget: 100,
|
||||||
|
tool_call_id: None
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(!root.path().join("nested/escaped").exists());
|
||||||
|
router.close_all().await.unwrap();
|
||||||
|
lease.close().await.unwrap();
|
||||||
|
broker.close().await.unwrap();
|
||||||
|
}
|
||||||
@@ -924,6 +924,8 @@ mod tests {
|
|||||||
chunks: vec![],
|
chunks: vec![],
|
||||||
next_cursor: 0,
|
next_cursor: 0,
|
||||||
terminal: Some(crate::CommandTerminal {
|
terminal: Some(crate::CommandTerminal {
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: crate::CommandOutputCollection::Complete,
|
||||||
status: crate::CommandStatus::Completed,
|
status: crate::CommandStatus::Completed,
|
||||||
exit_code: Some(0),
|
exit_code: Some(0),
|
||||||
stdout_bytes: 0,
|
stdout_bytes: 0,
|
||||||
@@ -1082,6 +1084,8 @@ mod tests {
|
|||||||
}],
|
}],
|
||||||
next_cursor: 3,
|
next_cursor: 3,
|
||||||
terminal: Some(crate::CommandTerminal {
|
terminal: Some(crate::CommandTerminal {
|
||||||
|
process_failure: None,
|
||||||
|
output_collection: crate::CommandOutputCollection::Complete,
|
||||||
status: crate::CommandStatus::Completed,
|
status: crate::CommandStatus::Completed,
|
||||||
exit_code: Some(0),
|
exit_code: Some(0),
|
||||||
stdout_bytes: 100 * 1024 * 1024,
|
stdout_bytes: 100 * 1024 * 1024,
|
||||||
|
|||||||
+113
-9
@@ -54,6 +54,7 @@ pub struct OpenWorkdirSessionResponse {
|
|||||||
pub session_id: WorkdirSessionId,
|
pub session_id: WorkdirSessionId,
|
||||||
pub workdir_id: WorkdirId,
|
pub workdir_id: WorkdirId,
|
||||||
pub capabilities: WorkdirSessionCapabilities,
|
pub capabilities: WorkdirSessionCapabilities,
|
||||||
|
pub environment: crate::WorkdirSessionEnvironment,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One provider-side Workdir operation.
|
/// One provider-side Workdir operation.
|
||||||
@@ -119,6 +120,51 @@ pub enum WorkdirSessionOperationResult {
|
|||||||
CommandCancel,
|
CommandCancel,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Audit persistence is independent of the provider result and never authorizes a resend.
|
||||||
|
/// Only server-generated correlation and closed classifications cross this boundary.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct WorkdirAuditFailure {
|
||||||
|
pub operation_id: String,
|
||||||
|
pub stage: WorkdirAuditStage,
|
||||||
|
pub operation_outcome: WorkdirAuditOperationOutcome,
|
||||||
|
pub reason: WorkdirAuditFailureReason,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum WorkdirAuditStage {
|
||||||
|
BeforeDispatch,
|
||||||
|
DispatchRejected,
|
||||||
|
ResponseReceived,
|
||||||
|
ResponseUnknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||||
|
pub enum WorkdirAuditOperationOutcome {
|
||||||
|
NotDispatched,
|
||||||
|
Succeeded,
|
||||||
|
Failed { code: WorkdirTransportErrorCode },
|
||||||
|
Unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum WorkdirAuditFailureReason {
|
||||||
|
AuditStoreUnavailable,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A successful operation can still have failed audit persistence. Consumers must
|
||||||
|
/// surface that degradation separately, without discarding the result or resending.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct WorkdirSessionOperationResponse {
|
||||||
|
#[serde(flatten)]
|
||||||
|
pub result: WorkdirSessionOperationResult,
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
pub audit_failures: Vec<WorkdirAuditFailure>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Dispatch one typed operation against any provider-neutral Workdir session.
|
/// Dispatch one typed operation against any provider-neutral Workdir session.
|
||||||
///
|
///
|
||||||
/// Runtime HTTP handlers, outbound External providers, and Backend brokers can
|
/// Runtime HTTP handlers, outbound External providers, and Backend brokers can
|
||||||
@@ -208,6 +254,7 @@ pub enum WorkdirTransportErrorCode {
|
|||||||
NotFound,
|
NotFound,
|
||||||
Conflict,
|
Conflict,
|
||||||
OutcomeUnknown,
|
OutcomeUnknown,
|
||||||
|
NotDispatched,
|
||||||
Unsupported,
|
Unsupported,
|
||||||
InvalidRequest,
|
InvalidRequest,
|
||||||
Denied,
|
Denied,
|
||||||
@@ -231,6 +278,7 @@ impl WorkdirTransportErrorCode {
|
|||||||
Self::NotFound => "not_found",
|
Self::NotFound => "not_found",
|
||||||
Self::Conflict => "conflict",
|
Self::Conflict => "conflict",
|
||||||
Self::OutcomeUnknown => "outcome_unknown",
|
Self::OutcomeUnknown => "outcome_unknown",
|
||||||
|
Self::NotDispatched => "not_dispatched",
|
||||||
Self::Unsupported => "unsupported",
|
Self::Unsupported => "unsupported",
|
||||||
Self::InvalidRequest => "invalid_request",
|
Self::InvalidRequest => "invalid_request",
|
||||||
Self::Denied => "denied",
|
Self::Denied => "denied",
|
||||||
@@ -261,7 +309,7 @@ impl WorkdirTransportErrorCode {
|
|||||||
| Self::SymlinkTargetIsDirectory
|
| Self::SymlinkTargetIsDirectory
|
||||||
| Self::IsDirectory
|
| Self::IsDirectory
|
||||||
| Self::SymlinkDirectoryNotTraversed => 400,
|
| Self::SymlinkDirectoryNotTraversed => 400,
|
||||||
Self::Unavailable => 503,
|
Self::Unavailable | Self::NotDispatched => 503,
|
||||||
Self::Io | Self::Internal | Self::OutcomeUnknown => 500,
|
Self::Io | Self::Internal | Self::OutcomeUnknown => 500,
|
||||||
Self::Transport => 502,
|
Self::Transport => 502,
|
||||||
}
|
}
|
||||||
@@ -298,6 +346,9 @@ impl WorkdirTransportError {
|
|||||||
Code::OutcomeUnknown,
|
Code::OutcomeUnknown,
|
||||||
"Workdir operation may have effects; inspect before retrying",
|
"Workdir operation may have effects; inspect before retrying",
|
||||||
),
|
),
|
||||||
|
WorkdirError::NotDispatched => {
|
||||||
|
(Code::NotDispatched, "Workdir operation was not dispatched")
|
||||||
|
}
|
||||||
WorkdirError::NotFound(_) => (Code::NotFound, "Workdir path was not found"),
|
WorkdirError::NotFound(_) => (Code::NotFound, "Workdir path was not found"),
|
||||||
WorkdirError::Conflict(_) => (
|
WorkdirError::Conflict(_) => (
|
||||||
Code::Conflict,
|
Code::Conflict,
|
||||||
@@ -375,12 +426,14 @@ impl WorkdirTransportError {
|
|||||||
use WorkdirTransportErrorCode as Code;
|
use WorkdirTransportErrorCode as Code;
|
||||||
let error = match self.code {
|
let error = match self.code {
|
||||||
Code::NotFound => WorkdirError::NotFound("<remote>".into()),
|
Code::NotFound => WorkdirError::NotFound("<remote>".into()),
|
||||||
Code::Conflict => WorkdirError::Conflict(self.message),
|
Code::Conflict => WorkdirError::Conflict(
|
||||||
|
"The target file's content or existence changed since it was last observed; read the file again before retrying".into(),
|
||||||
|
),
|
||||||
Code::OutcomeUnknown => WorkdirError::OutcomeUnknown(
|
Code::OutcomeUnknown => WorkdirError::OutcomeUnknown(
|
||||||
"Workdir operation may have effects; inspect before retrying".into(),
|
"Workdir operation may have effects; inspect before retrying".into(),
|
||||||
),
|
),
|
||||||
Code::Unsupported => WorkdirError::UnsupportedOperation(self.message),
|
Code::Unsupported => WorkdirError::UnsupportedOperation("Workdir operation is not supported".into()),
|
||||||
Code::InvalidRequest => WorkdirError::InvalidArgument(self.message),
|
Code::InvalidRequest => WorkdirError::InvalidArgument("Workdir operation request is invalid".into()),
|
||||||
Code::Denied => WorkdirError::Denied(crate::WorkdirDenial {
|
Code::Denied => WorkdirError::Denied(crate::WorkdirDenial {
|
||||||
reason: self.denial_reason,
|
reason: self.denial_reason,
|
||||||
// Provider-authored text is not trusted diagnostic data.
|
// Provider-authored text is not trusted diagnostic data.
|
||||||
@@ -409,12 +462,13 @@ impl WorkdirTransportError {
|
|||||||
target: "<remote-target>".into(),
|
target: "<remote-target>".into(),
|
||||||
},
|
},
|
||||||
Code::UnknownCommand => WorkdirError::UnknownCommand("<remote>".to_string()),
|
Code::UnknownCommand => WorkdirError::UnknownCommand("<remote>".to_string()),
|
||||||
Code::Unavailable => WorkdirError::Unavailable(self.message),
|
Code::Unavailable => WorkdirError::Unavailable("Workdir session is unavailable".into()),
|
||||||
|
Code::NotDispatched => WorkdirError::NotDispatched,
|
||||||
Code::Io => WorkdirError::Io {
|
Code::Io => WorkdirError::Io {
|
||||||
path: "<remote>".into(),
|
path: "<remote>".into(),
|
||||||
source: std::io::Error::other(self.message),
|
source: std::io::Error::other("Workdir I/O operation failed"),
|
||||||
},
|
},
|
||||||
Code::Transport => WorkdirError::Transport(self.message),
|
Code::Transport => WorkdirError::Transport("Workdir transport failed".into()),
|
||||||
Code::Internal => WorkdirError::OperationFailed,
|
Code::Internal => WorkdirError::OperationFailed,
|
||||||
};
|
};
|
||||||
let error = if let Some(reason) = self.failure_reason {
|
let error = if let Some(reason) = self.failure_reason {
|
||||||
@@ -500,6 +554,7 @@ mod client {
|
|||||||
workdir: Workdir,
|
workdir: Workdir,
|
||||||
session_id: WorkdirSessionId,
|
session_id: WorkdirSessionId,
|
||||||
capabilities: WorkdirSessionCapabilities,
|
capabilities: WorkdirSessionCapabilities,
|
||||||
|
environment: crate::WorkdirSessionEnvironment,
|
||||||
closed: AtomicBool,
|
closed: AtomicBool,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -549,7 +604,8 @@ mod client {
|
|||||||
if opened.workdir_id.as_str() != workdir_id.as_str() {
|
if opened.workdir_id.as_str() != workdir_id.as_str() {
|
||||||
return Err(WorkdirError::Unavailable(
|
return Err(WorkdirError::Unavailable(
|
||||||
"Runtime opened a session for a different Workdir".to_string(),
|
"Runtime opened a session for a different Workdir".to_string(),
|
||||||
));
|
)
|
||||||
|
.with_failure_reason(crate::WorkdirFailureReason::SessionIdentityMismatch));
|
||||||
}
|
}
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
client,
|
client,
|
||||||
@@ -558,6 +614,7 @@ mod client {
|
|||||||
workdir: Workdir::new(opened.workdir_id.as_str()),
|
workdir: Workdir::new(opened.workdir_id.as_str()),
|
||||||
session_id: opened.session_id,
|
session_id: opened.session_id,
|
||||||
capabilities: opened.capabilities,
|
capabilities: opened.capabilities,
|
||||||
|
environment: opened.environment,
|
||||||
closed: AtomicBool::new(false),
|
closed: AtomicBool::new(false),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -608,6 +665,7 @@ mod client {
|
|||||||
WorkdirError::Unavailable(format!(
|
WorkdirError::Unavailable(format!(
|
||||||
"Runtime returned a mismatched Workdir operation result; expected {expected}"
|
"Runtime returned a mismatched Workdir operation result; expected {expected}"
|
||||||
))
|
))
|
||||||
|
.with_failure_reason(crate::WorkdirFailureReason::OperationResultMismatch)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -621,6 +679,12 @@ mod client {
|
|||||||
self.capabilities
|
self.capabilities
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn environment(&self) -> crate::WorkdirSessionEnvironment {
|
||||||
|
let mut environment = self.environment.clone();
|
||||||
|
environment.capabilities = self.capabilities();
|
||||||
|
environment
|
||||||
|
}
|
||||||
|
|
||||||
async fn authorize_scope_path(
|
async fn authorize_scope_path(
|
||||||
&self,
|
&self,
|
||||||
request: WorkdirScopeAuthorizationRequest,
|
request: WorkdirScopeAuthorizationRequest,
|
||||||
@@ -770,7 +834,8 @@ mod client {
|
|||||||
}
|
}
|
||||||
Ok(_) if mutation => Err(WorkdirError::OutcomeUnknown(
|
Ok(_) if mutation => Err(WorkdirError::OutcomeUnknown(
|
||||||
"mismatched response after checkout mutation".into(),
|
"mismatched response after checkout mutation".into(),
|
||||||
)),
|
)
|
||||||
|
.with_failure_reason(crate::WorkdirFailureReason::OperationResultMismatch)),
|
||||||
Ok(_) => Err(Self::mismatch("checkout_execute")),
|
Ok(_) => Err(Self::mismatch("checkout_execute")),
|
||||||
Err(error)
|
Err(error)
|
||||||
if mutation
|
if mutation
|
||||||
@@ -1153,6 +1218,9 @@ mod client {
|
|||||||
workdir,
|
workdir,
|
||||||
session_id: WorkdirSessionId::new("s").unwrap(),
|
session_id: WorkdirSessionId::new("s").unwrap(),
|
||||||
capabilities: WorkdirSessionCapabilities::EMPTY,
|
capabilities: WorkdirSessionCapabilities::EMPTY,
|
||||||
|
environment: crate::WorkdirSessionEnvironment::root(
|
||||||
|
WorkdirSessionCapabilities::EMPTY,
|
||||||
|
),
|
||||||
closed: AtomicBool::new(false),
|
closed: AtomicBool::new(false),
|
||||||
};
|
};
|
||||||
let stat = StatRequest {
|
let stat = StatRequest {
|
||||||
@@ -1210,6 +1278,37 @@ pub use client::{
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn operation_response_keeps_success_and_independent_audit_failure() {
|
||||||
|
let original =
|
||||||
|
WorkdirSessionOperationResult::CommandStart(crate::CommandHandle("command-a".into()));
|
||||||
|
let response = WorkdirSessionOperationResponse {
|
||||||
|
result: original.clone(),
|
||||||
|
audit_failures: vec![WorkdirAuditFailure {
|
||||||
|
operation_id: "op-00000000-0000-7000-8000-000000000001".into(),
|
||||||
|
stage: WorkdirAuditStage::ResponseReceived,
|
||||||
|
operation_outcome: WorkdirAuditOperationOutcome::Succeeded,
|
||||||
|
reason: WorkdirAuditFailureReason::AuditStoreUnavailable,
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
let json = serde_json::to_value(&response).unwrap();
|
||||||
|
assert_eq!(json["operation"], "command_start");
|
||||||
|
assert_eq!(json["result"], "command-a");
|
||||||
|
assert_eq!(
|
||||||
|
json["audit_failures"][0]["operation_outcome"]["kind"],
|
||||||
|
"succeeded"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::from_value::<WorkdirSessionOperationResponse>(json).unwrap(),
|
||||||
|
response
|
||||||
|
);
|
||||||
|
let normal = serde_json::to_value(&original).unwrap();
|
||||||
|
let decoded: WorkdirSessionOperationResponse =
|
||||||
|
serde_json::from_value(normal.clone()).unwrap();
|
||||||
|
assert!(decoded.audit_failures.is_empty());
|
||||||
|
assert_eq!(serde_json::to_value(decoded).unwrap(), normal);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn transport_error_round_trip_keeps_public_classification() {
|
fn transport_error_round_trip_keeps_public_classification() {
|
||||||
for (code, expected_status, expected_error) in [
|
for (code, expected_status, expected_error) in [
|
||||||
@@ -1229,6 +1328,11 @@ mod tests {
|
|||||||
409,
|
409,
|
||||||
"The target file's content or existence changed since it was last observed",
|
"The target file's content or existence changed since it was last observed",
|
||||||
),
|
),
|
||||||
|
(
|
||||||
|
WorkdirTransportErrorCode::NotDispatched,
|
||||||
|
503,
|
||||||
|
"not dispatched",
|
||||||
|
),
|
||||||
(WorkdirTransportErrorCode::Unsupported, 400, "unsupported"),
|
(WorkdirTransportErrorCode::Unsupported, 400, "unsupported"),
|
||||||
(WorkdirTransportErrorCode::Denied, 403, "denied"),
|
(WorkdirTransportErrorCode::Denied, 403, "denied"),
|
||||||
(
|
(
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ pub use checkout::{
|
|||||||
CheckoutObservation, CheckoutOperation, CheckoutOutput, CheckoutRequest, CheckoutResult,
|
CheckoutObservation, CheckoutOperation, CheckoutOutput, CheckoutRequest, CheckoutResult,
|
||||||
CheckoutSearchOperation, CheckoutSearchRequest, CheckoutSearchResult,
|
CheckoutSearchOperation, CheckoutSearchRequest, CheckoutSearchResult,
|
||||||
};
|
};
|
||||||
|
mod environment;
|
||||||
|
pub use environment::WorkdirSessionEnvironment;
|
||||||
pub mod external;
|
pub mod external;
|
||||||
pub mod http;
|
pub mod http;
|
||||||
mod local;
|
mod local;
|
||||||
@@ -16,6 +18,8 @@ pub mod workspace;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod checkout_search_tests;
|
mod checkout_search_tests;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
mod environment_tests;
|
||||||
|
#[cfg(test)]
|
||||||
mod external_local_tests;
|
mod external_local_tests;
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
@@ -185,6 +189,9 @@ pub type WriteOutcome = WriteResult;
|
|||||||
/// Worker-bound operation session for a persistent [`Workdir`].
|
/// Worker-bound operation session for a persistent [`Workdir`].
|
||||||
///
|
///
|
||||||
/// Filesystem search and commands run on the host owning materialization.
|
/// Filesystem search and commands run on the host owning materialization.
|
||||||
|
/// Relative operation paths and reusable search results use the session's cwd;
|
||||||
|
/// root-coordinate providers are adapted once at attachment, not by each Tool.
|
||||||
|
/// Scope declarations retain their own root-coordinate authority contract.
|
||||||
/// Session closure is terminal but does not delete the Workdir or files.
|
/// Session closure is terminal but does not delete the Workdir or files.
|
||||||
/// Unsupported checkout/search implementations explicitly reject requests, with no ambient
|
/// Unsupported checkout/search implementations explicitly reject requests, with no ambient
|
||||||
/// filesystem fallback. Providers own identity validators; callers own read hash history
|
/// filesystem fallback. Providers own identity validators; callers own read hash history
|
||||||
@@ -194,6 +201,13 @@ pub trait WorkdirSession: std::fmt::Debug + Send + Sync {
|
|||||||
fn workdir(&self) -> &Workdir;
|
fn workdir(&self) -> &Workdir;
|
||||||
fn capabilities(&self) -> WorkdirSessionCapabilities;
|
fn capabilities(&self) -> WorkdirSessionCapabilities;
|
||||||
|
|
||||||
|
/// Describe the coordinates used by this session, not a caller's Profile
|
||||||
|
/// or filesystem authority. Root-coordinate providers use the default;
|
||||||
|
/// coordinate and authority wrappers must project their effective view.
|
||||||
|
fn environment(&self) -> WorkdirSessionEnvironment {
|
||||||
|
WorkdirSessionEnvironment::root(self.capabilities())
|
||||||
|
}
|
||||||
|
|
||||||
/// Validate an attenuated filesystem rule at the provider boundary without
|
/// Validate an attenuated filesystem rule at the provider boundary without
|
||||||
/// exposing the resolved host path. Providers that cannot resolve symbolic
|
/// exposing the resolved host path. Providers that cannot resolve symbolic
|
||||||
/// links must reject resolved-policy checks rather than downgrade them.
|
/// links must reject resolved-policy checks rather than downgrade them.
|
||||||
@@ -408,12 +422,36 @@ impl From<&str> for WorkdirDenial {
|
|||||||
pub enum WorkdirFailureReason {
|
pub enum WorkdirFailureReason {
|
||||||
SessionClosed,
|
SessionClosed,
|
||||||
ProviderUnavailable,
|
ProviderUnavailable,
|
||||||
|
ProviderGrantExpired,
|
||||||
|
ProviderAdmissionClosed,
|
||||||
|
ProviderDispatchRejected,
|
||||||
|
ProviderResponseLost,
|
||||||
|
ProviderOperationTimeout,
|
||||||
CommandTerminalMissing,
|
CommandTerminalMissing,
|
||||||
|
AuditStoreUnavailable,
|
||||||
HttpTimeout,
|
HttpTimeout,
|
||||||
HttpConnect,
|
HttpConnect,
|
||||||
HttpTransfer,
|
HttpTransfer,
|
||||||
HttpResponseInvalid,
|
HttpResponseInvalid,
|
||||||
HttpResponseTooLarge,
|
HttpResponseTooLarge,
|
||||||
|
OperationResultMismatch,
|
||||||
|
SessionIdentityMismatch,
|
||||||
|
SessionOpenTaskFailed,
|
||||||
|
SessionOpenFailed,
|
||||||
|
AuthorizationMissing,
|
||||||
|
AuthorizationStale,
|
||||||
|
AuthorizationUnavailable,
|
||||||
|
SigningIdentityUnavailable,
|
||||||
|
AuthorizationSignFailed,
|
||||||
|
RuntimeStopped,
|
||||||
|
RuntimeStateUnavailable,
|
||||||
|
RuntimeBackendUnavailable,
|
||||||
|
RuntimeOwnerMismatch,
|
||||||
|
RuntimeInvalidRequest,
|
||||||
|
WorkingDirectoryNotFound,
|
||||||
|
WorkingDirectoryNotActive,
|
||||||
|
WorkingDirectoryRecordInvalid,
|
||||||
|
WorkingDirectoryAccessUnavailable,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WorkdirFailureReason {
|
impl WorkdirFailureReason {
|
||||||
@@ -421,12 +459,107 @@ impl WorkdirFailureReason {
|
|||||||
match self {
|
match self {
|
||||||
Self::SessionClosed => "session_closed",
|
Self::SessionClosed => "session_closed",
|
||||||
Self::ProviderUnavailable => "provider_unavailable",
|
Self::ProviderUnavailable => "provider_unavailable",
|
||||||
|
Self::ProviderGrantExpired => "provider_grant_expired",
|
||||||
|
Self::ProviderAdmissionClosed => "provider_admission_closed",
|
||||||
|
Self::ProviderDispatchRejected => "provider_dispatch_rejected",
|
||||||
|
Self::ProviderResponseLost => "provider_response_lost",
|
||||||
|
Self::ProviderOperationTimeout => "provider_operation_timeout",
|
||||||
Self::CommandTerminalMissing => "command_terminal_missing",
|
Self::CommandTerminalMissing => "command_terminal_missing",
|
||||||
|
Self::AuditStoreUnavailable => "audit_store_unavailable",
|
||||||
Self::HttpTimeout => "http_timeout",
|
Self::HttpTimeout => "http_timeout",
|
||||||
Self::HttpConnect => "http_connect",
|
Self::HttpConnect => "http_connect",
|
||||||
Self::HttpTransfer => "http_transfer",
|
Self::HttpTransfer => "http_transfer",
|
||||||
Self::HttpResponseInvalid => "http_response_invalid",
|
Self::HttpResponseInvalid => "http_response_invalid",
|
||||||
Self::HttpResponseTooLarge => "http_response_too_large",
|
Self::HttpResponseTooLarge => "http_response_too_large",
|
||||||
|
Self::OperationResultMismatch => "operation_result_mismatch",
|
||||||
|
Self::SessionIdentityMismatch => "session_identity_mismatch",
|
||||||
|
Self::SessionOpenTaskFailed => "session_open_task_failed",
|
||||||
|
Self::SessionOpenFailed => "session_open_failed",
|
||||||
|
Self::AuthorizationMissing => "authorization_missing",
|
||||||
|
Self::AuthorizationStale => "authorization_stale",
|
||||||
|
Self::AuthorizationUnavailable => "authorization_unavailable",
|
||||||
|
Self::SigningIdentityUnavailable => "signing_identity_unavailable",
|
||||||
|
Self::AuthorizationSignFailed => "authorization_sign_failed",
|
||||||
|
Self::RuntimeStopped => "runtime_stopped",
|
||||||
|
Self::RuntimeStateUnavailable => "runtime_state_unavailable",
|
||||||
|
Self::RuntimeBackendUnavailable => "runtime_backend_unavailable",
|
||||||
|
Self::RuntimeOwnerMismatch => "runtime_owner_mismatch",
|
||||||
|
Self::RuntimeInvalidRequest => "runtime_invalid_request",
|
||||||
|
Self::WorkingDirectoryNotFound => "working_directory_not_found",
|
||||||
|
Self::WorkingDirectoryNotActive => "working_directory_not_active",
|
||||||
|
Self::WorkingDirectoryRecordInvalid => "working_directory_record_invalid",
|
||||||
|
Self::WorkingDirectoryAccessUnavailable => "working_directory_access_unavailable",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The observed failing boundary, not a dispatch or effects assertion.
|
||||||
|
pub const fn stage(self) -> WorkdirFailureStage {
|
||||||
|
use WorkdirFailureStage as Stage;
|
||||||
|
match self {
|
||||||
|
Self::SessionClosed => Stage::Session,
|
||||||
|
Self::ProviderUnavailable => Stage::Provider,
|
||||||
|
Self::ProviderGrantExpired | Self::ProviderAdmissionClosed => Stage::ProviderAdmission,
|
||||||
|
Self::ProviderDispatchRejected => Stage::ProviderDispatch,
|
||||||
|
Self::ProviderResponseLost | Self::ProviderOperationTimeout => Stage::ProviderResponse,
|
||||||
|
Self::AuditStoreUnavailable => Stage::Audit,
|
||||||
|
Self::CommandTerminalMissing => Stage::Command,
|
||||||
|
Self::HttpTimeout | Self::HttpConnect | Self::HttpTransfer => Stage::Http,
|
||||||
|
Self::HttpResponseInvalid
|
||||||
|
| Self::HttpResponseTooLarge
|
||||||
|
| Self::OperationResultMismatch => Stage::ResponseValidation,
|
||||||
|
Self::SessionIdentityMismatch
|
||||||
|
| Self::SessionOpenTaskFailed
|
||||||
|
| Self::SessionOpenFailed
|
||||||
|
| Self::WorkingDirectoryNotFound
|
||||||
|
| Self::WorkingDirectoryNotActive
|
||||||
|
| Self::WorkingDirectoryRecordInvalid
|
||||||
|
| Self::WorkingDirectoryAccessUnavailable => Stage::SessionOpen,
|
||||||
|
Self::AuthorizationMissing
|
||||||
|
| Self::AuthorizationStale
|
||||||
|
| Self::AuthorizationUnavailable
|
||||||
|
| Self::SigningIdentityUnavailable
|
||||||
|
| Self::AuthorizationSignFailed => Stage::AuthorizationPreparation,
|
||||||
|
Self::RuntimeStopped
|
||||||
|
| Self::RuntimeStateUnavailable
|
||||||
|
| Self::RuntimeBackendUnavailable
|
||||||
|
| Self::RuntimeOwnerMismatch
|
||||||
|
| Self::RuntimeInvalidRequest => Stage::Runtime,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Internal operational boundary derived only from a bounded reason.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum WorkdirFailureStage {
|
||||||
|
Session,
|
||||||
|
Provider,
|
||||||
|
ProviderAdmission,
|
||||||
|
ProviderDispatch,
|
||||||
|
ProviderResponse,
|
||||||
|
Audit,
|
||||||
|
Command,
|
||||||
|
Http,
|
||||||
|
ResponseValidation,
|
||||||
|
SessionOpen,
|
||||||
|
AuthorizationPreparation,
|
||||||
|
Runtime,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkdirFailureStage {
|
||||||
|
pub const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Session => "session",
|
||||||
|
Self::Provider => "provider",
|
||||||
|
Self::ProviderAdmission => "provider_admission",
|
||||||
|
Self::ProviderDispatch => "provider_dispatch",
|
||||||
|
Self::ProviderResponse => "provider_response",
|
||||||
|
Self::Audit => "audit",
|
||||||
|
Self::Command => "command",
|
||||||
|
Self::Http => "http",
|
||||||
|
Self::ResponseValidation => "response_validation",
|
||||||
|
Self::SessionOpen => "session_open",
|
||||||
|
Self::AuthorizationPreparation => "authorization_preparation",
|
||||||
|
Self::Runtime => "runtime",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -466,6 +599,10 @@ pub enum WorkdirError {
|
|||||||
#[error("Workdir session is unavailable: {0}")]
|
#[error("Workdir session is unavailable: {0}")]
|
||||||
Unavailable(String),
|
Unavailable(String),
|
||||||
|
|
||||||
|
/// The broker has proof that this operation never reached the provider.
|
||||||
|
#[error("Workdir operation was not dispatched")]
|
||||||
|
NotDispatched,
|
||||||
|
|
||||||
#[error("Workdir operation failed")]
|
#[error("Workdir operation failed")]
|
||||||
OperationFailed,
|
OperationFailed,
|
||||||
|
|
||||||
|
|||||||
@@ -1340,6 +1340,52 @@ impl WorkdirSession for LocalWorkdirSession {
|
|||||||
self.inner.capabilities
|
self.inner.capabilities
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn environment(&self) -> crate::WorkdirSessionEnvironment {
|
||||||
|
let scope = self.inner.scope.snapshot();
|
||||||
|
let project = |rule: manifest::ScopeRule| -> Option<crate::WorkdirToolScopeRule> {
|
||||||
|
let target = if self.inner.root.starts_with(&rule.target) && rule.recursive {
|
||||||
|
WorkdirPath::root()
|
||||||
|
} else {
|
||||||
|
let path = rule
|
||||||
|
.target
|
||||||
|
.strip_prefix(&self.inner.root)
|
||||||
|
.unwrap_or(&rule.target);
|
||||||
|
WorkdirPath::new_scoped(path.to_str()?).ok()?
|
||||||
|
};
|
||||||
|
Some(crate::WorkdirToolScopeRule {
|
||||||
|
target,
|
||||||
|
permission: match rule.permission {
|
||||||
|
Permission::Read => WorkdirToolScopePermission::Read,
|
||||||
|
Permission::Write => WorkdirToolScopePermission::Write,
|
||||||
|
},
|
||||||
|
recursive: rule.recursive,
|
||||||
|
symlink_policy: rule.symlink_policy,
|
||||||
|
})
|
||||||
|
};
|
||||||
|
crate::WorkdirSessionEnvironment {
|
||||||
|
cwd: WorkdirPath::root(),
|
||||||
|
scope: Some(
|
||||||
|
scope
|
||||||
|
.allow_rules()
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(project)
|
||||||
|
.collect(),
|
||||||
|
),
|
||||||
|
capabilities: self.capabilities(),
|
||||||
|
restrictions: scope
|
||||||
|
.deny_rules()
|
||||||
|
.into_iter()
|
||||||
|
.map(|rule| {
|
||||||
|
project(rule)
|
||||||
|
.map(|rule| format!("Deny: {rule:?}"))
|
||||||
|
.unwrap_or_else(|| {
|
||||||
|
"Unrepresentable provider deny rule remains enforced".into()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn authorize_scope_path(
|
async fn authorize_scope_path(
|
||||||
&self,
|
&self,
|
||||||
request: WorkdirScopeAuthorizationRequest,
|
request: WorkdirScopeAuthorizationRequest,
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
//! A replay page survives a lost HTTP response until its next cursor is sent.
|
//! A replay page survives a lost HTTP response until its next cursor is sent.
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::{CommandFailure, CommandFailureStage, CommandIoErrorKind, CommandOutputCollection};
|
||||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
|
|
||||||
@@ -146,6 +147,33 @@ pub(super) fn start(
|
|||||||
telemetry: CommandTelemetry,
|
telemetry: CommandTelemetry,
|
||||||
environment: BTreeMap<String, String>,
|
environment: BTreeMap<String, String>,
|
||||||
resources: Vec<Arc<dyn WorkdirSessionResource>>,
|
resources: Vec<Arc<dyn WorkdirSessionResource>>,
|
||||||
|
) -> LocalCommand {
|
||||||
|
let execution_id = command_id.clone();
|
||||||
|
let execution_telemetry = telemetry.clone();
|
||||||
|
start_with(
|
||||||
|
command_id,
|
||||||
|
telemetry,
|
||||||
|
resources,
|
||||||
|
move |cancel, sender| async move {
|
||||||
|
execute(
|
||||||
|
cwd,
|
||||||
|
request,
|
||||||
|
&execution_id,
|
||||||
|
&execution_telemetry,
|
||||||
|
environment,
|
||||||
|
cancel,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn start_with<F: std::future::Future<Output = crate::CommandTerminal> + Send + 'static>(
|
||||||
|
command_id: String,
|
||||||
|
telemetry: CommandTelemetry,
|
||||||
|
resources: Vec<Arc<dyn WorkdirSessionResource>>,
|
||||||
|
execution: impl FnOnce(watch::Receiver<bool>, mpsc::Sender<Message>) -> F + Send + 'static,
|
||||||
) -> LocalCommand {
|
) -> LocalCommand {
|
||||||
let (sender, receiver) = mpsc::channel(QUEUE_CHUNKS);
|
let (sender, receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||||
let (cancel, cancel_rx) = watch::channel(false);
|
let (cancel, cancel_rx) = watch::channel(false);
|
||||||
@@ -160,16 +188,7 @@ pub(super) fn start(
|
|||||||
}));
|
}));
|
||||||
let task = tokio::spawn(async move {
|
let task = tokio::spawn(async move {
|
||||||
let _resources = resources;
|
let _resources = resources;
|
||||||
let terminal = execute(
|
let terminal = execution(cancel_rx, sender.clone()).await;
|
||||||
cwd,
|
|
||||||
request,
|
|
||||||
&command_id,
|
|
||||||
&telemetry,
|
|
||||||
environment,
|
|
||||||
cancel_rx,
|
|
||||||
&sender,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
telemetry.terminal(&command_id, terminal.status, terminal.exit_code);
|
telemetry.terminal(&command_id, terminal.status, terminal.exit_code);
|
||||||
// Closing the byte queue fences this already-published terminal, without
|
// Closing the byte queue fences this already-published terminal, without
|
||||||
// requiring spare queue capacity or a receiver to release task resources.
|
// requiring spare queue capacity or a receiver to release task resources.
|
||||||
@@ -192,10 +211,18 @@ struct ByteAccounting {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ByteAccounting {
|
impl ByteAccounting {
|
||||||
fn terminal(&self, status: CommandStatus, exit_code: Option<i32>) -> crate::CommandTerminal {
|
fn terminal(
|
||||||
|
&self,
|
||||||
|
status: CommandStatus,
|
||||||
|
exit_code: Option<i32>,
|
||||||
|
process_failure: Option<CommandFailure>,
|
||||||
|
output_collection: CommandOutputCollection,
|
||||||
|
) -> crate::CommandTerminal {
|
||||||
crate::CommandTerminal {
|
crate::CommandTerminal {
|
||||||
status,
|
status,
|
||||||
exit_code,
|
exit_code,
|
||||||
|
process_failure,
|
||||||
|
output_collection,
|
||||||
stdout_bytes: self.stdout,
|
stdout_bytes: self.stdout,
|
||||||
stderr_bytes: self.stderr,
|
stderr_bytes: self.stderr,
|
||||||
stdout_delivered_bytes: self.stdout_delivered,
|
stdout_delivered_bytes: self.stdout_delivered,
|
||||||
@@ -215,7 +242,7 @@ async fn pump<R: AsyncRead + Unpin>(
|
|||||||
command_id: &str,
|
command_id: &str,
|
||||||
mut remaining: u64,
|
mut remaining: u64,
|
||||||
accounting: &mut ByteAccounting,
|
accounting: &mut ByteAccounting,
|
||||||
) -> std::io::Result<()> {
|
) -> Result<(), CommandFailure> {
|
||||||
let mut stdout_done = false;
|
let mut stdout_done = false;
|
||||||
let mut stderr_done = false;
|
let mut stderr_done = false;
|
||||||
let mut stdout_decoder = CommandOutputDecoder::default();
|
let mut stdout_decoder = CommandOutputDecoder::default();
|
||||||
@@ -228,7 +255,11 @@ async fn pump<R: AsyncRead + Unpin>(
|
|||||||
// Before quota, reserve first to retain bounded pipe backpressure.
|
// Before quota, reserve first to retain bounded pipe backpressure.
|
||||||
let permit = if remaining > 0 {
|
let permit = if remaining > 0 {
|
||||||
let Ok(permit) = sender.reserve().await else {
|
let Ok(permit) = sender.reserve().await else {
|
||||||
return Ok(());
|
return Err(CommandFailure {
|
||||||
|
stage: CommandFailureStage::OutputDelivery,
|
||||||
|
reason: CommandIoErrorKind::Unknown,
|
||||||
|
os_code: None,
|
||||||
|
});
|
||||||
};
|
};
|
||||||
Some(permit)
|
Some(permit)
|
||||||
} else {
|
} else {
|
||||||
@@ -242,7 +273,15 @@ async fn pump<R: AsyncRead + Unpin>(
|
|||||||
CommandStream::Stdout => &stdout_buffer,
|
CommandStream::Stdout => &stdout_buffer,
|
||||||
CommandStream::Stderr => &stderr_buffer,
|
CommandStream::Stderr => &stderr_buffer,
|
||||||
};
|
};
|
||||||
let count = result?;
|
let count = result.map_err(|error| {
|
||||||
|
CommandFailure::io(
|
||||||
|
match stream {
|
||||||
|
CommandStream::Stdout => CommandFailureStage::StdoutRead,
|
||||||
|
CommandStream::Stderr => CommandFailureStage::StderrRead,
|
||||||
|
},
|
||||||
|
&error,
|
||||||
|
)
|
||||||
|
})?;
|
||||||
let decoder = match stream {
|
let decoder = match stream {
|
||||||
CommandStream::Stdout => &mut stdout_decoder,
|
CommandStream::Stdout => &mut stdout_decoder,
|
||||||
CommandStream::Stderr => &mut stderr_decoder,
|
CommandStream::Stderr => &mut stderr_decoder,
|
||||||
@@ -288,7 +327,7 @@ async fn execute(
|
|||||||
command_id: &str,
|
command_id: &str,
|
||||||
telemetry: &CommandTelemetry,
|
telemetry: &CommandTelemetry,
|
||||||
environment: BTreeMap<String, String>,
|
environment: BTreeMap<String, String>,
|
||||||
mut cancel: watch::Receiver<bool>,
|
cancel: watch::Receiver<bool>,
|
||||||
sender: &mpsc::Sender<Message>,
|
sender: &mpsc::Sender<Message>,
|
||||||
) -> crate::CommandTerminal {
|
) -> crate::CommandTerminal {
|
||||||
telemetry.started(command_id, request.tool_call_id);
|
telemetry.started(command_id, request.tool_call_id);
|
||||||
@@ -304,74 +343,159 @@ async fn execute(
|
|||||||
.kill_on_drop(true);
|
.kill_on_drop(true);
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
command.as_std_mut().process_group(0);
|
command.as_std_mut().process_group(0);
|
||||||
let mut accounting = ByteAccounting::default();
|
let mut child = match command.spawn() {
|
||||||
let Ok(mut child) = command.spawn() else {
|
Ok(child) => child,
|
||||||
return accounting.terminal(CommandStatus::Failed, None);
|
Err(error) => {
|
||||||
|
return ByteAccounting::default().terminal(
|
||||||
|
CommandStatus::Failed,
|
||||||
|
None,
|
||||||
|
Some(CommandFailure::io(CommandFailureStage::Spawn, &error)),
|
||||||
|
CommandOutputCollection::NotStarted,
|
||||||
|
);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
let mut process_group = CommandProcessGroupGuard::new(child.id());
|
|
||||||
let stdout = child.stdout.take().expect("piped stdout");
|
let stdout = child.stdout.take().expect("piped stdout");
|
||||||
let stderr = child.stderr.take().expect("piped stderr");
|
let stderr = child.stderr.take().expect("piped stderr");
|
||||||
let (status, exit_code) = {
|
supervise(
|
||||||
|
child,
|
||||||
|
stdout,
|
||||||
|
stderr,
|
||||||
|
request.timeout_secs,
|
||||||
|
request.transport_byte_budget,
|
||||||
|
command_id,
|
||||||
|
telemetry,
|
||||||
|
cancel,
|
||||||
|
sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
// Substitute only syscalls and readers: injected faults run the production supervisor.
|
||||||
|
trait CommandProcess: Send {
|
||||||
|
fn id(&self) -> Option<u32>;
|
||||||
|
fn start_kill(&mut self);
|
||||||
|
fn wait(
|
||||||
|
&mut self,
|
||||||
|
) -> impl std::future::Future<Output = std::io::Result<std::process::ExitStatus>> + Send;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CommandProcess for tokio::process::Child {
|
||||||
|
fn id(&self) -> Option<u32> {
|
||||||
|
self.id()
|
||||||
|
}
|
||||||
|
fn start_kill(&mut self) {
|
||||||
|
let _ = self.start_kill();
|
||||||
|
}
|
||||||
|
async fn wait(&mut self) -> std::io::Result<std::process::ExitStatus> {
|
||||||
|
self.wait().await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn process_result(
|
||||||
|
result: std::io::Result<std::process::ExitStatus>,
|
||||||
|
) -> (CommandStatus, Option<i32>, Option<CommandFailure>) {
|
||||||
|
match result {
|
||||||
|
Ok(exit) => (
|
||||||
|
if exit.success() {
|
||||||
|
CommandStatus::Completed
|
||||||
|
} else {
|
||||||
|
CommandStatus::Failed
|
||||||
|
},
|
||||||
|
exit.code(),
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
Err(error) => (
|
||||||
|
CommandStatus::Failed,
|
||||||
|
None,
|
||||||
|
Some(CommandFailure::io(CommandFailureStage::Wait, &error)),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
async fn supervise(
|
||||||
|
mut child: impl CommandProcess,
|
||||||
|
stdout: impl AsyncRead + Unpin,
|
||||||
|
stderr: impl AsyncRead + Unpin,
|
||||||
|
timeout_secs: u64,
|
||||||
|
byte_budget: u64,
|
||||||
|
command_id: &str,
|
||||||
|
telemetry: &CommandTelemetry,
|
||||||
|
mut cancel: watch::Receiver<bool>,
|
||||||
|
sender: &mpsc::Sender<Message>,
|
||||||
|
) -> crate::CommandTerminal {
|
||||||
|
let mut process_group = CommandProcessGroupGuard::new(child.id());
|
||||||
|
let mut accounting = ByteAccounting::default();
|
||||||
|
let (status, exit_code, process_failure, output_collection) = {
|
||||||
let output = pump(
|
let output = pump(
|
||||||
stdout,
|
stdout,
|
||||||
stderr,
|
stderr,
|
||||||
sender,
|
sender,
|
||||||
telemetry,
|
telemetry,
|
||||||
command_id,
|
command_id,
|
||||||
request.transport_byte_budget,
|
byte_budget,
|
||||||
&mut accounting,
|
&mut accounting,
|
||||||
);
|
);
|
||||||
tokio::pin!(output);
|
tokio::pin!(output);
|
||||||
let mut drained = false;
|
let mut collected = None;
|
||||||
let timeout = tokio::time::sleep(Duration::from_secs(request.timeout_secs.max(1)));
|
let timeout = tokio::time::sleep(Duration::from_secs(timeout_secs.max(1)));
|
||||||
tokio::pin!(timeout);
|
tokio::pin!(timeout);
|
||||||
let terminal = loop {
|
let terminal = loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
result = &mut output, if !drained => {
|
result = &mut output, if collected.is_none() => {
|
||||||
drained = true;
|
let failed = result.is_err();
|
||||||
if result.is_err() || sender.is_closed() {
|
collected = Some(result);
|
||||||
|
if failed {
|
||||||
process_group.terminate();
|
process_group.terminate();
|
||||||
let _ = child.start_kill();
|
child.start_kill();
|
||||||
let _ = child.wait().await;
|
break process_result(child.wait().await);
|
||||||
break (CommandStatus::Failed, None);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
exit = child.wait() => {
|
exit = child.wait() => {
|
||||||
process_group.terminate();
|
process_group.terminate();
|
||||||
break match exit {
|
break process_result(exit);
|
||||||
Ok(exit) => (if exit.success() { CommandStatus::Completed } else { CommandStatus::Failed }, exit.code()),
|
|
||||||
Err(_) => (CommandStatus::Failed, None),
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
_ = &mut timeout => {
|
_ = &mut timeout => {
|
||||||
process_group.terminate();
|
process_group.terminate();
|
||||||
let _ = child.start_kill();
|
child.start_kill();
|
||||||
let code = child.wait().await.ok().and_then(|exit| exit.code());
|
let (_, code, failure) = process_result(child.wait().await);
|
||||||
break (CommandStatus::TimedOut, code);
|
break (CommandStatus::TimedOut, code, failure);
|
||||||
}
|
}
|
||||||
_ = sender.closed() => {
|
_ = sender.closed() => {
|
||||||
process_group.terminate();
|
process_group.terminate();
|
||||||
let _ = child.start_kill();
|
child.start_kill();
|
||||||
let _ = child.wait().await;
|
let (_, code, failure) = process_result(child.wait().await);
|
||||||
break (CommandStatus::Cancelled, None);
|
break (CommandStatus::Cancelled, code, failure);
|
||||||
}
|
}
|
||||||
changed = cancel.changed() => {
|
changed = cancel.changed() => {
|
||||||
if changed.is_err() || *cancel.borrow() {
|
if changed.is_err() || *cancel.borrow() {
|
||||||
process_group.terminate();
|
process_group.terminate();
|
||||||
let _ = child.start_kill();
|
child.start_kill();
|
||||||
let code = child.wait().await.ok().and_then(|exit| exit.code());
|
let (_, code, failure) = process_result(child.wait().await);
|
||||||
break (CommandStatus::Cancelled, code);
|
break (CommandStatus::Cancelled, code, failure);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
// Even after kill, preserve the bounded pipe tail before publishing terminal.
|
// Exit is not pipe EOF. Retain read failures first observed during drain.
|
||||||
if !drained {
|
let result = match collected {
|
||||||
let _ = output.await;
|
Some(result) => result,
|
||||||
}
|
None => output.await.map_err(|mut failure| {
|
||||||
terminal
|
failure.stage = match failure.stage {
|
||||||
|
CommandFailureStage::StdoutRead => CommandFailureStage::StdoutDrain,
|
||||||
|
CommandFailureStage::StderrRead => CommandFailureStage::StderrDrain,
|
||||||
|
stage => stage,
|
||||||
|
};
|
||||||
|
failure
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
let collection = match result {
|
||||||
|
Ok(()) => CommandOutputCollection::Complete,
|
||||||
|
Err(failure) => CommandOutputCollection::Incomplete { failure },
|
||||||
|
};
|
||||||
|
(terminal.0, terminal.1, terminal.2, collection)
|
||||||
};
|
};
|
||||||
accounting.terminal(status, exit_code)
|
accounting.terminal(status, exit_code, process_failure, output_collection)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -520,3 +520,349 @@ async fn slow_receiver_backpressures_before_quota_but_full_queue_does_not_block_
|
|||||||
assert_eq!(queued_bytes, QUEUE_CHUNKS * COMMAND_EVENT_CHUNK_BYTES);
|
assert_eq!(queued_bytes, QUEUE_CHUNKS * COMMAND_EVENT_CHUNK_BYTES);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
struct Gate(Arc<std::sync::Mutex<(bool, Option<std::task::Waker>)>>);
|
||||||
|
|
||||||
|
impl Gate {
|
||||||
|
fn open(&self) {
|
||||||
|
let mut state = self.0.lock().unwrap();
|
||||||
|
state.0 = true;
|
||||||
|
if let Some(waker) = state.1.take() {
|
||||||
|
waker.wake();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll(&self, cx: &mut std::task::Context<'_>) -> std::task::Poll<()> {
|
||||||
|
let mut state = self.0.lock().unwrap();
|
||||||
|
if state.0 {
|
||||||
|
std::task::Poll::Ready(())
|
||||||
|
} else {
|
||||||
|
state.1 = Some(cx.waker().clone());
|
||||||
|
std::task::Poll::Pending
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct FaultProcess {
|
||||||
|
ready: Gate,
|
||||||
|
exited: Gate,
|
||||||
|
wait_error: Option<std::io::ErrorKind>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CommandProcess for FaultProcess {
|
||||||
|
fn id(&self) -> Option<u32> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
fn start_kill(&mut self) {
|
||||||
|
self.ready.open();
|
||||||
|
}
|
||||||
|
async fn wait(&mut self) -> std::io::Result<std::process::ExitStatus> {
|
||||||
|
std::future::poll_fn(|cx| self.ready.poll(cx)).await;
|
||||||
|
self.exited.open();
|
||||||
|
match self.wait_error {
|
||||||
|
Some(kind) => Err(std::io::Error::new(
|
||||||
|
kind,
|
||||||
|
"/private/path TOKEN=secret command-body",
|
||||||
|
)),
|
||||||
|
None => Ok(std::process::ExitStatus::default()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct FaultReader {
|
||||||
|
prefix: &'static [u8],
|
||||||
|
prefix_read: Gate,
|
||||||
|
fail_ready: Gate,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AsyncRead for FaultReader {
|
||||||
|
fn poll_read(
|
||||||
|
mut self: std::pin::Pin<&mut Self>,
|
||||||
|
cx: &mut std::task::Context<'_>,
|
||||||
|
buffer: &mut tokio::io::ReadBuf<'_>,
|
||||||
|
) -> std::task::Poll<std::io::Result<()>> {
|
||||||
|
if !self.prefix.is_empty() {
|
||||||
|
let count = self.prefix.len().min(buffer.remaining());
|
||||||
|
buffer.put_slice(&self.prefix[..count]);
|
||||||
|
self.prefix = &self.prefix[count..];
|
||||||
|
self.prefix_read.open();
|
||||||
|
return std::task::Poll::Ready(Ok(()));
|
||||||
|
}
|
||||||
|
std::task::ready!(self.fail_ready.poll(cx));
|
||||||
|
std::task::Poll::Ready(Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::PermissionDenied,
|
||||||
|
"/private/path TOKEN=secret command-body",
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Faults replace only OS wait/read boundaries, not collection or finalization.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn read_and_post_exit_drain_faults_keep_prefix_exit_and_safe_original_reason() {
|
||||||
|
for stream in [CommandStream::Stdout, CommandStream::Stderr] {
|
||||||
|
for after_exit in [false, true] {
|
||||||
|
let ready = Gate::default();
|
||||||
|
let exited = Gate::default();
|
||||||
|
let fail_ready = if after_exit {
|
||||||
|
exited.clone()
|
||||||
|
} else {
|
||||||
|
let gate = Gate::default();
|
||||||
|
gate.open();
|
||||||
|
gate
|
||||||
|
};
|
||||||
|
let reader = FaultReader {
|
||||||
|
prefix: b"partial",
|
||||||
|
prefix_read: if after_exit {
|
||||||
|
ready.clone()
|
||||||
|
} else {
|
||||||
|
Gate::default()
|
||||||
|
},
|
||||||
|
fail_ready,
|
||||||
|
};
|
||||||
|
let process = FaultProcess {
|
||||||
|
ready,
|
||||||
|
exited,
|
||||||
|
wait_error: None,
|
||||||
|
};
|
||||||
|
let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||||
|
let (_cancel, cancel_rx) = watch::channel(false);
|
||||||
|
let telemetry = CommandTelemetry::new();
|
||||||
|
let terminal = tokio::time::timeout(Duration::from_secs(5), async {
|
||||||
|
match stream {
|
||||||
|
CommandStream::Stdout => {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
reader,
|
||||||
|
&b""[..],
|
||||||
|
10,
|
||||||
|
100,
|
||||||
|
"fault",
|
||||||
|
&telemetry,
|
||||||
|
cancel_rx,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
CommandStream::Stderr => {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
&b""[..],
|
||||||
|
reader,
|
||||||
|
10,
|
||||||
|
100,
|
||||||
|
"fault",
|
||||||
|
&telemetry,
|
||||||
|
cancel_rx,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let Message::Bytes(chunk) = receiver.recv().await.unwrap();
|
||||||
|
assert_eq!(chunk.bytes, b"partial");
|
||||||
|
assert_eq!(chunk.stream, stream);
|
||||||
|
assert_eq!(chunk.sequence, 0);
|
||||||
|
assert_eq!(chunk.offset, 0);
|
||||||
|
assert!(receiver.try_recv().is_err());
|
||||||
|
assert_eq!(terminal.status, CommandStatus::Completed);
|
||||||
|
assert_eq!(terminal.exit_code, Some(0));
|
||||||
|
assert_eq!(terminal.process_failure, None);
|
||||||
|
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes,
|
||||||
|
7
|
||||||
|
);
|
||||||
|
assert_eq!(terminal.truncation_reason, None);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
CommandOutputCollection::Incomplete {
|
||||||
|
failure: CommandFailure {
|
||||||
|
stage: match (stream, after_exit) {
|
||||||
|
(CommandStream::Stdout, false) => CommandFailureStage::StdoutRead,
|
||||||
|
(CommandStream::Stdout, true) => CommandFailureStage::StdoutDrain,
|
||||||
|
(CommandStream::Stderr, false) => CommandFailureStage::StderrRead,
|
||||||
|
(CommandStream::Stderr, true) => CommandFailureStage::StderrDrain,
|
||||||
|
},
|
||||||
|
reason: CommandIoErrorKind::PermissionDenied,
|
||||||
|
os_code: None
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let wire = serde_json::to_string(&terminal).unwrap();
|
||||||
|
for secret in ["/private", "TOKEN", "secret", "command-body"] {
|
||||||
|
assert!(!wire.contains(secret));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn wait_fault_keeps_collected_bytes_and_does_not_invent_exit_information() {
|
||||||
|
let ready = Gate::default();
|
||||||
|
ready.open();
|
||||||
|
let process = FaultProcess {
|
||||||
|
ready,
|
||||||
|
exited: Gate::default(),
|
||||||
|
wait_error: Some(std::io::ErrorKind::InvalidData),
|
||||||
|
};
|
||||||
|
let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||||
|
let (_cancel, cancel_rx) = watch::channel(false);
|
||||||
|
let terminal = supervise(
|
||||||
|
process,
|
||||||
|
&b"out"[..],
|
||||||
|
&b"err"[..],
|
||||||
|
10,
|
||||||
|
100,
|
||||||
|
"wait-fault",
|
||||||
|
&CommandTelemetry::new(),
|
||||||
|
cancel_rx,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(terminal.status, CommandStatus::Failed);
|
||||||
|
assert_eq!(terminal.exit_code, None);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.process_failure,
|
||||||
|
Some(CommandFailure {
|
||||||
|
stage: CommandFailureStage::Wait,
|
||||||
|
reason: CommandIoErrorKind::InvalidData,
|
||||||
|
os_code: None
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
CommandOutputCollection::Complete
|
||||||
|
);
|
||||||
|
assert_eq!(terminal.stdout_bytes, 3);
|
||||||
|
assert_eq!(terminal.stderr_bytes, 3);
|
||||||
|
let mut chunks = Vec::new();
|
||||||
|
while let Ok(Message::Bytes(chunk)) = receiver.try_recv() {
|
||||||
|
chunks.push(chunk);
|
||||||
|
}
|
||||||
|
assert_eq!(stream_bytes(&chunks, CommandStream::Stdout), b"out");
|
||||||
|
assert_eq!(stream_bytes(&chunks, CommandStream::Stderr), b"err");
|
||||||
|
assert!(!serde_json::to_string(&terminal).unwrap().contains("secret"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn spawn_fault_retains_safe_reason_without_claiming_output_collection() {
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
let (sender, _receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||||
|
let (_cancel, cancel_rx) = watch::channel(false);
|
||||||
|
let terminal = execute(
|
||||||
|
root.path().join("missing-secret-path"),
|
||||||
|
Harness::request("secret-command", 10, 100),
|
||||||
|
"spawn-fault",
|
||||||
|
&CommandTelemetry::new(),
|
||||||
|
BTreeMap::new(),
|
||||||
|
cancel_rx,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
terminal.process_failure,
|
||||||
|
Some(CommandFailure {
|
||||||
|
stage: CommandFailureStage::Spawn,
|
||||||
|
reason: CommandIoErrorKind::NotFound,
|
||||||
|
os_code: Some(libc::ENOENT)
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
CommandOutputCollection::NotStarted
|
||||||
|
);
|
||||||
|
assert_eq!(terminal.exit_code, None);
|
||||||
|
assert_eq!(terminal.status, CommandStatus::Failed);
|
||||||
|
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 0);
|
||||||
|
assert!(!serde_json::to_string(&terminal).unwrap().contains("secret"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-client")]
|
||||||
|
mod http;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn drain_failure_and_byte_budget_keep_independent_loss_diagnostics() {
|
||||||
|
for stream in [CommandStream::Stdout, CommandStream::Stderr] {
|
||||||
|
let ready = Gate::default();
|
||||||
|
let exited = Gate::default();
|
||||||
|
let process = FaultProcess {
|
||||||
|
ready: ready.clone(),
|
||||||
|
exited: exited.clone(),
|
||||||
|
wait_error: None,
|
||||||
|
};
|
||||||
|
let reader = FaultReader {
|
||||||
|
prefix: b"partial",
|
||||||
|
prefix_read: ready,
|
||||||
|
fail_ready: exited,
|
||||||
|
};
|
||||||
|
let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||||
|
let (_cancel, cancel_rx) = watch::channel(false);
|
||||||
|
let telemetry = CommandTelemetry::new();
|
||||||
|
let terminal = tokio::time::timeout(Duration::from_secs(5), async {
|
||||||
|
match stream {
|
||||||
|
CommandStream::Stdout => {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
reader,
|
||||||
|
&b""[..],
|
||||||
|
10,
|
||||||
|
3,
|
||||||
|
"quota-fault",
|
||||||
|
&telemetry,
|
||||||
|
cancel_rx,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
CommandStream::Stderr => {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
&b""[..],
|
||||||
|
reader,
|
||||||
|
10,
|
||||||
|
3,
|
||||||
|
"quota-fault",
|
||||||
|
&telemetry,
|
||||||
|
cancel_rx,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let Message::Bytes(chunk) = receiver.recv().await.unwrap();
|
||||||
|
assert_eq!(chunk.stream, stream);
|
||||||
|
assert_eq!(chunk.bytes, b"par");
|
||||||
|
assert!(receiver.try_recv().is_err());
|
||||||
|
assert_eq!(terminal.exit_code, Some(0));
|
||||||
|
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes,
|
||||||
|
3
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.truncation_reason,
|
||||||
|
Some(crate::CommandOutputTruncation::ByteBudget)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
CommandOutputCollection::Incomplete {
|
||||||
|
failure: CommandFailure {
|
||||||
|
stage: match stream {
|
||||||
|
CommandStream::Stdout => CommandFailureStage::StdoutDrain,
|
||||||
|
CommandStream::Stderr => CommandFailureStage::StderrDrain,
|
||||||
|
},
|
||||||
|
reason: CommandIoErrorKind::PermissionDenied,
|
||||||
|
os_code: None,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert!(terminal.metadata_valid());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,241 @@
|
|||||||
|
//! Loopback integration of the real supervisor, byte queue/replay, local session,
|
||||||
|
//! HTTP dispatcher/client. OS wait/read are
|
||||||
|
//! deterministic faults; session open and start bind that injected process fixture.
|
||||||
|
//! Authorization is not asserted here (Runtime's signed HTTP suite owns it).
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::http::*;
|
||||||
|
use axum::{Json, Router, extract::State, routing::post};
|
||||||
|
|
||||||
|
struct Server(tokio::task::JoinHandle<()>);
|
||||||
|
impl Drop for Server {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn operation(
|
||||||
|
State(session): State<Arc<LocalWorkdirSession>>,
|
||||||
|
Json(request): Json<WorkdirSessionOperationRequest>,
|
||||||
|
) -> Json<WorkdirSessionOperationResult> {
|
||||||
|
if matches!(request.operation, WorkdirSessionOperation::CommandStart(_)) {
|
||||||
|
return Json(WorkdirSessionOperationResult::CommandStart(CommandHandle(
|
||||||
|
"injected".into(),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Json(
|
||||||
|
dispatch_workdir_session_operation(session.as_ref(), request.operation)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn pipe_and_wait_faults_cross_real_http_with_prefix_and_terminal_replay() {
|
||||||
|
tokio::time::timeout(Duration::from_secs(20), async {
|
||||||
|
for (stream, after_exit, wait_fault) in [
|
||||||
|
(CommandStream::Stdout, false, false),
|
||||||
|
(CommandStream::Stderr, false, false),
|
||||||
|
(CommandStream::Stdout, true, false),
|
||||||
|
(CommandStream::Stderr, true, false),
|
||||||
|
(CommandStream::Stdout, false, true),
|
||||||
|
] {
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
let session = Arc::new(LocalWorkdirSession::new(
|
||||||
|
Scope::writable(root.path()).unwrap(),
|
||||||
|
root.path().into(),
|
||||||
|
));
|
||||||
|
let ready = Gate::default();
|
||||||
|
let exited = Gate::default();
|
||||||
|
if wait_fault {
|
||||||
|
ready.open();
|
||||||
|
}
|
||||||
|
let process = FaultProcess {
|
||||||
|
ready: ready.clone(),
|
||||||
|
exited: exited.clone(),
|
||||||
|
wait_error: wait_fault.then_some(std::io::ErrorKind::InvalidData),
|
||||||
|
};
|
||||||
|
let fail_ready = if after_exit {
|
||||||
|
exited
|
||||||
|
} else {
|
||||||
|
let gate = Gate::default();
|
||||||
|
gate.open();
|
||||||
|
gate
|
||||||
|
};
|
||||||
|
let reader = FaultReader {
|
||||||
|
prefix: b"partial",
|
||||||
|
prefix_read: if after_exit { ready } else { Gate::default() },
|
||||||
|
fail_ready,
|
||||||
|
};
|
||||||
|
let command = start_with(
|
||||||
|
"injected".into(),
|
||||||
|
CommandTelemetry::new(),
|
||||||
|
vec![],
|
||||||
|
move |cancel, sender| async move {
|
||||||
|
let telemetry = CommandTelemetry::new();
|
||||||
|
if wait_fault {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
&b"partial"[..],
|
||||||
|
&b""[..],
|
||||||
|
10,
|
||||||
|
100,
|
||||||
|
"injected",
|
||||||
|
&telemetry,
|
||||||
|
cancel,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
} else {
|
||||||
|
match stream {
|
||||||
|
CommandStream::Stdout => {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
reader,
|
||||||
|
&b""[..],
|
||||||
|
10,
|
||||||
|
100,
|
||||||
|
"injected",
|
||||||
|
&telemetry,
|
||||||
|
cancel,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
CommandStream::Stderr => {
|
||||||
|
supervise(
|
||||||
|
process,
|
||||||
|
&b""[..],
|
||||||
|
reader,
|
||||||
|
10,
|
||||||
|
100,
|
||||||
|
"injected",
|
||||||
|
&telemetry,
|
||||||
|
cancel,
|
||||||
|
&sender,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
session
|
||||||
|
.inner
|
||||||
|
.commands
|
||||||
|
.lock()
|
||||||
|
.await
|
||||||
|
.insert("injected".into(), command);
|
||||||
|
let app = Router::new()
|
||||||
|
.route(
|
||||||
|
"/v1/working-directories/{id}/sessions",
|
||||||
|
post(|| async {
|
||||||
|
Json(OpenWorkdirSessionResponse {
|
||||||
|
session_id: WorkdirSessionId::new("fault-session").unwrap(),
|
||||||
|
workdir_id: crate::Workdir::new("fault-workdir").id().clone(),
|
||||||
|
capabilities: crate::WorkdirSessionCapabilities::ALL,
|
||||||
|
environment: crate::WorkdirSessionEnvironment::root(
|
||||||
|
crate::WorkdirSessionCapabilities::ALL,
|
||||||
|
),
|
||||||
|
})
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.route("/v1/workdir-sessions/{id}/operations", post(operation))
|
||||||
|
.with_state(session.clone());
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||||
|
let _server = Server(tokio::spawn(async move {
|
||||||
|
axum::serve(listener, app).await.unwrap();
|
||||||
|
}));
|
||||||
|
let remote: crate::WorkdirSessionHandle = Arc::new(
|
||||||
|
RemoteWorkdirSession::open(
|
||||||
|
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||||
|
base.parse().unwrap(),
|
||||||
|
"fixture-token",
|
||||||
|
crate::Workdir::new("fault-workdir").id().clone(),
|
||||||
|
OpenWorkdirSessionRequest::default(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let handle = CommandHandle("injected".into());
|
||||||
|
let mut request = CommandOutputRequest {
|
||||||
|
handle,
|
||||||
|
cursor: 0,
|
||||||
|
limit: 1,
|
||||||
|
wait: true,
|
||||||
|
};
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
let terminal = loop {
|
||||||
|
let page = remote.command_output(request.clone()).await.unwrap();
|
||||||
|
assert_eq!(remote.command_output(request.clone()).await.unwrap(), page);
|
||||||
|
for chunk in &page.chunks {
|
||||||
|
assert_eq!(chunk.sequence, request.cursor);
|
||||||
|
assert_eq!(chunk.offset, request.cursor);
|
||||||
|
assert_eq!(chunk.stream, stream);
|
||||||
|
bytes.extend_from_slice(&chunk.bytes);
|
||||||
|
request.cursor += chunk.bytes.len() as u64;
|
||||||
|
}
|
||||||
|
assert_eq!(page.next_cursor, request.cursor);
|
||||||
|
if let Some(terminal) = page.terminal {
|
||||||
|
break terminal;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
assert_eq!(bytes, b"partial");
|
||||||
|
assert_eq!(terminal.truncation_reason, None);
|
||||||
|
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes,
|
||||||
|
7
|
||||||
|
);
|
||||||
|
if wait_fault {
|
||||||
|
assert_eq!(terminal.status, CommandStatus::Failed);
|
||||||
|
assert_eq!(terminal.exit_code, None);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.process_failure,
|
||||||
|
Some(CommandFailure {
|
||||||
|
stage: CommandFailureStage::Wait,
|
||||||
|
reason: CommandIoErrorKind::InvalidData,
|
||||||
|
os_code: None
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
CommandOutputCollection::Complete
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
assert_eq!(terminal.status, CommandStatus::Completed);
|
||||||
|
assert_eq!(terminal.exit_code, Some(0));
|
||||||
|
let stage = match (stream, after_exit) {
|
||||||
|
(CommandStream::Stdout, false) => CommandFailureStage::StdoutRead,
|
||||||
|
(CommandStream::Stderr, false) => CommandFailureStage::StderrRead,
|
||||||
|
(CommandStream::Stdout, true) => CommandFailureStage::StdoutDrain,
|
||||||
|
(CommandStream::Stderr, true) => CommandFailureStage::StderrDrain,
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
CommandOutputCollection::Incomplete {
|
||||||
|
failure: CommandFailure {
|
||||||
|
stage,
|
||||||
|
reason: CommandIoErrorKind::PermissionDenied,
|
||||||
|
os_code: None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let wire = serde_json::to_string(&terminal).unwrap();
|
||||||
|
for secret in [
|
||||||
|
"/private",
|
||||||
|
"TOKEN",
|
||||||
|
"secret",
|
||||||
|
"command-body",
|
||||||
|
"fixture-token",
|
||||||
|
] {
|
||||||
|
assert!(!wire.contains(secret));
|
||||||
|
}
|
||||||
|
session.close().await.unwrap();
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded loopback fault projection");
|
||||||
|
}
|
||||||
+257
-10
@@ -132,12 +132,151 @@ pub enum CommandOutputTruncation {
|
|||||||
ByteBudget,
|
ByteBudget,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Raw totals include discarded pipe bytes. Delivered totals count only prefixes
|
/// Closed diagnostic vocabulary: never serialize OS messages, paths, or errno text.
|
||||||
/// admitted to the transport queue, not page reads or retries by a consumer.
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum CommandIoErrorKind {
|
||||||
|
NotFound,
|
||||||
|
PermissionDenied,
|
||||||
|
ConnectionRefused,
|
||||||
|
ConnectionReset,
|
||||||
|
ConnectionAborted,
|
||||||
|
NotConnected,
|
||||||
|
BrokenPipe,
|
||||||
|
AlreadyExists,
|
||||||
|
WouldBlock,
|
||||||
|
InvalidInput,
|
||||||
|
InvalidData,
|
||||||
|
TimedOut,
|
||||||
|
WriteZero,
|
||||||
|
Interrupted,
|
||||||
|
UnexpectedEof,
|
||||||
|
OutOfMemory,
|
||||||
|
HostUnreachable,
|
||||||
|
NetworkUnreachable,
|
||||||
|
AddrInUse,
|
||||||
|
AddrNotAvailable,
|
||||||
|
NetworkDown,
|
||||||
|
NotADirectory,
|
||||||
|
IsADirectory,
|
||||||
|
DirectoryNotEmpty,
|
||||||
|
ReadOnlyFilesystem,
|
||||||
|
StaleNetworkFileHandle,
|
||||||
|
StorageFull,
|
||||||
|
NotSeekable,
|
||||||
|
QuotaExceeded,
|
||||||
|
FileTooLarge,
|
||||||
|
ResourceBusy,
|
||||||
|
ExecutableFileBusy,
|
||||||
|
Deadlock,
|
||||||
|
CrossesDevices,
|
||||||
|
TooManyLinks,
|
||||||
|
InvalidFilename,
|
||||||
|
ArgumentListTooLong,
|
||||||
|
Unsupported,
|
||||||
|
Other,
|
||||||
|
Unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<std::io::ErrorKind> for CommandIoErrorKind {
|
||||||
|
fn from(kind: std::io::ErrorKind) -> Self {
|
||||||
|
use std::io::ErrorKind as Io;
|
||||||
|
match kind {
|
||||||
|
Io::NotFound => Self::NotFound,
|
||||||
|
Io::PermissionDenied => Self::PermissionDenied,
|
||||||
|
Io::ConnectionRefused => Self::ConnectionRefused,
|
||||||
|
Io::ConnectionReset => Self::ConnectionReset,
|
||||||
|
Io::ConnectionAborted => Self::ConnectionAborted,
|
||||||
|
Io::NotConnected => Self::NotConnected,
|
||||||
|
Io::BrokenPipe => Self::BrokenPipe,
|
||||||
|
Io::AlreadyExists => Self::AlreadyExists,
|
||||||
|
Io::WouldBlock => Self::WouldBlock,
|
||||||
|
Io::InvalidInput => Self::InvalidInput,
|
||||||
|
Io::InvalidData => Self::InvalidData,
|
||||||
|
Io::TimedOut => Self::TimedOut,
|
||||||
|
Io::WriteZero => Self::WriteZero,
|
||||||
|
Io::Interrupted => Self::Interrupted,
|
||||||
|
Io::UnexpectedEof => Self::UnexpectedEof,
|
||||||
|
Io::OutOfMemory => Self::OutOfMemory,
|
||||||
|
Io::HostUnreachable => Self::HostUnreachable,
|
||||||
|
Io::NetworkUnreachable => Self::NetworkUnreachable,
|
||||||
|
Io::AddrInUse => Self::AddrInUse,
|
||||||
|
Io::AddrNotAvailable => Self::AddrNotAvailable,
|
||||||
|
Io::NetworkDown => Self::NetworkDown,
|
||||||
|
Io::NotADirectory => Self::NotADirectory,
|
||||||
|
Io::IsADirectory => Self::IsADirectory,
|
||||||
|
Io::DirectoryNotEmpty => Self::DirectoryNotEmpty,
|
||||||
|
Io::ReadOnlyFilesystem => Self::ReadOnlyFilesystem,
|
||||||
|
Io::StaleNetworkFileHandle => Self::StaleNetworkFileHandle,
|
||||||
|
Io::StorageFull => Self::StorageFull,
|
||||||
|
Io::NotSeekable => Self::NotSeekable,
|
||||||
|
Io::QuotaExceeded => Self::QuotaExceeded,
|
||||||
|
Io::FileTooLarge => Self::FileTooLarge,
|
||||||
|
Io::ResourceBusy => Self::ResourceBusy,
|
||||||
|
Io::ExecutableFileBusy => Self::ExecutableFileBusy,
|
||||||
|
Io::Deadlock => Self::Deadlock,
|
||||||
|
Io::CrossesDevices => Self::CrossesDevices,
|
||||||
|
Io::TooManyLinks => Self::TooManyLinks,
|
||||||
|
Io::InvalidFilename => Self::InvalidFilename,
|
||||||
|
Io::ArgumentListTooLong => Self::ArgumentListTooLong,
|
||||||
|
Io::Unsupported => Self::Unsupported,
|
||||||
|
Io::Other => Self::Other,
|
||||||
|
_ => Self::Unknown,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum CommandFailureStage {
|
||||||
|
Spawn,
|
||||||
|
Wait,
|
||||||
|
StdoutRead,
|
||||||
|
StderrRead,
|
||||||
|
StdoutDrain,
|
||||||
|
StderrDrain,
|
||||||
|
OutputDelivery,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct CommandFailure {
|
||||||
|
pub stage: CommandFailureStage,
|
||||||
|
pub reason: CommandIoErrorKind,
|
||||||
|
/// Observed numeric OS code, if supplied by the source. Never inferred from text.
|
||||||
|
pub os_code: Option<i32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CommandFailure {
|
||||||
|
pub(crate) fn io(stage: CommandFailureStage, error: &std::io::Error) -> Self {
|
||||||
|
Self {
|
||||||
|
stage,
|
||||||
|
reason: error.kind().into(),
|
||||||
|
os_code: error.raw_os_error(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// EOF collection is independent of exit status and the explicit byte budget.
|
||||||
|
/// Incomplete counts are lower bounds on bytes actually produced by the process.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)]
|
||||||
|
pub enum CommandOutputCollection {
|
||||||
|
NotStarted,
|
||||||
|
Complete,
|
||||||
|
Incomplete { failure: CommandFailure },
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Raw totals include discarded *observed* pipe bytes, not an unknown failed tail.
|
||||||
|
/// Delivered totals count only prefixes admitted to the transport queue, not page
|
||||||
|
/// reads or retries. Neither terminal arrival nor exit zero implies EOF collection.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct CommandTerminal {
|
pub struct CommandTerminal {
|
||||||
pub status: CommandStatus,
|
pub status: CommandStatus,
|
||||||
pub exit_code: Option<i32>,
|
pub exit_code: Option<i32>,
|
||||||
|
pub process_failure: Option<CommandFailure>,
|
||||||
|
pub output_collection: CommandOutputCollection,
|
||||||
pub stdout_bytes: u64,
|
pub stdout_bytes: u64,
|
||||||
pub stderr_bytes: u64,
|
pub stderr_bytes: u64,
|
||||||
pub stdout_delivered_bytes: u64,
|
pub stdout_delivered_bytes: u64,
|
||||||
@@ -145,6 +284,54 @@ pub struct CommandTerminal {
|
|||||||
pub truncation_reason: Option<CommandOutputTruncation>,
|
pub truncation_reason: Option<CommandOutputTruncation>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl CommandTerminal {
|
||||||
|
/// Validate provider-authored metadata before publishing it as collected evidence.
|
||||||
|
pub fn metadata_valid(&self) -> bool {
|
||||||
|
let truncated = self.stdout_bytes > self.stdout_delivered_bytes
|
||||||
|
|| self.stderr_bytes > self.stderr_delivered_bytes;
|
||||||
|
if self.status == CommandStatus::Running
|
||||||
|
|| self.stdout_delivered_bytes > self.stdout_bytes
|
||||||
|
|| self.stderr_delivered_bytes > self.stderr_bytes
|
||||||
|
|| self.truncation_reason != truncated.then_some(CommandOutputTruncation::ByteBudget)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
match self.process_failure {
|
||||||
|
Some(CommandFailure {
|
||||||
|
stage: CommandFailureStage::Spawn,
|
||||||
|
..
|
||||||
|
}) => {
|
||||||
|
self.status == CommandStatus::Failed
|
||||||
|
&& self.exit_code.is_none()
|
||||||
|
&& self.output_collection == CommandOutputCollection::NotStarted
|
||||||
|
&& self.stdout_bytes == 0
|
||||||
|
&& self.stderr_bytes == 0
|
||||||
|
}
|
||||||
|
Some(CommandFailure {
|
||||||
|
stage: CommandFailureStage::Wait,
|
||||||
|
..
|
||||||
|
}) => {
|
||||||
|
self.status != CommandStatus::Completed
|
||||||
|
&& self.exit_code.is_none()
|
||||||
|
&& self.collection_started()
|
||||||
|
}
|
||||||
|
Some(_) => false,
|
||||||
|
None => self.collection_started(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collection_started(&self) -> bool {
|
||||||
|
match self.output_collection {
|
||||||
|
CommandOutputCollection::NotStarted => false,
|
||||||
|
CommandOutputCollection::Complete => true,
|
||||||
|
CommandOutputCollection::Incomplete { failure } => !matches!(
|
||||||
|
failure.stage,
|
||||||
|
CommandFailureStage::Spawn | CommandFailureStage::Wait
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Sending next_cursor acknowledges this page. Only the current page may be
|
/// Sending next_cursor acknowledges this page. Only the current page may be
|
||||||
/// retried; acknowledged bytes expire. Terminal fences all captured bytes,
|
/// retried; acknowledged bytes expire. Terminal fences all captured bytes,
|
||||||
/// including on timeout/cancel. Consumers must pull while the process runs.
|
/// including on timeout/cancel. Consumers must pull while the process runs.
|
||||||
@@ -186,19 +373,13 @@ impl CommandOutput {
|
|||||||
self.next_cursor <= MAX_COMMAND_TRANSPORT_BYTES
|
self.next_cursor <= MAX_COMMAND_TRANSPORT_BYTES
|
||||||
&& end.is_none_or(|end| end == self.next_cursor)
|
&& end.is_none_or(|end| end == self.next_cursor)
|
||||||
&& self.terminal.as_ref().is_none_or(|terminal| {
|
&& self.terminal.as_ref().is_none_or(|terminal| {
|
||||||
let truncated = terminal.stdout_bytes > terminal.stdout_delivered_bytes
|
terminal.metadata_valid()
|
||||||
|| terminal.stderr_bytes > terminal.stderr_delivered_bytes;
|
&& stream_ends[0].is_none_or(|end| end == terminal.stdout_delivered_bytes)
|
||||||
stream_ends[0].is_none_or(|end| end == terminal.stdout_delivered_bytes)
|
|
||||||
&& stream_ends[1].is_none_or(|end| end == terminal.stderr_delivered_bytes)
|
&& stream_ends[1].is_none_or(|end| end == terminal.stderr_delivered_bytes)
|
||||||
&& terminal.status != CommandStatus::Running
|
|
||||||
&& terminal.stdout_delivered_bytes <= terminal.stdout_bytes
|
|
||||||
&& terminal.stderr_delivered_bytes <= terminal.stderr_bytes
|
|
||||||
&& terminal
|
&& terminal
|
||||||
.stdout_delivered_bytes
|
.stdout_delivered_bytes
|
||||||
.checked_add(terminal.stderr_delivered_bytes)
|
.checked_add(terminal.stderr_delivered_bytes)
|
||||||
== Some(self.next_cursor)
|
== Some(self.next_cursor)
|
||||||
&& terminal.truncation_reason
|
|
||||||
== truncated.then_some(CommandOutputTruncation::ByteBudget)
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -262,4 +443,70 @@ mod tests {
|
|||||||
.contains(&serde_json::json!("transport_byte_budget"))
|
.contains(&serde_json::json!("transport_byte_budget"))
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn failure_wire_is_closed_and_does_not_infer_os_codes_from_custom_messages() {
|
||||||
|
let error = std::io::Error::new(
|
||||||
|
std::io::ErrorKind::Other,
|
||||||
|
"/host/private TOKEN=secret EACCES command",
|
||||||
|
);
|
||||||
|
let failure = CommandFailure::io(CommandFailureStage::Wait, &error);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(failure).unwrap(),
|
||||||
|
serde_json::json!({"stage":"wait","reason":"other","os_code":null})
|
||||||
|
);
|
||||||
|
let os_error = std::io::Error::from_raw_os_error(libc::EIO);
|
||||||
|
let actual = CommandFailure::io(CommandFailureStage::StdoutDrain, &os_error);
|
||||||
|
assert_eq!(actual.os_code, Some(libc::EIO));
|
||||||
|
for invalid in [
|
||||||
|
serde_json::json!({"stage":"wait","reason":"/host/private TOKEN=secret"}),
|
||||||
|
serde_json::json!({"stage":"/host/private","reason":"other"}),
|
||||||
|
serde_json::json!({"stage":"wait","reason":"other","message":"secret"}),
|
||||||
|
] {
|
||||||
|
assert!(serde_json::from_value::<CommandFailure>(invalid).is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn provider_terminal_rejects_inconsistent_failure_domains_and_unstarted_byte_claims() {
|
||||||
|
let wire = serde_json::json!({"status":"completed","exit_code":0,"process_failure":null,"output_collection":{"state":"incomplete","failure":{"stage":"stdout_drain","reason":"other","os_code":null}},"stdout_bytes":3,"stderr_bytes":0,"stdout_delivered_bytes":3,"stderr_delivered_bytes":0,"truncation_reason":null});
|
||||||
|
let terminal: CommandTerminal = serde_json::from_value(wire.clone()).unwrap();
|
||||||
|
assert!(terminal.metadata_valid());
|
||||||
|
for (field, value) in [
|
||||||
|
(
|
||||||
|
"output_collection",
|
||||||
|
serde_json::json!({"state":"not_started"}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"output_collection",
|
||||||
|
serde_json::json!({"state":"incomplete","failure":{"stage":"wait","reason":"other"}}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"process_failure",
|
||||||
|
serde_json::json!({"stage":"stdout_read","reason":"other"}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"process_failure",
|
||||||
|
serde_json::json!({"stage":"spawn","reason":"not_found"}),
|
||||||
|
),
|
||||||
|
("truncation_reason", serde_json::json!("byte_budget")),
|
||||||
|
] {
|
||||||
|
let mut invalid = wire.clone();
|
||||||
|
invalid[field] = value;
|
||||||
|
assert!(
|
||||||
|
!serde_json::from_value::<CommandTerminal>(invalid)
|
||||||
|
.unwrap()
|
||||||
|
.metadata_valid()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let mut unclassified = wire.clone();
|
||||||
|
unclassified
|
||||||
|
.as_object_mut()
|
||||||
|
.unwrap()
|
||||||
|
.remove("output_collection");
|
||||||
|
assert!(serde_json::from_value::<CommandTerminal>(unclassified).is_err());
|
||||||
|
let mut raw = wire;
|
||||||
|
raw["message"] = serde_json::json!("/private TOKEN=secret");
|
||||||
|
assert!(serde_json::from_value::<CommandTerminal>(raw).is_err());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+236
-31
@@ -225,16 +225,27 @@ impl Drop for ActiveOperation<'_> {
|
|||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct RoutedWorkdirSession {
|
pub struct RoutedWorkdirSession {
|
||||||
state: Arc<RoutedSessionState>,
|
state: Arc<RoutedSessionState>,
|
||||||
|
view: Option<WorkdirSessionHandle>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RoutedWorkdirSession {
|
||||||
|
fn inner(&self) -> &WorkdirSessionHandle {
|
||||||
|
self.view.as_ref().unwrap_or(&self.state.session)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl WorkdirSession for RoutedWorkdirSession {
|
impl WorkdirSession for RoutedWorkdirSession {
|
||||||
fn workdir(&self) -> &Workdir {
|
fn workdir(&self) -> &Workdir {
|
||||||
self.state.session.workdir()
|
self.inner().workdir()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
||||||
self.state.session.capabilities()
|
self.inner().capabilities()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn environment(&self) -> crate::WorkdirSessionEnvironment {
|
||||||
|
self.inner().environment()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn authorize_scope_path(
|
async fn authorize_scope_path(
|
||||||
@@ -242,7 +253,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
|||||||
request: WorkdirScopeAuthorizationRequest,
|
request: WorkdirScopeAuthorizationRequest,
|
||||||
) -> Result<(), WorkdirError> {
|
) -> Result<(), WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.authorize_scope_path(request).await
|
self.inner().authorize_scope_path(request).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn scope_rules_overlap(
|
async fn scope_rules_overlap(
|
||||||
@@ -250,7 +261,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
|||||||
request: WorkdirScopeOverlapRequest,
|
request: WorkdirScopeOverlapRequest,
|
||||||
) -> Result<bool, WorkdirError> {
|
) -> Result<bool, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.scope_rules_overlap(request).await
|
self.inner().scope_rules_overlap(request).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn checkout_search(
|
async fn checkout_search(
|
||||||
@@ -258,7 +269,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
|||||||
request: crate::CheckoutSearchRequest,
|
request: crate::CheckoutSearchRequest,
|
||||||
) -> Result<crate::CheckoutSearchResult, WorkdirError> {
|
) -> Result<crate::CheckoutSearchResult, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.checkout_search(request).await
|
self.inner().checkout_search(request).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn checkout_observe(
|
async fn checkout_observe(
|
||||||
@@ -266,59 +277,59 @@ impl WorkdirSession for RoutedWorkdirSession {
|
|||||||
path: crate::WorkdirPath,
|
path: crate::WorkdirPath,
|
||||||
) -> Result<crate::CheckoutObservation, WorkdirError> {
|
) -> Result<crate::CheckoutObservation, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.checkout_observe(path).await
|
self.inner().checkout_observe(path).await
|
||||||
}
|
}
|
||||||
async fn checkout_execute(
|
async fn checkout_execute(
|
||||||
&self,
|
&self,
|
||||||
request: crate::CheckoutRequest,
|
request: crate::CheckoutRequest,
|
||||||
) -> Result<crate::CheckoutResult, WorkdirError> {
|
) -> Result<crate::CheckoutResult, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.checkout_execute(request).await
|
self.inner().checkout_execute(request).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn stat(&self, request: StatRequest) -> Result<StatResult, WorkdirError> {
|
async fn stat(&self, request: StatRequest) -> Result<StatResult, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.stat(request).await
|
self.inner().stat(request).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn read(&self, request: ReadRequest) -> Result<ReadResult, WorkdirError> {
|
async fn read(&self, request: ReadRequest) -> Result<ReadResult, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.read(request).await
|
self.inner().read(request).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn read_bytes(&self, request: ReadBytesRequest) -> Result<ReadBytesResult, WorkdirError> {
|
async fn read_bytes(&self, request: ReadBytesRequest) -> Result<ReadBytesResult, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.read_bytes(request).await
|
self.inner().read_bytes(request).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn write(&self, request: WriteRequest) -> Result<WriteResult, WorkdirError> {
|
async fn write(&self, request: WriteRequest) -> Result<WriteResult, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.write(request).await
|
self.inner().write(request).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn edit(&self, request: EditRequest) -> Result<EditResult, WorkdirError> {
|
async fn edit(&self, request: EditRequest) -> Result<EditResult, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.edit(request).await
|
self.inner().edit(request).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list(&self, request: ListRequest) -> Result<ListResult, WorkdirError> {
|
async fn list(&self, request: ListRequest) -> Result<ListResult, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.list(request).await
|
self.inner().list(request).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn glob(&self, request: GlobRequest) -> Result<GlobResult, WorkdirError> {
|
async fn glob(&self, request: GlobRequest) -> Result<GlobResult, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.glob(request).await
|
self.inner().glob(request).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn grep(&self, request: GrepRequest) -> Result<GrepResult, WorkdirError> {
|
async fn grep(&self, request: GrepRequest) -> Result<GrepResult, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.grep(request).await
|
self.inner().grep(request).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn start_command(&self, request: CommandRequest) -> Result<CommandHandle, WorkdirError> {
|
async fn start_command(&self, request: CommandRequest) -> Result<CommandHandle, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
let handle = self.state.session.start_command(request).await?;
|
let handle = self.inner().start_command(request).await?;
|
||||||
self.state
|
self.state
|
||||||
.active_commands
|
.active_commands
|
||||||
.lock()
|
.lock()
|
||||||
@@ -329,7 +340,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
|||||||
|
|
||||||
async fn command_status(&self, handle: CommandHandle) -> Result<CommandStatus, WorkdirError> {
|
async fn command_status(&self, handle: CommandHandle) -> Result<CommandStatus, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
let status = self.state.session.command_status(handle.clone()).await?;
|
let status = self.inner().command_status(handle.clone()).await?;
|
||||||
self.state.mark_terminal(&handle, status);
|
self.state.mark_terminal(&handle, status);
|
||||||
Ok(status)
|
Ok(status)
|
||||||
}
|
}
|
||||||
@@ -340,7 +351,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
|||||||
) -> Result<CommandOutput, WorkdirError> {
|
) -> Result<CommandOutput, WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
let handle = request.handle.clone();
|
let handle = request.handle.clone();
|
||||||
let output = self.state.session.command_output(request).await?;
|
let output = self.inner().command_output(request).await?;
|
||||||
if let Some(terminal) = &output.terminal {
|
if let Some(terminal) = &output.terminal {
|
||||||
self.state.mark_terminal(&handle, terminal.status);
|
self.state.mark_terminal(&handle, terminal.status);
|
||||||
}
|
}
|
||||||
@@ -349,7 +360,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
|||||||
|
|
||||||
async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError> {
|
async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError> {
|
||||||
let _active = self.state.enter()?;
|
let _active = self.state.enter()?;
|
||||||
self.state.session.cancel_command(handle.clone()).await?;
|
self.inner().cancel_command(handle.clone()).await?;
|
||||||
self.state
|
self.state
|
||||||
.active_commands
|
.active_commands
|
||||||
.lock()
|
.lock()
|
||||||
@@ -359,11 +370,11 @@ impl WorkdirSession for RoutedWorkdirSession {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<CommandEvent>> {
|
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<CommandEvent>> {
|
||||||
self.state.session.subscribe_command_events()
|
self.inner().subscribe_command_events()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn command_snapshot(&self) -> Vec<CommandSnapshot> {
|
fn command_snapshot(&self) -> Vec<CommandSnapshot> {
|
||||||
let snapshots = self.state.session.command_snapshot();
|
let snapshots = self.inner().command_snapshot();
|
||||||
self.state.reconcile_terminal_commands(&snapshots);
|
self.state.reconcile_terminal_commands(&snapshots);
|
||||||
snapshots
|
snapshots
|
||||||
}
|
}
|
||||||
@@ -374,7 +385,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
|||||||
"Workdir attachment has active operations or commands".to_string(),
|
"Workdir attachment has active operations or commands".to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
self.state.session.close().await
|
self.inner().close().await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -387,6 +398,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
|||||||
pub struct WorkdirSessionRouter {
|
pub struct WorkdirSessionRouter {
|
||||||
sessions: RwLock<BTreeMap<WorkdirAttachmentAlias, Arc<RoutedSessionState>>>,
|
sessions: RwLock<BTreeMap<WorkdirAttachmentAlias, Arc<RoutedSessionState>>>,
|
||||||
next_generation: AtomicU64,
|
next_generation: AtomicU64,
|
||||||
|
coordinate_brokers: RwLock<BTreeMap<WorkdirAttachmentAlias, (u64, crate::WorkdirToolBroker)>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WorkdirSessionRouter {
|
impl WorkdirSessionRouter {
|
||||||
@@ -473,9 +485,9 @@ impl WorkdirSessionRouter {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
Ok(ResolvedWorkdirSession {
|
Ok(ResolvedWorkdirSession {
|
||||||
|
session: self.routed(&selected.0, selected.1.clone()),
|
||||||
alias: selected.0,
|
alias: selected.0,
|
||||||
generation: selected.1.generation,
|
generation: selected.1.generation,
|
||||||
session: Arc::new(RoutedWorkdirSession { state: selected.1 }),
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -483,13 +495,22 @@ impl WorkdirSessionRouter {
|
|||||||
&self,
|
&self,
|
||||||
alias: WorkdirAttachmentAlias,
|
alias: WorkdirAttachmentAlias,
|
||||||
session: WorkdirSessionHandle,
|
session: WorkdirSessionHandle,
|
||||||
|
) -> Result<WorkdirSessionHandle, WorkdirError> {
|
||||||
|
self.attach_with_cwd(alias, session, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn attach_with_cwd(
|
||||||
|
&self,
|
||||||
|
alias: WorkdirAttachmentAlias,
|
||||||
|
session: WorkdirSessionHandle,
|
||||||
|
cwd: Option<crate::WorkdirPath>,
|
||||||
) -> Result<WorkdirSessionHandle, WorkdirError> {
|
) -> Result<WorkdirSessionHandle, WorkdirError> {
|
||||||
let mut sessions = self.sessions.write().map_err(|_| {
|
let mut sessions = self.sessions.write().map_err(|_| {
|
||||||
WorkdirError::Unavailable("Workdir attachment router is poisoned".to_string())
|
WorkdirError::Unavailable("Workdir attachment router is poisoned".into())
|
||||||
})?;
|
})?;
|
||||||
if sessions.contains_key(&alias) {
|
if sessions.contains_key(&alias) {
|
||||||
return Err(WorkdirError::Conflict(format!(
|
return Err(WorkdirError::Conflict(format!(
|
||||||
"Workdir attachment alias `{alias}` already exists"
|
"Workdir attachment alias \u{0060}{alias}\u{0060} already exists"
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
let state = Arc::new(RoutedSessionState {
|
let state = Arc::new(RoutedSessionState {
|
||||||
@@ -499,8 +520,61 @@ impl WorkdirSessionRouter {
|
|||||||
active_operations: AtomicUsize::new(0),
|
active_operations: AtomicUsize::new(0),
|
||||||
active_commands: Mutex::new(HashSet::new()),
|
active_commands: Mutex::new(HashSet::new()),
|
||||||
});
|
});
|
||||||
|
let view = if let Some(cwd) = cwd {
|
||||||
|
let source = Arc::new(RoutedWorkdirSession {
|
||||||
|
state: state.clone(),
|
||||||
|
view: None,
|
||||||
|
});
|
||||||
|
let broker = crate::WorkdirToolBroker::at_cwd(source, cwd)?;
|
||||||
|
let view = broker.tool_session();
|
||||||
|
self.coordinate_brokers
|
||||||
|
.write()
|
||||||
|
.map_err(|_| {
|
||||||
|
WorkdirError::Unavailable("Workdir coordinate broker map is poisoned".into())
|
||||||
|
})?
|
||||||
|
.insert(alias.clone(), (state.generation, broker));
|
||||||
|
Some(view)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
sessions.insert(alias, state.clone());
|
sessions.insert(alias, state.clone());
|
||||||
Ok(Arc::new(RoutedWorkdirSession { state }))
|
Ok(Arc::new(RoutedWorkdirSession { state, view }))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn routed(
|
||||||
|
&self,
|
||||||
|
alias: &WorkdirAttachmentAlias,
|
||||||
|
state: Arc<RoutedSessionState>,
|
||||||
|
) -> Arc<RoutedWorkdirSession> {
|
||||||
|
let view = self
|
||||||
|
.coordinate_broker(alias, state.generation)
|
||||||
|
.map(|broker| broker.tool_session());
|
||||||
|
Arc::new(RoutedWorkdirSession { state, view })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn coordinate_broker(
|
||||||
|
&self,
|
||||||
|
alias: &WorkdirAttachmentAlias,
|
||||||
|
generation: u64,
|
||||||
|
) -> Option<crate::WorkdirToolBroker> {
|
||||||
|
self.coordinate_brokers
|
||||||
|
.read()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.get(alias)
|
||||||
|
.filter(|(current, _)| *current == generation)
|
||||||
|
.map(|(_, broker)| broker.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bind tool coordinates once while retaining root-coordinate delegation
|
||||||
|
/// and provider authorization. Both views share the route's detach fence.
|
||||||
|
pub fn attach_at_cwd(
|
||||||
|
&self,
|
||||||
|
alias: WorkdirAttachmentAlias,
|
||||||
|
source: WorkdirSessionHandle,
|
||||||
|
cwd: crate::WorkdirPath,
|
||||||
|
) -> Result<WorkdirSessionHandle, WorkdirError> {
|
||||||
|
let cwd = crate::WorkdirPath::new(cwd.as_str())?;
|
||||||
|
self.attach_with_cwd(alias, source, Some(cwd))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn session(&self, alias: &WorkdirAttachmentAlias) -> Option<WorkdirSessionHandle> {
|
pub fn session(&self, alias: &WorkdirAttachmentAlias) -> Option<WorkdirSessionHandle> {
|
||||||
@@ -509,7 +583,7 @@ impl WorkdirSessionRouter {
|
|||||||
.ok()?
|
.ok()?
|
||||||
.get(alias)
|
.get(alias)
|
||||||
.cloned()
|
.cloned()
|
||||||
.map(|state| Arc::new(RoutedWorkdirSession { state }) as WorkdirSessionHandle)
|
.map(|state| self.routed(alias, state) as WorkdirSessionHandle)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Compatibility projection used only while filesystem tool schemas have no
|
/// Compatibility projection used only while filesystem tool schemas have no
|
||||||
@@ -521,10 +595,8 @@ impl WorkdirSessionRouter {
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
sessions
|
sessions
|
||||||
.values()
|
.first_key_value()
|
||||||
.next()
|
.map(|(alias, state)| self.routed(alias, state.clone()) as WorkdirSessionHandle)
|
||||||
.cloned()
|
|
||||||
.map(|state| Arc::new(RoutedWorkdirSession { state }) as WorkdirSessionHandle)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn can_detach(&self, alias: &WorkdirAttachmentAlias) -> Result<(), WorkdirError> {
|
pub fn can_detach(&self, alias: &WorkdirAttachmentAlias) -> Result<(), WorkdirError> {
|
||||||
@@ -620,6 +692,10 @@ impl WorkdirSessionRouter {
|
|||||||
.is_some_and(|current| Arc::ptr_eq(current, &state))
|
.is_some_and(|current| Arc::ptr_eq(current, &state))
|
||||||
{
|
{
|
||||||
sessions.remove(alias);
|
sessions.remove(alias);
|
||||||
|
self.coordinate_brokers
|
||||||
|
.write()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.remove(alias);
|
||||||
}
|
}
|
||||||
Ok(state.session.clone())
|
Ok(state.session.clone())
|
||||||
}
|
}
|
||||||
@@ -661,6 +737,10 @@ impl WorkdirSessionRouter {
|
|||||||
.is_some_and(|current| Arc::ptr_eq(current, &state))
|
.is_some_and(|current| Arc::ptr_eq(current, &state))
|
||||||
{
|
{
|
||||||
sessions.remove(&alias);
|
sessions.remove(&alias);
|
||||||
|
self.coordinate_brokers
|
||||||
|
.write()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.remove(&alias);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -684,6 +764,131 @@ mod tests {
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cwd_views_share_root_delegation_leases_and_attachment_fences() {
|
||||||
|
use crate::{
|
||||||
|
WorkdirPath as P, WorkdirToolBrokerRouter, WorkdirToolScope,
|
||||||
|
WorkdirToolScopePermission as Permission, WorkdirToolScopeRule,
|
||||||
|
};
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
for directory in ["work", "other", "other/inside"] {
|
||||||
|
std::fs::create_dir_all(root.path().join(directory)).unwrap();
|
||||||
|
std::fs::write(root.path().join(directory).join("marker"), directory).unwrap();
|
||||||
|
}
|
||||||
|
let router = Arc::new(WorkdirSessionRouter::new());
|
||||||
|
let alias = WorkdirAttachmentAlias::new("source").unwrap();
|
||||||
|
let parent = router
|
||||||
|
.attach_at_cwd(alias.clone(), session("wd", &root), P::new("work").unwrap())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(parent.environment().cwd, P::new("work").unwrap());
|
||||||
|
let request = |cwd: &str| WorkdirToolScope {
|
||||||
|
rules: vec![WorkdirToolScopeRule {
|
||||||
|
target: P::new(cwd).unwrap(),
|
||||||
|
permission: Permission::Write,
|
||||||
|
recursive: true,
|
||||||
|
symlink_policy: Default::default(),
|
||||||
|
}],
|
||||||
|
cwd: P::new(cwd).unwrap(),
|
||||||
|
command: false,
|
||||||
|
};
|
||||||
|
let brokers = WorkdirToolBrokerRouter::new(router.clone());
|
||||||
|
// Declarations stay root-relative, not work/other.
|
||||||
|
let child = brokers
|
||||||
|
.scope(vec![(alias.clone(), request("other"))])
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let grandchild = child
|
||||||
|
.broker_router()
|
||||||
|
.scope(vec![(alias.clone(), request("other/inside"))])
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
for (session, cwd) in [
|
||||||
|
(parent.clone(), "work"),
|
||||||
|
(
|
||||||
|
child.session_router().resolve(None).unwrap().session,
|
||||||
|
"other",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
grandchild.session_router().resolve(None).unwrap().session,
|
||||||
|
"other/inside",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
assert_eq!(session.environment().cwd, P::new(cwd).unwrap());
|
||||||
|
let result = session
|
||||||
|
.read(ReadRequest {
|
||||||
|
path: P::new("marker").unwrap(),
|
||||||
|
offset: 0,
|
||||||
|
limit: 10,
|
||||||
|
max_bytes: 100,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(result.bytes, cwd.as_bytes());
|
||||||
|
assert_eq!(result.path, P::new("marker").unwrap());
|
||||||
|
assert_eq!(
|
||||||
|
session
|
||||||
|
.stat(StatRequest {
|
||||||
|
path: P::new("marker").unwrap()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.path,
|
||||||
|
P::new("marker").unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// The model's tool view and delegation view share write-lease exclusion.
|
||||||
|
let absolute = P::new_scoped(root.path().join("other/marker").to_str().unwrap()).unwrap();
|
||||||
|
let denied = parent
|
||||||
|
.write(WriteRequest {
|
||||||
|
path: absolute,
|
||||||
|
content: b"forbidden".to_vec(),
|
||||||
|
expected_hash: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert_eq!(
|
||||||
|
denied.denial_reason(),
|
||||||
|
Some(crate::WorkdirDenialReason::ChildWriteLeaseConflict)
|
||||||
|
);
|
||||||
|
let old_child = child.session_router().resolve(None).unwrap().session;
|
||||||
|
router.begin_detach(&alias).await.unwrap();
|
||||||
|
assert!(
|
||||||
|
old_child
|
||||||
|
.read(ReadRequest {
|
||||||
|
path: P::new("marker").unwrap(),
|
||||||
|
offset: 0,
|
||||||
|
limit: 10,
|
||||||
|
max_bytes: 100
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
router.cancel_detach(&alias);
|
||||||
|
grandchild.close().await.unwrap();
|
||||||
|
child.close().await.unwrap();
|
||||||
|
router.detach(&alias).await.unwrap();
|
||||||
|
router
|
||||||
|
.attach_at_cwd(
|
||||||
|
alias,
|
||||||
|
session("replacement", &root),
|
||||||
|
P::new("other").unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
router.resolve(None).unwrap().session.environment().cwd,
|
||||||
|
P::new("other").unwrap()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
old_child
|
||||||
|
.stat(StatRequest {
|
||||||
|
path: P::new("marker").unwrap()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
router.close_all().await.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn checkout_routes_use_active_operation_detach_fence_and_old_handles_stay_closed() {
|
async fn checkout_routes_use_active_operation_detach_fence_and_old_handles_stay_closed() {
|
||||||
let root = TempDir::new().unwrap();
|
let root = TempDir::new().unwrap();
|
||||||
|
|||||||
@@ -108,17 +108,34 @@ impl WorkdirToolBroker {
|
|||||||
Self::with_capabilities(source, capabilities)
|
Self::with_capabilities(source, capabilities)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Set the tool coordinate base without changing provider authority.
|
||||||
|
/// The input is relative to the source session's coordinates; delegation
|
||||||
|
/// declarations retain their existing coordinate contract.
|
||||||
|
pub fn at_cwd(source: WorkdirSessionHandle, cwd: FsPath) -> Result<Self, WorkdirError> {
|
||||||
|
let cwd = FsPath::new(cwd.as_str())?;
|
||||||
|
let capabilities = source.capabilities();
|
||||||
|
Ok(Self::with_capabilities_and_cwd(source, capabilities, cwd))
|
||||||
|
}
|
||||||
|
|
||||||
/// Own a session behind an additional Backend-authored capability ceiling.
|
/// Own a session behind an additional Backend-authored capability ceiling.
|
||||||
/// The ceiling can only attenuate provider capabilities; it can never add one.
|
/// The ceiling can only attenuate provider capabilities; it can never add one.
|
||||||
pub fn with_capabilities(
|
pub fn with_capabilities(
|
||||||
source: WorkdirSessionHandle,
|
source: WorkdirSessionHandle,
|
||||||
capabilities: WorkdirSessionCapabilities,
|
capabilities: WorkdirSessionCapabilities,
|
||||||
|
) -> Self {
|
||||||
|
Self::with_capabilities_and_cwd(source, capabilities, FsPath::root())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_capabilities_and_cwd(
|
||||||
|
source: WorkdirSessionHandle,
|
||||||
|
capabilities: WorkdirSessionCapabilities,
|
||||||
|
cwd: FsPath,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let capabilities = source.capabilities().intersection(capabilities);
|
let capabilities = source.capabilities().intersection(capabilities);
|
||||||
let (command_events, _) = broadcast::channel(64);
|
let (command_events, _) = broadcast::channel(64);
|
||||||
let authority = Arc::new(ScopedWorkdirSession {
|
let authority = Arc::new(ScopedWorkdirSession {
|
||||||
source,
|
source,
|
||||||
cwd: FsPath::new("").expect("empty Workdir path is valid"),
|
cwd,
|
||||||
scope: None,
|
scope: None,
|
||||||
capabilities,
|
capabilities,
|
||||||
validity: SessionValidity::root(),
|
validity: SessionValidity::root(),
|
||||||
@@ -416,6 +433,9 @@ impl WorkdirToolBrokerRouter {
|
|||||||
.sessions
|
.sessions
|
||||||
.resolve(Some(alias.as_str()))
|
.resolve(Some(alias.as_str()))
|
||||||
.map_err(|error| WorkdirError::InvalidArgument(error.to_string()))?;
|
.map_err(|error| WorkdirError::InvalidArgument(error.to_string()))?;
|
||||||
|
if let Some(broker) = self.sessions.coordinate_broker(alias, selected.generation) {
|
||||||
|
return Ok(broker);
|
||||||
|
}
|
||||||
let mut brokers = self.brokers.lock().map_err(|_| {
|
let mut brokers = self.brokers.lock().map_err(|_| {
|
||||||
WorkdirError::Unavailable("Workdir tool broker router is poisoned".to_string())
|
WorkdirError::Unavailable("Workdir tool broker router is poisoned".to_string())
|
||||||
})?;
|
})?;
|
||||||
@@ -684,7 +704,7 @@ impl ScopedWorkdirSession {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn resolve_path(&self, path: &FsPath) -> Result<FsPath, WorkdirError> {
|
fn resolve_path(&self, path: &FsPath) -> Result<FsPath, WorkdirError> {
|
||||||
if self.cwd.as_str().is_empty() {
|
if self.cwd.as_str().is_empty() || Path::new(path.as_str()).is_absolute() {
|
||||||
return Ok(path.clone());
|
return Ok(path.clone());
|
||||||
}
|
}
|
||||||
let joined = Path::new(self.cwd.as_str()).join(path.as_str());
|
let joined = Path::new(self.cwd.as_str()).join(path.as_str());
|
||||||
@@ -1085,6 +1105,43 @@ impl WorkdirSession for ScopedWorkdirSession {
|
|||||||
self.capabilities
|
self.capabilities
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn environment(&self) -> crate::WorkdirSessionEnvironment {
|
||||||
|
let mut environment = self.source.environment();
|
||||||
|
let source_cwd = environment.cwd.clone();
|
||||||
|
environment.cwd = FsPath::new(
|
||||||
|
Path::new(source_cwd.as_str())
|
||||||
|
.join(self.cwd.as_str())
|
||||||
|
.to_str()
|
||||||
|
.expect("validated UTF-8 paths"),
|
||||||
|
)
|
||||||
|
.expect("validated session cwd");
|
||||||
|
if let Some(scope) = &self.scope {
|
||||||
|
if let Some(inherited) = environment.scope.take() {
|
||||||
|
environment
|
||||||
|
.restrictions
|
||||||
|
.push(format!("Provider allow ceiling: {inherited:?}"));
|
||||||
|
}
|
||||||
|
environment.scope = Some(
|
||||||
|
scope
|
||||||
|
.iter()
|
||||||
|
.cloned()
|
||||||
|
.map(|mut rule| {
|
||||||
|
rule.target = FsPath::new(
|
||||||
|
Path::new(source_cwd.as_str())
|
||||||
|
.join(rule.target.as_str())
|
||||||
|
.to_str()
|
||||||
|
.expect("validated UTF-8 paths"),
|
||||||
|
)
|
||||||
|
.expect("validated scope target");
|
||||||
|
rule
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
environment.capabilities = self.capabilities();
|
||||||
|
environment
|
||||||
|
}
|
||||||
|
|
||||||
async fn authorize_scope_path(
|
async fn authorize_scope_path(
|
||||||
&self,
|
&self,
|
||||||
mut request: WorkdirScopeAuthorizationRequest,
|
mut request: WorkdirScopeAuthorizationRequest,
|
||||||
@@ -1273,8 +1330,11 @@ impl WorkdirSession for ScopedWorkdirSession {
|
|||||||
.resolve_operation_path(&request.path, WorkdirToolScopePermission::Read)
|
.resolve_operation_path(&request.path, WorkdirToolScopePermission::Read)
|
||||||
.await?;
|
.await?;
|
||||||
self.ensure_read(&path, WorkdirSessionCapability::Read)?;
|
self.ensure_read(&path, WorkdirSessionCapability::Read)?;
|
||||||
|
let caller_path = request.path.clone();
|
||||||
request.path = path;
|
request.path = path;
|
||||||
self.source.stat(request).await
|
let mut result = self.source.stat(request).await?;
|
||||||
|
result.path = caller_path;
|
||||||
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn read(&self, mut request: ReadRequest) -> Result<ReadResult, WorkdirError> {
|
async fn read(&self, mut request: ReadRequest) -> Result<ReadResult, WorkdirError> {
|
||||||
@@ -1282,8 +1342,11 @@ impl WorkdirSession for ScopedWorkdirSession {
|
|||||||
.resolve_operation_path(&request.path, WorkdirToolScopePermission::Read)
|
.resolve_operation_path(&request.path, WorkdirToolScopePermission::Read)
|
||||||
.await?;
|
.await?;
|
||||||
self.ensure_read(&path, WorkdirSessionCapability::Read)?;
|
self.ensure_read(&path, WorkdirSessionCapability::Read)?;
|
||||||
|
let caller_path = request.path.clone();
|
||||||
request.path = path;
|
request.path = path;
|
||||||
self.source.read(request).await
|
let mut result = self.source.read(request).await?;
|
||||||
|
result.path = caller_path;
|
||||||
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn read_bytes(
|
async fn read_bytes(
|
||||||
@@ -1422,9 +1485,10 @@ impl WorkdirSession for ScopedWorkdirSession {
|
|||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
let tool_call_id = request.tool_call_id.clone();
|
let tool_call_id = request.tool_call_id.clone();
|
||||||
if self.scope.is_some() {
|
request.cwd = self
|
||||||
request.cwd = self.resolve_path(&request.cwd)?;
|
.resolve_operation_path(&request.cwd, WorkdirToolScopePermission::Read)
|
||||||
}
|
.await?;
|
||||||
|
self.ensure_path(&request.cwd, WorkdirToolScopePermission::Read)?;
|
||||||
if let Some(tool_call_id) = &tool_call_id {
|
if let Some(tool_call_id) = &tool_call_id {
|
||||||
self.starting_tool_calls
|
self.starting_tool_calls
|
||||||
.lock()
|
.lock()
|
||||||
@@ -1601,7 +1665,20 @@ impl WorkdirSession for ReadOnlyWorkdirSession {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
||||||
WorkdirSessionCapabilities::READ_ONLY
|
self.inner
|
||||||
|
.capabilities()
|
||||||
|
.intersection(WorkdirSessionCapabilities::READ_ONLY)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn environment(&self) -> crate::WorkdirSessionEnvironment {
|
||||||
|
let mut environment = self.inner.environment();
|
||||||
|
environment.capabilities = self.capabilities();
|
||||||
|
if let Some(scope) = &mut environment.scope {
|
||||||
|
for rule in scope {
|
||||||
|
rule.permission = WorkdirToolScopePermission::Read;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
environment
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn authorize_scope_path(
|
async fn authorize_scope_path(
|
||||||
|
|||||||
@@ -83,3 +83,34 @@ fn provider_cannot_inject_free_text_failure_reason() {
|
|||||||
.is_err()
|
.is_err()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn provider_authored_operational_messages_never_enter_error_display_or_debug() {
|
||||||
|
for code in [
|
||||||
|
Code::Conflict,
|
||||||
|
Code::OutcomeUnknown,
|
||||||
|
Code::Unsupported,
|
||||||
|
Code::InvalidRequest,
|
||||||
|
Code::Unavailable,
|
||||||
|
Code::Io,
|
||||||
|
Code::Transport,
|
||||||
|
Code::Internal,
|
||||||
|
] {
|
||||||
|
let wire = serde_json::json!({
|
||||||
|
"code": code, "message":"/host/private command-secret body-secret proof-secret key-secret Authorization: Bearer bearer-secret",
|
||||||
|
"failure_reason": "session_open_task_failed"
|
||||||
|
});
|
||||||
|
let decoded: WorkdirTransportError = serde_json::from_value(wire).unwrap();
|
||||||
|
let error = decoded.into_workdir_error();
|
||||||
|
for text in [
|
||||||
|
format!("{error}"),
|
||||||
|
format!("{error:?}"),
|
||||||
|
serde_json::to_string(&WorkdirTransportError::from_workdir_error(&error)).unwrap(),
|
||||||
|
] {
|
||||||
|
assert!(!text.contains("secret"), "{text}");
|
||||||
|
assert!(!text.contains("/host/private"), "{text}");
|
||||||
|
}
|
||||||
|
assert_eq!(error.failure_reason(), Some(Reason::SessionOpenTaskFailed));
|
||||||
|
assert_eq!(WorkdirTransportError::from_workdir_error(&error).code, code);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -14,6 +14,18 @@ use workdir::{
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
fn opened(workdir_id: &str) -> String {
|
||||||
|
serde_json::to_string(&workdir::http::OpenWorkdirSessionResponse {
|
||||||
|
session_id: workdir::http::WorkdirSessionId::new("s").unwrap(),
|
||||||
|
workdir_id: Workdir::new(workdir_id).id().clone(),
|
||||||
|
capabilities: workdir::WorkdirSessionCapabilities::ALL,
|
||||||
|
environment: workdir::WorkdirSessionEnvironment::root(
|
||||||
|
workdir::WorkdirSessionCapabilities::ALL,
|
||||||
|
),
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
fn reply(listener: &TcpListener, status: &str, body: &str) {
|
fn reply(listener: &TcpListener, status: &str, body: &str) {
|
||||||
listener.set_nonblocking(true).unwrap();
|
listener.set_nonblocking(true).unwrap();
|
||||||
let deadline = Instant::now() + Duration::from_secs(5);
|
let deadline = Instant::now() + Duration::from_secs(5);
|
||||||
@@ -73,11 +85,7 @@ async fn remote_http_denials_forward_only_typed_reasons_and_old_missing_reason()
|
|||||||
error["denial_reason"] = serde_json::to_value(reason).unwrap();
|
error["denial_reason"] = serde_json::to_value(reason).unwrap();
|
||||||
}
|
}
|
||||||
let server = thread::spawn(move || {
|
let server = thread::spawn(move || {
|
||||||
reply(
|
reply(&listener, "200 OK", &opened("w"));
|
||||||
&listener,
|
|
||||||
"200 OK",
|
|
||||||
r#"{"session_id":"s","workdir_id":"w","capabilities":{"bits":63}}"#,
|
|
||||||
);
|
|
||||||
reply(&listener, "403 Forbidden", &error.to_string());
|
reply(&listener, "403 Forbidden", &error.to_string());
|
||||||
});
|
});
|
||||||
let remote = RemoteWorkdirSession::open(
|
let remote = RemoteWorkdirSession::open(
|
||||||
@@ -163,11 +171,7 @@ async fn remote_checkout_diagnostics_preserve_mutation_outcome_and_read_failures
|
|||||||
let address = listener.local_addr().unwrap();
|
let address = listener.local_addr().unwrap();
|
||||||
let error = serde_json::json!({"code":code, "message":"/secret/host provider-secret", "denial_reason":reason});
|
let error = serde_json::json!({"code":code, "message":"/secret/host provider-secret", "denial_reason":reason});
|
||||||
let server = thread::spawn(move || {
|
let server = thread::spawn(move || {
|
||||||
reply(
|
reply(&listener, "200 OK", &opened("w"));
|
||||||
&listener,
|
|
||||||
"200 OK",
|
|
||||||
r#"{"session_id":"s","workdir_id":"w","capabilities":{"bits":63}}"#,
|
|
||||||
);
|
|
||||||
reply(
|
reply(
|
||||||
&listener,
|
&listener,
|
||||||
&format!("{} Fixture failure", code.http_status()),
|
&format!("{} Fixture failure", code.http_status()),
|
||||||
@@ -216,3 +220,106 @@ async fn remote_checkout_diagnostics_preserve_mutation_outcome_and_read_failures
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn mismatched_http_results_keep_cause_and_mutation_uncertainty() {
|
||||||
|
use workdir::{
|
||||||
|
CheckoutOperation, CheckoutRequest, CommandHandle, CommandOutputRequest,
|
||||||
|
WorkdirFailureReason as Failure,
|
||||||
|
};
|
||||||
|
for case in ["variant", "bytes", "mutation", "identity"] {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
let server = thread::spawn(move || {
|
||||||
|
reply(
|
||||||
|
&listener,
|
||||||
|
"200 OK",
|
||||||
|
&opened(if case == "identity" {
|
||||||
|
"wrong-secret"
|
||||||
|
} else {
|
||||||
|
"w"
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
if case != "identity" {
|
||||||
|
let result = if case == "bytes" {
|
||||||
|
serde_json::json!({"operation":"command_output","result":{
|
||||||
|
"chunks": [], "next_cursor": 123, "terminal": null
|
||||||
|
}})
|
||||||
|
} else {
|
||||||
|
serde_json::to_value(
|
||||||
|
workdir::http::WorkdirSessionOperationResult::CommandCancel,
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
reply(&listener, "200 OK", &result.to_string());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let opened = RemoteWorkdirSession::open(
|
||||||
|
reqwest::Client::builder()
|
||||||
|
.no_proxy()
|
||||||
|
.timeout(Duration::from_secs(5))
|
||||||
|
.build()
|
||||||
|
.unwrap(),
|
||||||
|
format!("http://{address}").parse().unwrap(),
|
||||||
|
"request-secret",
|
||||||
|
Workdir::new("w").id().clone(),
|
||||||
|
OpenWorkdirSessionRequest::default(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let error = if case == "identity" {
|
||||||
|
opened.unwrap_err()
|
||||||
|
} else {
|
||||||
|
let remote = opened.unwrap();
|
||||||
|
match case {
|
||||||
|
"variant" => remote
|
||||||
|
.stat(StatRequest {
|
||||||
|
path: WorkdirPath::new("file-secret").unwrap(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_err(),
|
||||||
|
"bytes" => remote
|
||||||
|
.command_output(CommandOutputRequest {
|
||||||
|
handle: CommandHandle("command-secret".into()),
|
||||||
|
cursor: 0,
|
||||||
|
limit: 16,
|
||||||
|
wait: false,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_err(),
|
||||||
|
"mutation" => remote
|
||||||
|
.checkout_execute(CheckoutRequest {
|
||||||
|
target: WorkdirPath::new("file-secret").unwrap(),
|
||||||
|
validator: vec![1],
|
||||||
|
operation: CheckoutOperation::Write {
|
||||||
|
content: b"body-secret".to_vec(),
|
||||||
|
expected_hash: [0; 32],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap_err(),
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
server.join().unwrap();
|
||||||
|
let expected = if case == "identity" {
|
||||||
|
Failure::SessionIdentityMismatch
|
||||||
|
} else {
|
||||||
|
Failure::OperationResultMismatch
|
||||||
|
};
|
||||||
|
assert_eq!(error.failure_reason(), Some(expected), "{case}: {error:?}");
|
||||||
|
let transport = WorkdirTransportError::from_workdir_error(&error);
|
||||||
|
assert_eq!(
|
||||||
|
transport.code,
|
||||||
|
if case == "mutation" {
|
||||||
|
Code::OutcomeUnknown
|
||||||
|
} else {
|
||||||
|
Code::Unavailable
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let wire = serde_json::to_string(&transport).unwrap();
|
||||||
|
assert!(!wire.contains("secret"), "{wire}");
|
||||||
|
assert!(!format!("{error:?}").contains("secret"));
|
||||||
|
let forwarded = transport.into_workdir_error();
|
||||||
|
assert_eq!(forwarded.failure_reason(), Some(expected));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -19,15 +19,19 @@ required-features = ["ws-server", "fs-store"]
|
|||||||
[features]
|
[features]
|
||||||
default = ["ws-server", "fs-store"]
|
default = ["ws-server", "fs-store"]
|
||||||
fs-store = ["dep:fs4"]
|
fs-store = ["dep:fs4"]
|
||||||
|
# Offline fixture tooling only; never enabled by the product defaults.
|
||||||
|
test-support = ["fs-store", "ws-server"]
|
||||||
|
# Explicit process-test synchronization; independent of offline fixture tooling.
|
||||||
|
restore-checkpoints = ["fs-store"]
|
||||||
http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
|
http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
|
||||||
ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"]
|
ws-server = ["http-server", "axum/ws", "tokio/sync"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
agen.workspace = true
|
agen.workspace = true
|
||||||
async-trait.workspace = true
|
async-trait.workspace = true
|
||||||
base64.workspace = true
|
base64.workspace = true
|
||||||
axum = { workspace = true, optional = true }
|
axum = { workspace = true, optional = true }
|
||||||
futures = { workspace = true, optional = true }
|
futures.workspace = true
|
||||||
decodal.workspace = true
|
decodal.workspace = true
|
||||||
flow = { path = "../flow" }
|
flow = { path = "../flow" }
|
||||||
fs4 = { workspace = true, optional = true }
|
fs4 = { workspace = true, optional = true }
|
||||||
@@ -56,6 +60,7 @@ server-api = { path = "../server-api" }
|
|||||||
workdir.workspace = true
|
workdir.workspace = true
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
|
tools.workspace = true
|
||||||
workdir = { workspace = true, features = ["http-client"] }
|
workdir = { workspace = true, features = ["http-client"] }
|
||||||
futures.workspace = true
|
futures.workspace = true
|
||||||
agen.workspace = true
|
agen.workspace = true
|
||||||
|
|||||||
@@ -8,3 +8,61 @@ Manages Worker creation, stopping, and Restore, execution connections within the
|
|||||||
- [`execution`] — execution backends and operation-specific results.
|
- [`execution`] — execution backends and operation-specific results.
|
||||||
- [`working_directory`] — repository materialization, session resources, and safe deletion.
|
- [`working_directory`] — repository materialization, session resources, and safe deletion.
|
||||||
- [`workspace_issuer`] — Workspace signature verification and replay prevention.
|
- [`workspace_issuer`] — Workspace signature verification and replay prevention.
|
||||||
|
|
||||||
|
## Offline stopped-Worker fixtures
|
||||||
|
|
||||||
|
The nondefault `test-support` feature exposes `test_support::StoppedWorkerTemplate`.
|
||||||
|
It is intentionally limited to current-schema, stopped, attachment-free artificial
|
||||||
|
conversations with one Segment, text history, and no pending execution/activation.
|
||||||
|
It does not add a product endpoint or change create/restore authorization.
|
||||||
|
|
||||||
|
1. With the source Runtime offline, call `StoppedWorkerTemplate::extract(worker_dir)`.
|
||||||
|
The source must have its existing Runtime owner lock and identity. Extraction is
|
||||||
|
read-only; the returned serde template excludes source auth, saved Manifest,
|
||||||
|
request/config references, run directories, and Runtime operation history.
|
||||||
|
2. Initialize **new** Runtime identity and Workspace trust in a fresh destination
|
||||||
|
using normal setup. Preserve the template's Runtime and Workspace IDs.
|
||||||
|
3. Call `template.install(&options, request, &resolved_manifest)` before starting
|
||||||
|
the Runtime. Supply a fresh typed `CreateWorkerRequest` (including config/profile
|
||||||
|
references and trusted memory settings) and `manifest::WorkerManifest` resolved
|
||||||
|
for that environment. Its Worker name must equal `template.worker_name()`.
|
||||||
|
4. Start normally and use the ordinary explicit restore operation. Installation
|
||||||
|
itself creates no execution and leaves the Worker stopped with a fresh guard.
|
||||||
|
|
||||||
|
The template preserves the original Session manifest/JSONL bytes, including stable
|
||||||
|
entry IDs, timestamps, and settled submission receipts. It rejects resumable queues
|
||||||
|
and Feature invocation history. A saved system prompt is required for restore;
|
||||||
|
checkpoint receipts must match the Worker's typed receipt schema. Preserved records
|
||||||
|
must use canonical writer field shapes: unknown/duplicate fields (including nested
|
||||||
|
fields), discarded optional fields, and opaque Feature history payloads are rejected,
|
||||||
|
not stripped or rewritten. Portability/secret-marker checks do not replace
|
||||||
|
review of the supplied artificial conversation or its independently verified hashes.
|
||||||
|
The installer refuses any destination content other than initialized `auth.toml`;
|
||||||
|
interrupted publication is fail-closed, not an overwrite/recovery facility.
|
||||||
|
|
||||||
|
## Owned-process Restore checkpoints
|
||||||
|
|
||||||
|
The independent, nondefault `restore-checkpoints` feature exists only for explicit
|
||||||
|
process tests in the shared `tests/e2e` harness. Offline `test-support` does not enable
|
||||||
|
it. Normal binaries reject `--test-restore-checkpoint`; no environment variable or
|
||||||
|
HTTP endpoint activates this facility. Do not deploy a checkpoint-enabled binary
|
||||||
|
as a replacement for the default product.
|
||||||
|
|
||||||
|
A test-enabled Runtime accepts `--test-restore-checkpoint <config.json>` with a
|
||||||
|
bounded regular JSON file selecting a loopback socket, Worker ID, one stage and a
|
||||||
|
diagnostic nonce. The nonce is only a handshake label, never execution identity,
|
||||||
|
request authority or routing state. The one-shot stages are real prepared candidate
|
||||||
|
with `NotCommitted`, and durable `Committed` before activation/terminal receipt.
|
||||||
|
Before acknowledging, the existing bounded canonical reader validates the exact
|
||||||
|
Worker/operation/decision, syncs its file/directories and requires no saved receipt.
|
||||||
|
No pending JSON is fabricated and no lifecycle work is skipped.
|
||||||
|
|
||||||
|
The Runtime sends bounded newline-delimited JSON evidence, then waits at most eight
|
||||||
|
seconds for byte `R`. EOF, invalid reply, unavailable peer or deadline releases the
|
||||||
|
wait into the original operation. The Runtime-state mutex is not held across that
|
||||||
|
wait; existing per-Worker ownership still covers the actual candidate and cleanup.
|
||||||
|
The harness verifies the ack, kills/reaps only its owned child and inspects the
|
||||||
|
canonical aggregate offline before restarting the same roots/identity without the
|
||||||
|
checkpoint flag or another fixture install. These tests are a separate cfg/binary
|
||||||
|
lane from featureless product regression; they do not certify arbitrary OS failure,
|
||||||
|
external power loss or a real model call.
|
||||||
|
|||||||
@@ -215,6 +215,38 @@ impl WorkerExecutionResult {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Private one-shot synchronization at real publication boundaries. Disconnect
|
||||||
|
// releases the waiter during unwinding; timeout bounds a forgotten test owner.
|
||||||
|
#[cfg(test)]
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct RestoreTestGate<T> {
|
||||||
|
pub(crate) entered: std::sync::mpsc::Sender<T>,
|
||||||
|
pub(crate) release: std::sync::mpsc::Receiver<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl<T> RestoreTestGate<T> {
|
||||||
|
pub(crate) fn wait(self, evidence: T) {
|
||||||
|
let _ = self.entered.send(evidence);
|
||||||
|
assert!(
|
||||||
|
!matches!(
|
||||||
|
self.release
|
||||||
|
.recv_timeout(std::time::Duration::from_secs(30)),
|
||||||
|
Err(std::sync::mpsc::RecvTimeoutError::Timeout)
|
||||||
|
),
|
||||||
|
"publication test gate timed out"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(test, feature = "ws-server"))]
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct RestoreFlushFault {
|
||||||
|
pub(crate) remaining_successes: usize,
|
||||||
|
pub(crate) gate: RestoreTestGate<protocol::Event>,
|
||||||
|
pub(crate) restored: std::sync::mpsc::Sender<Vec<protocol::Event>>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Runtime hooks available to an execution backend for one Worker.
|
/// Runtime hooks available to an execution backend for one Worker.
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
type WorkerObservationPublisher = Arc<
|
type WorkerObservationPublisher = Arc<
|
||||||
@@ -242,10 +274,15 @@ enum WorkerExecutionPublicationState {
|
|||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct WorkerExecutionContext {
|
pub struct WorkerExecutionContext {
|
||||||
worker_ref: WorkerRef,
|
worker_ref: WorkerRef,
|
||||||
|
restore_progress: Option<Arc<dyn Fn(WorkerLifecycleOperationId) + Send + Sync>>,
|
||||||
|
restore_repair_authorizer:
|
||||||
|
Option<Arc<dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync>>,
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
observation_publisher: WorkerObservationPublisher,
|
observation_publisher: WorkerObservationPublisher,
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
publication_state: Arc<Mutex<WorkerExecutionPublicationState>>,
|
publication_state: Arc<Mutex<WorkerExecutionPublicationState>>,
|
||||||
|
#[cfg(all(test, feature = "ws-server"))]
|
||||||
|
flush_fault: Arc<Mutex<Option<RestoreFlushFault>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WorkerExecutionContext {
|
impl WorkerExecutionContext {
|
||||||
@@ -256,7 +293,11 @@ impl WorkerExecutionContext {
|
|||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
worker_ref,
|
worker_ref,
|
||||||
|
restore_progress: None,
|
||||||
|
restore_repair_authorizer: None,
|
||||||
observation_publisher,
|
observation_publisher,
|
||||||
|
#[cfg(test)]
|
||||||
|
flush_fault: Arc::default(),
|
||||||
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Active)),
|
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Active)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -268,7 +309,11 @@ impl WorkerExecutionContext {
|
|||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
worker_ref,
|
worker_ref,
|
||||||
|
restore_progress: None,
|
||||||
|
restore_repair_authorizer: None,
|
||||||
observation_publisher,
|
observation_publisher,
|
||||||
|
#[cfg(test)]
|
||||||
|
flush_fault: Arc::default(),
|
||||||
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Pending(
|
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Pending(
|
||||||
VecDeque::new(),
|
VecDeque::new(),
|
||||||
))),
|
))),
|
||||||
@@ -277,12 +322,56 @@ impl WorkerExecutionContext {
|
|||||||
|
|
||||||
#[cfg(not(feature = "ws-server"))]
|
#[cfg(not(feature = "ws-server"))]
|
||||||
pub(crate) fn new(worker_ref: WorkerRef) -> Self {
|
pub(crate) fn new(worker_ref: WorkerRef) -> Self {
|
||||||
Self { worker_ref }
|
Self {
|
||||||
|
worker_ref,
|
||||||
|
restore_progress: None,
|
||||||
|
restore_repair_authorizer: None,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "ws-server"))]
|
#[cfg(not(feature = "ws-server"))]
|
||||||
pub(crate) fn candidate(worker_ref: WorkerRef) -> Self {
|
pub(crate) fn candidate(worker_ref: WorkerRef) -> Self {
|
||||||
Self { worker_ref }
|
Self {
|
||||||
|
worker_ref,
|
||||||
|
restore_progress: None,
|
||||||
|
restore_repair_authorizer: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn with_restore_repair_authorizer(
|
||||||
|
mut self,
|
||||||
|
authorizer: Arc<
|
||||||
|
dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync,
|
||||||
|
>,
|
||||||
|
) -> Self {
|
||||||
|
self.restore_repair_authorizer = Some(authorizer);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn authorize_restore_source_repair(
|
||||||
|
&self,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
match &self.restore_repair_authorizer {
|
||||||
|
Some(authorizer) => authorizer(operation),
|
||||||
|
// Direct factory/component users own their source repair separately;
|
||||||
|
// only Runtime contexts carry a Runtime aggregate authority callback.
|
||||||
|
None => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn with_restore_progress(
|
||||||
|
mut self,
|
||||||
|
progress: Arc<dyn Fn(WorkerLifecycleOperationId) + Send + Sync>,
|
||||||
|
) -> Self {
|
||||||
|
self.restore_progress = Some(progress);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn restore_progressed(&self, operation_id: WorkerLifecycleOperationId) {
|
||||||
|
if let Some(progress) = &self.restore_progress {
|
||||||
|
progress(operation_id);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn worker_ref(&self) -> &WorkerRef {
|
pub fn worker_ref(&self) -> &WorkerRef {
|
||||||
@@ -335,20 +424,65 @@ impl WorkerExecutionContext {
|
|||||||
let Some(event) = event else {
|
let Some(event) = event else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
if let Err(error) = (self.observation_publisher)(self.worker_ref.clone(), event.clone())
|
#[cfg(test)]
|
||||||
{
|
let fault = {
|
||||||
|
let mut fault = self.flush_fault.lock().unwrap();
|
||||||
|
if fault.as_ref().is_some_and(|f| f.remaining_successes == 0) {
|
||||||
|
fault.take()
|
||||||
|
} else {
|
||||||
|
if let Some(fault) = fault.as_mut() {
|
||||||
|
fault.remaining_successes -= 1;
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
};
|
||||||
|
#[cfg(test)]
|
||||||
|
let mut restored = None;
|
||||||
|
#[cfg(test)]
|
||||||
|
let publication = if let Some(fault) = fault {
|
||||||
|
restored = Some(fault.restored);
|
||||||
|
fault.gate.wait(event.clone());
|
||||||
|
Err(RuntimeError::InvalidRequest(
|
||||||
|
"injected buffered publication failure".into(),
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
(self.observation_publisher)(self.worker_ref.clone(), event.clone())
|
||||||
|
};
|
||||||
|
#[cfg(not(test))]
|
||||||
|
let publication = (self.observation_publisher)(self.worker_ref.clone(), event.clone());
|
||||||
|
if let Err(error) = publication {
|
||||||
let mut state = self
|
let mut state = self
|
||||||
.publication_state
|
.publication_state
|
||||||
.lock()
|
.lock()
|
||||||
.map_err(|_| RuntimeError::StatePoisoned)?;
|
.map_err(|_| RuntimeError::StatePoisoned)?;
|
||||||
if let WorkerExecutionPublicationState::Activating(events) = &mut *state {
|
if let WorkerExecutionPublicationState::Activating(events) = &mut *state {
|
||||||
events.push_front(event);
|
events.push_front(event);
|
||||||
|
#[cfg(test)]
|
||||||
|
if let Some(restored) = restored {
|
||||||
|
let _ = restored.send(events.iter().cloned().collect());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return Err(error);
|
return Err(error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(all(test, feature = "ws-server"))]
|
||||||
|
pub(crate) fn test_fail_flush(&self, fault: RestoreFlushFault) {
|
||||||
|
assert!(self.flush_fault.lock().unwrap().replace(fault).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(test, feature = "ws-server"))]
|
||||||
|
pub(crate) fn test_buffered_events(&self) -> Vec<protocol::Event> {
|
||||||
|
match &*self.publication_state.lock().unwrap() {
|
||||||
|
WorkerExecutionPublicationState::Pending(events)
|
||||||
|
| WorkerExecutionPublicationState::Activating(events) => {
|
||||||
|
events.iter().cloned().collect()
|
||||||
|
}
|
||||||
|
_ => Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "ws-server"))]
|
#[cfg(not(feature = "ws-server"))]
|
||||||
pub(crate) fn activate_candidate(&self) -> Result<(), RuntimeError> {
|
pub(crate) fn activate_candidate(&self) -> Result<(), RuntimeError> {
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -424,6 +558,7 @@ pub struct WorkerExecutionSpawnRequest {
|
|||||||
/// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker.
|
/// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker.
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct WorkerExecutionRestoreRequest {
|
pub struct WorkerExecutionRestoreRequest {
|
||||||
|
pub source_repair_possible: bool,
|
||||||
pub operation_id: WorkerLifecycleOperationId,
|
pub operation_id: WorkerLifecycleOperationId,
|
||||||
pub worker_ref: WorkerRef,
|
pub worker_ref: WorkerRef,
|
||||||
pub request: crate::catalog::CreateWorkerRequest,
|
pub request: crate::catalog::CreateWorkerRequest,
|
||||||
@@ -565,6 +700,19 @@ impl fmt::Debug for WorkerProtocolTransport {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Derived resource disposition for an existing restore owner. Not durable
|
||||||
|
/// authority: Runtime must reread its decision before choosing any action.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum RestoreRecoveryDisposition {
|
||||||
|
/// No live or pending owner for this exact operation. Not cleanup evidence.
|
||||||
|
Absent,
|
||||||
|
InFlight,
|
||||||
|
Continue,
|
||||||
|
Discard,
|
||||||
|
/// A retained cleanup-only owner; never reconstruct or activate it.
|
||||||
|
Reclaim,
|
||||||
|
}
|
||||||
|
|
||||||
/// The connection selects this backend; WorkerRef identifies an execution,
|
/// The connection selects this backend; WorkerRef identifies an execution,
|
||||||
/// never a capability or authorization token.
|
/// never a capability or authorization token.
|
||||||
pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
||||||
@@ -633,6 +781,33 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Completed resource-owner evidence retained while Runtime settlement may
|
||||||
|
/// still need retry. This is not a new request or execution authorization.
|
||||||
|
fn completed_restore_failure(
|
||||||
|
&self,
|
||||||
|
_operation_id: WorkerLifecycleOperationId,
|
||||||
|
_worker_ref: &WorkerRef,
|
||||||
|
) -> Option<WorkerExecutionSpawnResult> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opt in to Runtime-owned autonomous restore continuation. Pending tasks
|
||||||
|
/// signal completion through the original execution context; callbacks must
|
||||||
|
/// not wait for Runtime locks. Independent of resource disposition support.
|
||||||
|
fn supports_restore_continuation(&self) -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Describe exact operation resource ownership for warm/cold recovery
|
||||||
|
/// dispatch. This does not opt in to callbacks or autonomous continuation.
|
||||||
|
fn restore_recovery_disposition(
|
||||||
|
&self,
|
||||||
|
_operation_id: WorkerLifecycleOperationId,
|
||||||
|
_worker_ref: &WorkerRef,
|
||||||
|
) -> Option<RestoreRecoveryDisposition> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
fn reconcile_restore(
|
fn reconcile_restore(
|
||||||
&self,
|
&self,
|
||||||
request: WorkerExecutionRestoreRequest,
|
request: WorkerExecutionRestoreRequest,
|
||||||
@@ -640,6 +815,42 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
|||||||
self.restore_worker(request)
|
self.restore_worker(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reclaim an uncommitted operation after restart without preparing,
|
||||||
|
/// repairing, or activating a replacement candidate. Runtime must establish
|
||||||
|
/// the durable uncommitted decision before calling this seam. Unsupported
|
||||||
|
/// cleanup is uncertainty, never a successful rollback.
|
||||||
|
fn reclaim_uncommitted_restore(
|
||||||
|
&self,
|
||||||
|
_request: WorkerExecutionRestoreRequest,
|
||||||
|
) -> WorkerExecutionSpawnResult {
|
||||||
|
WorkerExecutionSpawnResult::ReconciliationRequired {
|
||||||
|
result: WorkerExecutionResult::unsupported(
|
||||||
|
WorkerExecutionOperation::Restore,
|
||||||
|
"backend cannot reclaim an uncommitted restore after restart",
|
||||||
|
),
|
||||||
|
worker_state: None,
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Discard only the exact unpublished operation after Runtime has proved
|
||||||
|
/// its durable decision uncommitted. This is not Stop and must not affect a
|
||||||
|
/// connected execution. RolledBack requires confirmed cleanup and no repair.
|
||||||
|
fn discard_uncommitted_restore(
|
||||||
|
&self,
|
||||||
|
_operation_id: WorkerLifecycleOperationId,
|
||||||
|
_worker_ref: &WorkerRef,
|
||||||
|
) -> WorkerExecutionSpawnResult {
|
||||||
|
WorkerExecutionSpawnResult::ReconciliationRequired {
|
||||||
|
result: WorkerExecutionResult::unsupported(
|
||||||
|
WorkerExecutionOperation::Restore,
|
||||||
|
"backend cannot confirm unpublished restore cleanup",
|
||||||
|
),
|
||||||
|
worker_state: None,
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn activate_restored_worker(
|
fn activate_restored_worker(
|
||||||
&self,
|
&self,
|
||||||
_operation_id: WorkerLifecycleOperationId,
|
_operation_id: WorkerLifecycleOperationId,
|
||||||
@@ -901,6 +1112,28 @@ impl WorkerExecutionBackendRef {
|
|||||||
self.backend.preflight_restore(request)
|
self.backend.preflight_restore(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn completed_restore_failure(
|
||||||
|
&self,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> Option<WorkerExecutionSpawnResult> {
|
||||||
|
self.backend
|
||||||
|
.completed_restore_failure(operation_id, worker_ref)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn supports_restore_continuation(&self) -> bool {
|
||||||
|
self.backend.supports_restore_continuation()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn restore_recovery_disposition(
|
||||||
|
&self,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> Option<RestoreRecoveryDisposition> {
|
||||||
|
self.backend
|
||||||
|
.restore_recovery_disposition(operation_id, worker_ref)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn reconcile_restore(
|
pub(crate) fn reconcile_restore(
|
||||||
&self,
|
&self,
|
||||||
request: WorkerExecutionRestoreRequest,
|
request: WorkerExecutionRestoreRequest,
|
||||||
@@ -908,6 +1141,22 @@ impl WorkerExecutionBackendRef {
|
|||||||
self.backend.reconcile_restore(request)
|
self.backend.reconcile_restore(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn reclaim_uncommitted_restore(
|
||||||
|
&self,
|
||||||
|
request: WorkerExecutionRestoreRequest,
|
||||||
|
) -> WorkerExecutionSpawnResult {
|
||||||
|
self.backend.reclaim_uncommitted_restore(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn discard_uncommitted_restore(
|
||||||
|
&self,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> WorkerExecutionSpawnResult {
|
||||||
|
self.backend
|
||||||
|
.discard_uncommitted_restore(operation_id, worker_ref)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn activate_restored_worker(
|
pub(crate) fn activate_restored_worker(
|
||||||
&self,
|
&self,
|
||||||
operation_id: WorkerLifecycleOperationId,
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -25,6 +25,7 @@ use crate::interaction::{WorkerInput, WorkerInteractionAck};
|
|||||||
use crate::management::{RuntimeSummary, WorkerDeleteResult};
|
use crate::management::{RuntimeSummary, WorkerDeleteResult};
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
use crate::observation::WorkerObservationCursor;
|
use crate::observation::WorkerObservationCursor;
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
use crate::retention::{
|
use crate::retention::{
|
||||||
WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory,
|
WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory,
|
||||||
};
|
};
|
||||||
@@ -869,6 +870,7 @@ async fn open_workdir_session(
|
|||||||
session_id: session_id.clone(),
|
session_id: session_id.clone(),
|
||||||
workdir_id: session.workdir().id().clone(),
|
workdir_id: session.workdir().id().clone(),
|
||||||
capabilities: session.capabilities(),
|
capabilities: session.capabilities(),
|
||||||
|
environment: session.environment(),
|
||||||
};
|
};
|
||||||
state
|
state
|
||||||
.workdir_sessions
|
.workdir_sessions
|
||||||
@@ -1567,6 +1569,7 @@ fn protocol_error_event(message: impl Into<String>) -> protocol::Event {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
async fn worker_retention_inventory(
|
async fn worker_retention_inventory(
|
||||||
State(state): State<RuntimeHttpState>,
|
State(state): State<RuntimeHttpState>,
|
||||||
@@ -1588,6 +1591,7 @@ async fn worker_retention_inventory(
|
|||||||
.map_err(RuntimeHttpRestError::runtime)
|
.map_err(RuntimeHttpRestError::runtime)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
async fn execute_worker_retention(
|
async fn execute_worker_retention(
|
||||||
State(state): State<RuntimeHttpState>,
|
State(state): State<RuntimeHttpState>,
|
||||||
@@ -2187,7 +2191,25 @@ impl RuntimeHttpWorkdirError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn runtime(error: RuntimeError) -> Self {
|
fn runtime(error: RuntimeError) -> Self {
|
||||||
match error {
|
use workdir::WorkdirFailureReason as Reason;
|
||||||
|
let reason = match &error {
|
||||||
|
RuntimeError::WorkingDirectory(diagnostic) => match diagnostic.code.as_str() {
|
||||||
|
"working_directory_not_found" => Reason::WorkingDirectoryNotFound,
|
||||||
|
"working_directory_not_active" => Reason::WorkingDirectoryNotActive,
|
||||||
|
"working_directory_record_invalid" => Reason::WorkingDirectoryRecordInvalid,
|
||||||
|
"working_directory_repository_access_unavailable" => {
|
||||||
|
Reason::WorkingDirectoryAccessUnavailable
|
||||||
|
}
|
||||||
|
_ => Reason::SessionOpenFailed,
|
||||||
|
},
|
||||||
|
RuntimeError::RuntimeStopped => Reason::RuntimeStopped,
|
||||||
|
RuntimeError::StatePoisoned => Reason::RuntimeStateUnavailable,
|
||||||
|
RuntimeError::ExecutionBackendUnavailable { .. } => Reason::RuntimeBackendUnavailable,
|
||||||
|
RuntimeError::WorkspaceOwnerMismatch { .. } => Reason::RuntimeOwnerMismatch,
|
||||||
|
RuntimeError::InvalidRequest(_) => Reason::RuntimeInvalidRequest,
|
||||||
|
_ => Reason::SessionOpenFailed,
|
||||||
|
};
|
||||||
|
let mut response = match error {
|
||||||
RuntimeError::WorkingDirectory(diagnostic)
|
RuntimeError::WorkingDirectory(diagnostic)
|
||||||
if diagnostic.code == "working_directory_not_found" =>
|
if diagnostic.code == "working_directory_not_found" =>
|
||||||
{
|
{
|
||||||
@@ -2200,7 +2222,9 @@ impl RuntimeHttpWorkdirError {
|
|||||||
WorkdirTransportErrorCode::Unavailable,
|
WorkdirTransportErrorCode::Unavailable,
|
||||||
"Workdir session is unavailable",
|
"Workdir session is unavailable",
|
||||||
),
|
),
|
||||||
}
|
};
|
||||||
|
response.payload.failure_reason = Some(reason);
|
||||||
|
response
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2222,6 +2246,7 @@ impl IntoResponse for RuntimeHttpWorkdirError {
|
|||||||
status = self.status.as_u16(),
|
status = self.status.as_u16(),
|
||||||
code = self.payload.code.as_str(),
|
code = self.payload.code.as_str(),
|
||||||
failure_reason = reason.as_str(),
|
failure_reason = reason.as_str(),
|
||||||
|
failure_stage = reason.stage().as_str(),
|
||||||
"Workdir provider operation failed"
|
"Workdir provider operation failed"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -2434,6 +2459,7 @@ pub enum RuntimeHttpServerError {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
mod command_transport_tests;
|
mod command_transport_tests;
|
||||||
|
mod runtime_workdir_failure_tests;
|
||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
mod session_material_tests;
|
mod session_material_tests;
|
||||||
|
|
||||||
@@ -2631,6 +2657,87 @@ mod tests {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Generated management routes and real Workspace capability middleware stay
|
||||||
|
// connected even when this process has no filesystem retention provider.
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn memory_retention_routes_preserve_auth_and_return_typed_unavailable() {
|
||||||
|
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
|
||||||
|
let trust = workspace_trust(&identity);
|
||||||
|
let runtime = Runtime::new_memory();
|
||||||
|
let app = runtime_http_router_with_workspace_auth(
|
||||||
|
runtime,
|
||||||
|
None,
|
||||||
|
workspace_test_auth(vec![trust.clone()]),
|
||||||
|
);
|
||||||
|
let worker_id = WorkerId::now_v7();
|
||||||
|
let body = runtime_api::WorkerRetentionExecutionRequest {
|
||||||
|
operation_id: "memory-retention".into(),
|
||||||
|
input_fingerprint: "input".into(),
|
||||||
|
archive_id: None,
|
||||||
|
workspace_id: "workspace-a".into(),
|
||||||
|
source_runtime_id: "runtime-test".into(),
|
||||||
|
worker_id,
|
||||||
|
source_created_at: "2026-01-01T00:00:00Z".into(),
|
||||||
|
removed_at: "2026-01-02T00:00:00Z".into(),
|
||||||
|
effective_profile: None,
|
||||||
|
retention_class: None,
|
||||||
|
policy_id: "policy".into(),
|
||||||
|
session_disposition: runtime_api::SessionDisposition::Purge,
|
||||||
|
diagnostics_disposition: runtime_api::DiagnosticsDisposition::Purge,
|
||||||
|
};
|
||||||
|
for (method, path, body) in [
|
||||||
|
(
|
||||||
|
"GET",
|
||||||
|
format!("/v1/workers/{worker_id}/retention/inventory"),
|
||||||
|
vec![],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"POST",
|
||||||
|
format!("/v1/workers/{worker_id}/retention/execute"),
|
||||||
|
serde_json::to_vec(&body).unwrap(),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let unsigned = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(method)
|
||||||
|
.uri(&path)
|
||||||
|
.header(header::CONTENT_TYPE, "application/json")
|
||||||
|
.body(Body::from(body.clone()))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
let claims = signed_claims(&trust, method, &path, &body);
|
||||||
|
let response =
|
||||||
|
signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||||
|
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||||
|
let error: RuntimeHttpErrorResponse = read_json(response).await;
|
||||||
|
assert_eq!(error.error.code, "worker_retention_unavailable");
|
||||||
|
}
|
||||||
|
for (mut value, expected_status, expected_code) in [
|
||||||
|
(body.clone(), StatusCode::NOT_FOUND, "worker_not_found"),
|
||||||
|
(body.clone(), StatusCode::BAD_REQUEST, "worker_id_mismatch"),
|
||||||
|
] {
|
||||||
|
if expected_status == StatusCode::NOT_FOUND {
|
||||||
|
value.workspace_id = "workspace-b".into();
|
||||||
|
} else {
|
||||||
|
value.worker_id = WorkerId::now_v7();
|
||||||
|
}
|
||||||
|
let path = format!("/v1/workers/{worker_id}/retention/execute");
|
||||||
|
let bytes = serde_json::to_vec(&value).unwrap();
|
||||||
|
let claims = signed_claims(&trust, "POST", &path, &bytes);
|
||||||
|
let response =
|
||||||
|
signed_request(app.clone(), &identity, &claims, bytes, "workspace-a").await;
|
||||||
|
assert_eq!(response.status(), expected_status);
|
||||||
|
let error: RuntimeHttpErrorResponse = read_json(response).await;
|
||||||
|
assert_eq!(error.error.code, expected_code);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn ordinary_signed_ping_succeeds_without_handshake_and_rejects_replay() {
|
async fn ordinary_signed_ping_succeeds_without_handshake_and_rejects_replay() {
|
||||||
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
|
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
|
||||||
@@ -3206,9 +3313,13 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.expect_err("close must be terminal");
|
.expect_err("close must be terminal");
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
closed_error,
|
closed_error.classification_source(),
|
||||||
workdir::WorkdirError::Unavailable(_)
|
workdir::WorkdirError::Unavailable(_)
|
||||||
));
|
));
|
||||||
|
assert_eq!(
|
||||||
|
closed_error.failure_reason(),
|
||||||
|
Some(workdir::WorkdirFailureReason::SessionClosed)
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
state
|
state
|
||||||
.workdir_sessions
|
.workdir_sessions
|
||||||
|
|||||||
@@ -444,6 +444,31 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
material_pin(&self.state, request, true)
|
material_pin(&self.state, request, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn session_recording_capture(
|
||||||
|
&self,
|
||||||
|
request: runtime_api::SessionRecordingCaptureRequest,
|
||||||
|
) -> Result<runtime_api::SessionRecordingCaptureAvailability, runtime_api::RuntimeApiError>
|
||||||
|
{
|
||||||
|
let scope = material_scope(&self.state, &request.workspace_id)?;
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
{
|
||||||
|
Ok(self
|
||||||
|
.state
|
||||||
|
.runtime
|
||||||
|
.session_recording_capture_scoped(&scope, &request))
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
{
|
||||||
|
let _ = scope;
|
||||||
|
Ok(
|
||||||
|
runtime_api::SessionRecordingCaptureAvailability::Unavailable {
|
||||||
|
reason: runtime_api::SessionPublicUnavailableReason::StorageUnavailable,
|
||||||
|
message: "Committed Session storage unavailable".into(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn session_material_catalog(
|
async fn session_material_catalog(
|
||||||
&self,
|
&self,
|
||||||
request: runtime_api::SessionMaterialCatalogRequest,
|
request: runtime_api::SessionMaterialCatalogRequest,
|
||||||
@@ -694,6 +719,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
server_api::WorkerRestoreState::RolledBack => {
|
server_api::WorkerRestoreState::RolledBack => {
|
||||||
runtime_api::WorkerRestoreState::RolledBack
|
runtime_api::WorkerRestoreState::RolledBack
|
||||||
}
|
}
|
||||||
|
server_api::WorkerRestoreState::Superseded => {
|
||||||
|
runtime_api::WorkerRestoreState::Superseded
|
||||||
|
}
|
||||||
server_api::WorkerRestoreState::ReconciliationRequired => {
|
server_api::WorkerRestoreState::ReconciliationRequired => {
|
||||||
runtime_api::WorkerRestoreState::ReconciliationRequired
|
runtime_api::WorkerRestoreState::ReconciliationRequired
|
||||||
}
|
}
|
||||||
@@ -744,6 +772,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
server_api::WorkerRestoreState::RolledBack => {
|
server_api::WorkerRestoreState::RolledBack => {
|
||||||
runtime_api::WorkerRestoreState::RolledBack
|
runtime_api::WorkerRestoreState::RolledBack
|
||||||
}
|
}
|
||||||
|
server_api::WorkerRestoreState::Superseded => {
|
||||||
|
runtime_api::WorkerRestoreState::Superseded
|
||||||
|
}
|
||||||
server_api::WorkerRestoreState::ReconciliationRequired => {
|
server_api::WorkerRestoreState::ReconciliationRequired => {
|
||||||
runtime_api::WorkerRestoreState::ReconciliationRequired
|
runtime_api::WorkerRestoreState::ReconciliationRequired
|
||||||
}
|
}
|
||||||
@@ -856,13 +887,25 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
||||||
let inventory = self
|
#[cfg(feature = "fs-store")]
|
||||||
.state
|
{
|
||||||
.runtime
|
let inventory = self
|
||||||
.worker_retention_inventory(&scope.workspace_id, &worker_ref)
|
.state
|
||||||
.map_err(RuntimeHttpRestError::runtime)
|
.runtime
|
||||||
.map_err(api_error)?;
|
.worker_retention_inventory(&scope.workspace_id, &worker_ref)
|
||||||
response(inventory)
|
.map_err(RuntimeHttpRestError::runtime)
|
||||||
|
.map_err(api_error)?;
|
||||||
|
response(inventory)
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
{
|
||||||
|
let _ = (scope, worker_ref);
|
||||||
|
Err(runtime_api::RuntimeApiError::new(
|
||||||
|
503,
|
||||||
|
"worker_retention_unavailable",
|
||||||
|
"Worker retention requires filesystem storage",
|
||||||
|
))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn execute_retention(
|
async fn execute_retention(
|
||||||
@@ -880,27 +923,36 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
||||||
let request: WorkerRetentionExecutionRequest = request(value)?;
|
if value.workspace_id != scope.workspace_id {
|
||||||
if request.workspace_id != scope.workspace_id {
|
|
||||||
return Err(runtime_api::RuntimeApiError::new(
|
return Err(runtime_api::RuntimeApiError::new(
|
||||||
StatusCode::NOT_FOUND.as_u16(),
|
StatusCode::NOT_FOUND.as_u16(),
|
||||||
"worker_not_found",
|
"worker_not_found",
|
||||||
"worker does not belong to the authenticated Workspace",
|
"worker does not belong to the authenticated Workspace",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if request.worker_id != worker_ref.worker_id {
|
if value.worker_id != worker_ref.worker_id {
|
||||||
return Err(runtime_api::RuntimeApiError::new(
|
return Err(runtime_api::RuntimeApiError::new(
|
||||||
StatusCode::BAD_REQUEST.as_u16(),
|
StatusCode::BAD_REQUEST.as_u16(),
|
||||||
"worker_id_mismatch",
|
"worker_id_mismatch",
|
||||||
"Worker retention request worker_id does not match route worker_id",
|
"Worker retention request worker_id does not match route worker_id",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let result = self
|
#[cfg(feature = "fs-store")]
|
||||||
.state
|
{
|
||||||
.runtime
|
let request: WorkerRetentionExecutionRequest = request(value)?;
|
||||||
.execute_worker_retention(&request)
|
let result = self
|
||||||
.map_err(RuntimeHttpRestError::runtime)
|
.state
|
||||||
.map_err(api_error)?;
|
.runtime
|
||||||
response(result)
|
.execute_worker_retention(&request)
|
||||||
|
.map_err(RuntimeHttpRestError::runtime)
|
||||||
|
.map_err(api_error)?;
|
||||||
|
response(result)
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
Err(runtime_api::RuntimeApiError::new(
|
||||||
|
503,
|
||||||
|
"worker_retention_unavailable",
|
||||||
|
"Worker retention requires filesystem storage",
|
||||||
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
//! pre-seeded session is a fixture; no Runtime Worker/materializer is claimed here.
|
//! pre-seeded session is a fixture; no Runtime Worker/materializer is claimed here.
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use workdir::http::{
|
use workdir::http::{
|
||||||
RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture,
|
RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture,
|
||||||
@@ -70,14 +71,23 @@ impl Harness {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn new() -> Self {
|
async fn new() -> Self {
|
||||||
let root = tempfile::tempdir().unwrap();
|
Self::with_environment(tempfile::tempdir().unwrap(), BTreeMap::new()).await
|
||||||
let session: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::materialized_bound(
|
}
|
||||||
Workdir::new("wd-transport"),
|
|
||||||
root.path().to_path_buf(),
|
async fn with_environment(
|
||||||
root.path().to_path_buf(),
|
root: tempfile::TempDir,
|
||||||
SharedScope::new(Scope::writable(root.path()).unwrap()),
|
command_environment: BTreeMap<String, String>,
|
||||||
WorkdirSessionCapabilities::ALL,
|
) -> Self {
|
||||||
));
|
let session: WorkdirSessionHandle =
|
||||||
|
Arc::new(LocalWorkdirSession::materialized_bound_with_environment(
|
||||||
|
Workdir::new("wd-transport"),
|
||||||
|
root.path().to_path_buf(),
|
||||||
|
root.path().to_path_buf(),
|
||||||
|
SharedScope::new(Scope::writable(root.path()).unwrap()),
|
||||||
|
WorkdirSessionCapabilities::ALL,
|
||||||
|
command_environment,
|
||||||
|
Vec::new(),
|
||||||
|
));
|
||||||
let identity = RuntimeIdentityMaterial::generate("transport-key").unwrap();
|
let identity = RuntimeIdentityMaterial::generate("transport-key").unwrap();
|
||||||
let trust = workspace_trust(&identity);
|
let trust = workspace_trust(&identity);
|
||||||
let state = RuntimeHttpState {
|
let state = RuntimeHttpState {
|
||||||
@@ -101,6 +111,9 @@ impl Harness {
|
|||||||
session_id: WorkdirSessionId::new("session-transport").unwrap(),
|
session_id: WorkdirSessionId::new("session-transport").unwrap(),
|
||||||
workdir_id: Workdir::new("wd-transport").id().clone(),
|
workdir_id: Workdir::new("wd-transport").id().clone(),
|
||||||
capabilities: WorkdirSessionCapabilities::ALL,
|
capabilities: WorkdirSessionCapabilities::ALL,
|
||||||
|
environment: workdir::WorkdirSessionEnvironment::root(
|
||||||
|
WorkdirSessionCapabilities::ALL,
|
||||||
|
),
|
||||||
})
|
})
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
@@ -222,6 +235,234 @@ async fn real_http_waiting_command_survives_long_poll_expiry() {
|
|||||||
.expect("bounded long-poll continuation over HTTP");
|
.expect("bounded long-poll continuation over HTTP");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn real_http_spawn_failure_preserves_safe_cause_counts_and_terminal_replay() {
|
||||||
|
tokio::time::timeout(Duration::from_secs(10), async {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let missing_path = root.path().join("private-spawn-path-must-not-exist");
|
||||||
|
assert!(!missing_path.exists());
|
||||||
|
let private_path = missing_path.to_str().unwrap().to_owned();
|
||||||
|
let environment_secret = "spawn-environment-secret-must-not-leak";
|
||||||
|
let command_secret = "spawn-command-secret-must-not-leak";
|
||||||
|
let tool_call_secret = "spawn-tool-call-secret-must-not-leak";
|
||||||
|
// This is an OS spawn failure in the production local provider, not a
|
||||||
|
// fabricated terminal: PATH contains no bash. The test process environment
|
||||||
|
// is untouched, so parallel command tests retain their normal PATH.
|
||||||
|
let harness = Harness::with_environment(
|
||||||
|
root,
|
||||||
|
BTreeMap::from([
|
||||||
|
("PATH".into(), private_path.clone()),
|
||||||
|
("PRIVATE_AUTH_TOKEN".into(), environment_secret.into()),
|
||||||
|
]),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let command = format!("printf '{command_secret}'; printf ran > must-not-run");
|
||||||
|
let handle = harness
|
||||||
|
.remote
|
||||||
|
.start_command(CommandRequest {
|
||||||
|
transport_byte_budget: 1024,
|
||||||
|
command: command.clone(),
|
||||||
|
timeout_secs: 5,
|
||||||
|
cwd: WorkdirPath::root(),
|
||||||
|
tool_call_id: Some(tool_call_secret.into()),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let request = CommandOutputRequest {
|
||||||
|
handle,
|
||||||
|
cursor: 0,
|
||||||
|
limit: 16,
|
||||||
|
wait: true,
|
||||||
|
};
|
||||||
|
let page = harness.remote.command_output(request.clone()).await.unwrap();
|
||||||
|
assert!(page.chunks.is_empty());
|
||||||
|
assert_eq!(page.next_cursor, 0);
|
||||||
|
let terminal = page.terminal.as_ref().expect("spawn failure is terminal");
|
||||||
|
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
||||||
|
assert_eq!(terminal.exit_code, None);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.process_failure,
|
||||||
|
Some(workdir::CommandFailure {
|
||||||
|
stage: workdir::CommandFailureStage::Spawn,
|
||||||
|
reason: workdir::CommandIoErrorKind::NotFound,
|
||||||
|
os_code: Some(libc::ENOENT),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
workdir::CommandOutputCollection::NotStarted
|
||||||
|
);
|
||||||
|
assert_eq!(terminal.stdout_bytes, 0);
|
||||||
|
assert_eq!(terminal.stderr_bytes, 0);
|
||||||
|
assert_eq!(terminal.stdout_delivered_bytes, 0);
|
||||||
|
assert_eq!(terminal.stderr_delivered_bytes, 0);
|
||||||
|
assert_eq!(terminal.truncation_reason, None);
|
||||||
|
|
||||||
|
// A zero-byte terminal page remains replayable at its unchanged cursor;
|
||||||
|
// neither a retry nor a smaller nonwaiting read changes its accounting.
|
||||||
|
assert_eq!(
|
||||||
|
harness.remote.command_output(request.clone()).await.unwrap(),
|
||||||
|
page
|
||||||
|
);
|
||||||
|
let replay_request = CommandOutputRequest {
|
||||||
|
wait: false,
|
||||||
|
limit: 1,
|
||||||
|
..request
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
harness
|
||||||
|
.remote
|
||||||
|
.command_output(replay_request.clone())
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
page
|
||||||
|
);
|
||||||
|
let response = harness
|
||||||
|
.raw_operation(
|
||||||
|
serde_json::to_value(WorkdirSessionOperation::CommandOutput(
|
||||||
|
replay_request.clone(),
|
||||||
|
))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), reqwest::StatusCode::OK);
|
||||||
|
let body = response.text().await.unwrap();
|
||||||
|
for secret in [
|
||||||
|
private_path.as_str(),
|
||||||
|
harness._root.path().to_str().unwrap(),
|
||||||
|
"private-spawn-path-must-not-exist",
|
||||||
|
"PRIVATE_AUTH_TOKEN",
|
||||||
|
environment_secret,
|
||||||
|
command_secret,
|
||||||
|
command.as_str(),
|
||||||
|
tool_call_secret,
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!body.contains(secret),
|
||||||
|
"private input leaked into HTTP terminal"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let wire: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
wire,
|
||||||
|
serde_json::json!({
|
||||||
|
"operation": "command_output",
|
||||||
|
"result": {
|
||||||
|
"chunks": [],
|
||||||
|
"next_cursor": 0,
|
||||||
|
"terminal": {
|
||||||
|
"status": "failed",
|
||||||
|
"exit_code": null,
|
||||||
|
"process_failure": {"stage": "spawn", "reason": "not_found", "os_code": libc::ENOENT},
|
||||||
|
"output_collection": {"state": "not_started"},
|
||||||
|
"stdout_bytes": 0,
|
||||||
|
"stderr_bytes": 0,
|
||||||
|
"stdout_delivered_bytes": 0,
|
||||||
|
"stderr_delivered_bytes": 0,
|
||||||
|
"truncation_reason": null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
harness.remote.command_output(replay_request).await.unwrap(),
|
||||||
|
page
|
||||||
|
);
|
||||||
|
// Exercise the same real spawn failure through Bash, not a terminal
|
||||||
|
// fixture. Opening another remote client reuses the pre-seeded provider
|
||||||
|
// session and still passes through the signed HTTP middleware.
|
||||||
|
let bash_remote: WorkdirSessionHandle = Arc::new(
|
||||||
|
RemoteWorkdirSession::open_with_authorization(
|
||||||
|
harness.client.clone(),
|
||||||
|
harness.operation_url.join("/").unwrap(),
|
||||||
|
harness.authorization.clone(),
|
||||||
|
Workdir::new("wd-transport").id().clone(),
|
||||||
|
OpenWorkdirSessionRequest::default(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let view = tools::bash_output::BashOutputView::new();
|
||||||
|
let (_, bash) = tools::bash_tool(bash_remote, view.clone())();
|
||||||
|
let input = serde_json::json!({
|
||||||
|
"command": command,
|
||||||
|
"cwd": ".",
|
||||||
|
"timeout": 5
|
||||||
|
})
|
||||||
|
.to_string();
|
||||||
|
let output = bash
|
||||||
|
.execute(
|
||||||
|
&input,
|
||||||
|
agen::tool::ToolExecutionContext::new(tool_call_secret, "http-spawn-attempt", 0),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("Bash returns a structured failed-command result");
|
||||||
|
let content = output.content.as_deref().expect("Bash ToolOutput body");
|
||||||
|
let tool_body: serde_json::Value = serde_json::from_str(content).unwrap();
|
||||||
|
assert_eq!(tool_body["stdout_preview"], "");
|
||||||
|
assert_eq!(tool_body["stderr_preview"], "");
|
||||||
|
assert_eq!(tool_body["preview_omitted"], false);
|
||||||
|
assert_eq!(
|
||||||
|
tool_body["output"]["terminal"],
|
||||||
|
serde_json::json!({
|
||||||
|
"status": "failed",
|
||||||
|
"exit_code": null,
|
||||||
|
"process_failure": {"stage": "spawn", "reason": "not_found", "os_code": libc::ENOENT},
|
||||||
|
"output_collection": {"state": "not_started"},
|
||||||
|
"transport_complete": false
|
||||||
|
})
|
||||||
|
);
|
||||||
|
let output_id = tool_body["output"]["id"].as_str().unwrap();
|
||||||
|
let retained = view.metadata(output_id).expect("retained spawn failure");
|
||||||
|
assert_eq!(retained.terminal.status, Some(workdir::CommandStatus::Failed));
|
||||||
|
assert_eq!(retained.terminal.exit_code, None);
|
||||||
|
assert_eq!(retained.terminal.process_failure, terminal.process_failure);
|
||||||
|
assert_eq!(
|
||||||
|
retained.terminal.output_collection,
|
||||||
|
Some(workdir::CommandOutputCollection::NotStarted)
|
||||||
|
);
|
||||||
|
assert!(!retained.terminal.transport_complete);
|
||||||
|
assert_eq!(retained.stdout_saved, 0);
|
||||||
|
assert_eq!(retained.stderr_saved, 0);
|
||||||
|
assert_eq!(retained.stdout_seen, 0);
|
||||||
|
assert_eq!(retained.stderr_seen, 0);
|
||||||
|
assert_eq!(retained.stdout_delivered, 0);
|
||||||
|
assert_eq!(retained.stderr_delivered, 0);
|
||||||
|
assert!(!retained.storage_truncated);
|
||||||
|
assert!(retained.retention_limit.is_none());
|
||||||
|
assert_eq!(retained.transport_truncation, None);
|
||||||
|
let retained_wire = serde_json::to_value(&retained).unwrap();
|
||||||
|
assert_eq!(retained_wire, tool_body["output"]);
|
||||||
|
for stream in [CommandStream::Stdout, CommandStream::Stderr] {
|
||||||
|
assert_eq!(view.read(output_id, stream, 0, 1), Some(Vec::new()));
|
||||||
|
}
|
||||||
|
for secret in [
|
||||||
|
private_path.as_str(),
|
||||||
|
harness._root.path().to_str().unwrap(),
|
||||||
|
"private-spawn-path-must-not-exist",
|
||||||
|
"PRIVATE_AUTH_TOKEN",
|
||||||
|
environment_secret,
|
||||||
|
command_secret,
|
||||||
|
command.as_str(),
|
||||||
|
tool_call_secret,
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!content.contains(secret),
|
||||||
|
"private input leaked into Bash ToolOutput body"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(!harness._root.path().join("must-not-run").exists());
|
||||||
|
harness.remote.close().await.unwrap();
|
||||||
|
assert!(harness.session.command_snapshot().is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(view.metadata(output_id).unwrap()).unwrap(),
|
||||||
|
retained_wire
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded real spawn failure over signed HTTP");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn real_http_closed_session_retains_failure_reason() {
|
async fn real_http_closed_session_retains_failure_reason() {
|
||||||
let harness = Harness::new().await;
|
let harness = Harness::new().await;
|
||||||
@@ -291,6 +532,11 @@ async fn real_http_command_pages_preserve_binary_streams_replay_and_nonzero_exit
|
|||||||
assert_eq!(stderr, b"stderr\xe6\x97\xa5");
|
assert_eq!(stderr, b"stderr\xe6\x97\xa5");
|
||||||
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
||||||
assert_eq!(terminal.exit_code, Some(9));
|
assert_eq!(terminal.exit_code, Some(9));
|
||||||
|
assert_eq!(terminal.process_failure, None);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
workdir::CommandOutputCollection::Complete
|
||||||
|
);
|
||||||
assert!(harness.remote.command_output(CommandOutputRequest { handle, cursor: 0, limit: 1, wait: false }).await.is_err());
|
assert!(harness.remote.command_output(CommandOutputRequest { handle, cursor: 0, limit: 1, wait: false }).await.is_err());
|
||||||
harness.remote.close().await.unwrap();
|
harness.remote.close().await.unwrap();
|
||||||
assert!(harness.session.command_snapshot().is_empty());
|
assert!(harness.session.command_snapshot().is_empty());
|
||||||
@@ -435,6 +681,11 @@ async fn real_http_command_budget_bounds_unique_payload_and_reports_larger_seen_
|
|||||||
);
|
);
|
||||||
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
||||||
assert_eq!(terminal.exit_code, Some(9));
|
assert_eq!(terminal.exit_code, Some(9));
|
||||||
|
assert_eq!(terminal.process_failure, None);
|
||||||
|
assert_eq!(
|
||||||
|
terminal.output_collection,
|
||||||
|
workdir::CommandOutputCollection::Complete
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
harness
|
harness
|
||||||
.remote
|
.remote
|
||||||
|
|||||||
@@ -0,0 +1,528 @@
|
|||||||
|
//! Session-open integration through the production signed router, Runtime owner/binding
|
||||||
|
//! checks, HTTP DTO and tracing boundary. Only the execution backend is replaced
|
||||||
|
//! for deterministic lookup/open faults and inert owner-binding Worker fixtures;
|
||||||
|
//! no handler, middleware or grant is replaced. Runs in the normal http-server lib
|
||||||
|
//! suite with isolated loopback sockets.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::working_directory::WorkingDirectoryDiagnostic;
|
||||||
|
use std::io::{self, Write};
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
const OPEN_PATH: &str = "/v1/working-directories/private-workdir-marker/sessions";
|
||||||
|
const PRIVATE_DIAGNOSTIC: &str = concat!(
|
||||||
|
"raw-runtime-error-marker /private/host-path-marker ",
|
||||||
|
"command=private-command-marker contents=private-file-marker ",
|
||||||
|
"proof=private-proof-marker Authorization: Bearer private-header-marker ",
|
||||||
|
"working_directory_not_found authorization_stale runtime_stopped",
|
||||||
|
);
|
||||||
|
const PRIVATE_MARKERS: &[&str] = &[
|
||||||
|
"raw-runtime-error-marker",
|
||||||
|
"/private/host-path-marker",
|
||||||
|
"private-command-marker",
|
||||||
|
"private-file-marker",
|
||||||
|
"private-proof-marker",
|
||||||
|
"private-header-marker",
|
||||||
|
"private-workdir-marker",
|
||||||
|
"private-owner-marker",
|
||||||
|
];
|
||||||
|
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
struct LogCapture(Arc<Mutex<Vec<u8>>>);
|
||||||
|
|
||||||
|
impl Write for LogCapture {
|
||||||
|
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||||
|
self.0.lock().unwrap().extend_from_slice(bytes);
|
||||||
|
Ok(bytes.len())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush(&mut self) -> io::Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LogCapture {
|
||||||
|
fn subscriber(&self) -> impl tracing::Subscriber + Send + Sync + 'static {
|
||||||
|
let writer = self.clone();
|
||||||
|
tracing_subscriber::fmt()
|
||||||
|
.with_ansi(false)
|
||||||
|
.without_time()
|
||||||
|
.with_max_level(tracing::Level::WARN)
|
||||||
|
.with_writer(move || writer.clone())
|
||||||
|
.finish()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn take(&self) -> String {
|
||||||
|
String::from_utf8(std::mem::take(&mut *self.0.lock().unwrap())).unwrap()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_private_absent(text: &str) {
|
||||||
|
for marker in PRIVATE_MARKERS {
|
||||||
|
assert!(
|
||||||
|
!text.contains(marker),
|
||||||
|
"private input escaped error boundary"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_failure_log(log: &str, status: StatusCode, code: &str, reason: &str, stage: &str) {
|
||||||
|
let events: Vec<_> = log
|
||||||
|
.lines()
|
||||||
|
.filter(|line| line.contains("workdir_provider_failure"))
|
||||||
|
.collect();
|
||||||
|
assert_eq!(events.len(), 1, "one diagnostic per failed request: {log}");
|
||||||
|
let event = events[0];
|
||||||
|
for field in [
|
||||||
|
"WARN".to_string(),
|
||||||
|
"worker_runtime::workdir".to_string(),
|
||||||
|
format!("status={}", status.as_u16()),
|
||||||
|
format!("code=\"{code}\""),
|
||||||
|
format!("failure_reason=\"{reason}\""),
|
||||||
|
format!("failure_stage=\"{stage}\""),
|
||||||
|
] {
|
||||||
|
assert!(event.contains(&field), "missing {field}: {event}");
|
||||||
|
}
|
||||||
|
assert_private_absent(log);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_failure_body(body: &str, code: &str, message: &str, reason: &str) {
|
||||||
|
assert_private_absent(body);
|
||||||
|
// Exact safe transport vocabulary; raw diagnostics and stage are not wire fields.
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::from_str::<serde_json::Value>(body).unwrap(),
|
||||||
|
serde_json::json!({"code": code, "message": message, "failure_reason": reason}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Harness {
|
||||||
|
server: tokio::task::JoinHandle<()>,
|
||||||
|
client: reqwest::Client,
|
||||||
|
base: String,
|
||||||
|
identity: RuntimeIdentityMaterial,
|
||||||
|
trust: WorkspaceIssuerTrustRecord,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for Harness {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.server.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Harness {
|
||||||
|
async fn new(runtime: Runtime) -> Self {
|
||||||
|
let identity = RuntimeIdentityMaterial::generate("runtime-failure-test-key").unwrap();
|
||||||
|
let trust = workspace_trust(&identity);
|
||||||
|
let app = runtime_http_router_with_workspace_auth(
|
||||||
|
runtime,
|
||||||
|
None,
|
||||||
|
workspace_test_auth(vec![trust.clone()]),
|
||||||
|
);
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||||
|
let server = tokio::spawn(async move {
|
||||||
|
axum::serve(listener, app).await.unwrap();
|
||||||
|
});
|
||||||
|
Self {
|
||||||
|
server,
|
||||||
|
client: reqwest::Client::builder()
|
||||||
|
.no_proxy()
|
||||||
|
.timeout(Duration::from_secs(5))
|
||||||
|
.build()
|
||||||
|
.unwrap(),
|
||||||
|
base,
|
||||||
|
identity,
|
||||||
|
trust,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn send(
|
||||||
|
&self,
|
||||||
|
claims: &WorkspaceCapabilityClaims,
|
||||||
|
body: Vec<u8>,
|
||||||
|
workspace: &str,
|
||||||
|
) -> (StatusCode, String, String) {
|
||||||
|
let token = issue_workspace_capability_token(&self.identity.signing_key().unwrap(), claims)
|
||||||
|
.unwrap();
|
||||||
|
let response = self
|
||||||
|
.client
|
||||||
|
.post(format!("{}{OPEN_PATH}", self.base))
|
||||||
|
.bearer_auth(&token)
|
||||||
|
.header(header::CONTENT_TYPE, "application/json")
|
||||||
|
.header(RUNTIME_WORKSPACE_SCOPE_HEADER, workspace)
|
||||||
|
.header("x-private-test-header", "private-header-marker")
|
||||||
|
.body(body)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let status = StatusCode::from_u16(response.status().as_u16()).unwrap();
|
||||||
|
let headers = format!("{:?}", response.headers());
|
||||||
|
let body = response.text().await.unwrap();
|
||||||
|
assert_private_absent(&headers);
|
||||||
|
for output in [&body, &headers] {
|
||||||
|
assert!(!output.contains(&token), "signed bearer token leaked");
|
||||||
|
assert!(
|
||||||
|
!output.contains(&self.identity.private_key),
|
||||||
|
"signing key leaked"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
(status, body, token)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn open(&self) -> (StatusCode, String, String) {
|
||||||
|
let body = serde_json::to_vec(&OpenWorkdirSessionRequest::default()).unwrap();
|
||||||
|
let claims = signed_claims(&self.trust, "POST", OPEN_PATH, &body);
|
||||||
|
self.send(&claims, body, "workspace-a").await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
enum FailureAt {
|
||||||
|
Lookup,
|
||||||
|
Open,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct FailingBackend {
|
||||||
|
at: FailureAt,
|
||||||
|
code: &'static str,
|
||||||
|
lookups: AtomicUsize,
|
||||||
|
opens: AtomicUsize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FailingBackend {
|
||||||
|
fn runtime(at: FailureAt, code: &'static str) -> (Runtime, Arc<Self>) {
|
||||||
|
let backend = Arc::new(Self {
|
||||||
|
at,
|
||||||
|
code,
|
||||||
|
lookups: AtomicUsize::new(0),
|
||||||
|
opens: AtomicUsize::new(0),
|
||||||
|
});
|
||||||
|
let runtime =
|
||||||
|
Runtime::with_execution_backend(RuntimeOptions::default(), backend.clone()).unwrap();
|
||||||
|
(runtime, backend)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn failure(&self) -> WorkingDirectoryDiagnostic {
|
||||||
|
WorkingDirectoryDiagnostic::rejected(self.code, PRIVATE_DIAGNOSTIC)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkerExecutionBackend for FailingBackend {
|
||||||
|
fn backend_id(&self) -> &str {
|
||||||
|
"runtime-failure-test"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||||
|
AcceptingBackend.spawn_worker(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dispatch_input(&self, worker: &WorkerRef, input: WorkerInput) -> WorkerExecutionResult {
|
||||||
|
AcceptingBackend.dispatch_input(worker, input)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stop_worker(&self, worker: &WorkerRef) -> WorkerExecutionResult {
|
||||||
|
AcceptingBackend.stop_worker(worker)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn working_directory(
|
||||||
|
&self,
|
||||||
|
id: &str,
|
||||||
|
) -> Result<WorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
|
||||||
|
self.lookups.fetch_add(1, Ordering::SeqCst);
|
||||||
|
match self.at {
|
||||||
|
FailureAt::Lookup => Err(self.failure()),
|
||||||
|
FailureAt::Open => Ok(WorkingDirectoryStatus {
|
||||||
|
summary: crate::catalog::WorkingDirectorySummary {
|
||||||
|
working_directory_id: id.into(),
|
||||||
|
display_name: None,
|
||||||
|
repository_id: "repository-test".into(),
|
||||||
|
creation_selector: None,
|
||||||
|
creation_ref: None,
|
||||||
|
creation_tree: None,
|
||||||
|
current_selector: None,
|
||||||
|
current_ref: None,
|
||||||
|
current_tree: None,
|
||||||
|
observed_at_epoch_seconds: None,
|
||||||
|
materializer_kind: crate::catalog::MaterializerKind::RuntimeGitClone,
|
||||||
|
cleanup_target: None,
|
||||||
|
status: crate::catalog::WorkingDirectoryStatusKind::Active,
|
||||||
|
cleanliness: None,
|
||||||
|
occupied_by: None,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open_workdir_session(
|
||||||
|
&self,
|
||||||
|
_: &str,
|
||||||
|
) -> Result<WorkdirSessionHandle, WorkingDirectoryDiagnostic> {
|
||||||
|
self.opens.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Err(self.failure())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Current-thread execution keeps the server tasks inside the thread-local tracing
|
||||||
|
// capture across awaits, without replacing the process-global subscriber.
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn signed_http_session_open_preserves_runtime_availability_and_owner_failures() {
|
||||||
|
let capture = LogCapture::default();
|
||||||
|
let _guard = tracing::subscriber::set_default(capture.subscriber());
|
||||||
|
tracing::callsite::rebuild_interest_cache();
|
||||||
|
tokio::time::timeout(Duration::from_secs(15), async {
|
||||||
|
let stopped = Runtime::new_memory();
|
||||||
|
stopped.stop_runtime().unwrap();
|
||||||
|
let (owned, backend) = FailingBackend::runtime(FailureAt::Lookup, "must_not_reach_backend");
|
||||||
|
let mut request = task_request("bind owner");
|
||||||
|
request.config_bundle = None;
|
||||||
|
request.memory_settings.as_mut().unwrap().workspace_id = "workspace-a".into();
|
||||||
|
owned
|
||||||
|
.create_worker_scoped(
|
||||||
|
&RuntimeWorkspaceScope::new("workspace-a", "private-owner-marker"),
|
||||||
|
request,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
for (runtime, status, code, message, reason) in [
|
||||||
|
(
|
||||||
|
stopped,
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"unavailable",
|
||||||
|
"Workdir session is unavailable",
|
||||||
|
"runtime_stopped",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
Runtime::new_memory(),
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"unavailable",
|
||||||
|
"Workdir session is unavailable",
|
||||||
|
"runtime_backend_unavailable",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
owned,
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
"not_found",
|
||||||
|
"Workdir session was not found",
|
||||||
|
"runtime_owner_mismatch",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let harness = Harness::new(runtime).await;
|
||||||
|
capture.take();
|
||||||
|
let (actual, body, token) = harness.open().await;
|
||||||
|
assert_eq!(actual, status);
|
||||||
|
assert_failure_body(&body, code, message, reason);
|
||||||
|
let log = capture.take();
|
||||||
|
assert_failure_log(&log, status, code, reason, "runtime");
|
||||||
|
assert!(!log.contains(&token));
|
||||||
|
assert!(!log.contains(&harness.identity.private_key));
|
||||||
|
}
|
||||||
|
assert_eq!(backend.lookups.load(Ordering::SeqCst), 0);
|
||||||
|
assert_eq!(backend.opens.load(Ordering::SeqCst), 0);
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded Runtime failure HTTP requests");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn signed_http_session_open_allowlists_diagnostic_codes_without_guessing_unknown_causes() {
|
||||||
|
let capture = LogCapture::default();
|
||||||
|
let _guard = tracing::subscriber::set_default(capture.subscriber());
|
||||||
|
tracing::callsite::rebuild_interest_cache();
|
||||||
|
tokio::time::timeout(Duration::from_secs(20), async {
|
||||||
|
for (diagnostic, reason) in [
|
||||||
|
("working_directory_not_found", "working_directory_not_found"),
|
||||||
|
(
|
||||||
|
"working_directory_not_active",
|
||||||
|
"working_directory_not_active",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"working_directory_record_invalid",
|
||||||
|
"working_directory_record_invalid",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"working_directory_repository_access_unavailable",
|
||||||
|
"working_directory_access_unavailable",
|
||||||
|
),
|
||||||
|
// Plausible provider text/code is not evidence of a typed cause.
|
||||||
|
("authorization_stale", "session_open_failed"),
|
||||||
|
(
|
||||||
|
"working_directory_not_found_private-proof-marker",
|
||||||
|
"session_open_failed",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
for at in [FailureAt::Lookup, FailureAt::Open] {
|
||||||
|
let (runtime, backend) = FailingBackend::runtime(at, diagnostic);
|
||||||
|
let harness = Harness::new(runtime).await;
|
||||||
|
capture.take();
|
||||||
|
let (status, body, token) = harness.open().await;
|
||||||
|
let (expected, code, message) = if diagnostic == "working_directory_not_found" {
|
||||||
|
(
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
"not_found",
|
||||||
|
"Workdir session was not found",
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"unavailable",
|
||||||
|
"Workdir session is unavailable",
|
||||||
|
)
|
||||||
|
};
|
||||||
|
assert_eq!(status, expected, "{diagnostic}");
|
||||||
|
assert_failure_body(&body, code, message, reason);
|
||||||
|
let log = capture.take();
|
||||||
|
assert_failure_log(&log, status, code, reason, "session_open");
|
||||||
|
assert!(!log.contains(&token));
|
||||||
|
assert!(!log.contains(&harness.identity.private_key));
|
||||||
|
assert_eq!(backend.lookups.load(Ordering::SeqCst), 1, "no hidden retry");
|
||||||
|
assert_eq!(
|
||||||
|
backend.opens.load(Ordering::SeqCst),
|
||||||
|
usize::from(matches!(at, FailureAt::Open)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded diagnostic fault matrix");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn signed_http_session_open_rejects_bad_capability_scope_and_owner_binding_before_backend() {
|
||||||
|
let capture = LogCapture::default();
|
||||||
|
let _guard = tracing::subscriber::set_default(capture.subscriber());
|
||||||
|
tracing::callsite::rebuild_interest_cache();
|
||||||
|
tokio::time::timeout(Duration::from_secs(15), async {
|
||||||
|
let (runtime, backend) =
|
||||||
|
FailingBackend::runtime(FailureAt::Open, "working_directory_record_invalid");
|
||||||
|
let mut request = task_request("foreign workspace binding");
|
||||||
|
request.config_bundle = None;
|
||||||
|
request.memory_settings.as_mut().unwrap().workspace_id = "other-workspace".into();
|
||||||
|
let foreign_worker = runtime
|
||||||
|
.create_worker_scoped(
|
||||||
|
&RuntimeWorkspaceScope::new("other-workspace", "https://backend.test"),
|
||||||
|
request,
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.worker_ref;
|
||||||
|
let harness = Harness::new(runtime).await;
|
||||||
|
for case in ["permission", "body", "workspace", "owner_binding"] {
|
||||||
|
let body = if case == "owner_binding" {
|
||||||
|
serde_json::to_vec(&OpenWorkdirSessionRequest {
|
||||||
|
owner_worker_id: Some(foreign_worker.worker_id.to_string()),
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
} else {
|
||||||
|
b"{}".to_vec()
|
||||||
|
};
|
||||||
|
let mut claims = signed_claims(&harness.trust, "POST", OPEN_PATH, &body);
|
||||||
|
if case == "permission" {
|
||||||
|
claims.operation = "workers:read".into();
|
||||||
|
}
|
||||||
|
if case == "workspace" {
|
||||||
|
claims.issuer_workspace_id = "other-workspace".into();
|
||||||
|
}
|
||||||
|
let sent = if case == "body" {
|
||||||
|
b"{ }".to_vec()
|
||||||
|
} else {
|
||||||
|
body
|
||||||
|
};
|
||||||
|
capture.take();
|
||||||
|
let (status, body, token) = harness.send(&claims, sent, "workspace-a").await;
|
||||||
|
assert_eq!(
|
||||||
|
status,
|
||||||
|
if case == "owner_binding" {
|
||||||
|
StatusCode::NOT_FOUND
|
||||||
|
} else {
|
||||||
|
StatusCode::UNAUTHORIZED
|
||||||
|
},
|
||||||
|
"{case}",
|
||||||
|
);
|
||||||
|
assert_private_absent(&body);
|
||||||
|
let log = capture.take();
|
||||||
|
assert_private_absent(&log);
|
||||||
|
assert!(!log.contains(&token));
|
||||||
|
assert!(!log.contains(&harness.identity.private_key));
|
||||||
|
if case == "owner_binding" {
|
||||||
|
assert_failure_body(
|
||||||
|
&body,
|
||||||
|
"not_found",
|
||||||
|
"Workdir session was not found",
|
||||||
|
"working_directory_not_found",
|
||||||
|
);
|
||||||
|
assert_failure_log(
|
||||||
|
&log,
|
||||||
|
status,
|
||||||
|
"not_found",
|
||||||
|
"working_directory_not_found",
|
||||||
|
"session_open",
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
let wire: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||||
|
assert_eq!(wire["error"]["code"], "unauthorized");
|
||||||
|
assert!(
|
||||||
|
!log.contains("workdir_provider_failure"),
|
||||||
|
"auth rejection is not a provider fault",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(backend.lookups.load(Ordering::SeqCst), 0);
|
||||||
|
assert_eq!(backend.opens.load(Ordering::SeqCst), 0);
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded authorization rejection requests");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn runtime_error_response_keeps_raw_details_private_and_unclassified_causes_generic() {
|
||||||
|
let capture = LogCapture::default();
|
||||||
|
let _guard = tracing::subscriber::set_default(capture.subscriber());
|
||||||
|
tracing::callsite::rebuild_interest_cache();
|
||||||
|
for (error, status, code, message, reason, stage) in [
|
||||||
|
(
|
||||||
|
RuntimeError::ExecutionBackendUnavailable {
|
||||||
|
message: PRIVATE_DIAGNOSTIC.into(),
|
||||||
|
},
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"unavailable",
|
||||||
|
"Workdir session is unavailable",
|
||||||
|
"runtime_backend_unavailable",
|
||||||
|
"runtime",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
RuntimeError::StatePoisoned,
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"unavailable",
|
||||||
|
"Workdir session is unavailable",
|
||||||
|
"runtime_state_unavailable",
|
||||||
|
"runtime",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
RuntimeError::InvalidRequest(PRIVATE_DIAGNOSTIC.into()),
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
"invalid_request",
|
||||||
|
"Workdir operation request is invalid",
|
||||||
|
"runtime_invalid_request",
|
||||||
|
"runtime",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
RuntimeError::StoreCorrupt {
|
||||||
|
operation: "session_open",
|
||||||
|
path: PathBuf::from("/private/host-path-marker"),
|
||||||
|
message: PRIVATE_DIAGNOSTIC.into(),
|
||||||
|
},
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"unavailable",
|
||||||
|
"Workdir session is unavailable",
|
||||||
|
"session_open_failed",
|
||||||
|
"session_open",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
capture.take();
|
||||||
|
let response = RuntimeHttpWorkdirError::runtime(error).into_response();
|
||||||
|
assert_eq!(response.status(), status);
|
||||||
|
let bytes = to_bytes(response.into_body(), 16384).await.unwrap();
|
||||||
|
let body = std::str::from_utf8(&bytes).unwrap();
|
||||||
|
assert_failure_body(body, code, message, reason);
|
||||||
|
assert_failure_log(&capture.take(), status, code, reason, stage);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -150,6 +150,27 @@ impl Fixture {
|
|||||||
limit: 1,
|
limit: 1,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
fn recording_request(&self) -> runtime_api::SessionRecordingCaptureRequest {
|
||||||
|
runtime_api::SessionRecordingCaptureRequest {
|
||||||
|
workspace_id: self.scope.workspace_id.clone(),
|
||||||
|
source: self.source(),
|
||||||
|
expected_session_id: self.session.to_string(),
|
||||||
|
start: Default::default(),
|
||||||
|
max_entries: 1000,
|
||||||
|
max_serialized_bytes: 128 * 1024,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn capture(&self, source: Source) -> session_store::capture::RecordingCaptureSlice {
|
||||||
|
let mut request = self.recording_request();
|
||||||
|
request.source = source;
|
||||||
|
match self
|
||||||
|
.runtime
|
||||||
|
.session_recording_capture_scoped(&self.scope, &request)
|
||||||
|
{
|
||||||
|
runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => slice,
|
||||||
|
other => panic!("{other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
fn catalog(&self) -> runtime_api::SessionMaterialCatalogPage {
|
fn catalog(&self) -> runtime_api::SessionMaterialCatalogPage {
|
||||||
match self
|
match self
|
||||||
.runtime
|
.runtime
|
||||||
@@ -226,6 +247,9 @@ fn pending_session_proof_is_not_missing_material_and_pin_release_allows_empty_ar
|
|||||||
.is_err(),
|
.is_err(),
|
||||||
"retention intent is not source availability"
|
"retention intent is not source availability"
|
||||||
);
|
);
|
||||||
|
assert!(
|
||||||
|
matches!(f.runtime.session_recording_capture_scoped(&f.scope, &f.recording_request()), runtime_api::SessionRecordingCaptureAvailability::NotMaterialized { session_id } if session_id == f.session.to_string())
|
||||||
|
);
|
||||||
// A missing committed source is never the same fact as a pending pointer.
|
// A missing committed source is never the same fact as a pending pointer.
|
||||||
metadata
|
metadata
|
||||||
.write(
|
.write(
|
||||||
@@ -479,6 +503,7 @@ fn source_scope_session_and_active_pointer_mismatches_never_return_material() {
|
|||||||
#[test]
|
#[test]
|
||||||
fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source() {
|
fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source() {
|
||||||
let f = Fixture::new();
|
let f = Fixture::new();
|
||||||
|
let frozen = f.capture(f.source());
|
||||||
let original: Vec<_> = f.reader(&f.source()).descriptors().cloned().collect();
|
let original: Vec<_> = f.reader(&f.source()).descriptors().cloned().collect();
|
||||||
let pin = SessionRetentionPin(uuid::Uuid::now_v7());
|
let pin = SessionRetentionPin(uuid::Uuid::now_v7());
|
||||||
f.runtime
|
f.runtime
|
||||||
@@ -516,6 +541,7 @@ fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source()
|
|||||||
let archived = Source::Archived {
|
let archived = Source::Archived {
|
||||||
archive_id: "canonical-archive".into(),
|
archive_id: "canonical-archive".into(),
|
||||||
};
|
};
|
||||||
|
assert_eq!(f.capture(archived.clone()), frozen);
|
||||||
let reader = f.reader(&archived);
|
let reader = f.reader(&archived);
|
||||||
assert_eq!(reader.descriptors().cloned().collect::<Vec<_>>(), original);
|
assert_eq!(reader.descriptors().cloned().collect::<Vec<_>>(), original);
|
||||||
let body = reader.read_content(&original[0], 0, 16384).unwrap();
|
let body = reader.read_content(&original[0], 0, 16384).unwrap();
|
||||||
@@ -951,11 +977,52 @@ async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router
|
|||||||
.authorizer(SignedAuthorizer { identity, trust })
|
.authorizer(SignedAuthorizer { identity, trust })
|
||||||
.build()
|
.build()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
let frozen = match client
|
||||||
|
.session_recording_capture(f.recording_request())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
{
|
||||||
|
runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => slice,
|
||||||
|
other => panic!("{other:?}"),
|
||||||
|
};
|
||||||
|
frozen.capture.validate(Default::default()).unwrap();
|
||||||
|
assert!(frozen.next.is_none());
|
||||||
|
assert!(
|
||||||
|
serde_json::to_vec(&frozen).unwrap().len()
|
||||||
|
<= f.recording_request().max_serialized_bytes
|
||||||
|
);
|
||||||
|
let snapshot = serde_json::to_vec(&frozen).unwrap();
|
||||||
|
assert!(!String::from_utf8_lossy(&snapshot).contains("private-system-prompt"));
|
||||||
|
f.store
|
||||||
|
.append(
|
||||||
|
f.session,
|
||||||
|
f.segment,
|
||||||
|
&LogEntry::AnnotatedAssistantItem {
|
||||||
|
ts: 6,
|
||||||
|
entry: message("late-answer", LoggedRole::Assistant, "not in frozen input"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(serde_json::to_vec(&frozen).unwrap(), snapshot);
|
||||||
let page = match client.session_material_catalog(f.request()).await.unwrap() {
|
let page = match client.session_material_catalog(f.request()).await.unwrap() {
|
||||||
Catalog::Page { page } => page,
|
Catalog::Page { page } => page,
|
||||||
other => panic!("{other:?}"),
|
other => panic!("{other:?}"),
|
||||||
};
|
};
|
||||||
let descriptor = page.descriptors[0].clone();
|
let descriptor = page.descriptors[0].clone();
|
||||||
|
assert_eq!(
|
||||||
|
frozen
|
||||||
|
.capture
|
||||||
|
.recording_text(0)
|
||||||
|
.unwrap()
|
||||||
|
.0
|
||||||
|
.identity
|
||||||
|
.entry_ref,
|
||||||
|
descriptor.entry_ref
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
frozen.capture.recording_text(0).unwrap().0.sha256,
|
||||||
|
descriptor.sha256
|
||||||
|
);
|
||||||
let read = client
|
let read = client
|
||||||
.session_material_read(runtime_api::SessionMaterialReadRequest {
|
.session_material_read(runtime_api::SessionMaterialReadRequest {
|
||||||
workspace_id: "workspace-a".into(),
|
workspace_id: "workspace-a".into(),
|
||||||
@@ -1019,6 +1086,13 @@ async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
assert!(!f.root().exists());
|
assert!(!f.root().exists());
|
||||||
|
assert!(matches!(
|
||||||
|
client
|
||||||
|
.session_recording_capture(f.recording_request())
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
runtime_api::SessionRecordingCaptureAvailability::Unavailable { .. }
|
||||||
|
));
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
client.session_material_catalog(f.request()).await.unwrap(),
|
client.session_material_catalog(f.request()).await.unwrap(),
|
||||||
Catalog::Unavailable { .. }
|
Catalog::Unavailable { .. }
|
||||||
@@ -1050,41 +1124,54 @@ async fn canonical_http_rejects_replay_tampered_body_and_workspace_mismatch() {
|
|||||||
None,
|
None,
|
||||||
workspace_test_auth(vec![trust.clone()]),
|
workspace_test_auth(vec![trust.clone()]),
|
||||||
);
|
);
|
||||||
let path = "/v1/session-public/material/catalog";
|
for (path, body) in [
|
||||||
let body = serde_json::to_vec(&f.request()).unwrap();
|
(
|
||||||
let claims = signed_claims(&trust, "POST", path, &body);
|
"/v1/session-public/material/catalog",
|
||||||
let first = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
|
serde_json::to_vec(&f.request()).unwrap(),
|
||||||
assert_eq!(first.status(), StatusCode::OK);
|
),
|
||||||
let replay = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
|
(
|
||||||
assert_eq!(replay.status(), StatusCode::UNAUTHORIZED);
|
"/v1/session-public/recording-capture",
|
||||||
let claims = signed_claims(&trust, "POST", path, &body);
|
serde_json::to_vec(&f.recording_request()).unwrap(),
|
||||||
let changed = signed_request(
|
),
|
||||||
app.clone(),
|
] {
|
||||||
&identity,
|
let claims = signed_claims(&trust, "POST", path, &body);
|
||||||
&claims,
|
let first =
|
||||||
b"{}".to_vec(),
|
signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
|
||||||
"workspace-a",
|
assert_eq!(first.status(), StatusCode::OK);
|
||||||
)
|
let replay =
|
||||||
.await;
|
signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
|
||||||
assert_eq!(changed.status(), StatusCode::UNAUTHORIZED);
|
assert_eq!(replay.status(), StatusCode::UNAUTHORIZED);
|
||||||
let mut wrong_scope = f.request();
|
let claims = signed_claims(&trust, "POST", path, &body);
|
||||||
wrong_scope.workspace_id = "other-workspace".into();
|
let changed = signed_request(
|
||||||
let wrong_body = serde_json::to_vec(&wrong_scope).unwrap();
|
app.clone(),
|
||||||
let claims = signed_claims(&trust, "POST", path, &wrong_body);
|
&identity,
|
||||||
let response = signed_request(app.clone(), &identity, &claims, wrong_body, "workspace-a").await;
|
&claims,
|
||||||
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
b"{}".to_vec(),
|
||||||
let unsigned = app
|
"workspace-a",
|
||||||
.oneshot(
|
|
||||||
Request::builder()
|
|
||||||
.method("POST")
|
|
||||||
.uri(path)
|
|
||||||
.header(header::CONTENT_TYPE, "application/json")
|
|
||||||
.body(Body::from(body))
|
|
||||||
.unwrap(),
|
|
||||||
)
|
)
|
||||||
.await
|
.await;
|
||||||
.unwrap();
|
assert_eq!(changed.status(), StatusCode::UNAUTHORIZED);
|
||||||
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
|
let mut wrong_scope: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||||
|
wrong_scope["workspace_id"] = "other-workspace".into();
|
||||||
|
let wrong_body = serde_json::to_vec(&wrong_scope).unwrap();
|
||||||
|
let claims = signed_claims(&trust, "POST", path, &wrong_body);
|
||||||
|
let response =
|
||||||
|
signed_request(app.clone(), &identity, &claims, wrong_body, "workspace-a").await;
|
||||||
|
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
||||||
|
let unsigned = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri(path)
|
||||||
|
.header(header::CONTENT_TYPE, "application/json")
|
||||||
|
.body(Body::from(body))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1170,3 +1257,152 @@ async fn explicit_material_erasure_requires_delete_capability_and_matching_scope
|
|||||||
);
|
);
|
||||||
assert!(f.root().exists());
|
assert!(f.root().exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn recording_capture_rejects_scope_session_cursor_and_budget_without_returning_bodies() {
|
||||||
|
use runtime_api::SessionRecordingCaptureAvailability as Capture;
|
||||||
|
let f = Fixture::new();
|
||||||
|
let request = f.recording_request();
|
||||||
|
let frozen = f.capture(f.source());
|
||||||
|
assert_eq!(frozen.capture.entries.len(), 3);
|
||||||
|
let mut request = request;
|
||||||
|
for invalid in 0..5 {
|
||||||
|
let mut bad = request.clone();
|
||||||
|
match invalid {
|
||||||
|
0 => bad.workspace_id = "other".into(),
|
||||||
|
1 => bad.expected_session_id = session_store::new_session_id().to_string(),
|
||||||
|
2 => bad.start.canonical_order = 100,
|
||||||
|
3 => bad.max_entries = 0,
|
||||||
|
4 => bad.max_serialized_bytes = 4 * 1024 * 1024 + 1,
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
assert!(matches!(
|
||||||
|
f.runtime.session_recording_capture_scoped(&f.scope, &bad),
|
||||||
|
Capture::Unavailable { .. }
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let text = frozen.capture.recording_text(1).unwrap().2;
|
||||||
|
request.start = session_store::capture::RecordingPosition {
|
||||||
|
canonical_order: 1,
|
||||||
|
byte_offset: text.find('🍎').unwrap() as u64 + 1,
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
f.runtime
|
||||||
|
.session_recording_capture_scoped(&f.scope, &request),
|
||||||
|
Capture::Unavailable {
|
||||||
|
reason: Reason::InvalidCursor,
|
||||||
|
..
|
||||||
|
}
|
||||||
|
));
|
||||||
|
request.start = Default::default();
|
||||||
|
request.max_entries = 1;
|
||||||
|
let Capture::Captured { slice } = f
|
||||||
|
.runtime
|
||||||
|
.session_recording_capture_scoped(&f.scope, &request)
|
||||||
|
else {
|
||||||
|
panic!()
|
||||||
|
};
|
||||||
|
assert_eq!(slice.capture.entries.len(), 1);
|
||||||
|
assert_eq!(slice.next.unwrap().canonical_order, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn recording_capture_compaction_preserves_original_descriptors_and_excludes_summary_and_new_tail() {
|
||||||
|
let f = Fixture::new();
|
||||||
|
let old = f.capture(f.source());
|
||||||
|
let compact = session_store::new_segment_id();
|
||||||
|
let mut summary = message(
|
||||||
|
"summary",
|
||||||
|
LoggedRole::System,
|
||||||
|
"derived summary is not original",
|
||||||
|
);
|
||||||
|
summary.metadata.input_metadata = Some(
|
||||||
|
session_store::HistoryInputMetadata::new(
|
||||||
|
session_store::HistoryInputUsePolicy {
|
||||||
|
recording_material: session_store::InputEvidenceUse::Exclude,
|
||||||
|
memory_evidence: session_store::InputEvidenceUse::Exclude,
|
||||||
|
observation: session_store::InputObservationUse::Hidden,
|
||||||
|
compaction: session_store::InputCompactionUse::OpeningSummary,
|
||||||
|
},
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
summary.metadata.derivation = Some(session_store::LoggedHistoryDerivation {
|
||||||
|
sources: vec![
|
||||||
|
LoggedSessionHistoryEntryId("user-1".into()),
|
||||||
|
LoggedSessionHistoryEntryId("answer-1".into()),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
f.store
|
||||||
|
.create_segment(
|
||||||
|
f.session,
|
||||||
|
compact,
|
||||||
|
&[
|
||||||
|
LogEntry::AnnotatedSegmentStart {
|
||||||
|
ts: 6,
|
||||||
|
session_id: f.session,
|
||||||
|
system_prompt: Some("private".into()),
|
||||||
|
config: Default::default(),
|
||||||
|
history: vec![summary],
|
||||||
|
forked_from: None,
|
||||||
|
compacted_from: Some(session_store::SegmentOrigin {
|
||||||
|
segment_id: f.segment,
|
||||||
|
at_turn_index: 0,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
LogEntry::AnnotatedAssistantItem {
|
||||||
|
ts: 7,
|
||||||
|
entry: message(
|
||||||
|
"new-answer",
|
||||||
|
LoggedRole::Assistant,
|
||||||
|
"new original after compaction",
|
||||||
|
),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
WorkerAggregateStore::new(
|
||||||
|
f.root().parent().unwrap(),
|
||||||
|
format!("worker-runtime-{}", f.worker.worker_id),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.set_active(
|
||||||
|
&format!("worker-runtime-{}", f.worker.worker_id),
|
||||||
|
Some(session_store::WorkerActiveSegmentRef::active_segment(
|
||||||
|
f.session, compact,
|
||||||
|
)),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let frozen = f.capture(f.source());
|
||||||
|
assert_eq!(frozen.capture.entries.len(), old.capture.entries.len() + 1);
|
||||||
|
for entry in &old.capture.entries {
|
||||||
|
assert_eq!(
|
||||||
|
frozen.capture.recording_text(entry.order).unwrap(),
|
||||||
|
old.capture.recording_text(entry.order).unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
!serde_json::to_string(&frozen)
|
||||||
|
.unwrap()
|
||||||
|
.contains("derived summary is not original")
|
||||||
|
);
|
||||||
|
let bytes = serde_json::to_vec(&frozen).unwrap();
|
||||||
|
f.store
|
||||||
|
.append(
|
||||||
|
f.session,
|
||||||
|
compact,
|
||||||
|
&LogEntry::AnnotatedAssistantItem {
|
||||||
|
ts: 8,
|
||||||
|
entry: message("tail", LoggedRole::Assistant, "next Job only"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(serde_json::to_vec(&frozen).unwrap(), bytes);
|
||||||
|
assert_eq!(
|
||||||
|
f.capture(f.source()).capture.entries.len(),
|
||||||
|
frozen.capture.entries.len() + 1
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -16,6 +16,11 @@ pub mod management;
|
|||||||
pub mod observation;
|
pub mod observation;
|
||||||
pub mod profile_archive;
|
pub mod profile_archive;
|
||||||
pub mod resource;
|
pub mod resource;
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
pub mod restore_checkpoints;
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "../../../tests/support/restore_environment.rs"]
|
||||||
|
mod restore_test_environment;
|
||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
pub mod retention;
|
pub mod retention;
|
||||||
mod runtime;
|
mod runtime;
|
||||||
@@ -32,6 +37,8 @@ pub mod workspace_issuer;
|
|||||||
#[cfg(feature = "http-server")]
|
#[cfg(feature = "http-server")]
|
||||||
pub mod workspace_request;
|
pub mod workspace_request;
|
||||||
|
|
||||||
|
#[cfg(feature = "test-support")]
|
||||||
|
pub use fs_store::test_support;
|
||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
pub use fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
|
pub use fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
|
||||||
pub use management::RuntimeOptions;
|
pub use management::RuntimeOptions;
|
||||||
|
|||||||
@@ -117,6 +117,12 @@ fn run() -> Result<(), ProcessError> {
|
|||||||
let listener = tokio::net::TcpListener::bind(config.http.bind_addr).await?;
|
let listener = tokio::net::TcpListener::bind(config.http.bind_addr).await?;
|
||||||
let local_addr = listener.local_addr()?;
|
let local_addr = listener.local_addr()?;
|
||||||
let worker_runtime = build_runtime(&config)?;
|
let worker_runtime = build_runtime(&config)?;
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
if let Some(checkpoint) = &config.test_restore_checkpoint {
|
||||||
|
worker_runtime
|
||||||
|
.install_test_restore_checkpoint(checkpoint.clone())
|
||||||
|
.map_err(ProcessError::Runtime)?;
|
||||||
|
}
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"yoi-runtime listening on {local_addr}; intended client is a trusted backend/proxy, not a browser"
|
"yoi-runtime listening on {local_addr}; intended client is a trusted backend/proxy, not a browser"
|
||||||
);
|
);
|
||||||
@@ -391,6 +397,21 @@ where
|
|||||||
let (flag, inline_value) = split_flag_value(arg)?;
|
let (flag, inline_value) = split_flag_value(arg)?;
|
||||||
match flag.as_str() {
|
match flag.as_str() {
|
||||||
"--help" | "-h" => return Ok(None),
|
"--help" | "-h" => return Ok(None),
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
"--test-restore-checkpoint" => {
|
||||||
|
let path = take_value(&flag, inline_value, &mut args)?;
|
||||||
|
if config.test_restore_checkpoint.is_some() {
|
||||||
|
return Err(ProcessError::usage(
|
||||||
|
"--test-restore-checkpoint may only be specified once".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
config.test_restore_checkpoint = Some(
|
||||||
|
worker_runtime::restore_checkpoints::RestoreCheckpointConfig::from_path(
|
||||||
|
Path::new(&path),
|
||||||
|
)
|
||||||
|
.map_err(ProcessError::Runtime)?,
|
||||||
|
);
|
||||||
|
}
|
||||||
"--bind" => {
|
"--bind" => {
|
||||||
let value = take_value(&flag, inline_value, &mut args)?;
|
let value = take_value(&flag, inline_value, &mut args)?;
|
||||||
config.http.bind_addr = value.parse::<SocketAddr>().map_err(|error| {
|
config.http.bind_addr = value.parse::<SocketAddr>().map_err(|error| {
|
||||||
@@ -465,6 +486,12 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
apply_store_selection(&mut config);
|
apply_store_selection(&mut config);
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
if config.test_restore_checkpoint.is_some() && config.no_store {
|
||||||
|
return Err(ProcessError::usage(
|
||||||
|
"--test-restore-checkpoint requires an fs Runtime store".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
Ok(Some(config))
|
Ok(Some(config))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -521,6 +548,8 @@ struct ProcessConfig {
|
|||||||
no_store: bool,
|
no_store: bool,
|
||||||
backend_resource_endpoint: Option<String>,
|
backend_resource_endpoint: Option<String>,
|
||||||
backend_resource_token: Option<String>,
|
backend_resource_token: Option<String>,
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
test_restore_checkpoint: Option<worker_runtime::restore_checkpoints::RestoreCheckpointConfig>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
@@ -542,6 +571,8 @@ impl ProcessConfig {
|
|||||||
no_store: false,
|
no_store: false,
|
||||||
backend_resource_endpoint: None,
|
backend_resource_endpoint: None,
|
||||||
backend_resource_token: None,
|
backend_resource_token: None,
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
test_restore_checkpoint: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1227,6 +1258,47 @@ Auth commands:
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_cli_is_absent_by_default() {
|
||||||
|
let config = parse_args([] as [&str; 0]).unwrap().unwrap();
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
assert!(config.test_restore_checkpoint.is_none());
|
||||||
|
#[cfg(not(feature = "restore-checkpoints"))]
|
||||||
|
{
|
||||||
|
let _ = config;
|
||||||
|
let error = parse_args(["--test-restore-checkpoint", "never-opened.json"]).unwrap_err();
|
||||||
|
assert!(error.to_string().contains("unknown argument"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_cli_requires_valid_explicit_file_and_fs_store() {
|
||||||
|
use worker_runtime::restore_checkpoints::{
|
||||||
|
RestoreCheckpointConfig, RestoreCheckpointStage,
|
||||||
|
};
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("checkpoint.json");
|
||||||
|
let checkpoint = RestoreCheckpointConfig {
|
||||||
|
endpoint: "127.0.0.1:1234".parse().unwrap(),
|
||||||
|
worker_id: protocol::WorkerId::now_v7(),
|
||||||
|
stage: RestoreCheckpointStage::CommittedPreterminal,
|
||||||
|
nonce: "cli-test".into(),
|
||||||
|
};
|
||||||
|
std::fs::write(&path, serde_json::to_vec(&checkpoint).unwrap()).unwrap();
|
||||||
|
let args = ["--test-restore-checkpoint", path.to_str().unwrap()];
|
||||||
|
let config = parse_args(args).unwrap().unwrap();
|
||||||
|
assert_eq!(config.test_restore_checkpoint, Some(checkpoint));
|
||||||
|
assert!(parse_args([args[0], args[1], "--no-store"]).is_err());
|
||||||
|
assert!(parse_args([args[0], args[1], args[0], args[1]]).is_err());
|
||||||
|
assert!(parse_args([args[0]]).is_err());
|
||||||
|
std::fs::write(&path, b"private-invalid-body").unwrap();
|
||||||
|
let error = parse_args(args).unwrap_err().to_string();
|
||||||
|
assert!(!error.contains("private-invalid-body"));
|
||||||
|
std::fs::remove_file(&path).unwrap();
|
||||||
|
assert!(parse_args(args).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn help_does_not_start_server() {
|
fn help_does_not_start_server() {
|
||||||
assert!(parse_args(["--help"]).unwrap().is_none());
|
assert!(parse_args(["--help"]).unwrap().is_none());
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
|
#[cfg(feature = "http-server")]
|
||||||
use crate::auth::BACKEND_RESOURCE_FETCH_PERMISSION;
|
use crate::auth::BACKEND_RESOURCE_FETCH_PERMISSION;
|
||||||
use crate::identity::WorkerId;
|
use crate::identity::WorkerId;
|
||||||
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef, sha256_hex};
|
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef, sha256_hex};
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
use crate::workspace_request::RuntimeWorkspaceRequestClient;
|
use crate::workspace_request::RuntimeWorkspaceRequestClient;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|||||||
@@ -0,0 +1,144 @@
|
|||||||
|
//! Explicit, ephemeral synchronization for owned Runtime process tests only.
|
||||||
|
//!
|
||||||
|
//! The nonce authenticates no Runtime action and is never a routing identity.
|
||||||
|
//! Socket failure, EOF and the fixed deadline only release the gate; the lifecycle
|
||||||
|
//! owner continues its original operation and remains responsible for cleanup.
|
||||||
|
|
||||||
|
use std::fs::OpenOptions;
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::net::{SocketAddr, TcpStream};
|
||||||
|
use std::path::Path;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use protocol::WorkerId;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::error::RuntimeError;
|
||||||
|
use crate::execution::WorkerLifecycleOperationId;
|
||||||
|
|
||||||
|
pub const RESTORE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(8);
|
||||||
|
pub const MAX_RESTORE_CHECKPOINT_CONFIG_BYTES: usize = 4096;
|
||||||
|
pub const MAX_RESTORE_CHECKPOINT_ACK_BYTES: usize = 4096;
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum RestoreCheckpointStage {
|
||||||
|
PreparedNotCommitted,
|
||||||
|
CommittedPreterminal,
|
||||||
|
ActivatedPreterminal,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum RestoreCheckpointCommit {
|
||||||
|
NotCommitted,
|
||||||
|
Committed,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct RestoreCheckpointConfig {
|
||||||
|
pub endpoint: SocketAddr,
|
||||||
|
pub worker_id: WorkerId,
|
||||||
|
pub stage: RestoreCheckpointStage,
|
||||||
|
pub nonce: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct RestoreCheckpointAck {
|
||||||
|
pub nonce: String,
|
||||||
|
pub stage: RestoreCheckpointStage,
|
||||||
|
pub runtime_id: String,
|
||||||
|
pub worker_id: WorkerId,
|
||||||
|
pub operation_id: WorkerLifecycleOperationId,
|
||||||
|
pub commit: RestoreCheckpointCommit,
|
||||||
|
pub receipt_absent: bool,
|
||||||
|
pub pid: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn invalid_config() -> RuntimeError {
|
||||||
|
// Do not include serde errors, paths or input contents in process diagnostics.
|
||||||
|
RuntimeError::InvalidRequest("invalid test restore checkpoint configuration".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RestoreCheckpointConfig {
|
||||||
|
pub fn validate(&self) -> Result<(), RuntimeError> {
|
||||||
|
if !self.endpoint.ip().is_loopback()
|
||||||
|
|| self.endpoint.port() == 0
|
||||||
|
|| self.nonce.is_empty()
|
||||||
|
|| self.nonce.len() > 256
|
||||||
|
|| self.nonce.chars().any(char::is_control)
|
||||||
|
{
|
||||||
|
return Err(invalid_config());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_path(path: &Path) -> Result<Self, RuntimeError> {
|
||||||
|
let mut options = OpenOptions::new();
|
||||||
|
options.read(true);
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
|
// A FIFO must not stall startup before its regular-file check.
|
||||||
|
options.custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW);
|
||||||
|
}
|
||||||
|
let file = options.open(path).map_err(|_| invalid_config())?;
|
||||||
|
let metadata = file.metadata().map_err(|_| invalid_config())?;
|
||||||
|
if !metadata.is_file() || metadata.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 {
|
||||||
|
return Err(invalid_config());
|
||||||
|
}
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
file.take(MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 + 1)
|
||||||
|
.read_to_end(&mut bytes)
|
||||||
|
.map_err(|_| invalid_config())?;
|
||||||
|
if bytes.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES {
|
||||||
|
return Err(invalid_config());
|
||||||
|
}
|
||||||
|
let config: Self = serde_json::from_slice(&bytes).map_err(|_| invalid_config())?;
|
||||||
|
config.validate()?;
|
||||||
|
Ok(config)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn wait_for_release(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) {
|
||||||
|
let _ = exchange(config, ack);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remaining(deadline: Instant) -> std::io::Result<Duration> {
|
||||||
|
deadline
|
||||||
|
.checked_duration_since(Instant::now())
|
||||||
|
.filter(|remaining| !remaining.is_zero())
|
||||||
|
.ok_or_else(|| std::io::Error::from(std::io::ErrorKind::TimedOut))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exchange(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) -> std::io::Result<()> {
|
||||||
|
let deadline = Instant::now() + RESTORE_CHECKPOINT_TIMEOUT;
|
||||||
|
let mut bytes = serde_json::to_vec(ack)?;
|
||||||
|
bytes.push(b'\n');
|
||||||
|
if bytes.len() > MAX_RESTORE_CHECKPOINT_ACK_BYTES {
|
||||||
|
return Err(std::io::Error::from(std::io::ErrorKind::InvalidData));
|
||||||
|
}
|
||||||
|
let mut stream = TcpStream::connect_timeout(&config.endpoint, remaining(deadline)?)?;
|
||||||
|
// Each syscall shares one budget; partial writes cannot reset the deadline.
|
||||||
|
let mut pending = bytes.as_slice();
|
||||||
|
while !pending.is_empty() {
|
||||||
|
stream.set_write_timeout(Some(remaining(deadline)?))?;
|
||||||
|
let written = stream.write(pending)?;
|
||||||
|
if written == 0 {
|
||||||
|
return Err(std::io::Error::from(std::io::ErrorKind::WriteZero));
|
||||||
|
}
|
||||||
|
pending = &pending[written..];
|
||||||
|
}
|
||||||
|
stream.set_read_timeout(Some(remaining(deadline)?))?;
|
||||||
|
let mut release = [0];
|
||||||
|
stream.read_exact(&mut release)?;
|
||||||
|
if release != [b'R'] {
|
||||||
|
return Err(std::io::Error::from(std::io::ErrorKind::InvalidData));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
use super::*;
|
||||||
|
use std::io::{BufRead, BufReader};
|
||||||
|
use std::net::TcpListener;
|
||||||
|
use std::thread;
|
||||||
|
|
||||||
|
fn config(endpoint: SocketAddr) -> RestoreCheckpointConfig {
|
||||||
|
RestoreCheckpointConfig {
|
||||||
|
endpoint,
|
||||||
|
worker_id: WorkerId::now_v7(),
|
||||||
|
stage: RestoreCheckpointStage::PreparedNotCommitted,
|
||||||
|
nonce: "checkpoint-test-handshake".into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ack(config: &RestoreCheckpointConfig) -> RestoreCheckpointAck {
|
||||||
|
RestoreCheckpointAck {
|
||||||
|
nonce: config.nonce.clone(),
|
||||||
|
stage: config.stage,
|
||||||
|
runtime_id: "checkpoint-runtime".into(),
|
||||||
|
worker_id: config.worker_id,
|
||||||
|
operation_id: WorkerLifecycleOperationId::new(),
|
||||||
|
commit: RestoreCheckpointCommit::NotCommitted,
|
||||||
|
receipt_absent: true,
|
||||||
|
pid: std::process::id(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn config_rejects_nonliteral_nonloopback_bad_stage_unknown_fields_and_invalid_worker() {
|
||||||
|
let valid = serde_json::to_value(config("127.0.0.1:1234".parse().unwrap())).unwrap();
|
||||||
|
for (field, value) in [
|
||||||
|
("endpoint", "localhost:1234"),
|
||||||
|
("endpoint", "192.0.2.1:1234"),
|
||||||
|
("endpoint", "0.0.0.0:1234"),
|
||||||
|
("endpoint", "127.0.0.1:0"),
|
||||||
|
("endpoint", "[::ffff:127.0.0.1]:1234"),
|
||||||
|
("worker_id", "not-a-worker-id"),
|
||||||
|
("stage", "after_terminal"),
|
||||||
|
("nonce", ""),
|
||||||
|
("nonce", "secret\nbody"),
|
||||||
|
("timeout", "unbounded"),
|
||||||
|
] {
|
||||||
|
let mut invalid = valid.clone();
|
||||||
|
invalid[field] = value.into();
|
||||||
|
let parsed = serde_json::from_value::<RestoreCheckpointConfig>(invalid);
|
||||||
|
assert!(
|
||||||
|
parsed.is_err() || parsed.unwrap().validate().is_err(),
|
||||||
|
"{field}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for endpoint in ["127.0.0.1:1234", "[::1]:1234"] {
|
||||||
|
config(endpoint.parse().unwrap()).validate().unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn config_file_bounds_and_errors_do_not_disclose_contents() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("config.json");
|
||||||
|
assert!(RestoreCheckpointConfig::from_path(&path).is_err());
|
||||||
|
std::fs::write(&path, vec![b'x'; MAX_RESTORE_CHECKPOINT_CONFIG_BYTES + 1]).unwrap();
|
||||||
|
assert!(RestoreCheckpointConfig::from_path(&path).is_err());
|
||||||
|
std::fs::write(
|
||||||
|
&path,
|
||||||
|
br#"{"nonce":"private-config-body","stage":"secret"}"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let error = RestoreCheckpointConfig::from_path(&path)
|
||||||
|
.unwrap_err()
|
||||||
|
.to_string();
|
||||||
|
assert!(!error.contains("private-config-body"));
|
||||||
|
assert!(!error.contains("secret"));
|
||||||
|
let mut valid = config("127.0.0.1:1234".parse().unwrap());
|
||||||
|
valid.nonce = "x".repeat(257);
|
||||||
|
assert!(valid.validate().is_err());
|
||||||
|
valid.nonce = "valid".into();
|
||||||
|
std::fs::write(&path, serde_json::to_vec(&valid).unwrap()).unwrap();
|
||||||
|
assert_eq!(RestoreCheckpointConfig::from_path(&path).unwrap(), valid);
|
||||||
|
assert!(RestoreCheckpointConfig::from_path(dir.path()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn explicit_release_eof_and_invalid_release_close_the_owned_stream() {
|
||||||
|
for release in [Some(b'R'), None, Some(b'X')] {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let config = config(listener.local_addr().unwrap());
|
||||||
|
let expected = ack(&config);
|
||||||
|
let sent = expected.clone();
|
||||||
|
let task = thread::spawn(move || exchange(&config, &sent));
|
||||||
|
let (mut stream, _) = listener.accept().unwrap();
|
||||||
|
stream
|
||||||
|
.set_read_timeout(Some(Duration::from_secs(2)))
|
||||||
|
.unwrap();
|
||||||
|
let mut line = String::new();
|
||||||
|
BufReader::new(&mut stream).read_line(&mut line).unwrap();
|
||||||
|
assert!(line.len() <= MAX_RESTORE_CHECKPOINT_ACK_BYTES);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::from_str::<RestoreCheckpointAck>(&line).unwrap(),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
if let Some(byte) = release {
|
||||||
|
stream.write_all(&[byte]).unwrap();
|
||||||
|
} else {
|
||||||
|
stream.shutdown(std::net::Shutdown::Write).unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(task.join().unwrap().is_ok(), release == Some(b'R'));
|
||||||
|
let mut byte = [0];
|
||||||
|
assert_eq!(stream.read(&mut byte).unwrap(), 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn absent_listener_resumes_without_waiting_or_retrying() {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let config = config(listener.local_addr().unwrap());
|
||||||
|
drop(listener);
|
||||||
|
let start = Instant::now();
|
||||||
|
wait_for_release(&config, &ack(&config));
|
||||||
|
assert!(start.elapsed() < Duration::from_secs(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn silent_peer_hits_fixed_deadline_and_stream_is_dropped() {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let config = config(listener.local_addr().unwrap());
|
||||||
|
let start = Instant::now();
|
||||||
|
let task = thread::spawn(move || exchange(&config, &ack(&config)));
|
||||||
|
let (mut stream, _) = listener.accept().unwrap();
|
||||||
|
stream
|
||||||
|
.set_read_timeout(Some(Duration::from_secs(12)))
|
||||||
|
.unwrap();
|
||||||
|
let mut reader = BufReader::new(&mut stream);
|
||||||
|
let mut line = String::new();
|
||||||
|
reader.read_line(&mut line).unwrap();
|
||||||
|
assert!(task.join().unwrap().is_err());
|
||||||
|
assert!(start.elapsed() >= RESTORE_CHECKPOINT_TIMEOUT - Duration::from_millis(100));
|
||||||
|
assert!(start.elapsed() < RESTORE_CHECKPOINT_TIMEOUT + Duration::from_secs(3));
|
||||||
|
assert_eq!(reader.read(&mut [0]).unwrap(), 0);
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
//! only the empty segment envelope is built with current APIs.
|
//! only the empty segment envelope is built with current APIs.
|
||||||
use super::tests::{request, write_json};
|
use super::tests::{request, write_json};
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
|
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions, SCHEMA_VERSION};
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use session_store::{LogEntry, SegmentId, SessionId, Store, WorkerSessionStore};
|
use session_store::{LogEntry, SegmentId, SessionId, Store, WorkerSessionStore};
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
@@ -226,7 +226,7 @@ fn completed_v4_archive(
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
||||||
for runtime_schema in [11, 12] {
|
for runtime_schema in [11, 12, SCHEMA_VERSION] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
||||||
@@ -329,7 +329,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
|||||||
(storage.checksum_sha256, storage.bytes, storage.file_count),
|
(storage.checksum_sha256, storage.bytes, storage.file_count),
|
||||||
(checksum, bytes, count)
|
(checksum, bytes, count)
|
||||||
);
|
);
|
||||||
if runtime_schema == 12 {
|
if runtime_schema == SCHEMA_VERSION {
|
||||||
for name in [
|
for name in [
|
||||||
PathBuf::from("runtime.json"),
|
PathBuf::from("runtime.json"),
|
||||||
PathBuf::from(format!("workers/{worker}/worker.json")),
|
PathBuf::from(format!("workers/{worker}/worker.json")),
|
||||||
@@ -346,8 +346,8 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
|||||||
);
|
);
|
||||||
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
|
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
|
||||||
assert_eq!(tree_bytes(&root), once, "retry rewrote migrated evidence");
|
assert_eq!(tree_bytes(&root), once, "retry rewrote migrated evidence");
|
||||||
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
|
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
|
||||||
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
|
assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists());
|
||||||
// session-storage, not the immutable receipt's old checksum, attests
|
// session-storage, not the immutable receipt's old checksum, attests
|
||||||
// current bytes. Tampering with those bytes must remain detectable.
|
// current bytes. Tampering with those bytes must remain detectable.
|
||||||
let path = archive_session
|
let path = archive_session
|
||||||
@@ -360,7 +360,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn corrupt_archive_does_not_block_startup_or_healthy_session_migration_and_repair_converges() {
|
fn corrupt_archive_does_not_block_startup_or_healthy_session_migration_and_repair_converges() {
|
||||||
for runtime_schema in [11, 12] {
|
for runtime_schema in [11, 12, SCHEMA_VERSION] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
||||||
@@ -440,8 +440,8 @@ fn unmigratable_archive_preserves_evidence_across_startup_and_reopen() {
|
|||||||
] {
|
] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (live_worker, live_session, live_segment) = runtime_fixture(&root, 12);
|
let (live_worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let (worker, _, segment) = stopped_worker(&root, 12);
|
let (worker, _, segment) = stopped_worker(&root, SCHEMA_VERSION);
|
||||||
let provider = FsWorkerRetentionProvider::new(&root);
|
let provider = FsWorkerRetentionProvider::new(&root);
|
||||||
let path = provider
|
let path = provider
|
||||||
.worker_dir(worker)
|
.worker_dir(worker)
|
||||||
@@ -542,8 +542,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() {
|
|||||||
] {
|
] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, _, _) = runtime_fixture(&root, 12);
|
let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
completed_v4_archive(&root, 12);
|
completed_v4_archive(&root, SCHEMA_VERSION);
|
||||||
let provider = FsWorkerRetentionProvider::new(&root);
|
let provider = FsWorkerRetentionProvider::new(&root);
|
||||||
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
|
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
|
||||||
let archive = provider.archive_dir("archive-a").unwrap();
|
let archive = provider.archive_dir("archive-a").unwrap();
|
||||||
@@ -575,8 +575,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() {
|
|||||||
fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
|
fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
runtime_fixture(&root, 12);
|
runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let (session, _, _, _) = completed_v4_archive(&root, 12);
|
let (session, _, _, _) = completed_v4_archive(&root, SCHEMA_VERSION);
|
||||||
let ledger = root.join("retention/session-sources");
|
let ledger = root.join("retention/session-sources");
|
||||||
fs::create_dir_all(&ledger).unwrap();
|
fs::create_dir_all(&ledger).unwrap();
|
||||||
fs::write(ledger.join(format!("{session}.json")), b"{broken\n").unwrap();
|
fs::write(ledger.join(format!("{session}.json")), b"{broken\n").unwrap();
|
||||||
@@ -591,7 +591,7 @@ fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
|
|||||||
fn runtime_v5_probe_and_staging_skip_erased_originals() {
|
fn runtime_v5_probe_and_staging_skip_erased_originals() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, live_session, live_segment) = runtime_fixture(&root, 12);
|
let (worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let erased = root
|
let erased = root
|
||||||
.join("orphaned-workers")
|
.join("orphaned-workers")
|
||||||
.join(WorkerId::now_v7().to_string());
|
.join(WorkerId::now_v7().to_string());
|
||||||
@@ -643,7 +643,7 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
|
|||||||
for corruption in ["broken-json", "future-schema"] {
|
for corruption in ["broken-json", "future-schema"] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, session, segment) = runtime_fixture(&root, 12);
|
let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let session_root = root
|
let session_root = root
|
||||||
.join("workers")
|
.join("workers")
|
||||||
.join(worker.to_string())
|
.join(worker.to_string())
|
||||||
@@ -674,17 +674,19 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
|
|||||||
assert_eq!(tree_bytes(&root), before);
|
assert_eq!(tree_bytes(&root), before);
|
||||||
assert!(FsRuntimeStore::migrate(&options).is_err());
|
assert!(FsRuntimeStore::migrate(&options).is_err());
|
||||||
assert_eq!(tree_bytes(&root), before);
|
assert_eq!(tree_bytes(&root), before);
|
||||||
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
|
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
|
||||||
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
|
assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
#[test]
|
#[test]
|
||||||
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
// The same TempDir owns the relative bind/connect reference. No cwd change
|
||||||
let root = temp.path().join("runtime");
|
// or alias is needed, and the real live-socket migration refusal stays tested.
|
||||||
let (worker, _, _) = runtime_fixture(&root, 12);
|
let temp = tempfile::tempdir_in(".").unwrap();
|
||||||
|
let root = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
||||||
|
let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let before = tree_bytes(&root);
|
let before = tree_bytes(&root);
|
||||||
let runs = root.join("workers").join(worker.to_string()).join("runs");
|
let runs = root.join("workers").join(worker.to_string()).join("runs");
|
||||||
fs::create_dir_all(runs.join("1")).unwrap();
|
fs::create_dir_all(runs.join("1")).unwrap();
|
||||||
@@ -695,7 +697,7 @@ fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
|||||||
drop(listener);
|
drop(listener);
|
||||||
fs::remove_dir_all(runs).unwrap();
|
fs::remove_dir_all(runs).unwrap();
|
||||||
assert_eq!(tree_bytes(&root), before);
|
assert_eq!(tree_bytes(&root), before);
|
||||||
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
|
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -703,7 +705,7 @@ fn current_runtime_upgrades_v5_role_metadata_through_staging_without_reserializi
|
|||||||
{
|
{
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, session, segment) = runtime_fixture(&root, 12);
|
let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let aggregate = root.join("workers").join(worker.to_string());
|
let aggregate = root.join("workers").join(worker.to_string());
|
||||||
let path = aggregate
|
let path = aggregate
|
||||||
.join("session/segments")
|
.join("session/segments")
|
||||||
|
|||||||
+2072
-496
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,112 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::restore_checkpoints::{
|
||||||
|
RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage,
|
||||||
|
wait_for_release,
|
||||||
|
};
|
||||||
|
|
||||||
|
impl Runtime {
|
||||||
|
/// Install an explicitly selected, one-shot process-test gate after normal startup.
|
||||||
|
/// It is not persisted or consulted by admission, routing or recovery authority.
|
||||||
|
pub fn install_test_restore_checkpoint(
|
||||||
|
&self,
|
||||||
|
config: RestoreCheckpointConfig,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
config.validate()?;
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
if !matches!(state.persistence, RuntimePersistence::Fs(_)) {
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"test restore checkpoint requires an fs Runtime store".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
state.restore_checkpoint = Some(config);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// The lifecycle caller holds the per-Worker operation lock. No Runtime-state
|
||||||
|
// mutex is retained while waiting; the real backend candidate remains owned by
|
||||||
|
// the same operation throughout this synchronous call.
|
||||||
|
pub(super) fn test_restore_checkpoint(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
stage: RestoreCheckpointStage,
|
||||||
|
) {
|
||||||
|
let config = {
|
||||||
|
let Ok(mut state) = self.lock() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let slot = &mut state.restore_checkpoint;
|
||||||
|
if !slot.as_ref().is_some_and(|config| {
|
||||||
|
config.worker_id == worker_ref.worker_id && config.stage == stage
|
||||||
|
}) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Consume even if canonical verification or the handshake fails.
|
||||||
|
slot.take().expect("matched checkpoint")
|
||||||
|
};
|
||||||
|
let Ok(ack) = self.restore_checkpoint_ack(&config, worker_ref, operation_id) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
wait_for_release(&config, &ack);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn restore_checkpoint_ack(
|
||||||
|
&self,
|
||||||
|
config: &RestoreCheckpointConfig,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
) -> Result<RestoreCheckpointAck, RuntimeError> {
|
||||||
|
let invalid = || {
|
||||||
|
RuntimeError::InvalidRequest("test restore checkpoint authority did not match".into())
|
||||||
|
};
|
||||||
|
let state = self.lock()?;
|
||||||
|
let RuntimePersistence::Fs(store) = &state.persistence else {
|
||||||
|
return Err(invalid());
|
||||||
|
};
|
||||||
|
// The existing helper validates the exact bounded canonical aggregate and
|
||||||
|
// syncs that same file plus its owning directories. Never manufacture a
|
||||||
|
// journal marker or infer commit from an in-memory projection.
|
||||||
|
let record = store.read_durable_worker_record(worker_ref)?;
|
||||||
|
let PersistedWorkerExecutionState::ReconciliationRequired(
|
||||||
|
PersistedWorkerLifecycleOperation::Restore(pending),
|
||||||
|
) = &record.execution_state
|
||||||
|
else {
|
||||||
|
return Err(invalid());
|
||||||
|
};
|
||||||
|
let expected = match config.stage {
|
||||||
|
RestoreCheckpointStage::PreparedNotCommitted => RestoreCommitFact::NotCommitted,
|
||||||
|
RestoreCheckpointStage::CommittedPreterminal
|
||||||
|
| RestoreCheckpointStage::ActivatedPreterminal => RestoreCommitFact::Committed,
|
||||||
|
};
|
||||||
|
if config.worker_id != worker_ref.worker_id
|
||||||
|
|| pending.operation_id != operation_id
|
||||||
|
|| pending.commit != expected
|
||||||
|
{
|
||||||
|
return Err(invalid());
|
||||||
|
}
|
||||||
|
let guard = &record.restore_guard;
|
||||||
|
let id = guard.active_request_id.as_ref().ok_or_else(invalid)?;
|
||||||
|
let owner = guard.owners.get(id).ok_or_else(invalid)?;
|
||||||
|
if owner.operation_id != operation_id
|
||||||
|
|| owner.request.request_id != *id
|
||||||
|
|| owner.receipt.is_some()
|
||||||
|
{
|
||||||
|
return Err(invalid());
|
||||||
|
}
|
||||||
|
let runtime_id = state.runtime_identity.clone().ok_or_else(invalid)?;
|
||||||
|
Ok(RestoreCheckpointAck {
|
||||||
|
nonce: config.nonce.clone(),
|
||||||
|
stage: config.stage,
|
||||||
|
runtime_id,
|
||||||
|
worker_id: record.worker_id,
|
||||||
|
operation_id,
|
||||||
|
commit: match expected {
|
||||||
|
RestoreCommitFact::NotCommitted => RestoreCheckpointCommit::NotCommitted,
|
||||||
|
RestoreCommitFact::Committed => RestoreCheckpointCommit::Committed,
|
||||||
|
RestoreCommitFact::LegacyUnknown => unreachable!(),
|
||||||
|
},
|
||||||
|
receipt_absent: true,
|
||||||
|
pid: std::process::id(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,304 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::restore_checkpoints::{
|
||||||
|
RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage,
|
||||||
|
};
|
||||||
|
use std::io::{BufRead, BufReader, Write};
|
||||||
|
use std::net::TcpListener;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_stages_ack_canonical_decision_and_eof_resumes_same_operation_once() {
|
||||||
|
// Component scope: durable Runtime with a substituted execution backend.
|
||||||
|
// Real prepared Controllers and process death are owned by the E2E suite.
|
||||||
|
for (stage, release) in [
|
||||||
|
(RestoreCheckpointStage::PreparedNotCommitted, "release"),
|
||||||
|
(RestoreCheckpointStage::CommittedPreterminal, "eof"),
|
||||||
|
(RestoreCheckpointStage::CommittedPreterminal, "timeout"),
|
||||||
|
(RestoreCheckpointStage::ActivatedPreterminal, "release"),
|
||||||
|
] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||||
|
FsRuntimeStoreOptions::new(dir.path().join("runtime"))
|
||||||
|
.with_runtime_id("checkpoint-runtime"),
|
||||||
|
backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker(task_request("checkpoint original operation"))
|
||||||
|
.unwrap();
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
runtime
|
||||||
|
.install_test_restore_checkpoint(RestoreCheckpointConfig {
|
||||||
|
endpoint: listener.local_addr().unwrap(),
|
||||||
|
worker_id: worker.worker_id,
|
||||||
|
stage,
|
||||||
|
nonce: "runtime-stage-test".into(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let cloned = runtime.clone();
|
||||||
|
let worker_ref = worker.worker_ref.clone();
|
||||||
|
let sent_request = request.clone();
|
||||||
|
let (sent, received) = std::sync::mpsc::channel();
|
||||||
|
let task = std::thread::spawn(move || {
|
||||||
|
sent.send(cloned.restore_worker_operation(&worker_ref, sent_request))
|
||||||
|
.unwrap();
|
||||||
|
});
|
||||||
|
listener.set_nonblocking(true).unwrap();
|
||||||
|
let stream = tokio::runtime::Builder::new_current_thread()
|
||||||
|
.enable_all()
|
||||||
|
.build()
|
||||||
|
.unwrap()
|
||||||
|
.block_on(async {
|
||||||
|
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
|
||||||
|
let (stream, _) = tokio::time::timeout(Duration::from_secs(5), listener.accept())
|
||||||
|
.await
|
||||||
|
.expect("checkpoint did not connect")
|
||||||
|
.unwrap();
|
||||||
|
stream.into_std().unwrap()
|
||||||
|
});
|
||||||
|
stream.set_nonblocking(false).unwrap();
|
||||||
|
stream
|
||||||
|
.set_read_timeout(Some(Duration::from_secs(3)))
|
||||||
|
.unwrap();
|
||||||
|
let mut reader = BufReader::new(stream);
|
||||||
|
let mut line = String::new();
|
||||||
|
reader.read_line(&mut line).unwrap();
|
||||||
|
let ack: RestoreCheckpointAck = serde_json::from_str(&line).unwrap();
|
||||||
|
assert_eq!(ack.stage, stage);
|
||||||
|
assert_eq!(ack.nonce, "runtime-stage-test");
|
||||||
|
assert_eq!(ack.runtime_id, "checkpoint-runtime");
|
||||||
|
assert_eq!(ack.worker_id, worker.worker_id);
|
||||||
|
assert_eq!(ack.pid, std::process::id());
|
||||||
|
assert!(ack.receipt_absent);
|
||||||
|
assert_eq!(
|
||||||
|
ack.commit,
|
||||||
|
match stage {
|
||||||
|
RestoreCheckpointStage::PreparedNotCommitted =>
|
||||||
|
RestoreCheckpointCommit::NotCommitted,
|
||||||
|
RestoreCheckpointStage::CommittedPreterminal
|
||||||
|
| RestoreCheckpointStage::ActivatedPreterminal => RestoreCheckpointCommit::Committed,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
backend.restore_operation_ids.lock().unwrap().as_slice(),
|
||||||
|
&[ack.operation_id]
|
||||||
|
);
|
||||||
|
if stage == RestoreCheckpointStage::ActivatedPreterminal {
|
||||||
|
assert_eq!(backend.activation_operation_ids.lock().unwrap().as_slice(), &[ack.operation_id]);
|
||||||
|
} else {
|
||||||
|
assert!(backend.activation_operation_ids.lock().unwrap().is_empty());
|
||||||
|
}
|
||||||
|
assert!(received.try_recv().is_err());
|
||||||
|
let canonical = runtime_store(&runtime)
|
||||||
|
.read_durable_worker_record(&worker.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
let id = canonical.restore_guard.active_request_id.as_ref().unwrap();
|
||||||
|
assert_eq!(id, &request.request_id);
|
||||||
|
assert!(canonical.restore_guard.owners[id].receipt.is_none());
|
||||||
|
// Release, EOF and timeout all resume the original lifecycle owner.
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
if release == "release" {
|
||||||
|
reader.get_mut().write_all(b"R").unwrap();
|
||||||
|
} else if release == "eof" {
|
||||||
|
reader
|
||||||
|
.get_mut()
|
||||||
|
.shutdown(std::net::Shutdown::Write)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let result = received
|
||||||
|
.recv_timeout(Duration::from_secs(12))
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(start.elapsed() < Duration::from_secs(11));
|
||||||
|
task.join().unwrap();
|
||||||
|
assert_eq!(result.state, WorkerRestoreState::Accepted);
|
||||||
|
assert_eq!(
|
||||||
|
backend.activation_operation_ids.lock().unwrap().as_slice(),
|
||||||
|
&[ack.operation_id]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request)
|
||||||
|
.unwrap(),
|
||||||
|
result
|
||||||
|
);
|
||||||
|
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
|
||||||
|
assert!(runtime.lock().unwrap().restore_checkpoint.is_none());
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
let next = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, next)
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert_eq!(*backend.restore_count.lock().unwrap(), 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_disabled_mismatched_or_unreachable_does_not_change_restore_result() {
|
||||||
|
for mode in ["disabled", "mismatch", "unreachable"] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||||
|
FsRuntimeStoreOptions::new(dir.path().join("runtime"))
|
||||||
|
.with_runtime_id("checkpoint-runtime"),
|
||||||
|
backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker(task_request("checkpoint absent"))
|
||||||
|
.unwrap();
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
if mode != "disabled" {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let endpoint = listener.local_addr().unwrap();
|
||||||
|
drop(listener);
|
||||||
|
runtime
|
||||||
|
.install_test_restore_checkpoint(RestoreCheckpointConfig {
|
||||||
|
endpoint,
|
||||||
|
worker_id: if mode == "mismatch" {
|
||||||
|
WorkerId::now_v7()
|
||||||
|
} else {
|
||||||
|
worker.worker_id
|
||||||
|
},
|
||||||
|
stage: RestoreCheckpointStage::PreparedNotCommitted,
|
||||||
|
nonce: "absent-test".into(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let result = runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(result.state, WorkerRestoreState::Accepted, "{mode}");
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request)
|
||||||
|
.unwrap(),
|
||||||
|
result
|
||||||
|
);
|
||||||
|
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
runtime.lock().unwrap().restore_checkpoint.is_some(),
|
||||||
|
mode == "mismatch"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_verification_requires_canonical_worker_operation_commit_and_absent_receipt() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let root = dir.path().join("runtime");
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||||
|
FsRuntimeStoreOptions::new(&root).with_runtime_id("checkpoint-runtime"),
|
||||||
|
backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker(task_request("checkpoint validation"))
|
||||||
|
.unwrap();
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
*backend.restore_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::Errored(
|
||||||
|
WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, "not yet connected"),
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request.clone())
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let operation = backend.restore_operation_ids.lock().unwrap()[0];
|
||||||
|
let mut config = RestoreCheckpointConfig {
|
||||||
|
endpoint: "127.0.0.1:1234".parse().unwrap(),
|
||||||
|
worker_id: worker.worker_id,
|
||||||
|
stage: RestoreCheckpointStage::PreparedNotCommitted,
|
||||||
|
nonce: "verification-test".into(),
|
||||||
|
};
|
||||||
|
let path = root
|
||||||
|
.join("workers")
|
||||||
|
.join(worker.worker_id.to_string())
|
||||||
|
.join("worker.json");
|
||||||
|
let before = std::fs::read(&path).unwrap();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(
|
||||||
|
&config,
|
||||||
|
&worker.worker_ref,
|
||||||
|
WorkerLifecycleOperationId::new()
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
config.worker_id = WorkerId::now_v7();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
config.worker_id = worker.worker_id;
|
||||||
|
config.stage = RestoreCheckpointStage::CommittedPreterminal;
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
config.stage = RestoreCheckpointStage::ActivatedPreterminal;
|
||||||
|
assert!(runtime.restore_checkpoint_ack(&config, &worker.worker_ref, operation).is_err());
|
||||||
|
config.stage = RestoreCheckpointStage::PreparedNotCommitted;
|
||||||
|
runtime_store(&runtime).fail_next_worker_reconcile_sync();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&path).unwrap(),
|
||||||
|
before,
|
||||||
|
"verification must not write authority"
|
||||||
|
);
|
||||||
|
// In-memory equality must not hide an unreadable canonical aggregate.
|
||||||
|
std::fs::write(&path, b"unreadable canonical aggregate").unwrap();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&path).unwrap(),
|
||||||
|
b"unreadable canonical aggregate"
|
||||||
|
);
|
||||||
|
std::fs::write(&path, before).unwrap();
|
||||||
|
backend.restore_result.lock().unwrap().take();
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request)
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err(),
|
||||||
|
"terminal operation cannot ack a preterminal checkpoint"
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,522 @@
|
|||||||
|
//! Runtime-owned continuation of an admitted restore, independent of its caller.
|
||||||
|
//!
|
||||||
|
//! Completion callbacks only signal a condition variable. They never acquire a
|
||||||
|
//! Runtime/lifecycle lock and are never joined by this driver while holding one.
|
||||||
|
//! Each step revalidates the canonical owner under the existing Worker lock;
|
||||||
|
//! waiting/backoff and driver teardown happen outside it. A repeated obstruction
|
||||||
|
//! parks this owner with its reason, rather than spinning or inventing rollback.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::execution::RestoreRecoveryDisposition;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::{Condvar, Weak};
|
||||||
|
use std::thread::JoinHandle;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
type Key = (WorkerRef, WorkerLifecycleOperationId);
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub(super) struct RestoreContinuations {
|
||||||
|
owners: Mutex<HashMap<Key, Arc<Continuation>>>,
|
||||||
|
#[cfg(test)]
|
||||||
|
paused: std::sync::atomic::AtomicBool,
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(super) terminal_receipt_gate:
|
||||||
|
Mutex<Option<crate::execution::RestoreTestGate<WorkerLifecycleOperationId>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct Schedule {
|
||||||
|
signaled: bool,
|
||||||
|
cancelled: bool,
|
||||||
|
finished: bool,
|
||||||
|
blocked_reason: Option<String>,
|
||||||
|
parked: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct Continuation {
|
||||||
|
schedule: Mutex<Schedule>,
|
||||||
|
wake: Condvar,
|
||||||
|
task: Mutex<Option<JoinHandle<()>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RestoreContinuations {
|
||||||
|
pub(super) fn signal(&self, worker_ref: &WorkerRef, operation: WorkerLifecycleOperationId) {
|
||||||
|
if let Some(owner) = self
|
||||||
|
.owners
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.get(&(worker_ref.clone(), operation))
|
||||||
|
{
|
||||||
|
let mut schedule = owner
|
||||||
|
.schedule
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
schedule.signaled = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for RestoreContinuations {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let owners = self
|
||||||
|
.owners
|
||||||
|
.get_mut()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
for owner in owners.values() {
|
||||||
|
let mut schedule = owner
|
||||||
|
.schedule
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
schedule.cancelled = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
}
|
||||||
|
for owner in owners.values() {
|
||||||
|
if let Some(task) = owner
|
||||||
|
.task
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.take()
|
||||||
|
{
|
||||||
|
if task.thread().id() != std::thread::current().id() {
|
||||||
|
let _ = task.join();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) struct ContinuationPause(Arc<RestoreContinuations>);
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl Drop for ContinuationPause {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0
|
||||||
|
.paused
|
||||||
|
.store(false, std::sync::atomic::Ordering::Release);
|
||||||
|
for owner in self.0.owners.lock().unwrap().values() {
|
||||||
|
owner.schedule.lock().unwrap().signaled = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enum Step {
|
||||||
|
Finished,
|
||||||
|
Waiting,
|
||||||
|
Contended,
|
||||||
|
Retry(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Runtime {
|
||||||
|
pub(super) fn restore_repair_authorizer(
|
||||||
|
&self,
|
||||||
|
worker_ref: WorkerRef,
|
||||||
|
) -> Arc<dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync> {
|
||||||
|
let inner = Arc::downgrade(&self.inner);
|
||||||
|
Arc::new(move |operation| {
|
||||||
|
let inner = inner.upgrade().ok_or(RuntimeError::RuntimeStopped)?;
|
||||||
|
// The factory caller owns the lifecycle lock. Never reacquire it
|
||||||
|
// here: it may be awaiting this exact factory task.
|
||||||
|
let mut state = inner.lock().map_err(|_| RuntimeError::StatePoisoned)?;
|
||||||
|
state.ensure_running()?;
|
||||||
|
state.refresh_uncertain_worker_write(&worker_ref)?;
|
||||||
|
let worker = state.worker(&worker_ref)?;
|
||||||
|
state.ensure_subject_not_reserved(&worker_ref, false)?;
|
||||||
|
if worker.pending_stop.is_some()
|
||||||
|
|| worker.pending_restore.is_none_or(|pending| {
|
||||||
|
pending.operation_id != operation
|
||||||
|
|| pending.commit == RestoreCommitFact::LegacyUnknown
|
||||||
|
})
|
||||||
|
{
|
||||||
|
return Err(RuntimeError::RestoreObservationConflict {
|
||||||
|
worker_id: worker_ref.worker_id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if let Some(owner) = worker
|
||||||
|
.restore_guard
|
||||||
|
.active_request_id
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|id| worker.restore_guard.owners.get(id))
|
||||||
|
{
|
||||||
|
owner.validate_workdir_preparation_authority(worker)?;
|
||||||
|
}
|
||||||
|
if worker
|
||||||
|
.pending_restore
|
||||||
|
.is_some_and(|pending| pending.source_repair_possible)
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let mut candidate = worker.clone();
|
||||||
|
candidate
|
||||||
|
.pending_restore
|
||||||
|
.as_mut()
|
||||||
|
.expect("validated restore owner")
|
||||||
|
.source_repair_possible = true;
|
||||||
|
// Grant before touching another canonical store. Unknown durability
|
||||||
|
// denies repair until authoritative reconciliation; no old-memory write.
|
||||||
|
state.persist_restore_worker_record(&candidate)?;
|
||||||
|
state.workers.insert(worker_ref.worker_id, candidate);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn test_wait_restore_continuation_parked(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
timeout: Duration,
|
||||||
|
) -> String {
|
||||||
|
let owner = self
|
||||||
|
.restore_continuations
|
||||||
|
.owners
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.get(&(worker_ref.clone(), operation))
|
||||||
|
.cloned()
|
||||||
|
.unwrap();
|
||||||
|
let (schedule, waited) = owner
|
||||||
|
.wake
|
||||||
|
.wait_timeout_while(owner.schedule.lock().unwrap(), timeout, |s| {
|
||||||
|
!s.parked && !s.finished
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
!waited.timed_out() && schedule.parked,
|
||||||
|
"continuation did not park: {:?}",
|
||||||
|
schedule.blocked_reason
|
||||||
|
);
|
||||||
|
schedule.blocked_reason.clone().expect("parked reason")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Gate autonomous steps when a component test observes a single fault
|
||||||
|
/// boundary. Dedicated continuation tests run without this gate.
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn test_pause_restore_continuation(&self) -> ContinuationPause {
|
||||||
|
assert!(
|
||||||
|
!self
|
||||||
|
.restore_continuations
|
||||||
|
.paused
|
||||||
|
.swap(true, std::sync::atomic::Ordering::AcqRel)
|
||||||
|
);
|
||||||
|
ContinuationPause(self.restore_continuations.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn test_wait_restore_continuation(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
timeout: Duration,
|
||||||
|
) {
|
||||||
|
let owner = self
|
||||||
|
.restore_continuations
|
||||||
|
.owners
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.get(&(worker_ref.clone(), operation))
|
||||||
|
.cloned()
|
||||||
|
.expect("continuation owner");
|
||||||
|
let schedule = owner.schedule.lock().unwrap();
|
||||||
|
let (schedule, waited) = owner
|
||||||
|
.wake
|
||||||
|
.wait_timeout_while(schedule, timeout, |s| !s.finished && !s.cancelled)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
!waited.timed_out() && schedule.finished,
|
||||||
|
"restore continuation did not settle: {:?}",
|
||||||
|
schedule.blocked_reason
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn ensure_restore_continuation(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
let operation = {
|
||||||
|
let state = self.lock()?;
|
||||||
|
let worker = state.worker(worker_ref)?;
|
||||||
|
worker
|
||||||
|
.pending_restore
|
||||||
|
.map(|pending| pending.operation_id)
|
||||||
|
.or_else(|| {
|
||||||
|
worker
|
||||||
|
.restore_guard
|
||||||
|
.active_request_id
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|id| worker.restore_guard.owners.get(id))
|
||||||
|
.filter(|owner| owner.receipt.is_none())
|
||||||
|
.map(|owner| owner.operation_id)
|
||||||
|
})
|
||||||
|
};
|
||||||
|
if let Some(operation) = operation {
|
||||||
|
self.start_restore_continuation(worker_ref.clone(), operation)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn start_restore_continuation(
|
||||||
|
&self,
|
||||||
|
worker_ref: WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
{
|
||||||
|
let state = self.lock()?;
|
||||||
|
let Some(backend) = &state.execution_backend else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
if !backend.supports_restore_continuation()
|
||||||
|
|| backend
|
||||||
|
.restore_recovery_disposition(operation, &worker_ref)
|
||||||
|
.is_none()
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let key = (worker_ref.clone(), operation);
|
||||||
|
let mut owners = self
|
||||||
|
.restore_continuations
|
||||||
|
.owners
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| RuntimeError::StatePoisoned)?;
|
||||||
|
if owners.contains_key(&key) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let owner = Arc::new(Continuation::default());
|
||||||
|
let retained = owner.clone();
|
||||||
|
let state = Arc::downgrade(&self.inner);
|
||||||
|
let operations = Arc::downgrade(&self.worker_operations);
|
||||||
|
let continuations = Arc::downgrade(&self.restore_continuations);
|
||||||
|
let task = std::thread::Builder::new()
|
||||||
|
.name("restore-continuation".into())
|
||||||
|
.spawn(move || {
|
||||||
|
run(
|
||||||
|
state,
|
||||||
|
operations,
|
||||||
|
continuations,
|
||||||
|
retained,
|
||||||
|
worker_ref,
|
||||||
|
operation,
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.map_err(|error| {
|
||||||
|
RuntimeError::InvalidRequest(format!(
|
||||||
|
"Restore continuation could not start: {error}"
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
*owner.task.lock().map_err(|_| RuntimeError::StatePoisoned)? = Some(task);
|
||||||
|
owners.insert(key, owner);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn continue_restore_operation(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
) -> Result<Step, RuntimeError> {
|
||||||
|
#[cfg(test)]
|
||||||
|
if self
|
||||||
|
.restore_continuations
|
||||||
|
.paused
|
||||||
|
.load(std::sync::atomic::Ordering::Acquire)
|
||||||
|
{
|
||||||
|
return Ok(Step::Contended);
|
||||||
|
}
|
||||||
|
let lease = self.worker_operation_lock(worker_ref.worker_id)?;
|
||||||
|
// Another caller may be waiting for a backend task. Do not queue a join
|
||||||
|
// of that caller/driver while retaining the lock it needs to complete.
|
||||||
|
let _guard = match lease.try_lock() {
|
||||||
|
Ok(guard) => guard,
|
||||||
|
Err(std::sync::TryLockError::WouldBlock) => return Ok(Step::Contended),
|
||||||
|
Err(_) => return Err(RuntimeError::StatePoisoned),
|
||||||
|
};
|
||||||
|
let stop_is_owned = {
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
state.ensure_running()?;
|
||||||
|
state.refresh_uncertain_worker_write(worker_ref)?;
|
||||||
|
let worker = state.worker(worker_ref)?;
|
||||||
|
worker.pending_stop.is_some()
|
||||||
|
&& worker
|
||||||
|
.pending_restore
|
||||||
|
.is_some_and(|pending| pending.operation_id == operation)
|
||||||
|
};
|
||||||
|
if stop_is_owned {
|
||||||
|
// Stop supersedes publication, not ownership of late resources.
|
||||||
|
// Retain this same continuation until its cleanup is confirmed.
|
||||||
|
self.reconcile_pending_stop_under_lock(worker_ref)?;
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
let (backend, pending) = {
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
state.ensure_running()?;
|
||||||
|
state.refresh_uncertain_worker_write(worker_ref)?;
|
||||||
|
let worker = state.worker(worker_ref)?;
|
||||||
|
if worker.pending_stop.is_some() {
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
let owner = worker
|
||||||
|
.restore_guard
|
||||||
|
.active_request_id
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|id| worker.restore_guard.owners.get(id));
|
||||||
|
if owner.is_some_and(|owner| owner.operation_id != operation || owner.receipt.is_some())
|
||||||
|
{
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
let Some(pending) = worker.pending_restore else {
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
};
|
||||||
|
if pending.operation_id != operation {
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
if pending.commit == RestoreCommitFact::LegacyUnknown {
|
||||||
|
return Ok(Step::Retry(
|
||||||
|
"Saved Restore has no authoritative commit evidence".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
state.ensure_subject_not_reserved(worker_ref, false)?;
|
||||||
|
if let Some(owner) = owner {
|
||||||
|
owner.validate_workdir_preparation_authority(worker)?;
|
||||||
|
}
|
||||||
|
let Some(backend) = state.execution_backend.clone() else {
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
};
|
||||||
|
(backend, pending)
|
||||||
|
};
|
||||||
|
match backend.restore_recovery_disposition(operation, worker_ref) {
|
||||||
|
None => return Ok(Step::Finished),
|
||||||
|
Some(RestoreRecoveryDisposition::InFlight) => return Ok(Step::Waiting),
|
||||||
|
Some(RestoreRecoveryDisposition::Discard)
|
||||||
|
if pending.commit == RestoreCommitFact::NotCommitted =>
|
||||||
|
{
|
||||||
|
match backend.discard_uncommitted_restore(operation, worker_ref) {
|
||||||
|
WorkerExecutionSpawnResult::RolledBack(result) => {
|
||||||
|
let receipt = RuntimeWorkerRestoreResult::failed(
|
||||||
|
WorkerRestoreState::RolledBack,
|
||||||
|
"worker_restore_rolled_back",
|
||||||
|
result.message_or_default(),
|
||||||
|
);
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
self.lock()?
|
||||||
|
.record_restore_failure(worker_ref, &result, &receipt)?;
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
self.settle_restore_rollback(worker_ref, &receipt)?;
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
result => {
|
||||||
|
return Ok(Step::Retry(format!(
|
||||||
|
"Restore cleanup remains unresolved: {result:?}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(_) => {}
|
||||||
|
}
|
||||||
|
let result = self.restore_admitted_worker_under_lock(worker_ref, pending.mode)?;
|
||||||
|
if backend.restore_recovery_disposition(operation, worker_ref)
|
||||||
|
== Some(RestoreRecoveryDisposition::InFlight)
|
||||||
|
{
|
||||||
|
return Ok(Step::Waiting);
|
||||||
|
}
|
||||||
|
if result.state == WorkerRestoreState::ReconciliationRequired {
|
||||||
|
Ok(Step::Retry(result.message.unwrap_or_else(|| {
|
||||||
|
"Restore reconciliation remains pending".into()
|
||||||
|
})))
|
||||||
|
} else {
|
||||||
|
Ok(Step::Finished)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run(
|
||||||
|
inner: Weak<Mutex<RuntimeState>>,
|
||||||
|
operations: Weak<Mutex<BTreeMap<WorkerId, Arc<Mutex<()>>>>>,
|
||||||
|
continuations: Weak<RestoreContinuations>,
|
||||||
|
owner: Arc<Continuation>,
|
||||||
|
worker_ref: WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
) {
|
||||||
|
let mut repeated = 0u32;
|
||||||
|
let mut last_reason = None;
|
||||||
|
loop {
|
||||||
|
let (Some(inner), Some(worker_operations), Some(restore_continuations)) = (
|
||||||
|
inner.upgrade(),
|
||||||
|
operations.upgrade(),
|
||||||
|
continuations.upgrade(),
|
||||||
|
) else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
owner
|
||||||
|
.schedule
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.parked = false;
|
||||||
|
let runtime = Runtime {
|
||||||
|
inner,
|
||||||
|
worker_operations,
|
||||||
|
restore_continuations,
|
||||||
|
};
|
||||||
|
let step = runtime.continue_restore_operation(&worker_ref, operation);
|
||||||
|
// No lifecycle lock or strong Runtime/registry reference survives waiting.
|
||||||
|
drop(runtime);
|
||||||
|
let mut schedule = owner
|
||||||
|
.schedule
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
if schedule.cancelled {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let contended = matches!(&step, Ok(Step::Contended));
|
||||||
|
let reason = match step {
|
||||||
|
Ok(Step::Finished) => {
|
||||||
|
schedule.finished = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Ok(Step::Waiting) | Ok(Step::Contended) => None,
|
||||||
|
Ok(Step::Retry(reason)) => Some(reason),
|
||||||
|
Err(error) => Some(sanitize_worker_create_failure_message(&error.to_string())),
|
||||||
|
};
|
||||||
|
if let Some(reason) = &reason {
|
||||||
|
repeated = repeated.saturating_add(1);
|
||||||
|
last_reason = Some(reason.clone());
|
||||||
|
schedule.blocked_reason = Some(reason.clone());
|
||||||
|
}
|
||||||
|
// Completed failed attempts can have signaled while this step joined
|
||||||
|
// them. Do not let our own unchanged failure reset the retry budget.
|
||||||
|
if reason.is_some() {
|
||||||
|
schedule.signaled = false;
|
||||||
|
}
|
||||||
|
if (!contended && reason.is_none()) || repeated >= 4 {
|
||||||
|
if repeated >= 4 {
|
||||||
|
schedule.parked = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
tracing::warn!(worker_id = %worker_ref.worker_id, operation_id = %operation, reason = ?last_reason,
|
||||||
|
"Restore continuation parked after bounded attempts; operation remains unresolved");
|
||||||
|
}
|
||||||
|
schedule = owner
|
||||||
|
.wake
|
||||||
|
.wait_while(schedule, |s| !s.signaled && !s.cancelled)
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
} else {
|
||||||
|
let delay = Duration::from_millis(100 * (1u64 << repeated.min(4)));
|
||||||
|
schedule = owner
|
||||||
|
.wake
|
||||||
|
.wait_timeout_while(schedule, delay, |s| !s.signaled && !s.cancelled)
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.0;
|
||||||
|
}
|
||||||
|
if schedule.cancelled {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if schedule.signaled {
|
||||||
|
schedule.signaled = false;
|
||||||
|
// A completion wakes this same owner, not a new request. Its next
|
||||||
|
// observed result, not the signal itself, determines progress.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -55,6 +55,10 @@ impl WorkdirSession for RetainedWorkdirSession {
|
|||||||
self.inner.capabilities()
|
self.inner.capabilities()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn environment(&self) -> workdir::WorkdirSessionEnvironment {
|
||||||
|
self.inner.environment()
|
||||||
|
}
|
||||||
|
|
||||||
async fn authorize_scope_path(
|
async fn authorize_scope_path(
|
||||||
&self,
|
&self,
|
||||||
request: WorkdirScopeAuthorizationRequest,
|
request: WorkdirScopeAuthorizationRequest,
|
||||||
|
|||||||
@@ -0,0 +1,872 @@
|
|||||||
|
//! Offline support for the approved, attachment-free stopped Worker fixture.
|
||||||
|
//!
|
||||||
|
//! Enabled only by `test-support`. This is not an import API: extraction accepts a
|
||||||
|
//! current-schema, settled, single-Segment aggregate; installation accepts only a
|
||||||
|
//! fresh Runtime directory containing its newly initialized `auth.toml`. No
|
||||||
|
//! Runtime or execution backend is constructed. Original Session bytes are never
|
||||||
|
//! rewritten. The caller must separately review/attest the artificial conversation;
|
||||||
|
//! structural checks and secret-marker checks cannot prove arbitrary prose safe.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::catalog::ProfileSourceArchiveSource;
|
||||||
|
use crate::workspace_issuer::{
|
||||||
|
WorkspaceIssuerTrustRecord, WorkspaceIssuerTrustState, validate_workspace_issuer_trust_records,
|
||||||
|
};
|
||||||
|
use session_store::{
|
||||||
|
SegmentId, SessionId, WorkerActiveSegmentRef, WorkerMetadata, WorkerSessionStore,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "test_support_tests.rs"]
|
||||||
|
mod tests;
|
||||||
|
|
||||||
|
const TEMPLATE_VERSION: u32 = 1;
|
||||||
|
const MAX_SESSION_BYTES: usize = 8 * 1024 * 1024;
|
||||||
|
|
||||||
|
/// Portable fixture, intentionally without Debug (Session material is private).
|
||||||
|
/// Fields are private and every deserialized template is validated on install.
|
||||||
|
/// Serialize this value to retain the fixture; do not serialize its source aggregate.
|
||||||
|
#[derive(Clone, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct StoppedWorkerTemplate {
|
||||||
|
template_version: u32,
|
||||||
|
runtime_schema_version: u32,
|
||||||
|
runtime_id: String,
|
||||||
|
workspace_id: String,
|
||||||
|
worker_id: WorkerId,
|
||||||
|
created_at_ms: Option<u64>,
|
||||||
|
session_id: SessionId,
|
||||||
|
segment_id: SegmentId,
|
||||||
|
session_manifest: Vec<u8>,
|
||||||
|
segment_jsonl: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct SessionManifest {
|
||||||
|
schema_version: u32,
|
||||||
|
session_id: SessionId,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read-only projection: never retain source key material in a template.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct AuthIdentity {
|
||||||
|
identity_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct SourceAuth {
|
||||||
|
identity: AuthIdentity,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct InitializedAuth {
|
||||||
|
identity: crate::auth::RuntimeIdentityMaterial,
|
||||||
|
workspace_issuers: Vec<WorkspaceIssuerTrustRecord>,
|
||||||
|
#[serde(default)]
|
||||||
|
archived_legacy_issuers: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct GuardState {
|
||||||
|
active_request_id: Option<String>,
|
||||||
|
owners: BTreeMap<String, GuardOwner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct GuardOwner {
|
||||||
|
receipt: Option<serde::de::IgnoredAny>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reject(message: &'static str) -> RuntimeError {
|
||||||
|
RuntimeError::InvalidRequest(format!("stopped Worker fixture: {message}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn io<T>(result: std::io::Result<T>) -> Result<T, RuntimeError> {
|
||||||
|
result.map_err(|_| reject("filesystem operation failed"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> Result<T, RuntimeError> {
|
||||||
|
serde_json::from_slice(bytes).map_err(|_| reject("invalid typed document"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn no_symlink_components(path: &Path) -> Result<(), RuntimeError> {
|
||||||
|
let mut current = PathBuf::new();
|
||||||
|
for part in path.components() {
|
||||||
|
if matches!(part, std::path::Component::ParentDir) {
|
||||||
|
return Err(reject("parent traversal is unsupported"));
|
||||||
|
}
|
||||||
|
current.push(part);
|
||||||
|
let metadata = io(fs::symlink_metadata(¤t))?;
|
||||||
|
if metadata.file_type().is_symlink() {
|
||||||
|
return Err(reject("symlinks are unsupported"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_tree(path: &Path, budget: &mut usize) -> Result<(), RuntimeError> {
|
||||||
|
if *budget == 0 {
|
||||||
|
return Err(reject("file count exceeds fixture limit"));
|
||||||
|
}
|
||||||
|
*budget -= 1;
|
||||||
|
let metadata = io(fs::symlink_metadata(path))?;
|
||||||
|
if metadata.is_dir() {
|
||||||
|
for entry in io(fs::read_dir(path))? {
|
||||||
|
check_tree(&io(entry)?.path(), budget)?;
|
||||||
|
}
|
||||||
|
} else if !metadata.is_file() {
|
||||||
|
return Err(reject("symlinks and special files are unsupported"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn regular_file(path: &Path) -> Result<(), RuntimeError> {
|
||||||
|
no_symlink_components(path)?;
|
||||||
|
if !io(fs::symlink_metadata(path))?.is_file() {
|
||||||
|
return Err(reject("expected a regular file"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bytes(path: &Path) -> Result<Vec<u8>, RuntimeError> {
|
||||||
|
regular_file(path)?;
|
||||||
|
let mut options = OpenOptions::new();
|
||||||
|
options.read(true);
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
{
|
||||||
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
|
options.custom_flags(libc::O_NOATIME | libc::O_NOFOLLOW);
|
||||||
|
}
|
||||||
|
let file = io(options.open(path))?;
|
||||||
|
let mut result = Vec::new();
|
||||||
|
io(file
|
||||||
|
.take(MAX_SESSION_BYTES as u64 + 1)
|
||||||
|
.read_to_end(&mut result))?;
|
||||||
|
if result.len() > MAX_SESSION_BYTES {
|
||||||
|
return Err(reject("file exceeds fixture limit"));
|
||||||
|
}
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn text(path: &Path) -> Result<String, RuntimeError> {
|
||||||
|
String::from_utf8(bytes(path)?).map_err(|_| reject("invalid UTF-8 document"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn portable_id(id: &str) -> bool {
|
||||||
|
!id.is_empty()
|
||||||
|
&& id.len() <= 256
|
||||||
|
&& id
|
||||||
|
.bytes()
|
||||||
|
.all(|c| c.is_ascii_alphanumeric() || b"-_".contains(&c))
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StoppedWorkerTemplate {
|
||||||
|
pub fn runtime_id(&self) -> &str {
|
||||||
|
&self.runtime_id
|
||||||
|
}
|
||||||
|
pub fn workspace_id(&self) -> &str {
|
||||||
|
&self.workspace_id
|
||||||
|
}
|
||||||
|
pub fn worker_id(&self) -> WorkerId {
|
||||||
|
self.worker_id
|
||||||
|
}
|
||||||
|
pub fn worker_name(&self) -> String {
|
||||||
|
runtime_worker_name(self.worker_id)
|
||||||
|
}
|
||||||
|
pub fn session_id(&self) -> SessionId {
|
||||||
|
self.session_id
|
||||||
|
}
|
||||||
|
pub fn segment_id(&self) -> SegmentId {
|
||||||
|
self.segment_id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract without migration, rewriting metadata, or copying credentials,
|
||||||
|
/// resolved manifests, execution-operation receipts, or run directories.
|
||||||
|
/// The source Runtime must be offline; its existing owner lock is held for
|
||||||
|
/// the entire read. The path must be `<runtime>/workers/<WorkerId>`.
|
||||||
|
pub fn extract(worker_dir: &Path) -> Result<Self, RuntimeError> {
|
||||||
|
no_symlink_components(worker_dir)?;
|
||||||
|
let workers = worker_dir
|
||||||
|
.parent()
|
||||||
|
.ok_or_else(|| reject("invalid aggregate path"))?;
|
||||||
|
if workers.file_name().and_then(|s| s.to_str()) != Some(WORKERS_DIR) {
|
||||||
|
return Err(reject("invalid aggregate path"));
|
||||||
|
}
|
||||||
|
let root = workers
|
||||||
|
.parent()
|
||||||
|
.ok_or_else(|| reject("invalid Runtime path"))?;
|
||||||
|
// Require the existing lock, rather than creating a file in the source.
|
||||||
|
let lock_path = root.with_file_name(format!(
|
||||||
|
".{}.runtime-owner.lock",
|
||||||
|
root.file_name()
|
||||||
|
.and_then(|s| s.to_str())
|
||||||
|
.ok_or_else(|| reject("invalid Runtime path"))?
|
||||||
|
));
|
||||||
|
regular_file(&lock_path)?;
|
||||||
|
let lock = io(OpenOptions::new().read(true).open(lock_path))?;
|
||||||
|
if !io(FileExt::try_lock_shared(&lock))? {
|
||||||
|
return Err(reject("source Runtime is not offline"));
|
||||||
|
}
|
||||||
|
let _lock = RuntimeStoreOwnerLock { file: lock };
|
||||||
|
check_tree(worker_dir, &mut 256)?;
|
||||||
|
let snapshot: RuntimeSnapshot = decode(&bytes(&root.join(RUNTIME_FILE))?)?;
|
||||||
|
snapshot
|
||||||
|
.validate(&root.join(RUNTIME_FILE))
|
||||||
|
.map_err(|_| reject("unsupported Runtime schema"))?;
|
||||||
|
if !snapshot.subject_create_recovery_fences.is_empty()
|
||||||
|
|| !snapshot.subject_releases.is_empty()
|
||||||
|
{
|
||||||
|
return Err(reject("source has lifecycle operations"));
|
||||||
|
}
|
||||||
|
let record: WorkerAggregateRecord = decode(&bytes(&worker_dir.join(WORKER_FILE))?)?;
|
||||||
|
let record = record
|
||||||
|
.validate(&worker_dir.join(WORKER_FILE))
|
||||||
|
.map_err(|_| reject("invalid Worker schema or identity"))?;
|
||||||
|
let guard: GuardState = decode(
|
||||||
|
&serde_json::to_vec(&record.restore_guard)
|
||||||
|
.map_err(|_| reject("invalid restore guard"))?,
|
||||||
|
)?;
|
||||||
|
if record.status != WorkerStatus::Stopped
|
||||||
|
|| guard.active_request_id.is_some()
|
||||||
|
|| guard.owners.values().any(|owner| owner.receipt.is_none())
|
||||||
|
{
|
||||||
|
return Err(reject("Worker is not settled and stopped"));
|
||||||
|
}
|
||||||
|
let PersistedWorkerExecutionState::Available(execution) = &record.execution_state else {
|
||||||
|
return Err(reject("pending or unavailable execution metadata"));
|
||||||
|
};
|
||||||
|
// A stopped aggregate may retain the empty historical binding marker.
|
||||||
|
// It is deliberately not copied to the installed Worker.
|
||||||
|
if execution.restore_intent != WorkerRestoreIntent::Explicit {
|
||||||
|
return Err(reject("Worker requires explicit restore"));
|
||||||
|
}
|
||||||
|
if !record.workdir_attachments.is_empty() || !record.logical_workdir_attachments.is_empty()
|
||||||
|
{
|
||||||
|
return Err(reject("attachments are unsupported"));
|
||||||
|
}
|
||||||
|
check_request(&execution.request, false)?;
|
||||||
|
let workspace_id = record
|
||||||
|
.workspace_id
|
||||||
|
.ok_or_else(|| reject("missing Workspace identity"))?;
|
||||||
|
let metadata: WorkerMetadata = decode(&bytes(&worker_dir.join(WORKER_METADATA_FILE))?)?;
|
||||||
|
crate::worker_backend::validate_fixture_restore_metadata(&metadata, &execution.request)
|
||||||
|
.map_err(|_| reject("invalid source restore Manifest or settings"))?;
|
||||||
|
if metadata.worker_name != runtime_worker_name(record.worker_id)
|
||||||
|
|| metadata.workspace_id.as_deref() != Some(&workspace_id)
|
||||||
|
|| !metadata.spawned_children.is_empty()
|
||||||
|
|| !metadata.reclaimed_children.is_empty()
|
||||||
|
|| !metadata.peers.is_empty()
|
||||||
|
|| metadata.subjektiv_session_attribution.is_some()
|
||||||
|
{
|
||||||
|
return Err(reject("unsupported or inconsistent Worker metadata"));
|
||||||
|
}
|
||||||
|
let active = metadata
|
||||||
|
.active
|
||||||
|
.ok_or_else(|| reject("missing active Session"))?;
|
||||||
|
let segment_id = active
|
||||||
|
.segment_id
|
||||||
|
.ok_or_else(|| reject("unmaterialized Session"))?;
|
||||||
|
let source_auth: SourceAuth = toml::from_str(&text(&root.join("auth.toml"))?)
|
||||||
|
.map_err(|_| reject("missing Runtime identity"))?;
|
||||||
|
let session = worker_dir.join("session");
|
||||||
|
let fixture = Self {
|
||||||
|
template_version: TEMPLATE_VERSION,
|
||||||
|
runtime_schema_version: SCHEMA_VERSION,
|
||||||
|
runtime_id: source_auth.identity.identity_id,
|
||||||
|
workspace_id,
|
||||||
|
worker_id: record.worker_id,
|
||||||
|
created_at_ms: record.created_at_ms,
|
||||||
|
session_id: active.session_id,
|
||||||
|
segment_id,
|
||||||
|
session_manifest: bytes(&session.join("session.json"))?,
|
||||||
|
segment_jsonl: bytes(&session.join("segments").join(format!("{segment_id}.jsonl")))?,
|
||||||
|
};
|
||||||
|
fixture.validate_shape()?;
|
||||||
|
fixture.validate_session(&session)?;
|
||||||
|
Ok(fixture)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Install after normal identity/trust initialization, before Runtime start.
|
||||||
|
/// `request` and `manifest` supply NEW backend/config/profile references and
|
||||||
|
/// freshly resolved policy, not a launch. The saved source Manifest is never
|
||||||
|
/// reused. `manifest.worker.name` must equal `worker_name()`.
|
||||||
|
/// Only `auth.toml` may already exist. The owner lock fences live Runtimes;
|
||||||
|
/// staged files are synced and published without overwriting any Worker.
|
||||||
|
/// An interrupted publication fails closed as an incomplete Runtime layout.
|
||||||
|
pub fn install(
|
||||||
|
&self,
|
||||||
|
options: &FsRuntimeStoreOptions,
|
||||||
|
request: CreateWorkerRequest,
|
||||||
|
manifest: &manifest::WorkerManifest,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
self.validate_shape()?;
|
||||||
|
check_request(&request, true)?;
|
||||||
|
if manifest.worker.name != self.worker_name() {
|
||||||
|
return Err(reject("resolved Manifest Worker name mismatch"));
|
||||||
|
}
|
||||||
|
let mut metadata = WorkerMetadata::new(
|
||||||
|
self.worker_name(),
|
||||||
|
Some(WorkerActiveSegmentRef {
|
||||||
|
session_id: self.session_id,
|
||||||
|
segment_id: Some(self.segment_id),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.with_workspace_id(&self.workspace_id);
|
||||||
|
metadata.resolved_manifest_snapshot = Some(
|
||||||
|
manifest::write_persisted_worker_manifest_snapshot(manifest)
|
||||||
|
.map_err(|_| reject("invalid resolved Manifest"))?,
|
||||||
|
);
|
||||||
|
crate::worker_backend::validate_fixture_restore_metadata(&metadata, &request)
|
||||||
|
.map_err(|_| reject("resolved Manifest does not match fresh request settings"))?;
|
||||||
|
if options.runtime_id != self.runtime_id
|
||||||
|
|| request.worker_id != self.worker_id
|
||||||
|
|| request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.map(|api| api.workspace_id.as_str())
|
||||||
|
!= Some(&self.workspace_id)
|
||||||
|
{
|
||||||
|
return Err(reject("installation identity mismatch"));
|
||||||
|
}
|
||||||
|
no_symlink_components(&options.root)?;
|
||||||
|
check_tree(&options.root, &mut 256)?;
|
||||||
|
// Validate the lock location before the ordinary owner-lock opener.
|
||||||
|
let lock_path = options.root.with_file_name(format!(
|
||||||
|
".{}.runtime-owner.lock",
|
||||||
|
options
|
||||||
|
.root
|
||||||
|
.file_name()
|
||||||
|
.and_then(|s| s.to_str())
|
||||||
|
.ok_or_else(|| reject("invalid Runtime path"))?
|
||||||
|
));
|
||||||
|
if fs::symlink_metadata(&lock_path).is_ok() {
|
||||||
|
regular_file(&lock_path)?;
|
||||||
|
}
|
||||||
|
let lock = acquire_runtime_store_owner_lock(&options.root)?;
|
||||||
|
validate_fresh_runtime_store_layout(&options.root)
|
||||||
|
.map_err(|_| reject("destination is not fresh; refusing overwrite"))?;
|
||||||
|
let auth: InitializedAuth = toml::from_str(&text(&options.root.join("auth.toml"))?)
|
||||||
|
.map_err(|_| reject("initialize Runtime identity and trust before install"))?;
|
||||||
|
if auth.identity.identity_id != self.runtime_id
|
||||||
|
|| !auth.archived_legacy_issuers.is_empty()
|
||||||
|
|| auth.identity.signing_key().is_err()
|
||||||
|
{
|
||||||
|
return Err(reject("invalid destination Runtime identity"));
|
||||||
|
}
|
||||||
|
validate_workspace_issuer_trust_records(&auth.workspace_issuers)
|
||||||
|
.map_err(|_| reject("invalid destination Workspace trust"))?;
|
||||||
|
let api = request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.expect("validated Workspace binding");
|
||||||
|
if !auth.workspace_issuers.iter().any(|trust| {
|
||||||
|
trust.workspace_id == self.workspace_id
|
||||||
|
&& trust.backend_url == api.base_url
|
||||||
|
&& trust.state == WorkspaceIssuerTrustState::Active
|
||||||
|
}) {
|
||||||
|
return Err(reject("request does not match initialized Workspace trust"));
|
||||||
|
}
|
||||||
|
let stage = options
|
||||||
|
.root
|
||||||
|
.join(format!(".fixture-{}", uuid::Uuid::now_v7()));
|
||||||
|
io(fs::create_dir(&stage))?;
|
||||||
|
let store = FsRuntimeStore {
|
||||||
|
root: stage.clone(),
|
||||||
|
_owner_lock: Some(lock),
|
||||||
|
worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)),
|
||||||
|
#[cfg(test)]
|
||||||
|
worker_reconcile_sync_fault: Default::default(),
|
||||||
|
#[cfg(test)]
|
||||||
|
worker_reconcile_read_fault: Default::default(),
|
||||||
|
#[cfg(test)]
|
||||||
|
worker_reconcile_counts: Default::default(),
|
||||||
|
};
|
||||||
|
let result = (|| {
|
||||||
|
let worker = PersistedWorkerRecord {
|
||||||
|
restore_guard: crate::runtime::RestoreGuard::default(),
|
||||||
|
worker_ref: WorkerRef::new(self.worker_id),
|
||||||
|
worker_id: self.worker_id,
|
||||||
|
profile: request.profile.clone(),
|
||||||
|
display_name: request.display_name.clone(),
|
||||||
|
profile_source: request.profile_source.reference(),
|
||||||
|
config_bundle: request.config_bundle.clone(),
|
||||||
|
created_at_ms: self.created_at_ms,
|
||||||
|
status: WorkerStatus::Stopped,
|
||||||
|
last_finished_submission_request_id: None,
|
||||||
|
workspace_id: Some(self.workspace_id.clone()),
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
logical_workdir_attachments: Vec::new(),
|
||||||
|
execution_state: PersistedWorkerExecutionState::Available(
|
||||||
|
PersistedWorkerExecution {
|
||||||
|
request,
|
||||||
|
binding: None,
|
||||||
|
restore_intent: WorkerRestoreIntent::Explicit,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
};
|
||||||
|
WorkerAggregateRecord::from_persisted(&worker)
|
||||||
|
.validate(&store.worker_dir(&self.worker_id).join(WORKER_FILE))
|
||||||
|
.map_err(|_| reject("invalid installation request"))?;
|
||||||
|
store.write_worker_record(&worker)?;
|
||||||
|
let worker_dir = store.worker_dir(&self.worker_id);
|
||||||
|
atomic_write_json(
|
||||||
|
&worker_dir.join(WORKER_METADATA_FILE),
|
||||||
|
&metadata,
|
||||||
|
"write fixture metadata",
|
||||||
|
)?;
|
||||||
|
let session = worker_dir.join("session");
|
||||||
|
io(fs::create_dir_all(session.join("segments")))?;
|
||||||
|
write_new(&session.join("session.json"), &self.session_manifest)?;
|
||||||
|
write_new(
|
||||||
|
&session
|
||||||
|
.join("segments")
|
||||||
|
.join(format!("{}.jsonl", self.segment_id)),
|
||||||
|
&self.segment_jsonl,
|
||||||
|
)?;
|
||||||
|
self.validate_session(&session)?;
|
||||||
|
store.write_runtime_snapshot(&PersistedRuntimeState {
|
||||||
|
subject_create_recovery_fences: BTreeMap::new(),
|
||||||
|
subject_releases: Vec::new(),
|
||||||
|
// Match normal store initialization: this is catalog availability,
|
||||||
|
// not a process or Worker execution start.
|
||||||
|
display_name: options.display_name.clone(),
|
||||||
|
status: RuntimeStatus::Running,
|
||||||
|
next_diagnostic_id: 1,
|
||||||
|
workers: BTreeMap::new(),
|
||||||
|
workspace_owners: BTreeMap::new(),
|
||||||
|
diagnostics: Vec::new(),
|
||||||
|
})?;
|
||||||
|
let loaded = store.load_runtime_state()?;
|
||||||
|
if loaded.workers.len() != 1 || !loaded.diagnostics.is_empty() {
|
||||||
|
return Err(reject("staged aggregate failed ordinary store roundtrip"));
|
||||||
|
}
|
||||||
|
sync_runtime_tree(&stage)?;
|
||||||
|
// No destination existed when checked under the owner lock. Hard-link
|
||||||
|
// the snapshot last with create-new semantics; never replace a file.
|
||||||
|
// Reserve the destination catalog with create-new semantics. Even
|
||||||
|
// an empty preexisting workers directory must never be replaced.
|
||||||
|
let destination_workers = options.root.join(WORKERS_DIR);
|
||||||
|
io(fs::create_dir(&destination_workers))?;
|
||||||
|
io(fs::rename(
|
||||||
|
store.worker_dir(&self.worker_id),
|
||||||
|
destination_workers.join(self.worker_id.to_string()),
|
||||||
|
))?;
|
||||||
|
sync_directory(&destination_workers, "publish stopped Worker aggregate")?;
|
||||||
|
io(fs::hard_link(
|
||||||
|
stage.join(RUNTIME_FILE),
|
||||||
|
options.root.join(RUNTIME_FILE),
|
||||||
|
))?;
|
||||||
|
sync_directory(&options.root, "publish stopped fixture")?;
|
||||||
|
Ok(())
|
||||||
|
})();
|
||||||
|
// Only remove our private staging directory, never existing destination data.
|
||||||
|
let cleanup = fs::remove_dir_all(&stage);
|
||||||
|
result?;
|
||||||
|
io(cleanup)?;
|
||||||
|
sync_directory(&options.root, "finish stopped fixture")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_shape(&self) -> Result<(), RuntimeError> {
|
||||||
|
// Version 1 templates carry only settled Session material, never a
|
||||||
|
// lifecycle operation. Schema 12 -> 13 changes pending decision evidence,
|
||||||
|
// not that payload. Accept the fixed schema-12 artifact read-only and
|
||||||
|
// install through the current aggregate owner; never rewrite its bytes.
|
||||||
|
if self.template_version != TEMPLATE_VERSION
|
||||||
|
|| !matches!(
|
||||||
|
self.runtime_schema_version,
|
||||||
|
RESTORE_ADMISSION_SCHEMA_VERSION | SCHEMA_VERSION
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return Err(reject("unsupported template schema"));
|
||||||
|
}
|
||||||
|
if !portable_id(&self.runtime_id)
|
||||||
|
|| !portable_id(&self.workspace_id)
|
||||||
|
|| self.session_id.get_version_num() != 7
|
||||||
|
|| self.segment_id.get_version_num() != 7
|
||||||
|
|| self.session_manifest.len() > MAX_SESSION_BYTES
|
||||||
|
|| self.segment_jsonl.len() > MAX_SESSION_BYTES
|
||||||
|
{
|
||||||
|
return Err(reject("invalid portable identity or size"));
|
||||||
|
}
|
||||||
|
let manifest: SessionManifest = decode(&self.session_manifest)?;
|
||||||
|
if manifest.session_id != self.session_id
|
||||||
|
|| manifest.schema_version == 0
|
||||||
|
|| !self.segment_jsonl.ends_with(b"\n")
|
||||||
|
{
|
||||||
|
return Err(reject("Session identity or committed boundary mismatch"));
|
||||||
|
}
|
||||||
|
// Read-only JSON traversal, never mutation/reconstruction of Session bytes.
|
||||||
|
// Reject known secret/host markers even in otherwise valid typed records.
|
||||||
|
for line in self
|
||||||
|
.segment_jsonl
|
||||||
|
.split(|b| *b == b'\n')
|
||||||
|
.filter(|line| !line.is_empty())
|
||||||
|
{
|
||||||
|
let _: PortableJson = decode(line)?;
|
||||||
|
validate_preserved_record(line)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_history_entry(
|
||||||
|
&self,
|
||||||
|
entry: &session_store::LoggedHistoryEntry,
|
||||||
|
ids: &mut std::collections::BTreeSet<String>,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
if !portable_id(&entry.metadata.entry_id.0)
|
||||||
|
|| !ids.insert(entry.metadata.entry_id.0.clone())
|
||||||
|
{
|
||||||
|
return Err(reject("duplicate or invalid entry identity"));
|
||||||
|
}
|
||||||
|
if entry
|
||||||
|
.metadata
|
||||||
|
.input_metadata
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|metadata| {
|
||||||
|
metadata.feature_payload().is_some() || metadata.public_item().is_some()
|
||||||
|
})
|
||||||
|
{
|
||||||
|
// These are opaque owner-specific payloads, not part of the narrow
|
||||||
|
// artificial conversation contract. Never preserve unchecked JSON.
|
||||||
|
return Err(reject("Feature history payloads are unsupported"));
|
||||||
|
}
|
||||||
|
if !matches!(
|
||||||
|
entry.item,
|
||||||
|
session_store::LoggedItem::Message { .. }
|
||||||
|
| session_store::LoggedItem::AssistantResponseBoundary { .. }
|
||||||
|
) {
|
||||||
|
return Err(reject("only text conversation records are supported"));
|
||||||
|
}
|
||||||
|
if let session_store::LoggedSessionHistoryOrigin::ModelOutput { worker } =
|
||||||
|
&entry.metadata.origin
|
||||||
|
{
|
||||||
|
// Session producer IDs identify the original execution, not the
|
||||||
|
// Workspace catalog Worker. Preserve them; do not rebind/rewrite them.
|
||||||
|
if !portable_id(&worker.worker_id)
|
||||||
|
|| worker
|
||||||
|
.workspace_id
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|id| id != &self.workspace_id)
|
||||||
|
|| worker
|
||||||
|
.runtime_id
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|id| id != &self.runtime_id)
|
||||||
|
{
|
||||||
|
return Err(reject("Session origin identity mismatch"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_session(&self, root: &Path) -> Result<(), RuntimeError> {
|
||||||
|
for entry in io(fs::read_dir(root))? {
|
||||||
|
let entry = io(entry)?;
|
||||||
|
match entry.file_name().to_str() {
|
||||||
|
Some("session.json" | "segments") => {}
|
||||||
|
Some("artifacts") => check_empty_artifacts(&entry.path())?,
|
||||||
|
_ => return Err(reject("unsupported Session files")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let entries = io(fs::read_dir(root.join("segments")))?.collect::<Result<Vec<_>, _>>();
|
||||||
|
let entries = io(entries)?;
|
||||||
|
if entries.len() != 1
|
||||||
|
|| entries[0].file_name() != format!("{}.jsonl", self.segment_id).as_str()
|
||||||
|
{
|
||||||
|
return Err(reject("only the fixed single-Segment fixture is supported"));
|
||||||
|
}
|
||||||
|
let store = WorkerSessionStore::open_read_only(root)
|
||||||
|
.map_err(|_| reject("unsupported Session schema"))?;
|
||||||
|
let logs = store
|
||||||
|
.read_all_read_only_bounded(self.session_id, self.segment_id, MAX_SESSION_BYTES as u64)
|
||||||
|
.map_err(|_| reject("invalid canonical Session records"))?
|
||||||
|
.0;
|
||||||
|
if !matches!(logs.first(), Some(session_store::LogEntry::AnnotatedSegmentStart {
|
||||||
|
session_id, system_prompt: Some(_), forked_from: None, compacted_from: None, ..
|
||||||
|
}) if *session_id == self.session_id)
|
||||||
|
{
|
||||||
|
return Err(reject(
|
||||||
|
"missing system prompt, invalid Segment seed, or unsupported lineage",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut ids = std::collections::BTreeSet::new();
|
||||||
|
for (index, log) in logs.iter().enumerate() {
|
||||||
|
use session_store::LogEntry;
|
||||||
|
match log {
|
||||||
|
LogEntry::AnnotatedSegmentStart { history, .. }
|
||||||
|
if index == 0 && history.is_empty() => {}
|
||||||
|
LogEntry::AnnotatedUserInput {
|
||||||
|
history,
|
||||||
|
extensions,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
for extension in extensions {
|
||||||
|
validate_settled_activation(extension)?;
|
||||||
|
}
|
||||||
|
for entry in history {
|
||||||
|
self.validate_history_entry(entry, &mut ids)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LogEntry::AnnotatedAssistantItem { entry, .. } => {
|
||||||
|
self.validate_history_entry(entry, &mut ids)?
|
||||||
|
}
|
||||||
|
LogEntry::RunCompleted {
|
||||||
|
entry_id: Some(id), ..
|
||||||
|
} => {
|
||||||
|
if !portable_id(&id.0) || !ids.insert(id.0.clone()) {
|
||||||
|
return Err(reject("duplicate or invalid entry identity"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LogEntry::Invoke { .. } | LogEntry::LlmUsage { .. } | LogEntry::TurnEnd { .. } => {}
|
||||||
|
_ => {
|
||||||
|
return Err(reject(
|
||||||
|
"unsupported conversation record or pending operation",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let state = session_store::collect_state(&logs);
|
||||||
|
if state.last_run_interrupted || state.active_run_turn_count.is_some() {
|
||||||
|
return Err(reject("Session has pending execution"));
|
||||||
|
}
|
||||||
|
for segments in &state.user_segments {
|
||||||
|
if segments
|
||||||
|
.iter()
|
||||||
|
.any(|segment| !matches!(segment, protocol::Segment::Text { .. }))
|
||||||
|
{
|
||||||
|
return Err(reject("Session attachments are unsupported"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare *fields*, not re-encoded bytes or scalar values (e.g. f32 JSON
|
||||||
|
// normalization). Serde's permissive/untagged/flattened compatibility readers
|
||||||
|
// can drop unknown data without surfacing it to an ignored-field callback. A
|
||||||
|
// retained field must survive the owner's typed projection at every depth.
|
||||||
|
// This intentionally accepts only canonical writer shapes: even an explicit
|
||||||
|
// empty optional field omitted by that writer is rejected, not repaired.
|
||||||
|
// The projection is validation-only; installation always writes original bytes.
|
||||||
|
fn validate_preserved_record(bytes: &[u8]) -> Result<(), RuntimeError> {
|
||||||
|
let original: serde_json::Value = decode(bytes)?;
|
||||||
|
let record: session_store::LogEntry = decode(bytes)?;
|
||||||
|
let projection =
|
||||||
|
serde_json::to_value(&record).map_err(|_| reject("invalid typed Session record"))?;
|
||||||
|
require_preserved_fields(&original, &projection)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_preserved_fields(
|
||||||
|
original: &serde_json::Value,
|
||||||
|
projection: &serde_json::Value,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
use serde_json::Value;
|
||||||
|
match (original, projection) {
|
||||||
|
(Value::Object(original), Value::Object(projection)) => {
|
||||||
|
for (key, value) in original {
|
||||||
|
let projected = projection
|
||||||
|
.get(key)
|
||||||
|
.ok_or_else(|| reject("unknown or noncanonical Session field"))?;
|
||||||
|
require_preserved_fields(value, projected)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(Value::Array(original), Value::Array(projection))
|
||||||
|
if original.len() == projection.len() =>
|
||||||
|
{
|
||||||
|
for (value, projected) in original.iter().zip(projection) {
|
||||||
|
require_preserved_fields(value, projected)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(Value::Null, Value::Null)
|
||||||
|
| (Value::Bool(_), Value::Bool(_))
|
||||||
|
| (Value::Number(_), Value::Number(_))
|
||||||
|
| (Value::String(_), Value::String(_)) => {}
|
||||||
|
_ => return Err(reject("noncanonical Session field shape")),
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exact receipt wire fields owned by worker::PendingActivationState. Keep these
|
||||||
|
// fixture-only mirrors strict; do not change the production compatibility reader.
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct SubmissionReceipt {
|
||||||
|
submission_request_id: String,
|
||||||
|
source_namespace: String,
|
||||||
|
submission_id: String,
|
||||||
|
payload_digest: String,
|
||||||
|
disposition: protocol::SubmissionDisposition,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct NotificationReceipt {
|
||||||
|
notification_request_id: String,
|
||||||
|
source_namespace: String,
|
||||||
|
payload_digest: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
// The approved conversation has committed submission checkpoints in its original
|
||||||
|
// Session records. Keep their receipt bytes, but reject any resumable queue or
|
||||||
|
// Feature invocation history. This projection confers no execution authority.
|
||||||
|
fn validate_settled_activation(
|
||||||
|
extension: &session_store::SessionExtension,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct Checkpoint {
|
||||||
|
#[serde(rename = "next_activation_sequence")]
|
||||||
|
_next_activation_sequence: u64,
|
||||||
|
activating: Option<serde::de::IgnoredAny>,
|
||||||
|
activating_notification: Vec<serde::de::IgnoredAny>,
|
||||||
|
pending: Vec<serde::de::IgnoredAny>,
|
||||||
|
pending_notifications: Vec<serde::de::IgnoredAny>,
|
||||||
|
#[serde(rename = "receipts")]
|
||||||
|
_receipts: Vec<SubmissionReceipt>,
|
||||||
|
#[serde(rename = "notification_receipts")]
|
||||||
|
_notification_receipts: Vec<NotificationReceipt>,
|
||||||
|
invocation_receipts: Vec<serde::de::IgnoredAny>,
|
||||||
|
invocation_recovery_blocked: bool,
|
||||||
|
}
|
||||||
|
if extension.domain != "worker.pending_activations.v1" {
|
||||||
|
return Err(reject("unsupported Session extension"));
|
||||||
|
}
|
||||||
|
let checkpoint: Checkpoint = serde_json::from_value(extension.payload.clone())
|
||||||
|
.map_err(|_| reject("invalid Session activation checkpoint"))?;
|
||||||
|
if checkpoint.activating.is_some()
|
||||||
|
|| !checkpoint.activating_notification.is_empty()
|
||||||
|
|| !checkpoint.pending.is_empty()
|
||||||
|
|| !checkpoint.pending_notifications.is_empty()
|
||||||
|
|| !checkpoint.invocation_receipts.is_empty()
|
||||||
|
|| checkpoint.invocation_recovery_blocked
|
||||||
|
{
|
||||||
|
return Err(reject(
|
||||||
|
"Session has pending activation or invocation history",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_request(request: &CreateWorkerRequest, install: bool) -> Result<(), RuntimeError> {
|
||||||
|
if !request.workdir_attachment_requests.is_empty()
|
||||||
|
|| !request.workdir_attachments.is_empty()
|
||||||
|
|| request.backend_job.is_some()
|
||||||
|
|| request.subjektiv_attached
|
||||||
|
|| request.worker_observation_enabled
|
||||||
|
|| !request.worker_observation_grants.is_empty()
|
||||||
|
|| (install && request.initial_input.is_some())
|
||||||
|
{
|
||||||
|
return Err(reject(
|
||||||
|
"unsupported capabilities, attachments, or initial input",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if install
|
||||||
|
&& (request.create_fingerprint.trim().is_empty()
|
||||||
|
|| request.config_bundle.is_none()
|
||||||
|
|| !matches!(
|
||||||
|
request.profile_source,
|
||||||
|
ProfileSourceArchiveSource::WorkspaceConfig { .. }
|
||||||
|
))
|
||||||
|
{
|
||||||
|
return Err(reject(
|
||||||
|
"fresh config/profile references and create fingerprint are required",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_empty_artifacts(root: &Path) -> Result<(), RuntimeError> {
|
||||||
|
for entry in io(fs::read_dir(root))? {
|
||||||
|
let entry = io(entry)?;
|
||||||
|
let metadata = io(entry.metadata())?;
|
||||||
|
if metadata.is_dir() {
|
||||||
|
check_empty_artifacts(&entry.path())?;
|
||||||
|
} else if entry.file_name() != ".aggregate.lock" || metadata.len() != 0 {
|
||||||
|
return Err(reject("Session artifacts are unsupported"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_new(path: &Path, bytes: &[u8]) -> Result<(), RuntimeError> {
|
||||||
|
let mut file = io(OpenOptions::new().write(true).create_new(true).open(path))?;
|
||||||
|
io(file.write_all(bytes))?;
|
||||||
|
io(file.sync_all())
|
||||||
|
}
|
||||||
|
|
||||||
|
// A streaming validation visitor, not a generic document available to callers.
|
||||||
|
struct PortableJson;
|
||||||
|
impl<'de> Deserialize<'de> for PortableJson {
|
||||||
|
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||||
|
struct Visitor;
|
||||||
|
impl<'de> serde::de::Visitor<'de> for Visitor {
|
||||||
|
type Value = PortableJson;
|
||||||
|
fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.write_str("portable fixture JSON")
|
||||||
|
}
|
||||||
|
fn visit_bool<E: serde::de::Error>(self, _: bool) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_i64<E: serde::de::Error>(self, _: i64) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_u64<E: serde::de::Error>(self, _: u64) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_f64<E: serde::de::Error>(self, _: f64) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_unit<E: serde::de::Error>(self) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<PortableJson, E> {
|
||||||
|
let lower = value.to_ascii_lowercase();
|
||||||
|
if [
|
||||||
|
"://",
|
||||||
|
"private key",
|
||||||
|
"bearer ",
|
||||||
|
"sk-",
|
||||||
|
"api_key",
|
||||||
|
"access_token",
|
||||||
|
"refresh_token",
|
||||||
|
"private_key",
|
||||||
|
"authorization",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|marker| lower.contains(marker))
|
||||||
|
|| value
|
||||||
|
.split_whitespace()
|
||||||
|
.any(|word| word.starts_with('/') || word.contains(":\\"))
|
||||||
|
{
|
||||||
|
return Err(E::custom("nonportable or sensitive fixture material"));
|
||||||
|
}
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_seq<A: serde::de::SeqAccess<'de>>(
|
||||||
|
self,
|
||||||
|
mut seq: A,
|
||||||
|
) -> Result<PortableJson, A::Error> {
|
||||||
|
while seq.next_element::<PortableJson>()?.is_some() {}
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_map<A: serde::de::MapAccess<'de>>(
|
||||||
|
self,
|
||||||
|
mut map: A,
|
||||||
|
) -> Result<PortableJson, A::Error> {
|
||||||
|
let mut keys = std::collections::BTreeSet::new();
|
||||||
|
while let Some(key) = map.next_key::<String>()? {
|
||||||
|
// A duplicate key can hide discarded data even inside an
|
||||||
|
// otherwise opaque extension payload. Never retain it.
|
||||||
|
if !keys.insert(key.clone()) {
|
||||||
|
return Err(serde::de::Error::custom("duplicate fixture JSON field"));
|
||||||
|
}
|
||||||
|
serde::de::Visitor::visit_str::<A::Error>(Visitor, &key)?;
|
||||||
|
map.next_value::<PortableJson>()?;
|
||||||
|
}
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
deserializer.deserialize_any(Visitor)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,919 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::auth::RuntimeIdentityMaterial;
|
||||||
|
use crate::catalog::{ProfileSelector, WorkspaceApiRef};
|
||||||
|
use crate::profile_archive::ProfileSourceGraphSummary;
|
||||||
|
use session_store::{LogEntry, Store};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
const RUNTIME: &str = "fixture-runtime";
|
||||||
|
const WORKSPACE: &str = "fixture-workspace";
|
||||||
|
const NEW_URL: &str = "http://127.0.0.1:34567";
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
enum ReceiptFixture<T> {
|
||||||
|
Valid(T),
|
||||||
|
Malformed(u8),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default, Serialize)]
|
||||||
|
struct CheckpointFixture {
|
||||||
|
next_activation_sequence: u64,
|
||||||
|
activating: Option<u8>,
|
||||||
|
activating_notification: Vec<u8>,
|
||||||
|
pending: Vec<u8>,
|
||||||
|
pending_notifications: Vec<u8>,
|
||||||
|
receipts: Vec<ReceiptFixture<SubmissionReceipt>>,
|
||||||
|
notification_receipts: Vec<ReceiptFixture<NotificationReceipt>>,
|
||||||
|
invocation_receipts: Vec<u8>,
|
||||||
|
invocation_recovery_blocked: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn settled_checkpoint() -> CheckpointFixture {
|
||||||
|
CheckpointFixture {
|
||||||
|
next_activation_sequence: 2,
|
||||||
|
receipts: vec![ReceiptFixture::Valid(SubmissionReceipt {
|
||||||
|
submission_request_id: "completed-submission-request".into(),
|
||||||
|
source_namespace: "fixture-source".into(),
|
||||||
|
submission_id: "completed-submission".into(),
|
||||||
|
payload_digest: "fixture-payload-digest".into(),
|
||||||
|
disposition: protocol::SubmissionDisposition::Started,
|
||||||
|
})],
|
||||||
|
notification_receipts: vec![ReceiptFixture::Valid(NotificationReceipt {
|
||||||
|
notification_request_id: "completed-notification-request".into(),
|
||||||
|
source_namespace: "fixture-source".into(),
|
||||||
|
payload_digest: "fixture-notification-digest".into(),
|
||||||
|
})],
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn checkpoint_extension(checkpoint: &CheckpointFixture) -> session_store::SessionExtension {
|
||||||
|
session_store::SessionExtension::new(
|
||||||
|
"worker.pending_activations.v1",
|
||||||
|
serde_json::to_value(checkpoint).unwrap(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn worker_id() -> WorkerId {
|
||||||
|
"01a1657d-b762-7791-87ae-a49f28cadbce".parse().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolved() -> manifest::WorkerManifest {
|
||||||
|
let mut config = manifest::WorkerManifestConfig::resolution_defaults();
|
||||||
|
config.worker.name = Some(runtime_worker_name(worker_id()));
|
||||||
|
manifest::WorkerManifest::try_from(config).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request() -> CreateWorkerRequest {
|
||||||
|
CreateWorkerRequest {
|
||||||
|
worker_id: worker_id(),
|
||||||
|
create_fingerprint: "new-create-fingerprint".into(),
|
||||||
|
profile: ProfileSelector::Builtin("default".into()),
|
||||||
|
display_name: Some("fixture".into()),
|
||||||
|
profile_source: ProfileSourceArchiveSource::WorkspaceConfig {
|
||||||
|
archive: ProfileSourceArchiveRef {
|
||||||
|
id: "new-profile".into(),
|
||||||
|
digest: "new-profile-digest".into(),
|
||||||
|
size_bytes: 1,
|
||||||
|
source_graph: ProfileSourceGraphSummary {
|
||||||
|
source_count: 1,
|
||||||
|
total_source_bytes: 1,
|
||||||
|
entrypoints: BTreeMap::from([("builtin:default".into(), "default.dcl".into())]),
|
||||||
|
import_count: 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
config_bundle: Some(ConfigBundleRef {
|
||||||
|
id: "new-config".into(),
|
||||||
|
digest: "new-config-digest".into(),
|
||||||
|
}),
|
||||||
|
workspace_api: Some(WorkspaceApiRef {
|
||||||
|
workspace_id: WORKSPACE.into(),
|
||||||
|
base_url: NEW_URL.into(),
|
||||||
|
}),
|
||||||
|
initial_input: None,
|
||||||
|
workdir_attachment_requests: Vec::new(),
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
worker_observation_enabled: false,
|
||||||
|
worker_observation_grants: Vec::new(),
|
||||||
|
memory_settings: Some(manifest::WorkspaceMemorySettingsSnapshot {
|
||||||
|
workspace_id: WORKSPACE.into(),
|
||||||
|
language: "en".into(),
|
||||||
|
}),
|
||||||
|
subjektiv_attached: false,
|
||||||
|
backend_job: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn initialize(root: &Path, runtime_id: &str, workspace_id: &str, url: &str) {
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct Auth {
|
||||||
|
identity: RuntimeIdentityMaterial,
|
||||||
|
workspace_issuers: Vec<WorkspaceIssuerTrustRecord>,
|
||||||
|
}
|
||||||
|
let issuer = RuntimeIdentityMaterial::generate("fixture-issuer").unwrap();
|
||||||
|
let digest = Sha256::digest(crate::auth::decode_public_key(&issuer.public_key).unwrap());
|
||||||
|
let fingerprint = format!(
|
||||||
|
"sha256:{}",
|
||||||
|
digest
|
||||||
|
.iter()
|
||||||
|
.map(|byte| format!("{byte:02x}"))
|
||||||
|
.collect::<String>()
|
||||||
|
);
|
||||||
|
let trust = WorkspaceIssuerTrustRecord::from_bundle(
|
||||||
|
server_api::WorkspacePublicIdentityBundle {
|
||||||
|
workspace_id: workspace_id.into(),
|
||||||
|
backend_url: url.into(),
|
||||||
|
key_id: "fixture-issuer".into(),
|
||||||
|
algorithm: "ed25519".into(),
|
||||||
|
public_key: issuer.public_key,
|
||||||
|
public_key_fingerprint: fingerprint,
|
||||||
|
},
|
||||||
|
"fixture-trust".into(),
|
||||||
|
100,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fs::create_dir_all(root).unwrap();
|
||||||
|
fs::write(
|
||||||
|
root.join("auth.toml"),
|
||||||
|
toml::to_string(&Auth {
|
||||||
|
identity: RuntimeIdentityMaterial::generate(runtime_id).unwrap(),
|
||||||
|
workspace_issuers: vec![trust],
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record(request: CreateWorkerRequest) -> PersistedWorkerRecord {
|
||||||
|
PersistedWorkerRecord {
|
||||||
|
restore_guard: crate::runtime::RestoreGuard::default(),
|
||||||
|
worker_ref: WorkerRef::new(request.worker_id),
|
||||||
|
worker_id: request.worker_id,
|
||||||
|
profile: request.profile.clone(),
|
||||||
|
display_name: request.display_name.clone(),
|
||||||
|
profile_source: request.profile_source.reference(),
|
||||||
|
config_bundle: request.config_bundle.clone(),
|
||||||
|
created_at_ms: Some(123456),
|
||||||
|
status: WorkerStatus::Stopped,
|
||||||
|
last_finished_submission_request_id: Some("old-submission-operation".into()),
|
||||||
|
workspace_id: Some(WORKSPACE.into()),
|
||||||
|
workdir_attachments: vec![],
|
||||||
|
logical_workdir_attachments: vec![],
|
||||||
|
execution_state: PersistedWorkerExecutionState::Available(PersistedWorkerExecution {
|
||||||
|
request,
|
||||||
|
binding: Some(PersistedWorkerExecutionBinding {}),
|
||||||
|
restore_intent: WorkerRestoreIntent::Explicit,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn source() -> (tempfile::TempDir, PathBuf, StoppedWorkerTemplate) {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let root = temp.path().join("runtime");
|
||||||
|
initialize(&root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456");
|
||||||
|
drop(
|
||||||
|
crate::Runtime::with_fs_store(FsRuntimeStoreOptions::new(&root).with_runtime_id(RUNTIME))
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let opened = FsRuntimeStore::open_or_create(root.clone(), RUNTIME).unwrap();
|
||||||
|
let mut old = request();
|
||||||
|
old.workspace_api.as_mut().unwrap().base_url = "http://127.0.0.1:23456".into();
|
||||||
|
old.config_bundle.as_mut().unwrap().id = "old-config".into();
|
||||||
|
opened.store.write_worker_record(&record(old)).unwrap();
|
||||||
|
let worker = opened.store.worker_dir(&worker_id());
|
||||||
|
let session_id = "01a1657d-b800-7777-8888-111111111111".parse().unwrap();
|
||||||
|
let segment_id = "01a1657d-b818-77c1-bd73-176172a4bd82".parse().unwrap();
|
||||||
|
let mut metadata = WorkerMetadata::new(
|
||||||
|
runtime_worker_name(worker_id()),
|
||||||
|
Some(WorkerActiveSegmentRef {
|
||||||
|
session_id,
|
||||||
|
segment_id: Some(segment_id),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.with_workspace_id(WORKSPACE)
|
||||||
|
.with_workspace_root(PathBuf::from("/old/host/path"));
|
||||||
|
metadata.resolved_manifest_snapshot =
|
||||||
|
Some(manifest::write_persisted_worker_manifest_snapshot(&resolved()).unwrap());
|
||||||
|
atomic_write_json(
|
||||||
|
&worker.join(WORKER_METADATA_FILE),
|
||||||
|
&metadata,
|
||||||
|
"test metadata",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let sessions = WorkerSessionStore::new(worker.join("session")).unwrap();
|
||||||
|
let history = session_store::LoggedHistoryEntry {
|
||||||
|
item: session_store::LoggedItem::Message {
|
||||||
|
role: session_store::LoggedRole::Assistant,
|
||||||
|
content: vec![session_store::LoggedContentPart::Text {
|
||||||
|
text: "artificial fixture reply".into(),
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||||
|
entry_id: session_store::LoggedSessionHistoryEntryId("fixed-entry-id".into()),
|
||||||
|
origin: session_store::LoggedSessionHistoryOrigin::ModelOutput {
|
||||||
|
worker: session_store::LoggedWorkerSubject {
|
||||||
|
workspace_id: Some(WORKSPACE.into()),
|
||||||
|
runtime_id: Some(RUNTIME.into()),
|
||||||
|
worker_id: worker_id().to_string(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
derivation: None,
|
||||||
|
input_metadata: None,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
sessions
|
||||||
|
.create_segment(
|
||||||
|
session_id,
|
||||||
|
segment_id,
|
||||||
|
&[
|
||||||
|
LogEntry::AnnotatedSegmentStart {
|
||||||
|
ts: 123,
|
||||||
|
session_id,
|
||||||
|
system_prompt: Some("Harmless artificial fixture instructions.".into()),
|
||||||
|
config: Default::default(),
|
||||||
|
history: vec![],
|
||||||
|
forked_from: None,
|
||||||
|
compacted_from: None,
|
||||||
|
},
|
||||||
|
LogEntry::Invoke {
|
||||||
|
ts: 124,
|
||||||
|
trigger: protocol::InvokeKind::UserSend,
|
||||||
|
},
|
||||||
|
LogEntry::AnnotatedUserInput {
|
||||||
|
ts: 124,
|
||||||
|
segments: vec![protocol::Segment::text(
|
||||||
|
"Harmless artificial fixture input.",
|
||||||
|
)],
|
||||||
|
extensions: vec![checkpoint_extension(&settled_checkpoint())],
|
||||||
|
history: vec![session_store::LoggedHistoryEntry {
|
||||||
|
item: session_store::LoggedItem::Message {
|
||||||
|
role: session_store::LoggedRole::User,
|
||||||
|
content: vec![session_store::LoggedContentPart::Text {
|
||||||
|
text: "Harmless artificial fixture input.".into(),
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||||
|
entry_id: session_store::LoggedSessionHistoryEntryId(
|
||||||
|
"fixed-user-id".into(),
|
||||||
|
),
|
||||||
|
origin: session_store::LoggedSessionHistoryOrigin::BackendInstruction {
|
||||||
|
operation_id: None,
|
||||||
|
},
|
||||||
|
derivation: None,
|
||||||
|
input_metadata: None,
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
LogEntry::AnnotatedAssistantItem {
|
||||||
|
ts: 125,
|
||||||
|
entry: history,
|
||||||
|
},
|
||||||
|
LogEntry::LlmUsage {
|
||||||
|
ts: 125,
|
||||||
|
history_len: 2,
|
||||||
|
input_total_tokens: 1,
|
||||||
|
cache_read_tokens: 0,
|
||||||
|
cache_write_tokens: 0,
|
||||||
|
output_tokens: 1,
|
||||||
|
},
|
||||||
|
LogEntry::TurnEnd {
|
||||||
|
ts: 126,
|
||||||
|
turn_count: 1,
|
||||||
|
},
|
||||||
|
LogEntry::RunCompleted {
|
||||||
|
entry_id: Some(session_store::LoggedSessionHistoryEntryId(
|
||||||
|
"fixed-terminal-id".into(),
|
||||||
|
)),
|
||||||
|
ts: 126,
|
||||||
|
interrupted: false,
|
||||||
|
result: agen::EngineResult::Finished,
|
||||||
|
active_run_turn_count: None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fs::create_dir_all(worker.join("runs/old-run")).unwrap();
|
||||||
|
fs::write(worker.join("runs/old-run/private.log"), "not exported").unwrap();
|
||||||
|
drop(opened);
|
||||||
|
let template = StoppedWorkerTemplate::extract(&worker).unwrap();
|
||||||
|
(temp, worker, template)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn destination(template: &StoppedWorkerTemplate) -> (tempfile::TempDir, FsRuntimeStoreOptions) {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let root = temp.path().join("runtime");
|
||||||
|
initialize(
|
||||||
|
&root,
|
||||||
|
template.runtime_id(),
|
||||||
|
template.workspace_id(),
|
||||||
|
NEW_URL,
|
||||||
|
);
|
||||||
|
(
|
||||||
|
temp,
|
||||||
|
FsRuntimeStoreOptions::new(root).with_runtime_id(template.runtime_id()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_fresh(options: &FsRuntimeStoreOptions) {
|
||||||
|
assert_eq!(fs::read_dir(&options.root).unwrap().count(), 1);
|
||||||
|
assert!(options.root.join("auth.toml").is_file());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extraction_install_roundtrip_preserves_bytes_and_is_inactive() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
let original_worker = fs::read(worker.join(WORKER_FILE)).unwrap();
|
||||||
|
let original_metadata = fs::read(worker.join(WORKER_METADATA_FILE)).unwrap();
|
||||||
|
let exported = serde_json::to_vec(&template).unwrap();
|
||||||
|
let exported_text = String::from_utf8(exported.clone()).unwrap();
|
||||||
|
for forbidden in [
|
||||||
|
"old-config",
|
||||||
|
"old-submission-operation",
|
||||||
|
"restore_guard",
|
||||||
|
"private_key",
|
||||||
|
"http://",
|
||||||
|
"/old/host",
|
||||||
|
"private.log",
|
||||||
|
"resolved_manifest_snapshot",
|
||||||
|
] {
|
||||||
|
assert!(!exported_text.contains(forbidden));
|
||||||
|
}
|
||||||
|
let template: StoppedWorkerTemplate = serde_json::from_slice(&exported).unwrap();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let auth_before = fs::read(options.root.join("auth.toml")).unwrap();
|
||||||
|
template.install(&options, request(), &resolved()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(options.root.join("auth.toml")).unwrap(),
|
||||||
|
auth_before
|
||||||
|
);
|
||||||
|
assert_eq!(fs::read(worker.join(WORKER_FILE)).unwrap(), original_worker);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(worker.join(WORKER_METADATA_FILE)).unwrap(),
|
||||||
|
original_metadata
|
||||||
|
);
|
||||||
|
let installed = options
|
||||||
|
.root
|
||||||
|
.join(WORKERS_DIR)
|
||||||
|
.join(template.worker_id().to_string());
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(installed.join("session/session.json")).unwrap(),
|
||||||
|
template.session_manifest
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(installed.join(format!("session/segments/{}.jsonl", template.segment_id())))
|
||||||
|
.unwrap(),
|
||||||
|
template.segment_jsonl
|
||||||
|
);
|
||||||
|
assert!(!installed.join("runs").exists());
|
||||||
|
let metadata: WorkerMetadata =
|
||||||
|
decode(&fs::read(installed.join(WORKER_METADATA_FILE)).unwrap()).unwrap();
|
||||||
|
assert_eq!(metadata.active.unwrap().session_id, template.session_id());
|
||||||
|
assert!(metadata.workspace_root.is_none() && metadata.resolved_manifest_snapshot.is_some());
|
||||||
|
|
||||||
|
let opened = FsRuntimeStore::open_or_create(options.root.clone(), RUNTIME).unwrap();
|
||||||
|
let state = opened.state.unwrap();
|
||||||
|
assert!(state.diagnostics.is_empty());
|
||||||
|
let saved = state.workers.get(&worker_id()).unwrap();
|
||||||
|
assert_eq!(saved.status, WorkerStatus::Stopped);
|
||||||
|
assert!(saved.last_finished_submission_request_id.is_none());
|
||||||
|
let PersistedWorkerExecutionState::Available(execution) = &saved.execution_state else {
|
||||||
|
panic!("not available")
|
||||||
|
};
|
||||||
|
assert_eq!(execution.restore_intent, WorkerRestoreIntent::Explicit);
|
||||||
|
assert!(execution.binding.is_none());
|
||||||
|
assert_eq!(execution.request, request());
|
||||||
|
// Fresh restore guard, no operation owner/receipt history.
|
||||||
|
assert_ne!(
|
||||||
|
serde_json::to_vec(&saved.restore_guard).unwrap(),
|
||||||
|
serde_json::to_vec(
|
||||||
|
&decode::<WorkerAggregateRecord>(&original_worker)
|
||||||
|
.unwrap()
|
||||||
|
.restore_guard
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
let guard: GuardState = decode(&serde_json::to_vec(&saved.restore_guard).unwrap()).unwrap();
|
||||||
|
assert!(guard.active_request_id.is_none() && guard.owners.is_empty());
|
||||||
|
drop(opened.store);
|
||||||
|
|
||||||
|
struct NoExecution;
|
||||||
|
impl crate::execution::WorkerExecutionBackend for NoExecution {
|
||||||
|
fn backend_id(&self) -> &str {
|
||||||
|
"fixture-no-execution"
|
||||||
|
}
|
||||||
|
fn spawn_worker(
|
||||||
|
&self,
|
||||||
|
_: crate::execution::WorkerExecutionSpawnRequest,
|
||||||
|
) -> crate::execution::WorkerExecutionSpawnResult {
|
||||||
|
panic!("fixture executed")
|
||||||
|
}
|
||||||
|
fn dispatch_input(
|
||||||
|
&self,
|
||||||
|
_: &WorkerRef,
|
||||||
|
_: crate::interaction::WorkerInput,
|
||||||
|
) -> crate::execution::WorkerExecutionResult {
|
||||||
|
panic!("fixture submitted")
|
||||||
|
}
|
||||||
|
fn restore_worker(
|
||||||
|
&self,
|
||||||
|
_: crate::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> crate::execution::WorkerExecutionSpawnResult {
|
||||||
|
panic!("fixture auto-restored")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let runtime =
|
||||||
|
crate::Runtime::with_fs_store_and_execution_backend(options.clone(), Arc::new(NoExecution))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(runtime.list_workers().unwrap().len(), 1);
|
||||||
|
assert_eq!(runtime.summary().unwrap().active_worker_count, 0);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.worker_detail(&WorkerRef::new(worker_id()))
|
||||||
|
.unwrap()
|
||||||
|
.status,
|
||||||
|
WorkerStatus::Stopped
|
||||||
|
);
|
||||||
|
drop(runtime);
|
||||||
|
let runtime = crate::Runtime::with_fs_store(options).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
runtime.list_workers().unwrap()[0].status,
|
||||||
|
WorkerStatus::Stopped
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn install_rejects_schema_identity_input_and_overwrite() {
|
||||||
|
let (_source, _, template) = source();
|
||||||
|
for change in [
|
||||||
|
"template",
|
||||||
|
"runtime-schema",
|
||||||
|
"session-schema",
|
||||||
|
"session-id",
|
||||||
|
"segment-id",
|
||||||
|
"runtime-id",
|
||||||
|
"workspace-id",
|
||||||
|
"worker-id",
|
||||||
|
"origin-id",
|
||||||
|
"duplicate-id",
|
||||||
|
"truncated",
|
||||||
|
] {
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let mut invalid = template.clone();
|
||||||
|
match change {
|
||||||
|
"template" => invalid.template_version += 1,
|
||||||
|
"runtime-schema" => invalid.runtime_schema_version += 1,
|
||||||
|
"session-schema" => {
|
||||||
|
invalid.session_manifest = format!(
|
||||||
|
"{{\"schema_version\":999,\"session_id\":\"{}\"}}",
|
||||||
|
invalid.session_id
|
||||||
|
)
|
||||||
|
.into_bytes()
|
||||||
|
}
|
||||||
|
"session-id" => invalid.session_id = uuid::Uuid::now_v7(),
|
||||||
|
"segment-id" => invalid.segment_id = uuid::Uuid::nil(), // no external attestation: valid new Segment ID is otherwise portable
|
||||||
|
"runtime-id" => invalid.runtime_id = "other-runtime".into(),
|
||||||
|
"workspace-id" => invalid.workspace_id = "other-workspace".into(),
|
||||||
|
"worker-id" => invalid.worker_id = WorkerId::now_v7(),
|
||||||
|
"origin-id" => {
|
||||||
|
invalid.segment_jsonl = String::from_utf8(invalid.segment_jsonl)
|
||||||
|
.unwrap()
|
||||||
|
.replace(RUNTIME, "other-runtime")
|
||||||
|
.into_bytes()
|
||||||
|
}
|
||||||
|
"duplicate-id" => {
|
||||||
|
let last = invalid
|
||||||
|
.segment_jsonl
|
||||||
|
.split_inclusive(|b| *b == b'\n')
|
||||||
|
.last()
|
||||||
|
.unwrap()
|
||||||
|
.to_vec();
|
||||||
|
invalid.segment_jsonl.extend(last);
|
||||||
|
}
|
||||||
|
"truncated" => {
|
||||||
|
invalid.segment_jsonl.pop();
|
||||||
|
}
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
invalid.install(&options, request(), &resolved()).is_err(),
|
||||||
|
"accepted {change}"
|
||||||
|
);
|
||||||
|
assert_fresh(&options);
|
||||||
|
}
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let mut launch = request();
|
||||||
|
launch.initial_input = Some(crate::interaction::WorkerInput::user("do not execute"));
|
||||||
|
assert!(template.install(&options, launch, &resolved()).is_err());
|
||||||
|
assert_fresh(&options);
|
||||||
|
template.install(&options, request(), &resolved()).unwrap();
|
||||||
|
let worker_path = options
|
||||||
|
.root
|
||||||
|
.join(WORKERS_DIR)
|
||||||
|
.join(worker_id().to_string())
|
||||||
|
.join(WORKER_FILE);
|
||||||
|
let before = fs::read(&worker_path).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
assert_eq!(fs::read(worker_path).unwrap(), before);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_uninitialized_untrusted_live_and_partial_destination() {
|
||||||
|
let (_source, _, template) = source();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let held = acquire_runtime_store_owner_lock(&options.root).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
assert_fresh(&options);
|
||||||
|
drop(held);
|
||||||
|
initialize(&options.root, "wrong-runtime", WORKSPACE, NEW_URL);
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
initialize(&options.root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456");
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
fs::remove_file(options.root.join("auth.toml")).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
initialize(&options.root, RUNTIME, WORKSPACE, NEW_URL);
|
||||||
|
fs::create_dir(options.root.join(WORKERS_DIR)).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() {
|
||||||
|
for change in [
|
||||||
|
"running",
|
||||||
|
"pending",
|
||||||
|
"guard-pending",
|
||||||
|
"attachment",
|
||||||
|
"schema",
|
||||||
|
"worker-id",
|
||||||
|
"metadata-workspace",
|
||||||
|
"metadata-session",
|
||||||
|
"artifact",
|
||||||
|
"secret",
|
||||||
|
"url",
|
||||||
|
"path",
|
||||||
|
] {
|
||||||
|
let (_source, worker, _) = source();
|
||||||
|
let mut aggregate: WorkerAggregateRecord =
|
||||||
|
decode(&fs::read(worker.join(WORKER_FILE)).unwrap()).unwrap();
|
||||||
|
match change {
|
||||||
|
"running" => aggregate.status = WorkerStatus::Running,
|
||||||
|
"pending" => {
|
||||||
|
aggregate.execution_state = WorkerExecutionStateRecord::ReconciliationRequired(
|
||||||
|
PersistedWorkerLifecycleOperation::Restore(PersistedWorkerRestoreOperation {
|
||||||
|
operation_id: WorkerLifecycleOperationId::new(),
|
||||||
|
mode: PersistedWorkerRestoreMode::Explicit,
|
||||||
|
commit: crate::runtime::RestoreCommitFact::NotCommitted,
|
||||||
|
source_repair_possible: false,
|
||||||
|
request: request(),
|
||||||
|
binding: None,
|
||||||
|
last_settled_status: WorkerStatus::Stopped,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
"guard-pending" => aggregate.restore_guard = serde_json::from_str(r#"{"observation_token":"old-token","active_request_id":"pending-restore","owners":{}}"#).unwrap(),
|
||||||
|
"attachment" => aggregate.logical_workdir_attachments.push(LogicalWorkdirAttachment {
|
||||||
|
alias: workdir::WorkdirAttachmentAlias::new("source").unwrap(),
|
||||||
|
working_directory_id: "old-workdir".into(),
|
||||||
|
capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
}),
|
||||||
|
"schema" => aggregate.schema_version += 1,
|
||||||
|
"worker-id" => aggregate.worker_id = WorkerId::now_v7(),
|
||||||
|
"metadata-workspace" | "metadata-session" => {
|
||||||
|
let mut metadata: WorkerMetadata =
|
||||||
|
decode(&fs::read(worker.join(WORKER_METADATA_FILE)).unwrap()).unwrap();
|
||||||
|
if change == "metadata-workspace" {
|
||||||
|
metadata.workspace_id = Some("wrong-workspace".into());
|
||||||
|
} else {
|
||||||
|
metadata.active.as_mut().unwrap().session_id = uuid::Uuid::now_v7();
|
||||||
|
}
|
||||||
|
atomic_write_json(
|
||||||
|
&worker.join(WORKER_METADATA_FILE),
|
||||||
|
&metadata,
|
||||||
|
"test mismatch",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
"artifact" => {
|
||||||
|
fs::create_dir_all(worker.join("session/artifacts/paste")).unwrap();
|
||||||
|
fs::write(
|
||||||
|
worker.join("session/artifacts/paste/payload"),
|
||||||
|
"unsupported",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
"secret" | "url" | "path" => {
|
||||||
|
let segment = fs::read_dir(worker.join("session/segments"))
|
||||||
|
.unwrap()
|
||||||
|
.next()
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.path();
|
||||||
|
let replacement = match change {
|
||||||
|
"secret" => "Bearer secret",
|
||||||
|
"url" => "https://old-host.invalid",
|
||||||
|
_ => "/old/host/path",
|
||||||
|
};
|
||||||
|
let text = fs::read_to_string(&segment)
|
||||||
|
.unwrap()
|
||||||
|
.replace("artificial fixture reply", replacement);
|
||||||
|
fs::write(segment, text).unwrap();
|
||||||
|
}
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
atomic_write_json(&worker.join(WORKER_FILE), &aggregate, "test aggregate").unwrap();
|
||||||
|
assert!(
|
||||||
|
StoppedWorkerTemplate::extract(&worker).is_err(),
|
||||||
|
"accepted {change}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let (_source, worker, _) = source();
|
||||||
|
let held =
|
||||||
|
acquire_runtime_store_owner_lock(worker.parent().unwrap().parent().unwrap()).unwrap();
|
||||||
|
assert!(StoppedWorkerTemplate::extract(&worker).is_err());
|
||||||
|
drop(held);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_v12_settled_template_installs_current_aggregate_without_rewriting_session() {
|
||||||
|
let (_source, _, mut template) = source();
|
||||||
|
template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION;
|
||||||
|
let bytes = serde_json::to_vec(&template).unwrap();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
template.install(&options, request(), &resolved()).unwrap();
|
||||||
|
assert_eq!(serde_json::to_vec(&template).unwrap(), bytes);
|
||||||
|
let worker = options.root.join(WORKERS_DIR).join(worker_id().to_string());
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(worker.join("session/session.json")).unwrap(),
|
||||||
|
template.session_manifest
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(worker.join(format!("session/segments/{}.jsonl", template.segment_id))).unwrap(),
|
||||||
|
template.segment_jsonl
|
||||||
|
);
|
||||||
|
let aggregate: WorkerAggregateRecord =
|
||||||
|
read_bounded_json(&worker.join(WORKER_FILE), "read installed fixture").unwrap();
|
||||||
|
let saved = aggregate.validate(&worker.join(WORKER_FILE)).unwrap();
|
||||||
|
assert_eq!(saved.status, WorkerStatus::Stopped);
|
||||||
|
assert!(matches!(
|
||||||
|
saved.execution_state,
|
||||||
|
PersistedWorkerExecutionState::Available(_)
|
||||||
|
));
|
||||||
|
// Unsupported template schemas do not enter the versioned reader.
|
||||||
|
template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION - 1;
|
||||||
|
assert!(template.validate_shape().is_err());
|
||||||
|
template.runtime_schema_version = SCHEMA_VERSION + 1;
|
||||||
|
assert!(template.validate_shape().is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fresh_manifest_and_memory_settings_are_required() {
|
||||||
|
let (_source, _, template) = source();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let mut manifest = resolved();
|
||||||
|
manifest.worker.name = "wrong-worker".into();
|
||||||
|
assert!(template.install(&options, request(), &manifest).is_err());
|
||||||
|
let mut binding = request();
|
||||||
|
binding.memory_settings = None;
|
||||||
|
assert!(template.install(&options, binding, &resolved()).is_err());
|
||||||
|
let mut binding = request();
|
||||||
|
binding.memory_settings.as_mut().unwrap().workspace_id = "other-workspace".into();
|
||||||
|
assert!(template.install(&options, binding, &resolved()).is_err());
|
||||||
|
assert_fresh(&options);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn settled_checkpoints_preserve_receipts_but_refuse_pending_work() {
|
||||||
|
let mut checkpoint = settled_checkpoint();
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_ok());
|
||||||
|
checkpoint.pending.push(1);
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||||
|
checkpoint.pending.clear();
|
||||||
|
checkpoint.activating = Some(1);
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||||
|
checkpoint.activating = None;
|
||||||
|
checkpoint.invocation_receipts.push(1);
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||||
|
checkpoint.invocation_receipts.clear();
|
||||||
|
checkpoint.invocation_recovery_blocked = true;
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deliberately corrupt only test-owned synthetic source files, never the lab.
|
||||||
|
fn assert_rejected_session_bytes(worker: &Path, template: &StoppedWorkerTemplate, bytes: Vec<u8>) {
|
||||||
|
let path = worker.join(format!("session/segments/{}.jsonl", template.segment_id()));
|
||||||
|
fs::write(path, &bytes).unwrap();
|
||||||
|
let extraction_error = StoppedWorkerTemplate::extract(worker)
|
||||||
|
.err()
|
||||||
|
.expect("invalid source accepted");
|
||||||
|
assert!(
|
||||||
|
!extraction_error
|
||||||
|
.to_string()
|
||||||
|
.contains("synthetic-discarded-material")
|
||||||
|
);
|
||||||
|
let (_target, options) = destination(template);
|
||||||
|
let mut invalid = template.clone();
|
||||||
|
invalid.segment_jsonl = bytes;
|
||||||
|
let installation_error = invalid
|
||||||
|
.install(&options, request(), &resolved())
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
!installation_error
|
||||||
|
.to_string()
|
||||||
|
.contains("synthetic-discarded-material")
|
||||||
|
);
|
||||||
|
assert_fresh(&options);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn synthetic_logs(template: &StoppedWorkerTemplate) -> Vec<LogEntry> {
|
||||||
|
template
|
||||||
|
.segment_jsonl
|
||||||
|
.split(|byte| *byte == b'\n')
|
||||||
|
.filter(|line| !line.is_empty())
|
||||||
|
.map(|line| serde_json::from_slice(line).unwrap())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encode_synthetic_logs(logs: &[LogEntry]) -> Vec<u8> {
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
for log in logs {
|
||||||
|
serde_json::to_writer(&mut bytes, log).unwrap();
|
||||||
|
bytes.push(b'\n');
|
||||||
|
}
|
||||||
|
bytes
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn missing_system_prompt_matches_worker_restore_rejection() {
|
||||||
|
let (_source, worker_dir, template) = source();
|
||||||
|
let mut logs = synthetic_logs(&template);
|
||||||
|
assert!(session_store::collect_state(&logs).system_prompt.is_some());
|
||||||
|
let LogEntry::AnnotatedSegmentStart { system_prompt, .. } = &mut logs[0] else {
|
||||||
|
panic!("missing seed")
|
||||||
|
};
|
||||||
|
*system_prompt = None;
|
||||||
|
assert_rejected_session_bytes(&worker_dir, &template, encode_synthetic_logs(&logs));
|
||||||
|
// The owning Worker constructor rejects this before model-client creation,
|
||||||
|
// allocation, or execution. Do not start a controller or submit any input.
|
||||||
|
let store = session_store::CombinedStore::new(
|
||||||
|
WorkerSessionStore::open_read_only(worker_dir.join("session")).unwrap(),
|
||||||
|
session_store::WorkerAggregateStore::open_read_only(&worker_dir, template.worker_name())
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let restored = worker::Worker::restore_from_manifest(
|
||||||
|
template.session_id(),
|
||||||
|
template.segment_id(),
|
||||||
|
resolved(),
|
||||||
|
store,
|
||||||
|
worker::PromptCatalogSource::builtins_only(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(matches!(
|
||||||
|
restored,
|
||||||
|
Err(worker::WorkerError::SegmentSystemPromptMissing { .. })
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn malformed_submission_and_notification_receipts_fail_extraction_and_install() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
for notification in [false, true] {
|
||||||
|
let mut checkpoint = settled_checkpoint();
|
||||||
|
if notification {
|
||||||
|
checkpoint.notification_receipts =
|
||||||
|
vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)];
|
||||||
|
} else {
|
||||||
|
checkpoint.receipts = vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)];
|
||||||
|
}
|
||||||
|
let mut logs = synthetic_logs(&template);
|
||||||
|
for log in &mut logs {
|
||||||
|
if let LogEntry::AnnotatedUserInput { extensions, .. } = log {
|
||||||
|
*extensions = vec![checkpoint_extension(&checkpoint)];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_fields_at_every_retained_record_depth_are_rejected_without_secret_diagnostics() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
let original = String::from_utf8(template.segment_jsonl.clone()).unwrap();
|
||||||
|
// These values are harmless synthetic markers, not lab/private material.
|
||||||
|
// The compatibility decoder normally discards many of these fields.
|
||||||
|
let locations = [
|
||||||
|
r#""kind":"annotated_segment_start""#,
|
||||||
|
r#""kind":"invoke""#,
|
||||||
|
r#""kind":"annotated_user_input""#,
|
||||||
|
r#""kind":"annotated_assistant_item""#,
|
||||||
|
r#""kind":"llm_usage""#,
|
||||||
|
r#""kind":"turn_end""#,
|
||||||
|
r#""kind":"run_completed""#,
|
||||||
|
r#""max_tokens":null"#,
|
||||||
|
r#""kind":"message","role":"assistant""#,
|
||||||
|
r#""kind":"text","text":"artificial fixture reply""#,
|
||||||
|
r#""kind":"text","content":"Harmless artificial fixture input.""#,
|
||||||
|
r#""entry_id":"fixed-entry-id""#,
|
||||||
|
r#""kind":"model_output""#,
|
||||||
|
r#""worker_id":"01a1657d-b762-7791-87ae-a49f28cadbce""#,
|
||||||
|
r#""domain":"worker.pending_activations.v1""#,
|
||||||
|
r#""next_activation_sequence":2"#,
|
||||||
|
r#""submission_request_id":"completed-submission-request""#,
|
||||||
|
r#""notification_request_id":"completed-notification-request""#,
|
||||||
|
];
|
||||||
|
for location in locations {
|
||||||
|
assert!(original.contains(location));
|
||||||
|
for extra in [
|
||||||
|
r#", "reasoning":{"text":"synthetic-discarded-material"}"#,
|
||||||
|
r#", "encrypted_content":"synthetic-discarded-material""#,
|
||||||
|
] {
|
||||||
|
let altered = original.replacen(location, &format!("{location}{extra}"), 1);
|
||||||
|
assert_rejected_session_bytes(&worker, &template, altered.into_bytes());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let duplicate = original.replacen(
|
||||||
|
r#""role":"assistant""#,
|
||||||
|
r#""role":"assistant","role":"assistant""#,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
assert_rejected_session_bytes(&worker, &template, duplicate.into_bytes());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn opaque_feature_payloads_cannot_bypass_strict_session_validation() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
let mut logs = synthetic_logs(&template);
|
||||||
|
for log in &mut logs {
|
||||||
|
if let LogEntry::AnnotatedUserInput { history, .. } = log {
|
||||||
|
if let session_store::LoggedItem::Message { role, .. } = &mut history[0].item {
|
||||||
|
*role = session_store::LoggedRole::System;
|
||||||
|
}
|
||||||
|
history[0].metadata.input_metadata = Some(
|
||||||
|
session_store::HistoryInputMetadata::new(
|
||||||
|
session_store::HistoryInputUsePolicy {
|
||||||
|
recording_material: session_store::InputEvidenceUse::Exclude,
|
||||||
|
memory_evidence: session_store::InputEvidenceUse::Exclude,
|
||||||
|
observation: session_store::InputObservationUse::Hidden,
|
||||||
|
compaction: session_store::InputCompactionUse::Retain,
|
||||||
|
},
|
||||||
|
Some(
|
||||||
|
session_store::FeatureInputPayload::encode(
|
||||||
|
"fixture.test",
|
||||||
|
1,
|
||||||
|
&BTreeMap::from([("reasoning", "harmless synthetic payload")]),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn refuses_source_and_destination_symlinks() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
std::os::unix::fs::symlink(worker.join(WORKER_FILE), options.root.join("linked")).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
fs::remove_file(options.root.join("linked")).unwrap();
|
||||||
|
std::os::unix::fs::symlink(worker.join(WORKER_FILE), worker.join("runs/link")).unwrap();
|
||||||
|
assert!(StoppedWorkerTemplate::extract(&worker).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opt-in only: source path supplied explicitly; assertions never display content.
|
||||||
|
#[test]
|
||||||
|
#[ignore = "requires the approved artificial lab aggregate via YOI_STOPPED_FIXTURE_SOURCE"]
|
||||||
|
fn approved_lab_extract_install() {
|
||||||
|
let path = PathBuf::from(
|
||||||
|
std::env::var_os("YOI_STOPPED_FIXTURE_SOURCE").expect("explicit source required"),
|
||||||
|
);
|
||||||
|
let template = StoppedWorkerTemplate::extract(&path).unwrap();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let mut bind = request();
|
||||||
|
bind.worker_id = template.worker_id();
|
||||||
|
bind.workspace_api.as_mut().unwrap().workspace_id = template.workspace_id().into();
|
||||||
|
bind.memory_settings.as_mut().unwrap().workspace_id = template.workspace_id().into();
|
||||||
|
let mut manifest = resolved();
|
||||||
|
manifest.worker.name = template.worker_name();
|
||||||
|
template.install(&options, bind, &manifest).unwrap();
|
||||||
|
let installed = options
|
||||||
|
.root
|
||||||
|
.join(WORKERS_DIR)
|
||||||
|
.join(template.worker_id().to_string());
|
||||||
|
for relative in [
|
||||||
|
"session/session.json".to_string(),
|
||||||
|
format!("session/segments/{}.jsonl", template.segment_id()),
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
fs::read(path.join(&relative)).unwrap() == fs::read(installed.join(relative)).unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let runtime = crate::Runtime::with_fs_store(options).unwrap();
|
||||||
|
assert_eq!(runtime.list_workers().unwrap().len(), 1);
|
||||||
|
assert_eq!(runtime.summary().unwrap().active_worker_count, 0);
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -66,13 +66,32 @@ impl DeferredWorkdirSession {
|
|||||||
let open = self.open.clone();
|
let open = self.open.clone();
|
||||||
let session = tokio::task::spawn_blocking(move || open())
|
let session = tokio::task::spawn_blocking(move || open())
|
||||||
.await
|
.await
|
||||||
.map_err(|_| WorkdirError::Unavailable("Workdir session opener failed".into()))?
|
.map_err(|_| {
|
||||||
.map_err(WorkdirError::Unavailable)?;
|
WorkdirError::Unavailable("Workdir session opener failed".into())
|
||||||
|
.with_failure_reason(WorkdirFailureReason::SessionOpenTaskFailed)
|
||||||
|
})?
|
||||||
|
.map_err(|_| {
|
||||||
|
// The opener's string contract cannot prove a more specific cause.
|
||||||
|
WorkdirError::Unavailable("Workdir session could not be opened".into())
|
||||||
|
.with_failure_reason(WorkdirFailureReason::SessionOpenFailed)
|
||||||
|
})?;
|
||||||
if session.workdir().id() != self.workdir.id() {
|
if session.workdir().id() != self.workdir.id() {
|
||||||
session.close().await?;
|
if let Err(error) = session.close().await {
|
||||||
return Err(WorkdirError::Unavailable(
|
// Retain a failed-close owner for explicit close; never use it.
|
||||||
"Workdir session identity mismatch".into(),
|
state.closed = true;
|
||||||
));
|
state.session = Some(session);
|
||||||
|
tracing::warn!(
|
||||||
|
target: "worker_runtime::workdir",
|
||||||
|
event = "workdir_session_identity_cleanup_failed",
|
||||||
|
failure_reason = error.failure_reason().map(WorkdirFailureReason::as_str),
|
||||||
|
denial_reason = error.denial_reason().map(WorkdirDenialReason::as_str),
|
||||||
|
"Mismatched Workdir session cleanup failed"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return Err(
|
||||||
|
WorkdirError::Unavailable("Workdir session identity mismatch".into())
|
||||||
|
.with_failure_reason(WorkdirFailureReason::SessionIdentityMismatch),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
if let Some(mut source) = session.subscribe_command_events() {
|
if let Some(mut source) = session.subscribe_command_events() {
|
||||||
let events = self.events.clone();
|
let events = self.events.clone();
|
||||||
@@ -105,6 +124,18 @@ macro_rules! impl_deferred_session {
|
|||||||
self.capabilities
|
self.capabilities
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn environment(&self) -> WorkdirSessionEnvironment {
|
||||||
|
let mut environment = self.state.try_lock().ok().and_then(|state| {
|
||||||
|
state.session.as_ref().map(|session| session.environment())
|
||||||
|
}).unwrap_or_else(|| {
|
||||||
|
let mut environment = WorkdirSessionEnvironment::root(self.capabilities);
|
||||||
|
environment.restrictions.push("Deferred provider: scope is checked on connection and every operation".into());
|
||||||
|
environment
|
||||||
|
});
|
||||||
|
environment.capabilities = self.capabilities;
|
||||||
|
environment
|
||||||
|
}
|
||||||
|
|
||||||
$(
|
$(
|
||||||
async fn $method(&self, $request: $request_type) -> Result<$result, WorkdirError> {
|
async fn $method(&self, $request: $request_type) -> Result<$result, WorkdirError> {
|
||||||
self.session().await?.$method($request).await
|
self.session().await?.$method($request).await
|
||||||
@@ -162,6 +193,21 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
fn short_tempdir() -> tempfile::TempDir {
|
||||||
|
let target = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.join("target");
|
||||||
|
std::fs::create_dir_all(&target).unwrap();
|
||||||
|
tempfile::Builder::new()
|
||||||
|
.prefix("d")
|
||||||
|
.rand_bytes(3)
|
||||||
|
.tempdir_in(target)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
fn read_request() -> ReadRequest {
|
fn read_request() -> ReadRequest {
|
||||||
ReadRequest {
|
ReadRequest {
|
||||||
path: WorkdirPath::new("README.md").unwrap(),
|
path: WorkdirPath::new("README.md").unwrap(),
|
||||||
@@ -194,9 +240,60 @@ mod tests {
|
|||||||
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn router_cleanup_of_unopened_session_never_connects_or_follows_reused_alias() {
|
||||||
|
let attempts = Arc::new(AtomicUsize::new(0));
|
||||||
|
let observed = attempts.clone();
|
||||||
|
let session = DeferredWorkdirSession::handle(
|
||||||
|
"old",
|
||||||
|
WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
move || {
|
||||||
|
observed.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Err("offline".into())
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let router = WorkdirSessionRouter::new();
|
||||||
|
let alias = WorkdirAttachmentAlias::new("w").unwrap();
|
||||||
|
let routed = router.attach(alias.clone(), session.clone()).unwrap();
|
||||||
|
assert!(routed.subscribe_command_events().is_some());
|
||||||
|
assert!(routed.command_snapshot().is_empty());
|
||||||
|
router.close_all().await.unwrap();
|
||||||
|
router.close_all().await.unwrap();
|
||||||
|
assert!(router.aliases().is_empty());
|
||||||
|
assert!(matches!(
|
||||||
|
session.read(read_request()).await,
|
||||||
|
Err(WorkdirError::SessionClosed)
|
||||||
|
));
|
||||||
|
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||||
|
|
||||||
|
let new_attempts = Arc::new(AtomicUsize::new(0));
|
||||||
|
let new_observed = new_attempts.clone();
|
||||||
|
let replacement = DeferredWorkdirSession::handle(
|
||||||
|
"new",
|
||||||
|
WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
move || {
|
||||||
|
new_observed.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Err("offline".into())
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let new_routed = router.attach(alias, replacement).unwrap();
|
||||||
|
// A retained old connection cannot open the replacement provider.
|
||||||
|
assert!(routed.read(read_request()).await.is_err());
|
||||||
|
assert_eq!(new_attempts.load(Ordering::SeqCst), 0);
|
||||||
|
assert!(matches!(
|
||||||
|
new_routed.read(read_request()).await,
|
||||||
|
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
|
||||||
|
));
|
||||||
|
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
|
||||||
|
// Even after a failed open, cleanup must not try connecting again.
|
||||||
|
router.close_all().await.unwrap();
|
||||||
|
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
|
||||||
|
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn failed_open_retries_on_use_and_keeps_read_only_capabilities() {
|
async fn failed_open_retries_on_use_and_keeps_read_only_capabilities() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = short_tempdir();
|
||||||
std::fs::write(temp.path().join("README.md"), "restored").unwrap();
|
std::fs::write(temp.path().join("README.md"), "restored").unwrap();
|
||||||
let root = temp.path().to_path_buf();
|
let root = temp.path().to_path_buf();
|
||||||
let attempts = Arc::new(AtomicUsize::new(0));
|
let attempts = Arc::new(AtomicUsize::new(0));
|
||||||
@@ -224,10 +321,16 @@ mod tests {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||||
|
let failed = session.read(read_request()).await.unwrap_err();
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
session.read(read_request()).await,
|
failed.classification_source(),
|
||||||
Err(WorkdirError::Unavailable(_))
|
WorkdirError::Unavailable(_)
|
||||||
));
|
));
|
||||||
|
assert_eq!(
|
||||||
|
failed.failure_reason(),
|
||||||
|
Some(WorkdirFailureReason::SessionOpenFailed)
|
||||||
|
);
|
||||||
|
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
session.read(read_request()).await.unwrap().bytes,
|
session.read(read_request()).await.unwrap().bytes,
|
||||||
b"restored"
|
b"restored"
|
||||||
@@ -253,4 +356,126 @@ mod tests {
|
|||||||
);
|
);
|
||||||
session.close().await.unwrap();
|
session.close().await.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn opener_task_and_open_errors_keep_distinct_safe_causes() {
|
||||||
|
for panic in [false, true] {
|
||||||
|
let attempts = Arc::new(AtomicUsize::new(0));
|
||||||
|
let observed = attempts.clone();
|
||||||
|
let session = DeferredWorkdirSession::handle(
|
||||||
|
"saved",
|
||||||
|
WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
move || {
|
||||||
|
observed.fetch_add(1, Ordering::SeqCst);
|
||||||
|
if panic {
|
||||||
|
panic!(
|
||||||
|
"/host/private bearer-secret command-secret proof-secret key-secret"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err("/host/private bearer-secret command-secret proof-secret key-secret".into())
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let error = session.read(read_request()).await.unwrap_err();
|
||||||
|
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||||
|
let reason = if panic {
|
||||||
|
WorkdirFailureReason::SessionOpenTaskFailed
|
||||||
|
} else {
|
||||||
|
WorkdirFailureReason::SessionOpenFailed
|
||||||
|
};
|
||||||
|
assert_eq!(error.failure_reason(), Some(reason));
|
||||||
|
assert_eq!(reason.stage().as_str(), "session_open");
|
||||||
|
let transport = workdir::http::WorkdirTransportError::from_workdir_error(&error);
|
||||||
|
assert_eq!(transport.code.http_status(), 503);
|
||||||
|
assert!(
|
||||||
|
!serde_json::to_string(&transport)
|
||||||
|
.unwrap()
|
||||||
|
.contains("secret")
|
||||||
|
);
|
||||||
|
assert!(!format!("{error:?}").contains("secret"));
|
||||||
|
session.close().await.unwrap();
|
||||||
|
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct MismatchedSession {
|
||||||
|
workdir: Workdir,
|
||||||
|
closes: Arc<AtomicUsize>,
|
||||||
|
fail_close: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! mismatched_session {
|
||||||
|
($( $method:ident($request:ident: $request_type:ty) -> $result:ty; )*) => {
|
||||||
|
#[async_trait]
|
||||||
|
impl WorkdirSession for MismatchedSession {
|
||||||
|
fn workdir(&self) -> &Workdir { &self.workdir }
|
||||||
|
fn capabilities(&self) -> WorkdirSessionCapabilities { WorkdirSessionCapabilities::READ_ONLY }
|
||||||
|
$(async fn $method(&self, _: $request_type) -> Result<$result, WorkdirError> {
|
||||||
|
panic!("mismatched session must never be used");
|
||||||
|
})*
|
||||||
|
async fn close(&self) -> Result<(), WorkdirError> {
|
||||||
|
let attempt = self.closes.fetch_add(1, Ordering::SeqCst);
|
||||||
|
if self.fail_close && attempt == 0 {
|
||||||
|
Err(WorkdirError::Transport("/host/private key-secret".into())
|
||||||
|
.with_failure_reason(WorkdirFailureReason::HttpTransfer))
|
||||||
|
} else { Ok(()) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
mismatched_session! {
|
||||||
|
stat(request: StatRequest) -> StatResult;
|
||||||
|
read(request: ReadRequest) -> ReadResult;
|
||||||
|
write(request: WriteRequest) -> WriteResult;
|
||||||
|
edit(request: EditRequest) -> EditResult;
|
||||||
|
list(request: ListRequest) -> ListResult;
|
||||||
|
glob(request: GlobRequest) -> GlobResult;
|
||||||
|
grep(request: GrepRequest) -> GrepResult;
|
||||||
|
start_command(request: CommandRequest) -> CommandHandle;
|
||||||
|
command_status(handle: CommandHandle) -> CommandStatus;
|
||||||
|
command_output(request: CommandOutputRequest) -> CommandOutput;
|
||||||
|
cancel_command(handle: CommandHandle) -> ();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn identity_mismatch_preserves_primary_cause_and_failed_close_owner() {
|
||||||
|
for fail_close in [false, true] {
|
||||||
|
let closes = Arc::new(AtomicUsize::new(0));
|
||||||
|
let raw: WorkdirSessionHandle = Arc::new(MismatchedSession {
|
||||||
|
workdir: Workdir::new("wrong-secret"),
|
||||||
|
closes: closes.clone(),
|
||||||
|
fail_close,
|
||||||
|
});
|
||||||
|
let attempts = Arc::new(AtomicUsize::new(0));
|
||||||
|
let observed = attempts.clone();
|
||||||
|
let session = DeferredWorkdirSession::handle(
|
||||||
|
"saved",
|
||||||
|
WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
move || {
|
||||||
|
observed.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Ok(raw.clone())
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let error = session.read(read_request()).await.unwrap_err();
|
||||||
|
assert_eq!(
|
||||||
|
error.failure_reason(),
|
||||||
|
Some(WorkdirFailureReason::SessionIdentityMismatch)
|
||||||
|
);
|
||||||
|
assert!(!format!("{error:?}").contains("secret"));
|
||||||
|
assert_eq!(closes.load(Ordering::SeqCst), 1);
|
||||||
|
if fail_close {
|
||||||
|
assert!(matches!(
|
||||||
|
session.read(read_request()).await,
|
||||||
|
Err(WorkdirError::SessionClosed)
|
||||||
|
));
|
||||||
|
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||||
|
}
|
||||||
|
session.close().await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
closes.load(Ordering::SeqCst),
|
||||||
|
if fail_close { 2 } else { 1 }
|
||||||
|
);
|
||||||
|
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,185 @@
|
|||||||
|
//! Explicit, operation-owned repair of saved Session material. This is not
|
||||||
|
//! read-only candidate preparation: the canonical Session/attribution is the
|
||||||
|
//! receipt for replay after a lost response, and effects are reported as such.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
impl ProfileRuntimeWorkerFactory {
|
||||||
|
pub(super) async fn recover_saved_restore_source(
|
||||||
|
&self,
|
||||||
|
request: &WorkerExecutionRestoreRequest,
|
||||||
|
) -> Result<bool, RuntimeRestoreFailure> {
|
||||||
|
let cause = |cause| RuntimeRestoreFailure::started(RuntimeRestoreStage::Recovery, cause);
|
||||||
|
let name = Self::runtime_worker_name_for_ref(&request.worker_ref);
|
||||||
|
let root = self
|
||||||
|
.worker_aggregate_dir(&request.worker_ref)
|
||||||
|
.map_err(|_| cause("restore recovery aggregate unavailable"))?;
|
||||||
|
let metadata_store = WorkerAggregateStore::open_read_only(&root, &name)
|
||||||
|
.map_err(|_| cause("restore recovery metadata open failed"))?;
|
||||||
|
let metadata = metadata_store
|
||||||
|
.read_by_name(&name)
|
||||||
|
.map_err(|_| cause("restore recovery metadata read failed"))?
|
||||||
|
.ok_or_else(|| cause("restore recovery metadata missing"))?;
|
||||||
|
let active = metadata
|
||||||
|
.active
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| cause("restore recovery Session missing"))?;
|
||||||
|
let pending = active.segment_id.is_none();
|
||||||
|
let incomplete = request.request.subjektiv_attached
|
||||||
|
&& !matches!(
|
||||||
|
metadata.subjektiv_session_attribution,
|
||||||
|
Some(session_store::SubjektivSessionAttributionState::Confirmed { .. })
|
||||||
|
);
|
||||||
|
if !pending && !incomplete {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
request
|
||||||
|
.context
|
||||||
|
.authorize_restore_source_repair(request.operation_id)
|
||||||
|
.map_err(|_| cause("canonical source repair authority is not durably confirmed"))?;
|
||||||
|
// The accepted Runtime operation already owns this Session. Its stable
|
||||||
|
// Session identity and sole committed seed, not a new restore request,
|
||||||
|
// are used to reconcile an interrupted first-head publication.
|
||||||
|
let session_store = WorkerSessionStore::new_with_retention(
|
||||||
|
root.join("session"),
|
||||||
|
self.session_retention()
|
||||||
|
.map_err(|_| cause("restore recovery retention unavailable"))?,
|
||||||
|
)
|
||||||
|
.map_err(|_| cause("restore recovery Session open failed"))?;
|
||||||
|
let (manifest, mut loader) = Self::manifest_for_restore(&metadata, &request.request)
|
||||||
|
.map_err(|_| cause("restore recovery saved manifest invalid"))?;
|
||||||
|
let workspace = RuntimeWorkspaceBackendRef::from_worker_request(
|
||||||
|
&request.request,
|
||||||
|
self.runtime_id.as_deref(),
|
||||||
|
)
|
||||||
|
.map_err(cause)?;
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
let request_client = request
|
||||||
|
.request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|api| self.workspace_request_clients.get(&api.workspace_id));
|
||||||
|
let context = workspace.worker_context(
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
&request.worker_ref,
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
request.workspace_scope.as_ref(),
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
self.worker_mutation_identity.as_ref(),
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
request_client,
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
self.embedded_worker_mutation_dispatcher.as_ref(),
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
Some(self.prompt_projection_cache.clone()),
|
||||||
|
);
|
||||||
|
if pending && context.workspace_id().is_some() {
|
||||||
|
let bundle = request
|
||||||
|
.config_bundle
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| cause("pending recovery launch material missing"))?;
|
||||||
|
let projection = self
|
||||||
|
.observe_bundle_prompt_projection(
|
||||||
|
bundle,
|
||||||
|
request
|
||||||
|
.request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.map(|api| api.workspace_id.as_str()),
|
||||||
|
)
|
||||||
|
.map_err(|_| cause("pending recovery Prompt projection invalid"))?
|
||||||
|
.ok_or_else(|| cause("pending recovery Prompt projection missing"))?;
|
||||||
|
loader = loader.with_effective_catalog(projection.projection.catalog.clone());
|
||||||
|
}
|
||||||
|
self.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.insert(
|
||||||
|
request.worker_ref.clone(),
|
||||||
|
FailedRestoreResources {
|
||||||
|
operation_id: request.operation_id,
|
||||||
|
sessions: Vec::new(),
|
||||||
|
candidate_workdir_cleanup: None,
|
||||||
|
allocation: None,
|
||||||
|
feature_tasks: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let store = CombinedStore::new(session_store, metadata_store);
|
||||||
|
let worker_result = Worker::restore_pending_from_worker_metadata_with_context(
|
||||||
|
&name,
|
||||||
|
manifest,
|
||||||
|
store,
|
||||||
|
loader,
|
||||||
|
context,
|
||||||
|
WorkerFilesystemAuthority::None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let mut worker = worker_result.map_err(|error| {
|
||||||
|
self.retain_construction_failure(request, &error);
|
||||||
|
cause(restore_worker_cause(&error))
|
||||||
|
})?;
|
||||||
|
let tasks = worker.retain_startup_feature_cleanup();
|
||||||
|
let sessions = worker.workdir_sessions();
|
||||||
|
let allocation = worker.restore_allocation_cleanup_handle();
|
||||||
|
self.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.insert(
|
||||||
|
request.worker_ref.clone(),
|
||||||
|
FailedRestoreResources {
|
||||||
|
operation_id: request.operation_id,
|
||||||
|
sessions: vec![sessions.clone()],
|
||||||
|
candidate_workdir_cleanup: None,
|
||||||
|
allocation: allocation.clone(),
|
||||||
|
feature_tasks: Some(tasks.clone()),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let result = async {
|
||||||
|
if let Some(binding) = request.request.backend_job.as_ref() {
|
||||||
|
worker
|
||||||
|
.bind_backend_job(binding.clone())
|
||||||
|
.map_err(|_| cause("recovery Backend Job binding failed"))?;
|
||||||
|
}
|
||||||
|
worker
|
||||||
|
.finalize_subjektiv_session_attribution(
|
||||||
|
request.request.subjektiv_attached,
|
||||||
|
SubjektivSessionAttributionLifecycle::RestoredSession,
|
||||||
|
)
|
||||||
|
.map_err(|error| cause(restore_worker_cause(&error)))?;
|
||||||
|
if pending {
|
||||||
|
worker::WorkerController::recover_pending_head(&mut worker, &root)
|
||||||
|
.await
|
||||||
|
.map_err(|_| cause("pending Session head recovery failed"))?;
|
||||||
|
}
|
||||||
|
Ok::<_, RuntimeRestoreFailure>(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
// Never release allocation while the Worker guard is attached. Retained
|
||||||
|
// handles in the same operation remain available on every cleanup error.
|
||||||
|
drop(worker);
|
||||||
|
let cleanup = async {
|
||||||
|
tasks
|
||||||
|
.shutdown()
|
||||||
|
.await
|
||||||
|
.map_err(|_| cause("recovery Feature settlement failed"))?;
|
||||||
|
sessions
|
||||||
|
.close_all()
|
||||||
|
.await
|
||||||
|
.map_err(|_| cause("recovery Workdir close failed"))?;
|
||||||
|
if let Some(allocation) = allocation {
|
||||||
|
allocation
|
||||||
|
.release()
|
||||||
|
.map_err(|_| cause("recovery allocation release failed"))?;
|
||||||
|
}
|
||||||
|
Ok::<_, RuntimeRestoreFailure>(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
result?;
|
||||||
|
cleanup?;
|
||||||
|
self.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.remove(&request.worker_ref);
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user