Compare commits
68
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2d7da04502 | ||
|
|
b4099ddda4 | ||
|
|
04410a5022 | ||
|
|
682747e9f7 | ||
|
|
00f327da4b | ||
|
|
aa7bb6282d | ||
|
|
33d7ef8fa0 | ||
|
|
0ea6308dae | ||
|
|
e7351c16b0 | ||
|
|
00d96b7163 | ||
|
|
2b4bf4b7bd | ||
|
|
e7ebc92f83 | ||
|
|
181b0d637e | ||
|
|
b0733e53b2 | ||
|
|
618880d38d | ||
|
|
79b76c8789 | ||
|
|
897c025b85 | ||
|
|
5b12c3e085 | ||
|
|
623c31615d | ||
|
|
af719e387d | ||
|
|
dddb21bd70 | ||
|
|
e8d4e964cb | ||
|
|
eb24cb04dc | ||
|
|
e84057f402 | ||
|
|
15c3006d09 | ||
|
|
b3209c2012 | ||
|
|
6f5a424661 | ||
|
|
8d47d2670b | ||
|
|
b19de45a82 | ||
|
|
1e88893ef9 | ||
|
|
1fd2e09971 | ||
|
|
f85d831104 | ||
|
|
2b24db5a15 | ||
|
|
e1945d977c | ||
|
|
a10875d68e | ||
|
|
f35dcef4d0 | ||
|
|
f5ebae1134 | ||
|
|
6be4e8388b | ||
|
|
9e3f87ac52 | ||
|
|
ceb01f6eaa | ||
|
|
1b13563e19 | ||
|
|
848a1e9a5b | ||
|
|
a77453338c | ||
|
|
d12bc2ea0b | ||
|
|
a1df3742b1 | ||
|
|
d8818b776b | ||
|
|
40c36ccb1d | ||
|
|
80c151d084 | ||
|
|
c6a8759e7b | ||
|
|
f17b5212c7 | ||
|
|
85f70d4bc1 | ||
|
|
b284934e34 | ||
|
|
2d33321fab | ||
|
|
0975471986 | ||
|
|
214d6e21dd | ||
|
|
fca5c23345 | ||
|
|
f9b1500ba2 | ||
|
|
931bf3b27d | ||
|
|
6bb671be21 | ||
|
|
b8f2fa479c | ||
|
|
1639be9841 | ||
|
|
907392cce8 | ||
|
|
6776973e4c | ||
|
|
73a3e674c9 | ||
|
|
994e75233b | ||
|
|
b5c105545c | ||
|
|
61c20b0995 | ||
|
|
b7a22146a0 |
Generated
+9
@@ -3990,6 +3990,7 @@ dependencies = [
|
||||
"protocol",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"session-store",
|
||||
"tokio",
|
||||
"workdir",
|
||||
]
|
||||
@@ -4783,12 +4784,14 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"feature-storage",
|
||||
"job",
|
||||
"manifest",
|
||||
"memory",
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"server-api",
|
||||
"session-store",
|
||||
"sha2 0.11.0",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
@@ -6709,15 +6712,21 @@ name = "yoi-e2e"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"config-source",
|
||||
"libc",
|
||||
"manifest",
|
||||
"protocol",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"server-api",
|
||||
"session-store",
|
||||
"sha2 0.11.0",
|
||||
"tempfile",
|
||||
"tungstenite 0.29.0",
|
||||
"webauthn-authenticator-rs",
|
||||
"webauthn-rs-proto",
|
||||
"worker-runtime",
|
||||
"yoi-workspace-server",
|
||||
]
|
||||
|
||||
|
||||
@@ -72,6 +72,41 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compiled_handoff_uses_host_owned_retry_identity() {
|
||||
let flow = builtin_flow_source(CODER_REVIEW_FLOW_SLUG)
|
||||
.unwrap()
|
||||
.compile()
|
||||
.unwrap();
|
||||
let complete = flow
|
||||
.state(&crate::StateId::new("complete").unwrap())
|
||||
.unwrap();
|
||||
assert!(
|
||||
complete
|
||||
.instructions
|
||||
.contains("The Host owns transport retry identity")
|
||||
);
|
||||
assert!(
|
||||
complete
|
||||
.instructions
|
||||
.contains("do not supply operation_key")
|
||||
);
|
||||
assert!(!complete.instructions.contains("same key and full input"));
|
||||
let fix = flow.state(&crate::StateId::new("fix").unwrap()).unwrap();
|
||||
assert!(
|
||||
fix.instructions
|
||||
.contains("rerun validation affected by the fixes")
|
||||
);
|
||||
assert!(
|
||||
fix.instructions
|
||||
.contains("Cite reusable evidence for unaffected scope")
|
||||
);
|
||||
assert!(
|
||||
fix.instructions
|
||||
.contains("Source movement invalidates the prior verdict")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_builtin_flow_compiles_and_matches_catalog_identity() {
|
||||
assert!(!builtin_flow_sources().is_empty());
|
||||
|
||||
+171
-70
@@ -617,7 +617,8 @@ impl fmt::Display for McpRequestStage {
|
||||
}
|
||||
}
|
||||
|
||||
/// A running initialized stdio MCP client.
|
||||
/// An owned stdio MCP process, initialized by `connect` or explicitly after
|
||||
/// `spawn_uninitialized` when startup cleanup must retain it before any await.
|
||||
pub struct McpStdioClient {
|
||||
server_name: String,
|
||||
limits: McpStdioLimits,
|
||||
@@ -636,6 +637,9 @@ pub struct McpStdioClient {
|
||||
|
||||
impl McpStdioClient {
|
||||
/// Spawn, initialize, negotiate capabilities, and send notifications/initialized.
|
||||
/// Initialization errors attempt shutdown, but this convenience API cannot
|
||||
/// return an unreaped client on failure/cancellation. Its error is not proof
|
||||
/// of cleanup; use `spawn_uninitialized` with retained ownership when needed.
|
||||
pub async fn connect(
|
||||
spec: McpStdioServerSpec,
|
||||
limits: McpStdioLimits,
|
||||
@@ -667,6 +671,21 @@ impl McpStdioClient {
|
||||
async fn spawn(
|
||||
spec: McpStdioServerSpec,
|
||||
limits: McpStdioLimits,
|
||||
) -> Result<Self, McpClientError> {
|
||||
Self::spawn_uninitialized(spec, limits)
|
||||
}
|
||||
|
||||
/// Spawn without awaiting the handshake, so a cleanup owner can retain the
|
||||
/// client before initialization can fail or be cancelled. Call
|
||||
/// `initialize_with_timeout` before discovery; on error/cancellation, shut
|
||||
/// down this same client rather than spawning a replacement.
|
||||
///
|
||||
/// Once the process spawns, this returns its owner even if an expected pipe
|
||||
/// is missing. Initialization then fails, but cleanup can still reap the
|
||||
/// actual child rather than relying on kill-on-drop for a post-spawn error.
|
||||
pub fn spawn_uninitialized(
|
||||
spec: McpStdioServerSpec,
|
||||
limits: McpStdioLimits,
|
||||
) -> Result<Self, McpClientError> {
|
||||
let redactor = spec.redactor();
|
||||
let mut command = Command::new(&spec.command);
|
||||
@@ -688,29 +707,23 @@ impl McpStdioClient {
|
||||
McpErrorKind::Io(redactor.redact(&err.to_string())),
|
||||
)
|
||||
})?;
|
||||
let stdin = child.stdin.take().ok_or_else(|| {
|
||||
McpClientError::new(
|
||||
&spec.name,
|
||||
McpPhase::Spawn,
|
||||
McpErrorKind::Protocol("child stdin was not piped".into()),
|
||||
)
|
||||
})?;
|
||||
let stdout = child.stdout.take().ok_or_else(|| {
|
||||
McpClientError::new(
|
||||
&spec.name,
|
||||
McpPhase::Spawn,
|
||||
McpErrorKind::Protocol("child stdout was not piped".into()),
|
||||
)
|
||||
})?;
|
||||
let stderr = child.stderr.take().ok_or_else(|| {
|
||||
McpClientError::new(
|
||||
&spec.name,
|
||||
McpPhase::Spawn,
|
||||
McpErrorKind::Protocol("child stderr was not piped".into()),
|
||||
)
|
||||
})?;
|
||||
let stdin = child.stdin.take();
|
||||
let stdout = child.stdout.take();
|
||||
let stderr = child.stderr.take();
|
||||
let missing_pipe = if stdin.is_none() {
|
||||
Some("stdin")
|
||||
} else if stdout.is_none() {
|
||||
Some("stdout")
|
||||
} else if stderr.is_none() {
|
||||
Some("stderr")
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let stdin: SharedStdin = Arc::new(Mutex::new(Some(Box::new(stdin))));
|
||||
let stdin: SharedStdin =
|
||||
Arc::new(Mutex::new(stdin.map(|stdin| {
|
||||
Box::new(stdin) as Box<dyn AsyncWrite + Send + Unpin>
|
||||
})));
|
||||
let diagnostics = Arc::new(Mutex::new(BoundedDiagnostics::new(
|
||||
spec.name.clone(),
|
||||
limits.max_diagnostic_lines,
|
||||
@@ -718,7 +731,18 @@ impl McpStdioClient {
|
||||
)));
|
||||
let list_changes = Arc::new(Mutex::new(BoundedListChanged::new(spec.name.clone())));
|
||||
let (tx, rx) = mpsc::channel(16);
|
||||
let reader_task = spawn_stdout_reader(
|
||||
// Do not return an error that discards an already spawned child. Queue
|
||||
// setup failure before reader events so initialization cannot succeed.
|
||||
if let Some(pipe) = missing_pipe {
|
||||
tx.try_send(ReaderEvent::Error(McpClientError::new(
|
||||
&spec.name,
|
||||
McpPhase::Spawn,
|
||||
McpErrorKind::Protocol(format!("child {pipe} was not piped")),
|
||||
)))
|
||||
.expect("new response channel has capacity");
|
||||
}
|
||||
let reader_task = match stdout {
|
||||
Some(stdout) => spawn_stdout_reader(
|
||||
spec.name.clone(),
|
||||
stdout,
|
||||
stdin.clone(),
|
||||
@@ -726,8 +750,13 @@ impl McpStdioClient {
|
||||
limits.clone(),
|
||||
redactor.clone(),
|
||||
list_changes.clone(),
|
||||
);
|
||||
let stderr_task = spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone());
|
||||
),
|
||||
None => tokio::spawn(async {}),
|
||||
};
|
||||
let stderr_task = match stderr {
|
||||
Some(stderr) => spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone()),
|
||||
None => tokio::spawn(async {}),
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
server_name: spec.name,
|
||||
@@ -746,6 +775,25 @@ impl McpStdioClient {
|
||||
})
|
||||
}
|
||||
|
||||
/// Perform the bounded handshake on an already retained client. Failure or
|
||||
/// cancellation does not drop or shut down the client: its owner must do so.
|
||||
/// Do not retry initialization on a partially initialized connection.
|
||||
pub async fn initialize_with_timeout(&mut self) -> Result<(), McpClientError> {
|
||||
let started = Instant::now();
|
||||
match timeout(self.limits.startup_timeout, self.initialize()).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(McpClientError::new(
|
||||
&self.server_name,
|
||||
McpPhase::Initialize,
|
||||
McpErrorKind::Timeout {
|
||||
operation: "startup".to_string(),
|
||||
elapsed: started.elapsed(),
|
||||
},
|
||||
)
|
||||
.with_diagnostics(self.snapshot_diagnostics().await)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn initialize(&mut self) -> Result<(), McpClientError> {
|
||||
let result: InitializeResult = self
|
||||
.request(
|
||||
@@ -1080,6 +1128,8 @@ impl McpStdioClient {
|
||||
}
|
||||
|
||||
/// Close stdin and wait for process exit, falling back to terminate and kill.
|
||||
/// Errors and cancellation retain the child for another attempt on this
|
||||
/// client. Only confirmed reaping releases the process handle.
|
||||
pub async fn shutdown(&mut self) -> Result<ShutdownReport, McpClientError> {
|
||||
self.shutdown_started = true;
|
||||
let deadline = Instant::now() + self.limits.shutdown_timeout;
|
||||
@@ -1091,64 +1141,55 @@ impl McpStdioClient {
|
||||
.await
|
||||
.is_ok();
|
||||
|
||||
let mut child = match self.child.take() {
|
||||
Some(child) => child,
|
||||
None => return Ok(ShutdownReport::already_finished()),
|
||||
};
|
||||
// Keep the actual process handle in the client across every await and
|
||||
// error. Taking it before wait loses retry/reaping authority on cancel.
|
||||
if self.child.is_none() {
|
||||
return Ok(ShutdownReport::already_finished());
|
||||
}
|
||||
if !closed {
|
||||
self.reader_task.abort();
|
||||
return self.terminate_then_kill(child).await;
|
||||
return self.terminate_then_kill().await;
|
||||
}
|
||||
|
||||
let child = self.child.as_mut().expect("checked child");
|
||||
if let Ok(Some(status)) = child.try_wait() {
|
||||
self.reader_task.abort();
|
||||
self.stderr_task.abort();
|
||||
return Ok(ShutdownReport {
|
||||
exit_status: Some(status),
|
||||
terminated: false,
|
||||
killed: false,
|
||||
});
|
||||
return Ok(self.finish_shutdown(status, false, false));
|
||||
}
|
||||
|
||||
match timeout_at(deadline, child.wait()).await {
|
||||
Ok(Ok(status)) => {
|
||||
self.reader_task.abort();
|
||||
self.stderr_task.abort();
|
||||
Ok(ShutdownReport {
|
||||
exit_status: Some(status),
|
||||
terminated: false,
|
||||
killed: false,
|
||||
})
|
||||
}
|
||||
Ok(Ok(status)) => Ok(self.finish_shutdown(status, false, false)),
|
||||
Ok(Err(err)) => Err(McpClientError::new(
|
||||
&self.server_name,
|
||||
McpPhase::Shutdown,
|
||||
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
||||
)
|
||||
.with_diagnostics(self.snapshot_diagnostics().await)),
|
||||
Err(_) => self.terminate_then_kill(child).await,
|
||||
Err(_) => self.terminate_then_kill().await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn terminate_then_kill(
|
||||
fn finish_shutdown(
|
||||
&mut self,
|
||||
mut child: Child,
|
||||
) -> Result<ShutdownReport, McpClientError> {
|
||||
let mut terminated = false;
|
||||
let mut killed = false;
|
||||
if send_terminate(&mut child).is_ok() {
|
||||
terminated = true;
|
||||
}
|
||||
match timeout(self.limits.kill_timeout, child.wait()).await {
|
||||
Ok(Ok(status)) => {
|
||||
status: ExitStatus,
|
||||
terminated: bool,
|
||||
killed: bool,
|
||||
) -> ShutdownReport {
|
||||
// No await separates confirmed reaping from releasing the handle.
|
||||
drop(self.child.take());
|
||||
self.reader_task.abort();
|
||||
self.stderr_task.abort();
|
||||
Ok(ShutdownReport {
|
||||
ShutdownReport {
|
||||
exit_status: Some(status),
|
||||
terminated,
|
||||
killed,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn terminate_then_kill(&mut self) -> Result<ShutdownReport, McpClientError> {
|
||||
let child = self.child.as_mut().expect("shutdown retains child");
|
||||
let terminated = send_terminate(child).is_ok();
|
||||
match timeout(self.limits.kill_timeout, child.wait()).await {
|
||||
Ok(Ok(status)) => Ok(self.finish_shutdown(status, terminated, false)),
|
||||
Ok(Err(err)) => Err(McpClientError::new(
|
||||
&self.server_name,
|
||||
McpPhase::Shutdown,
|
||||
@@ -1163,7 +1204,6 @@ impl McpStdioClient {
|
||||
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
||||
)
|
||||
})?;
|
||||
killed = true;
|
||||
let status = timeout(self.limits.kill_timeout, child.wait())
|
||||
.await
|
||||
.map_err(|_| {
|
||||
@@ -1183,13 +1223,7 @@ impl McpStdioClient {
|
||||
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
||||
)
|
||||
})?;
|
||||
self.reader_task.abort();
|
||||
self.stderr_task.abort();
|
||||
Ok(ShutdownReport {
|
||||
exit_status: Some(status),
|
||||
terminated,
|
||||
killed,
|
||||
})
|
||||
Ok(self.finish_shutdown(status, terminated, true))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1197,11 +1231,11 @@ impl McpStdioClient {
|
||||
|
||||
impl Drop for McpStdioClient {
|
||||
fn drop(&mut self) {
|
||||
if !self.shutdown_started {
|
||||
// An attempted shutdown is not proof of exit. Errors and cancellation
|
||||
// leave the child here so Drop must still make a best-effort kill.
|
||||
if let Some(child) = &mut self.child {
|
||||
let _ = child.start_kill();
|
||||
}
|
||||
}
|
||||
self.reader_task.abort();
|
||||
self.stderr_task.abort();
|
||||
}
|
||||
@@ -1784,3 +1818,70 @@ fn send_terminate(child: &mut Child) -> Result<(), ()> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod deadline_tests;
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod shutdown_retention_tests {
|
||||
//! The real client and OS process are used; only the external server is a
|
||||
//! local shell that acknowledges initialization and ignores TERM/EOF.
|
||||
use super::*;
|
||||
use std::future::{Future, poll_fn};
|
||||
use std::task::Poll;
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_graceful_or_terminate_wait_retries_reaping_the_same_child() {
|
||||
for cancel_in_fallback in [false, true] {
|
||||
let script = r#"trap '' TERM
|
||||
read init || exit 1
|
||||
printf '%s\n' '{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-11-25","capabilities":{},"serverInfo":{"name":"retained","version":"1"}}}'
|
||||
while :; do :; done
|
||||
"#;
|
||||
let mut client = McpStdioClient::connect(
|
||||
McpStdioServerSpec::new("retained", "/bin/sh").args(["-c", script]),
|
||||
McpStdioLimits {
|
||||
shutdown_timeout: if cancel_in_fallback {
|
||||
Duration::ZERO
|
||||
} else {
|
||||
Duration::from_secs(30)
|
||||
},
|
||||
kill_timeout: Duration::from_secs(30),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("initialize after installing TERM handler");
|
||||
let pid = client.child.as_ref().unwrap().id();
|
||||
let mut shutdown = Box::pin(client.shutdown());
|
||||
poll_fn(|cx| {
|
||||
assert!(shutdown.as_mut().poll(cx).is_pending());
|
||||
Poll::Ready(())
|
||||
})
|
||||
.await;
|
||||
drop(shutdown);
|
||||
assert_eq!(client.child.as_ref().and_then(Child::id), pid);
|
||||
assert!(client.shutdown_started);
|
||||
|
||||
client.limits.shutdown_timeout = Duration::from_millis(20);
|
||||
client.limits.kill_timeout = Duration::from_millis(50);
|
||||
let report = timeout(Duration::from_secs(2), client.shutdown())
|
||||
.await
|
||||
.expect("retry bounded")
|
||||
.expect("retry reaps original child");
|
||||
assert!(
|
||||
report.exit_status.is_some(),
|
||||
"not an already-finished receipt"
|
||||
);
|
||||
assert!(report.killed, "TERM was ignored");
|
||||
assert!(client.child.is_none());
|
||||
let mut status = 0;
|
||||
assert_eq!(
|
||||
unsafe { libc::waitpid(pid.unwrap() as libc::pid_t, &mut status, libc::WNOHANG) },
|
||||
-1
|
||||
);
|
||||
assert_eq!(
|
||||
std::io::Error::last_os_error().raw_os_error(),
|
||||
Some(libc::ECHILD),
|
||||
"shutdown, not a later observer, reaped the original process"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -210,12 +210,7 @@ impl MergeRequestThreadEvent {
|
||||
}
|
||||
fn bound_bodies(&mut self) {
|
||||
match self {
|
||||
Self::Review(value) => {
|
||||
truncate_body(&mut value.body);
|
||||
for finding in &mut value.findings {
|
||||
truncate_body(&mut finding.body);
|
||||
}
|
||||
}
|
||||
Self::Review(_) => {}
|
||||
Self::ReviewRevoked(value) => truncate_body(&mut value.reason),
|
||||
Self::ReviewCancelled(value) => truncate_body(&mut value.reason),
|
||||
Self::Comment(value) => truncate_body(&mut value.body),
|
||||
@@ -939,10 +934,6 @@ impl MergeRequestStore {
|
||||
&self,
|
||||
i: SubmitMergeRequestReview,
|
||||
) -> Result<ReviewEvent, MergeRequestError> {
|
||||
bounded_body("review body", &i.body)?;
|
||||
for finding in &i.findings {
|
||||
bounded_body("review finding", &finding.body)?;
|
||||
}
|
||||
let mut c = self.lock()?;
|
||||
let t = c.transaction()?;
|
||||
let g: Option<(String, String, String, String, String, String)> = t
|
||||
@@ -1747,7 +1738,23 @@ fn ticket_event(
|
||||
) -> Result<(), MergeRequestError> {
|
||||
let ticket = &mr.ticket_ids[0];
|
||||
let n:i64=t.query_row("SELECT COALESCE(MAX(event_index),-1)+1 FROM typed_ticket_events WHERE workspace_id=?1 AND ticket_id=?2",params![mr.workspace_id,ticket],|r|r.get(0))?;
|
||||
t.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,author,at,from_state,to_state,heading,body)VALUES(?1,?2,?3,'comment',?4,?5,NULL,NULL,'Merge Request integrated',?6)",params![mr.workspace_id,ticket,n,format!("worker:{}:{}",e.merged_by.runtime_id,e.merged_by.worker_id),e.created_at.to_rfc3339(),format!("Merge Request `{}` integrated approved source ref `{}`; Ticket remains in progress until guarded Ticket completion.",mr.merge_request_id,e.approved_source_ref)])?;
|
||||
t.execute(
|
||||
"INSERT INTO typed_ticket_events(
|
||||
workspace_id,ticket_id,event_index,kind,author,at,from_state,to_state,heading,body,event_ref
|
||||
) VALUES(?1,?2,?3,'comment',?4,?5,NULL,NULL,'Merge Request integrated',?6,?7)",
|
||||
params![
|
||||
mr.workspace_id,
|
||||
ticket,
|
||||
n,
|
||||
format!("worker:{}:{}", e.merged_by.runtime_id, e.merged_by.worker_id),
|
||||
e.created_at.to_rfc3339(),
|
||||
format!(
|
||||
"Merge Request `{}` integrated approved source ref `{}`; Ticket remains in progress until guarded Ticket completion.",
|
||||
mr.merge_request_id, e.approved_source_ref
|
||||
),
|
||||
format!("{ticket}:{n}"),
|
||||
],
|
||||
)?;
|
||||
for (k, v) in [
|
||||
("implementation_assignment_id", a),
|
||||
("merge_request_id", &mr.merge_request_id),
|
||||
|
||||
@@ -53,8 +53,8 @@ fn fixture() -> (tempfile::TempDir, MergeRequestStore) {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let p = d.path().join("db");
|
||||
let c = Connection::open(&p).unwrap();
|
||||
c.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY);CREATE TABLE repositories(workspace_id TEXT,repository_id TEXT,PRIMARY KEY(workspace_id,repository_id));CREATE TABLE ticket_current_worker_assignments(workspace_id TEXT,ticket_id TEXT,assignment_id TEXT,runtime_id TEXT,worker_id TEXT,updated_at TEXT,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_tickets(workspace_id TEXT,ticket_id TEXT,workflow_state TEXT,workflow_state_explicit INTEGER,updated_at TEXT,title TEXT NOT NULL,body TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_ticket_targets(workspace_id TEXT,ticket_id TEXT,ordinal INTEGER,repository_key TEXT,ref_selector TEXT,access TEXT);CREATE TABLE typed_ticket_events(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,kind TEXT,author TEXT,at TEXT,from_state TEXT,to_state TEXT,heading TEXT,body TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index));CREATE TABLE typed_ticket_event_attributes(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,key TEXT,value TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index,key));INSERT INTO workspaces VALUES('W');INSERT INTO repositories VALUES('W','R');INSERT INTO repositories VALUES('W','R2');INSERT INTO ticket_current_worker_assignments VALUES('W','T','A','runtime','coder','t');INSERT INTO typed_tickets VALUES('W','T','inprogress',1,'t','Title','Requirements');CREATE VIEW ticket_active_worker_assignments AS SELECT current.* FROM ticket_current_worker_assignments current JOIN typed_tickets ticket ON ticket.workspace_id=current.workspace_id AND ticket.ticket_id=current.ticket_id WHERE ticket.workflow_state NOT IN ('done','closed');").unwrap();
|
||||
c.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body) VALUES('W','T',-1,'comment','Requirements')",[]).unwrap();
|
||||
c.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY);CREATE TABLE repositories(workspace_id TEXT,repository_id TEXT,PRIMARY KEY(workspace_id,repository_id));CREATE TABLE ticket_current_worker_assignments(workspace_id TEXT,ticket_id TEXT,assignment_id TEXT,runtime_id TEXT,worker_id TEXT,updated_at TEXT,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_tickets(workspace_id TEXT,ticket_id TEXT,workflow_state TEXT,workflow_state_explicit INTEGER,updated_at TEXT,title TEXT NOT NULL,body TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_ticket_targets(workspace_id TEXT,ticket_id TEXT,ordinal INTEGER,repository_key TEXT,ref_selector TEXT,access TEXT);CREATE TABLE typed_ticket_events(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,kind TEXT,author TEXT,at TEXT,from_state TEXT,to_state TEXT,heading TEXT,body TEXT,event_ref TEXT NOT NULL DEFAULT '',PRIMARY KEY(workspace_id,ticket_id,event_index),UNIQUE(workspace_id,ticket_id,event_ref));CREATE TABLE typed_ticket_event_attributes(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,key TEXT,value TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index,key));INSERT INTO workspaces VALUES('W');INSERT INTO repositories VALUES('W','R');INSERT INTO repositories VALUES('W','R2');INSERT INTO ticket_current_worker_assignments VALUES('W','T','A','runtime','coder','t');INSERT INTO typed_tickets VALUES('W','T','inprogress',1,'t','Title','Requirements');CREATE VIEW ticket_active_worker_assignments AS SELECT current.* FROM ticket_current_worker_assignments current JOIN typed_tickets ticket ON ticket.workspace_id=current.workspace_id AND ticket.ticket_id=current.ticket_id WHERE ticket.workflow_state NOT IN ('done','closed');").unwrap();
|
||||
c.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body,event_ref) VALUES('W','T',-1,'comment','Requirements','T:-1')",[]).unwrap();
|
||||
drop(c);
|
||||
let a = Assignments(Arc::new(Mutex::new(CurrentAssignment {
|
||||
assignment_id: "A".into(),
|
||||
@@ -376,6 +376,49 @@ fn approve_for(
|
||||
fn approve(s: &MergeRequestStore, subject: &str, token: &str) -> ReviewEvent {
|
||||
approve_for(s, "MR", "R", subject, token)
|
||||
}
|
||||
#[test]
|
||||
fn review_submission_preserves_long_body_and_findings_on_reload() {
|
||||
for (body, finding_body) in [
|
||||
("b".repeat(16 * 1024 + 1), "finding".into()),
|
||||
("review".into(), "指摘".repeat(4096)),
|
||||
] {
|
||||
let (_dir, store) = fixture();
|
||||
open(&store);
|
||||
request(&store, "source", "token");
|
||||
let findings = vec![ReviewFinding {
|
||||
severity: FindingSeverity::Major,
|
||||
code: None,
|
||||
path: None,
|
||||
line: None,
|
||||
body: finding_body,
|
||||
}];
|
||||
let submitted = store
|
||||
.submit_review(SubmitMergeRequestReview {
|
||||
merge_request_id: "MR".into(),
|
||||
ticket_id: "T".into(),
|
||||
current_subject_ref: "source".into(),
|
||||
capability_token: "token".into(),
|
||||
decision: ReviewDecision::RequestChanges,
|
||||
body: body.clone(),
|
||||
findings: findings.clone(),
|
||||
now: at(4),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(submitted.body, body);
|
||||
assert_eq!(submitted.findings, findings);
|
||||
let reloaded = store.get_by_id("W", "MR").unwrap();
|
||||
let saved = reloaded
|
||||
.thread
|
||||
.iter()
|
||||
.find_map(|event| match event {
|
||||
MergeRequestThreadEvent::Review(review) => Some(review),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(saved, &submitted);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn review_submission_authorization_rejects_invalid_grants_before_side_effects() {
|
||||
let (_d, store) = fixture();
|
||||
@@ -457,6 +500,18 @@ fn integration_preserves_selectors_ticket_state_and_assignment_and_replays_its_m
|
||||
.unwrap();
|
||||
assert_eq!(replayed, merged);
|
||||
let connection = Connection::open(d.path().join("db")).unwrap();
|
||||
let ticket_refs = connection
|
||||
.prepare("SELECT event_ref FROM typed_ticket_events WHERE event_index >= 0")
|
||||
.unwrap()
|
||||
.query_map([], |row| row.get::<_, String>(0))
|
||||
.unwrap()
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
ticket_refs,
|
||||
vec!["T:0"],
|
||||
"replay must not append another post"
|
||||
);
|
||||
let (state, active): (String, bool) = connection.query_row(
|
||||
"SELECT workflow_state, EXISTS(SELECT 1 FROM ticket_active_worker_assignments WHERE workspace_id='W' AND ticket_id='T') FROM typed_tickets WHERE workspace_id='W' AND ticket_id='T'",
|
||||
[],
|
||||
@@ -477,6 +532,87 @@ fn integration_preserves_selectors_ticket_state_and_assignment_and_replays_its_m
|
||||
assert!(!json.contains(banned), "{banned} in {json}")
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn integration_post_failure_rolls_back_and_same_operation_recovers_with_legacy_empty_ref() {
|
||||
let (dir, store) = fixture();
|
||||
open(&store);
|
||||
let review = approve(&store, "source", "approved");
|
||||
request(&store, "source", "pending");
|
||||
let completion = CompleteMergeRequest {
|
||||
merge_request_id: "MR".into(),
|
||||
ticket_id: "T".into(),
|
||||
operation_id: "recover-integration".into(),
|
||||
approval_event_id: review.event_id,
|
||||
current_subject_ref: "source".into(),
|
||||
target_ref_before: "before".into(),
|
||||
target_ref_after: "after".into(),
|
||||
strategy: MergeStrategy::FastForward,
|
||||
resolution: ConflictResolution::None,
|
||||
auth: auth(),
|
||||
now: at(5),
|
||||
};
|
||||
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||
// Retain a comment written before integration populated event_ref.
|
||||
connection.execute_batch(
|
||||
"INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body,event_ref)
|
||||
VALUES('W','T',0,'comment','Legacy integration','');
|
||||
CREATE TRIGGER reject_integration_post BEFORE INSERT ON typed_ticket_events
|
||||
WHEN NEW.heading='Merge Request integrated'
|
||||
BEGIN SELECT RAISE(ABORT,'injected integration post failure'); END;",
|
||||
).unwrap();
|
||||
let before = store.get_by_id("W", "MR").unwrap();
|
||||
let error = store.complete(completion.clone()).unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("injected integration post failure")
|
||||
);
|
||||
let failed = store.get_by_id("W", "MR").unwrap();
|
||||
assert_eq!(failed.state, MergeRequestState::Open);
|
||||
assert_eq!(failed.thread, before.thread);
|
||||
assert!(store.authorize_review_submission("MR", "pending").is_ok());
|
||||
assert_eq!(
|
||||
connection
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM typed_ticket_event_attributes",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
|
||||
connection
|
||||
.execute_batch("DROP TRIGGER reject_integration_post")
|
||||
.unwrap();
|
||||
let merged = store.complete(completion.clone()).unwrap();
|
||||
assert_eq!(store.complete(completion).unwrap(), merged);
|
||||
assert_eq!(
|
||||
store.get_by_id("W", "MR").unwrap().state,
|
||||
MergeRequestState::Merged
|
||||
);
|
||||
assert!(store.authorize_review_submission("MR", "pending").is_err());
|
||||
let posts = connection
|
||||
.prepare("SELECT event_ref,body FROM typed_ticket_events WHERE event_index >= 0 ORDER BY event_index")
|
||||
.unwrap()
|
||||
.query_map([], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)))
|
||||
.unwrap()
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
posts.len(),
|
||||
2,
|
||||
"retry and replay must append exactly one post"
|
||||
);
|
||||
assert_eq!(posts[0], (String::new(), "Legacy integration".into()));
|
||||
assert_eq!(posts[1].0, "T:1");
|
||||
assert!(posts[1].1.contains("`MR`"));
|
||||
assert_eq!(connection.query_row(
|
||||
"SELECT value FROM typed_ticket_event_attributes WHERE event_index=1 AND key='operation_id'",
|
||||
[], |row| row.get::<_, String>(0),
|
||||
).unwrap(), "recover-integration");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_move_cancels_submission_and_old_approval_is_reusable_when_source_returns() {
|
||||
let (_d, s) = fixture();
|
||||
@@ -955,7 +1091,7 @@ fn ticket_rescope_requires_fresh_review_before_merge_and_remains_recoverable() {
|
||||
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||
connection
|
||||
.execute_batch(
|
||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL)",
|
||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL,'T:1')",
|
||||
)
|
||||
.unwrap();
|
||||
connection
|
||||
@@ -1082,7 +1218,7 @@ fn ticket_rescope_after_integration_accepts_fresh_requirement_attestation() {
|
||||
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||
connection
|
||||
.execute_batch(
|
||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL)",
|
||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL,'T:1')",
|
||||
)
|
||||
.unwrap();
|
||||
connection
|
||||
@@ -1307,6 +1443,24 @@ fn partial_and_full_integration_retain_ticket_state_and_active_assignment() {
|
||||
assert_eq!(second_request.merged_result().unwrap(), &second_merge);
|
||||
assert_eq!(first_request.integration_approval().unwrap(), &first);
|
||||
assert_eq!(second_request.integration_approval().unwrap(), &second);
|
||||
let posts = connection
|
||||
.prepare(
|
||||
"SELECT event.event_ref, attribute.value
|
||||
FROM typed_ticket_events event
|
||||
JOIN typed_ticket_event_attributes attribute USING(workspace_id,ticket_id,event_index)
|
||||
WHERE attribute.key='merge_request_id' ORDER BY event.event_index",
|
||||
)
|
||||
.unwrap()
|
||||
.query_map([], |row| {
|
||||
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
|
||||
})
|
||||
.unwrap()
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
posts,
|
||||
vec![("T:0".into(), "MR".into()), ("T:1".into(), "MR-2".into())]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1324,7 +1478,7 @@ fn historical_ticket_completion_payload_remains_readable_and_unchanged_on_migrat
|
||||
}"#;
|
||||
connection.execute_batch(
|
||||
"UPDATE typed_tickets SET workflow_state='done' WHERE workspace_id='W' AND ticket_id='T';
|
||||
INSERT INTO typed_ticket_events VALUES('W','T',0,'state_changed','worker:runtime:coder','2026-07-26T12:00:07Z','inprogress','done','Ticket requirements completed','historical body');"
|
||||
INSERT INTO typed_ticket_events VALUES('W','T',0,'state_changed','worker:runtime:coder','2026-07-26T12:00:07Z','inprogress','done','Ticket requirements completed','historical body','T:0');"
|
||||
).unwrap();
|
||||
connection
|
||||
.execute(
|
||||
|
||||
@@ -9,6 +9,7 @@ license.workspace = true
|
||||
api-macros = { workspace = true, features = ["reqwest", "axum"] }
|
||||
manifest.workspace = true
|
||||
protocol.workspace = true
|
||||
session-store.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
workdir.workspace = true
|
||||
|
||||
|
||||
@@ -86,6 +86,8 @@ pub enum WorkerRestoreState {
|
||||
Accepted,
|
||||
Rejected,
|
||||
RolledBack,
|
||||
/// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.
|
||||
Superseded,
|
||||
ReconciliationRequired,
|
||||
}
|
||||
|
||||
@@ -836,6 +838,34 @@ pub struct SessionMaterialPinResponse {
|
||||
pub pin_id: String,
|
||||
}
|
||||
|
||||
/// One body-bearing immutable slice. The byte budget includes slice metadata,
|
||||
/// but not the enclosing availability enum or the caller's durable Job envelope.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SessionRecordingCaptureRequest {
|
||||
pub workspace_id: String,
|
||||
pub source: SessionPublicSource,
|
||||
pub expected_session_id: String,
|
||||
pub start: session_store::capture::RecordingPosition,
|
||||
pub max_entries: usize,
|
||||
pub max_serialized_bytes: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "availability", rename_all = "snake_case")]
|
||||
pub enum SessionRecordingCaptureAvailability {
|
||||
NotMaterialized {
|
||||
session_id: String,
|
||||
},
|
||||
Captured {
|
||||
slice: session_store::capture::RecordingCaptureSlice,
|
||||
},
|
||||
Unavailable {
|
||||
reason: SessionPublicUnavailableReason,
|
||||
message: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SessionMaterialCatalogRequest {
|
||||
@@ -1246,6 +1276,16 @@ pub trait RuntimeApi {
|
||||
#[body] request: SessionMaterialPinRequest,
|
||||
) -> Result<SessionMaterialPinResponse, RuntimeApiError>;
|
||||
|
||||
#[post(
|
||||
"/v1/session-public/recording-capture",
|
||||
status = 200,
|
||||
error_status = 400
|
||||
)]
|
||||
async fn session_recording_capture(
|
||||
&self,
|
||||
#[body] request: SessionRecordingCaptureRequest,
|
||||
) -> Result<SessionRecordingCaptureAvailability, RuntimeApiError>;
|
||||
|
||||
#[post(
|
||||
"/v1/session-public/material/catalog",
|
||||
status = 200,
|
||||
@@ -1540,13 +1580,24 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
assert!(request.preparation.is_none());
|
||||
for state in [
|
||||
"accepted",
|
||||
"rejected",
|
||||
"rolled_back",
|
||||
for (state, wire) in [
|
||||
(WorkerRestoreState::Accepted, "accepted"),
|
||||
(WorkerRestoreState::Rejected, "rejected"),
|
||||
(WorkerRestoreState::RolledBack, "rolled_back"),
|
||||
(WorkerRestoreState::Superseded, "superseded"),
|
||||
(
|
||||
WorkerRestoreState::ReconciliationRequired,
|
||||
"reconciliation_required",
|
||||
),
|
||||
] {
|
||||
assert!(serde_json::from_value::<WorkerRestoreState>(serde_json::json!(state)).is_ok());
|
||||
assert_eq!(
|
||||
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(wire)).unwrap(),
|
||||
state
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(state).unwrap(),
|
||||
serde_json::json!(wire)
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(
|
||||
@@ -1730,6 +1781,13 @@ mod tests {
|
||||
Err(test_error(501))
|
||||
}
|
||||
|
||||
async fn session_recording_capture(
|
||||
&self,
|
||||
_request: SessionRecordingCaptureRequest,
|
||||
) -> Result<SessionRecordingCaptureAvailability, RuntimeApiError> {
|
||||
Err(test_error(501))
|
||||
}
|
||||
|
||||
async fn session_material_catalog(
|
||||
&self,
|
||||
_request: SessionMaterialCatalogRequest,
|
||||
|
||||
+201
-10
@@ -229,6 +229,10 @@ impl_openapi_schema!(
|
||||
PasskeyLoginOptionsResponse,
|
||||
PasskeyLoginCompleteRequest,
|
||||
PasskeyAdditionOptionsRequest,
|
||||
PasskeySummary,
|
||||
PasskeyListResponse,
|
||||
PasskeyAdditionResultRequest,
|
||||
PasskeyAdditionResultResponse,
|
||||
PasskeyRegistrationOptionsRequest,
|
||||
PasskeyRegistrationOptionsResponse,
|
||||
PasskeyRegistrationCompleteRequest,
|
||||
@@ -552,8 +556,9 @@ pub struct WorkspaceWorkerSessionHistoryResponse {
|
||||
/// comparisons do not replace admission checks under the Runtime operation lock; conflicts do
|
||||
/// not trigger automatic token replacement.
|
||||
/// [`WorkerRestoreState`] Accepted means Restore committed, not that the subsequent run succeeded.
|
||||
/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects, and
|
||||
/// ReconciliationRequired needing settled-state checks. Response loss does not imply rollback.
|
||||
/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects,
|
||||
/// Superseded by a later Stop, and ReconciliationRequired needing settled-state checks.
|
||||
/// Response loss does not imply rollback.
|
||||
#[api(reqwest, axum, openapi)]
|
||||
pub trait ServerApi {
|
||||
#[get("/health", status = 200, error_status = 400)]
|
||||
@@ -652,6 +657,33 @@ pub trait ServerApi {
|
||||
#[body] request: PasskeyRegistrationCompleteRequest,
|
||||
) -> Result<server_api_responses::AuthPasskeyRegistrationComplete, RepositoryApiError>;
|
||||
|
||||
#[get(
|
||||
"/api/auth/passkeys",
|
||||
status = 200,
|
||||
error_status = 400,
|
||||
additional_error_statuses = [403, 500],
|
||||
bearer_auth = true,
|
||||
browser_auth = true
|
||||
)]
|
||||
async fn auth_passkey_list(
|
||||
&self,
|
||||
#[extension] context: ServerRequestContext,
|
||||
) -> Result<PasskeyListResponse, RepositoryApiError>;
|
||||
|
||||
#[post(
|
||||
"/api/auth/passkeys/addition/result",
|
||||
status = 200,
|
||||
error_status = 400,
|
||||
additional_error_statuses = [403, 500],
|
||||
bearer_auth = true,
|
||||
browser_auth = true
|
||||
)]
|
||||
async fn auth_passkey_addition_result(
|
||||
&self,
|
||||
#[extension] context: ServerRequestContext,
|
||||
#[body] request: PasskeyAdditionResultRequest,
|
||||
) -> Result<PasskeyAdditionResultResponse, RepositoryApiError>;
|
||||
|
||||
#[post(
|
||||
"/api/auth/passkeys/addition/options",
|
||||
status = 200,
|
||||
@@ -4119,6 +4151,8 @@ impl RequestActor {
|
||||
pub struct WhoamiResponse {
|
||||
pub actor: Option<RequestActor>,
|
||||
pub session: BrowserSessionState,
|
||||
/// Non-secret comparison evidence, never an authentication credential.
|
||||
pub authentication_reference: Option<String>,
|
||||
}
|
||||
|
||||
/// Browser navigation evidence, not an authorization grant or a generic 401 classification.
|
||||
@@ -4146,6 +4180,8 @@ pub struct AuthBootstrapUserRequest {
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AuthUserResponse {
|
||||
pub user: AuthenticatedUser,
|
||||
/// Comparison evidence for the exact newly issued session, not a credential.
|
||||
pub authentication_reference: String,
|
||||
}
|
||||
|
||||
/// Starts new-account registration only; an existing handle is never interpreted as key addition.
|
||||
@@ -4162,16 +4198,64 @@ pub struct PasskeyRegistrationOptionsRequest {
|
||||
pub browser_origin: Option<String>,
|
||||
}
|
||||
|
||||
/// Adds a credential to the current authenticated actor; target identity is never public input.
|
||||
/// Adds to the current actor. Expected identity/binding fields only fence stale UI;
|
||||
/// they never select an owner or authorize an addition.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PasskeyAdditionOptionsRequest {
|
||||
pub expected_user_id: String,
|
||||
pub expected_account_id: String,
|
||||
pub expected_authentication_reference: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
|
||||
pub browser_origin: Option<String>,
|
||||
}
|
||||
|
||||
/// Safe account metadata. Neither the WebAuthn credential ID nor key material is exposed.
|
||||
/// Normal-login verification is observed verifier success, not inferred discoverability.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PasskeySummary {
|
||||
pub key_reference: String,
|
||||
pub created_at: String,
|
||||
pub last_used_at: Option<String>,
|
||||
pub normal_login_verified_at: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PasskeyListResponse {
|
||||
pub user: AuthenticatedUser,
|
||||
pub keys: Vec<PasskeySummary>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PasskeyAdditionResultRequest {
|
||||
pub challenge_id: String,
|
||||
pub expected_user_id: String,
|
||||
pub expected_account_id: String,
|
||||
/// Original options authentication reference; the current same-owner session may differ.
|
||||
pub expected_authentication_reference: String,
|
||||
}
|
||||
|
||||
/// Unknown is not proof that registration failed: completion may still be in flight.
|
||||
/// Saved proves the credential commit, not session issuance or current session health.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum PasskeyAdditionResultResponse {
|
||||
Unknown {},
|
||||
Saved {
|
||||
user: AuthenticatedUser,
|
||||
key: PasskeySummary,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -6980,6 +7064,8 @@ pub struct WorkingDirectoryCleanupTarget {
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WorkingDirectoryRemovalRequest {
|
||||
/// Stable for an exact resend; a new explicit request uses a new identity.
|
||||
pub operation_id: String,
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
@@ -7518,6 +7604,9 @@ pub struct CurrentWorkerWorkdirAttachmentResponse {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub environment: Option<workdir::WorkdirSessionEnvironment>,
|
||||
pub attached: bool,
|
||||
/// The Workdir was already attached to this Worker; alias is its existing name.
|
||||
#[serde(default)]
|
||||
pub already_attached: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
@@ -8889,6 +8978,9 @@ pub struct MergeRequestListQuery {
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MergeRequestThreadQuery {
|
||||
/// Show only: summary omits history; detail returns an exact bounded thread page.
|
||||
/// Omission retains the existing HTTP detail response.
|
||||
pub view: Option<MergeRequestReadView>,
|
||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
||||
pub after: Option<u64>,
|
||||
#[schemars(range(min = 0, max = 1000))]
|
||||
@@ -8916,6 +9008,73 @@ pub struct MergeRequestLinkedTicketResponse {
|
||||
pub key: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum MergeRequestReadView {
|
||||
Summary,
|
||||
#[default]
|
||||
Detail,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MergeRequestEventReference {
|
||||
pub event_id: String,
|
||||
/// Exact event: read detail with after = sequence - 1 and limit = 1.
|
||||
#[cfg_attr(feature = "typescript", ts(type = "number"))]
|
||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
||||
pub sequence: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MergeRequestReviewSummary {
|
||||
pub event: MergeRequestEventReference,
|
||||
pub request_event_id: String,
|
||||
pub subject_ref: String,
|
||||
pub ticket_content_digest: String,
|
||||
pub decision: ReviewDecision,
|
||||
/// Findings retain their order in the exact review event; no finding body is truncated here.
|
||||
#[cfg_attr(feature = "typescript", ts(type = "number"))]
|
||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_usize))]
|
||||
pub findings_count: usize,
|
||||
}
|
||||
|
||||
/// Computed from complete persisted evidence, never from the returned history page.
|
||||
/// This observation is not integration authority or current-requirements approval.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MergeRequestStatusSummary {
|
||||
/// unknown, none, pending, approved, or request_changes for the observed source.
|
||||
pub review_status: String,
|
||||
pub review: Option<MergeRequestReviewSummary>,
|
||||
pub latest_review_subject_ref: Option<String>,
|
||||
pub latest_event: Option<MergeRequestEventReference>,
|
||||
#[cfg_attr(feature = "typescript", ts(type = "number"))]
|
||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_usize))]
|
||||
pub thread_event_count: usize,
|
||||
/// Existing readiness semantics; review bodies are omitted in this compact projection.
|
||||
pub readiness: MergeRequestReadinessResponse,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MergeRequestEventPage {
|
||||
/// Summary deliberately omits events. Read detail from the beginning to obtain history.
|
||||
pub omitted: bool,
|
||||
pub has_more: bool,
|
||||
#[cfg_attr(feature = "typescript", ts(type = "number | null"))]
|
||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
||||
pub next_after: Option<u64>,
|
||||
#[schemars(range(min = 1, max = 200))]
|
||||
pub limit: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -8925,6 +9084,9 @@ pub struct MergeRequestDetailResponse {
|
||||
pub source: MergeRequestRefResponse,
|
||||
pub target: MergeRequestRefResponse,
|
||||
pub linked_tickets: Vec<MergeRequestLinkedTicketResponse>,
|
||||
pub view: MergeRequestReadView,
|
||||
pub summary: MergeRequestStatusSummary,
|
||||
pub event_page: MergeRequestEventPage,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
@@ -10579,8 +10741,8 @@ pub enum WorkerOperationState {
|
||||
/// Cross-layer outcome of an explicit Worker restore operation.
|
||||
///
|
||||
/// `Rejected` is reserved for read-only preflight failures. Once live restore
|
||||
/// work starts, failure is either a confirmed `RolledBack` operation or a
|
||||
/// `ReconciliationRequired` result whose commit state must be reread/retried.
|
||||
/// work starts, distinguish confirmed `RolledBack`, unresolved
|
||||
/// `ReconciliationRequired`, and a later Stop's `Superseded` outcome.
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
@@ -10588,6 +10750,8 @@ pub enum WorkerRestoreState {
|
||||
Accepted,
|
||||
Rejected,
|
||||
RolledBack,
|
||||
/// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.
|
||||
Superseded,
|
||||
ReconciliationRequired,
|
||||
}
|
||||
|
||||
@@ -11651,6 +11815,10 @@ pub fn auth_api_typescript() -> String {
|
||||
AuthBootstrapUserRequest::decl(&config),
|
||||
AuthUserResponse::decl(&config),
|
||||
PasskeyAdditionOptionsRequest::decl(&config),
|
||||
PasskeySummary::decl(&config),
|
||||
PasskeyListResponse::decl(&config),
|
||||
PasskeyAdditionResultRequest::decl(&config),
|
||||
PasskeyAdditionResultResponse::decl(&config),
|
||||
PasskeyRegistrationOptionsRequest::decl(&config),
|
||||
PasskeyRegistrationOptionsResponse::decl(&config),
|
||||
PasskeyRegistrationCompleteRequest::decl(&config),
|
||||
@@ -12681,6 +12849,7 @@ mod tests {
|
||||
(WorkerRestoreState::Accepted, "accepted"),
|
||||
(WorkerRestoreState::Rejected, "rejected"),
|
||||
(WorkerRestoreState::RolledBack, "rolled_back"),
|
||||
(WorkerRestoreState::Superseded, "superseded"),
|
||||
(
|
||||
WorkerRestoreState::ReconciliationRequired,
|
||||
"reconciliation_required",
|
||||
@@ -13291,9 +13460,11 @@ mod tests {
|
||||
capabilities: workdir::WorkdirSessionCapabilities::EMPTY,
|
||||
environment: None,
|
||||
attached: false,
|
||||
already_attached: false,
|
||||
};
|
||||
let value = serde_json::to_value(&response).unwrap();
|
||||
assert_eq!(value.as_object().unwrap().len(), 5);
|
||||
assert_eq!(value.as_object().unwrap().len(), 6);
|
||||
assert_eq!(value["already_attached"], false);
|
||||
assert!(value.get("connection_id").is_none());
|
||||
assert_eq!(
|
||||
serde_json::from_value::<CurrentWorkerWorkdirAttachmentResponse>(value).unwrap(),
|
||||
@@ -14079,7 +14250,8 @@ mod tests {
|
||||
"display_name": "Hare",
|
||||
"auth_method": "browser_session"
|
||||
},
|
||||
"session": { "kind": "authenticated", "expires_at": "2026-10-20T00:00:00Z" }
|
||||
"session": { "kind": "authenticated", "expires_at": "2026-10-20T00:00:00Z" },
|
||||
"authentication_reference": "synthetic-session-reference"
|
||||
});
|
||||
let decoded = serde_json::from_value::<WhoamiResponse>(whoami.clone())
|
||||
.expect("server whoami fixture should match shared DTO");
|
||||
@@ -14101,7 +14273,8 @@ mod tests {
|
||||
"account_id": "account-1",
|
||||
"handle": "hare",
|
||||
"display_name": "Hare"
|
||||
}
|
||||
},
|
||||
"authentication_reference": "synthetic-session-reference"
|
||||
});
|
||||
let decoded = serde_json::from_value::<AuthUserResponse>(auth_user.clone())
|
||||
.expect("server auth-user fixture should match shared DTO");
|
||||
@@ -14274,7 +14447,7 @@ mod tests {
|
||||
serde_json::json!({"kind":"expired"}),
|
||||
serde_json::json!({"kind":"authenticated", "expires_at":null}),
|
||||
] {
|
||||
let value = serde_json::json!({"actor":null, "session":session});
|
||||
let value = serde_json::json!({"actor":null, "session":session, "authentication_reference": null});
|
||||
let response: WhoamiResponse = serde_json::from_value(value.clone()).unwrap();
|
||||
assert_eq!(serde_json::to_value(response).unwrap(), value);
|
||||
}
|
||||
@@ -14812,10 +14985,28 @@ mod openapi_artifact_tests {
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
serde_json::from_value::<PasskeyAdditionOptionsRequest>(serde_json::json!({})).is_ok()
|
||||
serde_json::from_value::<PasskeyAdditionOptionsRequest>(serde_json::json!({})).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passkey_addition_requires_expected_owner_and_binding() {
|
||||
let expected = serde_json::json!({
|
||||
"expected_user_id": "user-a", "expected_account_id": "account-a",
|
||||
"expected_authentication_reference": "reference"
|
||||
});
|
||||
assert!(serde_json::from_value::<PasskeyAdditionOptionsRequest>(expected.clone()).is_ok());
|
||||
for field in [
|
||||
"expected_user_id",
|
||||
"expected_account_id",
|
||||
"expected_authentication_reference",
|
||||
] {
|
||||
let mut missing = expected.clone();
|
||||
missing.as_object_mut().unwrap().remove(field);
|
||||
assert!(serde_json::from_value::<PasskeyAdditionOptionsRequest>(missing).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auth_workspace_catalog_and_typed_header_contract_is_strict() {
|
||||
let document = canonical_openapi_document().expect("canonical OpenAPI contract must build");
|
||||
|
||||
@@ -11,5 +11,7 @@ checks or Runtime catalog lifecycle management.
|
||||
- [`worker_session_store`]: storage boundary for a Session bound to a Worker
|
||||
- [`worker_metadata`]: current Worker state and active Segment switching
|
||||
- [`system_item`]: records of system input presented to the model
|
||||
- [`capture`]: Worker-independent immutable public values and bounded body transport;
|
||||
authorization, source I/O, and delivery lifecycle stay with the Host
|
||||
- [`public_snapshot`] / [`public_index`]: public projections of stored history
|
||||
- [`event_trace`]: diagnostic records independent of history replay
|
||||
|
||||
@@ -0,0 +1,967 @@
|
||||
//! Exploration domain for immutable Session captures, with no Workspace/Memory mutation capability.
|
||||
//!
|
||||
//! A capture is an ordered projection of committed entries; exploration excludes reasoning and
|
||||
//! system prompts, including Host compaction summaries and other derived context. Full inherited
|
||||
//! summary input belongs to compaction, not public observation or Memory evidence. Overview,
|
||||
//! search, bounded reads, and evidence conversion share `SessionEntryRef`.
|
||||
//! `SessionEntryRef` is a contextual UI reference. `ScopedSessionEntryRef` additionally binds
|
||||
//! the Session and exact immutable capture, and is required for portable checked operations.
|
||||
//! Appends preserve existing entry identities. Sparse real user/assistant anchors and intervening
|
||||
//! entry counts index long histories. Search with an empty filter also provides bounded range lists.
|
||||
//!
|
||||
//! Host-fixed snapshot exploration, Memory candidate evidence validation, and current Worker
|
||||
//! observation share only this domain; they do not enable or modify each other's Features.
|
||||
//! The provider owns authorization to recapture a current Worker. A capture itself proves neither
|
||||
//! completion nor approval and grants no business-mutation authority.
|
||||
//!
|
||||
//! Recording Hosts use the common committed replay through the guarded APIs below.
|
||||
//! [`recording_catalog_worker_guarded`] and [`recording_catalog_fs_guarded`] expose
|
||||
//! descriptors only to an admission callback under the caller's existing publication/
|
||||
//! erasure guard and persisted active selector, without allocating a full body DTO.
|
||||
//! [`capture_recording_slice_worker_guarded`] and [`capture_recording_slice_fs_guarded`]
|
||||
//! separately construct bounded body-bearing inputs. Admission descriptors are not
|
||||
//! sufficient Job inputs; saved recording DTOs are read/sliced without live storage.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::{
|
||||
LoggedSessionHistoryMetadata as SessionHistoryMetadata,
|
||||
LoggedSessionHistoryOrigin as HistoryOrigin,
|
||||
};
|
||||
mod body;
|
||||
mod recording;
|
||||
use agen::{HistoryEntry, Item, Role};
|
||||
pub use body::*;
|
||||
use protocol::{
|
||||
SessionContentPart, SessionEntryProvenance, SessionMessageRole, SessionSnapshot,
|
||||
SessionSnapshotEntryData,
|
||||
};
|
||||
pub use recording::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
const DEFAULT_SEARCH_LIMIT: usize = 20;
|
||||
const MAX_SEARCH_LIMIT: usize = 50;
|
||||
const DEFAULT_READ_MAX_ITEMS: usize = 40;
|
||||
const MAX_READ_MAX_ITEMS: usize = 80;
|
||||
const DEFAULT_READ_MAX_BYTES: usize = 32 * 1024;
|
||||
const OVERVIEW_ANCHOR_STRIDE: usize = 8;
|
||||
|
||||
/// Append-stable contextual UI reference derived from a logical entry identity.
|
||||
/// Portable consumers must bind it with `SessionCapture::scoped_ref` before use.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct SessionEntryRef(String);
|
||||
|
||||
impl SessionEntryRef {
|
||||
pub fn from_history_entry_id(entry_id: &crate::LoggedSessionHistoryEntryId) -> Self {
|
||||
Self(format!("E{}", entry_id.0))
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
let suffix = value.strip_prefix('E')?;
|
||||
if suffix.is_empty()
|
||||
// Legacy Host boundary identities contain both Session and Segment UUIDs.
|
||||
|| suffix.len() > 256
|
||||
|| !suffix
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(Self(value.to_string()))
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SessionEntryRef {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReferenceKind {
|
||||
User,
|
||||
Assistant,
|
||||
System,
|
||||
Tool,
|
||||
}
|
||||
|
||||
impl ReferenceKind {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::User => "user",
|
||||
Self::Assistant => "assistant",
|
||||
Self::System => "system",
|
||||
Self::Tool => "tool",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"user" => Some(Self::User),
|
||||
"assistant" | "agent" => Some(Self::Assistant),
|
||||
"system" => Some(Self::System),
|
||||
"tool" => Some(Self::Tool),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ToolPart {
|
||||
Input,
|
||||
Output,
|
||||
Both,
|
||||
}
|
||||
|
||||
impl ToolPart {
|
||||
pub fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"input" => Some(Self::Input),
|
||||
"output" => Some(Self::Output),
|
||||
"both" => Some(Self::Both),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Input => "input",
|
||||
Self::Output => "output",
|
||||
Self::Both => "both",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn matches(self, actual: ToolPart) -> bool {
|
||||
matches!(self, ToolPart::Both) || self == actual
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OverviewItem {
|
||||
pub id: SessionEntryRef,
|
||||
pub origin: SessionEntryProvenance,
|
||||
pub entry_range: [u64; 2],
|
||||
pub kind: ReferenceKind,
|
||||
pub label: String,
|
||||
pub text: String,
|
||||
pub intervening_entries: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ReferenceEntry {
|
||||
pub attachment_count: usize,
|
||||
use_policy: Option<protocol::SessionEntryUsePolicy>,
|
||||
pub id: SessionEntryRef,
|
||||
pub origin: SessionEntryProvenance,
|
||||
pub entry_range: [u64; 2],
|
||||
pub kind: ReferenceKind,
|
||||
pub tool_part: Option<ToolPart>,
|
||||
pub tool_name: Option<String>,
|
||||
pub label: String,
|
||||
pub summary: String,
|
||||
search_text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct SearchOptions {
|
||||
pub query: String,
|
||||
pub kind: Option<ReferenceKind>,
|
||||
pub tool_part: Option<ToolPart>,
|
||||
pub tool_name: Option<String>,
|
||||
pub limit: Option<usize>,
|
||||
pub min_entry_index: Option<u64>,
|
||||
pub from: Option<SessionEntryRef>,
|
||||
pub through: Option<SessionEntryRef>,
|
||||
pub offset: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SearchHit {
|
||||
pub id: SessionEntryRef,
|
||||
pub origin: SessionEntryProvenance,
|
||||
pub kind: ReferenceKind,
|
||||
pub tool_part: Option<ToolPart>,
|
||||
pub tool_name: Option<String>,
|
||||
pub entry_range: [u64; 2],
|
||||
pub label: String,
|
||||
pub summary: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum ReadSelector<'a> {
|
||||
Id(&'a str),
|
||||
EntryRange([u64; 2]),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReadDetail {
|
||||
Compact,
|
||||
Full,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ReadOptions {
|
||||
pub include_tools: bool,
|
||||
pub tool_part: ToolPart,
|
||||
pub detail: ReadDetail,
|
||||
pub max_items: usize,
|
||||
pub max_bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for ReadOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
include_tools: true,
|
||||
tool_part: ToolPart::Both,
|
||||
detail: ReadDetail::Compact,
|
||||
max_items: DEFAULT_READ_MAX_ITEMS,
|
||||
max_bytes: DEFAULT_READ_MAX_BYTES,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ReadEntry {
|
||||
pub id: SessionEntryRef,
|
||||
pub origin: SessionEntryProvenance,
|
||||
pub kind: ReferenceKind,
|
||||
pub tool_part: Option<ToolPart>,
|
||||
pub tool_name: Option<String>,
|
||||
pub entry_range: [u64; 2],
|
||||
pub label: String,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ReadResult {
|
||||
pub entries: Vec<ReadEntry>,
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SessionEntryEvidence {
|
||||
pub segment_id: String,
|
||||
pub entry_ref: SessionEntryRef,
|
||||
pub origin: SessionEntryProvenance,
|
||||
/// Inclusive original-source positions in `segment_id`, not view/UI order.
|
||||
pub entry_range: [u64; 2],
|
||||
pub kind: ReferenceKind,
|
||||
pub tool_part: Option<ToolPart>,
|
||||
pub label: String,
|
||||
pub summary: String,
|
||||
pub excerpt: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct CapturedHistoryEntry {
|
||||
use_policy: Option<protocol::SessionEntryUsePolicy>,
|
||||
item: Item,
|
||||
entry_id: crate::LoggedSessionHistoryEntryId,
|
||||
origin: SessionEntryProvenance,
|
||||
dto: CaptureEntryDto,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SessionCapture {
|
||||
session_id: String,
|
||||
segment_id: String,
|
||||
capture_id: String,
|
||||
view_entry_count: u64,
|
||||
entries: Arc<Vec<CapturedHistoryEntry>>,
|
||||
overview: Vec<OverviewItem>,
|
||||
index: Vec<ReferenceEntry>,
|
||||
}
|
||||
|
||||
impl SessionCapture {
|
||||
pub fn from_session_snapshot(
|
||||
session_id: impl Into<String>,
|
||||
segment_id: impl Into<String>,
|
||||
snapshot: SessionSnapshot,
|
||||
) -> Self {
|
||||
let segment_id = segment_id.into();
|
||||
let view_entry_count = snapshot.entries.len() as u64;
|
||||
let entries = snapshot
|
||||
.entries
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, entry)| {
|
||||
let policy = CaptureUsePolicy {
|
||||
recording_material: false,
|
||||
memory_evidence: entry.use_policy.is_none_or(|p| p.memory_evidence),
|
||||
observation_readable: entry.use_policy.is_none_or(|p| p.observation_readable),
|
||||
};
|
||||
let explicit = entry.use_policy.is_some();
|
||||
let item = match entry.data {
|
||||
SessionSnapshotEntryData::UserInput { segments } => {
|
||||
Item::user_message(protocol::Segment::flatten_to_text(&segments))
|
||||
}
|
||||
SessionSnapshotEntryData::Message { role, content } => Item::Message {
|
||||
id: None,
|
||||
status: None,
|
||||
role: match role {
|
||||
SessionMessageRole::User => Role::User,
|
||||
SessionMessageRole::Assistant => Role::Assistant,
|
||||
},
|
||||
content: content
|
||||
.into_iter()
|
||||
.map(|part| match part {
|
||||
SessionContentPart::Text { text } => {
|
||||
agen::ContentPart::Text { text }
|
||||
}
|
||||
SessionContentPart::Refusal { refusal } => {
|
||||
agen::ContentPart::Refusal { refusal }
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
SessionSnapshotEntryData::ToolCall {
|
||||
call_id,
|
||||
name,
|
||||
arguments,
|
||||
} => Item::tool_call(call_id, name, arguments),
|
||||
SessionSnapshotEntryData::ToolResult {
|
||||
call_id,
|
||||
summary,
|
||||
content,
|
||||
is_error,
|
||||
..
|
||||
} => Item::tool_result_item(call_id, summary, content, is_error),
|
||||
SessionSnapshotEntryData::SystemItem { content, .. } if explicit => {
|
||||
Item::system_message(content)
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
let body = CaptureBody::project(&item, &entry.provenance, explicit)?;
|
||||
if !policy.any() {
|
||||
return None;
|
||||
}
|
||||
Some(CaptureEntryDto {
|
||||
entry_id: entry.entry_id,
|
||||
order: index as u64,
|
||||
source: CaptureSource {
|
||||
segment_id: segment_id.clone(),
|
||||
entry_index: index as u64,
|
||||
provenance: entry.provenance,
|
||||
history_origin: None,
|
||||
derived_from: entry.derived_from,
|
||||
},
|
||||
policy,
|
||||
body,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
Self::from_projected(session_id.into(), segment_id, view_entry_count, entries)
|
||||
}
|
||||
|
||||
pub fn new(
|
||||
session_id: impl Into<String>,
|
||||
segment_id: impl Into<String>,
|
||||
items: Vec<Item>,
|
||||
) -> Self {
|
||||
let segment_id = segment_id.into();
|
||||
let view_entry_count = items.len() as u64;
|
||||
let entries = items
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, item)| {
|
||||
let origin = SessionEntryProvenance::LegacyUnknown;
|
||||
let body = CaptureBody::project(&item, &origin, false)?;
|
||||
Some(CaptureEntryDto {
|
||||
entry_id: format!("{index:08}"),
|
||||
order: index as u64,
|
||||
source: CaptureSource {
|
||||
segment_id: segment_id.clone(),
|
||||
entry_index: index as u64,
|
||||
provenance: origin,
|
||||
history_origin: None,
|
||||
derived_from: Vec::new(),
|
||||
},
|
||||
policy: CaptureUsePolicy::PUBLIC,
|
||||
body,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
Self::from_projected(session_id.into(), segment_id, view_entry_count, entries)
|
||||
}
|
||||
|
||||
/// Project a complete Host-fixed history whose source positions start at zero.
|
||||
/// Select a suffix with `select_order_range` on the resulting value instead
|
||||
/// of passing a rebased history vector: annotations do not carry source positions.
|
||||
pub fn from_complete_history(
|
||||
session_id: impl Into<String>,
|
||||
segment_id: impl Into<String>,
|
||||
entries: Vec<HistoryEntry<SessionHistoryMetadata>>,
|
||||
) -> Self {
|
||||
let segment_id = segment_id.into();
|
||||
let view_entry_count = entries.len() as u64;
|
||||
let entries = entries
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, entry)| {
|
||||
let origin = public_provenance(&entry.annotation.origin);
|
||||
let mut item = entry.item;
|
||||
let mut explicit = false;
|
||||
let policy = if let Some(input) = entry.annotation.input_metadata {
|
||||
explicit = true;
|
||||
if matches!(
|
||||
item,
|
||||
Item::Message {
|
||||
role: Role::System,
|
||||
..
|
||||
}
|
||||
) {
|
||||
// Only the explicit public projection crosses this boundary.
|
||||
let public = input.public_item()?;
|
||||
item = Item::system_message(public.body.clone());
|
||||
} else if input.feature_payload().is_some() || input.public_item().is_some() {
|
||||
return None;
|
||||
}
|
||||
CaptureUsePolicy::from(input.usage())
|
||||
} else {
|
||||
if matches!(entry.annotation.origin, HistoryOrigin::Provided { .. }) {
|
||||
return None;
|
||||
}
|
||||
CaptureUsePolicy::PUBLIC
|
||||
};
|
||||
if !policy.any() {
|
||||
return None;
|
||||
}
|
||||
let body = CaptureBody::project(&item, &origin, explicit)?;
|
||||
Some(CaptureEntryDto {
|
||||
entry_id: entry.annotation.entry_id.0,
|
||||
order: index as u64,
|
||||
source: CaptureSource {
|
||||
segment_id: segment_id.clone(),
|
||||
entry_index: index as u64,
|
||||
provenance: origin,
|
||||
history_origin: Some(entry.annotation.origin),
|
||||
derived_from: entry
|
||||
.annotation
|
||||
.derivation
|
||||
.map(|d| d.sources.into_iter().map(|id| id.0).collect())
|
||||
.unwrap_or_default(),
|
||||
},
|
||||
policy,
|
||||
body,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
Self::from_projected(session_id.into(), segment_id, view_entry_count, entries)
|
||||
}
|
||||
|
||||
fn from_projected(
|
||||
session_id: String,
|
||||
segment_id: String,
|
||||
view_entry_count: u64,
|
||||
dtos: Vec<CaptureEntryDto>,
|
||||
) -> Self {
|
||||
let dto = CaptureDto {
|
||||
version: CAPTURE_VERSION,
|
||||
session_id: session_id.clone(),
|
||||
segment_id: segment_id.clone(),
|
||||
view_entry_count,
|
||||
entries: dtos,
|
||||
};
|
||||
let capture_id = dto.fingerprint();
|
||||
let entries: Vec<_> = dto
|
||||
.entries
|
||||
.into_iter()
|
||||
.map(|dto| CapturedHistoryEntry {
|
||||
use_policy: Some(protocol::SessionEntryUsePolicy {
|
||||
observation_readable: dto.policy.observation_readable,
|
||||
memory_evidence: dto.policy.memory_evidence,
|
||||
}),
|
||||
item: dto.body.to_item(),
|
||||
entry_id: crate::LoggedSessionHistoryEntryId(dto.entry_id.clone()),
|
||||
origin: dto.source.provenance.clone(),
|
||||
dto,
|
||||
})
|
||||
.collect();
|
||||
let mut overview = Vec::new();
|
||||
let mut index = Vec::new();
|
||||
|
||||
for entry in &entries {
|
||||
let item = &entry.item;
|
||||
let entry_range = [entry.dto.order; 2];
|
||||
let attachment_count = match entry.dto.body {
|
||||
CaptureBody::ToolResult {
|
||||
attachment_count, ..
|
||||
} => attachment_count,
|
||||
_ => 0,
|
||||
};
|
||||
match item {
|
||||
Item::Message { role, content, .. } => {
|
||||
let Some(kind) = message_reference_kind(&entry.origin, role, entry.use_policy)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let text = content
|
||||
.iter()
|
||||
.map(|p| p.as_text())
|
||||
.collect::<Vec<_>>()
|
||||
.join("");
|
||||
let label = format!("{} message", kind.as_str());
|
||||
let summary = truncate_chars(&text, 240);
|
||||
let id = SessionEntryRef::from_history_entry_id(&entry.entry_id);
|
||||
index.push(ReferenceEntry {
|
||||
use_policy: entry.use_policy,
|
||||
attachment_count,
|
||||
id: id.clone(),
|
||||
origin: entry.origin.clone(),
|
||||
entry_range,
|
||||
kind,
|
||||
tool_part: None,
|
||||
tool_name: None,
|
||||
label: label.clone(),
|
||||
summary: summary.clone(),
|
||||
search_text: text.clone(),
|
||||
});
|
||||
if entry.use_policy.is_none_or(|p| p.observation_readable)
|
||||
&& matches!(kind, ReferenceKind::User | ReferenceKind::Assistant)
|
||||
{
|
||||
overview.push(OverviewItem {
|
||||
id: id.clone(),
|
||||
origin: entry.origin.clone(),
|
||||
entry_range,
|
||||
kind,
|
||||
label,
|
||||
text,
|
||||
intervening_entries: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
Item::ToolCall {
|
||||
name, arguments, ..
|
||||
} => {
|
||||
let text = format!("{name}\n{arguments}");
|
||||
index.push(ReferenceEntry {
|
||||
use_policy: entry.use_policy,
|
||||
attachment_count,
|
||||
id: SessionEntryRef::from_history_entry_id(&entry.entry_id),
|
||||
origin: entry.origin.clone(),
|
||||
entry_range,
|
||||
kind: ReferenceKind::Tool,
|
||||
tool_part: Some(ToolPart::Input),
|
||||
tool_name: Some(name.clone()),
|
||||
label: format!("tool input: {name}"),
|
||||
summary: format!("Tool call {name}"),
|
||||
search_text: text,
|
||||
});
|
||||
}
|
||||
Item::ToolResult {
|
||||
summary,
|
||||
content,
|
||||
attachments: _,
|
||||
..
|
||||
} => {
|
||||
let attachment_marker = if attachment_count == 0 {
|
||||
String::new()
|
||||
} else {
|
||||
format!("\n[{} image attachment(s)]", attachment_count)
|
||||
};
|
||||
let text = format!(
|
||||
"{summary}\n{}{attachment_marker}",
|
||||
content.as_deref().unwrap_or_default(),
|
||||
);
|
||||
index.push(ReferenceEntry {
|
||||
use_policy: entry.use_policy,
|
||||
attachment_count,
|
||||
id: SessionEntryRef::from_history_entry_id(&entry.entry_id),
|
||||
origin: entry.origin.clone(),
|
||||
entry_range,
|
||||
kind: ReferenceKind::Tool,
|
||||
tool_part: Some(ToolPart::Output),
|
||||
tool_name: None,
|
||||
label: "tool output".to_string(),
|
||||
summary: truncate_chars(summary, 240),
|
||||
search_text: text,
|
||||
});
|
||||
}
|
||||
Item::AssistantResponseBoundary { .. } | Item::Reasoning { .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
if overview.len() > 2 {
|
||||
let last = overview.len() - 1;
|
||||
overview = overview
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, entry)| {
|
||||
(index == 0 || index == last || index % OVERVIEW_ANCHOR_STRIDE == 0)
|
||||
.then_some(entry)
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
|
||||
for overview_index in 0..overview.len().saturating_sub(1) {
|
||||
let current_entry = overview[overview_index].entry_range[0];
|
||||
let next_entry = overview[overview_index + 1].entry_range[0];
|
||||
overview[overview_index].intervening_entries = index
|
||||
.iter()
|
||||
.filter(|entry| {
|
||||
let entry_index = entry.entry_range[0];
|
||||
entry_index > current_entry && entry_index < next_entry
|
||||
})
|
||||
.count();
|
||||
}
|
||||
|
||||
Self {
|
||||
session_id,
|
||||
segment_id,
|
||||
capture_id,
|
||||
view_entry_count,
|
||||
entries: Arc::new(entries),
|
||||
overview,
|
||||
index,
|
||||
}
|
||||
}
|
||||
|
||||
/// Public source entries are immutable; callers retain their own business decisions.
|
||||
pub fn entries(&self) -> impl ExactSizeIterator<Item = &CaptureEntryDto> {
|
||||
self.entries.iter().map(|entry| &entry.dto)
|
||||
}
|
||||
|
||||
pub fn overview(&self) -> &[OverviewItem] {
|
||||
&self.overview
|
||||
}
|
||||
|
||||
/// Capture-local UI position, not the original source position used by evidence.
|
||||
pub fn view_order_for_ref(&self, reference: &SessionEntryRef) -> Option<u64> {
|
||||
self.index
|
||||
.iter()
|
||||
.find(|entry| entry.id == *reference)
|
||||
.map(|entry| entry.entry_range[0])
|
||||
}
|
||||
|
||||
pub fn search(&self, options: &SearchOptions) -> Vec<SearchHit> {
|
||||
if self.validate_search(options).is_err() {
|
||||
return Vec::new();
|
||||
}
|
||||
let query = options.query.trim().to_lowercase();
|
||||
let limit = options
|
||||
.limit
|
||||
.unwrap_or(DEFAULT_SEARCH_LIMIT)
|
||||
.clamp(1, MAX_SEARCH_LIMIT);
|
||||
let tool_name = options.tool_name.as_deref();
|
||||
let min_entry_index = options
|
||||
.from
|
||||
.as_ref()
|
||||
.and_then(|reference| self.view_order_for_ref(reference))
|
||||
.unwrap_or_else(|| options.min_entry_index.unwrap_or(0));
|
||||
let max_entry_index = options
|
||||
.through
|
||||
.as_ref()
|
||||
.and_then(|reference| self.view_order_for_ref(reference))
|
||||
.unwrap_or(u64::MAX);
|
||||
let mut skipped = 0usize;
|
||||
let mut hits = Vec::new();
|
||||
|
||||
for entry in &self.index {
|
||||
if entry.use_policy.is_some_and(|p| !p.observation_readable) {
|
||||
continue;
|
||||
}
|
||||
if entry.entry_range[0] < min_entry_index || entry.entry_range[0] > max_entry_index {
|
||||
continue;
|
||||
}
|
||||
if let Some(kind) = options.kind {
|
||||
if entry.kind != kind {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if let Some(part) = options.tool_part {
|
||||
if entry.kind != ReferenceKind::Tool {
|
||||
continue;
|
||||
}
|
||||
let Some(actual) = entry.tool_part else {
|
||||
continue;
|
||||
};
|
||||
if !part.matches(actual) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if let Some(name) = tool_name {
|
||||
if entry.tool_name.as_deref() != Some(name) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if !query.is_empty() && !entry.search_text.to_lowercase().contains(&query) {
|
||||
continue;
|
||||
}
|
||||
if skipped < options.offset {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
hits.push(SearchHit {
|
||||
id: entry.id.clone(),
|
||||
origin: entry.origin.clone(),
|
||||
kind: entry.kind,
|
||||
tool_part: entry.tool_part,
|
||||
tool_name: entry.tool_name.clone(),
|
||||
entry_range: entry.entry_range,
|
||||
label: entry.label.clone(),
|
||||
summary: entry.summary.clone(),
|
||||
});
|
||||
if hits.len() >= limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
hits
|
||||
}
|
||||
|
||||
pub fn read(&self, selector: ReadSelector<'_>, options: ReadOptions) -> ReadResult {
|
||||
let max_items = options.max_items.clamp(1, MAX_READ_MAX_ITEMS);
|
||||
let max_bytes = options.max_bytes.max(1);
|
||||
let mut entries = Vec::new();
|
||||
let mut bytes = 0usize;
|
||||
let mut truncated = false;
|
||||
|
||||
let selected: Vec<&ReferenceEntry> = match selector {
|
||||
ReadSelector::Id(id) => self
|
||||
.index
|
||||
.iter()
|
||||
.filter(|entry| entry.id.as_str() == id)
|
||||
.collect(),
|
||||
ReadSelector::EntryRange([start, end]) => self
|
||||
.index
|
||||
.iter()
|
||||
.filter(|entry| entry.entry_range[0] >= start && entry.entry_range[0] <= end)
|
||||
.collect(),
|
||||
};
|
||||
|
||||
for entry in selected {
|
||||
if entry.use_policy.is_some_and(|p| !p.observation_readable) {
|
||||
continue;
|
||||
}
|
||||
if entries.len() >= max_items {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
if entry.kind == ReferenceKind::Tool {
|
||||
if !options.include_tools {
|
||||
continue;
|
||||
}
|
||||
if let Some(actual) = entry.tool_part {
|
||||
if !options.tool_part.matches(actual) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
let Some(item) = self
|
||||
.entries
|
||||
.iter()
|
||||
.find(|captured| captured.dto.order == entry.entry_range[0])
|
||||
.map(|entry| &entry.item)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let remaining = max_bytes.saturating_sub(bytes);
|
||||
if remaining == 0 {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
let rendered = render_item(item, entry, options.detail, usize::MAX);
|
||||
truncated |= rendered.len() > remaining;
|
||||
let text = truncate_bytes(&rendered, remaining);
|
||||
bytes = bytes.saturating_add(text.len());
|
||||
entries.push(ReadEntry {
|
||||
id: entry.id.clone(),
|
||||
origin: entry.origin.clone(),
|
||||
kind: entry.kind,
|
||||
tool_part: entry.tool_part,
|
||||
tool_name: entry.tool_name.clone(),
|
||||
entry_range: entry.entry_range,
|
||||
label: entry.label.clone(),
|
||||
text,
|
||||
});
|
||||
if truncated {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ReadResult { entries, truncated }
|
||||
}
|
||||
|
||||
/// Resolves the committed tool-call input for a provider call id. This is
|
||||
/// used by explicit-memory receipts only after the containing run is durable,
|
||||
/// so a model-authored claim can cite its own committed tool invocation
|
||||
/// without inventing a SessionEntryRef.
|
||||
pub fn evidence_for_tool_call(&self, call_id: &str) -> Option<SessionEntryEvidence> {
|
||||
let captured = self.entries.iter().find(|entry| {
|
||||
matches!(
|
||||
&entry.item,
|
||||
Item::ToolCall {
|
||||
call_id: stored_call_id,
|
||||
..
|
||||
} if stored_call_id == call_id
|
||||
)
|
||||
})?;
|
||||
let reference = self.index.iter().find(|entry| {
|
||||
entry.entry_range[0] == captured.dto.order
|
||||
&& entry.kind == ReferenceKind::Tool
|
||||
&& entry.tool_part == Some(ToolPart::Input)
|
||||
})?;
|
||||
self.evidence_for(reference.id.as_str())
|
||||
}
|
||||
|
||||
pub fn evidence_for(&self, id: &str) -> Option<SessionEntryEvidence> {
|
||||
let entry = self.index.iter().find(|entry| entry.id.as_str() == id)?;
|
||||
if entry.use_policy.is_some_and(|p| !p.memory_evidence) {
|
||||
return None;
|
||||
}
|
||||
let captured = self
|
||||
.entries
|
||||
.iter()
|
||||
.find(|captured| captured.dto.order == entry.entry_range[0])?;
|
||||
let excerpt = render_item(&captured.item, entry, ReadDetail::Compact, 2 * 1024);
|
||||
Some(SessionEntryEvidence {
|
||||
segment_id: captured.dto.source.segment_id.clone(),
|
||||
entry_ref: entry.id.clone(),
|
||||
origin: entry.origin.clone(),
|
||||
entry_range: [captured.dto.source.entry_index; 2],
|
||||
kind: entry.kind,
|
||||
tool_part: entry.tool_part,
|
||||
label: entry.label.clone(),
|
||||
summary: entry.summary.clone(),
|
||||
excerpt,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn public_provenance(origin: &HistoryOrigin) -> SessionEntryProvenance {
|
||||
match origin {
|
||||
HistoryOrigin::HumanInput { .. } | HistoryOrigin::LocalHumanInput => {
|
||||
SessionEntryProvenance::HumanInput
|
||||
}
|
||||
HistoryOrigin::WorkerInput { .. } => SessionEntryProvenance::WorkerInput,
|
||||
HistoryOrigin::FlowInstruction { .. } => SessionEntryProvenance::FlowInstruction,
|
||||
HistoryOrigin::BackendInstruction { .. } | HistoryOrigin::Provided { .. } => {
|
||||
SessionEntryProvenance::BackendInstruction
|
||||
}
|
||||
HistoryOrigin::ModelOutput { .. } => SessionEntryProvenance::ModelOutput,
|
||||
HistoryOrigin::ToolOutput { .. } => SessionEntryProvenance::ToolOutput,
|
||||
HistoryOrigin::CompactionSummary
|
||||
| HistoryOrigin::DerivedSummary
|
||||
| HistoryOrigin::JournalContext => SessionEntryProvenance::DerivedSummary,
|
||||
HistoryOrigin::LegacyUnknown => SessionEntryProvenance::LegacyUnknown,
|
||||
}
|
||||
}
|
||||
|
||||
fn message_reference_kind(
|
||||
origin: &SessionEntryProvenance,
|
||||
provider_role: &Role,
|
||||
use_policy: Option<protocol::SessionEntryUsePolicy>,
|
||||
) -> Option<ReferenceKind> {
|
||||
if use_policy.is_some() {
|
||||
return match provider_role {
|
||||
Role::System => Some(ReferenceKind::System),
|
||||
Role::User => Some(ReferenceKind::User),
|
||||
Role::Assistant => Some(ReferenceKind::Assistant),
|
||||
};
|
||||
}
|
||||
match origin {
|
||||
SessionEntryProvenance::HumanInput | SessionEntryProvenance::WorkerInput => {
|
||||
Some(ReferenceKind::User)
|
||||
}
|
||||
SessionEntryProvenance::ModelOutput => Some(ReferenceKind::Assistant),
|
||||
SessionEntryProvenance::ToolOutput => Some(ReferenceKind::Tool),
|
||||
SessionEntryProvenance::LegacyUnknown => match provider_role {
|
||||
Role::User => Some(ReferenceKind::User),
|
||||
Role::Assistant => Some(ReferenceKind::Assistant),
|
||||
Role::System => None,
|
||||
},
|
||||
// Flow/backend/system content remains out of the observation surface
|
||||
// even when represented with a provider user/system role.
|
||||
SessionEntryProvenance::FlowInstruction
|
||||
| SessionEntryProvenance::BackendInstruction
|
||||
| SessionEntryProvenance::DerivedSummary => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn render_item(
|
||||
item: &Item,
|
||||
entry: &ReferenceEntry,
|
||||
detail: ReadDetail,
|
||||
max_bytes: usize,
|
||||
) -> String {
|
||||
let text = match item {
|
||||
Item::Message { role, content, .. } => {
|
||||
let text = content
|
||||
.iter()
|
||||
.map(|p| p.as_text())
|
||||
.collect::<Vec<_>>()
|
||||
.join("");
|
||||
format!("[{} {:?}] {text}", entry.id, role)
|
||||
}
|
||||
Item::ToolCall {
|
||||
name, arguments, ..
|
||||
} => match detail {
|
||||
ReadDetail::Compact => format!("[{} ToolInput {name}] (arguments omitted)", entry.id),
|
||||
ReadDetail::Full => format!("[{} ToolInput {name}]\narguments: {arguments}", entry.id),
|
||||
},
|
||||
Item::ToolResult {
|
||||
summary,
|
||||
content,
|
||||
attachments: _,
|
||||
is_error,
|
||||
..
|
||||
} => {
|
||||
let attachment_line = if entry.attachment_count == 0 {
|
||||
String::new()
|
||||
} else {
|
||||
format!("\nattachments: {} image(s)", entry.attachment_count)
|
||||
};
|
||||
match detail {
|
||||
ReadDetail::Compact => format!(
|
||||
"[{} ToolOutput{}]\nsummary: {summary}\ncontent: (omitted){attachment_line}",
|
||||
entry.id,
|
||||
if *is_error { " error" } else { "" },
|
||||
),
|
||||
ReadDetail::Full => format!(
|
||||
"[{} ToolOutput{}]\nsummary: {summary}\ncontent: {}{attachment_line}",
|
||||
entry.id,
|
||||
if *is_error { " error" } else { "" },
|
||||
content.as_deref().unwrap_or_default(),
|
||||
),
|
||||
}
|
||||
}
|
||||
Item::AssistantResponseBoundary { .. } => {
|
||||
format!("[{} Assistant response boundary omitted]", entry.id)
|
||||
}
|
||||
Item::Reasoning { .. } => format!("[{} Reasoning omitted]", entry.id),
|
||||
};
|
||||
truncate_bytes(&text, max_bytes)
|
||||
}
|
||||
|
||||
fn truncate_chars(text: &str, max_chars: usize) -> String {
|
||||
if max_chars == 0 {
|
||||
return "… [truncated]".to_string();
|
||||
}
|
||||
if text.chars().count() <= max_chars {
|
||||
return text.to_string();
|
||||
}
|
||||
let mut out = text.chars().take(max_chars).collect::<String>();
|
||||
out.push_str("… [truncated]");
|
||||
out
|
||||
}
|
||||
|
||||
fn truncate_bytes(text: &str, max_bytes: usize) -> String {
|
||||
if text.len() <= max_bytes {
|
||||
return text.to_owned();
|
||||
}
|
||||
let marker = "… [truncated]";
|
||||
let mut end = max_bytes.saturating_sub(marker.len());
|
||||
while !text.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
let mut out = text[..end].to_owned();
|
||||
if max_bytes >= marker.len() {
|
||||
out.push_str(marker);
|
||||
}
|
||||
out
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,425 @@
|
||||
//! Host capture and pure canonical slicing. No Job state or live-reader fallback lives here.
|
||||
|
||||
use super::*;
|
||||
use crate::committed_replay::CommittedPublicReplay;
|
||||
use crate::public_material::{
|
||||
PublicMaterialDescriptor, PublicMaterialError, PublicMaterialLimits, PublicMaterialUse,
|
||||
};
|
||||
use crate::retention::SessionRetentionStore;
|
||||
use crate::{SegmentId, SessionId};
|
||||
use std::path::Path;
|
||||
|
||||
/// Continuation in canonical Journal order, not an observation/Memory view cursor.
|
||||
/// A Host must reconcile the descriptor at this position with its saved checkpoint.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RecordingPosition {
|
||||
pub canonical_order: u64,
|
||||
pub byte_offset: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RecordingCaptureSlice {
|
||||
pub capture: CaptureDto,
|
||||
pub next: Option<RecordingPosition>,
|
||||
}
|
||||
|
||||
/// Inspect recording admission descriptors without constructing a body-bearing DTO.
|
||||
///
|
||||
/// The Host must authorize the source and hold the **same** Session publication/erasure
|
||||
/// guard from before reading its persisted active pointer until this function returns
|
||||
/// (including the callback). Pass that pointer's Segment, not a filename/mtime-derived
|
||||
/// selector. This function verifies the guard and stored Session identities but does
|
||||
/// not authorize the caller or establish that the supplied guard fences this root.
|
||||
/// No lock is recursively acquired, no active pointer is reread, and no replay handle
|
||||
/// or body-read capability is exposed to `inspect`.
|
||||
///
|
||||
/// The common committed recording replay fixes per-file ends before decoding. It
|
||||
/// retains only descriptors/locations, decoding one record at a time and streaming
|
||||
/// canonical bodies into their digests. `replay_limits` bound record allocation,
|
||||
/// aggregate storage extent, entries and Segments; there is no full-capture text/JSON
|
||||
/// allocation or `CaptureLimits` ceiling. The callback receives only descriptors in
|
||||
/// canonical order and may return a Host-specific mapping (including a `Result`). It
|
||||
/// runs once on success and not at all on replay/identity/resource errors.
|
||||
///
|
||||
/// This catalog is for Host admission, **not** a descriptor-only Job input. Construct
|
||||
/// the Job's bounded body input with the guarded slice API at its own authorized
|
||||
/// boundary; after saving it, execute only that immutable value. Neither descriptors
|
||||
/// nor a completed callback authorize delayed adoption after erasure.
|
||||
///
|
||||
/// ```no_run
|
||||
/// # use std::path::Path;
|
||||
/// # use session_store::{SessionId, SegmentId};
|
||||
/// # use session_store::retention::SessionRetentionGuard;
|
||||
/// # use session_store::public_material::PublicMaterialError;
|
||||
/// # fn admit(root: &Path, session: SessionId, active: SegmentId,
|
||||
/// # guard: &SessionRetentionGuard) -> Result<Vec<u64>, PublicMaterialError> {
|
||||
/// // `active` was resolved from persisted Host metadata while holding `guard`.
|
||||
/// session_store::capture::recording_catalog_worker_guarded(
|
||||
/// root, session, active, Default::default(), guard,
|
||||
/// |descriptors| descriptors.iter().map(|d| d.total_bytes).collect(),
|
||||
/// )
|
||||
/// # }
|
||||
/// ```
|
||||
pub fn recording_catalog_worker_guarded<T>(
|
||||
session_root: &Path,
|
||||
expected_session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
guard: &crate::retention::SessionRetentionGuard,
|
||||
inspect: impl FnOnce(&[PublicMaterialDescriptor]) -> T,
|
||||
) -> Result<T, PublicMaterialError> {
|
||||
if guard.session_id() != expected_session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
let replay = CommittedPublicReplay::open_worker_guarded(
|
||||
session_root,
|
||||
active_segment,
|
||||
replay_limits,
|
||||
None,
|
||||
PublicMaterialUse::Recording,
|
||||
)?;
|
||||
if replay.session_id() != expected_session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
Ok(inspect(&replay.into_recording_descriptors()))
|
||||
}
|
||||
|
||||
/// FsStore-layout counterpart of [`recording_catalog_worker_guarded`].
|
||||
///
|
||||
/// `sessions_root` is the directory containing per-Session directories. Invoke this
|
||||
/// inside the Host's `with_recording_boundary` callback, passing **its existing guard
|
||||
/// and persisted active Segment**. That same external retention control store must
|
||||
/// fence publishers, pointer changes, archive moves and erasure. Do not reacquire the
|
||||
/// guard or use a separately opened legacy public reader inside the boundary.
|
||||
/// All descriptor-only, allocation and callback guarantees of the Worker variant
|
||||
/// apply; this function does not construct an unbounded transient `CaptureDto`.
|
||||
pub fn recording_catalog_fs_guarded<T>(
|
||||
sessions_root: &Path,
|
||||
session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
guard: &crate::retention::SessionRetentionGuard,
|
||||
inspect: impl FnOnce(&[PublicMaterialDescriptor]) -> T,
|
||||
) -> Result<T, PublicMaterialError> {
|
||||
if guard.session_id() != session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
let replay = CommittedPublicReplay::open_fs_guarded(
|
||||
sessions_root,
|
||||
session,
|
||||
active_segment,
|
||||
replay_limits,
|
||||
)?;
|
||||
Ok(inspect(&replay.into_recording_descriptors()))
|
||||
}
|
||||
|
||||
/// Capture the selected original lineage, including typed Host run boundaries.
|
||||
/// The caller must hold the Session publication/erasure guard while selecting the
|
||||
/// active pointer and through this call. Runtime publishers use this same lock
|
||||
/// for appends and Segment creation. No lock is recursively acquired here.
|
||||
pub fn capture_recording_worker_guarded(
|
||||
session_root: &Path,
|
||||
expected_session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
capture_limits: CaptureLimits,
|
||||
guard: &crate::retention::SessionRetentionGuard,
|
||||
) -> Result<CaptureDto, PublicMaterialError> {
|
||||
if guard.session_id() != expected_session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
let replay = CommittedPublicReplay::open_worker_guarded(
|
||||
session_root,
|
||||
active_segment,
|
||||
replay_limits,
|
||||
None,
|
||||
PublicMaterialUse::Recording,
|
||||
)?;
|
||||
if replay.session_id() != expected_session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
replay.into_recording_dto(capture_limits)
|
||||
}
|
||||
|
||||
/// Standalone convenience entry; Hosts resolving a mutable active pointer should
|
||||
/// instead hold the fence while resolving it and call the guarded entry.
|
||||
pub fn capture_recording_worker(
|
||||
session_root: &Path,
|
||||
expected_session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
retention: &SessionRetentionStore,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
capture_limits: CaptureLimits,
|
||||
) -> Result<CaptureDto, PublicMaterialError> {
|
||||
let guard = retention
|
||||
.lock_live(expected_session)
|
||||
.map_err(|error| match error {
|
||||
crate::StoreError::SessionErased(_) => PublicMaterialError::Erased,
|
||||
_ => PublicMaterialError::Unavailable,
|
||||
})?;
|
||||
capture_recording_worker_guarded(
|
||||
session_root,
|
||||
expected_session,
|
||||
active_segment,
|
||||
replay_limits,
|
||||
capture_limits,
|
||||
&guard,
|
||||
)
|
||||
}
|
||||
|
||||
/// Bounded construction for one Job input: descriptor replay is bounded separately;
|
||||
/// only at most the requested body budget is materialized from the fixed files.
|
||||
pub fn capture_recording_slice_worker_guarded(
|
||||
session_root: &Path,
|
||||
expected_session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
start: RecordingPosition,
|
||||
max_entries: usize,
|
||||
max_serialized_bytes: usize,
|
||||
guard: &crate::retention::SessionRetentionGuard,
|
||||
) -> Result<RecordingCaptureSlice, PublicMaterialError> {
|
||||
if guard.session_id() != expected_session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
let replay = CommittedPublicReplay::open_worker_guarded(
|
||||
session_root,
|
||||
active_segment,
|
||||
replay_limits,
|
||||
None,
|
||||
PublicMaterialUse::Recording,
|
||||
)?;
|
||||
if replay.session_id() != expected_session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
replay.recording_slice(start, max_entries, max_serialized_bytes)
|
||||
}
|
||||
|
||||
/// The standalone FsStore layout uses the same committed replay and fixed values.
|
||||
pub fn capture_recording_fs_guarded(
|
||||
sessions_root: &Path,
|
||||
session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
capture_limits: CaptureLimits,
|
||||
guard: &crate::retention::SessionRetentionGuard,
|
||||
) -> Result<CaptureDto, PublicMaterialError> {
|
||||
if guard.session_id() != session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
CommittedPublicReplay::open_fs_guarded(sessions_root, session, active_segment, replay_limits)?
|
||||
.into_recording_dto(capture_limits)
|
||||
}
|
||||
|
||||
/// Bounded standalone capture; the caller holds its FsStore retention fence.
|
||||
pub fn capture_recording_slice_fs_guarded(
|
||||
sessions_root: &Path,
|
||||
session: SessionId,
|
||||
active_segment: SegmentId,
|
||||
replay_limits: PublicMaterialLimits,
|
||||
start: RecordingPosition,
|
||||
max_entries: usize,
|
||||
max_serialized_bytes: usize,
|
||||
guard: &crate::retention::SessionRetentionGuard,
|
||||
) -> Result<RecordingCaptureSlice, PublicMaterialError> {
|
||||
if guard.session_id() != session {
|
||||
return Err(PublicMaterialError::Inconsistent);
|
||||
}
|
||||
CommittedPublicReplay::open_fs_guarded(sessions_root, session, active_segment, replay_limits)?
|
||||
.recording_slice(start, max_entries, max_serialized_bytes)
|
||||
}
|
||||
|
||||
impl CaptureDto {
|
||||
/// Select the oldest canonical prefix fitting both budgets, including JSON
|
||||
/// escaping, metadata and continuation. Only immutable owned bytes are read.
|
||||
/// A later Job can recapture at `next`; never combine live pages into this value.
|
||||
pub fn recording_slice(
|
||||
&self,
|
||||
start: RecordingPosition,
|
||||
max_entries: usize,
|
||||
max_serialized_bytes: usize,
|
||||
) -> Result<RecordingCaptureSlice, CaptureError> {
|
||||
if max_entries == 0 {
|
||||
return Err(CaptureError::BudgetTooSmall);
|
||||
}
|
||||
if start.canonical_order > self.view_entry_count {
|
||||
return Err(CaptureError::InvalidRange);
|
||||
}
|
||||
let mut result = RecordingCaptureSlice {
|
||||
capture: CaptureDto {
|
||||
version: self.version,
|
||||
session_id: self.session_id.clone(),
|
||||
segment_id: self.segment_id.clone(),
|
||||
view_entry_count: self.view_entry_count,
|
||||
entries: Vec::new(),
|
||||
},
|
||||
next: (start.canonical_order < self.view_entry_count).then_some(start),
|
||||
};
|
||||
if body::json_size(&result, max_serialized_bytes).is_err() {
|
||||
return Err(CaptureError::BudgetTooSmall);
|
||||
}
|
||||
if start.canonical_order == self.view_entry_count {
|
||||
if start.byte_offset != 0 {
|
||||
return Err(CaptureError::InvalidRange);
|
||||
}
|
||||
result.capture.validate(CaptureLimits {
|
||||
max_bytes: max_serialized_bytes,
|
||||
max_text_bytes: max_serialized_bytes,
|
||||
max_entries,
|
||||
..Default::default()
|
||||
})?;
|
||||
return Ok(result);
|
||||
}
|
||||
let first = self
|
||||
.entries
|
||||
.iter()
|
||||
.position(|entry| entry.order == start.canonical_order)
|
||||
.ok_or(CaptureError::InvalidRange)?;
|
||||
let mut expected_order = start.canonical_order;
|
||||
for entry in self.entries.iter().skip(first).take(max_entries) {
|
||||
if entry.order != expected_order {
|
||||
return Err(CaptureError::InvalidRange);
|
||||
}
|
||||
let CaptureBody::CanonicalPublic {
|
||||
descriptor,
|
||||
byte_offset,
|
||||
text,
|
||||
boundary,
|
||||
} = &entry.body
|
||||
else {
|
||||
return Err(CaptureError::Invalid("not a recording capture"));
|
||||
};
|
||||
if !entry.policy.recording_material {
|
||||
return Err(CaptureError::PolicyDenied);
|
||||
}
|
||||
let offset = if entry.order == start.canonical_order {
|
||||
start.byte_offset
|
||||
} else {
|
||||
*byte_offset
|
||||
};
|
||||
let local = offset
|
||||
.checked_sub(*byte_offset)
|
||||
.and_then(|o| usize::try_from(o).ok())
|
||||
.ok_or(CaptureError::InvalidRange)?;
|
||||
if local > text.len()
|
||||
|| !text.is_char_boundary(local)
|
||||
|| offset >= descriptor.total_bytes
|
||||
{
|
||||
return Err(CaptureError::InvalidRange);
|
||||
}
|
||||
let item = CaptureEntryDto {
|
||||
entry_id: entry.entry_id.clone(),
|
||||
order: entry.order,
|
||||
source: entry.source.clone(),
|
||||
policy: entry.policy,
|
||||
body: CaptureBody::CanonicalPublic {
|
||||
descriptor: descriptor.clone(),
|
||||
byte_offset: offset,
|
||||
text: String::new(),
|
||||
boundary: *boundary,
|
||||
},
|
||||
};
|
||||
result.capture.entries.push(item);
|
||||
// At most budget bytes are considered before allocating a trial fragment.
|
||||
let mut low = 0;
|
||||
let mut high = text.len().saturating_sub(local).min(max_serialized_bytes);
|
||||
let prior_next = result.next;
|
||||
while low < high {
|
||||
let mid = low + (high - low).div_ceil(2);
|
||||
let mut end = local + mid;
|
||||
while !text.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
if let CaptureBody::CanonicalPublic { text: fragment, .. } =
|
||||
&mut result.capture.entries.last_mut().unwrap().body
|
||||
{
|
||||
*fragment = text[local..end].to_owned();
|
||||
}
|
||||
let next_offset = offset + (end - local) as u64;
|
||||
result.next = if next_offset < descriptor.total_bytes {
|
||||
Some(RecordingPosition {
|
||||
canonical_order: entry.order,
|
||||
byte_offset: next_offset,
|
||||
})
|
||||
} else {
|
||||
(entry.order + 1 < self.view_entry_count).then_some(RecordingPosition {
|
||||
canonical_order: entry.order + 1,
|
||||
byte_offset: 0,
|
||||
})
|
||||
};
|
||||
if body::json_size(&result, max_serialized_bytes).is_ok() {
|
||||
low = mid;
|
||||
} else {
|
||||
high = mid - 1;
|
||||
}
|
||||
}
|
||||
let mut end = local + low;
|
||||
while !text.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
if end == local {
|
||||
result.capture.entries.pop();
|
||||
result.next = prior_next;
|
||||
if result.capture.entries.is_empty() {
|
||||
return Err(CaptureError::BudgetTooSmall);
|
||||
}
|
||||
break;
|
||||
}
|
||||
if let CaptureBody::CanonicalPublic { text: fragment, .. } =
|
||||
&mut result.capture.entries.last_mut().unwrap().body
|
||||
{
|
||||
*fragment = text[local..end].to_owned();
|
||||
}
|
||||
let next_offset = offset + (end - local) as u64;
|
||||
result.next = if next_offset < descriptor.total_bytes {
|
||||
Some(RecordingPosition {
|
||||
canonical_order: entry.order,
|
||||
byte_offset: next_offset,
|
||||
})
|
||||
} else {
|
||||
(entry.order + 1 < self.view_entry_count).then_some(RecordingPosition {
|
||||
canonical_order: entry.order + 1,
|
||||
byte_offset: 0,
|
||||
})
|
||||
};
|
||||
if next_offset < descriptor.total_bytes {
|
||||
break;
|
||||
}
|
||||
expected_order = entry.order + 1;
|
||||
}
|
||||
body::json_size(&result, max_serialized_bytes).map_err(|_| CaptureError::BudgetTooSmall)?;
|
||||
result.capture.validate(CaptureLimits {
|
||||
max_bytes: max_serialized_bytes,
|
||||
max_text_bytes: max_serialized_bytes,
|
||||
max_entries,
|
||||
..Default::default()
|
||||
})?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Resolve canonical text already in the value. No original storage is consulted.
|
||||
pub fn recording_text(
|
||||
&self,
|
||||
order: u64,
|
||||
) -> Result<(&crate::public_material::PublicMaterialDescriptor, u64, &str), CaptureError> {
|
||||
let entry = self
|
||||
.entries
|
||||
.iter()
|
||||
.find(|entry| entry.order == order)
|
||||
.ok_or(CaptureError::UnknownReference)?;
|
||||
if !entry.policy.recording_material {
|
||||
return Err(CaptureError::PolicyDenied);
|
||||
}
|
||||
match &entry.body {
|
||||
CaptureBody::CanonicalPublic {
|
||||
descriptor,
|
||||
byte_offset,
|
||||
text,
|
||||
..
|
||||
} => Ok((descriptor, *byte_offset, text)),
|
||||
_ => Err(CaptureError::Invalid("not canonical recording text")),
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -38,6 +38,8 @@ pub(crate) fn read_without_atime_bounded(path: &Path, max_bytes: u64) -> io::Res
|
||||
}
|
||||
}
|
||||
|
||||
pub mod capture;
|
||||
mod committed_replay;
|
||||
pub mod event_trace;
|
||||
pub mod fs_store;
|
||||
pub mod history;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -46,6 +46,13 @@ pub struct SessionRetentionStore {
|
||||
/// Do not recursively acquire the same Session lock inside a guarded callback.
|
||||
pub struct SessionRetentionGuard {
|
||||
_file: File,
|
||||
session_id: SessionId,
|
||||
}
|
||||
|
||||
impl SessionRetentionGuard {
|
||||
pub fn session_id(&self) -> SessionId {
|
||||
self.session_id
|
||||
}
|
||||
}
|
||||
|
||||
impl SessionRetentionStore {
|
||||
@@ -71,7 +78,10 @@ impl SessionRetentionStore {
|
||||
}
|
||||
let file = options.open(self.root.join(format!("{session}.lock")))?;
|
||||
FileExt::lock_exclusive(&file)?;
|
||||
Ok(SessionRetentionGuard { _file: file })
|
||||
Ok(SessionRetentionGuard {
|
||||
_file: file,
|
||||
session_id: session,
|
||||
})
|
||||
}
|
||||
|
||||
fn read(&self, session: SessionId) -> Result<RetentionState, StoreError> {
|
||||
|
||||
@@ -194,6 +194,21 @@ impl WorkerSessionStore {
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
/// Open a current retained Session without creating or migrating anything.
|
||||
/// The live constraint is checked here and again by every later publication;
|
||||
/// attaching the control store itself does not acquire publication authority.
|
||||
pub fn open_existing_with_retention(
|
||||
root: impl Into<PathBuf>,
|
||||
retention: crate::retention::SessionRetentionStore,
|
||||
) -> Result<Self, StoreError> {
|
||||
let mut store = Self::open_read_only(root)?;
|
||||
if let Some(session) = *store.session_id.lock().unwrap() {
|
||||
retention.ensure_live(session)?;
|
||||
}
|
||||
store.retention = Some(retention);
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
fn lock_publication(
|
||||
&self,
|
||||
session: SessionId,
|
||||
|
||||
@@ -0,0 +1,711 @@
|
||||
//! Public consumer boundary: real capture adapters, checked DTO codec and pure readers.
|
||||
//! No Worker, persistence, live Session provider, or substituted capture implementation.
|
||||
|
||||
use agen::{HistoryEntry, Item};
|
||||
use protocol::{
|
||||
SessionEntryProvenance, SessionSnapshot, SessionSnapshotEntry, SessionSnapshotEntryData,
|
||||
};
|
||||
use session_store::capture::*;
|
||||
use session_store::{
|
||||
FeatureInputPayload, HistoryInputMetadata, HistoryInputUsePolicy, InputCompactionUse,
|
||||
InputEvidenceUse, InputObservationUse, LoggedHistoryDerivation, LoggedSessionHistoryEntryId,
|
||||
LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin, PublicSystemItem,
|
||||
PublicSystemPresentation,
|
||||
};
|
||||
|
||||
fn history(
|
||||
id: &str,
|
||||
item: Item,
|
||||
input_metadata: Option<HistoryInputMetadata>,
|
||||
) -> HistoryEntry<LoggedSessionHistoryMetadata> {
|
||||
HistoryEntry::new(
|
||||
item,
|
||||
LoggedSessionHistoryMetadata {
|
||||
entry_id: LoggedSessionHistoryEntryId(id.into()),
|
||||
origin: LoggedSessionHistoryOrigin::LocalHumanInput,
|
||||
derivation: Some(LoggedHistoryDerivation {
|
||||
sources: vec![LoggedSessionHistoryEntryId("earlier-entry".into())],
|
||||
}),
|
||||
input_metadata,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn usage(recording: bool, memory: bool, observation: bool) -> HistoryInputUsePolicy {
|
||||
HistoryInputUsePolicy {
|
||||
recording_material: if recording {
|
||||
InputEvidenceUse::Include
|
||||
} else {
|
||||
InputEvidenceUse::Exclude
|
||||
},
|
||||
memory_evidence: if memory {
|
||||
InputEvidenceUse::Include
|
||||
} else {
|
||||
InputEvidenceUse::Exclude
|
||||
},
|
||||
observation: if observation {
|
||||
InputObservationUse::Readable
|
||||
} else {
|
||||
InputObservationUse::Hidden
|
||||
},
|
||||
compaction: InputCompactionUse::Retain,
|
||||
}
|
||||
}
|
||||
|
||||
fn reference(capture: &SessionCapture, id: &str) -> ScopedSessionEntryRef {
|
||||
capture
|
||||
.scoped_ref(&SessionEntryRef::parse(id).unwrap())
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_consumer_roundtrip_retains_body_original_source_order_origin_and_independent_policy() {
|
||||
let entries = vec![
|
||||
history(
|
||||
"recording",
|
||||
Item::user_message("record-only"),
|
||||
Some(HistoryInputMetadata::new(usage(true, false, false), None, None).unwrap()),
|
||||
),
|
||||
history(
|
||||
"memory",
|
||||
Item::user_message("memory-only"),
|
||||
Some(HistoryInputMetadata::new(usage(false, true, false), None, None).unwrap()),
|
||||
),
|
||||
history(
|
||||
"observation",
|
||||
Item::user_message("observation-only"),
|
||||
Some(HistoryInputMetadata::new(usage(false, false, true), None, None).unwrap()),
|
||||
),
|
||||
];
|
||||
let capture = SessionCapture::from_complete_history("synthetic-session", "selected", entries);
|
||||
let mut dto = capture.to_dto();
|
||||
// The checked boundary also accepts entries retained from different original segments.
|
||||
dto.entries[0].source.segment_id = "original-a".into();
|
||||
dto.entries[0].source.entry_index = 80;
|
||||
dto.entries[1].source.segment_id = "original-b".into();
|
||||
dto.entries[1].source.entry_index = 3;
|
||||
let capture = SessionCapture::try_from_dto(dto.clone(), CaptureLimits::default()).unwrap();
|
||||
let encoded = capture.encode(CaptureLimits::default()).unwrap();
|
||||
let restored = SessionCapture::decode(&encoded, CaptureLimits::default()).unwrap();
|
||||
assert_eq!(restored.to_dto(), dto);
|
||||
assert_eq!(restored.capture_id(), capture.capture_id());
|
||||
for (id, allowed) in [
|
||||
("Erecording", CapturePurpose::Recording),
|
||||
("Ememory", CapturePurpose::MemoryEvidence),
|
||||
("Eobservation", CapturePurpose::Observation),
|
||||
] {
|
||||
let reference = reference(&capture, id);
|
||||
assert_eq!(
|
||||
restored.resolve_ref(&reference, allowed).unwrap(),
|
||||
capture.resolve_ref(&reference, allowed).unwrap()
|
||||
);
|
||||
for purpose in [
|
||||
CapturePurpose::Recording,
|
||||
CapturePurpose::MemoryEvidence,
|
||||
CapturePurpose::Observation,
|
||||
] {
|
||||
assert_eq!(
|
||||
restored.resolve_ref(&reference, purpose).is_ok(),
|
||||
purpose == allowed
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_eq!(restored.overview().len(), 1);
|
||||
assert_eq!(restored.search(&SearchOptions::default()).len(), 1);
|
||||
assert!(restored.evidence_for("Erecording").is_none());
|
||||
let evidence = restored.evidence_for("Ememory").unwrap();
|
||||
assert_eq!(evidence.segment_id, "original-b");
|
||||
assert_eq!(evidence.entry_range, [3, 3]);
|
||||
assert!(evidence.excerpt.contains("memory-only"));
|
||||
let recorded = restored
|
||||
.slice(CapturePurpose::Recording, None, 10, 16_000)
|
||||
.unwrap();
|
||||
assert_eq!(recorded.slice().items.len(), 1);
|
||||
assert_eq!(recorded.slice().items[0].text, "record-only");
|
||||
for (purpose, expected) in [
|
||||
(CapturePurpose::MemoryEvidence, "memory-only"),
|
||||
(CapturePurpose::Observation, "observation-only"),
|
||||
] {
|
||||
let slice = restored.slice(purpose, None, 10, 16_000).unwrap();
|
||||
assert_eq!(slice.slice().items.len(), 1);
|
||||
assert_eq!(slice.slice().items[0].text, expected);
|
||||
assert!(
|
||||
restored
|
||||
.read_raw(
|
||||
&reference(&restored, "Erecording"),
|
||||
purpose,
|
||||
RawField::MessagePart(0),
|
||||
0,
|
||||
100
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn common_data_drops_reasoning_raw_system_and_private_payload_even_when_unindexed() {
|
||||
let payload = FeatureInputPayload::encode(
|
||||
"capture-test",
|
||||
1,
|
||||
&serde_json::json!({"secret": "PRIVATE-PAYLOAD"}),
|
||||
)
|
||||
.unwrap();
|
||||
let public = PublicSystemItem {
|
||||
kind: "notice".into(),
|
||||
body: "public notice".into(),
|
||||
presentation: PublicSystemPresentation::Ignored,
|
||||
};
|
||||
let system =
|
||||
HistoryInputMetadata::new(usage(true, true, true), Some(payload.clone()), Some(public))
|
||||
.unwrap();
|
||||
let private = HistoryInputMetadata::new(usage(true, true, true), Some(payload), None).unwrap();
|
||||
let capture = SessionCapture::from_complete_history(
|
||||
"s",
|
||||
"segment",
|
||||
vec![
|
||||
history("raw", Item::system_message("RAW-SYSTEM-PROMPT"), None),
|
||||
history("reasoning", Item::reasoning("HIDDEN-REASONING"), None),
|
||||
history(
|
||||
"private",
|
||||
Item::system_message("PRIVATE-SYSTEM-BODY"),
|
||||
Some(private),
|
||||
),
|
||||
history(
|
||||
"public",
|
||||
Item::system_message("MODEL-SYSTEM-BODY"),
|
||||
Some(system),
|
||||
),
|
||||
history("ordinary", Item::user_message("ordinary"), None),
|
||||
],
|
||||
);
|
||||
let encoded = String::from_utf8(capture.encode(CaptureLimits::default()).unwrap()).unwrap();
|
||||
let retained_debug = format!("{capture:?}");
|
||||
for secret in [
|
||||
"RAW-SYSTEM-PROMPT",
|
||||
"HIDDEN-REASONING",
|
||||
"PRIVATE-SYSTEM-BODY",
|
||||
"MODEL-SYSTEM-BODY",
|
||||
"PRIVATE-PAYLOAD",
|
||||
] {
|
||||
assert!(!encoded.contains(secret));
|
||||
assert!(!retained_debug.contains(secret));
|
||||
}
|
||||
assert_eq!(capture.to_dto().entries.len(), 2);
|
||||
assert!(encoded.contains("public notice"));
|
||||
// Filtered entries do not renumber the public source positions.
|
||||
assert_eq!(capture.to_dto().entries[0].order, 3);
|
||||
assert_eq!(
|
||||
capture.view_order_for_ref(&SessionEntryRef::parse("Eordinary").unwrap()),
|
||||
Some(4)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protocol_projection_does_not_invent_recording_permission() {
|
||||
let capture = SessionCapture::from_session_snapshot(
|
||||
"s",
|
||||
"seg",
|
||||
SessionSnapshot {
|
||||
pending_submissions: Default::default(),
|
||||
entries: vec![SessionSnapshotEntry {
|
||||
entry_id: "projected".into(),
|
||||
timestamp: 0,
|
||||
provenance: SessionEntryProvenance::HumanInput,
|
||||
derived_from: vec!["source".into()],
|
||||
use_policy: Some(protocol::SessionEntryUsePolicy {
|
||||
observation_readable: true,
|
||||
memory_evidence: false,
|
||||
}),
|
||||
data: SessionSnapshotEntryData::Message {
|
||||
role: protocol::SessionMessageRole::User,
|
||||
content: vec![protocol::SessionContentPart::Text {
|
||||
text: "projected".into(),
|
||||
}],
|
||||
},
|
||||
}],
|
||||
},
|
||||
);
|
||||
let reference = reference(&capture, "Eprojected");
|
||||
assert!(matches!(
|
||||
capture.resolve_ref(&reference, CapturePurpose::Recording),
|
||||
Err(CaptureError::PolicyDenied)
|
||||
));
|
||||
assert!(
|
||||
capture
|
||||
.resolve_ref(&reference, CapturePurpose::Observation)
|
||||
.is_ok()
|
||||
);
|
||||
assert!(capture.evidence_for("Eprojected").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_dtos_and_oversized_representations_are_rejected() {
|
||||
let capture = SessionCapture::new("s", "seg", vec![Item::user_message("body")]);
|
||||
let original = serde_json::to_value(capture.to_dto()).unwrap();
|
||||
let mut invalids = Vec::new();
|
||||
let mut value = original.clone();
|
||||
value["version"] = 999.into();
|
||||
invalids.push(value);
|
||||
let mut value = original.clone();
|
||||
value["unexpected"] = true.into();
|
||||
invalids.push(value);
|
||||
let mut value = original.clone();
|
||||
value["entries"][0]["policy"]["recording_material"] = "include".into();
|
||||
invalids.push(value);
|
||||
let mut value = original.clone();
|
||||
value["entries"][0]["policy"]
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("memory_evidence");
|
||||
invalids.push(value);
|
||||
let mut value = original.clone();
|
||||
value["entries"][0]["body"] = serde_json::json!({"kind":"reasoning","text":"secret"});
|
||||
invalids.push(value);
|
||||
let mut value = original.clone();
|
||||
value["entries"][0]["body"]["feature_payload"] = serde_json::json!({"private":true});
|
||||
invalids.push(value);
|
||||
let mut value = original.clone();
|
||||
value["entries"][0]["body"]["role"] = "system".into();
|
||||
invalids.push(value);
|
||||
let mut value = original.clone();
|
||||
value["entries"][0]["source"]["unknown"] = true.into();
|
||||
invalids.push(value);
|
||||
let mut value = original.clone();
|
||||
value["entries"][0]["source"]["history_origin"] =
|
||||
serde_json::json!({"kind":"legacy_unknown","unknown":true});
|
||||
invalids.push(value);
|
||||
let mut value = original.clone();
|
||||
value["entries"][0]["entry_id"] = "bad/id".into();
|
||||
invalids.push(value);
|
||||
let mut value = original.clone();
|
||||
value["entries"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
.push(original["entries"][0].clone());
|
||||
invalids.push(value);
|
||||
for invalid in invalids {
|
||||
let bytes = serde_json::to_vec(&invalid).unwrap();
|
||||
assert!(
|
||||
SessionCapture::decode(&bytes, CaptureLimits::default()).is_err(),
|
||||
"accepted {invalid}"
|
||||
);
|
||||
}
|
||||
let bytes = capture.encode(CaptureLimits::default()).unwrap();
|
||||
assert!(
|
||||
SessionCapture::decode(
|
||||
&bytes,
|
||||
CaptureLimits {
|
||||
max_bytes: bytes.len() - 1,
|
||||
..CaptureLimits::default()
|
||||
}
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
capture
|
||||
.encode(CaptureLimits {
|
||||
max_entries: 0,
|
||||
..CaptureLimits::default()
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
capture
|
||||
.encode(CaptureLimits {
|
||||
max_text_bytes: 3,
|
||||
..CaptureLimits::default()
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
let mut reversed = capture.to_dto();
|
||||
reversed.view_entry_count = 2;
|
||||
let mut second = reversed.entries[0].clone();
|
||||
second.entry_id = "second".into();
|
||||
second.source.entry_index = 4;
|
||||
second.order = 0;
|
||||
reversed.entries[0].order = 1;
|
||||
reversed.entries.push(second);
|
||||
assert!(SessionCapture::try_from_dto(reversed, CaptureLimits::default()).is_err());
|
||||
let mut duplicate_source = capture.to_dto();
|
||||
duplicate_source.view_entry_count = 2;
|
||||
let mut second = duplicate_source.entries[0].clone();
|
||||
second.entry_id = "distinct-id".into();
|
||||
second.order = 1;
|
||||
duplicate_source.entries.push(second);
|
||||
assert!(matches!(
|
||||
SessionCapture::try_from_dto(duplicate_source, CaptureLimits::default()),
|
||||
Err(CaptureError::Invalid("duplicate source position"))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_utf8_tool_input_and_result_are_lossless_without_preview_markers() {
|
||||
let arguments = "\"quoted\"\\path\n日本語🙂".repeat(300);
|
||||
let result = "result\n\t\\\"λ🙂".repeat(400);
|
||||
let capture = SessionCapture::new(
|
||||
"s",
|
||||
"seg",
|
||||
vec![
|
||||
Item::tool_call("call", "tool", arguments.clone()),
|
||||
Item::tool_result_item("call", "summary", Some(result.clone()), false),
|
||||
],
|
||||
);
|
||||
for (id, field, expected) in [
|
||||
("E00000000", RawField::ToolArguments, arguments),
|
||||
("E00000001", RawField::ToolContent, result),
|
||||
] {
|
||||
let reference = reference(&capture, id);
|
||||
let mut offset = 0;
|
||||
let mut actual = String::new();
|
||||
loop {
|
||||
let part = capture
|
||||
.read_raw(&reference, CapturePurpose::Observation, field, offset, 17)
|
||||
.unwrap();
|
||||
assert!(part.text.len() <= 17);
|
||||
assert_eq!(part.offset, offset);
|
||||
actual.push_str(&part.text);
|
||||
match part.next_offset {
|
||||
Some(next) => {
|
||||
assert!(next > offset);
|
||||
offset = next;
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
let unicode = SessionCapture::new("s", "seg", vec![Item::tool_call("call", "tool", "🙂")]);
|
||||
let reference = reference(&unicode, "E00000000");
|
||||
assert!(matches!(
|
||||
unicode.read_raw(
|
||||
&reference,
|
||||
CapturePurpose::Observation,
|
||||
RawField::ToolArguments,
|
||||
1,
|
||||
8
|
||||
),
|
||||
Err(CaptureError::InvalidRange)
|
||||
));
|
||||
assert!(matches!(
|
||||
unicode.read_raw(
|
||||
&reference,
|
||||
CapturePurpose::Observation,
|
||||
RawField::ToolArguments,
|
||||
0,
|
||||
3
|
||||
),
|
||||
Err(CaptureError::BudgetTooSmall)
|
||||
));
|
||||
assert!(matches!(
|
||||
unicode.read_raw(
|
||||
&reference,
|
||||
CapturePurpose::Observation,
|
||||
RawField::ToolArguments,
|
||||
5,
|
||||
8
|
||||
),
|
||||
Err(CaptureError::InvalidRange)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn escaped_json_slice_budget_includes_metadata_and_bound_continuation() {
|
||||
let body = "\"\\\n\t🙂日本語".repeat(500);
|
||||
let capture = SessionCapture::new(
|
||||
"synthetic-session",
|
||||
"segment",
|
||||
vec![Item::tool_call("call", "tool", body.clone())],
|
||||
);
|
||||
let other_session = SessionCapture::new(
|
||||
"other-session",
|
||||
"segment",
|
||||
vec![Item::tool_call("call", "tool", body.clone())],
|
||||
);
|
||||
let changed = SessionCapture::new(
|
||||
"synthetic-session",
|
||||
"segment",
|
||||
vec![Item::tool_call("call", "tool", "changed")],
|
||||
);
|
||||
let mut cursor = None;
|
||||
let mut actual = String::new();
|
||||
let budget = 1400;
|
||||
loop {
|
||||
let part = capture
|
||||
.slice(CapturePurpose::Recording, cursor.as_ref(), 1, budget)
|
||||
.unwrap();
|
||||
assert!(part.encoded_len() <= budget);
|
||||
assert_eq!(part.encoded_len(), part.as_json().len());
|
||||
assert_eq!(serde_json::to_vec(part.slice()).unwrap(), part.as_json());
|
||||
assert_eq!(part.slice().items.len(), 1);
|
||||
let item = &part.slice().items[0];
|
||||
assert_eq!(item.offset, actual.len());
|
||||
actual.push_str(&item.text);
|
||||
cursor = part.slice().next_cursor.clone();
|
||||
if let Some(cursor) = &cursor {
|
||||
assert!(matches!(
|
||||
other_session.slice(CapturePurpose::Recording, Some(cursor), 1, budget),
|
||||
Err(CaptureError::ForeignReference)
|
||||
));
|
||||
assert!(matches!(
|
||||
changed.slice(CapturePurpose::Recording, Some(cursor), 1, budget),
|
||||
Err(CaptureError::ForeignReference)
|
||||
));
|
||||
assert!(matches!(
|
||||
capture.slice(CapturePurpose::Observation, Some(cursor), 1, budget),
|
||||
Err(CaptureError::ForeignReference)
|
||||
));
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert_eq!(actual, body);
|
||||
assert!(matches!(
|
||||
capture.slice(CapturePurpose::Recording, None, 1, 8),
|
||||
Err(CaptureError::BudgetTooSmall)
|
||||
));
|
||||
assert!(matches!(
|
||||
capture.slice(CapturePurpose::Recording, None, 0, budget),
|
||||
Err(CaptureError::BudgetTooSmall)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_immutability_and_scoped_operations_reject_foreign_or_unknown_refs_and_bad_ranges() {
|
||||
let mut source = vec![
|
||||
Item::user_message("before"),
|
||||
Item::assistant_message("answer"),
|
||||
];
|
||||
let capture = SessionCapture::new("s", "seg", source.clone());
|
||||
let fixed_bytes = capture.encode(CaptureLimits::default()).unwrap();
|
||||
let first = reference(&capture, "E00000000");
|
||||
let second = reference(&capture, "E00000001");
|
||||
source.push(Item::user_message("appended"));
|
||||
source[0] = Item::user_message("compacted");
|
||||
source.clear();
|
||||
assert_eq!(
|
||||
capture.encode(CaptureLimits::default()).unwrap(),
|
||||
fixed_bytes
|
||||
);
|
||||
assert_eq!(
|
||||
capture.clone().encode(CaptureLimits::default()).unwrap(),
|
||||
fixed_bytes
|
||||
);
|
||||
let foreign = SessionCapture::new(
|
||||
"different",
|
||||
"seg",
|
||||
vec![
|
||||
Item::user_message("before"),
|
||||
Item::assistant_message("answer"),
|
||||
],
|
||||
);
|
||||
assert!(matches!(
|
||||
foreign.resolve_ref(&first, CapturePurpose::Observation),
|
||||
Err(CaptureError::ForeignReference)
|
||||
));
|
||||
assert!(matches!(
|
||||
capture.checked_range(&second, &first, CapturePurpose::Observation),
|
||||
Err(CaptureError::InvalidRange)
|
||||
));
|
||||
assert_eq!(
|
||||
capture
|
||||
.checked_range(&first, &second, CapturePurpose::Observation)
|
||||
.unwrap()
|
||||
.len(),
|
||||
2
|
||||
);
|
||||
let unknown = SessionEntryRef::parse("E99999999").unwrap();
|
||||
assert_eq!(capture.view_order_for_ref(&unknown), None);
|
||||
assert!(matches!(
|
||||
capture.scoped_ref(&unknown),
|
||||
Err(CaptureError::UnknownReference)
|
||||
));
|
||||
assert!(
|
||||
capture
|
||||
.search_checked(&SearchOptions {
|
||||
from: Some(unknown.clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
capture
|
||||
.search(&SearchOptions {
|
||||
through: Some(unknown),
|
||||
..Default::default()
|
||||
})
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
capture
|
||||
.read_checked(ReadSelector::EntryRange([1, 0]), ReadOptions::default())
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
capture
|
||||
.read_checked(ReadSelector::EntryRange([0, 99]), ReadOptions::default())
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slices_preserve_empty_and_long_result_fields_under_item_and_metadata_budgets() {
|
||||
let summary = "summary\n\"🙂".repeat(100);
|
||||
let content = "content\t\\日本語".repeat(200);
|
||||
let capture = SessionCapture::new(
|
||||
"s",
|
||||
"seg",
|
||||
vec![
|
||||
Item::tool_result_item("first", summary.clone(), Some(content.clone()), true),
|
||||
Item::tool_result_item("empty", "", Some(String::new()), false),
|
||||
],
|
||||
);
|
||||
let mut cursor = None;
|
||||
let mut fields: Vec<(String, RawField, String)> = Vec::new();
|
||||
loop {
|
||||
let part = capture
|
||||
.slice(CapturePurpose::Recording, cursor.as_ref(), 2, 1500)
|
||||
.unwrap();
|
||||
assert!(part.slice().items.len() <= 2);
|
||||
assert!(serde_json::to_vec(part.slice()).unwrap().len() <= 1500);
|
||||
for item in &part.slice().items {
|
||||
let id = item.reference.entry_ref.to_string();
|
||||
if let Some((last_id, last_field, text)) = fields.last_mut() {
|
||||
if *last_id == id && *last_field == item.field {
|
||||
assert_eq!(text.len(), item.offset);
|
||||
text.push_str(&item.text);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
assert_eq!(item.offset, 0);
|
||||
fields.push((id, item.field, item.text.clone()));
|
||||
}
|
||||
cursor = part.slice().next_cursor.clone();
|
||||
if cursor.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
fields,
|
||||
vec![
|
||||
("E00000000".into(), RawField::ToolSummary, summary),
|
||||
("E00000000".into(), RawField::ToolContent, content),
|
||||
("E00000001".into(), RawField::ToolSummary, String::new()),
|
||||
("E00000001".into(), RawField::ToolContent, String::new()),
|
||||
]
|
||||
);
|
||||
let metadata_heavy = SessionCapture::new(
|
||||
"s",
|
||||
"seg",
|
||||
vec![Item::tool_call("call", "name".repeat(200), "x")],
|
||||
);
|
||||
assert!(matches!(
|
||||
metadata_heavy.slice(CapturePurpose::Recording, None, 1, 700),
|
||||
Err(CaptureError::BudgetTooSmall)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn evidence_uses_original_segment_positions_after_roundtrip_and_view_selection() {
|
||||
let full = SessionCapture::new(
|
||||
"s",
|
||||
"selected",
|
||||
vec![
|
||||
Item::user_message("first"),
|
||||
Item::assistant_message("second"),
|
||||
],
|
||||
);
|
||||
let mut dto = full.to_dto();
|
||||
dto.entries[0].source.segment_id = "original-a".into();
|
||||
dto.entries[0].source.entry_index = 80;
|
||||
dto.entries[1].source.segment_id = "original-b".into();
|
||||
dto.entries[1].source.entry_index = 3;
|
||||
let encoded = dto.encode(CaptureLimits::default()).unwrap();
|
||||
let restored = SessionCapture::decode(&encoded, CaptureLimits::default()).unwrap();
|
||||
for (id, segment, index, order) in [
|
||||
("E00000000", "original-a", 80, 0),
|
||||
("E00000001", "original-b", 3, 1),
|
||||
] {
|
||||
let evidence = restored.evidence_for(id).unwrap();
|
||||
assert_eq!(evidence.segment_id, segment);
|
||||
assert_eq!(evidence.entry_range, [index; 2]);
|
||||
let read = restored.read(ReadSelector::Id(id), ReadOptions::default());
|
||||
assert_eq!(read.entries[0].entry_range, [order; 2]);
|
||||
}
|
||||
let selected = restored.select_order_range(1..2).unwrap();
|
||||
assert_eq!(selected.overview()[0].entry_range, [0, 0]);
|
||||
let evidence = selected.evidence_for("E00000001").unwrap();
|
||||
assert_eq!(evidence.segment_id, "original-b");
|
||||
assert_eq!(evidence.entry_range, [3, 3]);
|
||||
let roundtrip = SessionCapture::decode(
|
||||
&selected.encode(CaptureLimits::default()).unwrap(),
|
||||
CaptureLimits::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(roundtrip.to_dto(), selected.to_dto());
|
||||
assert_eq!(
|
||||
roundtrip.evidence_for("E00000001").unwrap().entry_range,
|
||||
[3, 3]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixed_history_selection_preserves_original_coordinates_and_checks_private_position_bounds() {
|
||||
let history = vec![
|
||||
history("old", Item::user_message("old"), None),
|
||||
history("hidden", Item::reasoning("PRIVATE"), None),
|
||||
history("selected", Item::user_message("selected"), None),
|
||||
history("private-tail", Item::system_message("PRIVATE"), None),
|
||||
];
|
||||
let full = SessionCapture::from_complete_history("s", "seg", history);
|
||||
let suffix = full.select_order_range(1..4).unwrap();
|
||||
let original = full
|
||||
.entries()
|
||||
.find(|entry| entry.entry_id == "selected")
|
||||
.unwrap();
|
||||
let selected = suffix.entries().next().unwrap();
|
||||
assert_eq!(selected.entry_id, original.entry_id);
|
||||
assert_eq!(selected.source, original.source);
|
||||
assert_eq!(selected.body, original.body);
|
||||
assert_eq!(selected.policy, original.policy);
|
||||
assert!(suffix.evidence_for("Eold").is_none());
|
||||
assert_eq!(suffix.search(&SearchOptions::default()).len(), 1);
|
||||
assert_eq!(
|
||||
suffix.view_order_for_ref(&SessionEntryRef::parse("Eselected").unwrap()),
|
||||
Some(1)
|
||||
);
|
||||
assert_eq!(selected.source.entry_index, 2);
|
||||
assert_eq!(selected.order, 1);
|
||||
assert_eq!(suffix.to_dto().view_entry_count, 3);
|
||||
assert_eq!(
|
||||
suffix.evidence_for("Eselected").unwrap().entry_range,
|
||||
[2, 2]
|
||||
);
|
||||
let nested = suffix.select_order_range(1..2).unwrap();
|
||||
assert_eq!(nested.entries().next().unwrap().order, 0);
|
||||
assert_eq!(nested.entries().next().unwrap().source, original.source);
|
||||
let restored = SessionCapture::decode(
|
||||
&suffix.encode(CaptureLimits::default()).unwrap(),
|
||||
CaptureLimits::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(restored.to_dto(), suffix.to_dto());
|
||||
assert!(full.select_order_range(4..5).is_err());
|
||||
let reversed = std::ops::Range { start: 3, end: 2 };
|
||||
assert!(full.select_order_range(reversed).is_err());
|
||||
let hidden_only = full.select_order_range(3..4).unwrap();
|
||||
assert_eq!(hidden_only.entries().len(), 0);
|
||||
assert_eq!(hidden_only.to_dto().view_entry_count, 1);
|
||||
let mut invalid = full.to_dto();
|
||||
invalid.view_entry_count = 2; // the selected entry's view order is outside this extent
|
||||
assert!(SessionCapture::try_from_dto(invalid, CaptureLimits::default()).is_err());
|
||||
assert_eq!(
|
||||
full.retain_entries(|entry| entry.entry_id == "selected")
|
||||
.entries()
|
||||
.next()
|
||||
.unwrap()
|
||||
.source,
|
||||
original.source
|
||||
);
|
||||
assert_eq!(full.to_dto().view_entry_count, 4);
|
||||
}
|
||||
@@ -25,7 +25,46 @@ use thiserror::Error;
|
||||
#[path = "job_store/legacy_migrations.rs"]
|
||||
mod legacy_migrations;
|
||||
|
||||
const SCHEMA_VERSION: i64 = 4;
|
||||
const SCHEMA_VERSION: i64 = 5;
|
||||
|
||||
/// Availability of the immutable input body, independent of execution outcome.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum JobInputDisposition {
|
||||
Available,
|
||||
Reclaimed,
|
||||
Erased,
|
||||
Invalidated,
|
||||
}
|
||||
|
||||
impl JobInputDisposition {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Available => "available",
|
||||
Self::Reclaimed => "reclaimed",
|
||||
Self::Erased => "erased",
|
||||
Self::Invalidated => "invalidated",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Host-authorized retirement; erasure and legacy invalidation do not change outcomes.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum JobInputRetirementReason {
|
||||
Completed,
|
||||
ExplicitErasure,
|
||||
LegacyInvalidation,
|
||||
}
|
||||
|
||||
impl JobInputRetirementReason {
|
||||
pub fn disposition(self) -> JobInputDisposition {
|
||||
match self {
|
||||
Self::Completed => JobInputDisposition::Reclaimed,
|
||||
Self::ExplicitErasure => JobInputDisposition::Erased,
|
||||
Self::LegacyInvalidation => JobInputDisposition::Invalidated,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
@@ -67,7 +106,11 @@ impl JobAttemptState {
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct JobSnapshot {
|
||||
/// Receipt metadata. After retirement `input` is null; use the verified loader to execute.
|
||||
pub request: JobRequest,
|
||||
pub input_digest: String,
|
||||
pub intent_fingerprint: String,
|
||||
pub input_disposition: JobInputDisposition,
|
||||
/// Durable Host-issued domain delegation, not model input or Profile identity.
|
||||
pub domain_grant: Option<Value>,
|
||||
pub state: JobState,
|
||||
@@ -77,14 +120,27 @@ pub struct JobSnapshot {
|
||||
}
|
||||
|
||||
impl JobSnapshot {
|
||||
/// Load only the original available body, checking both content and immutable intent.
|
||||
pub fn load_executable_request(&self) -> Result<&JobRequest, JobStoreError> {
|
||||
if self.input_disposition != JobInputDisposition::Available {
|
||||
return Err(JobStoreError::InvalidTransition("Job input is retired"));
|
||||
}
|
||||
self.request.validate()?;
|
||||
if self.request.input_digest()? != self.input_digest
|
||||
|| self.request.fingerprint()? != self.intent_fingerprint
|
||||
{
|
||||
return Err(JobStoreError::Corrupt(
|
||||
"immutable input or intent fingerprint mismatch".into(),
|
||||
));
|
||||
}
|
||||
Ok(&self.request)
|
||||
}
|
||||
|
||||
/// Shared Feature-facing read contract; no Workspace or execution identity leaks.
|
||||
pub fn outcome(&self) -> job::JobOutcome {
|
||||
job::JobOutcome {
|
||||
job_id: self.request.job_id.clone(),
|
||||
input_digest: self
|
||||
.request
|
||||
.input_digest()
|
||||
.expect("validated immutable Job input"),
|
||||
input_digest: self.input_digest.clone(),
|
||||
attempt_id: self.attempt.attempt_id.clone(),
|
||||
attempt: self.attempt.number,
|
||||
state: match self.state {
|
||||
@@ -204,9 +260,23 @@ impl JobStore {
|
||||
tx.execute_batch(RECONCILIATION_SCHEMA)?;
|
||||
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
|
||||
}
|
||||
SCHEMA_VERSION => {}
|
||||
4 | SCHEMA_VERSION => {}
|
||||
other => return Err(JobStoreError::UnsupportedSchema(other)),
|
||||
}
|
||||
if version < 5 {
|
||||
tx.execute_batch(INPUT_RETENTION_SCHEMA)?;
|
||||
let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?;
|
||||
if has_legacy_archive {
|
||||
tx.execute_batch("DROP TRIGGER legacy_job_archive_immutable;
|
||||
CREATE TRIGGER legacy_job_archive_immutable BEFORE UPDATE OF job_id,fingerprint,attempts_json ON legacy_job_archive
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable legacy Job archive'); END;
|
||||
CREATE TRIGGER legacy_job_archive_input_retirement BEFORE UPDATE OF request_json ON legacy_job_archive
|
||||
WHEN NOT EXISTS (SELECT 1 FROM job_intents WHERE job_id=OLD.job_id AND input_disposition<>'available')
|
||||
OR (json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input') AND json_type(NEW.request_json,'$.input')='null') IS NOT TRUE
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable legacy Job input'); END;")?;
|
||||
}
|
||||
tx.pragma_update(None, "user_version", SCHEMA_VERSION)?;
|
||||
}
|
||||
// An accepted result is the only evidence of success. Reserved attempts
|
||||
// were never started and remain pending; cancelled attempts never resume.
|
||||
tx.execute_batch(
|
||||
@@ -249,10 +319,7 @@ impl JobStore {
|
||||
[&request.job_id],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if stored != fingerprint
|
||||
|| snapshot.request != request
|
||||
|| snapshot.domain_grant != grant
|
||||
{
|
||||
if stored != fingerprint || snapshot.domain_grant != grant {
|
||||
return Err(JobStoreError::IntentConflict(request.job_id));
|
||||
}
|
||||
tx.commit()?;
|
||||
@@ -334,6 +401,7 @@ impl JobStore {
|
||||
.conn
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let previous = require_snapshot(&tx, job_id)?;
|
||||
previous.load_executable_request()?;
|
||||
if previous.state != JobState::Failed || previous.attempt.state != JobAttemptState::Failed {
|
||||
return Err(JobStoreError::InvalidTransition(
|
||||
"retry requires known failure",
|
||||
@@ -354,6 +422,81 @@ impl JobStore {
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
/// Historical snapshots for one domain Session, including terminal and retired Jobs.
|
||||
pub(crate) fn list_session_snapshots(
|
||||
&self,
|
||||
session_id: &str,
|
||||
after: Option<&str>,
|
||||
limit: usize,
|
||||
) -> Result<Vec<JobSnapshot>, JobStoreError> {
|
||||
let mut stmt = self.conn.prepare("SELECT i.job_id FROM job_intents i JOIN job_domain_grants g ON g.job_id=i.job_id WHERE json_extract(g.grant_json,'$.session_id')=?1 AND (?2 IS NULL OR i.job_id>?2) ORDER BY i.job_id LIMIT ?3")?;
|
||||
let ids = stmt
|
||||
.query_map(
|
||||
params![session_id, after, limit.clamp(1, 100) as i64],
|
||||
|r| r.get::<_, String>(0),
|
||||
)?
|
||||
.collect::<rusqlite::Result<Vec<_>>>()?;
|
||||
ids.into_iter()
|
||||
.map(|id| require_snapshot(&self.conn, &id))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The Host must fence domain mutation and delivery before erasure/invalidation.
|
||||
/// This atomically prevents dispatch/retry/result acceptance, without rewriting outcome.
|
||||
pub(crate) fn reclaim_input(
|
||||
&mut self,
|
||||
job_id: &str,
|
||||
expected_input_digest: &str,
|
||||
reason: JobInputRetirementReason,
|
||||
) -> Result<JobSnapshot, JobStoreError> {
|
||||
let tx = self
|
||||
.conn
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let snapshot = require_snapshot(&tx, job_id)?;
|
||||
if snapshot.input_digest != expected_input_digest {
|
||||
return Err(JobStoreError::DigestMismatch);
|
||||
}
|
||||
let desired = reason.disposition();
|
||||
if snapshot.input_disposition == desired {
|
||||
tx.commit()?;
|
||||
reclaim_retired_input_storage(&self.conn)?;
|
||||
return Ok(snapshot);
|
||||
}
|
||||
if reason == JobInputRetirementReason::Completed
|
||||
&& (snapshot.state != JobState::Completed
|
||||
|| snapshot.attempt.state != JobAttemptState::Completed
|
||||
|| snapshot.result.is_none())
|
||||
{
|
||||
return Err(JobStoreError::InvalidTransition(
|
||||
"reclaim requires known completed result",
|
||||
));
|
||||
}
|
||||
if snapshot.input_disposition != JobInputDisposition::Available
|
||||
&& reason != JobInputRetirementReason::ExplicitErasure
|
||||
{
|
||||
return Err(JobStoreError::InvalidTransition(
|
||||
"retired input cannot be restored or reclassified",
|
||||
));
|
||||
}
|
||||
if reason == JobInputRetirementReason::Completed {
|
||||
snapshot.load_executable_request()?;
|
||||
}
|
||||
let mut metadata = snapshot.request.clone();
|
||||
metadata.input = Value::Null;
|
||||
tx.execute(
|
||||
"UPDATE job_intents SET request_json=?2, input_disposition=?3 WHERE job_id=?1",
|
||||
params![job_id, serde_json::to_string(&metadata)?, desired.as_str()],
|
||||
)?;
|
||||
let has_legacy_archive: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type='table' AND name='legacy_job_archive')", [], |r| r.get(0))?;
|
||||
if has_legacy_archive {
|
||||
tx.execute("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input',json('null')) WHERE job_id=?1", [job_id])?;
|
||||
}
|
||||
let snapshot = require_snapshot(&tx, job_id)?;
|
||||
tx.commit()?;
|
||||
reclaim_retired_input_storage(&self.conn)?;
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
pub(crate) fn get(&self, job_id: &str) -> Result<JobSnapshot, JobStoreError> {
|
||||
require_snapshot(&self.conn, job_id)
|
||||
}
|
||||
@@ -383,7 +526,7 @@ impl JobStore {
|
||||
let mut statement = self.conn.prepare(
|
||||
"SELECT i.job_id FROM job_intents i JOIN job_attempts a
|
||||
ON a.job_id = i.job_id AND a.number = i.current_attempt
|
||||
WHERE i.state = 'pending' AND a.state = 'reserved' ORDER BY i.rowid",
|
||||
WHERE i.state = 'pending' AND i.input_disposition='available' AND a.state = 'reserved' ORDER BY i.rowid",
|
||||
)?;
|
||||
let ids = statement
|
||||
.query_map([], |row| row.get::<_, String>(0))?
|
||||
@@ -406,6 +549,7 @@ impl JobStore {
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let snapshot = require_snapshot(&tx, job_id)?;
|
||||
check_attempt(&snapshot, attempt_id)?;
|
||||
snapshot.load_executable_request()?;
|
||||
if snapshot.state != JobState::Pending
|
||||
|| snapshot.attempt.state != JobAttemptState::Reserved
|
||||
{
|
||||
@@ -447,7 +591,7 @@ impl JobStore {
|
||||
.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||
let snapshot = require_snapshot(&tx, &bound.job_id)?;
|
||||
check_attempt(&snapshot, &bound.attempt_id)?;
|
||||
if snapshot.request.input_digest()? != bound.input_digest {
|
||||
if snapshot.input_digest != bound.input_digest {
|
||||
return Err(JobStoreError::DigestMismatch);
|
||||
}
|
||||
let (digest, encoded) =
|
||||
@@ -466,6 +610,7 @@ impl JobStore {
|
||||
tx.commit()?;
|
||||
return Ok(snapshot);
|
||||
}
|
||||
snapshot.load_executable_request()?;
|
||||
if snapshot.state != JobState::Pending
|
||||
|| snapshot.attempt.state != JobAttemptState::Dispatched
|
||||
{
|
||||
@@ -649,7 +794,7 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
||||
let row = conn
|
||||
.query_row(
|
||||
"SELECT i.request_json, i.state, i.acknowledged, a.attempt_id, a.number,
|
||||
a.state, a.failure, a.result_json FROM job_intents i JOIN job_attempts a
|
||||
a.state, a.failure, a.result_json, a.input_digest, i.fingerprint, i.input_disposition FROM job_intents i JOIN job_attempts a
|
||||
ON a.job_id = i.job_id AND a.number = i.current_attempt WHERE i.job_id = ?1",
|
||||
[job_id],
|
||||
|row| {
|
||||
@@ -662,12 +807,27 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
||||
row.get::<_, String>(5)?,
|
||||
row.get::<_, Option<String>>(6)?,
|
||||
row.get::<_, Option<String>>(7)?,
|
||||
row.get::<_, String>(8)?,
|
||||
row.get::<_, String>(9)?,
|
||||
row.get::<_, String>(10)?,
|
||||
))
|
||||
},
|
||||
)
|
||||
.optional()?;
|
||||
row.map(
|
||||
|(request, state, acknowledged, attempt_id, number, attempt_state, failure, result)| {
|
||||
|(
|
||||
request,
|
||||
state,
|
||||
acknowledged,
|
||||
attempt_id,
|
||||
number,
|
||||
attempt_state,
|
||||
failure,
|
||||
result,
|
||||
input_digest,
|
||||
intent_fingerprint,
|
||||
disposition,
|
||||
)| {
|
||||
let state = match state.as_str() {
|
||||
"pending" => JobState::Pending,
|
||||
"completed" => JobState::Completed,
|
||||
@@ -693,8 +853,17 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
||||
)));
|
||||
}
|
||||
};
|
||||
let request_value: Value = serde_json::from_str(&request)?;
|
||||
if request_value.get("input").is_none() {
|
||||
return Err(JobStoreError::Corrupt(
|
||||
"stored Job input field is missing".into(),
|
||||
));
|
||||
}
|
||||
Ok(JobSnapshot {
|
||||
request: serde_json::from_str(&request)?,
|
||||
request: serde_json::from_value(request_value)?,
|
||||
input_digest,
|
||||
intent_fingerprint,
|
||||
input_disposition: serde_json::from_value(Value::String(disposition))?,
|
||||
domain_grant: conn
|
||||
.query_row(
|
||||
"SELECT grant_json FROM job_domain_grants WHERE job_id = ?1",
|
||||
@@ -719,6 +888,37 @@ fn read_snapshot(conn: &Connection, job_id: &str) -> Result<Option<JobSnapshot>,
|
||||
.transpose()
|
||||
}
|
||||
|
||||
// Retirement is already durable before this maintenance step. Failure must be
|
||||
// retried with the same Job/digest/reason; it must never restore input. VACUUM
|
||||
// removes historical free-page copies, including stores written before secure_delete.
|
||||
fn reclaim_retired_input_storage(conn: &Connection) -> Result<(), JobStoreError> {
|
||||
conn.execute_batch("PRAGMA secure_delete=ON; VACUUM;")?;
|
||||
let busy: i64 = conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| r.get(0))?;
|
||||
if busy != 0 {
|
||||
return Err(JobStoreError::InvalidTransition(
|
||||
"input retired; SQLite body cleanup blocked by a reader, retry retirement",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const INPUT_RETENTION_SCHEMA: &str = "
|
||||
ALTER TABLE job_intents ADD COLUMN input_disposition TEXT NOT NULL DEFAULT 'available'
|
||||
CHECK (input_disposition IN ('available','reclaimed','erased','invalidated'));
|
||||
DROP TRIGGER job_intent_immutable;
|
||||
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF
|
||||
job_id, fingerprint, serialization_key, max_concurrent ON job_intents
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
|
||||
CREATE TRIGGER job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON job_intents
|
||||
WHEN (
|
||||
((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated'))
|
||||
OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased'))
|
||||
AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input')
|
||||
AND json_type(NEW.request_json,'$.input') = 'null'
|
||||
) IS NOT TRUE
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Job input retirement'); END;
|
||||
";
|
||||
|
||||
const RECONCILIATION_SCHEMA: &str = "
|
||||
DROP INDEX job_serialization_singleflight;
|
||||
CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key)
|
||||
@@ -885,7 +1085,11 @@ mod tests {
|
||||
Some("unconfirmed"),
|
||||
)
|
||||
.unwrap();
|
||||
store.conn.execute_batch("DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap();
|
||||
store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable;
|
||||
ALTER TABLE job_intents DROP COLUMN input_disposition;
|
||||
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
|
||||
DROP TRIGGER job_reconciliation_immutable; DROP TRIGGER job_reconciliation_retained; DROP TABLE job_domain_reconciliations; DROP INDEX job_serialization_singleflight; CREATE UNIQUE INDEX job_serialization_singleflight ON job_intents(serialization_key) WHERE serialization_key IS NOT NULL AND state IN ('pending','unknown'); PRAGMA user_version=3;").unwrap();
|
||||
drop(store);
|
||||
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
|
||||
assert_eq!(reopened.get("first").unwrap(), unknown);
|
||||
@@ -911,10 +1115,7 @@ mod tests {
|
||||
JobResultSubmission {
|
||||
job_id: snapshot.request.job_id.clone(),
|
||||
attempt_id: snapshot.attempt.attempt_id.clone(),
|
||||
input_digest: snapshot
|
||||
.request
|
||||
.input_digest()
|
||||
.expect("validated immutable Job input"),
|
||||
input_digest: snapshot.input_digest.clone(),
|
||||
result,
|
||||
}
|
||||
}
|
||||
@@ -926,6 +1127,314 @@ mod tests {
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_reclaim_reopens_as_receipt_and_exact_replay_never_rehydrates() {
|
||||
let (dir, mut store) = fixture();
|
||||
let original = request("retained");
|
||||
let grant = Some(json!({"session_id":"session-a", "consumer":"fixture"}));
|
||||
let reserved = store
|
||||
.reserve_granted(original.clone(), grant.clone())
|
||||
.unwrap();
|
||||
let running = store
|
||||
.dispatch("retained", &reserved.attempt.attempt_id, 8)
|
||||
.unwrap();
|
||||
let result = submission(&running, json!({"saved":true}));
|
||||
let completed = store.accept(&result).unwrap();
|
||||
store.ack("retained").unwrap();
|
||||
let retired = store
|
||||
.reclaim_input(
|
||||
"retained",
|
||||
&completed.input_digest,
|
||||
JobInputRetirementReason::Completed,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(retired.input_disposition, JobInputDisposition::Reclaimed);
|
||||
assert!(retired.request.input.is_null());
|
||||
assert!(retired.load_executable_request().is_err());
|
||||
assert_eq!(
|
||||
retired.outcome().input_digest,
|
||||
original.input_digest().unwrap()
|
||||
);
|
||||
assert_eq!(retired.result, completed.result);
|
||||
drop(store);
|
||||
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
|
||||
assert_eq!(
|
||||
reopened
|
||||
.reserve_granted(original.clone(), grant.clone())
|
||||
.unwrap(),
|
||||
retired
|
||||
);
|
||||
assert_eq!(reopened.accept(&result).unwrap(), retired);
|
||||
assert_eq!(
|
||||
reopened
|
||||
.list_session_snapshots("session-a", None, 1)
|
||||
.unwrap(),
|
||||
vec![retired.clone()]
|
||||
);
|
||||
assert!(
|
||||
reopened
|
||||
.list_session_snapshots("session-b", None, 100)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
reopened
|
||||
.list_session_snapshots("session-a", Some("retained"), 100)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
let mut changed = original.clone();
|
||||
changed.input = json!({"changed":true});
|
||||
assert!(matches!(
|
||||
reopened.reserve_granted(changed, grant),
|
||||
Err(JobStoreError::IntentConflict(_))
|
||||
));
|
||||
assert!(reopened.retry("retained").is_err());
|
||||
assert!(
|
||||
reopened
|
||||
.dispatch("retained", &retired.attempt.attempt_id, 8)
|
||||
.is_err()
|
||||
);
|
||||
assert!(reopened.conn.execute("UPDATE job_intents SET request_json=?1,input_disposition='available' WHERE job_id='retained'", [serde_json::to_string(&original).unwrap()]).is_err());
|
||||
let erased = reopened
|
||||
.reclaim_input(
|
||||
"retained",
|
||||
&retired.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
|
||||
assert_eq!(erased.state, JobState::Completed);
|
||||
assert!(erased.acknowledged);
|
||||
assert_eq!(
|
||||
reopened
|
||||
.reserve_granted(original, erased.domain_grant.clone())
|
||||
.unwrap(),
|
||||
erased
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unfinished_unknown_and_failed_inputs_are_kept_until_explicit_invalidation() {
|
||||
let (_dir, mut store) = fixture();
|
||||
for (id, state) in [
|
||||
("pending", None),
|
||||
("unknown", Some(JobAttemptState::Unknown)),
|
||||
("failed", Some(JobAttemptState::Failed)),
|
||||
] {
|
||||
let original = request(id);
|
||||
let snapshot = store.reserve(original.clone()).unwrap();
|
||||
if let Some(state) = state {
|
||||
store.dispatch(id, &snapshot.attempt.attempt_id, 8).unwrap();
|
||||
store
|
||||
.finish(
|
||||
id,
|
||||
&snapshot.attempt.attempt_id,
|
||||
state,
|
||||
Some("not complete"),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let before = store.get(id).unwrap();
|
||||
assert!(
|
||||
store
|
||||
.reclaim_input(
|
||||
id,
|
||||
&before.input_digest,
|
||||
JobInputRetirementReason::Completed
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(store.get(id).unwrap(), before);
|
||||
assert_eq!(before.load_executable_request().unwrap(), &original);
|
||||
assert!(
|
||||
store
|
||||
.reclaim_input(id, "wrong", JobInputRetirementReason::ExplicitErasure)
|
||||
.is_err()
|
||||
);
|
||||
let retired = store
|
||||
.reclaim_input(
|
||||
id,
|
||||
&before.input_digest,
|
||||
JobInputRetirementReason::LegacyInvalidation,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(retired.state, before.state);
|
||||
assert_eq!(retired.attempt, before.attempt);
|
||||
assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated);
|
||||
assert!(store.retry(id).is_err());
|
||||
assert!(store.dispatch(id, &before.attempt.attempt_id, 8).is_err());
|
||||
assert_eq!(store.reserve(original).unwrap(), retired);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn erased_dispatched_input_refuses_delayed_result_without_rewriting_unknown() {
|
||||
let (dir, mut store) = fixture();
|
||||
let running = dispatched(&mut store, request("erase-running"));
|
||||
let result = submission(&running, json!({"late":true}));
|
||||
let erased = store
|
||||
.reclaim_input(
|
||||
"erase-running",
|
||||
&running.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(store.accept(&result).is_err());
|
||||
assert_eq!(erased.state, JobState::Pending);
|
||||
drop(store);
|
||||
let mut reopened = JobStore::open(&dir.path().join("jobs.sqlite3")).unwrap();
|
||||
let recovered = reopened.get("erase-running").unwrap();
|
||||
assert_eq!(recovered.state, JobState::Unknown);
|
||||
assert_eq!(recovered.input_disposition, JobInputDisposition::Erased);
|
||||
assert_eq!(recovered.input_digest, running.input_digest);
|
||||
assert!(reopened.accept(&result).is_err());
|
||||
assert!(reopened.retry("erase-running").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn available_input_loader_refuses_tamper_missing_body_and_envelope_changes() {
|
||||
for mutation in [
|
||||
"json_set(request_json,'$.input',json('{\"tampered\":true}'))",
|
||||
"json_remove(request_json,'$.input')",
|
||||
"json_set(request_json,'$.instruction','changed')",
|
||||
] {
|
||||
let (_dir, mut store) = fixture();
|
||||
store.reserve(request("tamper")).unwrap();
|
||||
// Simulate external corruption bypassing SQLite's normal write guard.
|
||||
store
|
||||
.conn
|
||||
.execute_batch("DROP TRIGGER job_input_retirement")
|
||||
.unwrap();
|
||||
store
|
||||
.conn
|
||||
.execute_batch(&format!("UPDATE job_intents SET request_json={mutation}"))
|
||||
.unwrap();
|
||||
match store.get("tamper") {
|
||||
Ok(snapshot) => {
|
||||
assert!(snapshot.load_executable_request().is_err());
|
||||
assert!(
|
||||
store
|
||||
.dispatch("tamper", &snapshot.attempt.attempt_id, 8)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
Err(JobStoreError::Json(_) | JobStoreError::Corrupt(_)) => {}
|
||||
other => panic!("unexpected corrupt input read: {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_retirement_cleanup_failure_is_durable_and_exact_retry_finishes_without_rehydration() {
|
||||
let (dir, mut store) = fixture();
|
||||
let mut original = request("physical-erasure");
|
||||
let private_body = "private-input-retirement-original-bytes".repeat(1000);
|
||||
original.input = json!({"body":private_body});
|
||||
let reserved = store.reserve(original.clone()).unwrap();
|
||||
// A reader retaining the old SQLite snapshot prevents truncation of the old WAL.
|
||||
let reader = Connection::open(dir.path().join("jobs.sqlite3")).unwrap();
|
||||
reader
|
||||
.execute_batch("BEGIN; SELECT request_json FROM job_intents;")
|
||||
.unwrap();
|
||||
store.conn.busy_timeout(std::time::Duration::ZERO).unwrap();
|
||||
assert!(
|
||||
store
|
||||
.reclaim_input(
|
||||
"physical-erasure",
|
||||
&reserved.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let retired = store.get("physical-erasure").unwrap();
|
||||
assert_eq!(retired.input_disposition, JobInputDisposition::Erased);
|
||||
assert!(retired.load_executable_request().is_err());
|
||||
reader.execute_batch("ROLLBACK").unwrap();
|
||||
drop(reader);
|
||||
let replay = store
|
||||
.reclaim_input(
|
||||
"physical-erasure",
|
||||
&reserved.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(replay, retired);
|
||||
assert_eq!(store.reserve(original).unwrap(), retired);
|
||||
let needle = b"private-input-retirement-original-bytes";
|
||||
for entry in std::fs::read_dir(dir.path()).unwrap() {
|
||||
let path = entry.unwrap().path();
|
||||
if path.is_file() {
|
||||
let bytes = std::fs::read(&path).unwrap();
|
||||
assert!(
|
||||
!bytes.windows(needle.len()).any(|bytes| bytes == needle),
|
||||
"retired body remained in {}",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_original_null_input_is_not_treated_as_available_null() {
|
||||
let (_dir, mut store) = fixture();
|
||||
let mut original = request("null-original");
|
||||
original.input = Value::Null;
|
||||
let reserved = store.reserve(original).unwrap();
|
||||
assert!(reserved.load_executable_request().is_ok());
|
||||
store.conn.execute_batch("DROP TRIGGER job_input_retirement; UPDATE job_intents SET request_json=json_remove(request_json,'$.input')").unwrap();
|
||||
assert!(store.get("null-original").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_retirement_migration_preserves_receipts_and_rolls_back_failed_ddl() {
|
||||
let (dir, mut store) = fixture();
|
||||
let original = store.reserve(request("upgrade")).unwrap();
|
||||
store.conn.execute_batch("DROP TRIGGER job_input_retirement; DROP TRIGGER job_intent_immutable;
|
||||
ALTER TABLE job_intents DROP COLUMN input_disposition;
|
||||
CREATE TRIGGER job_intent_immutable BEFORE UPDATE OF job_id,request_json,fingerprint,serialization_key,max_concurrent ON job_intents
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Job intent'); END;
|
||||
PRAGMA user_version=4;
|
||||
CREATE TRIGGER job_input_retirement BEFORE UPDATE ON job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;").unwrap();
|
||||
drop(store);
|
||||
let path = dir.path().join("jobs.sqlite3");
|
||||
assert!(JobStore::open(&path).is_err());
|
||||
let conn = Connection::open(&path).unwrap();
|
||||
let version: i64 = conn
|
||||
.pragma_query_value(None, "user_version", |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(version, 4);
|
||||
let present: bool = conn.query_row("SELECT EXISTS(SELECT 1 FROM pragma_table_info('job_intents') WHERE name='input_disposition')",[],|r|r.get(0)).unwrap();
|
||||
assert!(!present);
|
||||
conn.execute_batch("DROP TRIGGER job_input_retirement")
|
||||
.unwrap();
|
||||
drop(conn);
|
||||
let mut reopened = JobStore::open(&path).unwrap();
|
||||
assert_eq!(reopened.get("upgrade").unwrap(), original);
|
||||
assert_eq!(
|
||||
reopened
|
||||
.get("upgrade")
|
||||
.unwrap()
|
||||
.load_executable_request()
|
||||
.unwrap(),
|
||||
&request("upgrade")
|
||||
);
|
||||
let retired = reopened
|
||||
.reclaim_input(
|
||||
"upgrade",
|
||||
&original.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(retired.state, JobState::Pending);
|
||||
assert!(
|
||||
reopened
|
||||
.conn
|
||||
.execute_batch("UPDATE job_intents SET input_disposition='available'")
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn database_configuration_schema_and_identity_constraints() {
|
||||
let (_dir, mut store) = fixture();
|
||||
|
||||
@@ -143,6 +143,42 @@ mod tests {
|
||||
(original, receipt, grant)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_archive_input_erasure_preserves_fingerprints_and_attempt_receipts() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("jobs.sqlite3");
|
||||
seed(&path, 2);
|
||||
let mut store = JobStore::open(&path).unwrap();
|
||||
let before: (String, String) = store
|
||||
.conn
|
||||
.query_row(
|
||||
"SELECT fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.unwrap();
|
||||
let snapshot = store.get("done").unwrap();
|
||||
store
|
||||
.reclaim_input(
|
||||
"done",
|
||||
&snapshot.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
let (body, fingerprint, attempts): (String,String,String) = store.conn.query_row("SELECT request_json,fingerprint,attempts_json FROM legacy_job_archive WHERE job_id='done'",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap();
|
||||
assert!(serde_json::from_str::<Value>(&body).unwrap()["input"].is_null());
|
||||
assert_eq!((fingerprint, attempts), before);
|
||||
assert!(store.conn.execute_batch("UPDATE legacy_job_archive SET request_json=json_set(request_json,'$.input','restored') WHERE job_id='done'").is_err());
|
||||
drop(store);
|
||||
let mut reopened = JobStore::open(&path).unwrap();
|
||||
let receipt = reopened
|
||||
.reserve_granted(request("done"), snapshot.domain_grant)
|
||||
.unwrap();
|
||||
assert_eq!(receipt.state, JobState::Completed);
|
||||
assert_eq!(receipt.input_disposition, JobInputDisposition::Erased);
|
||||
assert!(receipt.load_executable_request().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_versions_preserve_intent_receipts_attempts_grants_and_replay() {
|
||||
for version in [1, 2] {
|
||||
|
||||
+105
-40
@@ -26,7 +26,10 @@ use job::{JobRequest, JobResultSubmission};
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::job_store::JobStore;
|
||||
pub use crate::job_store::{JobAttempt, JobAttemptState, JobSnapshot, JobState, JobStoreError};
|
||||
pub use crate::job_store::{
|
||||
JobAttempt, JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot,
|
||||
JobState, JobStoreError,
|
||||
};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum JobServiceError {
|
||||
@@ -40,6 +43,8 @@ pub enum JobServiceError {
|
||||
Capacity,
|
||||
#[error("standalone Job cleanup failed: {0}")]
|
||||
Cleanup(String),
|
||||
#[error("standalone Job shutdown is incomplete; execution resources are still owned")]
|
||||
ShutdownIncomplete,
|
||||
}
|
||||
|
||||
pub(crate) trait JobAttemptFence: Send + Sync {
|
||||
@@ -173,6 +178,36 @@ impl StandaloneJobs {
|
||||
Ok(self.with_store(|store| store.reserve(request))?)
|
||||
}
|
||||
|
||||
/// Enumerate historical domain snapshots for a Session, including retired input receipts.
|
||||
pub fn list_session_snapshots(
|
||||
&self,
|
||||
session_id: &str,
|
||||
after: Option<&str>,
|
||||
limit: usize,
|
||||
) -> Result<Vec<JobSnapshot>, JobServiceError> {
|
||||
self.with_store(|store| store.list_session_snapshots(session_id, after, limit))
|
||||
}
|
||||
|
||||
/// Retire body only. The caller fences domain effects/delivered snapshots before erasure.
|
||||
/// Normal reclamation also requires that execution cleanup has finished.
|
||||
pub fn reclaim_input(
|
||||
&self,
|
||||
job_id: &str,
|
||||
expected_input_digest: &str,
|
||||
reason: JobInputRetirementReason,
|
||||
) -> Result<JobSnapshot, JobServiceError> {
|
||||
let executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if reason == JobInputRetirementReason::Completed
|
||||
&& executions
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.is_some_and(|execution| !execution.task.is_finished())
|
||||
{
|
||||
return Err(JobServiceError::Capacity);
|
||||
}
|
||||
self.with_store(|store| store.reclaim_input(job_id, expected_input_digest, reason))
|
||||
}
|
||||
|
||||
pub fn get(&self, job_id: &str) -> Result<JobSnapshot, JobServiceError> {
|
||||
Ok(self.with_store(|store| store.get(job_id))?)
|
||||
}
|
||||
@@ -274,6 +309,12 @@ impl StandaloneJobs {
|
||||
job_id: &str,
|
||||
client: Option<Box<dyn LlmClient>>,
|
||||
) -> Result<(), JobServiceError> {
|
||||
// Preparation owns copies of the immutable input and domain adapter too. Exclude
|
||||
// owner shutdown before creating them, not only before publishing the task.
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
let client = client.or_else(|| {
|
||||
self.model_client
|
||||
.lock()
|
||||
@@ -282,16 +323,14 @@ impl StandaloneJobs {
|
||||
.map(|c| c.clone_boxed())
|
||||
});
|
||||
let snapshot = self.get(job_id)?;
|
||||
snapshot.load_executable_request()?;
|
||||
let timeout = Duration::from_secs(u64::from(snapshot.request.limits.timeout_seconds));
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
let fence: Arc<dyn JobAttemptFence> = Arc::new(BoundAttemptFence {
|
||||
store: self.store.clone(),
|
||||
job_id: job_id.into(),
|
||||
attempt_id: snapshot.attempt.attempt_id.clone(),
|
||||
input_digest: snapshot
|
||||
.request
|
||||
.input_digest()
|
||||
.map_err(JobStoreError::from)?,
|
||||
input_digest: snapshot.input_digest.clone(),
|
||||
deadline,
|
||||
});
|
||||
let grant = match (
|
||||
@@ -311,10 +350,6 @@ impl StandaloneJobs {
|
||||
let prepared =
|
||||
worker::job::prepare_job_with_grant(&snapshot.request, &self.cwd, client, grant)
|
||||
.map_err(|error| JobServiceError::Configuration(error.to_string()))?;
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions.closed {
|
||||
return Err(JobServiceError::Closed);
|
||||
}
|
||||
executions.jobs.retain(|_, run| {
|
||||
!(run.task.is_finished() && matches!(run.complete.borrow().as_ref(), Some(Ok(()))))
|
||||
});
|
||||
@@ -367,10 +402,7 @@ impl StandaloneJobs {
|
||||
store: store.clone(),
|
||||
job_id: id.clone(),
|
||||
attempt_id: attempt_id.clone(),
|
||||
input_digest: snapshot
|
||||
.request
|
||||
.input_digest()
|
||||
.map_err(JobStoreError::from)?,
|
||||
input_digest: snapshot.input_digest.clone(),
|
||||
deadline,
|
||||
});
|
||||
let (cancel, mut cancellation) = watch::channel(false);
|
||||
@@ -446,23 +478,25 @@ impl StandaloneJobs {
|
||||
.clone()
|
||||
.expect("completion");
|
||||
result.map_err(JobServiceError::Cleanup)?;
|
||||
let task = {
|
||||
// Completion is published immediately before the task returns. Do not remove
|
||||
// its ownership record until the runtime confirms that return (and future drop).
|
||||
while self
|
||||
.executions
|
||||
.lock()
|
||||
.expect("Job executions poisoned")
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.is_some_and(|run| run.attempt_id == attempt && !run.task.is_finished())
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
if executions
|
||||
.jobs
|
||||
.get(job_id)
|
||||
.is_some_and(|run| run.attempt_id == attempt)
|
||||
{
|
||||
executions.jobs.remove(job_id)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(execution) = task {
|
||||
execution
|
||||
.task
|
||||
.await
|
||||
.map_err(|error| JobServiceError::Cleanup(error.to_string()))??;
|
||||
executions.jobs.remove(job_id);
|
||||
}
|
||||
}
|
||||
self.get(job_id)
|
||||
@@ -476,33 +510,63 @@ impl StandaloneJobs {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> {
|
||||
/// Initiate owner shutdown without blocking a synchronous caller. Success confirms all
|
||||
/// local executions have returned, including request/history and Feature cleanup. An
|
||||
/// incomplete/error result retains the service resources and must retain the Host lease.
|
||||
pub(crate) fn try_shutdown(&self) -> Result<(), JobServiceError> {
|
||||
self.close();
|
||||
let tasks = {
|
||||
let mut executions = self.executions.lock().expect("Job executions poisoned");
|
||||
std::mem::take(&mut executions.jobs)
|
||||
};
|
||||
let mut error = None;
|
||||
for execution in tasks.into_values() {
|
||||
match execution.task.await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(failure)) => {
|
||||
error = Some(failure);
|
||||
for execution in executions.jobs.values() {
|
||||
if !execution.task.is_finished() {
|
||||
return Err(JobServiceError::ShutdownIncomplete);
|
||||
}
|
||||
Err(failure) => {
|
||||
error = Some(JobServiceError::Cleanup(failure.to_string()));
|
||||
match execution.complete.borrow().as_ref() {
|
||||
Some(Ok(())) => {}
|
||||
Some(Err(error)) => return Err(JobServiceError::Cleanup(error.clone())),
|
||||
None => {
|
||||
return Err(JobServiceError::Cleanup(
|
||||
"Internal Worker ended without cleanup confirmation".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Close SQLite before the Host releases its lease. Cloned handles may
|
||||
// outlive the Host but cannot mutate or retain the live connection.
|
||||
executions.jobs.clear();
|
||||
// Retained external handles share these slots; they cannot retain a live store,
|
||||
// provider or model client after successful owner shutdown.
|
||||
self.store.lock().expect("Job store poisoned").take();
|
||||
self.domain.lock().expect("Job domain poisoned").take();
|
||||
self.model_client
|
||||
.lock()
|
||||
.expect("Job model transport poisoned")
|
||||
.take();
|
||||
error.map_or(Ok(()), Err)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn shutdown(&self) -> Result<(), JobServiceError> {
|
||||
self.close();
|
||||
let completions: Vec<_> = self
|
||||
.executions
|
||||
.lock()
|
||||
.expect("Job executions poisoned")
|
||||
.jobs
|
||||
.values()
|
||||
.map(|execution| execution.complete.clone())
|
||||
.collect();
|
||||
// Never take execution ownership out of the shared service across an await: a
|
||||
// cancelled shutdown future or a concurrent close must still see the barrier.
|
||||
for mut completion in completions {
|
||||
completion
|
||||
.wait_for(Option::is_some)
|
||||
.await
|
||||
.map_err(|error| JobServiceError::Cleanup(error.to_string()))?;
|
||||
}
|
||||
loop {
|
||||
match self.try_shutdown() {
|
||||
// Only the task's final return follows its completion notification.
|
||||
Err(JobServiceError::ShutdownIncomplete) => tokio::task::yield_now().await,
|
||||
result => return result,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -574,7 +638,8 @@ impl JobAttemptFence for BoundAttemptFence {
|
||||
if snapshot.state != JobState::Pending
|
||||
|| snapshot.attempt.state != JobAttemptState::Dispatched
|
||||
|| snapshot.attempt.attempt_id != self.attempt_id
|
||||
|| snapshot.request.input_digest().map_err(|e| e.to_string())? != self.input_digest
|
||||
|| snapshot.input_digest != self.input_digest
|
||||
|| snapshot.input_disposition != JobInputDisposition::Available
|
||||
{
|
||||
return Err("Job domain grant is not bound to the current live attempt".into());
|
||||
}
|
||||
|
||||
@@ -468,3 +468,104 @@ async fn cleanup_cannot_reclassify_success_or_durable_cancellation_and_stale_han
|
||||
assert!(matches!(operation, Err(JobServiceError::Closed)));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_shutdown_future_keeps_execution_barrier_and_unknown_input() {
|
||||
// Actual generic runner + SQLite; only the provider stream is scripted/blocked.
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let path = temp.path().join("jobs.sqlite3");
|
||||
let jobs = StandaloneJobs::open(&path, temp.path().into()).unwrap();
|
||||
let reserved = jobs.request(request("cancel-shutdown-wait")).unwrap();
|
||||
let model = DomainTestClient {
|
||||
stall: true,
|
||||
..Default::default()
|
||||
};
|
||||
jobs.start_with_model_client(&reserved.request.job_id, model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
await_model(&model).await;
|
||||
{
|
||||
let shutdown = jobs.shutdown();
|
||||
tokio::pin!(shutdown);
|
||||
assert!(futures::poll!(shutdown.as_mut()).is_pending());
|
||||
// Drop the caller's waiter while cancellation/cleanup is still pending.
|
||||
}
|
||||
assert!(matches!(
|
||||
jobs.try_shutdown(),
|
||||
Err(JobServiceError::ShutdownIncomplete)
|
||||
));
|
||||
assert!(jobs.store.lock().unwrap().is_some());
|
||||
assert!(matches!(
|
||||
jobs.request(request("after-shutdown")),
|
||||
Err(JobServiceError::Closed)
|
||||
));
|
||||
tokio::time::timeout(Duration::from_secs(5), jobs.shutdown())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(jobs.store.lock().unwrap().is_none());
|
||||
let reopened = StandaloneJobs::open(&path, temp.path().into()).unwrap();
|
||||
let unknown = reopened.get(&reserved.request.job_id).unwrap();
|
||||
assert_eq!(unknown.attempt.state, JobAttemptState::Unknown);
|
||||
assert_eq!(unknown.input_disposition, JobInputDisposition::Available);
|
||||
assert_eq!(unknown.request, reserved.request);
|
||||
assert_eq!(unknown.input_digest, reserved.input_digest);
|
||||
assert!(reopened.retry(&unknown.request.job_id).is_err());
|
||||
reopened.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn input_retirement_service_fences_live_attempt_and_preserves_session_receipts() {
|
||||
// Production service + SQLite + attempt/result fences, without a model or Worker fixture.
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let path = temp.path().join("jobs.sqlite3");
|
||||
let jobs = StandaloneJobs::open(&path, temp.path().to_path_buf()).unwrap();
|
||||
let original = request("input-retirement");
|
||||
let grant = json!({"session_id":"session-a"});
|
||||
let reserved = jobs
|
||||
.request_granted(original.clone(), grant.clone())
|
||||
.unwrap();
|
||||
jobs.with_store(|store| store.dispatch("input-retirement", &reserved.attempt.attempt_id, 8))
|
||||
.unwrap();
|
||||
let fence = BoundAttemptFence {
|
||||
store: jobs.store.clone(),
|
||||
job_id: "input-retirement".into(),
|
||||
attempt_id: reserved.attempt.attempt_id.clone(),
|
||||
input_digest: reserved.input_digest.clone(),
|
||||
deadline: tokio::time::Instant::now() + Duration::from_secs(10),
|
||||
};
|
||||
assert!(fence.ensure_live().is_ok());
|
||||
assert!(
|
||||
jobs.reclaim_input(
|
||||
"input-retirement",
|
||||
&reserved.input_digest,
|
||||
JobInputRetirementReason::Completed
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let erased = jobs
|
||||
.reclaim_input(
|
||||
"input-retirement",
|
||||
&reserved.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(fence.ensure_live().is_err());
|
||||
assert!(
|
||||
sink(&jobs, "input-retirement", &reserved.attempt.attempt_id)
|
||||
.submit(result("input-retirement", &reserved.attempt.attempt_id))
|
||||
.is_err()
|
||||
);
|
||||
assert!(jobs.start("input-retirement").await.is_err());
|
||||
assert_eq!(
|
||||
jobs.list_session_snapshots("session-a", None, 100).unwrap(),
|
||||
vec![erased.clone()]
|
||||
);
|
||||
assert!(
|
||||
jobs.list_session_snapshots("session-b", None, 100)
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(jobs.request_granted(original, grant).unwrap(), erased);
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
@@ -54,6 +54,8 @@ pub enum SubjectError {
|
||||
Journal(#[from] subjektiv::journal::JournalError),
|
||||
#[error(transparent)]
|
||||
Storage(#[from] feature_storage::FeatureStorageError),
|
||||
#[error("local Subject Job shutdown: {0}")]
|
||||
JobShutdown(#[from] crate::jobs::JobServiceError),
|
||||
#[error("local Subject storage metadata: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error("local Subject storage permission or identity is invalid: {0}")]
|
||||
@@ -338,8 +340,16 @@ impl SubjectConnection {
|
||||
}
|
||||
}
|
||||
pub fn close(&mut self) -> Result<(), SubjectError> {
|
||||
self.host.jobs.lock().expect("Subject Jobs poisoned").take();
|
||||
self.host.live.store(false, Ordering::Release);
|
||||
{
|
||||
let mut jobs = self.host.jobs.lock().expect("Subject Jobs poisoned");
|
||||
if let Some(service) = jobs.as_ref() {
|
||||
// Signal cancellation, but retain the handle and exclusive lease until
|
||||
// the generic owner confirms local request/history/resource teardown.
|
||||
service.try_shutdown()?;
|
||||
}
|
||||
jobs.take();
|
||||
}
|
||||
self.host.catalog.manager.shutdown()?;
|
||||
if let Some(file) = &self.lease {
|
||||
file.unlock()?;
|
||||
|
||||
@@ -5,9 +5,12 @@
|
||||
//! recovery authority; shutdown does not delete either original Sessions or pending requests.
|
||||
|
||||
use super::*;
|
||||
use crate::jobs::{JobAttemptState, JobSnapshot, JobState};
|
||||
use crate::jobs::{
|
||||
JobAttemptState, JobInputDisposition, JobInputRetirementReason, JobSnapshot, JobState,
|
||||
};
|
||||
use session_store::capture::{CaptureDto, CaptureLimits, RecordingPosition};
|
||||
use subjektiv::api::JobAttemptBinding;
|
||||
use subjektiv::journal::recording::JournalMaterial;
|
||||
use subjektiv::journal::recording::JournalInput;
|
||||
use subjektiv::journal::transport::*;
|
||||
use subjektiv::journal::*;
|
||||
use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeature};
|
||||
@@ -16,7 +19,6 @@ use worker::subjektiv::{JournalJobFinalizer, JournalJobHost, JournalJobToolsFeat
|
||||
mod tests;
|
||||
|
||||
pub(super) const PURPOSE: &str = "subjektiv_journal";
|
||||
const INPUT_BUDGET: usize = 48 * 1024;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -27,6 +29,7 @@ struct LocalJournalGrant {
|
||||
session_id: String,
|
||||
journal_id: String,
|
||||
ranges: Vec<JournalSourceRange>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
origins: Vec<SourceOrigin>,
|
||||
}
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -35,11 +38,6 @@ struct SourceOrigin {
|
||||
pointer: JournalSourcePointer,
|
||||
origin: session_store::public_material::PublicMaterialOrigin,
|
||||
}
|
||||
impl LocalJournalGrant {
|
||||
fn input(&self) -> serde_json::Value {
|
||||
serde_json::json!({"selected_ranges": self.ranges, "origins": self.origins})
|
||||
}
|
||||
}
|
||||
|
||||
impl StandaloneSubjects {
|
||||
/// Explicit operator erasure, separate from ordinary Worker removal. Requires the
|
||||
@@ -84,33 +82,30 @@ impl StandaloneSubjects {
|
||||
self.state_root.clone(),
|
||||
)
|
||||
.map_err(|error| SubjectError::InvalidScope(error.to_string()))?;
|
||||
for snapshot in jobs
|
||||
.pending()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
||||
.into_iter()
|
||||
.chain(
|
||||
jobs.unknown_granted()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?,
|
||||
)
|
||||
{
|
||||
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
|
||||
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
|
||||
}) {
|
||||
let mut after = None;
|
||||
loop {
|
||||
let page = jobs
|
||||
.list_session_snapshots(session_id, after.as_deref(), 100)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
if page.is_empty() {
|
||||
break;
|
||||
}
|
||||
for snapshot in &page {
|
||||
jobs.cancel(&snapshot.request.job_id)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
}
|
||||
}
|
||||
for snapshot in jobs
|
||||
.unacknowledged_results()
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?
|
||||
{
|
||||
if snapshot.domain_grant.as_ref().is_some_and(|grant| {
|
||||
grant.get("session_id").and_then(|v| v.as_str()) == Some(session_id)
|
||||
}) {
|
||||
jobs.reclaim_input(
|
||||
&snapshot.request.job_id,
|
||||
&snapshot.input_digest,
|
||||
JobInputRetirementReason::ExplicitErasure,
|
||||
)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
if snapshot.state == JobState::Completed && !snapshot.acknowledged {
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(|e| SubjectError::InvalidScope(e.to_string()))?;
|
||||
}
|
||||
}
|
||||
after = page.last().map(|job| job.request.job_id.clone());
|
||||
}
|
||||
session_store::FsStore::new_with_retention(
|
||||
path,
|
||||
crate::store::session_retention(&self.state_root),
|
||||
@@ -590,17 +585,28 @@ impl LocalSubjectHost {
|
||||
self.ensure_live()?;
|
||||
let jobs = self.jobs()?;
|
||||
for saved in jobs.unacknowledged_results().map_err(service_error)? {
|
||||
if self.journal_grant(&saved).is_ok() {
|
||||
if self.journal_grant_scope(&saved).is_ok() {
|
||||
self.complete_journal_job(&jobs, &saved)?;
|
||||
}
|
||||
}
|
||||
for pending in jobs.pending().map_err(service_error)? {
|
||||
if self.journal_grant(&pending).is_ok() {
|
||||
if pending.request.purpose != PURPOSE {
|
||||
continue;
|
||||
}
|
||||
self.journal_grant_scope(&pending)?;
|
||||
if pending.request.input.get("capture").is_some() {
|
||||
self.journal_grant(&pending)?;
|
||||
return Ok(Some((jobs, pending.request.job_id)));
|
||||
}
|
||||
// Old descriptor jobs never acquire new execution authority. Their immutable
|
||||
// identity/input is retired, not upgraded in place; saved checkpoint wins.
|
||||
let replacement = self.replace_legacy_journal_job(&jobs, &pending)?;
|
||||
if let Some(id) = replacement {
|
||||
return Ok(Some((jobs, id)));
|
||||
}
|
||||
}
|
||||
for unknown in jobs.unknown_granted().map_err(service_error)? {
|
||||
let Ok(grant) = self.journal_grant(&unknown) else {
|
||||
let Ok(grant) = self.journal_grant_scope(&unknown) else {
|
||||
continue;
|
||||
};
|
||||
let context = authority(self, &grant.session_id)?;
|
||||
@@ -685,32 +691,96 @@ impl LocalSubjectHost {
|
||||
}
|
||||
}
|
||||
}
|
||||
let materials = self.select_journal_materials(&context, &journal.id)?;
|
||||
if materials.is_empty() {
|
||||
let mut retained = None;
|
||||
// Saved body snapshots always win over descriptor migration. In particular,
|
||||
// a failed continuation must not recapture its original text even if an
|
||||
// invalidated legacy grant still bounds a larger unprocessed suffix.
|
||||
for legacy_pass in [false, true] {
|
||||
let mut after_job = None;
|
||||
loop {
|
||||
let page = jobs
|
||||
.list_session_snapshots(
|
||||
&journal.source_session_id,
|
||||
after_job.as_deref(),
|
||||
100,
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
if page.is_empty() {
|
||||
break;
|
||||
}
|
||||
for saved in &page {
|
||||
if saved.request.purpose != PURPOSE {
|
||||
continue;
|
||||
}
|
||||
let legacy = saved.input_disposition
|
||||
== JobInputDisposition::Invalidated
|
||||
|| (saved.input_disposition == JobInputDisposition::Available
|
||||
&& saved.request.input.get("capture").is_none());
|
||||
if legacy != legacy_pass
|
||||
|| (!legacy
|
||||
&& saved.input_disposition != JobInputDisposition::Available)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let saved_grant = self.journal_grant_scope(saved)?;
|
||||
if saved_grant.journal_id != journal.id {
|
||||
continue;
|
||||
}
|
||||
if legacy {
|
||||
if let Some(id) = self.replace_legacy_journal_job(&jobs, saved)? {
|
||||
return Ok(Some((jobs, id)));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
self.journal_grant(saved)?;
|
||||
let original = JournalInput::decode(&saved.request.input)
|
||||
.map_err(journal_error)?;
|
||||
let continuation = self.build_journal_input(
|
||||
&context,
|
||||
&journal.id,
|
||||
Some(&original.capture),
|
||||
)?;
|
||||
if !continuation.selected_ranges.is_empty() {
|
||||
retained = Some(continuation);
|
||||
break;
|
||||
}
|
||||
if saved.state == JobState::Completed && saved.acknowledged {
|
||||
// Recover the acknowledgement-before-reclamation gap without
|
||||
// turning a retained receipt into an executable empty input.
|
||||
jobs.reclaim_input(
|
||||
&saved.request.job_id,
|
||||
&saved.input_digest,
|
||||
JobInputRetirementReason::Completed,
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
}
|
||||
}
|
||||
if retained.is_some() {
|
||||
break;
|
||||
}
|
||||
after_job = page.last().map(|job| job.request.job_id.clone());
|
||||
}
|
||||
if retained.is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let input = match retained {
|
||||
Some(input) => input,
|
||||
None => self.build_journal_input(&context, &journal.id, None)?,
|
||||
};
|
||||
|
||||
if input.selected_ranges.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let ranges = materials
|
||||
.into_iter()
|
||||
.map(|material| material.range)
|
||||
.collect::<Vec<_>>();
|
||||
let origins = self.journal_origins(&ranges)?;
|
||||
let grant = LocalJournalGrant {
|
||||
writer_binding_id: Some(self.writer.binding_id.clone()),
|
||||
binding: self.binding.clone(),
|
||||
session_id: journal.source_session_id,
|
||||
journal_id: journal.id.clone(),
|
||||
ranges,
|
||||
origins,
|
||||
};
|
||||
let request = job::JobRequest {
|
||||
job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()),
|
||||
purpose: PURPOSE.into(), input_ref: format!("subjektiv://{}/journal/{}", self.binding.subject_id, journal.id),
|
||||
input: grant.input(),
|
||||
instruction: "ReadJournalState first. Input descriptors are the frozen canonical public ranges authorized for this Job, not already read material. Read every selected source byte with bounded ReadJournalText before recording; then read initial/current position and open text in bounded pages. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(),
|
||||
profile: "builtin:subjektiv-journal-recording".into(),
|
||||
serialization_key: Some(format!("subjektiv-journal:{}", journal.id)),
|
||||
limits: job::JobLimits { timeout_seconds: 600, ..Default::default() },
|
||||
ranges: input.selected_ranges.clone(),
|
||||
origins: vec![],
|
||||
};
|
||||
let request = self.journal_request(&journal.id, &input)?;
|
||||
let saved = jobs
|
||||
.request_granted(
|
||||
request,
|
||||
@@ -727,6 +797,138 @@ impl LocalSubjectHost {
|
||||
}
|
||||
}
|
||||
|
||||
fn replace_legacy_journal_job(
|
||||
&self,
|
||||
jobs: &crate::jobs::StandaloneJobs,
|
||||
old: &JobSnapshot,
|
||||
) -> Result<Option<String>, SubjektivHostError> {
|
||||
let grant = self.journal_grant_scope(old)?;
|
||||
if old.input_disposition == JobInputDisposition::Available {
|
||||
old.load_executable_request()
|
||||
.map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?;
|
||||
if old.request.input
|
||||
!= serde_json::json!({"selected_ranges":grant.ranges,"origins":grant.origins})
|
||||
{
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"invalid legacy Journal descriptor input".into(),
|
||||
));
|
||||
}
|
||||
} else if old.input_disposition != JobInputDisposition::Invalidated {
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"legacy Journal input is unavailable".into(),
|
||||
));
|
||||
}
|
||||
let context = authority(self, &grant.session_id)?;
|
||||
if let Some(active) = self
|
||||
.catalog
|
||||
.store
|
||||
.active_journal_generation(&context, &grant.journal_id)
|
||||
.map_err(journal_error)?
|
||||
{
|
||||
if active.job.job_id != old.request.job_id {
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"another generation owns legacy migration".into(),
|
||||
));
|
||||
}
|
||||
self.catalog
|
||||
.store
|
||||
.close_journal_generation(&context, &active.id, &active.job)
|
||||
.map_err(journal_error)?;
|
||||
}
|
||||
jobs.cancel(&old.request.job_id).map_err(service_error)?;
|
||||
let current = self
|
||||
.catalog
|
||||
.store
|
||||
.read_journal(&context, &grant.journal_id)
|
||||
.map_err(journal_error)?;
|
||||
let end = grant
|
||||
.ranges
|
||||
.last()
|
||||
.ok_or_else(|| SubjektivHostError::InvalidArgument("empty legacy selection".into()))?;
|
||||
let end_position = if end.bytes.end == end.source.total_bytes {
|
||||
(end.source.order + 1, 0)
|
||||
} else {
|
||||
(end.source.order, end.bytes.end)
|
||||
};
|
||||
let replacement = if (current.process_order, current.process_byte) >= end_position {
|
||||
None
|
||||
} else {
|
||||
let mut capture = self.recording_capture(&grant.session_id)?;
|
||||
capture
|
||||
.entries
|
||||
.retain(|entry| grant.ranges.iter().any(|r| r.source.order == entry.order));
|
||||
for entry in &mut capture.entries {
|
||||
let range = grant
|
||||
.ranges
|
||||
.iter()
|
||||
.find(|r| r.source.order == entry.order)
|
||||
.unwrap();
|
||||
let session_store::capture::CaptureBody::CanonicalPublic {
|
||||
descriptor,
|
||||
byte_offset,
|
||||
text,
|
||||
..
|
||||
} = &mut entry.body
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
if source_descriptor(&grant.session_id, descriptor) != range.source
|
||||
|| !grant.origins.iter().any(|origin| {
|
||||
origin.pointer == range.source.pointer && origin.origin == descriptor.origin
|
||||
})
|
||||
|| !text.is_char_boundary(range.bytes.start as usize)
|
||||
|| !text.is_char_boundary(range.bytes.end as usize)
|
||||
{
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"legacy original source identity/range changed".into(),
|
||||
));
|
||||
}
|
||||
*text = text[range.bytes.start as usize..range.bytes.end as usize].to_owned();
|
||||
*byte_offset = range.bytes.start;
|
||||
}
|
||||
let input = self.build_journal_input(&context, &grant.journal_id, Some(&capture))?;
|
||||
let request = self.journal_request(&grant.journal_id, &input)?;
|
||||
let next_grant = LocalJournalGrant {
|
||||
ranges: input.selected_ranges.clone(),
|
||||
origins: vec![],
|
||||
..grant
|
||||
};
|
||||
Some(
|
||||
jobs.request_granted(
|
||||
request,
|
||||
serde_json::to_value(next_grant)
|
||||
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?,
|
||||
)
|
||||
.map_err(service_error)?
|
||||
.request
|
||||
.job_id,
|
||||
)
|
||||
};
|
||||
jobs.reclaim_input(
|
||||
&old.request.job_id,
|
||||
&old.input_digest,
|
||||
JobInputRetirementReason::LegacyInvalidation,
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
Ok(replacement)
|
||||
}
|
||||
|
||||
fn journal_request(
|
||||
&self,
|
||||
journal_id: &str,
|
||||
input: &JournalInput,
|
||||
) -> Result<job::JobRequest, SubjektivHostError> {
|
||||
Ok(job::JobRequest {
|
||||
job_id: format!("subjektiv-journal:{}", uuid::Uuid::now_v7()), purpose: PURPOSE.into(),
|
||||
input_ref: format!("subjektiv://{}/journal/{journal_id}", self.binding.subject_id),
|
||||
input: serde_json::to_value(input).map_err(|e| SubjektivHostError::Unavailable(e.to_string()))?,
|
||||
instruction: "Input contains the frozen canonical public source bodies, provenance and byte ranges. Use only this saved input, never re-fetch Source. ReadJournalState first and read initial/current Journal position/open text in bounded pages if needed. Reconcile saved operation receipts before new creates. Save sequential Create/Rewrite/Fix and UpdateJournalPosition operations, then CheckpointJournal backed by saved record/no-change receipts for all selected input. Do not fix an ongoing event just because this Job/turn ended. SubmitJobResult with result: {} only after the entire selected prefix is reflected. Never infer tool success or user intent not evidenced by public input.".into(),
|
||||
profile: "builtin:subjektiv-journal-recording".into(),
|
||||
serialization_key: Some(format!("subjektiv-journal:{journal_id}")),
|
||||
limits: job::JobLimits { timeout_seconds: 600, ..Default::default() },
|
||||
})
|
||||
}
|
||||
|
||||
fn pin_journal_session(&self, session: &str) -> Result<(), SubjektivHostError> {
|
||||
let path = crate::subjektiv_sessions::authorized_sessions_path(
|
||||
&self.catalog.state_root,
|
||||
@@ -869,11 +1071,16 @@ impl LocalSubjectHost {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn material_reader(
|
||||
fn with_recording_boundary<T>(
|
||||
&self,
|
||||
session: &str,
|
||||
) -> Result<session_store::public_material::SessionPublicMaterialReader, SubjektivHostError>
|
||||
{
|
||||
read: impl FnOnce(
|
||||
&Path,
|
||||
session_store::SessionId,
|
||||
session_store::SegmentId,
|
||||
&session_store::retention::SessionRetentionGuard,
|
||||
) -> Result<T, SubjektivHostError>,
|
||||
) -> Result<T, SubjektivHostError> {
|
||||
let sessions = crate::subjektiv_sessions::authorized_sessions_path(
|
||||
&self.catalog.state_root,
|
||||
&self.binding.scope_id,
|
||||
@@ -906,6 +1113,11 @@ impl LocalSubjectHost {
|
||||
.ok_or_else(|| SubjektivHostError::InvalidArgument("invalid Session storage".into()))?
|
||||
.join("worker")
|
||||
.join(&record.storage_key);
|
||||
let session_id = session
|
||||
.parse()
|
||||
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?;
|
||||
let retention = crate::store::session_retention(&self.catalog.state_root);
|
||||
let guard = retention.lock_live(session_id).map_err(source_error)?;
|
||||
// This is the persisted active selector, not the newest filename/mtime or a model claim.
|
||||
let metadata =
|
||||
session_store::WorkerAggregateStore::open_read_only(metadata_root, &record.storage_key)
|
||||
@@ -933,29 +1145,42 @@ impl LocalSubjectHost {
|
||||
"no attested active lineage for the retained Session".into(),
|
||||
)
|
||||
})?;
|
||||
session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention(
|
||||
&sessions,
|
||||
session
|
||||
.parse()
|
||||
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
|
||||
read(&sessions, session_id, active, &guard)
|
||||
}
|
||||
|
||||
fn recording_capture(&self, session: &str) -> Result<CaptureDto, SubjektivHostError> {
|
||||
self.with_recording_boundary(session, |sessions, session_id, active, guard| {
|
||||
session_store::capture::capture_recording_fs_guarded(
|
||||
sessions,
|
||||
session_id,
|
||||
active,
|
||||
session_store::public_material::PublicMaterialLimits {
|
||||
max_segments: 256,
|
||||
session_store::public_material::PublicMaterialLimits::default(),
|
||||
CaptureLimits {
|
||||
max_bytes: 128 * 1024 * 1024,
|
||||
max_entries: 100_000,
|
||||
max_record_bytes: 64 * 1024 * 1024,
|
||||
max_text_bytes: 64 * 1024 * 1024,
|
||||
..Default::default()
|
||||
},
|
||||
crate::store::session_retention(&self.catalog.state_root),
|
||||
guard,
|
||||
)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
|
||||
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
|
||||
})
|
||||
}
|
||||
|
||||
fn journal_sources(
|
||||
&self,
|
||||
session: &str,
|
||||
) -> Result<Vec<JournalSourceRange>, SubjektivHostError> {
|
||||
let reader = self.material_reader(session)?;
|
||||
Ok(reader
|
||||
.descriptors()
|
||||
self.with_recording_boundary(session, |sessions, session_id, active, guard| {
|
||||
session_store::capture::recording_catalog_fs_guarded(
|
||||
sessions,
|
||||
session_id,
|
||||
active,
|
||||
session_store::public_material::PublicMaterialLimits::default(),
|
||||
guard,
|
||||
|descriptors| {
|
||||
descriptors
|
||||
.iter()
|
||||
.map(|descriptor| JournalSourceRange {
|
||||
source: source_descriptor(session, descriptor),
|
||||
bytes: JournalByteRange {
|
||||
@@ -963,124 +1188,160 @@ impl LocalSubjectHost {
|
||||
end: descriptor.total_bytes,
|
||||
},
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn source_reader(
|
||||
&self,
|
||||
source: &JournalSource,
|
||||
) -> Result<session_store::public_material::SessionPublicMaterialReader, SubjektivHostError>
|
||||
{
|
||||
let sessions = crate::subjektiv_sessions::authorized_sessions_path(
|
||||
&self.catalog.state_root,
|
||||
&self.binding.scope_id,
|
||||
&self.binding.subject_id,
|
||||
&self.catalog.store,
|
||||
&source.session_id,
|
||||
)?;
|
||||
// Frozen, already admitted original pointers do not follow the Worker's later active
|
||||
// Session/Segment. This reads that original lineage and still checks descriptor identity,
|
||||
// canonical order and digest below; the shared service enforces the frozen byte range.
|
||||
session_store::public_material::SessionPublicMaterialReader::open_fs_with_retention(
|
||||
&sessions,
|
||||
source
|
||||
.session_id
|
||||
.parse()
|
||||
.map_err(|_| SubjektivHostError::InvalidArgument("invalid Session".into()))?,
|
||||
source.pointer.segment_id,
|
||||
session_store::public_material::PublicMaterialLimits {
|
||||
max_segments: 256,
|
||||
max_entries: 100_000,
|
||||
max_record_bytes: 64 * 1024 * 1024,
|
||||
},
|
||||
crate::store::session_retention(&self.catalog.state_root),
|
||||
)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
|
||||
}
|
||||
|
||||
fn journal_origins(
|
||||
&self,
|
||||
ranges: &[JournalSourceRange],
|
||||
) -> Result<Vec<SourceOrigin>, SubjektivHostError> {
|
||||
if ranges.is_empty() {
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"empty material selection".into(),
|
||||
));
|
||||
}
|
||||
ranges
|
||||
.iter()
|
||||
.map(|range| {
|
||||
let reader = self.source_reader(&range.source)?;
|
||||
reader
|
||||
.descriptors()
|
||||
.find(|descriptor| {
|
||||
source_descriptor(&range.source.session_id, descriptor) == range.source
|
||||
})
|
||||
.map(|descriptor| SourceOrigin {
|
||||
pointer: range.source.pointer.clone(),
|
||||
origin: descriptor.origin.clone(),
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
SubjektivHostError::InvalidArgument(
|
||||
"source provenance/identity changed".into(),
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn journal_source_window(
|
||||
&self,
|
||||
source: &JournalSource,
|
||||
offset: u64,
|
||||
max_bytes: usize,
|
||||
) -> Result<String, SubjektivHostError> {
|
||||
let reader = self.source_reader(source)?;
|
||||
let descriptor = reader.descriptors().find(|descriptor| source_descriptor(&source.session_id, descriptor) == *source)
|
||||
.ok_or_else(|| SubjektivHostError::InvalidArgument("canonical source identity/order/digest changed or is outside the authorized Session".into()))?;
|
||||
reader
|
||||
.read_content(descriptor, offset, max_bytes)
|
||||
.map(|window| window.content)
|
||||
.map_err(|error| SubjektivHostError::Unavailable(error.to_string()))
|
||||
}
|
||||
|
||||
fn journal_source_material(
|
||||
&self,
|
||||
source: &JournalSource,
|
||||
start: u64,
|
||||
limit: usize,
|
||||
) -> JournalResult<JournalMaterial> {
|
||||
let text = self
|
||||
.journal_source_window(source, start, limit.max(4))
|
||||
.map_err(|error| JournalError::Conflict(error.to_string()))?;
|
||||
Ok(JournalMaterial {
|
||||
range: JournalSourceRange {
|
||||
source: source.clone(),
|
||||
bytes: JournalByteRange {
|
||||
start,
|
||||
end: start + text.len() as u64,
|
||||
},
|
||||
},
|
||||
text,
|
||||
)
|
||||
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
|
||||
})
|
||||
}
|
||||
|
||||
fn select_journal_materials(
|
||||
fn build_journal_input(
|
||||
&self,
|
||||
context: &JournalHostContext,
|
||||
id: &str,
|
||||
) -> Result<Vec<JournalMaterial>, SubjektivHostError> {
|
||||
subjektiv::journal::recording::select_materials_bounded(
|
||||
&self.catalog.store,
|
||||
context,
|
||||
id,
|
||||
INPUT_BUDGET,
|
||||
|source, start, limit| self.journal_source_material(source, start, limit),
|
||||
saved: Option<&CaptureDto>,
|
||||
) -> Result<JournalInput, SubjektivHostError> {
|
||||
let session = self
|
||||
.catalog
|
||||
.store
|
||||
.read_journal(context, id)
|
||||
.map_err(journal_error)?
|
||||
.source_session_id;
|
||||
JournalInput::build(&self.catalog.store, context, id, |order, offset, budget| {
|
||||
let capture = match saved {
|
||||
Some(capture) => capture.clone(),
|
||||
None => {
|
||||
let capture = self
|
||||
.with_recording_boundary(&session, |sessions, session_id, active, guard| {
|
||||
session_store::capture::capture_recording_slice_fs_guarded(
|
||||
sessions,
|
||||
session_id,
|
||||
active,
|
||||
session_store::public_material::PublicMaterialLimits::default(),
|
||||
RecordingPosition {
|
||||
canonical_order: order,
|
||||
byte_offset: offset,
|
||||
},
|
||||
MAX_JOURNAL_RANGES,
|
||||
budget,
|
||||
guard,
|
||||
)
|
||||
.map(|slice| slice.capture)
|
||||
.map_err(|e| SubjektivHostError::Unavailable(e.to_string()))
|
||||
})
|
||||
.map_err(|e| JournalError::Conflict(e.to_string()))?;
|
||||
// Admission and capture are separate saves. An append between them
|
||||
// may be in replay but has not yet been accepted for this Journal.
|
||||
return self.accepted_capture_prefix(context, id, capture);
|
||||
}
|
||||
};
|
||||
let end = capture
|
||||
.entries
|
||||
.last()
|
||||
.map(|entry| {
|
||||
capture
|
||||
.recording_text(entry.order)
|
||||
.map(|(descriptor, start, text)| {
|
||||
if start + text.len() as u64 == descriptor.total_bytes {
|
||||
RecordingPosition {
|
||||
canonical_order: entry.order + 1,
|
||||
byte_offset: 0,
|
||||
}
|
||||
} else {
|
||||
RecordingPosition {
|
||||
canonical_order: entry.order,
|
||||
byte_offset: start + text.len() as u64,
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
.map_err(|e| JournalError::Conflict(e.to_string()))?;
|
||||
if end.is_none_or(|end| (order, offset) >= (end.canonical_order, end.byte_offset)) {
|
||||
return Ok(CaptureDto {
|
||||
entries: vec![],
|
||||
..capture
|
||||
});
|
||||
}
|
||||
capture
|
||||
.recording_slice(
|
||||
RecordingPosition {
|
||||
canonical_order: order,
|
||||
byte_offset: offset,
|
||||
},
|
||||
MAX_JOURNAL_RANGES,
|
||||
budget,
|
||||
)
|
||||
.map(|slice| slice.capture)
|
||||
.map_err(|e| JournalError::Conflict(e.to_string()))
|
||||
})
|
||||
.map_err(journal_error)
|
||||
}
|
||||
|
||||
fn journal_grant(
|
||||
fn accepted_capture_prefix(
|
||||
&self,
|
||||
context: &JournalHostContext,
|
||||
id: &str,
|
||||
mut capture: CaptureDto,
|
||||
) -> JournalResult<CaptureDto> {
|
||||
let mut selected = Vec::new();
|
||||
for mut entry in capture.entries {
|
||||
let status = match self
|
||||
.catalog
|
||||
.store
|
||||
.read_journal_source(context, id, entry.order)
|
||||
{
|
||||
Ok(status) => status,
|
||||
Err(JournalError::NotFound(_)) => break,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let session_store::capture::CaptureBody::CanonicalPublic {
|
||||
descriptor,
|
||||
byte_offset,
|
||||
text,
|
||||
..
|
||||
} = &mut entry.body
|
||||
else {
|
||||
return Err(JournalError::InvalidInput("non-recording capture".into()));
|
||||
};
|
||||
if source_descriptor(&capture.session_id, descriptor) != status.source {
|
||||
return Err(JournalError::Conflict(
|
||||
"captured source differs from admitted identity".into(),
|
||||
));
|
||||
}
|
||||
let Some(accepted) = status
|
||||
.accepted
|
||||
.iter()
|
||||
.find(|range| range.start <= *byte_offset && *byte_offset < range.end)
|
||||
else {
|
||||
break;
|
||||
};
|
||||
let end = status
|
||||
.unavailable
|
||||
.iter()
|
||||
.filter(|range| range.end > *byte_offset)
|
||||
.fold(accepted.end, |end, range| {
|
||||
end.min(range.start.max(*byte_offset))
|
||||
});
|
||||
let mut length = text.len().min(end.saturating_sub(*byte_offset) as usize);
|
||||
while !text.is_char_boundary(length) {
|
||||
length -= 1;
|
||||
}
|
||||
if length == 0 {
|
||||
break;
|
||||
}
|
||||
text.truncate(length);
|
||||
let partial = *byte_offset + (text.len() as u64) < descriptor.total_bytes;
|
||||
selected.push(entry);
|
||||
if partial {
|
||||
break;
|
||||
}
|
||||
}
|
||||
capture.entries = selected;
|
||||
Ok(capture)
|
||||
}
|
||||
|
||||
fn journal_grant_scope(
|
||||
&self,
|
||||
snapshot: &JobSnapshot,
|
||||
) -> Result<LocalJournalGrant, SubjektivHostError> {
|
||||
@@ -1097,7 +1358,6 @@ impl LocalSubjectHost {
|
||||
|| snapshot.request.purpose != PURPOSE
|
||||
|| snapshot.request.serialization_key.as_deref()
|
||||
!= Some(&format!("subjektiv-journal:{}", grant.journal_id))
|
||||
|| snapshot.request.input != grant.input()
|
||||
{
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"Job input/grant does not match this Journal capability".into(),
|
||||
@@ -1135,12 +1395,33 @@ impl LocalSubjectHost {
|
||||
Ok(grant)
|
||||
}
|
||||
|
||||
fn journal_grant(
|
||||
&self,
|
||||
snapshot: &JobSnapshot,
|
||||
) -> Result<LocalJournalGrant, SubjektivHostError> {
|
||||
snapshot
|
||||
.load_executable_request()
|
||||
.map_err(|e| SubjektivHostError::InvalidArgument(e.to_string()))?;
|
||||
let grant = self.journal_grant_scope(snapshot)?;
|
||||
let input = JournalInput::decode(&snapshot.request.input).map_err(journal_error)?;
|
||||
if input.subject_id != self.binding.subject_id
|
||||
|| input.journal_id != grant.journal_id
|
||||
|| input.capture.session_id != grant.session_id
|
||||
|| input.selected_ranges != grant.ranges
|
||||
{
|
||||
return Err(SubjektivHostError::InvalidArgument(
|
||||
"saved Journal body identity/ranges differ from grant".into(),
|
||||
));
|
||||
}
|
||||
Ok(grant)
|
||||
}
|
||||
|
||||
fn complete_journal_job(
|
||||
&self,
|
||||
jobs: &crate::jobs::StandaloneJobs,
|
||||
snapshot: &JobSnapshot,
|
||||
) -> Result<(), SubjektivHostError> {
|
||||
let grant = self.journal_grant(snapshot)?;
|
||||
let grant = self.journal_grant_scope(snapshot)?;
|
||||
let context = authority(self, &grant.session_id)?;
|
||||
let completion: JournalJobCompletion =
|
||||
serde_json::from_value(snapshot.result.clone().ok_or_else(|| {
|
||||
@@ -1174,6 +1455,12 @@ impl LocalSubjectHost {
|
||||
}
|
||||
jobs.acknowledge(&snapshot.request.job_id)
|
||||
.map_err(service_error)?;
|
||||
jobs.reclaim_input(
|
||||
&snapshot.request.job_id,
|
||||
&snapshot.input_digest,
|
||||
JobInputRetirementReason::Completed,
|
||||
)
|
||||
.map_err(service_error)?;
|
||||
if let Err(error) = self.prune_retained_workers() {
|
||||
// Pruning is independent of a successful checkpoint/result. Keep the originals
|
||||
// and continue draining other saved prefixes; a retention error is not model failure.
|
||||
@@ -1192,47 +1479,18 @@ pub(super) fn grant(
|
||||
.journal_grant(snapshot)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let context = authority(&host, &grant.session_id).map_err(|error| error.to_string())?;
|
||||
// Re-fetch selected identities before opening mutation authority. Only descriptors are
|
||||
// retained in Job storage; the model must fetch actual source bytes with bounded tools.
|
||||
for range in &grant.ranges {
|
||||
let length = usize::try_from(
|
||||
range
|
||||
.bytes
|
||||
.end
|
||||
.checked_sub(range.bytes.start)
|
||||
.ok_or("invalid source range")?,
|
||||
)
|
||||
.map_err(|_| "source range overflow")?;
|
||||
if length > INPUT_BUDGET {
|
||||
return Err("source range exceeds frozen input budget".into());
|
||||
}
|
||||
let text = host
|
||||
.journal_source_window(&range.source, range.bytes.start, length.max(4))
|
||||
.map_err(|error| error.to_string())?;
|
||||
if text.len() != length {
|
||||
return Err("frozen Journal source byte range changed".into());
|
||||
}
|
||||
}
|
||||
if host
|
||||
.journal_origins(&grant.ranges)
|
||||
.map_err(|error| error.to_string())?
|
||||
!= grant.origins
|
||||
{
|
||||
return Err(
|
||||
"frozen Journal provenance does not match the authorized original Session".into(),
|
||||
);
|
||||
}
|
||||
let input = JournalInput::decode(&snapshot.request.input).map_err(|e| e.to_string())?;
|
||||
let input_digest = snapshot.input_digest.clone();
|
||||
let generation = host
|
||||
.catalog
|
||||
.store
|
||||
.prepare_journal_generation(
|
||||
.prepare_journal_input(
|
||||
&context,
|
||||
&grant.journal_id,
|
||||
&input,
|
||||
JobAttemptBinding {
|
||||
job_id: snapshot.request.job_id.clone(),
|
||||
attempt_id: snapshot.attempt.attempt_id.clone(),
|
||||
},
|
||||
grant.ranges.clone(),
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let job_context = JournalJobContext::validated(
|
||||
@@ -1242,12 +1500,17 @@ pub(super) fn grant(
|
||||
generation.job.clone(),
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
input
|
||||
.validate_generation(&generation)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let adapter = Arc::new(LocalJournalHost {
|
||||
host,
|
||||
grant,
|
||||
context,
|
||||
job_context,
|
||||
#[cfg(test)]
|
||||
generation,
|
||||
input_digest,
|
||||
fence,
|
||||
});
|
||||
Ok(worker::job::JobFeatureGrant {
|
||||
@@ -1263,7 +1526,9 @@ struct LocalJournalHost {
|
||||
grant: LocalJournalGrant,
|
||||
context: JournalHostContext,
|
||||
job_context: JournalJobContext,
|
||||
#[cfg(test)]
|
||||
generation: JournalGeneration,
|
||||
input_digest: String,
|
||||
fence: Arc<dyn crate::jobs::JobAttemptFence>,
|
||||
}
|
||||
impl LocalJournalHost {
|
||||
@@ -1296,18 +1561,20 @@ impl JournalJobHost for LocalJournalHost {
|
||||
request: JournalJobRequest,
|
||||
) -> Result<JournalJobResponse, SubjektivHostError> {
|
||||
let context = self.live()?;
|
||||
subjektiv::journal::service::execute_bounded(
|
||||
// The generic runner has supplied this exact immutable input before it can
|
||||
// invoke this bound tool. Preparation alone is not evidence of delivery.
|
||||
subjektiv::journal::service::attest_input_delivery(
|
||||
&self.host.catalog.store,
|
||||
&context,
|
||||
&self.input_digest,
|
||||
&self.grant.ranges,
|
||||
)
|
||||
.map_err(journal_error)?;
|
||||
subjektiv::journal::service::execute_snapshot(
|
||||
&self.host.catalog.store,
|
||||
&context,
|
||||
&self.input_digest,
|
||||
request,
|
||||
|source, start, limit| {
|
||||
if source.session_id != self.generation.initial_journal.source_session_id {
|
||||
return Err(JournalError::PermissionDenied(
|
||||
"foreign original Session".into(),
|
||||
));
|
||||
}
|
||||
self.host.journal_source_material(source, start, limit)
|
||||
},
|
||||
)
|
||||
.map_err(journal_error)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Integration of standalone attribution, original FsStore material, shared Journal storage
|
||||
//! and the local Job adapter. Only the outer model/execution lease is replaced in these cases.
|
||||
//! The generic Job runner is exercised separately by the standalone Jobs integration suite.
|
||||
//! The scripted model also exercises saved direct input through the actual generic Job runner.
|
||||
|
||||
use super::*;
|
||||
use crate::store::{StaleLeasePolicy, StandaloneWorkerAllocation, StandaloneWorkerStore};
|
||||
@@ -144,8 +144,8 @@ impl Fixture {
|
||||
.unwrap()
|
||||
.contains(sentinel)
|
||||
);
|
||||
// Include SQLite/WAL/journal files: immutable request/grant storage must never hold
|
||||
// original text, even in freed pages left behind by explicit source erasure.
|
||||
// Include SQLite/WAL/journal files: erasure must scrub retired original bodies,
|
||||
// including any freed pages or journal left by the immutable request store.
|
||||
for entry in fs::read_dir(self.connection.jobs_path.parent().unwrap()).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
if entry.file_type().unwrap().is_file() {
|
||||
@@ -194,9 +194,9 @@ impl Fixture {
|
||||
context,
|
||||
job_context,
|
||||
generation,
|
||||
input_digest: snapshot.input_digest.clone(),
|
||||
fence: fence.clone(),
|
||||
};
|
||||
adapter.read_all_sources();
|
||||
(adapter, fence, snapshot)
|
||||
}
|
||||
}
|
||||
@@ -220,37 +220,6 @@ fn message(id: &str, text: &str) -> session_store::LoggedHistoryEntry {
|
||||
}
|
||||
}
|
||||
impl LocalJournalHost {
|
||||
fn read_all_sources(&self) {
|
||||
for range in &self.generation.selected_ranges {
|
||||
let mut offset = range.bytes.start as usize;
|
||||
let mut content_token = None;
|
||||
loop {
|
||||
let response = self
|
||||
.execute(JournalJobRequest::ReadText {
|
||||
read: JournalTextRead {
|
||||
target: JournalTextTarget::Source {
|
||||
pointer: range.source.pointer.clone(),
|
||||
},
|
||||
offset,
|
||||
limit: MAX_JOURNAL_READ_BYTES,
|
||||
content_token: content_token.clone(),
|
||||
},
|
||||
})
|
||||
.unwrap();
|
||||
let JournalJobResponse::Text(text) = response else {
|
||||
panic!()
|
||||
};
|
||||
match text.next_byte {
|
||||
Some(next) => {
|
||||
assert!(next > offset);
|
||||
offset = next;
|
||||
content_token = Some(text.content_token);
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fn state(&self) -> Result<JournalJobState, SubjektivHostError> {
|
||||
match self.execute(JournalJobRequest::ReadState)? {
|
||||
JournalJobResponse::State(state) => Ok(state),
|
||||
@@ -362,8 +331,8 @@ fn duplicate_admission_reserves_one_job_and_excludes_private_prompt() {
|
||||
assert_eq!(jobs.pending().unwrap().len(), 1);
|
||||
let input = jobs.get(&id).unwrap().request.input.to_string();
|
||||
assert!(
|
||||
!input.contains("short request"),
|
||||
"Job storage retains descriptors, never a second copy of original text"
|
||||
input.contains("short request"),
|
||||
"Job storage supplies the canonical body directly"
|
||||
);
|
||||
assert!(!input.contains("PRIVATE PROMPT"));
|
||||
let foreign = SubjektivHostContext {
|
||||
@@ -498,14 +467,16 @@ fn generation_input_stays_frozen_and_next_prefix_includes_last_input() {
|
||||
&adapter.generation.job,
|
||||
)
|
||||
.unwrap();
|
||||
let materials = fixture
|
||||
let next = fixture
|
||||
.host()
|
||||
.select_journal_materials(&adapter.context, &adapter.grant.journal_id)
|
||||
.build_journal_input(&adapter.context, &adapter.grant.journal_id, None)
|
||||
.unwrap();
|
||||
assert_eq!(materials[0].range.source.pointer.entry_ref, "Elast");
|
||||
assert_eq!(next.selected_ranges[0].source.pointer.entry_ref, "Elast");
|
||||
assert!(
|
||||
materials[0]
|
||||
.text
|
||||
next.capture
|
||||
.recording_text(next.selected_ranges[0].source.order)
|
||||
.unwrap()
|
||||
.2
|
||||
.contains("last input while generation runs")
|
||||
);
|
||||
}
|
||||
@@ -547,7 +518,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() {
|
||||
let (adapter, _, _) = fixture.adapter();
|
||||
let range = &adapter.grant.ranges[0];
|
||||
assert!(range.bytes.end < range.source.total_bytes);
|
||||
assert!(range.bytes.end - range.bytes.start < INPUT_BUDGET as u64);
|
||||
assert!(range.bytes.end - range.bytes.start < job::MAX_JOB_INPUT_BYTES as u64);
|
||||
let snapshot = fixture
|
||||
.host()
|
||||
.jobs()
|
||||
@@ -557,7 +528,7 @@ fn large_original_is_chunked_without_claiming_unread_suffix() {
|
||||
.pop()
|
||||
.unwrap();
|
||||
assert!(serde_json::to_vec(&snapshot.request.input).unwrap().len() < 128 * 1024);
|
||||
assert!(!snapshot.request.input.to_string().contains("漢字"));
|
||||
assert!(snapshot.request.input.to_string().contains("漢字"));
|
||||
assert!(adapter.finish().is_err());
|
||||
}
|
||||
|
||||
@@ -628,43 +599,35 @@ impl JournalScript {
|
||||
)
|
||||
} else if let Some(state) = result("state") {
|
||||
let state = &state["value"];
|
||||
for (index, range) in state["selected_ranges"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
let input: serde_json::Value = request
|
||||
.items
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let mut offset = range["bytes"]["start"].as_u64().unwrap();
|
||||
let mut token = Value::Null;
|
||||
loop {
|
||||
let id = format!("source-{index}-{offset}");
|
||||
if let Some(page) = result(&id) {
|
||||
let page = &page["value"];
|
||||
let key = format!("{}:{id}", state["generation_id"].as_str().unwrap());
|
||||
let text = page["text"].as_str().unwrap().to_owned();
|
||||
if let Some(previous) =
|
||||
self.source_reads.lock().unwrap().insert(key, text.clone())
|
||||
{
|
||||
.filter_map(Item::as_text)
|
||||
.find_map(|text| {
|
||||
text.lines()
|
||||
.find_map(|line| line.strip_prefix("input_json: "))
|
||||
})
|
||||
.map(|json| serde_json::from_str(json).unwrap())
|
||||
.expect("direct saved Journal input");
|
||||
let input = JournalInput::decode(&input).unwrap();
|
||||
assert_eq!(
|
||||
previous, text,
|
||||
"frozen source read changed within a generation"
|
||||
serde_json::to_value(&input.selected_ranges).unwrap(),
|
||||
state["selected_ranges"]
|
||||
);
|
||||
}
|
||||
if let Some(next) = page["next_byte"].as_u64() {
|
||||
assert!(next > offset);
|
||||
offset = next;
|
||||
token = page["content_token"].clone();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
let input = json!({"target":{"target":"source","pointer":range["source"]["pointer"]},"offset":offset,"limit":MAX_JOURNAL_READ_BYTES,"content_token":token});
|
||||
return vec![
|
||||
Event::tool_use_start(0, &id, "ReadJournalText"),
|
||||
Event::tool_input_delta(0, input.to_string()),
|
||||
Event::tool_use_stop(0),
|
||||
];
|
||||
}
|
||||
for entry in &input.capture.entries {
|
||||
let (_, offset, text) = input.capture.recording_text(entry.order).unwrap();
|
||||
let key = format!(
|
||||
"{}:{}:{offset}",
|
||||
state["generation_id"].as_str().unwrap(),
|
||||
entry.order
|
||||
);
|
||||
if let Some(previous) = self
|
||||
.source_reads
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(key, text.to_owned())
|
||||
{
|
||||
assert_eq!(previous, text, "saved source changed within generation");
|
||||
}
|
||||
}
|
||||
let (name, action) = if state["current_open"].is_null() {
|
||||
@@ -1166,6 +1129,11 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
|
||||
.items;
|
||||
assert_eq!(entries.len(), 1);
|
||||
let saved_entry = entries[0].id.clone();
|
||||
let original_input = jobs.get(&id).unwrap().request.input;
|
||||
fixture.append(
|
||||
"newer-than-snapshot",
|
||||
"NEW INPUT MUST NOT MIX INTO RESUMED SNAPSHOT",
|
||||
);
|
||||
jobs.shutdown().await.unwrap();
|
||||
fixture.connection.close().unwrap();
|
||||
let mut reopened = StandaloneSubjects::open_existing(&state)
|
||||
@@ -1176,8 +1144,29 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
|
||||
.unwrap();
|
||||
reopened.host.bind_jobs(jobs.clone()).unwrap();
|
||||
assert_eq!(jobs.get(&id).unwrap().state, JobState::Unknown);
|
||||
// Domain reconciliation inspects the now-fenced prior attempt and preserves its receipts.
|
||||
// Deny every original-source read while preparing and running the replacement.
|
||||
let source_path = fixture
|
||||
.sessions
|
||||
.root_dir()
|
||||
.join(&fixture.context.session_id);
|
||||
let denied = fixture
|
||||
._temp
|
||||
.path()
|
||||
.join("temporarily-unavailable-original");
|
||||
fs::rename(&source_path, &denied).unwrap();
|
||||
let (_, replacement) = reopened.host.next_journal_job().unwrap().unwrap();
|
||||
let replacement_input = jobs.get(&replacement).unwrap().request.input;
|
||||
assert_eq!(replacement_input["capture"], original_input["capture"]);
|
||||
assert!(
|
||||
!replacement_input
|
||||
.to_string()
|
||||
.contains("NEW INPUT MUST NOT MIX")
|
||||
);
|
||||
assert_eq!(jobs.get(&id).unwrap().request.input, original_input);
|
||||
assert_eq!(
|
||||
jobs.get(&id).unwrap().input_disposition,
|
||||
JobInputDisposition::Available
|
||||
);
|
||||
assert_ne!(
|
||||
replacement, id,
|
||||
"unknown external execution is not blindly replayed"
|
||||
@@ -1205,8 +1194,26 @@ async fn shutdown_reopen_reconciles_partial_saved_operations_without_duplicate_c
|
||||
.items;
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].id, saved_entry);
|
||||
fs::rename(&denied, &source_path).unwrap();
|
||||
let journal = reopened
|
||||
.host
|
||||
.catalog
|
||||
.store
|
||||
.read_journal(&context, &journal_id)
|
||||
.unwrap();
|
||||
let original = JournalInput::decode(&original_input).unwrap();
|
||||
let end = original.selected_ranges.last().unwrap();
|
||||
assert_eq!(
|
||||
(journal.process_order, journal.process_byte),
|
||||
if end.bytes.end == end.source.total_bytes {
|
||||
(end.source.order + 1, 0)
|
||||
} else {
|
||||
(end.source.order, end.bytes.end)
|
||||
}
|
||||
);
|
||||
assert!(reopened.host.admit_journal(&fixture.context).unwrap());
|
||||
assert!(
|
||||
reopened
|
||||
!reopened
|
||||
.host
|
||||
.session_fully_reflected(&fixture.context.session_id)
|
||||
.unwrap()
|
||||
@@ -1302,9 +1309,9 @@ fn missing_original_material_blocks_worker_removal_without_erasing_state() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frozen_original_remains_readable_after_worker_switches_its_active_session() {
|
||||
fn frozen_input_survives_active_session_change_and_source_tool_remains_refused() {
|
||||
let fixture = Fixture::new("old Session original");
|
||||
let (adapter, _, _) = fixture.adapter();
|
||||
let (adapter, _, snapshot) = fixture.adapter();
|
||||
let workers = StandaloneWorkerStore::open_existing(&fixture.host().catalog.state_root).unwrap();
|
||||
let worker = fixture._allocation.as_ref().unwrap().worker_id();
|
||||
let record = workers.load(worker).unwrap();
|
||||
@@ -1318,22 +1325,30 @@ fn frozen_original_remains_readable_after_worker_switches_its_active_session() {
|
||||
)),
|
||||
))
|
||||
.unwrap();
|
||||
let response = adapter
|
||||
assert!(
|
||||
adapter
|
||||
.execute(JournalJobRequest::ReadText {
|
||||
read: JournalTextRead {
|
||||
target: JournalTextTarget::Source {
|
||||
pointer: adapter.generation.selected_ranges[0].source.pointer.clone(),
|
||||
pointer: adapter.generation.selected_ranges[0].source.pointer.clone()
|
||||
},
|
||||
offset: 0,
|
||||
limit: MAX_JOURNAL_READ_BYTES,
|
||||
content_token: None,
|
||||
},
|
||||
content_token: None
|
||||
}
|
||||
})
|
||||
.unwrap();
|
||||
let JournalJobResponse::Text(page) = response else {
|
||||
panic!()
|
||||
};
|
||||
assert!(page.text.contains("old Session original"));
|
||||
.is_err(),
|
||||
"original Source lookup is explicitly unavailable"
|
||||
);
|
||||
let input = JournalInput::decode(&snapshot.request.input).unwrap();
|
||||
assert!(
|
||||
input
|
||||
.capture
|
||||
.recording_text(0)
|
||||
.unwrap()
|
||||
.2
|
||||
.contains("old Session original")
|
||||
);
|
||||
assert!(
|
||||
fixture
|
||||
.host()
|
||||
@@ -1512,13 +1527,17 @@ fn relocated_erasure_fences_old_and_retained_readers_publishers_and_late_pins()
|
||||
const SENTINEL: &str = "original-plaintext-sentinel-must-never-enter-immutable-job-storage";
|
||||
let mut fixture = Fixture::new(SENTINEL);
|
||||
let (_, _, snapshot) = fixture.adapter();
|
||||
fixture.assert_job_storage_excludes(&snapshot, SENTINEL);
|
||||
assert!(snapshot.request.input.to_string().contains(SENTINEL));
|
||||
let state = fixture.host().catalog.state_root.clone();
|
||||
let binding = fixture.connection.binding.clone();
|
||||
let session: session_store::SessionId = fixture.context.session_id.parse().unwrap();
|
||||
let old_reader = fixture
|
||||
.host()
|
||||
.material_reader(&fixture.context.session_id)
|
||||
let old_reader = SessionPublicMaterialReader::open_fs_with_retention(
|
||||
fixture.sessions.root_dir(),
|
||||
session,
|
||||
fixture.segment,
|
||||
PublicMaterialLimits::default(),
|
||||
crate::store::session_retention(&state),
|
||||
)
|
||||
.unwrap();
|
||||
let descriptor = old_reader.descriptors().next().unwrap().clone();
|
||||
let (worker, workers) = fixture.stop_worker();
|
||||
@@ -1759,14 +1778,12 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job(
|
||||
let snapshot = jobs.get(&next).unwrap();
|
||||
let grant = fixture.host().journal_grant(&snapshot).unwrap();
|
||||
assert_eq!(grant.ranges.len(), 1);
|
||||
assert_eq!(
|
||||
grant.origins[0].origin,
|
||||
PublicMaterialOrigin::HostRunBoundary
|
||||
);
|
||||
let content = fixture
|
||||
.host()
|
||||
.journal_source_window(&grant.ranges[0].source, 0, 4096)
|
||||
let input = JournalInput::decode(&snapshot.request.input).unwrap();
|
||||
let (descriptor, _, content) = input
|
||||
.capture
|
||||
.recording_text(grant.ranges[0].source.order)
|
||||
.unwrap();
|
||||
assert_eq!(descriptor.origin, PublicMaterialOrigin::HostRunBoundary);
|
||||
let value: serde_json::Value = serde_json::from_str(&content).unwrap();
|
||||
assert_eq!(value["kind"], "run_boundary");
|
||||
assert_eq!(value["boundary"]["kind"], "errored");
|
||||
@@ -1791,3 +1808,621 @@ async fn late_terminal_boundary_after_reflected_task_prefix_is_read_by_next_job(
|
||||
assert!(fixture.host().next_journal_job().unwrap().is_none());
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
/// Original FsStore -> bounded common capture -> persisted Job -> actual generic runner ->
|
||||
/// sequential Journal tools. Only the LLM transport is scripted. Source files are deliberately
|
||||
/// unavailable after persistence, and input bodies are inspected in the provider request.
|
||||
#[tokio::test]
|
||||
async fn saved_body_jobs_process_over_a_hundred_entries_and_utf8_without_live_source_reads() {
|
||||
let fixture = Fixture::new("oldest original");
|
||||
for index in 0..115 {
|
||||
fixture.append(
|
||||
&format!("entry-{index}"),
|
||||
&format!("public original {index}"),
|
||||
);
|
||||
}
|
||||
fixture.append("long-utf8", &"漢字🙂\n\"".repeat(24_000));
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let expected = fixture
|
||||
.host()
|
||||
.recording_capture(&fixture.context.session_id)
|
||||
.unwrap();
|
||||
let source_path = fixture
|
||||
.sessions
|
||||
.root_dir()
|
||||
.join(&fixture.context.session_id);
|
||||
let denied = fixture._temp.path().join("denied-original-source");
|
||||
let mut seen: std::collections::BTreeMap<u64, String> = Default::default();
|
||||
let mut jobs_processed = 0;
|
||||
loop {
|
||||
let Some((jobs, id)) = fixture.host().next_journal_job().unwrap() else {
|
||||
break;
|
||||
};
|
||||
let saved = jobs.get(&id).unwrap();
|
||||
let input = JournalInput::decode(&saved.request.input).unwrap();
|
||||
assert!(
|
||||
serde_json::to_vec(&saved.request.input).unwrap().len() <= job::MAX_JOB_INPUT_BYTES
|
||||
);
|
||||
assert!(input.capture.entries.len() <= MAX_JOURNAL_RANGES);
|
||||
for entry in &input.capture.entries {
|
||||
let (_, offset, text) = input.capture.recording_text(entry.order).unwrap();
|
||||
let previous = seen.entry(entry.order).or_default();
|
||||
assert_eq!(
|
||||
previous.len() as u64,
|
||||
offset,
|
||||
"no gap, preview or duplicate prefix"
|
||||
);
|
||||
previous.push_str(text);
|
||||
}
|
||||
// Deny all catalogs/reads, including grant preparation. The model still receives
|
||||
// exactly the persisted body, and the Host must not follow the active pointer.
|
||||
fs::rename(&source_path, &denied).unwrap();
|
||||
let model = JournalScript::default();
|
||||
jobs.start_with_model_client(&id, model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(result.state, JobState::Completed);
|
||||
assert!(
|
||||
!model.source_reads.lock().unwrap().is_empty(),
|
||||
"provider saw saved source bodies"
|
||||
);
|
||||
fixture.host().complete_journal_job(&jobs, &result).unwrap();
|
||||
assert_eq!(
|
||||
jobs.get(&id).unwrap().input_disposition,
|
||||
JobInputDisposition::Reclaimed
|
||||
);
|
||||
fs::rename(&denied, &source_path).unwrap();
|
||||
jobs_processed += 1;
|
||||
assert!(jobs_processed < 20, "bounded prefix drain must progress");
|
||||
}
|
||||
assert!(
|
||||
jobs_processed > 2,
|
||||
"entry count and long text both require continuation"
|
||||
);
|
||||
assert_eq!(seen.len(), expected.entries.len());
|
||||
for entry in &expected.entries {
|
||||
assert_eq!(
|
||||
seen[&entry.order],
|
||||
expected.recording_text(entry.order).unwrap().2
|
||||
);
|
||||
}
|
||||
fixture.host().jobs().unwrap().shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn descriptor_job_migrates_to_new_identity_preserving_input_and_successful_records() {
|
||||
let fixture = Fixture::new("original descriptor-only input");
|
||||
let (adapter, _, modern) = fixture.adapter();
|
||||
let jobs = fixture.host().jobs().unwrap();
|
||||
fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.close_journal_generation(
|
||||
&adapter.context,
|
||||
&adapter.generation.id,
|
||||
&adapter.generation.job,
|
||||
)
|
||||
.unwrap();
|
||||
jobs.cancel(&modern.request.job_id).unwrap();
|
||||
let input = JournalInput::decode(&modern.request.input).unwrap();
|
||||
let origins: Vec<_> = input
|
||||
.capture
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
let (descriptor, _, _) = input.capture.recording_text(entry.order).unwrap();
|
||||
SourceOrigin {
|
||||
pointer: source_descriptor(&fixture.context.session_id, descriptor).pointer,
|
||||
origin: descriptor.origin.clone(),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let mut old_grant = adapter.grant.clone();
|
||||
old_grant.origins = origins;
|
||||
old_grant.writer_binding_id = None;
|
||||
let mut request = modern.request.clone();
|
||||
request.job_id = format!("legacy-descriptor:{}", uuid::Uuid::now_v7());
|
||||
request.input =
|
||||
serde_json::json!({"selected_ranges":old_grant.ranges,"origins":old_grant.origins});
|
||||
let old = jobs
|
||||
.request_granted(request, serde_json::to_value(&old_grant).unwrap())
|
||||
.unwrap();
|
||||
assert!(
|
||||
fixture.host().journal_grant(&old).is_err(),
|
||||
"descriptor jobs must not execute as snapshots"
|
||||
);
|
||||
let generation = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.prepare_journal_generation(
|
||||
&adapter.context,
|
||||
&old_grant.journal_id,
|
||||
JobAttemptBinding {
|
||||
job_id: old.request.job_id.clone(),
|
||||
attempt_id: old.attempt.attempt_id.clone(),
|
||||
},
|
||||
old_grant.ranges.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
let context = JournalJobContext::validated(
|
||||
&fixture.host().catalog.store,
|
||||
&adapter.context,
|
||||
&generation.id,
|
||||
generation.job.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
let saved = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.execute_journal(
|
||||
&context,
|
||||
JournalOperation {
|
||||
request_id: "legacy-record-success-response-lost".into(),
|
||||
expected_state: generation.initial_journal.state_token.clone(),
|
||||
action: JournalAction::Create {
|
||||
title: "already saved".into(),
|
||||
body: "legacy successful operation".into(),
|
||||
ongoing: "unfinished".into(),
|
||||
state: JournalEntryState::Open,
|
||||
source_end: None,
|
||||
correction_refs: vec![],
|
||||
},
|
||||
covered_ranges: generation.selected_ranges.clone(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
fixture.append(
|
||||
"new-not-selected",
|
||||
"LATEST SOURCE MUST WAIT UNTIL OLD INPUT FINISHES",
|
||||
);
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let (_, next) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
assert_ne!(next, old.request.job_id);
|
||||
let replacement = jobs.get(&next).unwrap();
|
||||
let replacement_input = JournalInput::decode(&replacement.request.input).unwrap();
|
||||
assert!(
|
||||
!replacement
|
||||
.request
|
||||
.input
|
||||
.to_string()
|
||||
.contains("LATEST SOURCE MUST WAIT")
|
||||
);
|
||||
assert_eq!(replacement_input.selected_ranges, old_grant.ranges);
|
||||
let retired = jobs.get(&old.request.job_id).unwrap();
|
||||
assert_eq!(retired.input_disposition, JobInputDisposition::Invalidated);
|
||||
assert_eq!(retired.input_digest, old.input_digest);
|
||||
assert_eq!(retired.intent_fingerprint, old.intent_fingerprint);
|
||||
assert!(retired.request.input.is_null());
|
||||
assert!(
|
||||
fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.execute_journal(
|
||||
&context,
|
||||
JournalOperation {
|
||||
request_id: "legacy-delayed".into(),
|
||||
expected_state: saved.state_token,
|
||||
action: JournalAction::ConfirmUnchanged,
|
||||
covered_ranges: old_grant.ranges,
|
||||
}
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
jobs.start_with_model_client(&next, JournalScript::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&next))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
fixture
|
||||
.host()
|
||||
.complete_journal_job(&jobs, &completed)
|
||||
.unwrap();
|
||||
let entries = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.list_journal_entries(&adapter.context, &adapter.grant.journal_id, None, 32)
|
||||
.unwrap();
|
||||
assert_eq!(entries.items.len(), 1);
|
||||
assert_eq!(entries.items[0].id, saved.entry_id.unwrap());
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_erase_retires_every_historical_body_across_job_history_pages() {
|
||||
const SENTINEL: &str = "historical-input-erasure-sentinel-781";
|
||||
let mut fixture = Fixture::new("ordinary original");
|
||||
let (adapter, _, first) = fixture.adapter();
|
||||
let jobs = fixture.host().jobs().unwrap();
|
||||
for index in 0..105 {
|
||||
let mut request = first.request.clone();
|
||||
request.job_id = format!("historical-{index:03}");
|
||||
request.purpose = "historical-other-domain-job".into();
|
||||
request.serialization_key = None;
|
||||
request.input = serde_json::json!({"body":SENTINEL, "index":index});
|
||||
let saved = jobs
|
||||
.request_granted(
|
||||
request,
|
||||
serde_json::json!({"session_id":fixture.context.session_id}),
|
||||
)
|
||||
.unwrap();
|
||||
jobs.cancel(&saved.request.job_id).unwrap();
|
||||
}
|
||||
let state = fixture.host().catalog.state_root.clone();
|
||||
let subject = fixture.connection.binding.subject_id.clone();
|
||||
fixture.connection.close().unwrap();
|
||||
StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.erase_session_completely(&subject, &fixture.context.session_id, "erase-all-history")
|
||||
.unwrap();
|
||||
let jobs = crate::jobs::StandaloneJobs::open(
|
||||
&fixture.connection.jobs_path,
|
||||
fixture._temp.path().into(),
|
||||
)
|
||||
.unwrap();
|
||||
let mut after = None;
|
||||
let mut count = 0;
|
||||
loop {
|
||||
let page = jobs
|
||||
.list_session_snapshots(&fixture.context.session_id, after.as_deref(), 100)
|
||||
.unwrap();
|
||||
if page.is_empty() {
|
||||
break;
|
||||
}
|
||||
for saved in &page {
|
||||
assert_eq!(saved.input_disposition, JobInputDisposition::Erased);
|
||||
assert!(saved.request.input.is_null());
|
||||
assert!(saved.load_executable_request().is_err());
|
||||
}
|
||||
count += page.len();
|
||||
after = page.last().map(|job| job.request.job_id.clone());
|
||||
}
|
||||
assert_eq!(count, 106);
|
||||
assert!(
|
||||
adapter.execute(JournalJobRequest::ReadState).is_err(),
|
||||
"prepared local adapter is invalidated"
|
||||
);
|
||||
fixture.assert_job_storage_excludes(&jobs.get(&first.request.job_id).unwrap(), SENTINEL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn saved_snapshot_missing_tampered_foreign_or_changed_initial_state_is_not_recaptured() {
|
||||
let fixture = Fixture::new("saved immutable original");
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
let saved = jobs.get(&id).unwrap();
|
||||
for changed in [serde_json::Value::Null, serde_json::json!({"capture":{}})] {
|
||||
let mut malformed = saved.clone();
|
||||
malformed.request.input = changed;
|
||||
assert!(fixture.host().journal_grant(&malformed).is_err());
|
||||
}
|
||||
let original = JournalInput::decode(&saved.request.input).unwrap();
|
||||
for field in ["subject_id", "journal_id", "initial_state_token"] {
|
||||
let mut changed = saved.request.clone();
|
||||
changed.job_id = format!("changed-{field}");
|
||||
changed.serialization_key = None;
|
||||
changed.input[field] = serde_json::json!("foreign-or-altered");
|
||||
// Even a separately reserved immutable Job may not adopt the original capability.
|
||||
let changed = jobs
|
||||
.request_granted(changed, saved.domain_grant.clone().unwrap())
|
||||
.unwrap();
|
||||
assert!(
|
||||
grant(
|
||||
fixture.host().clone(),
|
||||
&changed,
|
||||
Arc::new(TestFence(AtomicBool::new(true)))
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
let mut generation = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.prepare_journal_generation(
|
||||
&authority(fixture.host(), &fixture.context.session_id).unwrap(),
|
||||
&original.journal_id,
|
||||
JobAttemptBinding {
|
||||
job_id: id,
|
||||
attempt_id: saved.attempt.attempt_id.clone(),
|
||||
},
|
||||
original.selected_ranges.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
generation.initial_journal.state_token = "changed-after-snapshot".into();
|
||||
assert!(original.validate_generation(&generation).is_err());
|
||||
assert_eq!(
|
||||
saved.request.input,
|
||||
jobs.get(&saved.request.job_id).unwrap().request.input
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn append_between_admission_and_body_capture_does_not_publish_unaccepted_input() {
|
||||
let fixture = Fixture::new("accepted oldest prefix");
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
fixture.append("not-yet-admitted", "newly saved but not accepted");
|
||||
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
let input = jobs.get(&id).unwrap().request.input;
|
||||
assert!(input.to_string().contains("accepted oldest prefix"));
|
||||
assert!(!input.to_string().contains("newly saved but not accepted"));
|
||||
jobs.start_with_model_client(&id, JournalScript::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
fixture
|
||||
.host()
|
||||
.complete_journal_job(&jobs, &completed)
|
||||
.unwrap();
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let (_, next) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
let remaining = jobs.get(&next).unwrap().request.input;
|
||||
assert!(
|
||||
remaining
|
||||
.to_string()
|
||||
.contains("newly saved but not accepted")
|
||||
);
|
||||
assert!(!remaining.to_string().contains("accepted oldest prefix"));
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
/// Generic Job preparation must not attest delivery. The saved body is supplied by the
|
||||
/// runner before a bound Journal tool may execute; no model-provided digest is trusted.
|
||||
#[tokio::test]
|
||||
async fn actual_bound_tool_attests_delivery_but_prepared_blocked_job_does_not() {
|
||||
let fixture = Fixture::new("body delivered through generic runner");
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
let saved = jobs.get(&id).unwrap();
|
||||
let model = JournalScript {
|
||||
gate_first: true,
|
||||
pause_after_record: true,
|
||||
..Default::default()
|
||||
};
|
||||
jobs.start_with_model_client(&id, model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
let host_context = authority(fixture.host(), &fixture.context.session_id).unwrap();
|
||||
let journal_id = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.journal_for_session(&host_context)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let generation = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.active_journal_generation(&host_context, &journal_id)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let context = JournalJobContext::validated(
|
||||
&fixture.host().catalog.store,
|
||||
&host_context,
|
||||
&generation.id,
|
||||
generation.job.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
let undelivered = subjektiv::journal::service::execute_snapshot(
|
||||
&fixture.host().catalog.store,
|
||||
&context,
|
||||
&saved.input_digest,
|
||||
JournalJobRequest::ReadState,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
undelivered.to_string().contains("delivery"),
|
||||
"prepared grant must not satisfy delivery gate: {undelivered}"
|
||||
);
|
||||
let before = fixture
|
||||
.host()
|
||||
.catalog
|
||||
.store
|
||||
.read_journal(&host_context, &journal_id)
|
||||
.unwrap();
|
||||
assert_eq!((before.process_order, before.process_byte), (0, 0));
|
||||
model.release.notify_one();
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(20),
|
||||
model.record_saved.notified(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
subjektiv::journal::service::execute_snapshot(
|
||||
&fixture.host().catalog.store,
|
||||
&context,
|
||||
&saved.input_digest,
|
||||
JournalJobRequest::ReadState
|
||||
)
|
||||
.is_ok(),
|
||||
"actual bound tool execution attests exact saved input"
|
||||
);
|
||||
model.release.notify_one();
|
||||
let completed = tokio::time::timeout(std::time::Duration::from_secs(20), jobs.wait(&id))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(completed.state, JobState::Completed);
|
||||
fixture
|
||||
.host()
|
||||
.complete_journal_job(&jobs, &completed)
|
||||
.unwrap();
|
||||
jobs.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct BlockedJournalRequest {
|
||||
entered: Arc<tokio::sync::Notify>,
|
||||
dropped: Arc<tokio::sync::Notify>,
|
||||
request_dropped: Arc<AtomicBool>,
|
||||
}
|
||||
struct OwnedBlockedJournalRequest {
|
||||
// A local request copy, including the saved input, must be dropped before erasure
|
||||
// can acquire the lease. This does not claim remote/provider bytes are retractable.
|
||||
_request: agen::llm_client::Request,
|
||||
model: BlockedJournalRequest,
|
||||
}
|
||||
impl Drop for OwnedBlockedJournalRequest {
|
||||
fn drop(&mut self) {
|
||||
self.model.request_dropped.store(true, Ordering::Release);
|
||||
self.model.dropped.notify_one();
|
||||
}
|
||||
}
|
||||
#[async_trait::async_trait]
|
||||
impl agen::llm_client::LlmClient for BlockedJournalRequest {
|
||||
async fn stream(
|
||||
&self,
|
||||
request: agen::llm_client::Request,
|
||||
) -> Result<
|
||||
std::pin::Pin<
|
||||
Box<
|
||||
dyn futures::Stream<
|
||||
Item = Result<
|
||||
agen::llm_client::event::Event,
|
||||
agen::llm_client::error::ClientError,
|
||||
>,
|
||||
> + Send,
|
||||
>,
|
||||
>,
|
||||
agen::llm_client::error::ClientError,
|
||||
> {
|
||||
let owned = OwnedBlockedJournalRequest {
|
||||
_request: request,
|
||||
model: self.clone(),
|
||||
};
|
||||
assert!(
|
||||
owned
|
||||
._request
|
||||
.items
|
||||
.iter()
|
||||
.filter_map(agen::llm_client::types::Item::as_text)
|
||||
.any(|text| text.contains("local blocked source body"))
|
||||
);
|
||||
self.entered.notify_one();
|
||||
std::future::pending::<()>().await;
|
||||
drop(owned);
|
||||
unreachable!()
|
||||
}
|
||||
fn clone_boxed(&self) -> Box<dyn agen::llm_client::LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// Real generic Jobs and a retained public handle, with only the provider transport blocked.
|
||||
/// Connection close must initiate cancellation and preserve the exclusive lease until local
|
||||
/// request/history ownership is gone (or return an explicit incomplete barrier).
|
||||
#[tokio::test]
|
||||
async fn close_cancels_retained_local_job_before_erase_can_acquire_exclusive_lease() {
|
||||
let mut fixture = Fixture::new("local blocked source body");
|
||||
fixture.host().admit_journal(&fixture.context).unwrap();
|
||||
let (jobs, id) = fixture.host().next_journal_job().unwrap().unwrap();
|
||||
let retained = jobs.clone();
|
||||
let model = BlockedJournalRequest::default();
|
||||
retained
|
||||
.start_with_model_client(&id, model.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(20), model.entered.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
let state = fixture.host().catalog.state_root.clone();
|
||||
let subject = fixture.connection.binding.subject_id.clone();
|
||||
let close_error = fixture.connection.close().unwrap_err();
|
||||
assert!(matches!(
|
||||
close_error,
|
||||
SubjectError::JobShutdown(crate::jobs::JobServiceError::ShutdownIncomplete)
|
||||
));
|
||||
assert!(fixture.connection.lease.is_some());
|
||||
assert!(fixture.host().jobs.lock().unwrap().is_some());
|
||||
assert!(matches!(
|
||||
retained.start(&id).await,
|
||||
Err(crate::jobs::JobServiceError::Closed)
|
||||
));
|
||||
let locally_dropped_before_erase = model.request_dropped.load(Ordering::Acquire);
|
||||
let early_erasure = StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.erase_session_completely(
|
||||
&subject,
|
||||
&fixture.context.session_id,
|
||||
"erase-blocked-local-run",
|
||||
);
|
||||
let early_lease_was_retained = matches!(early_erasure, Err(SubjectError::Active(_)));
|
||||
let teardown_from_close = model.request_dropped.load(Ordering::Acquire)
|
||||
|| tokio::time::timeout(std::time::Duration::from_secs(5), model.dropped.notified())
|
||||
.await
|
||||
.is_ok();
|
||||
// Always tear down before asserting the observed violation, so a failed regression
|
||||
// cannot leak the blocked model future into another test.
|
||||
let _ = retained.shutdown().await;
|
||||
assert!(model.request_dropped.load(Ordering::Acquire));
|
||||
fixture.connection.close().unwrap();
|
||||
assert!(fixture.connection.lease.is_none());
|
||||
assert!(fixture.host().jobs.lock().unwrap().is_none());
|
||||
// Cancellation is not completion/erasure. Reopening retains the same unknown
|
||||
// attempt's immutable body until the explicit erasure transaction retires it.
|
||||
let reopened = crate::jobs::StandaloneJobs::open(
|
||||
&fixture.connection.jobs_path,
|
||||
fixture._temp.path().into(),
|
||||
)
|
||||
.unwrap();
|
||||
let unknown = reopened.get(&id).unwrap();
|
||||
assert_eq!(unknown.attempt.state, crate::jobs::JobAttemptState::Unknown);
|
||||
assert_eq!(unknown.input_disposition, JobInputDisposition::Available);
|
||||
assert!(
|
||||
unknown
|
||||
.request
|
||||
.input
|
||||
.to_string()
|
||||
.contains("local blocked source body")
|
||||
);
|
||||
reopened.shutdown().await.unwrap();
|
||||
StandaloneSubjects::open_existing(&state)
|
||||
.unwrap()
|
||||
.erase_session_completely(
|
||||
&subject,
|
||||
&fixture.context.session_id,
|
||||
"erase-blocked-local-run",
|
||||
)
|
||||
.unwrap();
|
||||
let erased_store = crate::jobs::StandaloneJobs::open(
|
||||
&fixture.connection.jobs_path,
|
||||
fixture._temp.path().into(),
|
||||
)
|
||||
.unwrap();
|
||||
let erased = erased_store.get(&id).unwrap();
|
||||
assert_eq!(erased.input_disposition, JobInputDisposition::Erased);
|
||||
fixture.assert_job_storage_excludes(&erased, "local blocked source body");
|
||||
erased_store.shutdown().await.unwrap();
|
||||
assert!(
|
||||
locally_dropped_before_erase || early_lease_was_retained,
|
||||
"close released exclusive lease while a retained generic Job still owned local input/history"
|
||||
);
|
||||
assert!(
|
||||
teardown_from_close,
|
||||
"connection close did not initiate cancellation of the externally retained local Job"
|
||||
);
|
||||
assert!(
|
||||
retained.get(&id).is_err(),
|
||||
"retained handles are closed after teardown"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -615,10 +615,14 @@ async fn headless_unix_client_uses_production_history_lease_and_local_authority(
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let temp = tempfile::tempdir()?;
|
||||
// Only the transport endpoint needs a short private pathname. Persistence remains the
|
||||
// same production store exercised by the in-process Host tests above.
|
||||
std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o700))?;
|
||||
let endpoint_path = temp.path().join("operator.sock");
|
||||
// Shorten only the owned endpoint's components, preserving the harness TMPDIR
|
||||
// even when it is checkout-local. No process-global cwd or external /tmp fallback.
|
||||
let socket_dir = tempfile::Builder::new()
|
||||
.prefix("")
|
||||
.rand_bytes(4)
|
||||
.tempdir()?;
|
||||
std::fs::set_permissions(socket_dir.path(), std::fs::Permissions::from_mode(0o700))?;
|
||||
let endpoint_path = socket_dir.path().join("s");
|
||||
let endpoint = standalone::UnixEndpoint::bind(&endpoint_path)?;
|
||||
let state_dir = temp.path().join("state");
|
||||
let launch = StandaloneLaunchConfig::new(
|
||||
|
||||
@@ -479,38 +479,28 @@ fn journal_response(request: &Request) -> Vec<LlmEvent> {
|
||||
}),
|
||||
);
|
||||
}
|
||||
for (index, range) in state["selected_ranges"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
let input: Value = request
|
||||
.items
|
||||
.iter()
|
||||
.enumerate()
|
||||
{
|
||||
let mut offset = range["bytes"]["start"].as_u64().unwrap();
|
||||
let mut token = Value::Null;
|
||||
loop {
|
||||
let id = format!("journal-source-{index}-{offset}");
|
||||
match result(&id) {
|
||||
Some(page) => match page["value"]["next_byte"].as_u64() {
|
||||
Some(next) => {
|
||||
assert!(next > offset);
|
||||
offset = next;
|
||||
token = page["value"]["content_token"].clone();
|
||||
}
|
||||
None => break,
|
||||
},
|
||||
None => {
|
||||
return tool(
|
||||
&id,
|
||||
"ReadJournalText",
|
||||
json!({
|
||||
"target":{"target":"source","pointer":range["source"]["pointer"]},
|
||||
"offset":offset,"limit":subjektiv::journal::MAX_JOURNAL_READ_BYTES,"content_token":token
|
||||
}),
|
||||
.filter_map(Item::as_text)
|
||||
.find_map(|text| {
|
||||
text.lines()
|
||||
.find_map(|line| line.strip_prefix("input_json: "))
|
||||
})
|
||||
.map(|json| serde_json::from_str(json).unwrap())
|
||||
.expect("saved direct Journal input");
|
||||
let input = subjektiv::journal::recording::JournalInput::decode(&input).unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(&input.selected_ranges).unwrap(),
|
||||
state["selected_ranges"]
|
||||
);
|
||||
assert!(
|
||||
input
|
||||
.capture
|
||||
.entries
|
||||
.iter()
|
||||
.all(|entry| input.capture.recording_text(entry.order).is_ok())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let (name, action) = if state["current_open"].is_null() {
|
||||
(
|
||||
"CreateJournalEntry",
|
||||
|
||||
@@ -15,6 +15,8 @@ serde = { workspace = true, features = ["derive"] }
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
server-api.workspace = true
|
||||
session-store.workspace = true
|
||||
job.workspace = true
|
||||
thiserror.workspace = true
|
||||
uuid = { workspace = true, features = ["v7"] }
|
||||
|
||||
|
||||
@@ -194,6 +194,29 @@ impl SubjektivStore {
|
||||
})
|
||||
}
|
||||
|
||||
/// End a body after the Host has confirmed its Runtime removal. This is not a
|
||||
/// completed Journal handoff: source coverage, positions and saved receipts stay
|
||||
/// unchanged. Revocation and closing generations are atomic, so no retained Job
|
||||
/// can write after deletion even while Backend Job cleanup is being retried.
|
||||
pub fn release_removed_journal_body(&self, host: &JournalHostContext) -> JournalResult<()> {
|
||||
self.database.try_transaction(|tx| {
|
||||
super::store::check_session_host(self, tx, host)?;
|
||||
let attr = host.attribution.as_ref().expect("checked Session authority");
|
||||
let binding: Option<String> = tx.query_row(
|
||||
"SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 ORDER BY rowid DESC LIMIT 1",
|
||||
params![host.subject_id, attr.runtime_id, attr.worker_id], |r| r.get(0),
|
||||
).optional()?;
|
||||
let binding = binding.unwrap_or_else(|| crate::issued_id("journal-removed-body"));
|
||||
tx.execute("INSERT OR IGNORE INTO journal_writer_bindings(binding_id,subject_id,runtime_id,worker_id,active) VALUES(?1,?2,?3,?4,0)",params![binding,host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||
tx.execute("UPDATE journal_writer_bindings SET active=0 WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||
// Bind legacy attributed Sessions to an inactive lifetime, never to the
|
||||
// successor. Retained unfinished history is not an active writer.
|
||||
tx.execute("INSERT OR IGNORE INTO journal_session_bindings(session_id,binding_id) SELECT session_id,?4 FROM subject_session_attributions WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id,binding])?;
|
||||
tx.execute("UPDATE journal_generations SET active=0 WHERE journal_id IN (SELECT j.id FROM journals j JOIN subject_session_attributions a ON a.session_id=j.source_session_id WHERE a.subject_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3)",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original
|
||||
/// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs.
|
||||
/// This transaction rejects retained writers, pending source bytes and active generations.
|
||||
|
||||
@@ -5,8 +5,221 @@
|
||||
//! original canonical identity while streaming and return only the requested bounded fragment.
|
||||
|
||||
use super::*;
|
||||
use session_store::capture::{CaptureDto, CaptureLimits};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// A Job owns only this bounded original-body prefix. The initial content condition binds
|
||||
/// the separate generation snapshot (whose potentially large Journal text remains paged).
|
||||
/// Neither possessing these bytes nor delivering them is a processing checkpoint.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JournalInput {
|
||||
pub version: u32,
|
||||
pub subject_id: String,
|
||||
pub journal_id: String,
|
||||
pub initial_state_token: String,
|
||||
pub initial_open_id: Option<String>,
|
||||
pub previous_fixed_end: Option<JournalSourcePointer>,
|
||||
pub selected_ranges: Vec<JournalSourceRange>,
|
||||
pub capture: CaptureDto,
|
||||
}
|
||||
|
||||
impl JournalInput {
|
||||
/// Capture callback runs once for this immutable Job, never once per page. Fitting uses
|
||||
/// only that captured value and measures the complete serialized input, including the
|
||||
/// initial Journal condition and ranges duplicated beside the capture's descriptors.
|
||||
pub fn build(
|
||||
store: &SubjektivStore,
|
||||
host: &JournalHostContext,
|
||||
journal_id: &str,
|
||||
capture_once: impl FnOnce(u64, u64, usize) -> JournalResult<CaptureDto>,
|
||||
) -> JournalResult<Self> {
|
||||
let (journal, open, previous_fixed_end) = store.database.try_with_connection(|tx| {
|
||||
super::store::check_host(store, tx, host, true)?;
|
||||
Ok::<_, JournalError>((
|
||||
super::store::journal(tx, host, journal_id, true)?,
|
||||
super::store::open(tx, journal_id)?,
|
||||
super::store::previous_fixed(tx, journal_id)?.map(|p| p.0),
|
||||
))
|
||||
})?;
|
||||
let mut input = Self {
|
||||
version: 1,
|
||||
subject_id: journal.subject_id.clone(),
|
||||
journal_id: journal_id.into(),
|
||||
initial_state_token: journal.state_token,
|
||||
initial_open_id: open.map(|e| e.id),
|
||||
previous_fixed_end,
|
||||
selected_ranges: Vec::new(),
|
||||
capture: CaptureDto {
|
||||
version: 1,
|
||||
session_id: journal.source_session_id,
|
||||
segment_id: "pending".into(),
|
||||
view_entry_count: 0,
|
||||
entries: Vec::new(),
|
||||
},
|
||||
};
|
||||
let envelope = serde_json::to_vec(&input)?.len();
|
||||
// Capture once at a fixed storage boundary. Any adjustment for the Journal envelope
|
||||
// below operates on this value only, never by making another live capture request.
|
||||
let budget = job::MAX_JOB_INPUT_BYTES.saturating_sub(envelope);
|
||||
input.capture = capture_once(journal.process_order, journal.process_byte, budget)?;
|
||||
input.selected_ranges = capture_ranges(&input.capture)?;
|
||||
if input.selected_ranges.is_empty() {
|
||||
return Ok(input);
|
||||
}
|
||||
input.fit()?;
|
||||
let value = serde_json::to_value(&input)?;
|
||||
Self::decode(&value)
|
||||
}
|
||||
|
||||
fn fit(&mut self) -> JournalResult<()> {
|
||||
let original = self.capture.clone();
|
||||
let first = self
|
||||
.selected_ranges
|
||||
.first()
|
||||
.ok_or_else(|| invalid("empty recording selection"))?;
|
||||
let start = session_store::capture::RecordingPosition {
|
||||
canonical_order: first.source.order,
|
||||
byte_offset: first.bytes.start,
|
||||
};
|
||||
let mut budget = serde_json::to_vec(&original)?.len();
|
||||
loop {
|
||||
let bytes = serde_json::to_vec(self)?.len();
|
||||
if bytes <= job::MAX_JOB_INPUT_BYTES {
|
||||
return Ok(());
|
||||
}
|
||||
budget = budget
|
||||
.checked_sub(bytes - job::MAX_JOB_INPUT_BYTES)
|
||||
.ok_or_else(|| invalid("Journal envelope cannot fit source bytes"))?;
|
||||
self.capture = original
|
||||
.recording_slice(start, MAX_JOURNAL_RANGES, budget)
|
||||
.map_err(|e| invalid(&e.to_string()))?
|
||||
.capture;
|
||||
self.selected_ranges = capture_ranges(&self.capture)?;
|
||||
if self.selected_ranges.is_empty() {
|
||||
return Err(invalid("Journal envelope cannot fit one source scalar"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode(value: &serde_json::Value) -> JournalResult<Self> {
|
||||
if serde_json::to_vec(value)?.len() > job::MAX_JOB_INPUT_BYTES {
|
||||
return Err(invalid(
|
||||
"Journal input exceeds whole serialized Job input limit",
|
||||
));
|
||||
}
|
||||
let input: Self = serde_json::from_value(value.clone())?;
|
||||
input
|
||||
.capture
|
||||
.validate(CaptureLimits {
|
||||
max_bytes: job::MAX_JOB_INPUT_BYTES,
|
||||
max_entries: MAX_JOURNAL_RANGES,
|
||||
max_text_bytes: job::MAX_JOB_INPUT_BYTES,
|
||||
..CaptureLimits::default()
|
||||
})
|
||||
.map_err(|e| invalid(&e.to_string()))?;
|
||||
if input.version != 1
|
||||
|| input.subject_id.is_empty()
|
||||
|| input.journal_id.is_empty()
|
||||
|| input.initial_state_token.is_empty()
|
||||
|| input.selected_ranges.is_empty()
|
||||
|| input.selected_ranges.len() > MAX_JOURNAL_RANGES
|
||||
|| input
|
||||
.selected_ranges
|
||||
.iter()
|
||||
.any(|r| r.source.session_id != input.capture.session_id)
|
||||
{
|
||||
return Err(invalid("invalid Journal input identity or selection"));
|
||||
}
|
||||
if capture_ranges(&input.capture)? != input.selected_ranges {
|
||||
return Err(conflict(
|
||||
"Journal body snapshot does not match selected canonical ranges",
|
||||
));
|
||||
}
|
||||
Ok(input)
|
||||
}
|
||||
|
||||
pub fn validate_generation(&self, generation: &JournalGeneration) -> JournalResult<()> {
|
||||
if generation.journal_id != self.journal_id
|
||||
|| generation.initial_journal.subject_id != self.subject_id
|
||||
|| generation.initial_journal.source_session_id != self.capture.session_id
|
||||
|| generation.initial_journal.state_token != self.initial_state_token
|
||||
|| generation.initial_open.as_ref().map(|e| &e.id) != self.initial_open_id.as_ref()
|
||||
|| generation.previous_fixed_end != self.previous_fixed_end
|
||||
|| generation.selected_ranges != self.selected_ranges
|
||||
{
|
||||
return Err(conflict(
|
||||
"saved Journal input differs from its initial generation snapshot",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts the shared recording projection into Journal's coverage language without
|
||||
/// interpreting model text or constructing a second capture representation.
|
||||
pub fn capture_ranges(capture: &CaptureDto) -> JournalResult<Vec<JournalSourceRange>> {
|
||||
use session_store::capture::CaptureBody;
|
||||
let mut ranges = Vec::new();
|
||||
for entry in &capture.entries {
|
||||
let CaptureBody::CanonicalPublic {
|
||||
descriptor,
|
||||
byte_offset,
|
||||
text,
|
||||
..
|
||||
} = &entry.body
|
||||
else {
|
||||
return Err(invalid(
|
||||
"Journal requires canonical original recording capture",
|
||||
));
|
||||
};
|
||||
if !entry.policy.recording_material
|
||||
|| text.is_empty()
|
||||
|| descriptor.identity.segment_id.to_string() != entry.source.segment_id
|
||||
|| descriptor.identity.entry_ref != format!("E{}", entry.entry_id)
|
||||
|| descriptor.canonical_order != entry.order
|
||||
|| byte_offset
|
||||
.checked_add(text.len() as u64)
|
||||
.is_none_or(|end| end > descriptor.total_bytes)
|
||||
{
|
||||
return Err(conflict(
|
||||
"recording body identity, policy or range mismatch",
|
||||
));
|
||||
}
|
||||
let source = JournalSource {
|
||||
session_id: capture.session_id.clone(),
|
||||
pointer: JournalSourcePointer {
|
||||
segment_id: descriptor.identity.segment_id,
|
||||
entry_ref: descriptor.identity.entry_ref.clone(),
|
||||
},
|
||||
order: descriptor.canonical_order,
|
||||
content_sha256: descriptor.sha256.clone(),
|
||||
total_bytes: descriptor.total_bytes,
|
||||
};
|
||||
if *byte_offset == 0 && text.len() as u64 == descriptor.total_bytes {
|
||||
verify_canonical_text(&source, text)?;
|
||||
}
|
||||
ranges.push(JournalSourceRange {
|
||||
source,
|
||||
bytes: JournalByteRange {
|
||||
start: *byte_offset,
|
||||
end: *byte_offset + text.len() as u64,
|
||||
},
|
||||
});
|
||||
}
|
||||
for pair in ranges.windows(2) {
|
||||
if pair[0].source.order.checked_add(1) != Some(pair[1].source.order)
|
||||
|| pair[0].bytes.end != pair[0].source.total_bytes
|
||||
|| pair[1].bytes.start != 0
|
||||
{
|
||||
return Err(conflict(
|
||||
"recording capture is not an oldest contiguous prefix",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(ranges)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JournalMaterial {
|
||||
|
||||
@@ -66,8 +66,8 @@ fn text_page(
|
||||
})
|
||||
}
|
||||
|
||||
/// Adapter for Host captures bounded by their storage contract. Production source reads should
|
||||
/// use [`execute_bounded`] to avoid allocating an entire source for one small read.
|
||||
/// Legacy adapter for Host captures bounded by their storage contract.
|
||||
/// Production Hosts use [`attest_input_delivery`] and [`execute_snapshot`] instead.
|
||||
pub fn execute(
|
||||
store: &SubjektivStore,
|
||||
context: &JournalJobContext,
|
||||
@@ -101,23 +101,149 @@ pub fn execute(
|
||||
})
|
||||
}
|
||||
|
||||
/// Reads are delivered through this service before mutation authority is enabled. The durable
|
||||
/// delivery ledger stores byte ranges only: Jobs may retain immutable descriptors without
|
||||
/// accumulating a permanent second copy of the original Session in their request ledgers.
|
||||
/// The Host verifies original provenance/digest/length and returns a bounded UTF-8 fragment.
|
||||
pub(crate) fn migrate_input_delivery(
|
||||
tx: &rusqlite::Transaction<'_>,
|
||||
) -> feature_storage::Result<()> {
|
||||
tx.execute_batch(
|
||||
"CREATE TABLE journal_generation_input_deliveries (
|
||||
generation_id TEXT PRIMARY KEY NOT NULL REFERENCES journal_generations(id) ON DELETE CASCADE,
|
||||
input_digest TEXT NOT NULL CHECK(length(input_digest)=71 AND substr(input_digest,1,7)='sha256:' AND substr(input_digest,8) NOT GLOB '*[^0-9a-f]*')
|
||||
);
|
||||
CREATE TRIGGER journal_generation_input_delivery_no_update
|
||||
BEFORE UPDATE ON journal_generation_input_deliveries
|
||||
BEGIN SELECT RAISE(ABORT, 'Journal input delivery bindings are immutable'); END;",
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_input_digest(input_digest: &str) -> JournalResult<()> {
|
||||
// Preserve JobRequest::input_digest() verbatim, including its algorithm prefix.
|
||||
if input_digest.strip_prefix("sha256:").is_none_or(|hex| {
|
||||
hex.len() != 64
|
||||
|| !hex
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||
}) {
|
||||
return Err(JournalError::InvalidInput(
|
||||
"immutable Job input digest must be sha256: followed by 64 lowercase hex digits".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Trusted Host-only delivery evidence, never a model tool or a claim of understanding.
|
||||
/// Before calling, the Host must validate the saved, immutable body-bearing input against the
|
||||
/// bound Job's input digest, its Session/provenance/ranges, and the actual dispatch plan that
|
||||
/// supplies that exact input. The service cannot verify bytes it does not own. This transaction
|
||||
/// stores only the digest bound to the generation's immutable selection; it neither copies the
|
||||
/// body nor changes records, position, reflection or checkpoint. Exact replay is allowed only
|
||||
/// while the same generation and writer binding remain active; changed digests/ranges conflict.
|
||||
pub fn attest_input_delivery(
|
||||
store: &SubjektivStore,
|
||||
context: &JournalJobContext,
|
||||
input_digest: &str,
|
||||
selected_ranges: &[JournalSourceRange],
|
||||
) -> JournalResult<()> {
|
||||
validate_input_digest(input_digest)?;
|
||||
store.database.try_transaction(|tx| {
|
||||
super::lifecycle::reject_erased(tx, &context.host)?;
|
||||
let generation = super::store::job(store, tx, context, true)?;
|
||||
if generation.selected_ranges != selected_ranges {
|
||||
return Err(JournalError::Conflict(
|
||||
"immutable Job input selection differs from generation".into(),
|
||||
));
|
||||
}
|
||||
let saved: Option<String> = tx.query_row(
|
||||
"SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1",
|
||||
[&generation.id], |row| row.get(0),
|
||||
).optional()?;
|
||||
if let Some(saved) = saved {
|
||||
if saved != input_digest {
|
||||
return Err(JournalError::Conflict(
|
||||
"generation is bound to a different immutable Job input digest".into(),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
tx.execute(
|
||||
"INSERT INTO journal_generation_input_deliveries(generation_id,input_digest) VALUES(?1,?2)",
|
||||
params![generation.id, input_digest],
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn require_input_delivery(
|
||||
tx: &rusqlite::Connection,
|
||||
context: &JournalJobContext,
|
||||
input_digest: &str,
|
||||
) -> JournalResult<()> {
|
||||
validate_input_digest(input_digest)?;
|
||||
let saved: Option<String> = tx
|
||||
.query_row(
|
||||
"SELECT input_digest FROM journal_generation_input_deliveries WHERE generation_id=?1",
|
||||
[&context.generation_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()?;
|
||||
if saved.as_deref() != Some(input_digest) {
|
||||
return Err(JournalError::Conflict(
|
||||
"trusted Host delivery of this immutable Job input is required".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Production sequential execution over Journal state and the Host-attested immutable input.
|
||||
/// The digest is taken from the bound saved Job by the Host, not from model arguments. Every
|
||||
/// request requires the matching delivery binding and an active generation/writer. Source text
|
||||
/// is already in immutable input: Source targets fail explicitly, with no Session reader or
|
||||
/// fallback. Receipt-backed record, position and checkpoint constraints remain unchanged.
|
||||
pub fn execute_snapshot(
|
||||
store: &SubjektivStore,
|
||||
context: &JournalJobContext,
|
||||
input_digest: &str,
|
||||
request: JournalJobRequest,
|
||||
) -> JournalResult<JournalJobResponse> {
|
||||
execute_inner(store, context, request, Some(input_digest), |_, _, _| {
|
||||
Err(JournalError::PermissionDenied(
|
||||
"Source reads are unavailable for immutable Journal input".into(),
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
/// Legacy/test source-read adapter. Production Hosts must use [`execute_snapshot`].
|
||||
/// This ledger attests bounded Source responses only and cannot satisfy the input-digest gate.
|
||||
pub fn execute_bounded(
|
||||
store: &SubjektivStore,
|
||||
context: &JournalJobContext,
|
||||
request: JournalJobRequest,
|
||||
read_source: impl FnMut(&JournalSource, u64, usize) -> JournalResult<recording::JournalMaterial>,
|
||||
) -> JournalResult<JournalJobResponse> {
|
||||
execute_inner(store, context, request, None, read_source)
|
||||
}
|
||||
|
||||
fn execute_inner(
|
||||
store: &SubjektivStore,
|
||||
context: &JournalJobContext,
|
||||
request: JournalJobRequest,
|
||||
input_digest: Option<&str>,
|
||||
mut read_source: impl FnMut(&JournalSource, u64, usize) -> JournalResult<recording::JournalMaterial>,
|
||||
) -> JournalResult<JournalJobResponse> {
|
||||
let generation = store
|
||||
.database
|
||||
.try_with_connection(|tx| super::store::job(store, tx, context, true))?;
|
||||
let generation = store.database.try_with_connection(|tx| {
|
||||
super::lifecycle::reject_erased(tx, &context.host)?;
|
||||
let generation = super::store::job(store, tx, context, true)?;
|
||||
if let Some(input_digest) = input_digest {
|
||||
require_input_delivery(tx, context, input_digest)?;
|
||||
}
|
||||
Ok::<_, JournalError>(generation)
|
||||
})?;
|
||||
let mut delivered = None;
|
||||
let response = match request {
|
||||
JournalJobRequest::Execute { operation } => {
|
||||
if input_digest.is_none() {
|
||||
require_delivery(store, context, &generation.selected_ranges)?;
|
||||
}
|
||||
let receipt = if matches!(operation.action, JournalAction::Checkpoint { .. }) {
|
||||
crate::api::checkpoint_journal(store, context, operation)?
|
||||
} else {
|
||||
@@ -197,6 +323,11 @@ pub fn execute_bounded(
|
||||
JournalJobRequest::ReadText { read } => {
|
||||
let text = match &read.target {
|
||||
JournalTextTarget::Source { pointer } => {
|
||||
if input_digest.is_some() {
|
||||
return Err(JournalError::PermissionDenied(
|
||||
"Source target is unavailable: raw source bodies are in immutable Job input".into(),
|
||||
));
|
||||
}
|
||||
let range = generation
|
||||
.selected_ranges
|
||||
.iter()
|
||||
|
||||
@@ -202,7 +202,7 @@ pub(super) fn open(tx: &Connection, journal_id: &str) -> JournalResult<Option<Jo
|
||||
.optional()?;
|
||||
id.map(|id| entry(tx, journal_id, &id)).transpose()
|
||||
}
|
||||
fn previous_fixed(
|
||||
pub(super) fn previous_fixed(
|
||||
tx: &Connection,
|
||||
journal_id: &str,
|
||||
) -> JournalResult<Option<(JournalSourcePointer, u64)>> {
|
||||
@@ -569,6 +569,60 @@ impl SubjektivStore {
|
||||
Ok(snapshot)
|
||||
})
|
||||
}
|
||||
/// Rebind a saved body input to a new attempt without resampling its initial state.
|
||||
/// The first binding freezes the full initial Journal snapshot; retries reuse it and
|
||||
/// retain operation receipts. Different Jobs cannot borrow that snapshot as authority.
|
||||
pub fn prepare_journal_input(
|
||||
&self,
|
||||
host: &JournalHostContext,
|
||||
input: &super::recording::JournalInput,
|
||||
binding: JobAttemptBinding,
|
||||
) -> JournalResult<JournalGeneration> {
|
||||
label(&binding.job_id)?;
|
||||
label(&binding.attempt_id)?;
|
||||
self.database.try_transaction(|tx| {
|
||||
check_host(self, tx, host, false)?;
|
||||
check_session_host(self, tx, host)?;
|
||||
let current = journal(tx, host, &input.journal_id, true)?;
|
||||
let existing: Option<String> = tx.query_row(
|
||||
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 AND attempt_id=?3",
|
||||
params![input.journal_id, binding.job_id, binding.attempt_id], |r| r.get(0)).optional()?;
|
||||
if let Some(raw) = existing {
|
||||
let g = serde_json::from_str(&raw)?;
|
||||
input.validate_generation(&g)?;
|
||||
return Ok(g);
|
||||
}
|
||||
// Retained receipt lookup does not reacquire writer authority. Only a new
|
||||
// generation requires the current active Subject/Session writer lifetime.
|
||||
check_host(self, tx, host, true)?;
|
||||
if tx.query_row("SELECT EXISTS(SELECT 1 FROM journal_generations WHERE journal_id=?1 AND active=1)", [&input.journal_id], |r| r.get::<_, bool>(0))? {
|
||||
return Err(conflict("another active generation owns this Journal"));
|
||||
}
|
||||
let prior: Option<String> = tx.query_row(
|
||||
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 ORDER BY rowid LIMIT 1",
|
||||
params![input.journal_id, binding.job_id], |r| r.get(0)).optional()?;
|
||||
let mut g = if let Some(raw) = prior {
|
||||
serde_json::from_str::<JournalGeneration>(&raw)?
|
||||
} else {
|
||||
JournalGeneration { id: String::new(), journal_id: input.journal_id.clone(), job: binding.clone(),
|
||||
selected_ranges: input.selected_ranges.clone(), initial_journal: current.clone(),
|
||||
initial_open: open(tx, &input.journal_id)?, previous_fixed_end: previous_fixed(tx, &input.journal_id)?.map(|p| p.0) }
|
||||
};
|
||||
input.validate_generation(&g)?;
|
||||
check_ranges(tx, ¤t, &input.selected_ranges, true)?;
|
||||
let end = prefix(&g.initial_journal, &input.selected_ranges)?;
|
||||
let cursor = (current.process_order, current.process_byte);
|
||||
if cursor < (g.initial_journal.process_order, g.initial_journal.process_byte) || cursor > end {
|
||||
return Err(conflict("saved input no longer owns the current processing cursor"));
|
||||
}
|
||||
g.id = issued_id("journal-generation");
|
||||
g.job = binding;
|
||||
tx.execute("INSERT INTO journal_generations(id,journal_id,job_id,attempt_id,request_json,snapshot_json,active) VALUES(?1,?2,?3,?4,?5,?6,1)",
|
||||
params![g.id,g.journal_id,g.job.job_id,g.job.attempt_id,serde_json::to_string(&g.selected_ranges)?,serde_json::to_string(&g)?])?;
|
||||
Ok(g)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn validate_journal_job(&self, ctx: &JournalJobContext) -> JournalResult<()> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
job(self, tx, ctx, true)?;
|
||||
@@ -597,6 +651,32 @@ impl SubjektivStore {
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
/// Read the exact retained Job/attempt snapshot without granting writer
|
||||
/// authority or creating a generation. Used for outcome recovery after removal.
|
||||
pub fn retained_journal_generation(
|
||||
&self,
|
||||
host: &JournalHostContext,
|
||||
id: &str,
|
||||
binding: &JobAttemptBinding,
|
||||
) -> JournalResult<Option<JournalGeneration>> {
|
||||
self.database.try_with_connection(|tx| {
|
||||
check_host(self, tx, host, false)?;
|
||||
check_session_host(self, tx, host)?;
|
||||
journal(tx, host, id, true)?;
|
||||
let raw: Option<String> = tx.query_row(
|
||||
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 AND attempt_id=?3",
|
||||
params![id, binding.job_id, binding.attempt_id], |r| r.get(0),
|
||||
).optional()?;
|
||||
raw.map(|raw| {
|
||||
let generation: JournalGeneration = serde_json::from_str(&raw)?;
|
||||
if generation.journal_id != id || &generation.job != binding {
|
||||
return Err(conflict("retained generation identity mismatch"));
|
||||
}
|
||||
Ok(generation)
|
||||
}).transpose()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn read_journal_generation(
|
||||
&self,
|
||||
host: &JournalHostContext,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,10 @@
|
||||
//! Bounded, Host-bound Journal recording and sequential Job transport.
|
||||
//!
|
||||
//! Subject, Journal and generation authority never come from these model inputs.
|
||||
//! Hosts validate live execution and public source bytes before selection; reading a
|
||||
//! preview must not select the enclosing entry. Operation retries retain the entire
|
||||
//! Hosts validate live execution and public source bytes before selection, save body-bearing
|
||||
//! immutable Job input, and attest its exact digest and dispatch plan through the service.
|
||||
//! Model requests cannot attest input delivery; Source targets are rejected by snapshot execution.
|
||||
//! Initial/current Journal reads remain separate from input delivery. Operation retries retain the entire
|
||||
//! JournalOperation, including its original expected_state and covered_ranges.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -50,8 +52,8 @@ pub struct JournalJobState {
|
||||
pub current_journal: JournalMetadata,
|
||||
pub current_open: Option<JournalEntrySummary>,
|
||||
pub previous_fixed_end: Option<JournalSourcePointer>,
|
||||
/// Frozen permission to fetch this prefix, not evidence of delivery or reflection.
|
||||
/// The sequential service gates mutations on its durable Source-read delivery ledger.
|
||||
/// Frozen source selection delivered in immutable Job input, not evidence of reflection.
|
||||
/// Production execution requires the trusted Host's matching input-digest attestation.
|
||||
pub selected_ranges: Vec<JournalSourceRange>,
|
||||
}
|
||||
|
||||
@@ -83,7 +85,8 @@ pub enum JournalTextTarget {
|
||||
OpenOngoing {
|
||||
snapshot: JournalSnapshot,
|
||||
},
|
||||
/// The Host restricts this pointer to this Job's frozen public source input.
|
||||
/// Legacy source-read adapter only. Production snapshot execution rejects this target;
|
||||
/// raw source bodies must already be present in the immutable Job input.
|
||||
Source {
|
||||
pointer: JournalSourcePointer,
|
||||
},
|
||||
|
||||
@@ -639,6 +639,11 @@ static MIGRATIONS: &[FeatureMigration] = &[
|
||||
"add explicit Subject permission sources and immutable child execution grants",
|
||||
permissions::migrate,
|
||||
),
|
||||
FeatureMigration::new(
|
||||
13,
|
||||
"bind Journal generation delivery to immutable Job input digest",
|
||||
journal::service::migrate_input_delivery,
|
||||
),
|
||||
];
|
||||
|
||||
pub const REGISTRATION: FeatureRegistration =
|
||||
|
||||
+315
-34
@@ -132,9 +132,7 @@ System events and case judgments cannot be edited. The Workspace backend supplie
|
||||
const LIST_DESCRIPTION: &str = "Query Tickets from the configured typed Ticket backend as a bounded \
|
||||
overview. Workspace-backed Workers replace this definition with the richer authoritative \
|
||||
text/event/evidence/relation/Objective/time/attention query.";
|
||||
const SHOW_DESCRIPTION: &str = "Show one Ticket by id or exact query through the configured typed \
|
||||
Ticket backend. Output includes the initial comment and recent events with stable event_ref, author/time and edit metadata, bounded Markdown body, resolution, and artifact \
|
||||
metadata; Workspace-backed Workers replace this definition with the richer authoritative evidence projection.";
|
||||
const SHOW_DESCRIPTION: &str = "Read one Ticket by id or exact query. Use view=summary for status/CAS without prose or history; detail (default) for canonical initial_post and bounded recent discussion; history with event_cursor for older thread pages; post with an observed event_ref for the exact full post. Never use truncated bodies for edit CAS or guess event identities. Workspace-backed Workers retain the same views with authoritative delegation and MR evidence.";
|
||||
const COMMENT_DESCRIPTION: &str = "Append a typed Ticket comment event. `body` is Markdown.";
|
||||
const PLAN_DESCRIPTION: &str = "Append a typed Ticket plan event. `body` is Markdown.";
|
||||
const DECISION_DESCRIPTION: &str = "Append a typed Ticket decision event. `body` is Markdown.";
|
||||
@@ -549,8 +547,50 @@ impl QueryTicketParams {
|
||||
}
|
||||
}
|
||||
|
||||
/// Model-facing read modes shared by generic Ticket and Workspace Tool/WIP reads.
|
||||
#[derive(
|
||||
Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, schemars::JsonSchema,
|
||||
)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ShowTicketView {
|
||||
Summary,
|
||||
#[default]
|
||||
Detail,
|
||||
History,
|
||||
Post,
|
||||
}
|
||||
|
||||
impl ShowTicketView {
|
||||
pub fn validate(
|
||||
self,
|
||||
event_cursor: Option<&str>,
|
||||
event_ref: Option<&str>,
|
||||
) -> Result<(), String> {
|
||||
if (self == Self::Post) != event_ref.is_some()
|
||||
|| event_ref.is_some_and(|value| value.trim().is_empty())
|
||||
{
|
||||
return Err(
|
||||
"event_ref is required only for view=post; use an observed identity".into(),
|
||||
);
|
||||
}
|
||||
if event_cursor.is_some() && !matches!(self, Self::Detail | Self::History) {
|
||||
return Err("event_cursor is only valid for detail/history".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||
struct ShowTicketParams {
|
||||
/// summary omits prose/history; detail is the default; history returns only a thread page; post returns an exact full post up to 1 MiB (larger posts fail, never truncate).
|
||||
#[serde(default)]
|
||||
view: ShowTicketView,
|
||||
/// Opaque continuation from event_page. Valid for detail/history only.
|
||||
#[serde(default)]
|
||||
event_cursor: Option<String>,
|
||||
/// Exact observed post identity. Required only for view=post.
|
||||
#[serde(default)]
|
||||
event_ref: Option<String>,
|
||||
/// Ticket reference. Prefer `T-*`; canonical internal ids remain accepted for compatibility. Exactly one of `id` or `query` must be provided.
|
||||
#[serde(default)]
|
||||
id: Option<String>,
|
||||
@@ -1096,6 +1136,10 @@ impl Tool for ShowTicketTool {
|
||||
_ctx: agen::tool::ToolExecutionContext,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let params: ShowTicketParams = parse_input("ShowTicket", input_json)?;
|
||||
params
|
||||
.view
|
||||
.validate(params.event_cursor.as_deref(), params.event_ref.as_deref())
|
||||
.map_err(ToolError::InvalidArgument)?;
|
||||
let query = id_or_query(params.id, params.query)?;
|
||||
let event_limit = bounded(params.event_limit, DEFAULT_EVENT_LIMIT, MAX_EVENT_LIMIT);
|
||||
let artifact_limit = bounded(
|
||||
@@ -1117,10 +1161,45 @@ impl Tool for ShowTicketTool {
|
||||
ticket.meta.id,
|
||||
ticket.meta.workflow_state.as_str()
|
||||
);
|
||||
Ok(json_output(
|
||||
summary,
|
||||
ticket_json(&ticket, event_limit, artifact_limit, body_max_bytes),
|
||||
))
|
||||
let value = match params.view {
|
||||
ShowTicketView::Summary => ticket_status_json(&ticket),
|
||||
ShowTicketView::Post => {
|
||||
let event = ticket
|
||||
.events
|
||||
.iter()
|
||||
.find(|event| Some(event.event_ref.as_str()) == params.event_ref.as_deref())
|
||||
.ok_or_else(|| {
|
||||
ToolError::InvalidArgument(
|
||||
"Observed post identity not found in this Ticket".into(),
|
||||
)
|
||||
})?;
|
||||
// Match the Workspace full-post route: never silently truncate CAS input.
|
||||
if event.body.as_str().len() > 1024 * 1024 {
|
||||
return Err(ToolError::ExecutionFailed(
|
||||
"Exact post exceeds the 1 MiB full-read limit; do not edit from a truncated preview".into(),
|
||||
));
|
||||
}
|
||||
json!({"ticket": ticket.meta.id, "post": ticket_event_json(event, usize::MAX)})
|
||||
}
|
||||
ShowTicketView::Detail | ShowTicketView::History => {
|
||||
let (events, page) = ticket_event_page(
|
||||
&ticket,
|
||||
params.event_cursor.as_deref(),
|
||||
event_limit,
|
||||
body_max_bytes,
|
||||
)?;
|
||||
if params.view == ShowTicketView::History {
|
||||
json!({"ticket": ticket.meta.id, "thread": events, "event_page": page})
|
||||
} else {
|
||||
let mut detail =
|
||||
ticket_json(&ticket, event_limit, artifact_limit, body_max_bytes);
|
||||
detail["events"] = events;
|
||||
detail["event_page"] = page;
|
||||
detail
|
||||
}
|
||||
}
|
||||
};
|
||||
Ok(json_output(summary, value))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1694,8 +1773,70 @@ fn ticket_relations_json(ticket: &Ticket) -> Value {
|
||||
})
|
||||
}
|
||||
|
||||
fn ticket_event_json(event: &crate::TicketEvent, body_max_bytes: usize) -> Value {
|
||||
fn ticket_status_json(ticket: &Ticket) -> Value {
|
||||
json!({
|
||||
"ticket": ticket.meta.resource_key.as_deref().unwrap_or(&ticket.meta.id),
|
||||
"title": ticket.meta.title,
|
||||
"state": ticket.meta.workflow_state,
|
||||
"content_digest": crate::ticket_content_digest(ticket),
|
||||
"closure_operation_key": ticket.closure_operation_key,
|
||||
"current_closure": ticket.current_closure.as_ref().map(|j| json!({
|
||||
"operation_key": j.operation_key, "reason": j.reason, "author": j.author, "at": j.at
|
||||
})),
|
||||
"targets": ticket.meta.targets,
|
||||
"tags": ticket.tags,
|
||||
"assignee": ticket.meta.assignee,
|
||||
"readiness": ticket.meta.readiness,
|
||||
"updated_at": ticket.meta.updated_at,
|
||||
"latest_event_ref": ticket.events.last().map(|event| &event.event_ref),
|
||||
})
|
||||
}
|
||||
|
||||
// Cursor is the persisted identity of the oldest returned event, never an offset.
|
||||
// Appending newer events between pages therefore cannot shift the continuation.
|
||||
fn ticket_event_page(
|
||||
ticket: &Ticket,
|
||||
cursor: Option<&str>,
|
||||
limit: usize,
|
||||
body_max_bytes: usize,
|
||||
) -> Result<(Value, Value), ToolError> {
|
||||
let events: Vec<_> = ticket
|
||||
.events
|
||||
.iter()
|
||||
.filter(|event| !event.is_initial)
|
||||
.collect();
|
||||
let end = match cursor {
|
||||
None => events.len(),
|
||||
Some(cursor) => events
|
||||
.iter()
|
||||
.position(|event| event.event_ref == cursor)
|
||||
.ok_or_else(|| {
|
||||
ToolError::InvalidArgument(
|
||||
"Unknown Ticket event_cursor; reuse a returned cursor".into(),
|
||||
)
|
||||
})?,
|
||||
};
|
||||
let start = end.saturating_sub(limit.max(1));
|
||||
let items: Vec<_> = events[start..end]
|
||||
.iter()
|
||||
.map(|event| ticket_event_json(event, body_max_bytes))
|
||||
.collect();
|
||||
let next_cursor = if start > 0 {
|
||||
Some(&events[start].event_ref)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok((
|
||||
json!({"count": events.len(), "returned": items.len(), "truncated": start > 0, "items": items}),
|
||||
json!({"next_cursor": next_cursor, "has_more": start > 0}),
|
||||
))
|
||||
}
|
||||
|
||||
fn ticket_event_json(event: &crate::TicketEvent, body_max_bytes: usize) -> Value {
|
||||
use sha2::{Digest, Sha256};
|
||||
json!({
|
||||
"body_digest": Sha256::digest(event.body.as_str().as_bytes()).iter().map(|byte| format!("{byte:02x}")).collect::<String>(),
|
||||
"editable": event.kind == TicketEventKind::Comment,
|
||||
"event_ref": event.event_ref,
|
||||
"is_initial": event.is_initial,
|
||||
"kind": event.kind.as_str(),
|
||||
@@ -1722,13 +1863,8 @@ fn ticket_json(
|
||||
artifact_limit: usize,
|
||||
body_max_bytes: usize,
|
||||
) -> Value {
|
||||
let event_count = ticket.events.len();
|
||||
let events: Vec<_> = ticket
|
||||
.events
|
||||
.iter()
|
||||
.skip(event_count.saturating_sub(event_limit))
|
||||
.map(|event| ticket_event_json(event, body_max_bytes))
|
||||
.collect();
|
||||
let (events, event_page) = ticket_event_page(ticket, None, event_limit, body_max_bytes)
|
||||
.expect("first thread page has no cursor to reject");
|
||||
let artifact_count = ticket.artifacts.len();
|
||||
let artifacts: Vec<_> = ticket
|
||||
.artifacts
|
||||
@@ -1753,17 +1889,12 @@ fn ticket_json(
|
||||
},
|
||||
"content_digest": crate::ticket_content_digest(ticket),
|
||||
"closure_operation_key": ticket.closure_operation_key,
|
||||
"current_closure": ticket.current_closure.as_ref().map(|j| json!({"operation_key": j.operation_key, "reason": j.reason, "explanation": truncate_text(&j.explanation, body_max_bytes), "author": j.author, "references": j.references, "at": j.at})),
|
||||
"case_history": { "count": ticket.case_history.len(), "items": ticket.case_history.iter().rev().take(event_limit).map(|j| json!({"operation_key": j.operation_key, "previous_closure_operation_key": j.previous_closure_operation_key, "action": j.action, "explanation": truncate_text(&j.explanation, body_max_bytes), "author": j.author, "references": j.references, "at": j.at})).collect::<Vec<_>>() },
|
||||
"current_closure": ticket.current_closure.as_ref().map(|j| json!({"operation_key": j.operation_key, "reason": j.reason, "author": j.author, "references": j.references, "at": j.at})),
|
||||
"case_history": { "count": ticket.case_history.len(), "items": ticket.case_history.iter().rev().take(event_limit).map(|j| json!({"operation_key": j.operation_key, "previous_closure_operation_key": j.previous_closure_operation_key, "action": j.action, "author": j.author, "references": j.references, "at": j.at})).collect::<Vec<_>>() },
|
||||
"legacy_workflow_snapshot": ticket.legacy_workflow_snapshot,
|
||||
"initial_post": ticket.events.iter().find(|event| event.is_initial).map(|event| ticket_event_json(event, body_max_bytes)),
|
||||
"body": truncate_text(ticket.document.body.as_str(), body_max_bytes),
|
||||
"events": {
|
||||
"count": event_count,
|
||||
"returned": events.len(),
|
||||
"truncated": event_count > events.len(),
|
||||
"items": events,
|
||||
},
|
||||
"events": events,
|
||||
"event_page": event_page,
|
||||
"artifacts": {
|
||||
"count": artifact_count,
|
||||
"returned": artifacts.len(),
|
||||
@@ -2063,13 +2194,17 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
let value: Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
||||
assert_eq!(value["events"]["count"], 2);
|
||||
let projected = value["events"]["items"]
|
||||
assert_eq!(value["events"]["count"], 1);
|
||||
let projected = if post.is_initial {
|
||||
&value["initial_post"]
|
||||
} else {
|
||||
value["events"]["items"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|e| e["event_ref"] == post.event_ref)
|
||||
.unwrap();
|
||||
.unwrap()
|
||||
};
|
||||
assert_eq!(projected["author"], json!(post.author));
|
||||
assert_eq!(projected["at"], json!(post.at));
|
||||
assert_eq!(projected["is_initial"], post.is_initial);
|
||||
@@ -2104,7 +2239,8 @@ mod tests {
|
||||
value["initial_post"]["event_ref"],
|
||||
before.events[0].event_ref
|
||||
);
|
||||
assert_eq!(value["initial_post"]["body"], value["body"]);
|
||||
assert_eq!(value["initial_post"]["body"], "Edited Initial body");
|
||||
assert!(value.get("body").is_none());
|
||||
assert_eq!(value["meta"]["created_by"], "creator");
|
||||
assert_eq!(value["initial_post"]["author"], "creator");
|
||||
let initial_keys = value["initial_post"]
|
||||
@@ -2518,7 +2654,10 @@ mod tests {
|
||||
.unwrap();
|
||||
let value: Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
value["body"].as_str().unwrap().trim_start_matches('\n'),
|
||||
value["initial_post"]["body"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.trim_start_matches('\n'),
|
||||
"ONE\ntwo\nONE\n"
|
||||
);
|
||||
|
||||
@@ -2980,13 +3119,13 @@ mod tests {
|
||||
assert_eq!(value["closure_operation_key"], "tool-correct");
|
||||
assert_eq!(value["case_history"]["count"], 2);
|
||||
assert_eq!(value["case_history"]["items"].as_array().unwrap().len(), 1);
|
||||
assert!(value["current_closure"].get("explanation").is_none());
|
||||
assert!(
|
||||
value["current_closure"]["explanation"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.len()
|
||||
< 256
|
||||
value["case_history"]["items"][0]
|
||||
.get("explanation")
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(value["events"]["items"][0]["body_truncated"], true);
|
||||
assert!(value["content_digest"].as_str().unwrap().len() == 64);
|
||||
let mut reopen = correction;
|
||||
reopen["expected_closure_operation_key"] = json!("tool-correct");
|
||||
@@ -3015,6 +3154,148 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn show_views_deduplicate_bound_history_and_read_exact_unicode_posts() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let backend = backend(&temp);
|
||||
let body = "long-requirement-".repeat(700);
|
||||
let mut input = NewTicket::new("Read projections");
|
||||
input.body = MarkdownText::new(body.clone());
|
||||
let reference = backend.create(input).unwrap();
|
||||
for index in 0..5 {
|
||||
backend
|
||||
.add_event(
|
||||
reference.id.clone().into(),
|
||||
NewTicketEvent::new(TicketEventKind::Comment, format!("comment-{index}")),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let before = backend.show(reference.id.clone().into()).unwrap();
|
||||
let detail = ticket_json(&before, 20, 20, DEFAULT_BODY_MAX_BYTES);
|
||||
assert_eq!(detail.to_string().matches(&body).count(), 1);
|
||||
assert!(detail.get("body").is_none());
|
||||
assert!(
|
||||
detail["events"]["items"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|event| event["is_initial"] == false)
|
||||
);
|
||||
let mut larger = before.clone();
|
||||
larger.document.body = MarkdownText::new("prose".repeat(100_000));
|
||||
larger.events[0].body = larger.document.body.clone();
|
||||
larger
|
||||
.events
|
||||
.extend(vec![before.events.last().unwrap().clone(); 1000]);
|
||||
let small_summary = ticket_status_json(&before).to_string();
|
||||
let large_summary = ticket_status_json(&larger).to_string();
|
||||
assert_eq!(small_summary.len(), large_summary.len());
|
||||
eprintln!(
|
||||
"Ticket generic bytes: detail={}, summary={} (unchanged with 500KB body / 1000 extra events)",
|
||||
detail.to_string().len(),
|
||||
small_summary.len()
|
||||
);
|
||||
|
||||
let show = tool_by_name(backend.clone(), "ShowTicket");
|
||||
let mut cursor = Value::Null;
|
||||
let mut seen = Vec::new();
|
||||
loop {
|
||||
let output = show.execute(&json!({"id": reference.id, "view": "history", "event_limit": 2, "event_cursor": cursor}).to_string(), Default::default()).await.unwrap();
|
||||
let page: Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
||||
assert_eq!(page.as_object().unwrap().len(), 3);
|
||||
for event in page["thread"]["items"].as_array().unwrap() {
|
||||
seen.push(event["event_ref"].as_str().unwrap().to_owned());
|
||||
}
|
||||
if cursor.is_null() {
|
||||
backend
|
||||
.add_event(
|
||||
reference.id.clone().into(),
|
||||
NewTicketEvent::new(TicketEventKind::Comment, "newer append"),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
cursor = page["event_page"]["next_cursor"].clone();
|
||||
if cursor.is_null() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
seen.sort();
|
||||
let mut expected: Vec<_> = before
|
||||
.events
|
||||
.iter()
|
||||
.filter(|event| !event.is_initial)
|
||||
.map(|event| event.event_ref.clone())
|
||||
.collect();
|
||||
expected.sort();
|
||||
assert_eq!(seen, expected);
|
||||
assert!(
|
||||
show.execute(
|
||||
&json!({"id": reference.id, "view":"history", "event_cursor":"guessed"})
|
||||
.to_string(),
|
||||
Default::default()
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let unicode = "全文🦀é\n".repeat(20_000);
|
||||
let post = &before.events[0];
|
||||
let edit = tool_by_name(backend.clone(), "TicketEditPost");
|
||||
let edit_args = json!({"ticket":reference.id, "event_ref":post.event_ref, "expected_body":post.body.as_str(), "body":unicode});
|
||||
edit.execute(&edit_args.to_string(), Default::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let full = show.execute(&json!({"id":reference.id, "view":"post", "event_ref":post.event_ref, "body_max_bytes":1}).to_string(), Default::default()).await.unwrap();
|
||||
let full: Value = serde_json::from_str(&full.content.unwrap()).unwrap();
|
||||
assert_eq!(full["post"]["body"], unicode);
|
||||
assert_eq!(full["post"]["body_truncated"], false);
|
||||
assert_eq!(full["post"]["editable"], true);
|
||||
assert!(
|
||||
edit.execute(&edit_args.to_string(), Default::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for args in [
|
||||
json!({"view":"post"}),
|
||||
json!({"view":"summary", "event_ref":post.event_ref}),
|
||||
json!({"view":"summary", "event_cursor":"x"}),
|
||||
] {
|
||||
let mut args = args;
|
||||
args["id"] = json!(reference.id);
|
||||
assert!(
|
||||
show.execute(&args.to_string(), Default::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exact_post_read_rejects_oversize_instead_of_truncating_edit_input() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let backend = backend(&temp);
|
||||
let mut input = NewTicket::new("Full-post limit");
|
||||
input.body = MarkdownText::new("x".repeat(1024 * 1024 + 1));
|
||||
let reference = backend.create(input).unwrap();
|
||||
let observed = backend.show(reference.id.clone().into()).unwrap();
|
||||
let event_ref = &observed
|
||||
.events
|
||||
.iter()
|
||||
.find(|event| event.is_initial)
|
||||
.unwrap()
|
||||
.event_ref;
|
||||
let show = tool_by_name(backend.clone(), "ShowTicket");
|
||||
let error = show
|
||||
.execute(
|
||||
&json!({"id":reference.id, "view":"post", "event_ref":event_ref}).to_string(),
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("1 MiB"));
|
||||
assert_eq!(backend.show(reference.id.into()).unwrap(), observed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn case_tool_rejects_legacy_states_and_unrecognized_input_fields() {
|
||||
let temp = TempDir::new().unwrap();
|
||||
|
||||
@@ -785,6 +785,12 @@ mod tests {
|
||||
#[derive(Default)]
|
||||
struct PendingRuntimeBackend {
|
||||
operations: std::sync::Mutex<Vec<worker_runtime::execution::WorkerLifecycleOperationId>>,
|
||||
owners: std::sync::Mutex<
|
||||
std::collections::HashMap<
|
||||
worker_runtime::execution::WorkerLifecycleOperationId,
|
||||
worker_runtime::identity::WorkerRef,
|
||||
>,
|
||||
>,
|
||||
}
|
||||
|
||||
impl worker_runtime::execution::WorkerExecutionBackend for PendingRuntimeBackend {
|
||||
@@ -806,6 +812,21 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
fn restore_recovery_disposition(
|
||||
&self,
|
||||
operation_id: worker_runtime::execution::WorkerLifecycleOperationId,
|
||||
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||
) -> Option<worker_runtime::execution::RestoreRecoveryDisposition> {
|
||||
use worker_runtime::execution::RestoreRecoveryDisposition as D;
|
||||
Some(
|
||||
if self.owners.lock().unwrap().get(&operation_id) == Some(worker_ref) {
|
||||
D::Continue
|
||||
} else {
|
||||
D::Absent
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn restore_worker(
|
||||
&self,
|
||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||
@@ -813,6 +834,10 @@ mod tests {
|
||||
use worker_runtime::execution::*;
|
||||
let mut operations = self.operations.lock().unwrap();
|
||||
operations.push(request.operation_id);
|
||||
self.owners
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(request.operation_id, request.worker_ref.clone());
|
||||
if operations.len() == 1 {
|
||||
WorkerExecutionSpawnResult::ReconciliationRequired {
|
||||
result: WorkerExecutionResult::errored(
|
||||
@@ -1119,9 +1144,14 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_resume_does_not_observe_or_open_retained_state() {
|
||||
async fn settled_nonaccepted_resume_does_not_retry_observe_or_open_retained_state() {
|
||||
for state in [
|
||||
BackendWorkerRestoreState::Rejected,
|
||||
BackendWorkerRestoreState::RolledBack,
|
||||
BackendWorkerRestoreState::Superseded,
|
||||
] {
|
||||
let lifecycle = FakeBackendWorkerLifecycle {
|
||||
restore_response: restore_response(BackendWorkerRestoreState::Rejected),
|
||||
restore_response: restore_response(state),
|
||||
observation: session_observation(serde_json::json!({
|
||||
"availability": "unavailable",
|
||||
"reason": "retention_missing",
|
||||
@@ -1144,15 +1174,16 @@ mod tests {
|
||||
expected_observation_token: "observed-worker-state".into(),
|
||||
request_id: "resume-operation".into(),
|
||||
}),
|
||||
|_| Ok(false),
|
||||
|_| panic!("settled nonaccepted restore must not offer recovery retry"),
|
||||
)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
|
||||
assert!(error.contains("Rejected"));
|
||||
assert!(error.contains(&format!("{state:?}")));
|
||||
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resume_candidates_have_no_current_worker_state() {
|
||||
@@ -1176,6 +1207,7 @@ mod tests {
|
||||
for state in [
|
||||
BackendWorkerRestoreState::Rejected,
|
||||
BackendWorkerRestoreState::RolledBack,
|
||||
BackendWorkerRestoreState::Superseded,
|
||||
BackendWorkerRestoreState::ReconciliationRequired,
|
||||
] {
|
||||
let mut response = restore_response(state);
|
||||
|
||||
@@ -19,15 +19,19 @@ required-features = ["ws-server", "fs-store"]
|
||||
[features]
|
||||
default = ["ws-server", "fs-store"]
|
||||
fs-store = ["dep:fs4"]
|
||||
# Offline fixture tooling only; never enabled by the product defaults.
|
||||
test-support = ["fs-store", "ws-server"]
|
||||
# Explicit process-test synchronization; independent of offline fixture tooling.
|
||||
restore-checkpoints = ["fs-store"]
|
||||
http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
|
||||
ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"]
|
||||
ws-server = ["http-server", "axum/ws", "tokio/sync"]
|
||||
|
||||
[dependencies]
|
||||
agen.workspace = true
|
||||
async-trait.workspace = true
|
||||
base64.workspace = true
|
||||
axum = { workspace = true, optional = true }
|
||||
futures = { workspace = true, optional = true }
|
||||
futures.workspace = true
|
||||
decodal.workspace = true
|
||||
flow = { path = "../flow" }
|
||||
fs4 = { workspace = true, optional = true }
|
||||
|
||||
@@ -8,3 +8,61 @@ Manages Worker creation, stopping, and Restore, execution connections within the
|
||||
- [`execution`] — execution backends and operation-specific results.
|
||||
- [`working_directory`] — repository materialization, session resources, and safe deletion.
|
||||
- [`workspace_issuer`] — Workspace signature verification and replay prevention.
|
||||
|
||||
## Offline stopped-Worker fixtures
|
||||
|
||||
The nondefault `test-support` feature exposes `test_support::StoppedWorkerTemplate`.
|
||||
It is intentionally limited to current-schema, stopped, attachment-free artificial
|
||||
conversations with one Segment, text history, and no pending execution/activation.
|
||||
It does not add a product endpoint or change create/restore authorization.
|
||||
|
||||
1. With the source Runtime offline, call `StoppedWorkerTemplate::extract(worker_dir)`.
|
||||
The source must have its existing Runtime owner lock and identity. Extraction is
|
||||
read-only; the returned serde template excludes source auth, saved Manifest,
|
||||
request/config references, run directories, and Runtime operation history.
|
||||
2. Initialize **new** Runtime identity and Workspace trust in a fresh destination
|
||||
using normal setup. Preserve the template's Runtime and Workspace IDs.
|
||||
3. Call `template.install(&options, request, &resolved_manifest)` before starting
|
||||
the Runtime. Supply a fresh typed `CreateWorkerRequest` (including config/profile
|
||||
references and trusted memory settings) and `manifest::WorkerManifest` resolved
|
||||
for that environment. Its Worker name must equal `template.worker_name()`.
|
||||
4. Start normally and use the ordinary explicit restore operation. Installation
|
||||
itself creates no execution and leaves the Worker stopped with a fresh guard.
|
||||
|
||||
The template preserves the original Session manifest/JSONL bytes, including stable
|
||||
entry IDs, timestamps, and settled submission receipts. It rejects resumable queues
|
||||
and Feature invocation history. A saved system prompt is required for restore;
|
||||
checkpoint receipts must match the Worker's typed receipt schema. Preserved records
|
||||
must use canonical writer field shapes: unknown/duplicate fields (including nested
|
||||
fields), discarded optional fields, and opaque Feature history payloads are rejected,
|
||||
not stripped or rewritten. Portability/secret-marker checks do not replace
|
||||
review of the supplied artificial conversation or its independently verified hashes.
|
||||
The installer refuses any destination content other than initialized `auth.toml`;
|
||||
interrupted publication is fail-closed, not an overwrite/recovery facility.
|
||||
|
||||
## Owned-process Restore checkpoints
|
||||
|
||||
The independent, nondefault `restore-checkpoints` feature exists only for explicit
|
||||
process tests in the shared `tests/e2e` harness. Offline `test-support` does not enable
|
||||
it. Normal binaries reject `--test-restore-checkpoint`; no environment variable or
|
||||
HTTP endpoint activates this facility. Do not deploy a checkpoint-enabled binary
|
||||
as a replacement for the default product.
|
||||
|
||||
A test-enabled Runtime accepts `--test-restore-checkpoint <config.json>` with a
|
||||
bounded regular JSON file selecting a loopback socket, Worker ID, one stage and a
|
||||
diagnostic nonce. The nonce is only a handshake label, never execution identity,
|
||||
request authority or routing state. The one-shot stages are real prepared candidate
|
||||
with `NotCommitted`, and durable `Committed` before activation/terminal receipt.
|
||||
Before acknowledging, the existing bounded canonical reader validates the exact
|
||||
Worker/operation/decision, syncs its file/directories and requires no saved receipt.
|
||||
No pending JSON is fabricated and no lifecycle work is skipped.
|
||||
|
||||
The Runtime sends bounded newline-delimited JSON evidence, then waits at most eight
|
||||
seconds for byte `R`. EOF, invalid reply, unavailable peer or deadline releases the
|
||||
wait into the original operation. The Runtime-state mutex is not held across that
|
||||
wait; existing per-Worker ownership still covers the actual candidate and cleanup.
|
||||
The harness verifies the ack, kills/reaps only its owned child and inspects the
|
||||
canonical aggregate offline before restarting the same roots/identity without the
|
||||
checkpoint flag or another fixture install. These tests are a separate cfg/binary
|
||||
lane from featureless product regression; they do not certify arbitrary OS failure,
|
||||
external power loss or a real model call.
|
||||
|
||||
@@ -215,6 +215,38 @@ impl WorkerExecutionResult {
|
||||
}
|
||||
}
|
||||
|
||||
// Private one-shot synchronization at real publication boundaries. Disconnect
|
||||
// releases the waiter during unwinding; timeout bounds a forgotten test owner.
|
||||
#[cfg(test)]
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct RestoreTestGate<T> {
|
||||
pub(crate) entered: std::sync::mpsc::Sender<T>,
|
||||
pub(crate) release: std::sync::mpsc::Receiver<()>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl<T> RestoreTestGate<T> {
|
||||
pub(crate) fn wait(self, evidence: T) {
|
||||
let _ = self.entered.send(evidence);
|
||||
assert!(
|
||||
!matches!(
|
||||
self.release
|
||||
.recv_timeout(std::time::Duration::from_secs(30)),
|
||||
Err(std::sync::mpsc::RecvTimeoutError::Timeout)
|
||||
),
|
||||
"publication test gate timed out"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "ws-server"))]
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct RestoreFlushFault {
|
||||
pub(crate) remaining_successes: usize,
|
||||
pub(crate) gate: RestoreTestGate<protocol::Event>,
|
||||
pub(crate) restored: std::sync::mpsc::Sender<Vec<protocol::Event>>,
|
||||
}
|
||||
|
||||
/// Runtime hooks available to an execution backend for one Worker.
|
||||
#[cfg(feature = "ws-server")]
|
||||
type WorkerObservationPublisher = Arc<
|
||||
@@ -242,10 +274,15 @@ enum WorkerExecutionPublicationState {
|
||||
#[derive(Clone)]
|
||||
pub struct WorkerExecutionContext {
|
||||
worker_ref: WorkerRef,
|
||||
restore_progress: Option<Arc<dyn Fn(WorkerLifecycleOperationId) + Send + Sync>>,
|
||||
restore_repair_authorizer:
|
||||
Option<Arc<dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync>>,
|
||||
#[cfg(feature = "ws-server")]
|
||||
observation_publisher: WorkerObservationPublisher,
|
||||
#[cfg(feature = "ws-server")]
|
||||
publication_state: Arc<Mutex<WorkerExecutionPublicationState>>,
|
||||
#[cfg(all(test, feature = "ws-server"))]
|
||||
flush_fault: Arc<Mutex<Option<RestoreFlushFault>>>,
|
||||
}
|
||||
|
||||
impl WorkerExecutionContext {
|
||||
@@ -256,7 +293,11 @@ impl WorkerExecutionContext {
|
||||
) -> Self {
|
||||
Self {
|
||||
worker_ref,
|
||||
restore_progress: None,
|
||||
restore_repair_authorizer: None,
|
||||
observation_publisher,
|
||||
#[cfg(test)]
|
||||
flush_fault: Arc::default(),
|
||||
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Active)),
|
||||
}
|
||||
}
|
||||
@@ -268,7 +309,11 @@ impl WorkerExecutionContext {
|
||||
) -> Self {
|
||||
Self {
|
||||
worker_ref,
|
||||
restore_progress: None,
|
||||
restore_repair_authorizer: None,
|
||||
observation_publisher,
|
||||
#[cfg(test)]
|
||||
flush_fault: Arc::default(),
|
||||
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Pending(
|
||||
VecDeque::new(),
|
||||
))),
|
||||
@@ -277,12 +322,56 @@ impl WorkerExecutionContext {
|
||||
|
||||
#[cfg(not(feature = "ws-server"))]
|
||||
pub(crate) fn new(worker_ref: WorkerRef) -> Self {
|
||||
Self { worker_ref }
|
||||
Self {
|
||||
worker_ref,
|
||||
restore_progress: None,
|
||||
restore_repair_authorizer: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "ws-server"))]
|
||||
pub(crate) fn candidate(worker_ref: WorkerRef) -> Self {
|
||||
Self { worker_ref }
|
||||
Self {
|
||||
worker_ref,
|
||||
restore_progress: None,
|
||||
restore_repair_authorizer: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_restore_repair_authorizer(
|
||||
mut self,
|
||||
authorizer: Arc<
|
||||
dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync,
|
||||
>,
|
||||
) -> Self {
|
||||
self.restore_repair_authorizer = Some(authorizer);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn authorize_restore_source_repair(
|
||||
&self,
|
||||
operation: WorkerLifecycleOperationId,
|
||||
) -> Result<(), RuntimeError> {
|
||||
match &self.restore_repair_authorizer {
|
||||
Some(authorizer) => authorizer(operation),
|
||||
// Direct factory/component users own their source repair separately;
|
||||
// only Runtime contexts carry a Runtime aggregate authority callback.
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_restore_progress(
|
||||
mut self,
|
||||
progress: Arc<dyn Fn(WorkerLifecycleOperationId) + Send + Sync>,
|
||||
) -> Self {
|
||||
self.restore_progress = Some(progress);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn restore_progressed(&self, operation_id: WorkerLifecycleOperationId) {
|
||||
if let Some(progress) = &self.restore_progress {
|
||||
progress(operation_id);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn worker_ref(&self) -> &WorkerRef {
|
||||
@@ -335,20 +424,65 @@ impl WorkerExecutionContext {
|
||||
let Some(event) = event else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Err(error) = (self.observation_publisher)(self.worker_ref.clone(), event.clone())
|
||||
{
|
||||
#[cfg(test)]
|
||||
let fault = {
|
||||
let mut fault = self.flush_fault.lock().unwrap();
|
||||
if fault.as_ref().is_some_and(|f| f.remaining_successes == 0) {
|
||||
fault.take()
|
||||
} else {
|
||||
if let Some(fault) = fault.as_mut() {
|
||||
fault.remaining_successes -= 1;
|
||||
}
|
||||
None
|
||||
}
|
||||
};
|
||||
#[cfg(test)]
|
||||
let mut restored = None;
|
||||
#[cfg(test)]
|
||||
let publication = if let Some(fault) = fault {
|
||||
restored = Some(fault.restored);
|
||||
fault.gate.wait(event.clone());
|
||||
Err(RuntimeError::InvalidRequest(
|
||||
"injected buffered publication failure".into(),
|
||||
))
|
||||
} else {
|
||||
(self.observation_publisher)(self.worker_ref.clone(), event.clone())
|
||||
};
|
||||
#[cfg(not(test))]
|
||||
let publication = (self.observation_publisher)(self.worker_ref.clone(), event.clone());
|
||||
if let Err(error) = publication {
|
||||
let mut state = self
|
||||
.publication_state
|
||||
.lock()
|
||||
.map_err(|_| RuntimeError::StatePoisoned)?;
|
||||
if let WorkerExecutionPublicationState::Activating(events) = &mut *state {
|
||||
events.push_front(event);
|
||||
#[cfg(test)]
|
||||
if let Some(restored) = restored {
|
||||
let _ = restored.send(events.iter().cloned().collect());
|
||||
}
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "ws-server"))]
|
||||
pub(crate) fn test_fail_flush(&self, fault: RestoreFlushFault) {
|
||||
assert!(self.flush_fault.lock().unwrap().replace(fault).is_none());
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "ws-server"))]
|
||||
pub(crate) fn test_buffered_events(&self) -> Vec<protocol::Event> {
|
||||
match &*self.publication_state.lock().unwrap() {
|
||||
WorkerExecutionPublicationState::Pending(events)
|
||||
| WorkerExecutionPublicationState::Activating(events) => {
|
||||
events.iter().cloned().collect()
|
||||
}
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "ws-server"))]
|
||||
pub(crate) fn activate_candidate(&self) -> Result<(), RuntimeError> {
|
||||
Ok(())
|
||||
@@ -424,6 +558,7 @@ pub struct WorkerExecutionSpawnRequest {
|
||||
/// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WorkerExecutionRestoreRequest {
|
||||
pub source_repair_possible: bool,
|
||||
pub operation_id: WorkerLifecycleOperationId,
|
||||
pub worker_ref: WorkerRef,
|
||||
pub request: crate::catalog::CreateWorkerRequest,
|
||||
@@ -565,6 +700,19 @@ impl fmt::Debug for WorkerProtocolTransport {
|
||||
}
|
||||
}
|
||||
|
||||
/// Derived resource disposition for an existing restore owner. Not durable
|
||||
/// authority: Runtime must reread its decision before choosing any action.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RestoreRecoveryDisposition {
|
||||
/// No live or pending owner for this exact operation. Not cleanup evidence.
|
||||
Absent,
|
||||
InFlight,
|
||||
Continue,
|
||||
Discard,
|
||||
/// A retained cleanup-only owner; never reconstruct or activate it.
|
||||
Reclaim,
|
||||
}
|
||||
|
||||
/// The connection selects this backend; WorkerRef identifies an execution,
|
||||
/// never a capability or authorization token.
|
||||
pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
||||
@@ -633,6 +781,33 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Completed resource-owner evidence retained while Runtime settlement may
|
||||
/// still need retry. This is not a new request or execution authorization.
|
||||
fn completed_restore_failure(
|
||||
&self,
|
||||
_operation_id: WorkerLifecycleOperationId,
|
||||
_worker_ref: &WorkerRef,
|
||||
) -> Option<WorkerExecutionSpawnResult> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Opt in to Runtime-owned autonomous restore continuation. Pending tasks
|
||||
/// signal completion through the original execution context; callbacks must
|
||||
/// not wait for Runtime locks. Independent of resource disposition support.
|
||||
fn supports_restore_continuation(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Describe exact operation resource ownership for warm/cold recovery
|
||||
/// dispatch. This does not opt in to callbacks or autonomous continuation.
|
||||
fn restore_recovery_disposition(
|
||||
&self,
|
||||
_operation_id: WorkerLifecycleOperationId,
|
||||
_worker_ref: &WorkerRef,
|
||||
) -> Option<RestoreRecoveryDisposition> {
|
||||
None
|
||||
}
|
||||
|
||||
fn reconcile_restore(
|
||||
&self,
|
||||
request: WorkerExecutionRestoreRequest,
|
||||
@@ -640,6 +815,42 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
||||
self.restore_worker(request)
|
||||
}
|
||||
|
||||
/// Reclaim an uncommitted operation after restart without preparing,
|
||||
/// repairing, or activating a replacement candidate. Runtime must establish
|
||||
/// the durable uncommitted decision before calling this seam. Unsupported
|
||||
/// cleanup is uncertainty, never a successful rollback.
|
||||
fn reclaim_uncommitted_restore(
|
||||
&self,
|
||||
_request: WorkerExecutionRestoreRequest,
|
||||
) -> WorkerExecutionSpawnResult {
|
||||
WorkerExecutionSpawnResult::ReconciliationRequired {
|
||||
result: WorkerExecutionResult::unsupported(
|
||||
WorkerExecutionOperation::Restore,
|
||||
"backend cannot reclaim an uncommitted restore after restart",
|
||||
),
|
||||
worker_state: None,
|
||||
workdir_attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Discard only the exact unpublished operation after Runtime has proved
|
||||
/// its durable decision uncommitted. This is not Stop and must not affect a
|
||||
/// connected execution. RolledBack requires confirmed cleanup and no repair.
|
||||
fn discard_uncommitted_restore(
|
||||
&self,
|
||||
_operation_id: WorkerLifecycleOperationId,
|
||||
_worker_ref: &WorkerRef,
|
||||
) -> WorkerExecutionSpawnResult {
|
||||
WorkerExecutionSpawnResult::ReconciliationRequired {
|
||||
result: WorkerExecutionResult::unsupported(
|
||||
WorkerExecutionOperation::Restore,
|
||||
"backend cannot confirm unpublished restore cleanup",
|
||||
),
|
||||
worker_state: None,
|
||||
workdir_attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn activate_restored_worker(
|
||||
&self,
|
||||
_operation_id: WorkerLifecycleOperationId,
|
||||
@@ -901,6 +1112,28 @@ impl WorkerExecutionBackendRef {
|
||||
self.backend.preflight_restore(request)
|
||||
}
|
||||
|
||||
pub(crate) fn completed_restore_failure(
|
||||
&self,
|
||||
operation_id: WorkerLifecycleOperationId,
|
||||
worker_ref: &WorkerRef,
|
||||
) -> Option<WorkerExecutionSpawnResult> {
|
||||
self.backend
|
||||
.completed_restore_failure(operation_id, worker_ref)
|
||||
}
|
||||
|
||||
pub(crate) fn supports_restore_continuation(&self) -> bool {
|
||||
self.backend.supports_restore_continuation()
|
||||
}
|
||||
|
||||
pub(crate) fn restore_recovery_disposition(
|
||||
&self,
|
||||
operation_id: WorkerLifecycleOperationId,
|
||||
worker_ref: &WorkerRef,
|
||||
) -> Option<RestoreRecoveryDisposition> {
|
||||
self.backend
|
||||
.restore_recovery_disposition(operation_id, worker_ref)
|
||||
}
|
||||
|
||||
pub(crate) fn reconcile_restore(
|
||||
&self,
|
||||
request: WorkerExecutionRestoreRequest,
|
||||
@@ -908,6 +1141,22 @@ impl WorkerExecutionBackendRef {
|
||||
self.backend.reconcile_restore(request)
|
||||
}
|
||||
|
||||
pub(crate) fn reclaim_uncommitted_restore(
|
||||
&self,
|
||||
request: WorkerExecutionRestoreRequest,
|
||||
) -> WorkerExecutionSpawnResult {
|
||||
self.backend.reclaim_uncommitted_restore(request)
|
||||
}
|
||||
|
||||
pub(crate) fn discard_uncommitted_restore(
|
||||
&self,
|
||||
operation_id: WorkerLifecycleOperationId,
|
||||
worker_ref: &WorkerRef,
|
||||
) -> WorkerExecutionSpawnResult {
|
||||
self.backend
|
||||
.discard_uncommitted_restore(operation_id, worker_ref)
|
||||
}
|
||||
|
||||
pub(crate) fn activate_restored_worker(
|
||||
&self,
|
||||
operation_id: WorkerLifecycleOperationId,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -25,6 +25,7 @@ use crate::interaction::{WorkerInput, WorkerInteractionAck};
|
||||
use crate::management::{RuntimeSummary, WorkerDeleteResult};
|
||||
#[cfg(feature = "ws-server")]
|
||||
use crate::observation::WorkerObservationCursor;
|
||||
#[cfg(feature = "fs-store")]
|
||||
use crate::retention::{
|
||||
WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory,
|
||||
};
|
||||
@@ -1568,6 +1569,7 @@ fn protocol_error_event(message: impl Into<String>) -> protocol::Event {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "fs-store")]
|
||||
#[allow(dead_code)]
|
||||
async fn worker_retention_inventory(
|
||||
State(state): State<RuntimeHttpState>,
|
||||
@@ -1589,6 +1591,7 @@ async fn worker_retention_inventory(
|
||||
.map_err(RuntimeHttpRestError::runtime)
|
||||
}
|
||||
|
||||
#[cfg(feature = "fs-store")]
|
||||
#[allow(dead_code)]
|
||||
async fn execute_worker_retention(
|
||||
State(state): State<RuntimeHttpState>,
|
||||
@@ -2654,6 +2657,87 @@ mod tests {
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
// Generated management routes and real Workspace capability middleware stay
|
||||
// connected even when this process has no filesystem retention provider.
|
||||
#[cfg(not(feature = "fs-store"))]
|
||||
#[tokio::test]
|
||||
async fn memory_retention_routes_preserve_auth_and_return_typed_unavailable() {
|
||||
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
|
||||
let trust = workspace_trust(&identity);
|
||||
let runtime = Runtime::new_memory();
|
||||
let app = runtime_http_router_with_workspace_auth(
|
||||
runtime,
|
||||
None,
|
||||
workspace_test_auth(vec![trust.clone()]),
|
||||
);
|
||||
let worker_id = WorkerId::now_v7();
|
||||
let body = runtime_api::WorkerRetentionExecutionRequest {
|
||||
operation_id: "memory-retention".into(),
|
||||
input_fingerprint: "input".into(),
|
||||
archive_id: None,
|
||||
workspace_id: "workspace-a".into(),
|
||||
source_runtime_id: "runtime-test".into(),
|
||||
worker_id,
|
||||
source_created_at: "2026-01-01T00:00:00Z".into(),
|
||||
removed_at: "2026-01-02T00:00:00Z".into(),
|
||||
effective_profile: None,
|
||||
retention_class: None,
|
||||
policy_id: "policy".into(),
|
||||
session_disposition: runtime_api::SessionDisposition::Purge,
|
||||
diagnostics_disposition: runtime_api::DiagnosticsDisposition::Purge,
|
||||
};
|
||||
for (method, path, body) in [
|
||||
(
|
||||
"GET",
|
||||
format!("/v1/workers/{worker_id}/retention/inventory"),
|
||||
vec![],
|
||||
),
|
||||
(
|
||||
"POST",
|
||||
format!("/v1/workers/{worker_id}/retention/execute"),
|
||||
serde_json::to_vec(&body).unwrap(),
|
||||
),
|
||||
] {
|
||||
let unsigned = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method(method)
|
||||
.uri(&path)
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.body(Body::from(body.clone()))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
|
||||
let claims = signed_claims(&trust, method, &path, &body);
|
||||
let response =
|
||||
signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
let error: RuntimeHttpErrorResponse = read_json(response).await;
|
||||
assert_eq!(error.error.code, "worker_retention_unavailable");
|
||||
}
|
||||
for (mut value, expected_status, expected_code) in [
|
||||
(body.clone(), StatusCode::NOT_FOUND, "worker_not_found"),
|
||||
(body.clone(), StatusCode::BAD_REQUEST, "worker_id_mismatch"),
|
||||
] {
|
||||
if expected_status == StatusCode::NOT_FOUND {
|
||||
value.workspace_id = "workspace-b".into();
|
||||
} else {
|
||||
value.worker_id = WorkerId::now_v7();
|
||||
}
|
||||
let path = format!("/v1/workers/{worker_id}/retention/execute");
|
||||
let bytes = serde_json::to_vec(&value).unwrap();
|
||||
let claims = signed_claims(&trust, "POST", &path, &bytes);
|
||||
let response =
|
||||
signed_request(app.clone(), &identity, &claims, bytes, "workspace-a").await;
|
||||
assert_eq!(response.status(), expected_status);
|
||||
let error: RuntimeHttpErrorResponse = read_json(response).await;
|
||||
assert_eq!(error.error.code, expected_code);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_signed_ping_succeeds_without_handshake_and_rejects_replay() {
|
||||
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
|
||||
|
||||
@@ -444,6 +444,31 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
||||
material_pin(&self.state, request, true)
|
||||
}
|
||||
|
||||
async fn session_recording_capture(
|
||||
&self,
|
||||
request: runtime_api::SessionRecordingCaptureRequest,
|
||||
) -> Result<runtime_api::SessionRecordingCaptureAvailability, runtime_api::RuntimeApiError>
|
||||
{
|
||||
let scope = material_scope(&self.state, &request.workspace_id)?;
|
||||
#[cfg(feature = "fs-store")]
|
||||
{
|
||||
Ok(self
|
||||
.state
|
||||
.runtime
|
||||
.session_recording_capture_scoped(&scope, &request))
|
||||
}
|
||||
#[cfg(not(feature = "fs-store"))]
|
||||
{
|
||||
let _ = scope;
|
||||
Ok(
|
||||
runtime_api::SessionRecordingCaptureAvailability::Unavailable {
|
||||
reason: runtime_api::SessionPublicUnavailableReason::StorageUnavailable,
|
||||
message: "Committed Session storage unavailable".into(),
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async fn session_material_catalog(
|
||||
&self,
|
||||
request: runtime_api::SessionMaterialCatalogRequest,
|
||||
@@ -694,6 +719,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
||||
server_api::WorkerRestoreState::RolledBack => {
|
||||
runtime_api::WorkerRestoreState::RolledBack
|
||||
}
|
||||
server_api::WorkerRestoreState::Superseded => {
|
||||
runtime_api::WorkerRestoreState::Superseded
|
||||
}
|
||||
server_api::WorkerRestoreState::ReconciliationRequired => {
|
||||
runtime_api::WorkerRestoreState::ReconciliationRequired
|
||||
}
|
||||
@@ -744,6 +772,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
||||
server_api::WorkerRestoreState::RolledBack => {
|
||||
runtime_api::WorkerRestoreState::RolledBack
|
||||
}
|
||||
server_api::WorkerRestoreState::Superseded => {
|
||||
runtime_api::WorkerRestoreState::Superseded
|
||||
}
|
||||
server_api::WorkerRestoreState::ReconciliationRequired => {
|
||||
runtime_api::WorkerRestoreState::ReconciliationRequired
|
||||
}
|
||||
@@ -856,6 +887,8 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
||||
)
|
||||
})?;
|
||||
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
||||
#[cfg(feature = "fs-store")]
|
||||
{
|
||||
let inventory = self
|
||||
.state
|
||||
.runtime
|
||||
@@ -864,6 +897,16 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
||||
.map_err(api_error)?;
|
||||
response(inventory)
|
||||
}
|
||||
#[cfg(not(feature = "fs-store"))]
|
||||
{
|
||||
let _ = (scope, worker_ref);
|
||||
Err(runtime_api::RuntimeApiError::new(
|
||||
503,
|
||||
"worker_retention_unavailable",
|
||||
"Worker retention requires filesystem storage",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_retention(
|
||||
&self,
|
||||
@@ -880,21 +923,23 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
||||
)
|
||||
})?;
|
||||
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
||||
let request: WorkerRetentionExecutionRequest = request(value)?;
|
||||
if request.workspace_id != scope.workspace_id {
|
||||
if value.workspace_id != scope.workspace_id {
|
||||
return Err(runtime_api::RuntimeApiError::new(
|
||||
StatusCode::NOT_FOUND.as_u16(),
|
||||
"worker_not_found",
|
||||
"worker does not belong to the authenticated Workspace",
|
||||
));
|
||||
}
|
||||
if request.worker_id != worker_ref.worker_id {
|
||||
if value.worker_id != worker_ref.worker_id {
|
||||
return Err(runtime_api::RuntimeApiError::new(
|
||||
StatusCode::BAD_REQUEST.as_u16(),
|
||||
"worker_id_mismatch",
|
||||
"Worker retention request worker_id does not match route worker_id",
|
||||
));
|
||||
}
|
||||
#[cfg(feature = "fs-store")]
|
||||
{
|
||||
let request: WorkerRetentionExecutionRequest = request(value)?;
|
||||
let result = self
|
||||
.state
|
||||
.runtime
|
||||
@@ -903,4 +948,11 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
||||
.map_err(api_error)?;
|
||||
response(result)
|
||||
}
|
||||
#[cfg(not(feature = "fs-store"))]
|
||||
Err(runtime_api::RuntimeApiError::new(
|
||||
503,
|
||||
"worker_retention_unavailable",
|
||||
"Worker retention requires filesystem storage",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,6 +150,27 @@ impl Fixture {
|
||||
limit: 1,
|
||||
}
|
||||
}
|
||||
fn recording_request(&self) -> runtime_api::SessionRecordingCaptureRequest {
|
||||
runtime_api::SessionRecordingCaptureRequest {
|
||||
workspace_id: self.scope.workspace_id.clone(),
|
||||
source: self.source(),
|
||||
expected_session_id: self.session.to_string(),
|
||||
start: Default::default(),
|
||||
max_entries: 1000,
|
||||
max_serialized_bytes: 128 * 1024,
|
||||
}
|
||||
}
|
||||
fn capture(&self, source: Source) -> session_store::capture::RecordingCaptureSlice {
|
||||
let mut request = self.recording_request();
|
||||
request.source = source;
|
||||
match self
|
||||
.runtime
|
||||
.session_recording_capture_scoped(&self.scope, &request)
|
||||
{
|
||||
runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => slice,
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
fn catalog(&self) -> runtime_api::SessionMaterialCatalogPage {
|
||||
match self
|
||||
.runtime
|
||||
@@ -226,6 +247,9 @@ fn pending_session_proof_is_not_missing_material_and_pin_release_allows_empty_ar
|
||||
.is_err(),
|
||||
"retention intent is not source availability"
|
||||
);
|
||||
assert!(
|
||||
matches!(f.runtime.session_recording_capture_scoped(&f.scope, &f.recording_request()), runtime_api::SessionRecordingCaptureAvailability::NotMaterialized { session_id } if session_id == f.session.to_string())
|
||||
);
|
||||
// A missing committed source is never the same fact as a pending pointer.
|
||||
metadata
|
||||
.write(
|
||||
@@ -479,6 +503,7 @@ fn source_scope_session_and_active_pointer_mismatches_never_return_material() {
|
||||
#[test]
|
||||
fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source() {
|
||||
let f = Fixture::new();
|
||||
let frozen = f.capture(f.source());
|
||||
let original: Vec<_> = f.reader(&f.source()).descriptors().cloned().collect();
|
||||
let pin = SessionRetentionPin(uuid::Uuid::now_v7());
|
||||
f.runtime
|
||||
@@ -516,6 +541,7 @@ fn pinned_stopped_worker_archives_before_removal_and_job_reads_archived_source()
|
||||
let archived = Source::Archived {
|
||||
archive_id: "canonical-archive".into(),
|
||||
};
|
||||
assert_eq!(f.capture(archived.clone()), frozen);
|
||||
let reader = f.reader(&archived);
|
||||
assert_eq!(reader.descriptors().cloned().collect::<Vec<_>>(), original);
|
||||
let body = reader.read_content(&original[0], 0, 16384).unwrap();
|
||||
@@ -951,11 +977,52 @@ async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router
|
||||
.authorizer(SignedAuthorizer { identity, trust })
|
||||
.build()
|
||||
.unwrap();
|
||||
let frozen = match client
|
||||
.session_recording_capture(f.recording_request())
|
||||
.await
|
||||
.unwrap()
|
||||
{
|
||||
runtime_api::SessionRecordingCaptureAvailability::Captured { slice } => slice,
|
||||
other => panic!("{other:?}"),
|
||||
};
|
||||
frozen.capture.validate(Default::default()).unwrap();
|
||||
assert!(frozen.next.is_none());
|
||||
assert!(
|
||||
serde_json::to_vec(&frozen).unwrap().len()
|
||||
<= f.recording_request().max_serialized_bytes
|
||||
);
|
||||
let snapshot = serde_json::to_vec(&frozen).unwrap();
|
||||
assert!(!String::from_utf8_lossy(&snapshot).contains("private-system-prompt"));
|
||||
f.store
|
||||
.append(
|
||||
f.session,
|
||||
f.segment,
|
||||
&LogEntry::AnnotatedAssistantItem {
|
||||
ts: 6,
|
||||
entry: message("late-answer", LoggedRole::Assistant, "not in frozen input"),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(serde_json::to_vec(&frozen).unwrap(), snapshot);
|
||||
let page = match client.session_material_catalog(f.request()).await.unwrap() {
|
||||
Catalog::Page { page } => page,
|
||||
other => panic!("{other:?}"),
|
||||
};
|
||||
let descriptor = page.descriptors[0].clone();
|
||||
assert_eq!(
|
||||
frozen
|
||||
.capture
|
||||
.recording_text(0)
|
||||
.unwrap()
|
||||
.0
|
||||
.identity
|
||||
.entry_ref,
|
||||
descriptor.entry_ref
|
||||
);
|
||||
assert_eq!(
|
||||
frozen.capture.recording_text(0).unwrap().0.sha256,
|
||||
descriptor.sha256
|
||||
);
|
||||
let read = client
|
||||
.session_material_read(runtime_api::SessionMaterialReadRequest {
|
||||
workspace_id: "workspace-a".into(),
|
||||
@@ -1019,6 +1086,13 @@ async fn signed_generated_client_reads_canonical_pages_and_pins_with_real_router
|
||||
);
|
||||
}
|
||||
assert!(!f.root().exists());
|
||||
assert!(matches!(
|
||||
client
|
||||
.session_recording_capture(f.recording_request())
|
||||
.await
|
||||
.unwrap(),
|
||||
runtime_api::SessionRecordingCaptureAvailability::Unavailable { .. }
|
||||
));
|
||||
assert!(matches!(
|
||||
client.session_material_catalog(f.request()).await.unwrap(),
|
||||
Catalog::Unavailable { .. }
|
||||
@@ -1050,12 +1124,22 @@ async fn canonical_http_rejects_replay_tampered_body_and_workspace_mismatch() {
|
||||
None,
|
||||
workspace_test_auth(vec![trust.clone()]),
|
||||
);
|
||||
let path = "/v1/session-public/material/catalog";
|
||||
let body = serde_json::to_vec(&f.request()).unwrap();
|
||||
for (path, body) in [
|
||||
(
|
||||
"/v1/session-public/material/catalog",
|
||||
serde_json::to_vec(&f.request()).unwrap(),
|
||||
),
|
||||
(
|
||||
"/v1/session-public/recording-capture",
|
||||
serde_json::to_vec(&f.recording_request()).unwrap(),
|
||||
),
|
||||
] {
|
||||
let claims = signed_claims(&trust, "POST", path, &body);
|
||||
let first = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
|
||||
let first =
|
||||
signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
|
||||
assert_eq!(first.status(), StatusCode::OK);
|
||||
let replay = signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
|
||||
let replay =
|
||||
signed_request(app.clone(), &identity, &claims, body.clone(), "workspace-a").await;
|
||||
assert_eq!(replay.status(), StatusCode::UNAUTHORIZED);
|
||||
let claims = signed_claims(&trust, "POST", path, &body);
|
||||
let changed = signed_request(
|
||||
@@ -1067,13 +1151,15 @@ async fn canonical_http_rejects_replay_tampered_body_and_workspace_mismatch() {
|
||||
)
|
||||
.await;
|
||||
assert_eq!(changed.status(), StatusCode::UNAUTHORIZED);
|
||||
let mut wrong_scope = f.request();
|
||||
wrong_scope.workspace_id = "other-workspace".into();
|
||||
let mut wrong_scope: serde_json::Value = serde_json::from_slice(&body).unwrap();
|
||||
wrong_scope["workspace_id"] = "other-workspace".into();
|
||||
let wrong_body = serde_json::to_vec(&wrong_scope).unwrap();
|
||||
let claims = signed_claims(&trust, "POST", path, &wrong_body);
|
||||
let response = signed_request(app.clone(), &identity, &claims, wrong_body, "workspace-a").await;
|
||||
let response =
|
||||
signed_request(app.clone(), &identity, &claims, wrong_body, "workspace-a").await;
|
||||
assert_eq!(response.status(), StatusCode::FORBIDDEN);
|
||||
let unsigned = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
@@ -1086,6 +1172,7 @@ async fn canonical_http_rejects_replay_tampered_body_and_workspace_mismatch() {
|
||||
.unwrap();
|
||||
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subject_create_recovery_signed_http_requires_delete_and_matching_workspace() {
|
||||
@@ -1170,3 +1257,152 @@ async fn explicit_material_erasure_requires_delete_capability_and_matching_scope
|
||||
);
|
||||
assert!(f.root().exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recording_capture_rejects_scope_session_cursor_and_budget_without_returning_bodies() {
|
||||
use runtime_api::SessionRecordingCaptureAvailability as Capture;
|
||||
let f = Fixture::new();
|
||||
let request = f.recording_request();
|
||||
let frozen = f.capture(f.source());
|
||||
assert_eq!(frozen.capture.entries.len(), 3);
|
||||
let mut request = request;
|
||||
for invalid in 0..5 {
|
||||
let mut bad = request.clone();
|
||||
match invalid {
|
||||
0 => bad.workspace_id = "other".into(),
|
||||
1 => bad.expected_session_id = session_store::new_session_id().to_string(),
|
||||
2 => bad.start.canonical_order = 100,
|
||||
3 => bad.max_entries = 0,
|
||||
4 => bad.max_serialized_bytes = 4 * 1024 * 1024 + 1,
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert!(matches!(
|
||||
f.runtime.session_recording_capture_scoped(&f.scope, &bad),
|
||||
Capture::Unavailable { .. }
|
||||
));
|
||||
}
|
||||
let text = frozen.capture.recording_text(1).unwrap().2;
|
||||
request.start = session_store::capture::RecordingPosition {
|
||||
canonical_order: 1,
|
||||
byte_offset: text.find('🍎').unwrap() as u64 + 1,
|
||||
};
|
||||
assert!(matches!(
|
||||
f.runtime
|
||||
.session_recording_capture_scoped(&f.scope, &request),
|
||||
Capture::Unavailable {
|
||||
reason: Reason::InvalidCursor,
|
||||
..
|
||||
}
|
||||
));
|
||||
request.start = Default::default();
|
||||
request.max_entries = 1;
|
||||
let Capture::Captured { slice } = f
|
||||
.runtime
|
||||
.session_recording_capture_scoped(&f.scope, &request)
|
||||
else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(slice.capture.entries.len(), 1);
|
||||
assert_eq!(slice.next.unwrap().canonical_order, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recording_capture_compaction_preserves_original_descriptors_and_excludes_summary_and_new_tail() {
|
||||
let f = Fixture::new();
|
||||
let old = f.capture(f.source());
|
||||
let compact = session_store::new_segment_id();
|
||||
let mut summary = message(
|
||||
"summary",
|
||||
LoggedRole::System,
|
||||
"derived summary is not original",
|
||||
);
|
||||
summary.metadata.input_metadata = Some(
|
||||
session_store::HistoryInputMetadata::new(
|
||||
session_store::HistoryInputUsePolicy {
|
||||
recording_material: session_store::InputEvidenceUse::Exclude,
|
||||
memory_evidence: session_store::InputEvidenceUse::Exclude,
|
||||
observation: session_store::InputObservationUse::Hidden,
|
||||
compaction: session_store::InputCompactionUse::OpeningSummary,
|
||||
},
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
summary.metadata.derivation = Some(session_store::LoggedHistoryDerivation {
|
||||
sources: vec![
|
||||
LoggedSessionHistoryEntryId("user-1".into()),
|
||||
LoggedSessionHistoryEntryId("answer-1".into()),
|
||||
],
|
||||
});
|
||||
f.store
|
||||
.create_segment(
|
||||
f.session,
|
||||
compact,
|
||||
&[
|
||||
LogEntry::AnnotatedSegmentStart {
|
||||
ts: 6,
|
||||
session_id: f.session,
|
||||
system_prompt: Some("private".into()),
|
||||
config: Default::default(),
|
||||
history: vec![summary],
|
||||
forked_from: None,
|
||||
compacted_from: Some(session_store::SegmentOrigin {
|
||||
segment_id: f.segment,
|
||||
at_turn_index: 0,
|
||||
}),
|
||||
},
|
||||
LogEntry::AnnotatedAssistantItem {
|
||||
ts: 7,
|
||||
entry: message(
|
||||
"new-answer",
|
||||
LoggedRole::Assistant,
|
||||
"new original after compaction",
|
||||
),
|
||||
},
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
WorkerAggregateStore::new(
|
||||
f.root().parent().unwrap(),
|
||||
format!("worker-runtime-{}", f.worker.worker_id),
|
||||
)
|
||||
.unwrap()
|
||||
.set_active(
|
||||
&format!("worker-runtime-{}", f.worker.worker_id),
|
||||
Some(session_store::WorkerActiveSegmentRef::active_segment(
|
||||
f.session, compact,
|
||||
)),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let frozen = f.capture(f.source());
|
||||
assert_eq!(frozen.capture.entries.len(), old.capture.entries.len() + 1);
|
||||
for entry in &old.capture.entries {
|
||||
assert_eq!(
|
||||
frozen.capture.recording_text(entry.order).unwrap(),
|
||||
old.capture.recording_text(entry.order).unwrap()
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
!serde_json::to_string(&frozen)
|
||||
.unwrap()
|
||||
.contains("derived summary is not original")
|
||||
);
|
||||
let bytes = serde_json::to_vec(&frozen).unwrap();
|
||||
f.store
|
||||
.append(
|
||||
f.session,
|
||||
compact,
|
||||
&LogEntry::AnnotatedAssistantItem {
|
||||
ts: 8,
|
||||
entry: message("tail", LoggedRole::Assistant, "next Job only"),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(serde_json::to_vec(&frozen).unwrap(), bytes);
|
||||
assert_eq!(
|
||||
f.capture(f.source()).capture.entries.len(),
|
||||
frozen.capture.entries.len() + 1
|
||||
);
|
||||
}
|
||||
|
||||
@@ -16,6 +16,11 @@ pub mod management;
|
||||
pub mod observation;
|
||||
pub mod profile_archive;
|
||||
pub mod resource;
|
||||
#[cfg(feature = "restore-checkpoints")]
|
||||
pub mod restore_checkpoints;
|
||||
#[cfg(test)]
|
||||
#[path = "../../../tests/support/restore_environment.rs"]
|
||||
mod restore_test_environment;
|
||||
#[cfg(feature = "fs-store")]
|
||||
pub mod retention;
|
||||
mod runtime;
|
||||
@@ -32,6 +37,8 @@ pub mod workspace_issuer;
|
||||
#[cfg(feature = "http-server")]
|
||||
pub mod workspace_request;
|
||||
|
||||
#[cfg(feature = "test-support")]
|
||||
pub use fs_store::test_support;
|
||||
#[cfg(feature = "fs-store")]
|
||||
pub use fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
|
||||
pub use management::RuntimeOptions;
|
||||
|
||||
@@ -117,6 +117,12 @@ fn run() -> Result<(), ProcessError> {
|
||||
let listener = tokio::net::TcpListener::bind(config.http.bind_addr).await?;
|
||||
let local_addr = listener.local_addr()?;
|
||||
let worker_runtime = build_runtime(&config)?;
|
||||
#[cfg(feature = "restore-checkpoints")]
|
||||
if let Some(checkpoint) = &config.test_restore_checkpoint {
|
||||
worker_runtime
|
||||
.install_test_restore_checkpoint(checkpoint.clone())
|
||||
.map_err(ProcessError::Runtime)?;
|
||||
}
|
||||
eprintln!(
|
||||
"yoi-runtime listening on {local_addr}; intended client is a trusted backend/proxy, not a browser"
|
||||
);
|
||||
@@ -391,6 +397,21 @@ where
|
||||
let (flag, inline_value) = split_flag_value(arg)?;
|
||||
match flag.as_str() {
|
||||
"--help" | "-h" => return Ok(None),
|
||||
#[cfg(feature = "restore-checkpoints")]
|
||||
"--test-restore-checkpoint" => {
|
||||
let path = take_value(&flag, inline_value, &mut args)?;
|
||||
if config.test_restore_checkpoint.is_some() {
|
||||
return Err(ProcessError::usage(
|
||||
"--test-restore-checkpoint may only be specified once".into(),
|
||||
));
|
||||
}
|
||||
config.test_restore_checkpoint = Some(
|
||||
worker_runtime::restore_checkpoints::RestoreCheckpointConfig::from_path(
|
||||
Path::new(&path),
|
||||
)
|
||||
.map_err(ProcessError::Runtime)?,
|
||||
);
|
||||
}
|
||||
"--bind" => {
|
||||
let value = take_value(&flag, inline_value, &mut args)?;
|
||||
config.http.bind_addr = value.parse::<SocketAddr>().map_err(|error| {
|
||||
@@ -465,6 +486,12 @@ where
|
||||
}
|
||||
|
||||
apply_store_selection(&mut config);
|
||||
#[cfg(feature = "restore-checkpoints")]
|
||||
if config.test_restore_checkpoint.is_some() && config.no_store {
|
||||
return Err(ProcessError::usage(
|
||||
"--test-restore-checkpoint requires an fs Runtime store".into(),
|
||||
));
|
||||
}
|
||||
Ok(Some(config))
|
||||
}
|
||||
|
||||
@@ -521,6 +548,8 @@ struct ProcessConfig {
|
||||
no_store: bool,
|
||||
backend_resource_endpoint: Option<String>,
|
||||
backend_resource_token: Option<String>,
|
||||
#[cfg(feature = "restore-checkpoints")]
|
||||
test_restore_checkpoint: Option<worker_runtime::restore_checkpoints::RestoreCheckpointConfig>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
@@ -542,6 +571,8 @@ impl ProcessConfig {
|
||||
no_store: false,
|
||||
backend_resource_endpoint: None,
|
||||
backend_resource_token: None,
|
||||
#[cfg(feature = "restore-checkpoints")]
|
||||
test_restore_checkpoint: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1227,6 +1258,47 @@ Auth commands:
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn checkpoint_cli_is_absent_by_default() {
|
||||
let config = parse_args([] as [&str; 0]).unwrap().unwrap();
|
||||
#[cfg(feature = "restore-checkpoints")]
|
||||
assert!(config.test_restore_checkpoint.is_none());
|
||||
#[cfg(not(feature = "restore-checkpoints"))]
|
||||
{
|
||||
let _ = config;
|
||||
let error = parse_args(["--test-restore-checkpoint", "never-opened.json"]).unwrap_err();
|
||||
assert!(error.to_string().contains("unknown argument"));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "restore-checkpoints")]
|
||||
#[test]
|
||||
fn checkpoint_cli_requires_valid_explicit_file_and_fs_store() {
|
||||
use worker_runtime::restore_checkpoints::{
|
||||
RestoreCheckpointConfig, RestoreCheckpointStage,
|
||||
};
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("checkpoint.json");
|
||||
let checkpoint = RestoreCheckpointConfig {
|
||||
endpoint: "127.0.0.1:1234".parse().unwrap(),
|
||||
worker_id: protocol::WorkerId::now_v7(),
|
||||
stage: RestoreCheckpointStage::CommittedPreterminal,
|
||||
nonce: "cli-test".into(),
|
||||
};
|
||||
std::fs::write(&path, serde_json::to_vec(&checkpoint).unwrap()).unwrap();
|
||||
let args = ["--test-restore-checkpoint", path.to_str().unwrap()];
|
||||
let config = parse_args(args).unwrap().unwrap();
|
||||
assert_eq!(config.test_restore_checkpoint, Some(checkpoint));
|
||||
assert!(parse_args([args[0], args[1], "--no-store"]).is_err());
|
||||
assert!(parse_args([args[0], args[1], args[0], args[1]]).is_err());
|
||||
assert!(parse_args([args[0]]).is_err());
|
||||
std::fs::write(&path, b"private-invalid-body").unwrap();
|
||||
let error = parse_args(args).unwrap_err().to_string();
|
||||
assert!(!error.contains("private-invalid-body"));
|
||||
std::fs::remove_file(&path).unwrap();
|
||||
assert!(parse_args(args).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn help_does_not_start_server() {
|
||||
assert!(parse_args(["--help"]).unwrap().is_none());
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#[cfg(feature = "http-server")]
|
||||
use crate::auth::BACKEND_RESOURCE_FETCH_PERMISSION;
|
||||
use crate::identity::WorkerId;
|
||||
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef, sha256_hex};
|
||||
#[cfg(feature = "http-server")]
|
||||
use crate::workspace_request::RuntimeWorkspaceRequestClient;
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
//! Explicit, ephemeral synchronization for owned Runtime process tests only.
|
||||
//!
|
||||
//! The nonce authenticates no Runtime action and is never a routing identity.
|
||||
//! Socket failure, EOF and the fixed deadline only release the gate; the lifecycle
|
||||
//! owner continues its original operation and remains responsible for cleanup.
|
||||
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::{SocketAddr, TcpStream};
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use protocol::WorkerId;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::RuntimeError;
|
||||
use crate::execution::WorkerLifecycleOperationId;
|
||||
|
||||
pub const RESTORE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(8);
|
||||
pub const MAX_RESTORE_CHECKPOINT_CONFIG_BYTES: usize = 4096;
|
||||
pub const MAX_RESTORE_CHECKPOINT_ACK_BYTES: usize = 4096;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RestoreCheckpointStage {
|
||||
PreparedNotCommitted,
|
||||
CommittedPreterminal,
|
||||
ActivatedPreterminal,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RestoreCheckpointCommit {
|
||||
NotCommitted,
|
||||
Committed,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RestoreCheckpointConfig {
|
||||
pub endpoint: SocketAddr,
|
||||
pub worker_id: WorkerId,
|
||||
pub stage: RestoreCheckpointStage,
|
||||
pub nonce: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RestoreCheckpointAck {
|
||||
pub nonce: String,
|
||||
pub stage: RestoreCheckpointStage,
|
||||
pub runtime_id: String,
|
||||
pub worker_id: WorkerId,
|
||||
pub operation_id: WorkerLifecycleOperationId,
|
||||
pub commit: RestoreCheckpointCommit,
|
||||
pub receipt_absent: bool,
|
||||
pub pid: u32,
|
||||
}
|
||||
|
||||
fn invalid_config() -> RuntimeError {
|
||||
// Do not include serde errors, paths or input contents in process diagnostics.
|
||||
RuntimeError::InvalidRequest("invalid test restore checkpoint configuration".into())
|
||||
}
|
||||
|
||||
impl RestoreCheckpointConfig {
|
||||
pub fn validate(&self) -> Result<(), RuntimeError> {
|
||||
if !self.endpoint.ip().is_loopback()
|
||||
|| self.endpoint.port() == 0
|
||||
|| self.nonce.is_empty()
|
||||
|| self.nonce.len() > 256
|
||||
|| self.nonce.chars().any(char::is_control)
|
||||
{
|
||||
return Err(invalid_config());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn from_path(path: &Path) -> Result<Self, RuntimeError> {
|
||||
let mut options = OpenOptions::new();
|
||||
options.read(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
// A FIFO must not stall startup before its regular-file check.
|
||||
options.custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW);
|
||||
}
|
||||
let file = options.open(path).map_err(|_| invalid_config())?;
|
||||
let metadata = file.metadata().map_err(|_| invalid_config())?;
|
||||
if !metadata.is_file() || metadata.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 {
|
||||
return Err(invalid_config());
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
file.take(MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|_| invalid_config())?;
|
||||
if bytes.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES {
|
||||
return Err(invalid_config());
|
||||
}
|
||||
let config: Self = serde_json::from_slice(&bytes).map_err(|_| invalid_config())?;
|
||||
config.validate()?;
|
||||
Ok(config)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn wait_for_release(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) {
|
||||
let _ = exchange(config, ack);
|
||||
}
|
||||
|
||||
fn remaining(deadline: Instant) -> std::io::Result<Duration> {
|
||||
deadline
|
||||
.checked_duration_since(Instant::now())
|
||||
.filter(|remaining| !remaining.is_zero())
|
||||
.ok_or_else(|| std::io::Error::from(std::io::ErrorKind::TimedOut))
|
||||
}
|
||||
|
||||
fn exchange(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) -> std::io::Result<()> {
|
||||
let deadline = Instant::now() + RESTORE_CHECKPOINT_TIMEOUT;
|
||||
let mut bytes = serde_json::to_vec(ack)?;
|
||||
bytes.push(b'\n');
|
||||
if bytes.len() > MAX_RESTORE_CHECKPOINT_ACK_BYTES {
|
||||
return Err(std::io::Error::from(std::io::ErrorKind::InvalidData));
|
||||
}
|
||||
let mut stream = TcpStream::connect_timeout(&config.endpoint, remaining(deadline)?)?;
|
||||
// Each syscall shares one budget; partial writes cannot reset the deadline.
|
||||
let mut pending = bytes.as_slice();
|
||||
while !pending.is_empty() {
|
||||
stream.set_write_timeout(Some(remaining(deadline)?))?;
|
||||
let written = stream.write(pending)?;
|
||||
if written == 0 {
|
||||
return Err(std::io::Error::from(std::io::ErrorKind::WriteZero));
|
||||
}
|
||||
pending = &pending[written..];
|
||||
}
|
||||
stream.set_read_timeout(Some(remaining(deadline)?))?;
|
||||
let mut release = [0];
|
||||
stream.read_exact(&mut release)?;
|
||||
if release != [b'R'] {
|
||||
return Err(std::io::Error::from(std::io::ErrorKind::InvalidData));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,139 @@
|
||||
use super::*;
|
||||
use std::io::{BufRead, BufReader};
|
||||
use std::net::TcpListener;
|
||||
use std::thread;
|
||||
|
||||
fn config(endpoint: SocketAddr) -> RestoreCheckpointConfig {
|
||||
RestoreCheckpointConfig {
|
||||
endpoint,
|
||||
worker_id: WorkerId::now_v7(),
|
||||
stage: RestoreCheckpointStage::PreparedNotCommitted,
|
||||
nonce: "checkpoint-test-handshake".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn ack(config: &RestoreCheckpointConfig) -> RestoreCheckpointAck {
|
||||
RestoreCheckpointAck {
|
||||
nonce: config.nonce.clone(),
|
||||
stage: config.stage,
|
||||
runtime_id: "checkpoint-runtime".into(),
|
||||
worker_id: config.worker_id,
|
||||
operation_id: WorkerLifecycleOperationId::new(),
|
||||
commit: RestoreCheckpointCommit::NotCommitted,
|
||||
receipt_absent: true,
|
||||
pid: std::process::id(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_rejects_nonliteral_nonloopback_bad_stage_unknown_fields_and_invalid_worker() {
|
||||
let valid = serde_json::to_value(config("127.0.0.1:1234".parse().unwrap())).unwrap();
|
||||
for (field, value) in [
|
||||
("endpoint", "localhost:1234"),
|
||||
("endpoint", "192.0.2.1:1234"),
|
||||
("endpoint", "0.0.0.0:1234"),
|
||||
("endpoint", "127.0.0.1:0"),
|
||||
("endpoint", "[::ffff:127.0.0.1]:1234"),
|
||||
("worker_id", "not-a-worker-id"),
|
||||
("stage", "after_terminal"),
|
||||
("nonce", ""),
|
||||
("nonce", "secret\nbody"),
|
||||
("timeout", "unbounded"),
|
||||
] {
|
||||
let mut invalid = valid.clone();
|
||||
invalid[field] = value.into();
|
||||
let parsed = serde_json::from_value::<RestoreCheckpointConfig>(invalid);
|
||||
assert!(
|
||||
parsed.is_err() || parsed.unwrap().validate().is_err(),
|
||||
"{field}"
|
||||
);
|
||||
}
|
||||
for endpoint in ["127.0.0.1:1234", "[::1]:1234"] {
|
||||
config(endpoint.parse().unwrap()).validate().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_file_bounds_and_errors_do_not_disclose_contents() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("config.json");
|
||||
assert!(RestoreCheckpointConfig::from_path(&path).is_err());
|
||||
std::fs::write(&path, vec![b'x'; MAX_RESTORE_CHECKPOINT_CONFIG_BYTES + 1]).unwrap();
|
||||
assert!(RestoreCheckpointConfig::from_path(&path).is_err());
|
||||
std::fs::write(
|
||||
&path,
|
||||
br#"{"nonce":"private-config-body","stage":"secret"}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let error = RestoreCheckpointConfig::from_path(&path)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(!error.contains("private-config-body"));
|
||||
assert!(!error.contains("secret"));
|
||||
let mut valid = config("127.0.0.1:1234".parse().unwrap());
|
||||
valid.nonce = "x".repeat(257);
|
||||
assert!(valid.validate().is_err());
|
||||
valid.nonce = "valid".into();
|
||||
std::fs::write(&path, serde_json::to_vec(&valid).unwrap()).unwrap();
|
||||
assert_eq!(RestoreCheckpointConfig::from_path(&path).unwrap(), valid);
|
||||
assert!(RestoreCheckpointConfig::from_path(dir.path()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_release_eof_and_invalid_release_close_the_owned_stream() {
|
||||
for release in [Some(b'R'), None, Some(b'X')] {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let config = config(listener.local_addr().unwrap());
|
||||
let expected = ack(&config);
|
||||
let sent = expected.clone();
|
||||
let task = thread::spawn(move || exchange(&config, &sent));
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(2)))
|
||||
.unwrap();
|
||||
let mut line = String::new();
|
||||
BufReader::new(&mut stream).read_line(&mut line).unwrap();
|
||||
assert!(line.len() <= MAX_RESTORE_CHECKPOINT_ACK_BYTES);
|
||||
assert_eq!(
|
||||
serde_json::from_str::<RestoreCheckpointAck>(&line).unwrap(),
|
||||
expected
|
||||
);
|
||||
if let Some(byte) = release {
|
||||
stream.write_all(&[byte]).unwrap();
|
||||
} else {
|
||||
stream.shutdown(std::net::Shutdown::Write).unwrap();
|
||||
}
|
||||
assert_eq!(task.join().unwrap().is_ok(), release == Some(b'R'));
|
||||
let mut byte = [0];
|
||||
assert_eq!(stream.read(&mut byte).unwrap(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_listener_resumes_without_waiting_or_retrying() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let config = config(listener.local_addr().unwrap());
|
||||
drop(listener);
|
||||
let start = Instant::now();
|
||||
wait_for_release(&config, &ack(&config));
|
||||
assert!(start.elapsed() < Duration::from_secs(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn silent_peer_hits_fixed_deadline_and_stream_is_dropped() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let config = config(listener.local_addr().unwrap());
|
||||
let start = Instant::now();
|
||||
let task = thread::spawn(move || exchange(&config, &ack(&config)));
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(12)))
|
||||
.unwrap();
|
||||
let mut reader = BufReader::new(&mut stream);
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).unwrap();
|
||||
assert!(task.join().unwrap().is_err());
|
||||
assert!(start.elapsed() >= RESTORE_CHECKPOINT_TIMEOUT - Duration::from_millis(100));
|
||||
assert!(start.elapsed() < RESTORE_CHECKPOINT_TIMEOUT + Duration::from_secs(3));
|
||||
assert_eq!(reader.read(&mut [0]).unwrap(), 0);
|
||||
}
|
||||
@@ -2069,6 +2069,103 @@ mod tests {
|
||||
compact
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_removal_preserves_history_larger_than_256_mib_and_replays_receipt() {
|
||||
use session_store::{
|
||||
LogEntry, LoggedContentPart, LoggedHistoryEntry, LoggedItem, LoggedRole,
|
||||
LoggedSessionHistoryEntryId, LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin,
|
||||
Store,
|
||||
};
|
||||
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let worker = WorkerId::now_v7();
|
||||
let worker_dir = temp.path().join("workers").join(worker.to_string());
|
||||
write_json(
|
||||
&worker_dir.join("worker.json"),
|
||||
&serde_json::json!({"workspace_id": "workspace-a"}),
|
||||
);
|
||||
let session = session_store::new_session_id();
|
||||
let segment = session_store::new_segment_id();
|
||||
let store = session_store::WorkerSessionStore::new(worker_dir.join("session")).unwrap();
|
||||
store
|
||||
.create_segment(
|
||||
session,
|
||||
segment,
|
||||
&[LogEntry::AnnotatedSegmentStart {
|
||||
ts: 1,
|
||||
session_id: session,
|
||||
system_prompt: None,
|
||||
config: Default::default(),
|
||||
history: Vec::new(),
|
||||
forked_from: None,
|
||||
compacted_from: None,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
// Each record is small enough to decode independently. Only the lifetime
|
||||
// total exceeds the former ceiling; no individual resource limit is relaxed.
|
||||
let text = "x".repeat(1024 * 1024);
|
||||
for index in 0..257 {
|
||||
store
|
||||
.append(
|
||||
session,
|
||||
segment,
|
||||
&LogEntry::AnnotatedUserInput {
|
||||
ts: index + 2,
|
||||
segments: Vec::new(),
|
||||
extensions: Vec::new(),
|
||||
history: vec![LoggedHistoryEntry {
|
||||
item: LoggedItem::Message {
|
||||
role: LoggedRole::User,
|
||||
content: vec![LoggedContentPart::Text { text: text.clone() }],
|
||||
},
|
||||
metadata: LoggedSessionHistoryMetadata {
|
||||
input_metadata: None,
|
||||
entry_id: LoggedSessionHistoryEntryId(format!("input-{index}")),
|
||||
origin: LoggedSessionHistoryOrigin::LocalHumanInput,
|
||||
derivation: None,
|
||||
},
|
||||
}],
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let mut metadata = session_store::WorkerMetadata::new(
|
||||
worker.to_string(),
|
||||
Some(session_store::WorkerActiveSegmentRef::active_segment(
|
||||
session, segment,
|
||||
)),
|
||||
);
|
||||
metadata.workspace_id = Some("workspace-a".into());
|
||||
write_json(&worker_dir.join("metadata.json"), &metadata);
|
||||
let segment_path = format!("session/segments/{segment}.jsonl");
|
||||
let source_bytes = fs::metadata(worker_dir.join(&segment_path)).unwrap().len();
|
||||
assert!(source_bytes > 256 * 1024 * 1024);
|
||||
|
||||
let provider = FsWorkerRetentionProvider::new(temp.path());
|
||||
let request = request(worker, SessionDisposition::Archive);
|
||||
let result = provider.execute(&request).unwrap();
|
||||
assert!(result.source_removed);
|
||||
assert!(!worker_dir.exists());
|
||||
let archive = provider.archive_dir("archive-a").unwrap();
|
||||
assert_eq!(
|
||||
fs::metadata(archive.join(&segment_path)).unwrap().len(),
|
||||
source_bytes
|
||||
);
|
||||
let reader = session_store::public_material::SessionPublicMaterialReader::open_worker(
|
||||
&archive.join("session"),
|
||||
segment,
|
||||
Default::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(reader.session_id(), session);
|
||||
assert_eq!(reader.descriptors().len(), 257);
|
||||
let retry = provider.execute(&request).unwrap();
|
||||
assert!(retry.source_removed);
|
||||
assert_eq!(retry.archive, result.archive);
|
||||
assert_eq!(retry.operation_id, result.operation_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_migration_repairs_failed_archive_and_preserves_ingestion_receipt_on_retry() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! only the empty segment envelope is built with current APIs.
|
||||
use super::tests::{request, write_json};
|
||||
use super::*;
|
||||
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
|
||||
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions, SCHEMA_VERSION};
|
||||
use serde_json::{Value, json};
|
||||
use session_store::{LogEntry, SegmentId, SessionId, Store, WorkerSessionStore};
|
||||
use std::collections::BTreeMap;
|
||||
@@ -226,7 +226,7 @@ fn completed_v4_archive(
|
||||
|
||||
#[test]
|
||||
fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
||||
for runtime_schema in [11, 12] {
|
||||
for runtime_schema in [11, 12, SCHEMA_VERSION] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
||||
@@ -329,7 +329,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
||||
(storage.checksum_sha256, storage.bytes, storage.file_count),
|
||||
(checksum, bytes, count)
|
||||
);
|
||||
if runtime_schema == 12 {
|
||||
if runtime_schema == SCHEMA_VERSION {
|
||||
for name in [
|
||||
PathBuf::from("runtime.json"),
|
||||
PathBuf::from(format!("workers/{worker}/worker.json")),
|
||||
@@ -346,8 +346,8 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
||||
);
|
||||
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
|
||||
assert_eq!(tree_bytes(&root), once, "retry rewrote migrated evidence");
|
||||
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
|
||||
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
|
||||
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
|
||||
assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists());
|
||||
// session-storage, not the immutable receipt's old checksum, attests
|
||||
// current bytes. Tampering with those bytes must remain detectable.
|
||||
let path = archive_session
|
||||
@@ -360,7 +360,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
||||
|
||||
#[test]
|
||||
fn corrupt_archive_does_not_block_startup_or_healthy_session_migration_and_repair_converges() {
|
||||
for runtime_schema in [11, 12] {
|
||||
for runtime_schema in [11, 12, SCHEMA_VERSION] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
||||
@@ -440,8 +440,8 @@ fn unmigratable_archive_preserves_evidence_across_startup_and_reopen() {
|
||||
] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
let (live_worker, live_session, live_segment) = runtime_fixture(&root, 12);
|
||||
let (worker, _, segment) = stopped_worker(&root, 12);
|
||||
let (live_worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||
let (worker, _, segment) = stopped_worker(&root, SCHEMA_VERSION);
|
||||
let provider = FsWorkerRetentionProvider::new(&root);
|
||||
let path = provider
|
||||
.worker_dir(worker)
|
||||
@@ -542,8 +542,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() {
|
||||
] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
let (worker, _, _) = runtime_fixture(&root, 12);
|
||||
completed_v4_archive(&root, 12);
|
||||
let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||
completed_v4_archive(&root, SCHEMA_VERSION);
|
||||
let provider = FsWorkerRetentionProvider::new(&root);
|
||||
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
|
||||
let archive = provider.archive_dir("archive-a").unwrap();
|
||||
@@ -575,8 +575,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() {
|
||||
fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
runtime_fixture(&root, 12);
|
||||
let (session, _, _, _) = completed_v4_archive(&root, 12);
|
||||
runtime_fixture(&root, SCHEMA_VERSION);
|
||||
let (session, _, _, _) = completed_v4_archive(&root, SCHEMA_VERSION);
|
||||
let ledger = root.join("retention/session-sources");
|
||||
fs::create_dir_all(&ledger).unwrap();
|
||||
fs::write(ledger.join(format!("{session}.json")), b"{broken\n").unwrap();
|
||||
@@ -591,7 +591,7 @@ fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
|
||||
fn runtime_v5_probe_and_staging_skip_erased_originals() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
let (worker, live_session, live_segment) = runtime_fixture(&root, 12);
|
||||
let (worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||
let erased = root
|
||||
.join("orphaned-workers")
|
||||
.join(WorkerId::now_v7().to_string());
|
||||
@@ -643,7 +643,7 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
|
||||
for corruption in ["broken-json", "future-schema"] {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
let (worker, session, segment) = runtime_fixture(&root, 12);
|
||||
let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||
let session_root = root
|
||||
.join("workers")
|
||||
.join(worker.to_string())
|
||||
@@ -674,19 +674,19 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
|
||||
assert_eq!(tree_bytes(&root), before);
|
||||
assert!(FsRuntimeStore::migrate(&options).is_err());
|
||||
assert_eq!(tree_bytes(&root), before);
|
||||
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
|
||||
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
|
||||
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
|
||||
assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
||||
// Relative socket names keep this fixture independent of the checkout/TMPDIR
|
||||
// length. The TempDir still owns the same tree; no process cwd is changed.
|
||||
// The same TempDir owns the relative bind/connect reference. No cwd change
|
||||
// or alias is needed, and the real live-socket migration refusal stays tested.
|
||||
let temp = tempfile::tempdir_in(".").unwrap();
|
||||
let root = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
||||
let (worker, _, _) = runtime_fixture(&root, 12);
|
||||
let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||
let before = tree_bytes(&root);
|
||||
let runs = root.join("workers").join(worker.to_string()).join("runs");
|
||||
fs::create_dir_all(runs.join("1")).unwrap();
|
||||
@@ -697,7 +697,7 @@ fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
||||
drop(listener);
|
||||
fs::remove_dir_all(runs).unwrap();
|
||||
assert_eq!(tree_bytes(&root), before);
|
||||
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
|
||||
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -705,7 +705,7 @@ fn current_runtime_upgrades_v5_role_metadata_through_staging_without_reserializi
|
||||
{
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
let (worker, session, segment) = runtime_fixture(&root, 12);
|
||||
let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||
let aggregate = root.join("workers").join(worker.to_string());
|
||||
let path = aggregate
|
||||
.join("session/segments")
|
||||
|
||||
+2064
-488
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,112 @@
|
||||
use super::*;
|
||||
use crate::restore_checkpoints::{
|
||||
RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage,
|
||||
wait_for_release,
|
||||
};
|
||||
|
||||
impl Runtime {
|
||||
/// Install an explicitly selected, one-shot process-test gate after normal startup.
|
||||
/// It is not persisted or consulted by admission, routing or recovery authority.
|
||||
pub fn install_test_restore_checkpoint(
|
||||
&self,
|
||||
config: RestoreCheckpointConfig,
|
||||
) -> Result<(), RuntimeError> {
|
||||
config.validate()?;
|
||||
let mut state = self.lock()?;
|
||||
if !matches!(state.persistence, RuntimePersistence::Fs(_)) {
|
||||
return Err(RuntimeError::InvalidRequest(
|
||||
"test restore checkpoint requires an fs Runtime store".into(),
|
||||
));
|
||||
}
|
||||
state.restore_checkpoint = Some(config);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// The lifecycle caller holds the per-Worker operation lock. No Runtime-state
|
||||
// mutex is retained while waiting; the real backend candidate remains owned by
|
||||
// the same operation throughout this synchronous call.
|
||||
pub(super) fn test_restore_checkpoint(
|
||||
&self,
|
||||
worker_ref: &WorkerRef,
|
||||
operation_id: WorkerLifecycleOperationId,
|
||||
stage: RestoreCheckpointStage,
|
||||
) {
|
||||
let config = {
|
||||
let Ok(mut state) = self.lock() else {
|
||||
return;
|
||||
};
|
||||
let slot = &mut state.restore_checkpoint;
|
||||
if !slot.as_ref().is_some_and(|config| {
|
||||
config.worker_id == worker_ref.worker_id && config.stage == stage
|
||||
}) {
|
||||
return;
|
||||
}
|
||||
// Consume even if canonical verification or the handshake fails.
|
||||
slot.take().expect("matched checkpoint")
|
||||
};
|
||||
let Ok(ack) = self.restore_checkpoint_ack(&config, worker_ref, operation_id) else {
|
||||
return;
|
||||
};
|
||||
wait_for_release(&config, &ack);
|
||||
}
|
||||
|
||||
pub(super) fn restore_checkpoint_ack(
|
||||
&self,
|
||||
config: &RestoreCheckpointConfig,
|
||||
worker_ref: &WorkerRef,
|
||||
operation_id: WorkerLifecycleOperationId,
|
||||
) -> Result<RestoreCheckpointAck, RuntimeError> {
|
||||
let invalid = || {
|
||||
RuntimeError::InvalidRequest("test restore checkpoint authority did not match".into())
|
||||
};
|
||||
let state = self.lock()?;
|
||||
let RuntimePersistence::Fs(store) = &state.persistence else {
|
||||
return Err(invalid());
|
||||
};
|
||||
// The existing helper validates the exact bounded canonical aggregate and
|
||||
// syncs that same file plus its owning directories. Never manufacture a
|
||||
// journal marker or infer commit from an in-memory projection.
|
||||
let record = store.read_durable_worker_record(worker_ref)?;
|
||||
let PersistedWorkerExecutionState::ReconciliationRequired(
|
||||
PersistedWorkerLifecycleOperation::Restore(pending),
|
||||
) = &record.execution_state
|
||||
else {
|
||||
return Err(invalid());
|
||||
};
|
||||
let expected = match config.stage {
|
||||
RestoreCheckpointStage::PreparedNotCommitted => RestoreCommitFact::NotCommitted,
|
||||
RestoreCheckpointStage::CommittedPreterminal
|
||||
| RestoreCheckpointStage::ActivatedPreterminal => RestoreCommitFact::Committed,
|
||||
};
|
||||
if config.worker_id != worker_ref.worker_id
|
||||
|| pending.operation_id != operation_id
|
||||
|| pending.commit != expected
|
||||
{
|
||||
return Err(invalid());
|
||||
}
|
||||
let guard = &record.restore_guard;
|
||||
let id = guard.active_request_id.as_ref().ok_or_else(invalid)?;
|
||||
let owner = guard.owners.get(id).ok_or_else(invalid)?;
|
||||
if owner.operation_id != operation_id
|
||||
|| owner.request.request_id != *id
|
||||
|| owner.receipt.is_some()
|
||||
{
|
||||
return Err(invalid());
|
||||
}
|
||||
let runtime_id = state.runtime_identity.clone().ok_or_else(invalid)?;
|
||||
Ok(RestoreCheckpointAck {
|
||||
nonce: config.nonce.clone(),
|
||||
stage: config.stage,
|
||||
runtime_id,
|
||||
worker_id: record.worker_id,
|
||||
operation_id,
|
||||
commit: match expected {
|
||||
RestoreCommitFact::NotCommitted => RestoreCheckpointCommit::NotCommitted,
|
||||
RestoreCommitFact::Committed => RestoreCheckpointCommit::Committed,
|
||||
RestoreCommitFact::LegacyUnknown => unreachable!(),
|
||||
},
|
||||
receipt_absent: true,
|
||||
pid: std::process::id(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
use super::*;
|
||||
use crate::restore_checkpoints::{
|
||||
RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage,
|
||||
};
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn checkpoint_stages_ack_canonical_decision_and_eof_resumes_same_operation_once() {
|
||||
// Component scope: durable Runtime with a substituted execution backend.
|
||||
// Real prepared Controllers and process death are owned by the E2E suite.
|
||||
for (stage, release) in [
|
||||
(RestoreCheckpointStage::PreparedNotCommitted, "release"),
|
||||
(RestoreCheckpointStage::CommittedPreterminal, "eof"),
|
||||
(RestoreCheckpointStage::CommittedPreterminal, "timeout"),
|
||||
(RestoreCheckpointStage::ActivatedPreterminal, "release"),
|
||||
] {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let backend = Arc::new(TestExecutionBackend::default());
|
||||
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||
FsRuntimeStoreOptions::new(dir.path().join("runtime"))
|
||||
.with_runtime_id("checkpoint-runtime"),
|
||||
backend.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
|
||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||
let worker = runtime
|
||||
.create_worker(task_request("checkpoint original operation"))
|
||||
.unwrap();
|
||||
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||
let request = runtime.test_restore_request(&worker.worker_ref);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
runtime
|
||||
.install_test_restore_checkpoint(RestoreCheckpointConfig {
|
||||
endpoint: listener.local_addr().unwrap(),
|
||||
worker_id: worker.worker_id,
|
||||
stage,
|
||||
nonce: "runtime-stage-test".into(),
|
||||
})
|
||||
.unwrap();
|
||||
let cloned = runtime.clone();
|
||||
let worker_ref = worker.worker_ref.clone();
|
||||
let sent_request = request.clone();
|
||||
let (sent, received) = std::sync::mpsc::channel();
|
||||
let task = std::thread::spawn(move || {
|
||||
sent.send(cloned.restore_worker_operation(&worker_ref, sent_request))
|
||||
.unwrap();
|
||||
});
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let stream = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(async {
|
||||
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
|
||||
let (stream, _) = tokio::time::timeout(Duration::from_secs(5), listener.accept())
|
||||
.await
|
||||
.expect("checkpoint did not connect")
|
||||
.unwrap();
|
||||
stream.into_std().unwrap()
|
||||
});
|
||||
stream.set_nonblocking(false).unwrap();
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(3)))
|
||||
.unwrap();
|
||||
let mut reader = BufReader::new(stream);
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).unwrap();
|
||||
let ack: RestoreCheckpointAck = serde_json::from_str(&line).unwrap();
|
||||
assert_eq!(ack.stage, stage);
|
||||
assert_eq!(ack.nonce, "runtime-stage-test");
|
||||
assert_eq!(ack.runtime_id, "checkpoint-runtime");
|
||||
assert_eq!(ack.worker_id, worker.worker_id);
|
||||
assert_eq!(ack.pid, std::process::id());
|
||||
assert!(ack.receipt_absent);
|
||||
assert_eq!(
|
||||
ack.commit,
|
||||
match stage {
|
||||
RestoreCheckpointStage::PreparedNotCommitted =>
|
||||
RestoreCheckpointCommit::NotCommitted,
|
||||
RestoreCheckpointStage::CommittedPreterminal
|
||||
| RestoreCheckpointStage::ActivatedPreterminal => RestoreCheckpointCommit::Committed,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
backend.restore_operation_ids.lock().unwrap().as_slice(),
|
||||
&[ack.operation_id]
|
||||
);
|
||||
if stage == RestoreCheckpointStage::ActivatedPreterminal {
|
||||
assert_eq!(backend.activation_operation_ids.lock().unwrap().as_slice(), &[ack.operation_id]);
|
||||
} else {
|
||||
assert!(backend.activation_operation_ids.lock().unwrap().is_empty());
|
||||
}
|
||||
assert!(received.try_recv().is_err());
|
||||
let canonical = runtime_store(&runtime)
|
||||
.read_durable_worker_record(&worker.worker_ref)
|
||||
.unwrap();
|
||||
let id = canonical.restore_guard.active_request_id.as_ref().unwrap();
|
||||
assert_eq!(id, &request.request_id);
|
||||
assert!(canonical.restore_guard.owners[id].receipt.is_none());
|
||||
// Release, EOF and timeout all resume the original lifecycle owner.
|
||||
let start = std::time::Instant::now();
|
||||
if release == "release" {
|
||||
reader.get_mut().write_all(b"R").unwrap();
|
||||
} else if release == "eof" {
|
||||
reader
|
||||
.get_mut()
|
||||
.shutdown(std::net::Shutdown::Write)
|
||||
.unwrap();
|
||||
}
|
||||
let result = received
|
||||
.recv_timeout(Duration::from_secs(12))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(start.elapsed() < Duration::from_secs(11));
|
||||
task.join().unwrap();
|
||||
assert_eq!(result.state, WorkerRestoreState::Accepted);
|
||||
assert_eq!(
|
||||
backend.activation_operation_ids.lock().unwrap().as_slice(),
|
||||
&[ack.operation_id]
|
||||
);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.restore_worker_operation(&worker.worker_ref, request)
|
||||
.unwrap(),
|
||||
result
|
||||
);
|
||||
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
|
||||
assert!(runtime.lock().unwrap().restore_checkpoint.is_none());
|
||||
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||
let next = runtime.test_restore_request(&worker.worker_ref);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.restore_worker_operation(&worker.worker_ref, next)
|
||||
.unwrap()
|
||||
.state,
|
||||
WorkerRestoreState::Accepted
|
||||
);
|
||||
assert_eq!(*backend.restore_count.lock().unwrap(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_disabled_mismatched_or_unreachable_does_not_change_restore_result() {
|
||||
for mode in ["disabled", "mismatch", "unreachable"] {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let backend = Arc::new(TestExecutionBackend::default());
|
||||
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||
FsRuntimeStoreOptions::new(dir.path().join("runtime"))
|
||||
.with_runtime_id("checkpoint-runtime"),
|
||||
backend.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
|
||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||
let worker = runtime
|
||||
.create_worker(task_request("checkpoint absent"))
|
||||
.unwrap();
|
||||
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||
let request = runtime.test_restore_request(&worker.worker_ref);
|
||||
if mode != "disabled" {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let endpoint = listener.local_addr().unwrap();
|
||||
drop(listener);
|
||||
runtime
|
||||
.install_test_restore_checkpoint(RestoreCheckpointConfig {
|
||||
endpoint,
|
||||
worker_id: if mode == "mismatch" {
|
||||
WorkerId::now_v7()
|
||||
} else {
|
||||
worker.worker_id
|
||||
},
|
||||
stage: RestoreCheckpointStage::PreparedNotCommitted,
|
||||
nonce: "absent-test".into(),
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
let result = runtime
|
||||
.restore_worker_operation(&worker.worker_ref, request.clone())
|
||||
.unwrap();
|
||||
assert_eq!(result.state, WorkerRestoreState::Accepted, "{mode}");
|
||||
assert_eq!(
|
||||
runtime
|
||||
.restore_worker_operation(&worker.worker_ref, request)
|
||||
.unwrap(),
|
||||
result
|
||||
);
|
||||
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
|
||||
assert_eq!(
|
||||
runtime.lock().unwrap().restore_checkpoint.is_some(),
|
||||
mode == "mismatch"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_verification_requires_canonical_worker_operation_commit_and_absent_receipt() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dir.path().join("runtime");
|
||||
let backend = Arc::new(TestExecutionBackend::default());
|
||||
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||
FsRuntimeStoreOptions::new(&root).with_runtime_id("checkpoint-runtime"),
|
||||
backend.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
|
||||
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||
let worker = runtime
|
||||
.create_worker(task_request("checkpoint validation"))
|
||||
.unwrap();
|
||||
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||
let request = runtime.test_restore_request(&worker.worker_ref);
|
||||
*backend.restore_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::Errored(
|
||||
WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, "not yet connected"),
|
||||
));
|
||||
assert_eq!(
|
||||
runtime
|
||||
.restore_worker_operation(&worker.worker_ref, request.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
WorkerRestoreState::ReconciliationRequired
|
||||
);
|
||||
let operation = backend.restore_operation_ids.lock().unwrap()[0];
|
||||
let mut config = RestoreCheckpointConfig {
|
||||
endpoint: "127.0.0.1:1234".parse().unwrap(),
|
||||
worker_id: worker.worker_id,
|
||||
stage: RestoreCheckpointStage::PreparedNotCommitted,
|
||||
nonce: "verification-test".into(),
|
||||
};
|
||||
let path = root
|
||||
.join("workers")
|
||||
.join(worker.worker_id.to_string())
|
||||
.join("worker.json");
|
||||
let before = std::fs::read(&path).unwrap();
|
||||
assert!(
|
||||
runtime
|
||||
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.restore_checkpoint_ack(
|
||||
&config,
|
||||
&worker.worker_ref,
|
||||
WorkerLifecycleOperationId::new()
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
config.worker_id = WorkerId::now_v7();
|
||||
assert!(
|
||||
runtime
|
||||
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||
.is_err()
|
||||
);
|
||||
config.worker_id = worker.worker_id;
|
||||
config.stage = RestoreCheckpointStage::CommittedPreterminal;
|
||||
assert!(
|
||||
runtime
|
||||
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||
.is_err()
|
||||
);
|
||||
config.stage = RestoreCheckpointStage::ActivatedPreterminal;
|
||||
assert!(runtime.restore_checkpoint_ack(&config, &worker.worker_ref, operation).is_err());
|
||||
config.stage = RestoreCheckpointStage::PreparedNotCommitted;
|
||||
runtime_store(&runtime).fail_next_worker_reconcile_sync();
|
||||
assert!(
|
||||
runtime
|
||||
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(&path).unwrap(),
|
||||
before,
|
||||
"verification must not write authority"
|
||||
);
|
||||
// In-memory equality must not hide an unreadable canonical aggregate.
|
||||
std::fs::write(&path, b"unreadable canonical aggregate").unwrap();
|
||||
assert!(
|
||||
runtime
|
||||
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(&path).unwrap(),
|
||||
b"unreadable canonical aggregate"
|
||||
);
|
||||
std::fs::write(&path, before).unwrap();
|
||||
backend.restore_result.lock().unwrap().take();
|
||||
assert_eq!(
|
||||
runtime
|
||||
.restore_worker_operation(&worker.worker_ref, request)
|
||||
.unwrap()
|
||||
.state,
|
||||
WorkerRestoreState::Accepted
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||
.is_err(),
|
||||
"terminal operation cannot ack a preterminal checkpoint"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,522 @@
|
||||
//! Runtime-owned continuation of an admitted restore, independent of its caller.
|
||||
//!
|
||||
//! Completion callbacks only signal a condition variable. They never acquire a
|
||||
//! Runtime/lifecycle lock and are never joined by this driver while holding one.
|
||||
//! Each step revalidates the canonical owner under the existing Worker lock;
|
||||
//! waiting/backoff and driver teardown happen outside it. A repeated obstruction
|
||||
//! parks this owner with its reason, rather than spinning or inventing rollback.
|
||||
|
||||
use super::*;
|
||||
use crate::execution::RestoreRecoveryDisposition;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Condvar, Weak};
|
||||
use std::thread::JoinHandle;
|
||||
use std::time::Duration;
|
||||
|
||||
type Key = (WorkerRef, WorkerLifecycleOperationId);
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct RestoreContinuations {
|
||||
owners: Mutex<HashMap<Key, Arc<Continuation>>>,
|
||||
#[cfg(test)]
|
||||
paused: std::sync::atomic::AtomicBool,
|
||||
#[cfg(test)]
|
||||
pub(super) terminal_receipt_gate:
|
||||
Mutex<Option<crate::execution::RestoreTestGate<WorkerLifecycleOperationId>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Schedule {
|
||||
signaled: bool,
|
||||
cancelled: bool,
|
||||
finished: bool,
|
||||
blocked_reason: Option<String>,
|
||||
parked: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Continuation {
|
||||
schedule: Mutex<Schedule>,
|
||||
wake: Condvar,
|
||||
task: Mutex<Option<JoinHandle<()>>>,
|
||||
}
|
||||
|
||||
impl RestoreContinuations {
|
||||
pub(super) fn signal(&self, worker_ref: &WorkerRef, operation: WorkerLifecycleOperationId) {
|
||||
if let Some(owner) = self
|
||||
.owners
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(&(worker_ref.clone(), operation))
|
||||
{
|
||||
let mut schedule = owner
|
||||
.schedule
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
schedule.signaled = true;
|
||||
owner.wake.notify_all();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RestoreContinuations {
|
||||
fn drop(&mut self) {
|
||||
let owners = self
|
||||
.owners
|
||||
.get_mut()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for owner in owners.values() {
|
||||
let mut schedule = owner
|
||||
.schedule
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
schedule.cancelled = true;
|
||||
owner.wake.notify_all();
|
||||
}
|
||||
for owner in owners.values() {
|
||||
if let Some(task) = owner
|
||||
.task
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.take()
|
||||
{
|
||||
if task.thread().id() != std::thread::current().id() {
|
||||
let _ = task.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct ContinuationPause(Arc<RestoreContinuations>);
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for ContinuationPause {
|
||||
fn drop(&mut self) {
|
||||
self.0
|
||||
.paused
|
||||
.store(false, std::sync::atomic::Ordering::Release);
|
||||
for owner in self.0.owners.lock().unwrap().values() {
|
||||
owner.schedule.lock().unwrap().signaled = true;
|
||||
owner.wake.notify_all();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum Step {
|
||||
Finished,
|
||||
Waiting,
|
||||
Contended,
|
||||
Retry(String),
|
||||
}
|
||||
|
||||
impl Runtime {
|
||||
pub(super) fn restore_repair_authorizer(
|
||||
&self,
|
||||
worker_ref: WorkerRef,
|
||||
) -> Arc<dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync> {
|
||||
let inner = Arc::downgrade(&self.inner);
|
||||
Arc::new(move |operation| {
|
||||
let inner = inner.upgrade().ok_or(RuntimeError::RuntimeStopped)?;
|
||||
// The factory caller owns the lifecycle lock. Never reacquire it
|
||||
// here: it may be awaiting this exact factory task.
|
||||
let mut state = inner.lock().map_err(|_| RuntimeError::StatePoisoned)?;
|
||||
state.ensure_running()?;
|
||||
state.refresh_uncertain_worker_write(&worker_ref)?;
|
||||
let worker = state.worker(&worker_ref)?;
|
||||
state.ensure_subject_not_reserved(&worker_ref, false)?;
|
||||
if worker.pending_stop.is_some()
|
||||
|| worker.pending_restore.is_none_or(|pending| {
|
||||
pending.operation_id != operation
|
||||
|| pending.commit == RestoreCommitFact::LegacyUnknown
|
||||
})
|
||||
{
|
||||
return Err(RuntimeError::RestoreObservationConflict {
|
||||
worker_id: worker_ref.worker_id,
|
||||
});
|
||||
}
|
||||
if let Some(owner) = worker
|
||||
.restore_guard
|
||||
.active_request_id
|
||||
.as_ref()
|
||||
.and_then(|id| worker.restore_guard.owners.get(id))
|
||||
{
|
||||
owner.validate_workdir_preparation_authority(worker)?;
|
||||
}
|
||||
if worker
|
||||
.pending_restore
|
||||
.is_some_and(|pending| pending.source_repair_possible)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let mut candidate = worker.clone();
|
||||
candidate
|
||||
.pending_restore
|
||||
.as_mut()
|
||||
.expect("validated restore owner")
|
||||
.source_repair_possible = true;
|
||||
// Grant before touching another canonical store. Unknown durability
|
||||
// denies repair until authoritative reconciliation; no old-memory write.
|
||||
state.persist_restore_worker_record(&candidate)?;
|
||||
state.workers.insert(worker_ref.worker_id, candidate);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test_wait_restore_continuation_parked(
|
||||
&self,
|
||||
worker_ref: &WorkerRef,
|
||||
operation: WorkerLifecycleOperationId,
|
||||
timeout: Duration,
|
||||
) -> String {
|
||||
let owner = self
|
||||
.restore_continuations
|
||||
.owners
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(&(worker_ref.clone(), operation))
|
||||
.cloned()
|
||||
.unwrap();
|
||||
let (schedule, waited) = owner
|
||||
.wake
|
||||
.wait_timeout_while(owner.schedule.lock().unwrap(), timeout, |s| {
|
||||
!s.parked && !s.finished
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
!waited.timed_out() && schedule.parked,
|
||||
"continuation did not park: {:?}",
|
||||
schedule.blocked_reason
|
||||
);
|
||||
schedule.blocked_reason.clone().expect("parked reason")
|
||||
}
|
||||
|
||||
/// Gate autonomous steps when a component test observes a single fault
|
||||
/// boundary. Dedicated continuation tests run without this gate.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test_pause_restore_continuation(&self) -> ContinuationPause {
|
||||
assert!(
|
||||
!self
|
||||
.restore_continuations
|
||||
.paused
|
||||
.swap(true, std::sync::atomic::Ordering::AcqRel)
|
||||
);
|
||||
ContinuationPause(self.restore_continuations.clone())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test_wait_restore_continuation(
|
||||
&self,
|
||||
worker_ref: &WorkerRef,
|
||||
operation: WorkerLifecycleOperationId,
|
||||
timeout: Duration,
|
||||
) {
|
||||
let owner = self
|
||||
.restore_continuations
|
||||
.owners
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(&(worker_ref.clone(), operation))
|
||||
.cloned()
|
||||
.expect("continuation owner");
|
||||
let schedule = owner.schedule.lock().unwrap();
|
||||
let (schedule, waited) = owner
|
||||
.wake
|
||||
.wait_timeout_while(schedule, timeout, |s| !s.finished && !s.cancelled)
|
||||
.unwrap();
|
||||
assert!(
|
||||
!waited.timed_out() && schedule.finished,
|
||||
"restore continuation did not settle: {:?}",
|
||||
schedule.blocked_reason
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn ensure_restore_continuation(
|
||||
&self,
|
||||
worker_ref: &WorkerRef,
|
||||
) -> Result<(), RuntimeError> {
|
||||
let operation = {
|
||||
let state = self.lock()?;
|
||||
let worker = state.worker(worker_ref)?;
|
||||
worker
|
||||
.pending_restore
|
||||
.map(|pending| pending.operation_id)
|
||||
.or_else(|| {
|
||||
worker
|
||||
.restore_guard
|
||||
.active_request_id
|
||||
.as_ref()
|
||||
.and_then(|id| worker.restore_guard.owners.get(id))
|
||||
.filter(|owner| owner.receipt.is_none())
|
||||
.map(|owner| owner.operation_id)
|
||||
})
|
||||
};
|
||||
if let Some(operation) = operation {
|
||||
self.start_restore_continuation(worker_ref.clone(), operation)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn start_restore_continuation(
|
||||
&self,
|
||||
worker_ref: WorkerRef,
|
||||
operation: WorkerLifecycleOperationId,
|
||||
) -> Result<(), RuntimeError> {
|
||||
{
|
||||
let state = self.lock()?;
|
||||
let Some(backend) = &state.execution_backend else {
|
||||
return Ok(());
|
||||
};
|
||||
if !backend.supports_restore_continuation()
|
||||
|| backend
|
||||
.restore_recovery_disposition(operation, &worker_ref)
|
||||
.is_none()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
let key = (worker_ref.clone(), operation);
|
||||
let mut owners = self
|
||||
.restore_continuations
|
||||
.owners
|
||||
.lock()
|
||||
.map_err(|_| RuntimeError::StatePoisoned)?;
|
||||
if owners.contains_key(&key) {
|
||||
return Ok(());
|
||||
}
|
||||
let owner = Arc::new(Continuation::default());
|
||||
let retained = owner.clone();
|
||||
let state = Arc::downgrade(&self.inner);
|
||||
let operations = Arc::downgrade(&self.worker_operations);
|
||||
let continuations = Arc::downgrade(&self.restore_continuations);
|
||||
let task = std::thread::Builder::new()
|
||||
.name("restore-continuation".into())
|
||||
.spawn(move || {
|
||||
run(
|
||||
state,
|
||||
operations,
|
||||
continuations,
|
||||
retained,
|
||||
worker_ref,
|
||||
operation,
|
||||
);
|
||||
})
|
||||
.map_err(|error| {
|
||||
RuntimeError::InvalidRequest(format!(
|
||||
"Restore continuation could not start: {error}"
|
||||
))
|
||||
})?;
|
||||
*owner.task.lock().map_err(|_| RuntimeError::StatePoisoned)? = Some(task);
|
||||
owners.insert(key, owner);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn continue_restore_operation(
|
||||
&self,
|
||||
worker_ref: &WorkerRef,
|
||||
operation: WorkerLifecycleOperationId,
|
||||
) -> Result<Step, RuntimeError> {
|
||||
#[cfg(test)]
|
||||
if self
|
||||
.restore_continuations
|
||||
.paused
|
||||
.load(std::sync::atomic::Ordering::Acquire)
|
||||
{
|
||||
return Ok(Step::Contended);
|
||||
}
|
||||
let lease = self.worker_operation_lock(worker_ref.worker_id)?;
|
||||
// Another caller may be waiting for a backend task. Do not queue a join
|
||||
// of that caller/driver while retaining the lock it needs to complete.
|
||||
let _guard = match lease.try_lock() {
|
||||
Ok(guard) => guard,
|
||||
Err(std::sync::TryLockError::WouldBlock) => return Ok(Step::Contended),
|
||||
Err(_) => return Err(RuntimeError::StatePoisoned),
|
||||
};
|
||||
let stop_is_owned = {
|
||||
let mut state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
state.refresh_uncertain_worker_write(worker_ref)?;
|
||||
let worker = state.worker(worker_ref)?;
|
||||
worker.pending_stop.is_some()
|
||||
&& worker
|
||||
.pending_restore
|
||||
.is_some_and(|pending| pending.operation_id == operation)
|
||||
};
|
||||
if stop_is_owned {
|
||||
// Stop supersedes publication, not ownership of late resources.
|
||||
// Retain this same continuation until its cleanup is confirmed.
|
||||
self.reconcile_pending_stop_under_lock(worker_ref)?;
|
||||
return Ok(Step::Finished);
|
||||
}
|
||||
let (backend, pending) = {
|
||||
let mut state = self.lock()?;
|
||||
state.ensure_running()?;
|
||||
state.refresh_uncertain_worker_write(worker_ref)?;
|
||||
let worker = state.worker(worker_ref)?;
|
||||
if worker.pending_stop.is_some() {
|
||||
return Ok(Step::Finished);
|
||||
}
|
||||
let owner = worker
|
||||
.restore_guard
|
||||
.active_request_id
|
||||
.as_ref()
|
||||
.and_then(|id| worker.restore_guard.owners.get(id));
|
||||
if owner.is_some_and(|owner| owner.operation_id != operation || owner.receipt.is_some())
|
||||
{
|
||||
return Ok(Step::Finished);
|
||||
}
|
||||
let Some(pending) = worker.pending_restore else {
|
||||
return Ok(Step::Finished);
|
||||
};
|
||||
if pending.operation_id != operation {
|
||||
return Ok(Step::Finished);
|
||||
}
|
||||
if pending.commit == RestoreCommitFact::LegacyUnknown {
|
||||
return Ok(Step::Retry(
|
||||
"Saved Restore has no authoritative commit evidence".into(),
|
||||
));
|
||||
}
|
||||
state.ensure_subject_not_reserved(worker_ref, false)?;
|
||||
if let Some(owner) = owner {
|
||||
owner.validate_workdir_preparation_authority(worker)?;
|
||||
}
|
||||
let Some(backend) = state.execution_backend.clone() else {
|
||||
return Ok(Step::Finished);
|
||||
};
|
||||
(backend, pending)
|
||||
};
|
||||
match backend.restore_recovery_disposition(operation, worker_ref) {
|
||||
None => return Ok(Step::Finished),
|
||||
Some(RestoreRecoveryDisposition::InFlight) => return Ok(Step::Waiting),
|
||||
Some(RestoreRecoveryDisposition::Discard)
|
||||
if pending.commit == RestoreCommitFact::NotCommitted =>
|
||||
{
|
||||
match backend.discard_uncommitted_restore(operation, worker_ref) {
|
||||
WorkerExecutionSpawnResult::RolledBack(result) => {
|
||||
let receipt = RuntimeWorkerRestoreResult::failed(
|
||||
WorkerRestoreState::RolledBack,
|
||||
"worker_restore_rolled_back",
|
||||
result.message_or_default(),
|
||||
);
|
||||
#[cfg(feature = "fs-store")]
|
||||
self.lock()?
|
||||
.record_restore_failure(worker_ref, &result, &receipt)?;
|
||||
#[cfg(not(feature = "fs-store"))]
|
||||
self.settle_restore_rollback(worker_ref, &receipt)?;
|
||||
return Ok(Step::Finished);
|
||||
}
|
||||
result => {
|
||||
return Ok(Step::Retry(format!(
|
||||
"Restore cleanup remains unresolved: {result:?}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(_) => {}
|
||||
}
|
||||
let result = self.restore_admitted_worker_under_lock(worker_ref, pending.mode)?;
|
||||
if backend.restore_recovery_disposition(operation, worker_ref)
|
||||
== Some(RestoreRecoveryDisposition::InFlight)
|
||||
{
|
||||
return Ok(Step::Waiting);
|
||||
}
|
||||
if result.state == WorkerRestoreState::ReconciliationRequired {
|
||||
Ok(Step::Retry(result.message.unwrap_or_else(|| {
|
||||
"Restore reconciliation remains pending".into()
|
||||
})))
|
||||
} else {
|
||||
Ok(Step::Finished)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run(
|
||||
inner: Weak<Mutex<RuntimeState>>,
|
||||
operations: Weak<Mutex<BTreeMap<WorkerId, Arc<Mutex<()>>>>>,
|
||||
continuations: Weak<RestoreContinuations>,
|
||||
owner: Arc<Continuation>,
|
||||
worker_ref: WorkerRef,
|
||||
operation: WorkerLifecycleOperationId,
|
||||
) {
|
||||
let mut repeated = 0u32;
|
||||
let mut last_reason = None;
|
||||
loop {
|
||||
let (Some(inner), Some(worker_operations), Some(restore_continuations)) = (
|
||||
inner.upgrade(),
|
||||
operations.upgrade(),
|
||||
continuations.upgrade(),
|
||||
) else {
|
||||
break;
|
||||
};
|
||||
owner
|
||||
.schedule
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.parked = false;
|
||||
let runtime = Runtime {
|
||||
inner,
|
||||
worker_operations,
|
||||
restore_continuations,
|
||||
};
|
||||
let step = runtime.continue_restore_operation(&worker_ref, operation);
|
||||
// No lifecycle lock or strong Runtime/registry reference survives waiting.
|
||||
drop(runtime);
|
||||
let mut schedule = owner
|
||||
.schedule
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if schedule.cancelled {
|
||||
break;
|
||||
}
|
||||
let contended = matches!(&step, Ok(Step::Contended));
|
||||
let reason = match step {
|
||||
Ok(Step::Finished) => {
|
||||
schedule.finished = true;
|
||||
owner.wake.notify_all();
|
||||
break;
|
||||
}
|
||||
Ok(Step::Waiting) | Ok(Step::Contended) => None,
|
||||
Ok(Step::Retry(reason)) => Some(reason),
|
||||
Err(error) => Some(sanitize_worker_create_failure_message(&error.to_string())),
|
||||
};
|
||||
if let Some(reason) = &reason {
|
||||
repeated = repeated.saturating_add(1);
|
||||
last_reason = Some(reason.clone());
|
||||
schedule.blocked_reason = Some(reason.clone());
|
||||
}
|
||||
// Completed failed attempts can have signaled while this step joined
|
||||
// them. Do not let our own unchanged failure reset the retry budget.
|
||||
if reason.is_some() {
|
||||
schedule.signaled = false;
|
||||
}
|
||||
if (!contended && reason.is_none()) || repeated >= 4 {
|
||||
if repeated >= 4 {
|
||||
schedule.parked = true;
|
||||
owner.wake.notify_all();
|
||||
tracing::warn!(worker_id = %worker_ref.worker_id, operation_id = %operation, reason = ?last_reason,
|
||||
"Restore continuation parked after bounded attempts; operation remains unresolved");
|
||||
}
|
||||
schedule = owner
|
||||
.wake
|
||||
.wait_while(schedule, |s| !s.signaled && !s.cancelled)
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
} else {
|
||||
let delay = Duration::from_millis(100 * (1u64 << repeated.min(4)));
|
||||
schedule = owner
|
||||
.wake
|
||||
.wait_timeout_while(schedule, delay, |s| !s.signaled && !s.cancelled)
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.0;
|
||||
}
|
||||
if schedule.cancelled {
|
||||
break;
|
||||
}
|
||||
if schedule.signaled {
|
||||
schedule.signaled = false;
|
||||
// A completion wakes this same owner, not a new request. Its next
|
||||
// observed result, not the signal itself, determines progress.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,872 @@
|
||||
//! Offline support for the approved, attachment-free stopped Worker fixture.
|
||||
//!
|
||||
//! Enabled only by `test-support`. This is not an import API: extraction accepts a
|
||||
//! current-schema, settled, single-Segment aggregate; installation accepts only a
|
||||
//! fresh Runtime directory containing its newly initialized `auth.toml`. No
|
||||
//! Runtime or execution backend is constructed. Original Session bytes are never
|
||||
//! rewritten. The caller must separately review/attest the artificial conversation;
|
||||
//! structural checks and secret-marker checks cannot prove arbitrary prose safe.
|
||||
|
||||
use super::*;
|
||||
use crate::catalog::ProfileSourceArchiveSource;
|
||||
use crate::workspace_issuer::{
|
||||
WorkspaceIssuerTrustRecord, WorkspaceIssuerTrustState, validate_workspace_issuer_trust_records,
|
||||
};
|
||||
use session_store::{
|
||||
SegmentId, SessionId, WorkerActiveSegmentRef, WorkerMetadata, WorkerSessionStore,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "test_support_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
const TEMPLATE_VERSION: u32 = 1;
|
||||
const MAX_SESSION_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// Portable fixture, intentionally without Debug (Session material is private).
|
||||
/// Fields are private and every deserialized template is validated on install.
|
||||
/// Serialize this value to retain the fixture; do not serialize its source aggregate.
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct StoppedWorkerTemplate {
|
||||
template_version: u32,
|
||||
runtime_schema_version: u32,
|
||||
runtime_id: String,
|
||||
workspace_id: String,
|
||||
worker_id: WorkerId,
|
||||
created_at_ms: Option<u64>,
|
||||
session_id: SessionId,
|
||||
segment_id: SegmentId,
|
||||
session_manifest: Vec<u8>,
|
||||
segment_jsonl: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SessionManifest {
|
||||
schema_version: u32,
|
||||
session_id: SessionId,
|
||||
}
|
||||
|
||||
// Read-only projection: never retain source key material in a template.
|
||||
#[derive(Deserialize)]
|
||||
struct AuthIdentity {
|
||||
identity_id: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SourceAuth {
|
||||
identity: AuthIdentity,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct InitializedAuth {
|
||||
identity: crate::auth::RuntimeIdentityMaterial,
|
||||
workspace_issuers: Vec<WorkspaceIssuerTrustRecord>,
|
||||
#[serde(default)]
|
||||
archived_legacy_issuers: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GuardState {
|
||||
active_request_id: Option<String>,
|
||||
owners: BTreeMap<String, GuardOwner>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GuardOwner {
|
||||
receipt: Option<serde::de::IgnoredAny>,
|
||||
}
|
||||
|
||||
fn reject(message: &'static str) -> RuntimeError {
|
||||
RuntimeError::InvalidRequest(format!("stopped Worker fixture: {message}"))
|
||||
}
|
||||
|
||||
fn io<T>(result: std::io::Result<T>) -> Result<T, RuntimeError> {
|
||||
result.map_err(|_| reject("filesystem operation failed"))
|
||||
}
|
||||
|
||||
fn decode<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> Result<T, RuntimeError> {
|
||||
serde_json::from_slice(bytes).map_err(|_| reject("invalid typed document"))
|
||||
}
|
||||
|
||||
fn no_symlink_components(path: &Path) -> Result<(), RuntimeError> {
|
||||
let mut current = PathBuf::new();
|
||||
for part in path.components() {
|
||||
if matches!(part, std::path::Component::ParentDir) {
|
||||
return Err(reject("parent traversal is unsupported"));
|
||||
}
|
||||
current.push(part);
|
||||
let metadata = io(fs::symlink_metadata(¤t))?;
|
||||
if metadata.file_type().is_symlink() {
|
||||
return Err(reject("symlinks are unsupported"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_tree(path: &Path, budget: &mut usize) -> Result<(), RuntimeError> {
|
||||
if *budget == 0 {
|
||||
return Err(reject("file count exceeds fixture limit"));
|
||||
}
|
||||
*budget -= 1;
|
||||
let metadata = io(fs::symlink_metadata(path))?;
|
||||
if metadata.is_dir() {
|
||||
for entry in io(fs::read_dir(path))? {
|
||||
check_tree(&io(entry)?.path(), budget)?;
|
||||
}
|
||||
} else if !metadata.is_file() {
|
||||
return Err(reject("symlinks and special files are unsupported"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn regular_file(path: &Path) -> Result<(), RuntimeError> {
|
||||
no_symlink_components(path)?;
|
||||
if !io(fs::symlink_metadata(path))?.is_file() {
|
||||
return Err(reject("expected a regular file"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn bytes(path: &Path) -> Result<Vec<u8>, RuntimeError> {
|
||||
regular_file(path)?;
|
||||
let mut options = OpenOptions::new();
|
||||
options.read(true);
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
options.custom_flags(libc::O_NOATIME | libc::O_NOFOLLOW);
|
||||
}
|
||||
let file = io(options.open(path))?;
|
||||
let mut result = Vec::new();
|
||||
io(file
|
||||
.take(MAX_SESSION_BYTES as u64 + 1)
|
||||
.read_to_end(&mut result))?;
|
||||
if result.len() > MAX_SESSION_BYTES {
|
||||
return Err(reject("file exceeds fixture limit"));
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn text(path: &Path) -> Result<String, RuntimeError> {
|
||||
String::from_utf8(bytes(path)?).map_err(|_| reject("invalid UTF-8 document"))
|
||||
}
|
||||
|
||||
fn portable_id(id: &str) -> bool {
|
||||
!id.is_empty()
|
||||
&& id.len() <= 256
|
||||
&& id
|
||||
.bytes()
|
||||
.all(|c| c.is_ascii_alphanumeric() || b"-_".contains(&c))
|
||||
}
|
||||
|
||||
impl StoppedWorkerTemplate {
|
||||
pub fn runtime_id(&self) -> &str {
|
||||
&self.runtime_id
|
||||
}
|
||||
pub fn workspace_id(&self) -> &str {
|
||||
&self.workspace_id
|
||||
}
|
||||
pub fn worker_id(&self) -> WorkerId {
|
||||
self.worker_id
|
||||
}
|
||||
pub fn worker_name(&self) -> String {
|
||||
runtime_worker_name(self.worker_id)
|
||||
}
|
||||
pub fn session_id(&self) -> SessionId {
|
||||
self.session_id
|
||||
}
|
||||
pub fn segment_id(&self) -> SegmentId {
|
||||
self.segment_id
|
||||
}
|
||||
|
||||
/// Extract without migration, rewriting metadata, or copying credentials,
|
||||
/// resolved manifests, execution-operation receipts, or run directories.
|
||||
/// The source Runtime must be offline; its existing owner lock is held for
|
||||
/// the entire read. The path must be `<runtime>/workers/<WorkerId>`.
|
||||
pub fn extract(worker_dir: &Path) -> Result<Self, RuntimeError> {
|
||||
no_symlink_components(worker_dir)?;
|
||||
let workers = worker_dir
|
||||
.parent()
|
||||
.ok_or_else(|| reject("invalid aggregate path"))?;
|
||||
if workers.file_name().and_then(|s| s.to_str()) != Some(WORKERS_DIR) {
|
||||
return Err(reject("invalid aggregate path"));
|
||||
}
|
||||
let root = workers
|
||||
.parent()
|
||||
.ok_or_else(|| reject("invalid Runtime path"))?;
|
||||
// Require the existing lock, rather than creating a file in the source.
|
||||
let lock_path = root.with_file_name(format!(
|
||||
".{}.runtime-owner.lock",
|
||||
root.file_name()
|
||||
.and_then(|s| s.to_str())
|
||||
.ok_or_else(|| reject("invalid Runtime path"))?
|
||||
));
|
||||
regular_file(&lock_path)?;
|
||||
let lock = io(OpenOptions::new().read(true).open(lock_path))?;
|
||||
if !io(FileExt::try_lock_shared(&lock))? {
|
||||
return Err(reject("source Runtime is not offline"));
|
||||
}
|
||||
let _lock = RuntimeStoreOwnerLock { file: lock };
|
||||
check_tree(worker_dir, &mut 256)?;
|
||||
let snapshot: RuntimeSnapshot = decode(&bytes(&root.join(RUNTIME_FILE))?)?;
|
||||
snapshot
|
||||
.validate(&root.join(RUNTIME_FILE))
|
||||
.map_err(|_| reject("unsupported Runtime schema"))?;
|
||||
if !snapshot.subject_create_recovery_fences.is_empty()
|
||||
|| !snapshot.subject_releases.is_empty()
|
||||
{
|
||||
return Err(reject("source has lifecycle operations"));
|
||||
}
|
||||
let record: WorkerAggregateRecord = decode(&bytes(&worker_dir.join(WORKER_FILE))?)?;
|
||||
let record = record
|
||||
.validate(&worker_dir.join(WORKER_FILE))
|
||||
.map_err(|_| reject("invalid Worker schema or identity"))?;
|
||||
let guard: GuardState = decode(
|
||||
&serde_json::to_vec(&record.restore_guard)
|
||||
.map_err(|_| reject("invalid restore guard"))?,
|
||||
)?;
|
||||
if record.status != WorkerStatus::Stopped
|
||||
|| guard.active_request_id.is_some()
|
||||
|| guard.owners.values().any(|owner| owner.receipt.is_none())
|
||||
{
|
||||
return Err(reject("Worker is not settled and stopped"));
|
||||
}
|
||||
let PersistedWorkerExecutionState::Available(execution) = &record.execution_state else {
|
||||
return Err(reject("pending or unavailable execution metadata"));
|
||||
};
|
||||
// A stopped aggregate may retain the empty historical binding marker.
|
||||
// It is deliberately not copied to the installed Worker.
|
||||
if execution.restore_intent != WorkerRestoreIntent::Explicit {
|
||||
return Err(reject("Worker requires explicit restore"));
|
||||
}
|
||||
if !record.workdir_attachments.is_empty() || !record.logical_workdir_attachments.is_empty()
|
||||
{
|
||||
return Err(reject("attachments are unsupported"));
|
||||
}
|
||||
check_request(&execution.request, false)?;
|
||||
let workspace_id = record
|
||||
.workspace_id
|
||||
.ok_or_else(|| reject("missing Workspace identity"))?;
|
||||
let metadata: WorkerMetadata = decode(&bytes(&worker_dir.join(WORKER_METADATA_FILE))?)?;
|
||||
crate::worker_backend::validate_fixture_restore_metadata(&metadata, &execution.request)
|
||||
.map_err(|_| reject("invalid source restore Manifest or settings"))?;
|
||||
if metadata.worker_name != runtime_worker_name(record.worker_id)
|
||||
|| metadata.workspace_id.as_deref() != Some(&workspace_id)
|
||||
|| !metadata.spawned_children.is_empty()
|
||||
|| !metadata.reclaimed_children.is_empty()
|
||||
|| !metadata.peers.is_empty()
|
||||
|| metadata.subjektiv_session_attribution.is_some()
|
||||
{
|
||||
return Err(reject("unsupported or inconsistent Worker metadata"));
|
||||
}
|
||||
let active = metadata
|
||||
.active
|
||||
.ok_or_else(|| reject("missing active Session"))?;
|
||||
let segment_id = active
|
||||
.segment_id
|
||||
.ok_or_else(|| reject("unmaterialized Session"))?;
|
||||
let source_auth: SourceAuth = toml::from_str(&text(&root.join("auth.toml"))?)
|
||||
.map_err(|_| reject("missing Runtime identity"))?;
|
||||
let session = worker_dir.join("session");
|
||||
let fixture = Self {
|
||||
template_version: TEMPLATE_VERSION,
|
||||
runtime_schema_version: SCHEMA_VERSION,
|
||||
runtime_id: source_auth.identity.identity_id,
|
||||
workspace_id,
|
||||
worker_id: record.worker_id,
|
||||
created_at_ms: record.created_at_ms,
|
||||
session_id: active.session_id,
|
||||
segment_id,
|
||||
session_manifest: bytes(&session.join("session.json"))?,
|
||||
segment_jsonl: bytes(&session.join("segments").join(format!("{segment_id}.jsonl")))?,
|
||||
};
|
||||
fixture.validate_shape()?;
|
||||
fixture.validate_session(&session)?;
|
||||
Ok(fixture)
|
||||
}
|
||||
|
||||
/// Install after normal identity/trust initialization, before Runtime start.
|
||||
/// `request` and `manifest` supply NEW backend/config/profile references and
|
||||
/// freshly resolved policy, not a launch. The saved source Manifest is never
|
||||
/// reused. `manifest.worker.name` must equal `worker_name()`.
|
||||
/// Only `auth.toml` may already exist. The owner lock fences live Runtimes;
|
||||
/// staged files are synced and published without overwriting any Worker.
|
||||
/// An interrupted publication fails closed as an incomplete Runtime layout.
|
||||
pub fn install(
|
||||
&self,
|
||||
options: &FsRuntimeStoreOptions,
|
||||
request: CreateWorkerRequest,
|
||||
manifest: &manifest::WorkerManifest,
|
||||
) -> Result<(), RuntimeError> {
|
||||
self.validate_shape()?;
|
||||
check_request(&request, true)?;
|
||||
if manifest.worker.name != self.worker_name() {
|
||||
return Err(reject("resolved Manifest Worker name mismatch"));
|
||||
}
|
||||
let mut metadata = WorkerMetadata::new(
|
||||
self.worker_name(),
|
||||
Some(WorkerActiveSegmentRef {
|
||||
session_id: self.session_id,
|
||||
segment_id: Some(self.segment_id),
|
||||
}),
|
||||
)
|
||||
.with_workspace_id(&self.workspace_id);
|
||||
metadata.resolved_manifest_snapshot = Some(
|
||||
manifest::write_persisted_worker_manifest_snapshot(manifest)
|
||||
.map_err(|_| reject("invalid resolved Manifest"))?,
|
||||
);
|
||||
crate::worker_backend::validate_fixture_restore_metadata(&metadata, &request)
|
||||
.map_err(|_| reject("resolved Manifest does not match fresh request settings"))?;
|
||||
if options.runtime_id != self.runtime_id
|
||||
|| request.worker_id != self.worker_id
|
||||
|| request
|
||||
.workspace_api
|
||||
.as_ref()
|
||||
.map(|api| api.workspace_id.as_str())
|
||||
!= Some(&self.workspace_id)
|
||||
{
|
||||
return Err(reject("installation identity mismatch"));
|
||||
}
|
||||
no_symlink_components(&options.root)?;
|
||||
check_tree(&options.root, &mut 256)?;
|
||||
// Validate the lock location before the ordinary owner-lock opener.
|
||||
let lock_path = options.root.with_file_name(format!(
|
||||
".{}.runtime-owner.lock",
|
||||
options
|
||||
.root
|
||||
.file_name()
|
||||
.and_then(|s| s.to_str())
|
||||
.ok_or_else(|| reject("invalid Runtime path"))?
|
||||
));
|
||||
if fs::symlink_metadata(&lock_path).is_ok() {
|
||||
regular_file(&lock_path)?;
|
||||
}
|
||||
let lock = acquire_runtime_store_owner_lock(&options.root)?;
|
||||
validate_fresh_runtime_store_layout(&options.root)
|
||||
.map_err(|_| reject("destination is not fresh; refusing overwrite"))?;
|
||||
let auth: InitializedAuth = toml::from_str(&text(&options.root.join("auth.toml"))?)
|
||||
.map_err(|_| reject("initialize Runtime identity and trust before install"))?;
|
||||
if auth.identity.identity_id != self.runtime_id
|
||||
|| !auth.archived_legacy_issuers.is_empty()
|
||||
|| auth.identity.signing_key().is_err()
|
||||
{
|
||||
return Err(reject("invalid destination Runtime identity"));
|
||||
}
|
||||
validate_workspace_issuer_trust_records(&auth.workspace_issuers)
|
||||
.map_err(|_| reject("invalid destination Workspace trust"))?;
|
||||
let api = request
|
||||
.workspace_api
|
||||
.as_ref()
|
||||
.expect("validated Workspace binding");
|
||||
if !auth.workspace_issuers.iter().any(|trust| {
|
||||
trust.workspace_id == self.workspace_id
|
||||
&& trust.backend_url == api.base_url
|
||||
&& trust.state == WorkspaceIssuerTrustState::Active
|
||||
}) {
|
||||
return Err(reject("request does not match initialized Workspace trust"));
|
||||
}
|
||||
let stage = options
|
||||
.root
|
||||
.join(format!(".fixture-{}", uuid::Uuid::now_v7()));
|
||||
io(fs::create_dir(&stage))?;
|
||||
let store = FsRuntimeStore {
|
||||
root: stage.clone(),
|
||||
_owner_lock: Some(lock),
|
||||
worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)),
|
||||
#[cfg(test)]
|
||||
worker_reconcile_sync_fault: Default::default(),
|
||||
#[cfg(test)]
|
||||
worker_reconcile_read_fault: Default::default(),
|
||||
#[cfg(test)]
|
||||
worker_reconcile_counts: Default::default(),
|
||||
};
|
||||
let result = (|| {
|
||||
let worker = PersistedWorkerRecord {
|
||||
restore_guard: crate::runtime::RestoreGuard::default(),
|
||||
worker_ref: WorkerRef::new(self.worker_id),
|
||||
worker_id: self.worker_id,
|
||||
profile: request.profile.clone(),
|
||||
display_name: request.display_name.clone(),
|
||||
profile_source: request.profile_source.reference(),
|
||||
config_bundle: request.config_bundle.clone(),
|
||||
created_at_ms: self.created_at_ms,
|
||||
status: WorkerStatus::Stopped,
|
||||
last_finished_submission_request_id: None,
|
||||
workspace_id: Some(self.workspace_id.clone()),
|
||||
workdir_attachments: Vec::new(),
|
||||
logical_workdir_attachments: Vec::new(),
|
||||
execution_state: PersistedWorkerExecutionState::Available(
|
||||
PersistedWorkerExecution {
|
||||
request,
|
||||
binding: None,
|
||||
restore_intent: WorkerRestoreIntent::Explicit,
|
||||
},
|
||||
),
|
||||
};
|
||||
WorkerAggregateRecord::from_persisted(&worker)
|
||||
.validate(&store.worker_dir(&self.worker_id).join(WORKER_FILE))
|
||||
.map_err(|_| reject("invalid installation request"))?;
|
||||
store.write_worker_record(&worker)?;
|
||||
let worker_dir = store.worker_dir(&self.worker_id);
|
||||
atomic_write_json(
|
||||
&worker_dir.join(WORKER_METADATA_FILE),
|
||||
&metadata,
|
||||
"write fixture metadata",
|
||||
)?;
|
||||
let session = worker_dir.join("session");
|
||||
io(fs::create_dir_all(session.join("segments")))?;
|
||||
write_new(&session.join("session.json"), &self.session_manifest)?;
|
||||
write_new(
|
||||
&session
|
||||
.join("segments")
|
||||
.join(format!("{}.jsonl", self.segment_id)),
|
||||
&self.segment_jsonl,
|
||||
)?;
|
||||
self.validate_session(&session)?;
|
||||
store.write_runtime_snapshot(&PersistedRuntimeState {
|
||||
subject_create_recovery_fences: BTreeMap::new(),
|
||||
subject_releases: Vec::new(),
|
||||
// Match normal store initialization: this is catalog availability,
|
||||
// not a process or Worker execution start.
|
||||
display_name: options.display_name.clone(),
|
||||
status: RuntimeStatus::Running,
|
||||
next_diagnostic_id: 1,
|
||||
workers: BTreeMap::new(),
|
||||
workspace_owners: BTreeMap::new(),
|
||||
diagnostics: Vec::new(),
|
||||
})?;
|
||||
let loaded = store.load_runtime_state()?;
|
||||
if loaded.workers.len() != 1 || !loaded.diagnostics.is_empty() {
|
||||
return Err(reject("staged aggregate failed ordinary store roundtrip"));
|
||||
}
|
||||
sync_runtime_tree(&stage)?;
|
||||
// No destination existed when checked under the owner lock. Hard-link
|
||||
// the snapshot last with create-new semantics; never replace a file.
|
||||
// Reserve the destination catalog with create-new semantics. Even
|
||||
// an empty preexisting workers directory must never be replaced.
|
||||
let destination_workers = options.root.join(WORKERS_DIR);
|
||||
io(fs::create_dir(&destination_workers))?;
|
||||
io(fs::rename(
|
||||
store.worker_dir(&self.worker_id),
|
||||
destination_workers.join(self.worker_id.to_string()),
|
||||
))?;
|
||||
sync_directory(&destination_workers, "publish stopped Worker aggregate")?;
|
||||
io(fs::hard_link(
|
||||
stage.join(RUNTIME_FILE),
|
||||
options.root.join(RUNTIME_FILE),
|
||||
))?;
|
||||
sync_directory(&options.root, "publish stopped fixture")?;
|
||||
Ok(())
|
||||
})();
|
||||
// Only remove our private staging directory, never existing destination data.
|
||||
let cleanup = fs::remove_dir_all(&stage);
|
||||
result?;
|
||||
io(cleanup)?;
|
||||
sync_directory(&options.root, "finish stopped fixture")
|
||||
}
|
||||
|
||||
fn validate_shape(&self) -> Result<(), RuntimeError> {
|
||||
// Version 1 templates carry only settled Session material, never a
|
||||
// lifecycle operation. Schema 12 -> 13 changes pending decision evidence,
|
||||
// not that payload. Accept the fixed schema-12 artifact read-only and
|
||||
// install through the current aggregate owner; never rewrite its bytes.
|
||||
if self.template_version != TEMPLATE_VERSION
|
||||
|| !matches!(
|
||||
self.runtime_schema_version,
|
||||
RESTORE_ADMISSION_SCHEMA_VERSION | SCHEMA_VERSION
|
||||
)
|
||||
{
|
||||
return Err(reject("unsupported template schema"));
|
||||
}
|
||||
if !portable_id(&self.runtime_id)
|
||||
|| !portable_id(&self.workspace_id)
|
||||
|| self.session_id.get_version_num() != 7
|
||||
|| self.segment_id.get_version_num() != 7
|
||||
|| self.session_manifest.len() > MAX_SESSION_BYTES
|
||||
|| self.segment_jsonl.len() > MAX_SESSION_BYTES
|
||||
{
|
||||
return Err(reject("invalid portable identity or size"));
|
||||
}
|
||||
let manifest: SessionManifest = decode(&self.session_manifest)?;
|
||||
if manifest.session_id != self.session_id
|
||||
|| manifest.schema_version == 0
|
||||
|| !self.segment_jsonl.ends_with(b"\n")
|
||||
{
|
||||
return Err(reject("Session identity or committed boundary mismatch"));
|
||||
}
|
||||
// Read-only JSON traversal, never mutation/reconstruction of Session bytes.
|
||||
// Reject known secret/host markers even in otherwise valid typed records.
|
||||
for line in self
|
||||
.segment_jsonl
|
||||
.split(|b| *b == b'\n')
|
||||
.filter(|line| !line.is_empty())
|
||||
{
|
||||
let _: PortableJson = decode(line)?;
|
||||
validate_preserved_record(line)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_history_entry(
|
||||
&self,
|
||||
entry: &session_store::LoggedHistoryEntry,
|
||||
ids: &mut std::collections::BTreeSet<String>,
|
||||
) -> Result<(), RuntimeError> {
|
||||
if !portable_id(&entry.metadata.entry_id.0)
|
||||
|| !ids.insert(entry.metadata.entry_id.0.clone())
|
||||
{
|
||||
return Err(reject("duplicate or invalid entry identity"));
|
||||
}
|
||||
if entry
|
||||
.metadata
|
||||
.input_metadata
|
||||
.as_ref()
|
||||
.is_some_and(|metadata| {
|
||||
metadata.feature_payload().is_some() || metadata.public_item().is_some()
|
||||
})
|
||||
{
|
||||
// These are opaque owner-specific payloads, not part of the narrow
|
||||
// artificial conversation contract. Never preserve unchecked JSON.
|
||||
return Err(reject("Feature history payloads are unsupported"));
|
||||
}
|
||||
if !matches!(
|
||||
entry.item,
|
||||
session_store::LoggedItem::Message { .. }
|
||||
| session_store::LoggedItem::AssistantResponseBoundary { .. }
|
||||
) {
|
||||
return Err(reject("only text conversation records are supported"));
|
||||
}
|
||||
if let session_store::LoggedSessionHistoryOrigin::ModelOutput { worker } =
|
||||
&entry.metadata.origin
|
||||
{
|
||||
// Session producer IDs identify the original execution, not the
|
||||
// Workspace catalog Worker. Preserve them; do not rebind/rewrite them.
|
||||
if !portable_id(&worker.worker_id)
|
||||
|| worker
|
||||
.workspace_id
|
||||
.as_ref()
|
||||
.is_some_and(|id| id != &self.workspace_id)
|
||||
|| worker
|
||||
.runtime_id
|
||||
.as_ref()
|
||||
.is_some_and(|id| id != &self.runtime_id)
|
||||
{
|
||||
return Err(reject("Session origin identity mismatch"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_session(&self, root: &Path) -> Result<(), RuntimeError> {
|
||||
for entry in io(fs::read_dir(root))? {
|
||||
let entry = io(entry)?;
|
||||
match entry.file_name().to_str() {
|
||||
Some("session.json" | "segments") => {}
|
||||
Some("artifacts") => check_empty_artifacts(&entry.path())?,
|
||||
_ => return Err(reject("unsupported Session files")),
|
||||
}
|
||||
}
|
||||
let entries = io(fs::read_dir(root.join("segments")))?.collect::<Result<Vec<_>, _>>();
|
||||
let entries = io(entries)?;
|
||||
if entries.len() != 1
|
||||
|| entries[0].file_name() != format!("{}.jsonl", self.segment_id).as_str()
|
||||
{
|
||||
return Err(reject("only the fixed single-Segment fixture is supported"));
|
||||
}
|
||||
let store = WorkerSessionStore::open_read_only(root)
|
||||
.map_err(|_| reject("unsupported Session schema"))?;
|
||||
let logs = store
|
||||
.read_all_read_only_bounded(self.session_id, self.segment_id, MAX_SESSION_BYTES as u64)
|
||||
.map_err(|_| reject("invalid canonical Session records"))?
|
||||
.0;
|
||||
if !matches!(logs.first(), Some(session_store::LogEntry::AnnotatedSegmentStart {
|
||||
session_id, system_prompt: Some(_), forked_from: None, compacted_from: None, ..
|
||||
}) if *session_id == self.session_id)
|
||||
{
|
||||
return Err(reject(
|
||||
"missing system prompt, invalid Segment seed, or unsupported lineage",
|
||||
));
|
||||
}
|
||||
let mut ids = std::collections::BTreeSet::new();
|
||||
for (index, log) in logs.iter().enumerate() {
|
||||
use session_store::LogEntry;
|
||||
match log {
|
||||
LogEntry::AnnotatedSegmentStart { history, .. }
|
||||
if index == 0 && history.is_empty() => {}
|
||||
LogEntry::AnnotatedUserInput {
|
||||
history,
|
||||
extensions,
|
||||
..
|
||||
} => {
|
||||
for extension in extensions {
|
||||
validate_settled_activation(extension)?;
|
||||
}
|
||||
for entry in history {
|
||||
self.validate_history_entry(entry, &mut ids)?;
|
||||
}
|
||||
}
|
||||
LogEntry::AnnotatedAssistantItem { entry, .. } => {
|
||||
self.validate_history_entry(entry, &mut ids)?
|
||||
}
|
||||
LogEntry::RunCompleted {
|
||||
entry_id: Some(id), ..
|
||||
} => {
|
||||
if !portable_id(&id.0) || !ids.insert(id.0.clone()) {
|
||||
return Err(reject("duplicate or invalid entry identity"));
|
||||
}
|
||||
}
|
||||
LogEntry::Invoke { .. } | LogEntry::LlmUsage { .. } | LogEntry::TurnEnd { .. } => {}
|
||||
_ => {
|
||||
return Err(reject(
|
||||
"unsupported conversation record or pending operation",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
let state = session_store::collect_state(&logs);
|
||||
if state.last_run_interrupted || state.active_run_turn_count.is_some() {
|
||||
return Err(reject("Session has pending execution"));
|
||||
}
|
||||
for segments in &state.user_segments {
|
||||
if segments
|
||||
.iter()
|
||||
.any(|segment| !matches!(segment, protocol::Segment::Text { .. }))
|
||||
{
|
||||
return Err(reject("Session attachments are unsupported"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// Compare *fields*, not re-encoded bytes or scalar values (e.g. f32 JSON
|
||||
// normalization). Serde's permissive/untagged/flattened compatibility readers
|
||||
// can drop unknown data without surfacing it to an ignored-field callback. A
|
||||
// retained field must survive the owner's typed projection at every depth.
|
||||
// This intentionally accepts only canonical writer shapes: even an explicit
|
||||
// empty optional field omitted by that writer is rejected, not repaired.
|
||||
// The projection is validation-only; installation always writes original bytes.
|
||||
fn validate_preserved_record(bytes: &[u8]) -> Result<(), RuntimeError> {
|
||||
let original: serde_json::Value = decode(bytes)?;
|
||||
let record: session_store::LogEntry = decode(bytes)?;
|
||||
let projection =
|
||||
serde_json::to_value(&record).map_err(|_| reject("invalid typed Session record"))?;
|
||||
require_preserved_fields(&original, &projection)
|
||||
}
|
||||
|
||||
fn require_preserved_fields(
|
||||
original: &serde_json::Value,
|
||||
projection: &serde_json::Value,
|
||||
) -> Result<(), RuntimeError> {
|
||||
use serde_json::Value;
|
||||
match (original, projection) {
|
||||
(Value::Object(original), Value::Object(projection)) => {
|
||||
for (key, value) in original {
|
||||
let projected = projection
|
||||
.get(key)
|
||||
.ok_or_else(|| reject("unknown or noncanonical Session field"))?;
|
||||
require_preserved_fields(value, projected)?;
|
||||
}
|
||||
}
|
||||
(Value::Array(original), Value::Array(projection))
|
||||
if original.len() == projection.len() =>
|
||||
{
|
||||
for (value, projected) in original.iter().zip(projection) {
|
||||
require_preserved_fields(value, projected)?;
|
||||
}
|
||||
}
|
||||
(Value::Null, Value::Null)
|
||||
| (Value::Bool(_), Value::Bool(_))
|
||||
| (Value::Number(_), Value::Number(_))
|
||||
| (Value::String(_), Value::String(_)) => {}
|
||||
_ => return Err(reject("noncanonical Session field shape")),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Exact receipt wire fields owned by worker::PendingActivationState. Keep these
|
||||
// fixture-only mirrors strict; do not change the production compatibility reader.
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SubmissionReceipt {
|
||||
submission_request_id: String,
|
||||
source_namespace: String,
|
||||
submission_id: String,
|
||||
payload_digest: String,
|
||||
disposition: protocol::SubmissionDisposition,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct NotificationReceipt {
|
||||
notification_request_id: String,
|
||||
source_namespace: String,
|
||||
payload_digest: String,
|
||||
}
|
||||
|
||||
// The approved conversation has committed submission checkpoints in its original
|
||||
// Session records. Keep their receipt bytes, but reject any resumable queue or
|
||||
// Feature invocation history. This projection confers no execution authority.
|
||||
fn validate_settled_activation(
|
||||
extension: &session_store::SessionExtension,
|
||||
) -> Result<(), RuntimeError> {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Checkpoint {
|
||||
#[serde(rename = "next_activation_sequence")]
|
||||
_next_activation_sequence: u64,
|
||||
activating: Option<serde::de::IgnoredAny>,
|
||||
activating_notification: Vec<serde::de::IgnoredAny>,
|
||||
pending: Vec<serde::de::IgnoredAny>,
|
||||
pending_notifications: Vec<serde::de::IgnoredAny>,
|
||||
#[serde(rename = "receipts")]
|
||||
_receipts: Vec<SubmissionReceipt>,
|
||||
#[serde(rename = "notification_receipts")]
|
||||
_notification_receipts: Vec<NotificationReceipt>,
|
||||
invocation_receipts: Vec<serde::de::IgnoredAny>,
|
||||
invocation_recovery_blocked: bool,
|
||||
}
|
||||
if extension.domain != "worker.pending_activations.v1" {
|
||||
return Err(reject("unsupported Session extension"));
|
||||
}
|
||||
let checkpoint: Checkpoint = serde_json::from_value(extension.payload.clone())
|
||||
.map_err(|_| reject("invalid Session activation checkpoint"))?;
|
||||
if checkpoint.activating.is_some()
|
||||
|| !checkpoint.activating_notification.is_empty()
|
||||
|| !checkpoint.pending.is_empty()
|
||||
|| !checkpoint.pending_notifications.is_empty()
|
||||
|| !checkpoint.invocation_receipts.is_empty()
|
||||
|| checkpoint.invocation_recovery_blocked
|
||||
{
|
||||
return Err(reject(
|
||||
"Session has pending activation or invocation history",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_request(request: &CreateWorkerRequest, install: bool) -> Result<(), RuntimeError> {
|
||||
if !request.workdir_attachment_requests.is_empty()
|
||||
|| !request.workdir_attachments.is_empty()
|
||||
|| request.backend_job.is_some()
|
||||
|| request.subjektiv_attached
|
||||
|| request.worker_observation_enabled
|
||||
|| !request.worker_observation_grants.is_empty()
|
||||
|| (install && request.initial_input.is_some())
|
||||
{
|
||||
return Err(reject(
|
||||
"unsupported capabilities, attachments, or initial input",
|
||||
));
|
||||
}
|
||||
if install
|
||||
&& (request.create_fingerprint.trim().is_empty()
|
||||
|| request.config_bundle.is_none()
|
||||
|| !matches!(
|
||||
request.profile_source,
|
||||
ProfileSourceArchiveSource::WorkspaceConfig { .. }
|
||||
))
|
||||
{
|
||||
return Err(reject(
|
||||
"fresh config/profile references and create fingerprint are required",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_empty_artifacts(root: &Path) -> Result<(), RuntimeError> {
|
||||
for entry in io(fs::read_dir(root))? {
|
||||
let entry = io(entry)?;
|
||||
let metadata = io(entry.metadata())?;
|
||||
if metadata.is_dir() {
|
||||
check_empty_artifacts(&entry.path())?;
|
||||
} else if entry.file_name() != ".aggregate.lock" || metadata.len() != 0 {
|
||||
return Err(reject("Session artifacts are unsupported"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_new(path: &Path, bytes: &[u8]) -> Result<(), RuntimeError> {
|
||||
let mut file = io(OpenOptions::new().write(true).create_new(true).open(path))?;
|
||||
io(file.write_all(bytes))?;
|
||||
io(file.sync_all())
|
||||
}
|
||||
|
||||
// A streaming validation visitor, not a generic document available to callers.
|
||||
struct PortableJson;
|
||||
impl<'de> Deserialize<'de> for PortableJson {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct Visitor;
|
||||
impl<'de> serde::de::Visitor<'de> for Visitor {
|
||||
type Value = PortableJson;
|
||||
fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("portable fixture JSON")
|
||||
}
|
||||
fn visit_bool<E: serde::de::Error>(self, _: bool) -> Result<PortableJson, E> {
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_i64<E: serde::de::Error>(self, _: i64) -> Result<PortableJson, E> {
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_u64<E: serde::de::Error>(self, _: u64) -> Result<PortableJson, E> {
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_f64<E: serde::de::Error>(self, _: f64) -> Result<PortableJson, E> {
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_unit<E: serde::de::Error>(self) -> Result<PortableJson, E> {
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<PortableJson, E> {
|
||||
let lower = value.to_ascii_lowercase();
|
||||
if [
|
||||
"://",
|
||||
"private key",
|
||||
"bearer ",
|
||||
"sk-",
|
||||
"api_key",
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
"private_key",
|
||||
"authorization",
|
||||
]
|
||||
.iter()
|
||||
.any(|marker| lower.contains(marker))
|
||||
|| value
|
||||
.split_whitespace()
|
||||
.any(|word| word.starts_with('/') || word.contains(":\\"))
|
||||
{
|
||||
return Err(E::custom("nonportable or sensitive fixture material"));
|
||||
}
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_seq<A: serde::de::SeqAccess<'de>>(
|
||||
self,
|
||||
mut seq: A,
|
||||
) -> Result<PortableJson, A::Error> {
|
||||
while seq.next_element::<PortableJson>()?.is_some() {}
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_map<A: serde::de::MapAccess<'de>>(
|
||||
self,
|
||||
mut map: A,
|
||||
) -> Result<PortableJson, A::Error> {
|
||||
let mut keys = std::collections::BTreeSet::new();
|
||||
while let Some(key) = map.next_key::<String>()? {
|
||||
// A duplicate key can hide discarded data even inside an
|
||||
// otherwise opaque extension payload. Never retain it.
|
||||
if !keys.insert(key.clone()) {
|
||||
return Err(serde::de::Error::custom("duplicate fixture JSON field"));
|
||||
}
|
||||
serde::de::Visitor::visit_str::<A::Error>(Visitor, &key)?;
|
||||
map.next_value::<PortableJson>()?;
|
||||
}
|
||||
Ok(PortableJson)
|
||||
}
|
||||
}
|
||||
deserializer.deserialize_any(Visitor)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,919 @@
|
||||
use super::*;
|
||||
use crate::auth::RuntimeIdentityMaterial;
|
||||
use crate::catalog::{ProfileSelector, WorkspaceApiRef};
|
||||
use crate::profile_archive::ProfileSourceGraphSummary;
|
||||
use session_store::{LogEntry, Store};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const RUNTIME: &str = "fixture-runtime";
|
||||
const WORKSPACE: &str = "fixture-workspace";
|
||||
const NEW_URL: &str = "http://127.0.0.1:34567";
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(untagged)]
|
||||
enum ReceiptFixture<T> {
|
||||
Valid(T),
|
||||
Malformed(u8),
|
||||
}
|
||||
|
||||
#[derive(Default, Serialize)]
|
||||
struct CheckpointFixture {
|
||||
next_activation_sequence: u64,
|
||||
activating: Option<u8>,
|
||||
activating_notification: Vec<u8>,
|
||||
pending: Vec<u8>,
|
||||
pending_notifications: Vec<u8>,
|
||||
receipts: Vec<ReceiptFixture<SubmissionReceipt>>,
|
||||
notification_receipts: Vec<ReceiptFixture<NotificationReceipt>>,
|
||||
invocation_receipts: Vec<u8>,
|
||||
invocation_recovery_blocked: bool,
|
||||
}
|
||||
|
||||
fn settled_checkpoint() -> CheckpointFixture {
|
||||
CheckpointFixture {
|
||||
next_activation_sequence: 2,
|
||||
receipts: vec![ReceiptFixture::Valid(SubmissionReceipt {
|
||||
submission_request_id: "completed-submission-request".into(),
|
||||
source_namespace: "fixture-source".into(),
|
||||
submission_id: "completed-submission".into(),
|
||||
payload_digest: "fixture-payload-digest".into(),
|
||||
disposition: protocol::SubmissionDisposition::Started,
|
||||
})],
|
||||
notification_receipts: vec![ReceiptFixture::Valid(NotificationReceipt {
|
||||
notification_request_id: "completed-notification-request".into(),
|
||||
source_namespace: "fixture-source".into(),
|
||||
payload_digest: "fixture-notification-digest".into(),
|
||||
})],
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn checkpoint_extension(checkpoint: &CheckpointFixture) -> session_store::SessionExtension {
|
||||
session_store::SessionExtension::new(
|
||||
"worker.pending_activations.v1",
|
||||
serde_json::to_value(checkpoint).unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
fn worker_id() -> WorkerId {
|
||||
"01a1657d-b762-7791-87ae-a49f28cadbce".parse().unwrap()
|
||||
}
|
||||
|
||||
fn resolved() -> manifest::WorkerManifest {
|
||||
let mut config = manifest::WorkerManifestConfig::resolution_defaults();
|
||||
config.worker.name = Some(runtime_worker_name(worker_id()));
|
||||
manifest::WorkerManifest::try_from(config).unwrap()
|
||||
}
|
||||
|
||||
fn request() -> CreateWorkerRequest {
|
||||
CreateWorkerRequest {
|
||||
worker_id: worker_id(),
|
||||
create_fingerprint: "new-create-fingerprint".into(),
|
||||
profile: ProfileSelector::Builtin("default".into()),
|
||||
display_name: Some("fixture".into()),
|
||||
profile_source: ProfileSourceArchiveSource::WorkspaceConfig {
|
||||
archive: ProfileSourceArchiveRef {
|
||||
id: "new-profile".into(),
|
||||
digest: "new-profile-digest".into(),
|
||||
size_bytes: 1,
|
||||
source_graph: ProfileSourceGraphSummary {
|
||||
source_count: 1,
|
||||
total_source_bytes: 1,
|
||||
entrypoints: BTreeMap::from([("builtin:default".into(), "default.dcl".into())]),
|
||||
import_count: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
config_bundle: Some(ConfigBundleRef {
|
||||
id: "new-config".into(),
|
||||
digest: "new-config-digest".into(),
|
||||
}),
|
||||
workspace_api: Some(WorkspaceApiRef {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
base_url: NEW_URL.into(),
|
||||
}),
|
||||
initial_input: None,
|
||||
workdir_attachment_requests: Vec::new(),
|
||||
workdir_attachments: Vec::new(),
|
||||
worker_observation_enabled: false,
|
||||
worker_observation_grants: Vec::new(),
|
||||
memory_settings: Some(manifest::WorkspaceMemorySettingsSnapshot {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
language: "en".into(),
|
||||
}),
|
||||
subjektiv_attached: false,
|
||||
backend_job: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn initialize(root: &Path, runtime_id: &str, workspace_id: &str, url: &str) {
|
||||
#[derive(Serialize)]
|
||||
struct Auth {
|
||||
identity: RuntimeIdentityMaterial,
|
||||
workspace_issuers: Vec<WorkspaceIssuerTrustRecord>,
|
||||
}
|
||||
let issuer = RuntimeIdentityMaterial::generate("fixture-issuer").unwrap();
|
||||
let digest = Sha256::digest(crate::auth::decode_public_key(&issuer.public_key).unwrap());
|
||||
let fingerprint = format!(
|
||||
"sha256:{}",
|
||||
digest
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect::<String>()
|
||||
);
|
||||
let trust = WorkspaceIssuerTrustRecord::from_bundle(
|
||||
server_api::WorkspacePublicIdentityBundle {
|
||||
workspace_id: workspace_id.into(),
|
||||
backend_url: url.into(),
|
||||
key_id: "fixture-issuer".into(),
|
||||
algorithm: "ed25519".into(),
|
||||
public_key: issuer.public_key,
|
||||
public_key_fingerprint: fingerprint,
|
||||
},
|
||||
"fixture-trust".into(),
|
||||
100,
|
||||
)
|
||||
.unwrap();
|
||||
fs::create_dir_all(root).unwrap();
|
||||
fs::write(
|
||||
root.join("auth.toml"),
|
||||
toml::to_string(&Auth {
|
||||
identity: RuntimeIdentityMaterial::generate(runtime_id).unwrap(),
|
||||
workspace_issuers: vec![trust],
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn record(request: CreateWorkerRequest) -> PersistedWorkerRecord {
|
||||
PersistedWorkerRecord {
|
||||
restore_guard: crate::runtime::RestoreGuard::default(),
|
||||
worker_ref: WorkerRef::new(request.worker_id),
|
||||
worker_id: request.worker_id,
|
||||
profile: request.profile.clone(),
|
||||
display_name: request.display_name.clone(),
|
||||
profile_source: request.profile_source.reference(),
|
||||
config_bundle: request.config_bundle.clone(),
|
||||
created_at_ms: Some(123456),
|
||||
status: WorkerStatus::Stopped,
|
||||
last_finished_submission_request_id: Some("old-submission-operation".into()),
|
||||
workspace_id: Some(WORKSPACE.into()),
|
||||
workdir_attachments: vec![],
|
||||
logical_workdir_attachments: vec![],
|
||||
execution_state: PersistedWorkerExecutionState::Available(PersistedWorkerExecution {
|
||||
request,
|
||||
binding: Some(PersistedWorkerExecutionBinding {}),
|
||||
restore_intent: WorkerRestoreIntent::Explicit,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn source() -> (tempfile::TempDir, PathBuf, StoppedWorkerTemplate) {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
initialize(&root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456");
|
||||
drop(
|
||||
crate::Runtime::with_fs_store(FsRuntimeStoreOptions::new(&root).with_runtime_id(RUNTIME))
|
||||
.unwrap(),
|
||||
);
|
||||
let opened = FsRuntimeStore::open_or_create(root.clone(), RUNTIME).unwrap();
|
||||
let mut old = request();
|
||||
old.workspace_api.as_mut().unwrap().base_url = "http://127.0.0.1:23456".into();
|
||||
old.config_bundle.as_mut().unwrap().id = "old-config".into();
|
||||
opened.store.write_worker_record(&record(old)).unwrap();
|
||||
let worker = opened.store.worker_dir(&worker_id());
|
||||
let session_id = "01a1657d-b800-7777-8888-111111111111".parse().unwrap();
|
||||
let segment_id = "01a1657d-b818-77c1-bd73-176172a4bd82".parse().unwrap();
|
||||
let mut metadata = WorkerMetadata::new(
|
||||
runtime_worker_name(worker_id()),
|
||||
Some(WorkerActiveSegmentRef {
|
||||
session_id,
|
||||
segment_id: Some(segment_id),
|
||||
}),
|
||||
)
|
||||
.with_workspace_id(WORKSPACE)
|
||||
.with_workspace_root(PathBuf::from("/old/host/path"));
|
||||
metadata.resolved_manifest_snapshot =
|
||||
Some(manifest::write_persisted_worker_manifest_snapshot(&resolved()).unwrap());
|
||||
atomic_write_json(
|
||||
&worker.join(WORKER_METADATA_FILE),
|
||||
&metadata,
|
||||
"test metadata",
|
||||
)
|
||||
.unwrap();
|
||||
let sessions = WorkerSessionStore::new(worker.join("session")).unwrap();
|
||||
let history = session_store::LoggedHistoryEntry {
|
||||
item: session_store::LoggedItem::Message {
|
||||
role: session_store::LoggedRole::Assistant,
|
||||
content: vec![session_store::LoggedContentPart::Text {
|
||||
text: "artificial fixture reply".into(),
|
||||
}],
|
||||
},
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId("fixed-entry-id".into()),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::ModelOutput {
|
||||
worker: session_store::LoggedWorkerSubject {
|
||||
workspace_id: Some(WORKSPACE.into()),
|
||||
runtime_id: Some(RUNTIME.into()),
|
||||
worker_id: worker_id().to_string(),
|
||||
},
|
||||
},
|
||||
derivation: None,
|
||||
input_metadata: None,
|
||||
},
|
||||
};
|
||||
sessions
|
||||
.create_segment(
|
||||
session_id,
|
||||
segment_id,
|
||||
&[
|
||||
LogEntry::AnnotatedSegmentStart {
|
||||
ts: 123,
|
||||
session_id,
|
||||
system_prompt: Some("Harmless artificial fixture instructions.".into()),
|
||||
config: Default::default(),
|
||||
history: vec![],
|
||||
forked_from: None,
|
||||
compacted_from: None,
|
||||
},
|
||||
LogEntry::Invoke {
|
||||
ts: 124,
|
||||
trigger: protocol::InvokeKind::UserSend,
|
||||
},
|
||||
LogEntry::AnnotatedUserInput {
|
||||
ts: 124,
|
||||
segments: vec![protocol::Segment::text(
|
||||
"Harmless artificial fixture input.",
|
||||
)],
|
||||
extensions: vec![checkpoint_extension(&settled_checkpoint())],
|
||||
history: vec![session_store::LoggedHistoryEntry {
|
||||
item: session_store::LoggedItem::Message {
|
||||
role: session_store::LoggedRole::User,
|
||||
content: vec![session_store::LoggedContentPart::Text {
|
||||
text: "Harmless artificial fixture input.".into(),
|
||||
}],
|
||||
},
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId(
|
||||
"fixed-user-id".into(),
|
||||
),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::BackendInstruction {
|
||||
operation_id: None,
|
||||
},
|
||||
derivation: None,
|
||||
input_metadata: None,
|
||||
},
|
||||
}],
|
||||
},
|
||||
LogEntry::AnnotatedAssistantItem {
|
||||
ts: 125,
|
||||
entry: history,
|
||||
},
|
||||
LogEntry::LlmUsage {
|
||||
ts: 125,
|
||||
history_len: 2,
|
||||
input_total_tokens: 1,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
output_tokens: 1,
|
||||
},
|
||||
LogEntry::TurnEnd {
|
||||
ts: 126,
|
||||
turn_count: 1,
|
||||
},
|
||||
LogEntry::RunCompleted {
|
||||
entry_id: Some(session_store::LoggedSessionHistoryEntryId(
|
||||
"fixed-terminal-id".into(),
|
||||
)),
|
||||
ts: 126,
|
||||
interrupted: false,
|
||||
result: agen::EngineResult::Finished,
|
||||
active_run_turn_count: None,
|
||||
},
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
fs::create_dir_all(worker.join("runs/old-run")).unwrap();
|
||||
fs::write(worker.join("runs/old-run/private.log"), "not exported").unwrap();
|
||||
drop(opened);
|
||||
let template = StoppedWorkerTemplate::extract(&worker).unwrap();
|
||||
(temp, worker, template)
|
||||
}
|
||||
|
||||
fn destination(template: &StoppedWorkerTemplate) -> (tempfile::TempDir, FsRuntimeStoreOptions) {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
initialize(
|
||||
&root,
|
||||
template.runtime_id(),
|
||||
template.workspace_id(),
|
||||
NEW_URL,
|
||||
);
|
||||
(
|
||||
temp,
|
||||
FsRuntimeStoreOptions::new(root).with_runtime_id(template.runtime_id()),
|
||||
)
|
||||
}
|
||||
|
||||
fn assert_fresh(options: &FsRuntimeStoreOptions) {
|
||||
assert_eq!(fs::read_dir(&options.root).unwrap().count(), 1);
|
||||
assert!(options.root.join("auth.toml").is_file());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_install_roundtrip_preserves_bytes_and_is_inactive() {
|
||||
let (_source, worker, template) = source();
|
||||
let original_worker = fs::read(worker.join(WORKER_FILE)).unwrap();
|
||||
let original_metadata = fs::read(worker.join(WORKER_METADATA_FILE)).unwrap();
|
||||
let exported = serde_json::to_vec(&template).unwrap();
|
||||
let exported_text = String::from_utf8(exported.clone()).unwrap();
|
||||
for forbidden in [
|
||||
"old-config",
|
||||
"old-submission-operation",
|
||||
"restore_guard",
|
||||
"private_key",
|
||||
"http://",
|
||||
"/old/host",
|
||||
"private.log",
|
||||
"resolved_manifest_snapshot",
|
||||
] {
|
||||
assert!(!exported_text.contains(forbidden));
|
||||
}
|
||||
let template: StoppedWorkerTemplate = serde_json::from_slice(&exported).unwrap();
|
||||
let (_target, options) = destination(&template);
|
||||
let auth_before = fs::read(options.root.join("auth.toml")).unwrap();
|
||||
template.install(&options, request(), &resolved()).unwrap();
|
||||
assert_eq!(
|
||||
fs::read(options.root.join("auth.toml")).unwrap(),
|
||||
auth_before
|
||||
);
|
||||
assert_eq!(fs::read(worker.join(WORKER_FILE)).unwrap(), original_worker);
|
||||
assert_eq!(
|
||||
fs::read(worker.join(WORKER_METADATA_FILE)).unwrap(),
|
||||
original_metadata
|
||||
);
|
||||
let installed = options
|
||||
.root
|
||||
.join(WORKERS_DIR)
|
||||
.join(template.worker_id().to_string());
|
||||
assert_eq!(
|
||||
fs::read(installed.join("session/session.json")).unwrap(),
|
||||
template.session_manifest
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(installed.join(format!("session/segments/{}.jsonl", template.segment_id())))
|
||||
.unwrap(),
|
||||
template.segment_jsonl
|
||||
);
|
||||
assert!(!installed.join("runs").exists());
|
||||
let metadata: WorkerMetadata =
|
||||
decode(&fs::read(installed.join(WORKER_METADATA_FILE)).unwrap()).unwrap();
|
||||
assert_eq!(metadata.active.unwrap().session_id, template.session_id());
|
||||
assert!(metadata.workspace_root.is_none() && metadata.resolved_manifest_snapshot.is_some());
|
||||
|
||||
let opened = FsRuntimeStore::open_or_create(options.root.clone(), RUNTIME).unwrap();
|
||||
let state = opened.state.unwrap();
|
||||
assert!(state.diagnostics.is_empty());
|
||||
let saved = state.workers.get(&worker_id()).unwrap();
|
||||
assert_eq!(saved.status, WorkerStatus::Stopped);
|
||||
assert!(saved.last_finished_submission_request_id.is_none());
|
||||
let PersistedWorkerExecutionState::Available(execution) = &saved.execution_state else {
|
||||
panic!("not available")
|
||||
};
|
||||
assert_eq!(execution.restore_intent, WorkerRestoreIntent::Explicit);
|
||||
assert!(execution.binding.is_none());
|
||||
assert_eq!(execution.request, request());
|
||||
// Fresh restore guard, no operation owner/receipt history.
|
||||
assert_ne!(
|
||||
serde_json::to_vec(&saved.restore_guard).unwrap(),
|
||||
serde_json::to_vec(
|
||||
&decode::<WorkerAggregateRecord>(&original_worker)
|
||||
.unwrap()
|
||||
.restore_guard
|
||||
)
|
||||
.unwrap()
|
||||
);
|
||||
let guard: GuardState = decode(&serde_json::to_vec(&saved.restore_guard).unwrap()).unwrap();
|
||||
assert!(guard.active_request_id.is_none() && guard.owners.is_empty());
|
||||
drop(opened.store);
|
||||
|
||||
struct NoExecution;
|
||||
impl crate::execution::WorkerExecutionBackend for NoExecution {
|
||||
fn backend_id(&self) -> &str {
|
||||
"fixture-no-execution"
|
||||
}
|
||||
fn spawn_worker(
|
||||
&self,
|
||||
_: crate::execution::WorkerExecutionSpawnRequest,
|
||||
) -> crate::execution::WorkerExecutionSpawnResult {
|
||||
panic!("fixture executed")
|
||||
}
|
||||
fn dispatch_input(
|
||||
&self,
|
||||
_: &WorkerRef,
|
||||
_: crate::interaction::WorkerInput,
|
||||
) -> crate::execution::WorkerExecutionResult {
|
||||
panic!("fixture submitted")
|
||||
}
|
||||
fn restore_worker(
|
||||
&self,
|
||||
_: crate::execution::WorkerExecutionRestoreRequest,
|
||||
) -> crate::execution::WorkerExecutionSpawnResult {
|
||||
panic!("fixture auto-restored")
|
||||
}
|
||||
}
|
||||
let runtime =
|
||||
crate::Runtime::with_fs_store_and_execution_backend(options.clone(), Arc::new(NoExecution))
|
||||
.unwrap();
|
||||
assert_eq!(runtime.list_workers().unwrap().len(), 1);
|
||||
assert_eq!(runtime.summary().unwrap().active_worker_count, 0);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.worker_detail(&WorkerRef::new(worker_id()))
|
||||
.unwrap()
|
||||
.status,
|
||||
WorkerStatus::Stopped
|
||||
);
|
||||
drop(runtime);
|
||||
let runtime = crate::Runtime::with_fs_store(options).unwrap();
|
||||
assert_eq!(
|
||||
runtime.list_workers().unwrap()[0].status,
|
||||
WorkerStatus::Stopped
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn install_rejects_schema_identity_input_and_overwrite() {
|
||||
let (_source, _, template) = source();
|
||||
for change in [
|
||||
"template",
|
||||
"runtime-schema",
|
||||
"session-schema",
|
||||
"session-id",
|
||||
"segment-id",
|
||||
"runtime-id",
|
||||
"workspace-id",
|
||||
"worker-id",
|
||||
"origin-id",
|
||||
"duplicate-id",
|
||||
"truncated",
|
||||
] {
|
||||
let (_target, options) = destination(&template);
|
||||
let mut invalid = template.clone();
|
||||
match change {
|
||||
"template" => invalid.template_version += 1,
|
||||
"runtime-schema" => invalid.runtime_schema_version += 1,
|
||||
"session-schema" => {
|
||||
invalid.session_manifest = format!(
|
||||
"{{\"schema_version\":999,\"session_id\":\"{}\"}}",
|
||||
invalid.session_id
|
||||
)
|
||||
.into_bytes()
|
||||
}
|
||||
"session-id" => invalid.session_id = uuid::Uuid::now_v7(),
|
||||
"segment-id" => invalid.segment_id = uuid::Uuid::nil(), // no external attestation: valid new Segment ID is otherwise portable
|
||||
"runtime-id" => invalid.runtime_id = "other-runtime".into(),
|
||||
"workspace-id" => invalid.workspace_id = "other-workspace".into(),
|
||||
"worker-id" => invalid.worker_id = WorkerId::now_v7(),
|
||||
"origin-id" => {
|
||||
invalid.segment_jsonl = String::from_utf8(invalid.segment_jsonl)
|
||||
.unwrap()
|
||||
.replace(RUNTIME, "other-runtime")
|
||||
.into_bytes()
|
||||
}
|
||||
"duplicate-id" => {
|
||||
let last = invalid
|
||||
.segment_jsonl
|
||||
.split_inclusive(|b| *b == b'\n')
|
||||
.last()
|
||||
.unwrap()
|
||||
.to_vec();
|
||||
invalid.segment_jsonl.extend(last);
|
||||
}
|
||||
"truncated" => {
|
||||
invalid.segment_jsonl.pop();
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert!(
|
||||
invalid.install(&options, request(), &resolved()).is_err(),
|
||||
"accepted {change}"
|
||||
);
|
||||
assert_fresh(&options);
|
||||
}
|
||||
let (_target, options) = destination(&template);
|
||||
let mut launch = request();
|
||||
launch.initial_input = Some(crate::interaction::WorkerInput::user("do not execute"));
|
||||
assert!(template.install(&options, launch, &resolved()).is_err());
|
||||
assert_fresh(&options);
|
||||
template.install(&options, request(), &resolved()).unwrap();
|
||||
let worker_path = options
|
||||
.root
|
||||
.join(WORKERS_DIR)
|
||||
.join(worker_id().to_string())
|
||||
.join(WORKER_FILE);
|
||||
let before = fs::read(&worker_path).unwrap();
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
assert_eq!(fs::read(worker_path).unwrap(), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_uninitialized_untrusted_live_and_partial_destination() {
|
||||
let (_source, _, template) = source();
|
||||
let (_target, options) = destination(&template);
|
||||
let held = acquire_runtime_store_owner_lock(&options.root).unwrap();
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
assert_fresh(&options);
|
||||
drop(held);
|
||||
initialize(&options.root, "wrong-runtime", WORKSPACE, NEW_URL);
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
initialize(&options.root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456");
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
fs::remove_file(options.root.join("auth.toml")).unwrap();
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
initialize(&options.root, RUNTIME, WORKSPACE, NEW_URL);
|
||||
fs::create_dir(options.root.join(WORKERS_DIR)).unwrap();
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() {
|
||||
for change in [
|
||||
"running",
|
||||
"pending",
|
||||
"guard-pending",
|
||||
"attachment",
|
||||
"schema",
|
||||
"worker-id",
|
||||
"metadata-workspace",
|
||||
"metadata-session",
|
||||
"artifact",
|
||||
"secret",
|
||||
"url",
|
||||
"path",
|
||||
] {
|
||||
let (_source, worker, _) = source();
|
||||
let mut aggregate: WorkerAggregateRecord =
|
||||
decode(&fs::read(worker.join(WORKER_FILE)).unwrap()).unwrap();
|
||||
match change {
|
||||
"running" => aggregate.status = WorkerStatus::Running,
|
||||
"pending" => {
|
||||
aggregate.execution_state = WorkerExecutionStateRecord::ReconciliationRequired(
|
||||
PersistedWorkerLifecycleOperation::Restore(PersistedWorkerRestoreOperation {
|
||||
operation_id: WorkerLifecycleOperationId::new(),
|
||||
mode: PersistedWorkerRestoreMode::Explicit,
|
||||
commit: crate::runtime::RestoreCommitFact::NotCommitted,
|
||||
source_repair_possible: false,
|
||||
request: request(),
|
||||
binding: None,
|
||||
last_settled_status: WorkerStatus::Stopped,
|
||||
}),
|
||||
)
|
||||
}
|
||||
"guard-pending" => aggregate.restore_guard = serde_json::from_str(r#"{"observation_token":"old-token","active_request_id":"pending-restore","owners":{}}"#).unwrap(),
|
||||
"attachment" => aggregate.logical_workdir_attachments.push(LogicalWorkdirAttachment {
|
||||
alias: workdir::WorkdirAttachmentAlias::new("source").unwrap(),
|
||||
working_directory_id: "old-workdir".into(),
|
||||
capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY,
|
||||
}),
|
||||
"schema" => aggregate.schema_version += 1,
|
||||
"worker-id" => aggregate.worker_id = WorkerId::now_v7(),
|
||||
"metadata-workspace" | "metadata-session" => {
|
||||
let mut metadata: WorkerMetadata =
|
||||
decode(&fs::read(worker.join(WORKER_METADATA_FILE)).unwrap()).unwrap();
|
||||
if change == "metadata-workspace" {
|
||||
metadata.workspace_id = Some("wrong-workspace".into());
|
||||
} else {
|
||||
metadata.active.as_mut().unwrap().session_id = uuid::Uuid::now_v7();
|
||||
}
|
||||
atomic_write_json(
|
||||
&worker.join(WORKER_METADATA_FILE),
|
||||
&metadata,
|
||||
"test mismatch",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
"artifact" => {
|
||||
fs::create_dir_all(worker.join("session/artifacts/paste")).unwrap();
|
||||
fs::write(
|
||||
worker.join("session/artifacts/paste/payload"),
|
||||
"unsupported",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
"secret" | "url" | "path" => {
|
||||
let segment = fs::read_dir(worker.join("session/segments"))
|
||||
.unwrap()
|
||||
.next()
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.path();
|
||||
let replacement = match change {
|
||||
"secret" => "Bearer secret",
|
||||
"url" => "https://old-host.invalid",
|
||||
_ => "/old/host/path",
|
||||
};
|
||||
let text = fs::read_to_string(&segment)
|
||||
.unwrap()
|
||||
.replace("artificial fixture reply", replacement);
|
||||
fs::write(segment, text).unwrap();
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
atomic_write_json(&worker.join(WORKER_FILE), &aggregate, "test aggregate").unwrap();
|
||||
assert!(
|
||||
StoppedWorkerTemplate::extract(&worker).is_err(),
|
||||
"accepted {change}"
|
||||
);
|
||||
}
|
||||
let (_source, worker, _) = source();
|
||||
let held =
|
||||
acquire_runtime_store_owner_lock(worker.parent().unwrap().parent().unwrap()).unwrap();
|
||||
assert!(StoppedWorkerTemplate::extract(&worker).is_err());
|
||||
drop(held);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_v12_settled_template_installs_current_aggregate_without_rewriting_session() {
|
||||
let (_source, _, mut template) = source();
|
||||
template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION;
|
||||
let bytes = serde_json::to_vec(&template).unwrap();
|
||||
let (_target, options) = destination(&template);
|
||||
template.install(&options, request(), &resolved()).unwrap();
|
||||
assert_eq!(serde_json::to_vec(&template).unwrap(), bytes);
|
||||
let worker = options.root.join(WORKERS_DIR).join(worker_id().to_string());
|
||||
assert_eq!(
|
||||
fs::read(worker.join("session/session.json")).unwrap(),
|
||||
template.session_manifest
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(worker.join(format!("session/segments/{}.jsonl", template.segment_id))).unwrap(),
|
||||
template.segment_jsonl
|
||||
);
|
||||
let aggregate: WorkerAggregateRecord =
|
||||
read_bounded_json(&worker.join(WORKER_FILE), "read installed fixture").unwrap();
|
||||
let saved = aggregate.validate(&worker.join(WORKER_FILE)).unwrap();
|
||||
assert_eq!(saved.status, WorkerStatus::Stopped);
|
||||
assert!(matches!(
|
||||
saved.execution_state,
|
||||
PersistedWorkerExecutionState::Available(_)
|
||||
));
|
||||
// Unsupported template schemas do not enter the versioned reader.
|
||||
template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION - 1;
|
||||
assert!(template.validate_shape().is_err());
|
||||
template.runtime_schema_version = SCHEMA_VERSION + 1;
|
||||
assert!(template.validate_shape().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fresh_manifest_and_memory_settings_are_required() {
|
||||
let (_source, _, template) = source();
|
||||
let (_target, options) = destination(&template);
|
||||
let mut manifest = resolved();
|
||||
manifest.worker.name = "wrong-worker".into();
|
||||
assert!(template.install(&options, request(), &manifest).is_err());
|
||||
let mut binding = request();
|
||||
binding.memory_settings = None;
|
||||
assert!(template.install(&options, binding, &resolved()).is_err());
|
||||
let mut binding = request();
|
||||
binding.memory_settings.as_mut().unwrap().workspace_id = "other-workspace".into();
|
||||
assert!(template.install(&options, binding, &resolved()).is_err());
|
||||
assert_fresh(&options);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settled_checkpoints_preserve_receipts_but_refuse_pending_work() {
|
||||
let mut checkpoint = settled_checkpoint();
|
||||
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_ok());
|
||||
checkpoint.pending.push(1);
|
||||
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||
checkpoint.pending.clear();
|
||||
checkpoint.activating = Some(1);
|
||||
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||
checkpoint.activating = None;
|
||||
checkpoint.invocation_receipts.push(1);
|
||||
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||
checkpoint.invocation_receipts.clear();
|
||||
checkpoint.invocation_recovery_blocked = true;
|
||||
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||
}
|
||||
|
||||
// Deliberately corrupt only test-owned synthetic source files, never the lab.
|
||||
fn assert_rejected_session_bytes(worker: &Path, template: &StoppedWorkerTemplate, bytes: Vec<u8>) {
|
||||
let path = worker.join(format!("session/segments/{}.jsonl", template.segment_id()));
|
||||
fs::write(path, &bytes).unwrap();
|
||||
let extraction_error = StoppedWorkerTemplate::extract(worker)
|
||||
.err()
|
||||
.expect("invalid source accepted");
|
||||
assert!(
|
||||
!extraction_error
|
||||
.to_string()
|
||||
.contains("synthetic-discarded-material")
|
||||
);
|
||||
let (_target, options) = destination(template);
|
||||
let mut invalid = template.clone();
|
||||
invalid.segment_jsonl = bytes;
|
||||
let installation_error = invalid
|
||||
.install(&options, request(), &resolved())
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
!installation_error
|
||||
.to_string()
|
||||
.contains("synthetic-discarded-material")
|
||||
);
|
||||
assert_fresh(&options);
|
||||
}
|
||||
|
||||
fn synthetic_logs(template: &StoppedWorkerTemplate) -> Vec<LogEntry> {
|
||||
template
|
||||
.segment_jsonl
|
||||
.split(|byte| *byte == b'\n')
|
||||
.filter(|line| !line.is_empty())
|
||||
.map(|line| serde_json::from_slice(line).unwrap())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn encode_synthetic_logs(logs: &[LogEntry]) -> Vec<u8> {
|
||||
let mut bytes = Vec::new();
|
||||
for log in logs {
|
||||
serde_json::to_writer(&mut bytes, log).unwrap();
|
||||
bytes.push(b'\n');
|
||||
}
|
||||
bytes
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_system_prompt_matches_worker_restore_rejection() {
|
||||
let (_source, worker_dir, template) = source();
|
||||
let mut logs = synthetic_logs(&template);
|
||||
assert!(session_store::collect_state(&logs).system_prompt.is_some());
|
||||
let LogEntry::AnnotatedSegmentStart { system_prompt, .. } = &mut logs[0] else {
|
||||
panic!("missing seed")
|
||||
};
|
||||
*system_prompt = None;
|
||||
assert_rejected_session_bytes(&worker_dir, &template, encode_synthetic_logs(&logs));
|
||||
// The owning Worker constructor rejects this before model-client creation,
|
||||
// allocation, or execution. Do not start a controller or submit any input.
|
||||
let store = session_store::CombinedStore::new(
|
||||
WorkerSessionStore::open_read_only(worker_dir.join("session")).unwrap(),
|
||||
session_store::WorkerAggregateStore::open_read_only(&worker_dir, template.worker_name())
|
||||
.unwrap(),
|
||||
);
|
||||
let restored = worker::Worker::restore_from_manifest(
|
||||
template.session_id(),
|
||||
template.segment_id(),
|
||||
resolved(),
|
||||
store,
|
||||
worker::PromptCatalogSource::builtins_only(),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(
|
||||
restored,
|
||||
Err(worker::WorkerError::SegmentSystemPromptMissing { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_submission_and_notification_receipts_fail_extraction_and_install() {
|
||||
let (_source, worker, template) = source();
|
||||
for notification in [false, true] {
|
||||
let mut checkpoint = settled_checkpoint();
|
||||
if notification {
|
||||
checkpoint.notification_receipts =
|
||||
vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)];
|
||||
} else {
|
||||
checkpoint.receipts = vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)];
|
||||
}
|
||||
let mut logs = synthetic_logs(&template);
|
||||
for log in &mut logs {
|
||||
if let LogEntry::AnnotatedUserInput { extensions, .. } = log {
|
||||
*extensions = vec![checkpoint_extension(&checkpoint)];
|
||||
}
|
||||
}
|
||||
assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_fields_at_every_retained_record_depth_are_rejected_without_secret_diagnostics() {
|
||||
let (_source, worker, template) = source();
|
||||
let original = String::from_utf8(template.segment_jsonl.clone()).unwrap();
|
||||
// These values are harmless synthetic markers, not lab/private material.
|
||||
// The compatibility decoder normally discards many of these fields.
|
||||
let locations = [
|
||||
r#""kind":"annotated_segment_start""#,
|
||||
r#""kind":"invoke""#,
|
||||
r#""kind":"annotated_user_input""#,
|
||||
r#""kind":"annotated_assistant_item""#,
|
||||
r#""kind":"llm_usage""#,
|
||||
r#""kind":"turn_end""#,
|
||||
r#""kind":"run_completed""#,
|
||||
r#""max_tokens":null"#,
|
||||
r#""kind":"message","role":"assistant""#,
|
||||
r#""kind":"text","text":"artificial fixture reply""#,
|
||||
r#""kind":"text","content":"Harmless artificial fixture input.""#,
|
||||
r#""entry_id":"fixed-entry-id""#,
|
||||
r#""kind":"model_output""#,
|
||||
r#""worker_id":"01a1657d-b762-7791-87ae-a49f28cadbce""#,
|
||||
r#""domain":"worker.pending_activations.v1""#,
|
||||
r#""next_activation_sequence":2"#,
|
||||
r#""submission_request_id":"completed-submission-request""#,
|
||||
r#""notification_request_id":"completed-notification-request""#,
|
||||
];
|
||||
for location in locations {
|
||||
assert!(original.contains(location));
|
||||
for extra in [
|
||||
r#", "reasoning":{"text":"synthetic-discarded-material"}"#,
|
||||
r#", "encrypted_content":"synthetic-discarded-material""#,
|
||||
] {
|
||||
let altered = original.replacen(location, &format!("{location}{extra}"), 1);
|
||||
assert_rejected_session_bytes(&worker, &template, altered.into_bytes());
|
||||
}
|
||||
}
|
||||
let duplicate = original.replacen(
|
||||
r#""role":"assistant""#,
|
||||
r#""role":"assistant","role":"assistant""#,
|
||||
1,
|
||||
);
|
||||
assert_rejected_session_bytes(&worker, &template, duplicate.into_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_feature_payloads_cannot_bypass_strict_session_validation() {
|
||||
let (_source, worker, template) = source();
|
||||
let mut logs = synthetic_logs(&template);
|
||||
for log in &mut logs {
|
||||
if let LogEntry::AnnotatedUserInput { history, .. } = log {
|
||||
if let session_store::LoggedItem::Message { role, .. } = &mut history[0].item {
|
||||
*role = session_store::LoggedRole::System;
|
||||
}
|
||||
history[0].metadata.input_metadata = Some(
|
||||
session_store::HistoryInputMetadata::new(
|
||||
session_store::HistoryInputUsePolicy {
|
||||
recording_material: session_store::InputEvidenceUse::Exclude,
|
||||
memory_evidence: session_store::InputEvidenceUse::Exclude,
|
||||
observation: session_store::InputObservationUse::Hidden,
|
||||
compaction: session_store::InputCompactionUse::Retain,
|
||||
},
|
||||
Some(
|
||||
session_store::FeatureInputPayload::encode(
|
||||
"fixture.test",
|
||||
1,
|
||||
&BTreeMap::from([("reasoning", "harmless synthetic payload")]),
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn refuses_source_and_destination_symlinks() {
|
||||
let (_source, worker, template) = source();
|
||||
let (_target, options) = destination(&template);
|
||||
std::os::unix::fs::symlink(worker.join(WORKER_FILE), options.root.join("linked")).unwrap();
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
fs::remove_file(options.root.join("linked")).unwrap();
|
||||
std::os::unix::fs::symlink(worker.join(WORKER_FILE), worker.join("runs/link")).unwrap();
|
||||
assert!(StoppedWorkerTemplate::extract(&worker).is_err());
|
||||
}
|
||||
|
||||
/// Opt-in only: source path supplied explicitly; assertions never display content.
|
||||
#[test]
|
||||
#[ignore = "requires the approved artificial lab aggregate via YOI_STOPPED_FIXTURE_SOURCE"]
|
||||
fn approved_lab_extract_install() {
|
||||
let path = PathBuf::from(
|
||||
std::env::var_os("YOI_STOPPED_FIXTURE_SOURCE").expect("explicit source required"),
|
||||
);
|
||||
let template = StoppedWorkerTemplate::extract(&path).unwrap();
|
||||
let (_target, options) = destination(&template);
|
||||
let mut bind = request();
|
||||
bind.worker_id = template.worker_id();
|
||||
bind.workspace_api.as_mut().unwrap().workspace_id = template.workspace_id().into();
|
||||
bind.memory_settings.as_mut().unwrap().workspace_id = template.workspace_id().into();
|
||||
let mut manifest = resolved();
|
||||
manifest.worker.name = template.worker_name();
|
||||
template.install(&options, bind, &manifest).unwrap();
|
||||
let installed = options
|
||||
.root
|
||||
.join(WORKERS_DIR)
|
||||
.join(template.worker_id().to_string());
|
||||
for relative in [
|
||||
"session/session.json".to_string(),
|
||||
format!("session/segments/{}.jsonl", template.segment_id()),
|
||||
] {
|
||||
assert!(
|
||||
fs::read(path.join(&relative)).unwrap() == fs::read(installed.join(relative)).unwrap()
|
||||
);
|
||||
}
|
||||
let runtime = crate::Runtime::with_fs_store(options).unwrap();
|
||||
assert_eq!(runtime.list_workers().unwrap().len(), 1);
|
||||
assert_eq!(runtime.summary().unwrap().active_worker_count, 0);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -193,6 +193,21 @@ mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
fn short_tempdir() -> tempfile::TempDir {
|
||||
let target = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.parent()
|
||||
.unwrap()
|
||||
.parent()
|
||||
.unwrap()
|
||||
.join("target");
|
||||
std::fs::create_dir_all(&target).unwrap();
|
||||
tempfile::Builder::new()
|
||||
.prefix("d")
|
||||
.rand_bytes(3)
|
||||
.tempdir_in(target)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn read_request() -> ReadRequest {
|
||||
ReadRequest {
|
||||
path: WorkdirPath::new("README.md").unwrap(),
|
||||
@@ -225,9 +240,60 @@ mod tests {
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn router_cleanup_of_unopened_session_never_connects_or_follows_reused_alias() {
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
let observed = attempts.clone();
|
||||
let session = DeferredWorkdirSession::handle(
|
||||
"old",
|
||||
WorkdirSessionCapabilities::READ_ONLY,
|
||||
move || {
|
||||
observed.fetch_add(1, Ordering::SeqCst);
|
||||
Err("offline".into())
|
||||
},
|
||||
);
|
||||
let router = WorkdirSessionRouter::new();
|
||||
let alias = WorkdirAttachmentAlias::new("w").unwrap();
|
||||
let routed = router.attach(alias.clone(), session.clone()).unwrap();
|
||||
assert!(routed.subscribe_command_events().is_some());
|
||||
assert!(routed.command_snapshot().is_empty());
|
||||
router.close_all().await.unwrap();
|
||||
router.close_all().await.unwrap();
|
||||
assert!(router.aliases().is_empty());
|
||||
assert!(matches!(
|
||||
session.read(read_request()).await,
|
||||
Err(WorkdirError::SessionClosed)
|
||||
));
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||
|
||||
let new_attempts = Arc::new(AtomicUsize::new(0));
|
||||
let new_observed = new_attempts.clone();
|
||||
let replacement = DeferredWorkdirSession::handle(
|
||||
"new",
|
||||
WorkdirSessionCapabilities::READ_ONLY,
|
||||
move || {
|
||||
new_observed.fetch_add(1, Ordering::SeqCst);
|
||||
Err("offline".into())
|
||||
},
|
||||
);
|
||||
let new_routed = router.attach(alias, replacement).unwrap();
|
||||
// A retained old connection cannot open the replacement provider.
|
||||
assert!(routed.read(read_request()).await.is_err());
|
||||
assert_eq!(new_attempts.load(Ordering::SeqCst), 0);
|
||||
assert!(matches!(
|
||||
new_routed.read(read_request()).await,
|
||||
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
|
||||
));
|
||||
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
|
||||
// Even after a failed open, cleanup must not try connecting again.
|
||||
router.close_all().await.unwrap();
|
||||
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_open_retries_on_use_and_keeps_read_only_capabilities() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let temp = short_tempdir();
|
||||
std::fs::write(temp.path().join("README.md"), "restored").unwrap();
|
||||
let root = temp.path().to_path_buf();
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
//! Explicit, operation-owned repair of saved Session material. This is not
|
||||
//! read-only candidate preparation: the canonical Session/attribution is the
|
||||
//! receipt for replay after a lost response, and effects are reported as such.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl ProfileRuntimeWorkerFactory {
|
||||
pub(super) async fn recover_saved_restore_source(
|
||||
&self,
|
||||
request: &WorkerExecutionRestoreRequest,
|
||||
) -> Result<bool, RuntimeRestoreFailure> {
|
||||
let cause = |cause| RuntimeRestoreFailure::started(RuntimeRestoreStage::Recovery, cause);
|
||||
let name = Self::runtime_worker_name_for_ref(&request.worker_ref);
|
||||
let root = self
|
||||
.worker_aggregate_dir(&request.worker_ref)
|
||||
.map_err(|_| cause("restore recovery aggregate unavailable"))?;
|
||||
let metadata_store = WorkerAggregateStore::open_read_only(&root, &name)
|
||||
.map_err(|_| cause("restore recovery metadata open failed"))?;
|
||||
let metadata = metadata_store
|
||||
.read_by_name(&name)
|
||||
.map_err(|_| cause("restore recovery metadata read failed"))?
|
||||
.ok_or_else(|| cause("restore recovery metadata missing"))?;
|
||||
let active = metadata
|
||||
.active
|
||||
.as_ref()
|
||||
.ok_or_else(|| cause("restore recovery Session missing"))?;
|
||||
let pending = active.segment_id.is_none();
|
||||
let incomplete = request.request.subjektiv_attached
|
||||
&& !matches!(
|
||||
metadata.subjektiv_session_attribution,
|
||||
Some(session_store::SubjektivSessionAttributionState::Confirmed { .. })
|
||||
);
|
||||
if !pending && !incomplete {
|
||||
return Ok(false);
|
||||
}
|
||||
request
|
||||
.context
|
||||
.authorize_restore_source_repair(request.operation_id)
|
||||
.map_err(|_| cause("canonical source repair authority is not durably confirmed"))?;
|
||||
// The accepted Runtime operation already owns this Session. Its stable
|
||||
// Session identity and sole committed seed, not a new restore request,
|
||||
// are used to reconcile an interrupted first-head publication.
|
||||
let session_store = WorkerSessionStore::new_with_retention(
|
||||
root.join("session"),
|
||||
self.session_retention()
|
||||
.map_err(|_| cause("restore recovery retention unavailable"))?,
|
||||
)
|
||||
.map_err(|_| cause("restore recovery Session open failed"))?;
|
||||
let (manifest, mut loader) = Self::manifest_for_restore(&metadata, &request.request)
|
||||
.map_err(|_| cause("restore recovery saved manifest invalid"))?;
|
||||
let workspace = RuntimeWorkspaceBackendRef::from_worker_request(
|
||||
&request.request,
|
||||
self.runtime_id.as_deref(),
|
||||
)
|
||||
.map_err(cause)?;
|
||||
#[cfg(feature = "http-server")]
|
||||
let request_client = request
|
||||
.request
|
||||
.workspace_api
|
||||
.as_ref()
|
||||
.and_then(|api| self.workspace_request_clients.get(&api.workspace_id));
|
||||
let context = workspace.worker_context(
|
||||
#[cfg(feature = "http-server")]
|
||||
&request.worker_ref,
|
||||
#[cfg(feature = "http-server")]
|
||||
request.workspace_scope.as_ref(),
|
||||
#[cfg(feature = "http-server")]
|
||||
self.worker_mutation_identity.as_ref(),
|
||||
#[cfg(feature = "http-server")]
|
||||
request_client,
|
||||
#[cfg(feature = "http-server")]
|
||||
self.embedded_worker_mutation_dispatcher.as_ref(),
|
||||
#[cfg(feature = "http-server")]
|
||||
Some(self.prompt_projection_cache.clone()),
|
||||
);
|
||||
if pending && context.workspace_id().is_some() {
|
||||
let bundle = request
|
||||
.config_bundle
|
||||
.as_ref()
|
||||
.ok_or_else(|| cause("pending recovery launch material missing"))?;
|
||||
let projection = self
|
||||
.observe_bundle_prompt_projection(
|
||||
bundle,
|
||||
request
|
||||
.request
|
||||
.workspace_api
|
||||
.as_ref()
|
||||
.map(|api| api.workspace_id.as_str()),
|
||||
)
|
||||
.map_err(|_| cause("pending recovery Prompt projection invalid"))?
|
||||
.ok_or_else(|| cause("pending recovery Prompt projection missing"))?;
|
||||
loader = loader.with_effective_catalog(projection.projection.catalog.clone());
|
||||
}
|
||||
self.failed_restore_resources
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(
|
||||
request.worker_ref.clone(),
|
||||
FailedRestoreResources {
|
||||
operation_id: request.operation_id,
|
||||
sessions: Vec::new(),
|
||||
candidate_workdir_cleanup: None,
|
||||
allocation: None,
|
||||
feature_tasks: None,
|
||||
},
|
||||
);
|
||||
let store = CombinedStore::new(session_store, metadata_store);
|
||||
let worker_result = Worker::restore_pending_from_worker_metadata_with_context(
|
||||
&name,
|
||||
manifest,
|
||||
store,
|
||||
loader,
|
||||
context,
|
||||
WorkerFilesystemAuthority::None,
|
||||
)
|
||||
.await;
|
||||
let mut worker = worker_result.map_err(|error| {
|
||||
self.retain_construction_failure(request, &error);
|
||||
cause(restore_worker_cause(&error))
|
||||
})?;
|
||||
let tasks = worker.retain_startup_feature_cleanup();
|
||||
let sessions = worker.workdir_sessions();
|
||||
let allocation = worker.restore_allocation_cleanup_handle();
|
||||
self.failed_restore_resources
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(
|
||||
request.worker_ref.clone(),
|
||||
FailedRestoreResources {
|
||||
operation_id: request.operation_id,
|
||||
sessions: vec![sessions.clone()],
|
||||
candidate_workdir_cleanup: None,
|
||||
allocation: allocation.clone(),
|
||||
feature_tasks: Some(tasks.clone()),
|
||||
},
|
||||
);
|
||||
let result = async {
|
||||
if let Some(binding) = request.request.backend_job.as_ref() {
|
||||
worker
|
||||
.bind_backend_job(binding.clone())
|
||||
.map_err(|_| cause("recovery Backend Job binding failed"))?;
|
||||
}
|
||||
worker
|
||||
.finalize_subjektiv_session_attribution(
|
||||
request.request.subjektiv_attached,
|
||||
SubjektivSessionAttributionLifecycle::RestoredSession,
|
||||
)
|
||||
.map_err(|error| cause(restore_worker_cause(&error)))?;
|
||||
if pending {
|
||||
worker::WorkerController::recover_pending_head(&mut worker, &root)
|
||||
.await
|
||||
.map_err(|_| cause("pending Session head recovery failed"))?;
|
||||
}
|
||||
Ok::<_, RuntimeRestoreFailure>(())
|
||||
}
|
||||
.await;
|
||||
// Never release allocation while the Worker guard is attached. Retained
|
||||
// handles in the same operation remain available on every cleanup error.
|
||||
drop(worker);
|
||||
let cleanup = async {
|
||||
tasks
|
||||
.shutdown()
|
||||
.await
|
||||
.map_err(|_| cause("recovery Feature settlement failed"))?;
|
||||
sessions
|
||||
.close_all()
|
||||
.await
|
||||
.map_err(|_| cause("recovery Workdir close failed"))?;
|
||||
if let Some(allocation) = allocation {
|
||||
allocation
|
||||
.release()
|
||||
.map_err(|_| cause("recovery allocation release failed"))?;
|
||||
}
|
||||
Ok::<_, RuntimeRestoreFailure>(())
|
||||
}
|
||||
.await;
|
||||
result?;
|
||||
cleanup?;
|
||||
self.failed_restore_resources
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&request.worker_ref);
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,927 @@
|
||||
// Included by the production Profile publication suite. Each fault is at a
|
||||
// distinct partial-publication boundary; no factory result is substituted.
|
||||
|
||||
fn publication_gate<T>() -> (
|
||||
mpsc::Sender<()>,
|
||||
mpsc::Receiver<T>,
|
||||
crate::execution::RestoreTestGate<T>,
|
||||
) {
|
||||
let (release, receiver) = mpsc::channel();
|
||||
let (entered, evidence) = mpsc::channel();
|
||||
(
|
||||
release,
|
||||
evidence,
|
||||
crate::execution::RestoreTestGate {
|
||||
entered,
|
||||
release: receiver,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn assert_single_publication(fixture: &Fixture, expected: usize) {
|
||||
let faults = &fixture.backend.factory.restore_faults;
|
||||
assert_eq!(
|
||||
faults.executions.load(Ordering::SeqCst),
|
||||
expected,
|
||||
"construction reran"
|
||||
);
|
||||
assert_eq!(
|
||||
faults.effects_calls.load(Ordering::SeqCst),
|
||||
expected,
|
||||
"restore effects reran"
|
||||
);
|
||||
assert_eq!(
|
||||
faults.publication_attempts.load(Ordering::SeqCst),
|
||||
expected,
|
||||
"Controller start reran"
|
||||
);
|
||||
fixture.assert_quiet();
|
||||
}
|
||||
|
||||
fn assert_real_attach(
|
||||
fixture: &Fixture,
|
||||
handle: &WorkerHandle,
|
||||
) -> crate::execution::WorkerProtocolTransport {
|
||||
let worker_ref = &fixture.request.worker_ref;
|
||||
assert!(matches!(
|
||||
fixture
|
||||
.backend
|
||||
.worker_session(crate::execution::WorkerSessionObservationRequest {
|
||||
worker_ref: worker_ref.clone(),
|
||||
}),
|
||||
runtime_api::WorkerSessionAvailability::LiveProtocol
|
||||
));
|
||||
let transport = fixture
|
||||
.backend
|
||||
.clone()
|
||||
.attach_worker_protocol(worker_ref)
|
||||
.unwrap();
|
||||
transport.validate().unwrap();
|
||||
let paired = subscribe_worker_protocol_session(handle);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&transport.snapshot).unwrap(),
|
||||
serde_json::to_value(paired.snapshot_event).unwrap()
|
||||
);
|
||||
assert!(
|
||||
matches!(&transport.snapshot, Event::Snapshot { session, greeting, state, in_flight, .. }
|
||||
if session == &session_store::public_snapshot::project_current_session_snapshot(&fixture.entries())
|
||||
&& !greeting.tools.is_empty() && state.state == protocol::WorkerState::Idle && in_flight.is_empty())
|
||||
);
|
||||
transport
|
||||
}
|
||||
|
||||
fn observe_publication_markers(
|
||||
fixture: &Fixture,
|
||||
observations: tokio::sync::broadcast::Receiver<crate::observation::WorkerObservationEvent>,
|
||||
markers: &'static [&'static str],
|
||||
) -> tokio::sync::broadcast::Receiver<crate::observation::WorkerObservationEvent> {
|
||||
fixture
|
||||
.backend
|
||||
.run_on_adapter_runtime(async move {
|
||||
let mut observations = observations;
|
||||
let mut counts = vec![0; markers.len()];
|
||||
let mut snapshots = 0;
|
||||
tokio::time::timeout(WAIT, async {
|
||||
loop {
|
||||
let observed = observations.recv().await.unwrap();
|
||||
if matches!(observed.payload, Event::Snapshot { .. }) {
|
||||
snapshots += 1;
|
||||
}
|
||||
for (index, marker) in markers.iter().enumerate() {
|
||||
if is_marker(&observed.payload, marker) {
|
||||
counts[index] += 1;
|
||||
}
|
||||
}
|
||||
if counts.last() == Some(&1) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| format!("lost original subscription markers: {counts:?}"))?;
|
||||
assert_eq!(
|
||||
counts,
|
||||
vec![1; markers.len()],
|
||||
"lost or duplicated prefix on original receiver"
|
||||
);
|
||||
assert_eq!(snapshots, 1, "initial full snapshot was lost or replayed");
|
||||
while let Ok(event) = observations.try_recv() {
|
||||
assert!(
|
||||
!markers
|
||||
.iter()
|
||||
.any(|marker| is_marker(&event.payload, marker)),
|
||||
"duplicate after tail fence"
|
||||
);
|
||||
}
|
||||
Ok(observations)
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn partial_registration_recovers_same_controller(hub: bool) {
|
||||
let mut fixture = Fixture::no_model();
|
||||
Arc::get_mut(&mut fixture.backend)
|
||||
.unwrap()
|
||||
.spawn_restore_timeout = Duration::from_millis(250);
|
||||
let runtime = stopped_runtime(&fixture);
|
||||
let pause = runtime.test_pause_restore_continuation();
|
||||
let worker_ref = fixture.request.worker_ref.clone();
|
||||
let scope = fixture.request.workspace_scope.as_ref().unwrap();
|
||||
let request = runtime.test_restore_request(&worker_ref);
|
||||
let observations = runtime.subscribe_worker_observation().unwrap();
|
||||
let (started_release, started_entered) = activation_gate(&fixture, true);
|
||||
let first = runtime
|
||||
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
first.state,
|
||||
server_api::WorkerRestoreState::ReconciliationRequired
|
||||
);
|
||||
let started = started_entered.recv_timeout(WAIT).unwrap();
|
||||
let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started);
|
||||
let old = started.handle.as_ref().unwrap().clone();
|
||||
let canonical = fixture.canonical_files();
|
||||
alert_marker(&old, "registration-original-paired-prefix");
|
||||
let (release, entered, fault) = publication_gate();
|
||||
if hub {
|
||||
*fixture
|
||||
.backend
|
||||
.factory
|
||||
.observation_hub
|
||||
.registration_gate
|
||||
.lock()
|
||||
.unwrap() = Some(RestoreHubRegistrationGate {
|
||||
gate: fault,
|
||||
fail: true,
|
||||
});
|
||||
} else {
|
||||
*fixture.backend.bridge_fault.lock().unwrap() = Some(fault);
|
||||
}
|
||||
started_release.release();
|
||||
drop(pause);
|
||||
let boundary_handle = entered.recv_timeout(WAIT).unwrap();
|
||||
assert!(boundary_handle.same_controller(&old));
|
||||
let durable = durable_operation(&runtime, &worker_ref).unwrap();
|
||||
assert_eq!(durable.operation_id, started.operation_id);
|
||||
assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed);
|
||||
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
|
||||
if hub {
|
||||
let (_, shared, _) = fixture
|
||||
.backend
|
||||
.factory
|
||||
.observation_hub
|
||||
.get(&worker_ref)
|
||||
.unwrap();
|
||||
assert!(
|
||||
Arc::ptr_eq(&shared, &old.shared_state),
|
||||
"real hub insert preceded fault"
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
fixture
|
||||
.backend
|
||||
.factory
|
||||
.observation_hub
|
||||
.get(&worker_ref)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
let (operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref);
|
||||
assert_eq!(operation, started.operation_id);
|
||||
assert!(!uncertain);
|
||||
assert!(
|
||||
context
|
||||
.test_buffered_events()
|
||||
.iter()
|
||||
.any(|event| matches!(event, Event::Snapshot { .. }))
|
||||
);
|
||||
alert_marker(&old, "registration-during-fault");
|
||||
let pause = runtime.test_pause_restore_continuation();
|
||||
release.send(()).unwrap();
|
||||
// This lock also observes the end of the actual registration attempt, not
|
||||
// merely the test's entered notification. Its retained scope owns cleanup.
|
||||
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||
assert!(old.same_controller(&execution.handle));
|
||||
assert_eq!(execution.restore_operation_id, Some(operation));
|
||||
assert_eq!(
|
||||
execution.tasks.tasks.lock().unwrap().len(),
|
||||
if hub { 2 } else { 1 }
|
||||
);
|
||||
let bridge = execution.tasks.protocol_bridge.lock().unwrap();
|
||||
if hub {
|
||||
assert!(bridge.is_none());
|
||||
} else {
|
||||
let bridge = bridge.as_ref().unwrap();
|
||||
assert!(
|
||||
!bridge.snapshot_pending,
|
||||
"snapshot already registered, do not replay it"
|
||||
);
|
||||
let streams = bridge.streams.lock().unwrap();
|
||||
assert!(
|
||||
streams.is_some(),
|
||||
"original paired streams survived failed task admission"
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&streams.as_ref().unwrap().snapshot_event).unwrap(),
|
||||
serde_json::to_value(&started.snapshot).unwrap()
|
||||
);
|
||||
}
|
||||
drop(bridge);
|
||||
assert!(
|
||||
!old.protocol_is_closed(),
|
||||
"publication failure must not clean a committed Controller"
|
||||
);
|
||||
assert_single_publication(&fixture, 1);
|
||||
drop(pause);
|
||||
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
|
||||
assert!(durable_operation(&runtime, &worker_ref).is_none());
|
||||
let owner = restore_owner(&runtime, &worker_ref, &request.request_id);
|
||||
assert_eq!(
|
||||
owner["operation_id"],
|
||||
serde_json::to_value(operation).unwrap()
|
||||
);
|
||||
assert_eq!(owner["receipt"]["state"], "accepted");
|
||||
assert!(
|
||||
fixture
|
||||
.backend
|
||||
.pending_restore(&worker_ref)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(!pending.task_in_flight.load(Ordering::Acquire));
|
||||
assert!(pending.activated.lock().unwrap().is_none());
|
||||
assert!(pending.result.lock().unwrap().is_none());
|
||||
assert!(pending.task.try_lock().unwrap().is_none());
|
||||
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
|
||||
assert!(old.same_controller(&fixture.backend.workers.lock().unwrap()[&worker_ref].handle));
|
||||
let transport = assert_real_attach(&fixture, &old);
|
||||
assert_eq!(fixture.canonical_files(), canonical);
|
||||
assert_single_publication(&fixture, 1);
|
||||
alert_marker(&old, "registration-tail-fence");
|
||||
let observations = observe_publication_markers(
|
||||
&fixture,
|
||||
observations,
|
||||
&[
|
||||
"registration-original-paired-prefix",
|
||||
"registration-during-fault",
|
||||
"registration-tail-fence",
|
||||
],
|
||||
);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::Accepted
|
||||
);
|
||||
assert_single_publication(&fixture, 1);
|
||||
// Retain the original transport/handle and scope through Stop/replacement.
|
||||
runtime
|
||||
.stop_worker_scoped(scope, &worker_ref, None)
|
||||
.unwrap();
|
||||
assert!(old.protocol_is_closed());
|
||||
assert!(transport.validate().is_err());
|
||||
let replacement_request = runtime.test_restore_request(&worker_ref);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.restore_worker_operation_scoped(scope, &worker_ref, replacement_request.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::Accepted
|
||||
);
|
||||
let replacement = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||
assert!(!old.same_controller(&replacement.handle));
|
||||
assert_ne!(replacement.restore_operation_id, Some(operation));
|
||||
assert!(
|
||||
transport
|
||||
.dispatch(Method::Pause {
|
||||
command: protocol::WorkerCommandEnvelope::new(801)
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
transport
|
||||
.dispatch(Method::Shutdown {
|
||||
command: protocol::WorkerCommandEnvelope::new(802)
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
for method in [
|
||||
Method::Pause {
|
||||
command: protocol::WorkerCommandEnvelope::new(803),
|
||||
},
|
||||
Method::Shutdown {
|
||||
command: protocol::WorkerCommandEnvelope::new(804),
|
||||
},
|
||||
] {
|
||||
let old_command = old.clone();
|
||||
assert!(
|
||||
fixture
|
||||
.backend
|
||||
.run_on_adapter_runtime(async move {
|
||||
old_command
|
||||
.send(method)
|
||||
.await
|
||||
.map_err(|error| error.to_string())
|
||||
})
|
||||
.is_err(),
|
||||
"closed original Controller command must not route to replacement"
|
||||
);
|
||||
}
|
||||
fixture
|
||||
.backend
|
||||
.factory
|
||||
.deactivate_stopped_controller(&worker_ref, &old)
|
||||
.unwrap();
|
||||
let (_, shared, _) = fixture
|
||||
.backend
|
||||
.factory
|
||||
.observation_hub
|
||||
.get(&worker_ref)
|
||||
.unwrap();
|
||||
assert!(Arc::ptr_eq(&shared, &replacement.handle.shared_state));
|
||||
alert_marker(&old, "registration-stopped-event-must-not-leak");
|
||||
alert_marker(&replacement.handle, "registration-replacement-fence");
|
||||
fixture
|
||||
.backend
|
||||
.run_on_adapter_runtime(async move {
|
||||
let mut observations = observations;
|
||||
tokio::time::timeout(WAIT, async {
|
||||
loop {
|
||||
let event = observations.recv().await.unwrap();
|
||||
assert!(!is_marker(
|
||||
&event.payload,
|
||||
"registration-stopped-event-must-not-leak"
|
||||
));
|
||||
if is_marker(&event.payload, "registration-replacement-fence") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| "replacement observation fence missing".to_string())?;
|
||||
drop(transport);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert_single_publication(&fixture, 2);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::Accepted
|
||||
);
|
||||
assert_eq!(restore_owner_count(&runtime, &worker_ref), 2);
|
||||
let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||
assert!(current.handle.same_controller(&replacement.handle));
|
||||
assert_eq!(
|
||||
current.restore_operation_id,
|
||||
replacement.restore_operation_id
|
||||
);
|
||||
assert_eq!(
|
||||
replacement.handle.shared_state.snapshot().state,
|
||||
protocol::WorkerState::Idle
|
||||
);
|
||||
drop(current);
|
||||
drop(execution);
|
||||
drop(replacement);
|
||||
drop(pending);
|
||||
runtime
|
||||
.stop_worker_scoped(scope, &worker_ref, None)
|
||||
.unwrap();
|
||||
fixture.assert_released();
|
||||
}
|
||||
|
||||
fn restore_owner_count(runtime: &EmbeddedRuntime, worker_ref: &WorkerRef) -> usize {
|
||||
let saved = runtime
|
||||
.test_runtime_store()
|
||||
.read_worker_record(worker_ref)
|
||||
.unwrap();
|
||||
serde_json::to_value(saved.restore_guard).unwrap()["owners"]
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.len()
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn bridge_partial_snapshot_registration_retains_original_streams_and_autonomously_recovers() {
|
||||
partial_registration_recovers_same_controller(false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn hub_partial_insert_registration_retains_original_controller_and_autonomously_recovers() {
|
||||
partial_registration_recovers_same_controller(true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn context_partial_flush_retains_only_undelivered_queue_and_recovers_without_repost() {
|
||||
let mut fixture = Fixture::no_model();
|
||||
Arc::get_mut(&mut fixture.backend)
|
||||
.unwrap()
|
||||
.spawn_restore_timeout = Duration::from_millis(250);
|
||||
let runtime = stopped_runtime(&fixture);
|
||||
let pause = runtime.test_pause_restore_continuation();
|
||||
let worker_ref = fixture.request.worker_ref.clone();
|
||||
let scope = fixture.request.workspace_scope.as_ref().unwrap();
|
||||
let request = runtime.test_restore_request(&worker_ref);
|
||||
let observations = runtime.subscribe_worker_observation().unwrap();
|
||||
let (started_release, started_entered) = activation_gate(&fixture, true);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||
.unwrap()
|
||||
.state,
|
||||
server_api::WorkerRestoreState::ReconciliationRequired
|
||||
);
|
||||
let started = started_entered.recv_timeout(WAIT).unwrap();
|
||||
let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started);
|
||||
let old = started.handle.unwrap();
|
||||
let (operation, context, _) = runtime.test_pending_restore_context(&worker_ref);
|
||||
// Stop after the actual bridge and hub registration, without failing that
|
||||
// boundary, so the flush begins with its real original full snapshot.
|
||||
let (hub_release, hub_entered, hub_gate) = publication_gate();
|
||||
*fixture
|
||||
.backend
|
||||
.factory
|
||||
.observation_hub
|
||||
.registration_gate
|
||||
.lock()
|
||||
.unwrap() = Some(RestoreHubRegistrationGate {
|
||||
gate: hub_gate,
|
||||
fail: false,
|
||||
});
|
||||
started_release.release();
|
||||
drop(pause);
|
||||
assert!(
|
||||
hub_entered
|
||||
.recv_timeout(WAIT)
|
||||
.unwrap()
|
||||
.same_controller(&old)
|
||||
);
|
||||
let buffered_prefix = context.test_buffered_events();
|
||||
assert!(matches!(
|
||||
buffered_prefix.first(),
|
||||
Some(Event::Snapshot { .. })
|
||||
));
|
||||
let prefix_length = buffered_prefix.len();
|
||||
let marker = |text: &str| Event::Error {
|
||||
code: protocol::ErrorCode::Internal,
|
||||
message: text.into(),
|
||||
};
|
||||
context
|
||||
.publish_protocol_event(marker("flush-delivered"))
|
||||
.unwrap();
|
||||
context
|
||||
.publish_protocol_event(marker("flush-failed"))
|
||||
.unwrap();
|
||||
context
|
||||
.publish_protocol_event(marker("flush-pending"))
|
||||
.unwrap();
|
||||
let (release, entered, gate) = publication_gate();
|
||||
let (restored, restored_queue) = mpsc::channel();
|
||||
context.test_fail_flush(crate::execution::RestoreFlushFault {
|
||||
remaining_successes: prefix_length + 1,
|
||||
gate,
|
||||
restored,
|
||||
});
|
||||
drop(hub_release);
|
||||
assert!(
|
||||
matches!(entered.recv_timeout(WAIT).unwrap(), Event::Error { message, .. } if message == "flush-failed")
|
||||
);
|
||||
let partial_execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||
assert!(old.same_controller(&partial_execution.handle));
|
||||
assert_eq!(partial_execution.restore_operation_id, Some(operation));
|
||||
assert_eq!(partial_execution.tasks.tasks.lock().unwrap().len(), 2);
|
||||
let (_, shared, _) = fixture
|
||||
.backend
|
||||
.factory
|
||||
.observation_hub
|
||||
.get(&worker_ref)
|
||||
.unwrap();
|
||||
assert!(Arc::ptr_eq(&shared, &old.shared_state));
|
||||
let pause = runtime.test_pause_restore_continuation();
|
||||
context
|
||||
.publish_protocol_event(marker("flush-arrived-during-error"))
|
||||
.unwrap();
|
||||
release.send(()).unwrap();
|
||||
let restored = restored_queue.recv_timeout(WAIT).unwrap();
|
||||
let undelivered: Vec<_> = restored
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
Event::Error { message, .. } if message.starts_with("flush-") => Some(message.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
undelivered,
|
||||
[
|
||||
"flush-failed",
|
||||
"flush-pending",
|
||||
"flush-arrived-during-error"
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
!restored
|
||||
.iter()
|
||||
.any(|event| matches!(event, Event::Snapshot { .. })),
|
||||
"delivered snapshot must not rejoin the failed suffix"
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(context.test_buffered_events()).unwrap(),
|
||||
serde_json::to_value(&restored).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
durable_operation(&runtime, &worker_ref)
|
||||
.unwrap()
|
||||
.operation_id,
|
||||
operation
|
||||
);
|
||||
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
|
||||
// The next terminal gate observes successful retry only after the same
|
||||
// queued owner has repaired its failed callback and remaining suffix.
|
||||
let (terminal_release, terminal_entered, gate) = publication_gate();
|
||||
runtime.test_terminal_receipt_gate(gate);
|
||||
drop(pause);
|
||||
assert_eq!(terminal_entered.recv_timeout(WAIT).unwrap(), operation);
|
||||
assert!(context.test_buffered_events().is_empty());
|
||||
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
|
||||
assert_eq!(
|
||||
durable_operation(&runtime, &worker_ref).unwrap().commit,
|
||||
crate::runtime::RestoreCommitFact::Committed
|
||||
);
|
||||
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||
assert!(old.same_controller(&execution.handle));
|
||||
assert_eq!(execution.restore_operation_id, Some(operation));
|
||||
drop(terminal_release); // RAII disconnect, not a second request.
|
||||
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
|
||||
assert_eq!(
|
||||
restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"],
|
||||
"accepted"
|
||||
);
|
||||
assert_real_attach(&fixture, &old);
|
||||
fixture
|
||||
.backend
|
||||
.run_on_adapter_runtime(async move {
|
||||
let mut observations = observations;
|
||||
let mut messages = Vec::new();
|
||||
let mut snapshots = 0;
|
||||
while let Ok(event) = observations.try_recv() {
|
||||
match event.payload {
|
||||
Event::Error { message, .. } if message.starts_with("flush-") => {
|
||||
messages.push(message)
|
||||
}
|
||||
Event::Snapshot { .. } => snapshots += 1,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
messages,
|
||||
[
|
||||
"flush-delivered",
|
||||
"flush-failed",
|
||||
"flush-pending",
|
||||
"flush-arrived-during-error"
|
||||
]
|
||||
);
|
||||
assert_eq!(snapshots, 1);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert_single_publication(&fixture, 1);
|
||||
drop(partial_execution);
|
||||
drop(execution);
|
||||
drop(pending);
|
||||
runtime
|
||||
.stop_worker_scoped(scope, &worker_ref, None)
|
||||
.unwrap();
|
||||
fixture.assert_released();
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum TerminalFault {
|
||||
BeforeRename,
|
||||
AfterRename,
|
||||
SyncUnknown,
|
||||
ReadUnknown,
|
||||
}
|
||||
|
||||
fn published_terminal_receipt_fault(fault: TerminalFault) {
|
||||
let mut fixture = Fixture::no_model();
|
||||
// Reuse the existing ordinary non-network model boundary, not the product
|
||||
// Codex profile. All fault/convergence assertions precede the sole input.
|
||||
let script = MockClient::sequential(vec![MockResponse::Complete(simple_text_events())]);
|
||||
let injected = script.clone();
|
||||
let factory = fixture
|
||||
.backend
|
||||
.factory
|
||||
.as_ref()
|
||||
.clone()
|
||||
.with_spawn_model_client_factory(move || Box::new(injected.clone()));
|
||||
fixture.backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap());
|
||||
let runtime = Arc::new(stopped_runtime(&fixture));
|
||||
let pause = runtime.test_pause_restore_continuation();
|
||||
let worker_ref = fixture.request.worker_ref.clone();
|
||||
let scope = fixture.request.workspace_scope.clone().unwrap();
|
||||
let request = runtime.test_restore_request(&worker_ref);
|
||||
let observations = runtime.subscribe_worker_observation().unwrap();
|
||||
let mut catalog = runtime
|
||||
.subscribe_event_selector_scoped(
|
||||
&scope,
|
||||
protocol::subscription::EventSubscriptionSelector::RuntimeWorkers,
|
||||
)
|
||||
.unwrap();
|
||||
let (release, entered, gate) = publication_gate();
|
||||
runtime.test_terminal_receipt_gate(gate);
|
||||
let restoring = {
|
||||
let runtime = runtime.clone();
|
||||
let worker_ref = worker_ref.clone();
|
||||
let scope = scope.clone();
|
||||
let request = request.clone();
|
||||
std::thread::spawn(move || {
|
||||
runtime.restore_worker_operation_scoped(&scope, &worker_ref, request)
|
||||
})
|
||||
};
|
||||
let operation = entered.recv_timeout(WAIT).unwrap();
|
||||
// Restore legitimately repairs derived metadata before publication. Receipt
|
||||
// failures must not rerun those effects; compare from this actual boundary.
|
||||
let canonical = fixture.canonical_files();
|
||||
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||
let old = execution.handle.clone();
|
||||
assert_eq!(execution.restore_operation_id, Some(operation));
|
||||
assert!(
|
||||
fixture
|
||||
.backend
|
||||
.pending_restore(&worker_ref)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(execution.tasks.tasks.lock().unwrap().len(), 2);
|
||||
assert!(execution.tasks.protocol_bridge.lock().unwrap().is_none());
|
||||
assert_eq!(
|
||||
durable_operation(&runtime, &worker_ref)
|
||||
.unwrap()
|
||||
.operation_id,
|
||||
operation
|
||||
);
|
||||
assert_eq!(
|
||||
durable_operation(&runtime, &worker_ref).unwrap().commit,
|
||||
crate::runtime::RestoreCommitFact::Committed
|
||||
);
|
||||
let owner = restore_owner(&runtime, &worker_ref, &request.request_id);
|
||||
assert!(owner["receipt"].is_null());
|
||||
let (same_operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref);
|
||||
assert_eq!(same_operation, operation);
|
||||
assert!(!uncertain);
|
||||
assert!(context.test_buffered_events().is_empty());
|
||||
let transport = assert_real_attach(&fixture, &old);
|
||||
assert_single_publication(&fixture, 1);
|
||||
alert_marker(&old, "terminal-before-receipt");
|
||||
let store = runtime.test_runtime_store();
|
||||
let reconcile_before = store.test_worker_reconcile_counts();
|
||||
store.fail_next_worker_write(if matches!(fault, TerminalFault::BeforeRename) {
|
||||
AtomicWriteFault::BeforeRename
|
||||
} else {
|
||||
AtomicWriteFault::AfterRename
|
||||
});
|
||||
match fault {
|
||||
TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(),
|
||||
TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(),
|
||||
_ => {}
|
||||
}
|
||||
release.send(()).unwrap();
|
||||
let first = restoring.join().unwrap().unwrap();
|
||||
let reconcile_after = store.test_worker_reconcile_counts();
|
||||
let reconciliation = (
|
||||
reconcile_after.0 - reconcile_before.0,
|
||||
reconcile_after.1 - reconcile_before.1,
|
||||
);
|
||||
assert_eq!(
|
||||
reconciliation,
|
||||
match fault {
|
||||
TerminalFault::BeforeRename => (0, 0),
|
||||
TerminalFault::AfterRename => (1, 4), // canonical file plus Worker/workers/root directories
|
||||
TerminalFault::SyncUnknown => (1, 1),
|
||||
TerminalFault::ReadUnknown => (1, 0),
|
||||
},
|
||||
"{fault:?}: actual canonical read/sync attempts"
|
||||
);
|
||||
assert_eq!(
|
||||
first.state,
|
||||
if matches!(fault, TerminalFault::AfterRename) {
|
||||
server_api::WorkerRestoreState::Accepted
|
||||
} else {
|
||||
server_api::WorkerRestoreState::ReconciliationRequired
|
||||
},
|
||||
"{fault:?}: {first:?}"
|
||||
);
|
||||
assert_eq!(fixture.canonical_files(), canonical);
|
||||
assert!(!old.protocol_is_closed());
|
||||
transport.validate().unwrap();
|
||||
assert_single_publication(&fixture, 1);
|
||||
let disk_owner = restore_owner(&runtime, &worker_ref, &request.request_id);
|
||||
assert_eq!(disk_owner["operation_id"], owner["operation_id"]);
|
||||
match fault {
|
||||
TerminalFault::BeforeRename => {
|
||||
assert!(disk_owner["receipt"].is_null());
|
||||
assert!(!runtime.test_pending_restore_context(&worker_ref).2);
|
||||
}
|
||||
TerminalFault::AfterRename => {
|
||||
assert_eq!(disk_owner["receipt"]["state"], "accepted");
|
||||
assert!(durable_operation(&runtime, &worker_ref).is_none());
|
||||
}
|
||||
TerminalFault::SyncUnknown | TerminalFault::ReadUnknown => {
|
||||
assert_eq!(
|
||||
disk_owner["receipt"]["state"], "accepted",
|
||||
"visible rename alone is not durable proof"
|
||||
);
|
||||
let (same_operation, _, fenced) = runtime.test_pending_restore_context(&worker_ref);
|
||||
assert_eq!(same_operation, operation);
|
||||
assert!(
|
||||
fenced,
|
||||
"unknown canonical outcome must retain the memory owner/write fence"
|
||||
);
|
||||
// Keep durability unproved for one competing Stop as well: it must
|
||||
// fail before any Shutdown, never manufacture a Stopped judgment.
|
||||
match fault {
|
||||
TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(),
|
||||
TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert!(matches!(
|
||||
runtime.stop_worker_scoped(&scope, &worker_ref, None),
|
||||
Err(crate::error::RuntimeError::StoreCommitOutcomeUnknown { .. })
|
||||
));
|
||||
assert!(!old.protocol_is_closed());
|
||||
assert!(!execution.shutdown_requested.load(Ordering::Acquire));
|
||||
assert!(runtime.test_pending_restore_context(&worker_ref).2);
|
||||
assert_single_publication(&fixture, 1);
|
||||
}
|
||||
}
|
||||
// Recovery is autonomous after the failed write / confirmed unknown fence.
|
||||
// No POST, fresh request, or input is needed to make it converge.
|
||||
drop(pause);
|
||||
if !matches!(fault, TerminalFault::AfterRename) {
|
||||
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
|
||||
}
|
||||
assert!(durable_operation(&runtime, &worker_ref).is_none());
|
||||
assert_eq!(
|
||||
restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"],
|
||||
"accepted"
|
||||
);
|
||||
let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||
assert!(old.same_controller(¤t.handle));
|
||||
assert_eq!(current.restore_operation_id, Some(operation));
|
||||
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
|
||||
assert_real_attach(&fixture, &old);
|
||||
assert_single_publication(&fixture, 1);
|
||||
// A canonical terminal reread may bypass the final redundant upsert. The
|
||||
// original subscriber must still have the current state and observation token.
|
||||
let fresh_catalog = runtime
|
||||
.subscribe_event_selector_scoped(
|
||||
&scope,
|
||||
protocol::subscription::EventSubscriptionSelector::RuntimeWorkers,
|
||||
)
|
||||
.unwrap();
|
||||
let protocol::subscription::SubscriptionSnapshot::Workers { workers } =
|
||||
fresh_catalog.snapshot()
|
||||
else {
|
||||
panic!("Runtime catalog must return Workers");
|
||||
};
|
||||
assert_eq!(workers.len(), 1);
|
||||
let mut last_applied = None;
|
||||
use futures::FutureExt;
|
||||
while let Some(update) = catalog.recv().now_or_never() {
|
||||
match update.unwrap().payload {
|
||||
protocol::subscription::SubscriptionEventPayload::WorkerUpserted { worker } => {
|
||||
last_applied = Some(worker);
|
||||
}
|
||||
payload => panic!("unexpected catalog update: {payload:?}"),
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
serde_json::to_value(last_applied.expect("restore must publish its catalog state"))
|
||||
.unwrap()
|
||||
== serde_json::to_value(&workers[0]).unwrap(),
|
||||
"original catalog subscriber diverged after {fault:?}"
|
||||
);
|
||||
drop(fresh_catalog);
|
||||
drop(catalog);
|
||||
alert_marker(&old, "terminal-after-recovery");
|
||||
let _observations = observe_publication_markers(
|
||||
&fixture,
|
||||
observations,
|
||||
&["terminal-before-receipt", "terminal-after-recovery"],
|
||||
);
|
||||
let accepted = runtime
|
||||
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
|
||||
.unwrap();
|
||||
assert_eq!(accepted.state, server_api::WorkerRestoreState::Accepted);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
|
||||
.unwrap(),
|
||||
accepted
|
||||
);
|
||||
assert_eq!(restore_owner_count(&runtime, &worker_ref), 1);
|
||||
let mut stale = request;
|
||||
stale.request_id = "terminal-fresh-stale-intent".into();
|
||||
assert!(matches!(
|
||||
runtime.restore_worker_operation_scoped(&scope, &worker_ref, stale),
|
||||
Err(crate::error::RuntimeError::RestoreObservationConflict { .. })
|
||||
));
|
||||
assert_single_publication(&fixture, 1);
|
||||
assert_eq!(
|
||||
script.call_count.load(Ordering::SeqCst),
|
||||
0,
|
||||
"receipt recovery cannot trigger input or a model call"
|
||||
);
|
||||
// This transport was paired before the failed receipt write. Exercise its
|
||||
// normal log-entry receiver after autonomous settlement, not a new attach.
|
||||
transport
|
||||
.dispatch(Method::submit_text(
|
||||
"terminal-ordinary-input",
|
||||
"terminal-ordinary-input",
|
||||
))
|
||||
.unwrap();
|
||||
fixture
|
||||
.backend
|
||||
.run_on_adapter_runtime(async move {
|
||||
let mut transport = transport;
|
||||
let (mut admitted, mut committed, mut text, mut ended) = (false, false, false, false);
|
||||
tokio::time::timeout(WAIT, async {
|
||||
while !(admitted && committed && text && ended) {
|
||||
match transport.events.recv().await.unwrap() {
|
||||
Event::SubmissionAccepted {
|
||||
submission_request_id,
|
||||
..
|
||||
} if submission_request_id == "terminal-ordinary-input" => admitted = true,
|
||||
Event::UserMessage { segments, .. } => {
|
||||
assert_eq!(
|
||||
segments,
|
||||
vec![protocol::Segment::text("terminal-ordinary-input")]
|
||||
);
|
||||
assert!(!committed, "paired log receiver duplicated input");
|
||||
committed = true;
|
||||
}
|
||||
Event::TextDone { text: output } => {
|
||||
assert_eq!(output, "hello from worker");
|
||||
text = true;
|
||||
}
|
||||
Event::RunEnd { .. } => ended = true,
|
||||
Event::SubmissionRejected { .. } | Event::Error { .. } => {
|
||||
panic!("ordinary input failed after receipt recovery")
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| {
|
||||
"ordinary input failed to settle on original paired transport".to_string()
|
||||
})?;
|
||||
drop(transport);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(script.call_count.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(fixture.entries().iter().filter(|entry| matches!(entry,
|
||||
LogEntry::AnnotatedUserInput { segments, .. } if segments == &vec![protocol::Segment::text("terminal-ordinary-input")]
|
||||
)).count(), 1);
|
||||
assert_single_publication(&fixture, 1);
|
||||
drop(execution);
|
||||
drop(current);
|
||||
runtime
|
||||
.stop_worker_scoped(&scope, &worker_ref, None)
|
||||
.unwrap();
|
||||
assert!(old.protocol_is_closed());
|
||||
fixture.assert_released();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn published_controller_terminal_receipt_before_rename_autonomously_settles_without_reconstruction()
|
||||
{
|
||||
published_terminal_receipt_fault(TerminalFault::BeforeRename);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn published_controller_terminal_receipt_after_rename_is_canonically_reread_and_synced() {
|
||||
published_terminal_receipt_fault(TerminalFault::AfterRename);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn published_controller_terminal_receipt_sync_unknown_retains_owner_and_write_fence() {
|
||||
published_terminal_receipt_fault(TerminalFault::SyncUnknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
fn published_controller_terminal_receipt_read_unknown_retains_owner_and_write_fence() {
|
||||
published_terminal_receipt_fault(TerminalFault::ReadUnknown);
|
||||
}
|
||||
@@ -1,23 +1,32 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
#[cfg(feature = "http-server")]
|
||||
use std::time::Duration;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
use crate::workspace_request::{
|
||||
RuntimeWorkspaceRequest, RuntimeWorkspaceRequestClient, RuntimeWorkspaceRequestError,
|
||||
};
|
||||
use worker::WorkspaceResponse;
|
||||
#[cfg(feature = "http-server")]
|
||||
use worker::{
|
||||
WorkspaceBinaryRequest, WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError,
|
||||
WorkspacePromptCatalogResolution, WorkspacePromptProjection, WorkspaceRequest,
|
||||
WorkspaceRequestMethod, WorkspaceResponse, WorkspaceServerOperation,
|
||||
WorkspaceRequestMethod, WorkspaceServerOperation,
|
||||
};
|
||||
|
||||
use crate::auth::{
|
||||
RuntimeAuthError, RuntimeIdentityMaterial, RuntimeWorkerMutationSourceSigner,
|
||||
WORKER_REMOVE_PERMISSION, WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION,
|
||||
WorkerMutationActorKind, WorkerMutationOperation, WorkerMutationSourceClaims, new_token_id,
|
||||
WORKER_REMOVE_PERMISSION, WorkerMutationActorKind, WorkerMutationOperation,
|
||||
WorkerMutationSourceClaims, new_token_id,
|
||||
};
|
||||
#[cfg(feature = "http-server")]
|
||||
use crate::auth::{WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION};
|
||||
use crate::runtime::RuntimeWorkspaceScope;
|
||||
#[cfg(feature = "http-server")]
|
||||
use crate::worker_backend::WorkspacePromptProjectionCache;
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
fn ticket_assignment_role_segment(role: server_api::TicketAssignmentRole) -> &'static str {
|
||||
match role {
|
||||
server_api::TicketAssignmentRole::Orchestrator => "orchestrator",
|
||||
@@ -144,6 +153,7 @@ pub trait EmbeddedWorkerMutationDispatcher: Send + Sync {
|
||||
|
||||
#[derive(Clone)]
|
||||
enum RuntimeWorkerMutationTransport {
|
||||
#[cfg(feature = "http-server")]
|
||||
Remote {
|
||||
request_client: RuntimeWorkspaceRequestClient,
|
||||
},
|
||||
@@ -161,6 +171,7 @@ pub struct RuntimeWorkerMutationForwarder {
|
||||
}
|
||||
|
||||
impl RuntimeWorkerMutationForwarder {
|
||||
#[cfg(feature = "http-server")]
|
||||
pub fn remote(
|
||||
identity: &RuntimeIdentityMaterial,
|
||||
scope: RuntimeWorkspaceScope,
|
||||
@@ -206,6 +217,7 @@ impl RuntimeWorkerMutationForwarder {
|
||||
target_worker_id,
|
||||
)?;
|
||||
match (&self.transport, proof) {
|
||||
#[cfg(feature = "http-server")]
|
||||
(
|
||||
RuntimeWorkerMutationTransport::Remote { request_client },
|
||||
RuntimeOwnedWorkerMutationProof::Remote(token),
|
||||
@@ -231,6 +243,7 @@ impl RuntimeWorkerMutationForwarder {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
struct RemoteWorkerRemoveHttpRequest {
|
||||
request_client: RuntimeWorkspaceRequestClient,
|
||||
source_worker_id: String,
|
||||
@@ -240,6 +253,7 @@ struct RemoteWorkerRemoveHttpRequest {
|
||||
reason: String,
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
fn execute_remote_worker_remove_http(
|
||||
request: RemoteWorkerRemoveHttpRequest,
|
||||
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
|
||||
@@ -263,6 +277,7 @@ fn execute_remote_worker_remove_http(
|
||||
execute_remote_worker_remove_http_blocking(request)
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
fn execute_remote_worker_remove_http_blocking(
|
||||
request: RemoteWorkerRemoveHttpRequest,
|
||||
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
|
||||
@@ -315,6 +330,7 @@ fn execute_remote_worker_remove_http_blocking(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[derive(Clone)]
|
||||
pub struct RuntimeOwnedWorkspaceClient {
|
||||
workspace_id: String,
|
||||
@@ -325,6 +341,7 @@ pub struct RuntimeOwnedWorkspaceClient {
|
||||
prompt_projection_cache: Option<Arc<WorkspacePromptProjectionCache>>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
impl RuntimeOwnedWorkspaceClient {
|
||||
pub fn new(
|
||||
workspace_id: impl Into<String>,
|
||||
@@ -495,6 +512,7 @@ impl RuntimeOwnedWorkspaceClient {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
@@ -510,6 +528,7 @@ impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
fn generated_workspace_response<T: serde::Serialize>(
|
||||
response: Result<T, server_api::client_support::ClientError<server_api::RepositoryApiError>>,
|
||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
@@ -530,6 +549,7 @@ fn generated_workspace_response<T: serde::Serialize>(
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
|
||||
fn workspace_id(&self) -> Option<&str> {
|
||||
Some(self.request_client.workspace_id())
|
||||
@@ -929,12 +949,17 @@ fn unix_now_seconds() -> u64 {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[cfg(feature = "http-server")]
|
||||
use crate::auth::{
|
||||
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WorkerMutationSourceExpectation,
|
||||
decode_runtime_request_source_claims, decode_worker_mutation_source_claims,
|
||||
request_body_digest, verify_worker_mutation_source_proof,
|
||||
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, decode_runtime_request_source_claims,
|
||||
request_body_digest,
|
||||
};
|
||||
use crate::auth::{
|
||||
WorkerMutationSourceExpectation, decode_worker_mutation_source_claims,
|
||||
verify_worker_mutation_source_proof,
|
||||
};
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
fn binary_loopback(response: Vec<u8>) -> (String, std::thread::JoinHandle<(String, Vec<u8>)>) {
|
||||
use std::io::{Read, Write};
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
@@ -972,6 +997,7 @@ mod tests {
|
||||
(base_url, server)
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn generated_ticket_assignment_transport_signs_exact_authoring_and_bounded_read_subjects() {
|
||||
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
|
||||
@@ -1104,6 +1130,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn binary_roundtrip_signs_exact_query_method_worker_and_raw_body() {
|
||||
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
|
||||
@@ -1194,6 +1221,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn binary_responses_enforce_declared_and_streamed_limits_with_fixed_errors() {
|
||||
for response in [
|
||||
@@ -1213,6 +1241,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn binary_redirect_returns_status_without_forwarding_authentication() {
|
||||
let (base_url, server) = binary_loopback(b"HTTP/1.1 307 Temporary Redirect\r\nLocation: http://127.0.0.1:1/private\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec());
|
||||
@@ -1231,6 +1260,7 @@ mod tests {
|
||||
assert!(result.body.is_empty());
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn binary_invalid_or_normalized_targets_fail_with_fixed_errors_before_http() {
|
||||
let client = RuntimeOwnedWorkspaceClient::new(
|
||||
@@ -1277,6 +1307,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn current_prompt_projection_uses_the_shared_runtime_cache_without_http() {
|
||||
let cache = Arc::new(WorkspacePromptProjectionCache::default());
|
||||
@@ -1312,6 +1343,7 @@ mod tests {
|
||||
assert!(Arc::ptr_eq(&projection.catalog, &second.catalog));
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn prompt_projection_cache_miss_fetches_content_identity() {
|
||||
use std::io::{Read, Write};
|
||||
@@ -1356,6 +1388,7 @@ mod tests {
|
||||
assert_eq!(fetched.projection.source_digest, "source-3");
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn concurrent_prompt_projection_miss_fetches_once_and_shares_catalog() {
|
||||
use std::io::{Read, Write};
|
||||
@@ -1427,6 +1460,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn current_prompt_projection_rejects_cross_workspace_response() {
|
||||
use std::io::{Read, Write};
|
||||
@@ -1473,6 +1507,7 @@ mod tests {
|
||||
assert!(error.to_string().contains("scope mismatch"));
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn ordinary_workspace_forwarding_stamps_runtime_identity_and_signs_path_and_query() {
|
||||
use std::io::{Read, Write};
|
||||
@@ -1531,6 +1566,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn workspace_worker_discovery_signs_dedicated_permission_and_encoded_query() {
|
||||
use std::io::{Read, Write};
|
||||
@@ -1652,6 +1688,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn remote_forwarder_is_safe_in_async_runtime_and_stamps_signed_proof() {
|
||||
use std::io::{Read, Write};
|
||||
@@ -1815,6 +1852,7 @@ mod tests {
|
||||
assert_eq!(reason, "retire obsolete Worker");
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn runtime_owned_workspace_client_returns_successful_response_without_request_timeout() {
|
||||
// Effective deadline clearing is exercised with a short inherited deadline in
|
||||
@@ -1838,6 +1876,7 @@ mod tests {
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn runtime_owned_workspace_client_honors_per_request_timeouts() {
|
||||
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
|
||||
@@ -1857,6 +1896,7 @@ mod tests {
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
#[test]
|
||||
fn runtime_owned_workspace_client_reports_request_timeouts() {
|
||||
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
|
||||
@@ -1878,6 +1918,7 @@ mod tests {
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-server")]
|
||||
fn delayed_workspace_response(delay: Duration) -> (String, std::thread::JoinHandle<()>) {
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpListener;
|
||||
|
||||
@@ -3676,6 +3676,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() {
|
||||
crate::restore_test_environment::require_restore_test_environment();
|
||||
let repo = create_clean_repo();
|
||||
// ssh-agent and the broker receive relative socket paths in this owned
|
||||
// fixture tree, so the test does not depend on checkout/TMPDIR length.
|
||||
|
||||
@@ -7,7 +7,8 @@
|
||||
//! identity only and report CleanupPending/unknown, not current cleanliness or removal permission.
|
||||
//!
|
||||
//! The cleanup walker never reads ordinary file bodies or computes content digests. It records
|
||||
//! device/inode, type, regular-file executable bits, and size/nanosecond mtime for files and symlinks.
|
||||
//! device/inode, type, regular-file executable bits, and size/nanosecond mtime for non-directory
|
||||
//! entries, including FIFOs and Unix sockets. IPC endpoints are never opened for I/O or connected.
|
||||
//! Directories retain identity/type only because child unlink changes their timestamps. Symlink
|
||||
//! targets are neither read nor followed. This is metadata protection, not byte equality:
|
||||
//! same-size writes restoring mtime are not guaranteed to be detected. Normal Git inspection may
|
||||
@@ -735,7 +736,7 @@ mod linux {
|
||||
use std::ffi::{CString, OsStr};
|
||||
use std::os::fd::{AsRawFd, FromRawFd};
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::os::unix::fs::{FileTypeExt, MetadataExt};
|
||||
|
||||
#[repr(C)]
|
||||
struct OpenHow {
|
||||
@@ -916,7 +917,16 @@ mod linux {
|
||||
io_observation::record(io_observation::Event::EntryInspection);
|
||||
let file = open(dir, name, libc::O_PATH)?;
|
||||
let metadata = file.metadata()?;
|
||||
if !metadata.is_file() && !metadata.is_dir() && !metadata.file_type().is_symlink() {
|
||||
let kind = metadata.file_type();
|
||||
// O_PATH inspects IPC entries without opening a FIFO stream or connecting
|
||||
// to a socket. They are unlinked, never traversed, under the same identity
|
||||
// and mount checks as regular entries; their type alone is not occupancy.
|
||||
if !kind.is_file()
|
||||
&& !kind.is_dir()
|
||||
&& !kind.is_symlink()
|
||||
&& !kind.is_fifo()
|
||||
&& !kind.is_socket()
|
||||
{
|
||||
return Err(io::Error::from_raw_os_error(libc::EBUSY));
|
||||
}
|
||||
Ok(Entry {
|
||||
@@ -1163,6 +1173,91 @@ pub(super) mod tests {
|
||||
(runtime, materializer, binding)
|
||||
}
|
||||
|
||||
fn create_fifo(path: &Path) {
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
let name = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap();
|
||||
// SAFETY: NUL-terminated path inside the isolated test directory.
|
||||
assert_eq!(unsafe { libc::mkfifo(name.as_ptr(), 0o600) }, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cleanup_unlinks_fifo_and_socket_without_opening_or_connecting() {
|
||||
use std::os::unix::net::UnixListener;
|
||||
for retry in [false, true] {
|
||||
let (runtime, materializer, binding) = fixture();
|
||||
fs::write(binding.root.join(".git/info/exclude"), "target/\n").unwrap();
|
||||
let output = binding.root.join("target");
|
||||
fs::create_dir(&output).unwrap();
|
||||
let fifo = output.join("ready.fifo");
|
||||
let socket = output.join("service.sock");
|
||||
create_fifo(&fifo);
|
||||
let listener = UnixListener::bind(&socket).unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let mut watch = BodyWatch::new(&[fifo, socket]);
|
||||
let id = &binding.working_directory.id;
|
||||
if retry {
|
||||
let (result, audit) =
|
||||
io_observation::observe(Some(1), || materializer.cleanup_working_directory(id));
|
||||
assert_eq!(
|
||||
result.unwrap_err().code,
|
||||
"working_directory_cleanup_permission_denied"
|
||||
);
|
||||
assert_eq!(audit.unlinks, 1);
|
||||
assert_eq!(audit.entry_readable_opens, 0);
|
||||
}
|
||||
drop(materializer);
|
||||
let materializer = RuntimeGitMaterializer::new(runtime.path());
|
||||
let (result, audit) =
|
||||
io_observation::observe(None, || materializer.cleanup_working_directory(id));
|
||||
assert_eq!(
|
||||
result.unwrap().summary.status,
|
||||
WorkingDirectoryStatusKind::NotFound
|
||||
);
|
||||
assert!(!binding.working_directory_root.exists());
|
||||
assert_eq!(audit.entry_readable_opens, 0);
|
||||
watch.assert_no_body_access();
|
||||
assert_eq!(
|
||||
listener.accept().unwrap_err().kind(),
|
||||
io::ErrorKind::WouldBlock
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cleanup_preserves_replaced_fifo_and_socket_before_unlink() {
|
||||
use std::os::unix::net::UnixListener;
|
||||
for socket in [false, true] {
|
||||
let (runtime, materializer, binding) = fixture();
|
||||
fs::write(binding.root.join(".git/info/exclude"), "target/\n").unwrap();
|
||||
let output = binding.root.join("target");
|
||||
fs::create_dir(&output).unwrap();
|
||||
let path = output.join("endpoint");
|
||||
let _original = if socket {
|
||||
Some(UnixListener::bind(&path).unwrap())
|
||||
} else {
|
||||
create_fifo(&path);
|
||||
None
|
||||
};
|
||||
let error = remove_with_eraser(
|
||||
&materializer,
|
||||
&binding.working_directory.id,
|
||||
|tree, relative, entries| {
|
||||
fs::rename(&path, runtime.path().join("original-endpoint")).unwrap();
|
||||
let _replacement = if socket {
|
||||
Some(UnixListener::bind(&path).unwrap())
|
||||
} else {
|
||||
create_fifo(&path);
|
||||
None
|
||||
};
|
||||
erase(tree, relative, entries)
|
||||
},
|
||||
)
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code, "working_directory_cleanup_changes_present");
|
||||
assert!(path.symlink_metadata().is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn fail_once(materializer: &RuntimeGitMaterializer, id: &str) {
|
||||
let error = remove_with_eraser(materializer, id, |_, _, _| {
|
||||
Err(EraseError::Io(io::Error::from_raw_os_error(libc::EACCES)))
|
||||
|
||||
@@ -28,7 +28,19 @@ pub(super) struct Lease {
|
||||
id: String,
|
||||
}
|
||||
|
||||
fn busy() -> WorkingDirectoryDiagnostic {
|
||||
fn busy(id: &str, action: &str, blocking: &Entry) -> WorkingDirectoryDiagnostic {
|
||||
let (busy_reason, active_use_count) = match blocking {
|
||||
Entry::Use(count) => ("active_use", *count),
|
||||
Entry::Create => ("creation_in_progress", 0),
|
||||
Entry::Cleanup => ("cleanup_in_progress", 0),
|
||||
};
|
||||
tracing::warn!(target: "yoi::workdir_cleanup",
|
||||
working_directory_id = id,
|
||||
action,
|
||||
busy_source = "materializer_occupancy",
|
||||
busy_reason,
|
||||
active_use_count,
|
||||
"Workdir operation blocked by busy resource");
|
||||
WorkingDirectoryDiagnostic::new(
|
||||
"working_directory_cleanup_resource_busy",
|
||||
"A filesystem resource is busy or changed during removal. Release the resource and retry.",
|
||||
@@ -42,7 +54,9 @@ impl Occupancy {
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
match entries.get_mut(id) {
|
||||
Some(Entry::Cleanup | Entry::Create) => return Err(busy()),
|
||||
Some(blocking @ (Entry::Cleanup | Entry::Create)) => {
|
||||
return Err(busy(id, "acquire_use", blocking));
|
||||
}
|
||||
Some(Entry::Use(count)) => *count += 1,
|
||||
None => {
|
||||
entries.insert(id.to_string(), Entry::Use(1));
|
||||
@@ -101,8 +115,8 @@ impl Occupancy {
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
if entries.contains_key(id) {
|
||||
return Err(busy());
|
||||
if let Some(blocking) = entries.get(id) {
|
||||
return Err(busy(id, "acquire_creation", blocking));
|
||||
}
|
||||
entries.insert(id.to_string(), Entry::Create);
|
||||
Ok(Lease {
|
||||
@@ -116,8 +130,8 @@ impl Occupancy {
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner());
|
||||
if entries.contains_key(id) {
|
||||
return Err(busy());
|
||||
if let Some(blocking) = entries.get(id) {
|
||||
return Err(busy(id, "acquire_cleanup", blocking));
|
||||
}
|
||||
entries.insert(id.to_string(), Entry::Cleanup);
|
||||
Ok(Lease {
|
||||
@@ -131,6 +145,71 @@ impl Occupancy {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn busy_admission_logs_blocker_and_use_count_without_changing_error() {
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
|
||||
let mut output = tempfile::tempfile().unwrap();
|
||||
let writer = output.try_clone().unwrap();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.with_writer(move || writer.try_clone().unwrap())
|
||||
.finish();
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
tracing::callsite::rebuild_interest_cache();
|
||||
let occupancy = Occupancy::default();
|
||||
for (id, leases) in [
|
||||
(
|
||||
"workdir-used",
|
||||
vec![
|
||||
occupancy.acquire_use("workdir-used").unwrap(),
|
||||
occupancy.acquire_use("workdir-used").unwrap(),
|
||||
],
|
||||
),
|
||||
(
|
||||
"workdir-creating",
|
||||
vec![occupancy.acquire_creation("workdir-creating").unwrap()],
|
||||
),
|
||||
(
|
||||
"workdir-removing",
|
||||
vec![occupancy.acquire_cleanup("workdir-removing").unwrap()],
|
||||
),
|
||||
] {
|
||||
let error = occupancy.acquire_cleanup(id).unwrap_err();
|
||||
assert_eq!(error.code, "working_directory_cleanup_resource_busy");
|
||||
assert_eq!(
|
||||
error.message,
|
||||
"A filesystem resource is busy or changed during removal. Release the resource and retry."
|
||||
);
|
||||
drop(leases);
|
||||
drop(occupancy.acquire_cleanup(id).unwrap());
|
||||
}
|
||||
});
|
||||
output.seek(SeekFrom::Start(0)).unwrap();
|
||||
let mut log = String::new();
|
||||
output.read_to_string(&mut log).unwrap();
|
||||
let events: Vec<_> = log.lines().collect();
|
||||
assert_eq!(events.len(), 3, "{log}");
|
||||
for (event, (id, reason, count)) in events.iter().zip([
|
||||
("workdir-used", "active_use", 2),
|
||||
("workdir-creating", "creation_in_progress", 0),
|
||||
("workdir-removing", "cleanup_in_progress", 0),
|
||||
]) {
|
||||
for field in [
|
||||
"WARN".to_string(),
|
||||
"yoi::workdir_cleanup".to_string(),
|
||||
format!("working_directory_id=\"{id}\""),
|
||||
"action=\"acquire_cleanup\"".to_string(),
|
||||
"busy_source=\"materializer_occupancy\"".to_string(),
|
||||
format!("busy_reason=\"{reason}\""),
|
||||
format!("active_use_count={count}"),
|
||||
] {
|
||||
assert!(event.contains(&field), "missing {field}: {event}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn use_and_cleanup_leases_release_during_unwind() {
|
||||
let occupancy = Occupancy::default();
|
||||
|
||||
@@ -7,6 +7,7 @@ use workdir::WorkdirSessionHandle;
|
||||
|
||||
use crate::PromptCatalogSource;
|
||||
use crate::controller::{ShutdownReceiver, WorkerController, WorkerHandle};
|
||||
pub use crate::worker::{PreparedRestoredWorker, RestoreCandidateWorkdirCleanup};
|
||||
use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext};
|
||||
use manifest::WorkerManifest;
|
||||
|
||||
@@ -161,6 +162,38 @@ where
|
||||
Ok(PreparedWorker::new(worker, self.layout))
|
||||
}
|
||||
|
||||
/// Reconstruct a saved materialized Worker without allocation, attribution
|
||||
/// repair, Feature installation, or controller exposure. The caller owns the
|
||||
/// later controller layout and must activate before starting it.
|
||||
pub async fn prepare_restored_candidate(
|
||||
self,
|
||||
worker_name: &str,
|
||||
) -> Result<PreparedRestoredWorker<St>, WorkerBootstrapError> {
|
||||
let mut candidate =
|
||||
Worker::prepare_restored_from_worker_metadata_with_context_and_model_client(
|
||||
worker_name,
|
||||
self.manifest,
|
||||
self.store,
|
||||
self.prompt_catalog,
|
||||
self.workspace_context,
|
||||
self.filesystem_authority,
|
||||
self.model_client,
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
if let Some(workdir_session) = self.workdir_session {
|
||||
candidate
|
||||
.worker_mut()
|
||||
.map_err(WorkerBootstrapError::Worker)?
|
||||
.bind_single_workdir_session(Some(workdir_session));
|
||||
}
|
||||
candidate
|
||||
.worker_mut()
|
||||
.map_err(WorkerBootstrapError::Worker)?
|
||||
.bind_subjektiv_host(self.subjektiv_host);
|
||||
Ok(candidate)
|
||||
}
|
||||
|
||||
pub async fn prepare_restored(
|
||||
self,
|
||||
worker_name: &str,
|
||||
@@ -243,6 +276,7 @@ where
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
let cleanup_sessions = worker.workdir_sessions();
|
||||
let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some();
|
||||
let controller = match layout {
|
||||
WorkerBootstrapLayout::Direct { runtime_base } => {
|
||||
WorkerController::spawn_owned(worker, &runtime_base).await
|
||||
@@ -259,7 +293,10 @@ where
|
||||
controller_task,
|
||||
}),
|
||||
Err(source) => {
|
||||
let cleanup_failed = cleanup_sessions.close_all().await.is_err();
|
||||
// Factory-owned task scopes must settle before sessions close.
|
||||
// Cleanup remains unconfirmed here; preserve the startup source.
|
||||
let cleanup_failed =
|
||||
factory_owned_cleanup || cleanup_sessions.close_all().await.is_err();
|
||||
Err(WorkerBootstrapError::Controller {
|
||||
source,
|
||||
cleanup_failed,
|
||||
@@ -267,3 +304,229 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Host bindings for one unpublished restored Controller. These do not expose
|
||||
/// Worker execution or permit arbitrary mutation during preparation.
|
||||
pub struct RestoredControllerBindings {
|
||||
pub backend_job: Option<crate::BackendJobExecutionBinding>,
|
||||
pub workdir_sessions: std::sync::Arc<workdir::WorkdirSessionRouter>,
|
||||
pub observation_provider:
|
||||
Option<std::sync::Arc<dyn crate::feature::builtin::WorkerObservationProvider>>,
|
||||
pub subjektiv_attached: bool,
|
||||
}
|
||||
|
||||
/// The same real receiver, snapshot sources and paired subscription retained
|
||||
/// privately across the Runtime's durable restore decision.
|
||||
pub struct UnpublishedRestoredController<St: Store> {
|
||||
initialized: Option<crate::controller::InitializedWorkerController<Box<dyn LlmClient>, St>>,
|
||||
activation: Option<crate::worker::RestoredWorkerActivation>,
|
||||
streams: Option<crate::ipc::protocol_session::WorkerProtocolSessionStreams>,
|
||||
subjektiv_attached: bool,
|
||||
effects_applied: bool,
|
||||
activation_failed: bool,
|
||||
closing: bool,
|
||||
}
|
||||
|
||||
pub struct ActivatedRestoredController {
|
||||
pub controller: BootstrappedWorker,
|
||||
pub streams: crate::ipc::protocol_session::WorkerProtocolSessionStreams,
|
||||
}
|
||||
|
||||
impl<St> PreparedRestoredWorker<St>
|
||||
where
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
pub fn workspace_client_handle(
|
||||
&mut self,
|
||||
) -> Result<std::sync::Arc<dyn crate::WorkspaceClient>, WorkerError> {
|
||||
Ok(self.worker_mut()?.workspace_client_handle())
|
||||
}
|
||||
|
||||
pub fn retain_startup_feature_cleanup(
|
||||
&mut self,
|
||||
) -> Result<crate::WorkerStartupFeatureCleanup, WorkerError> {
|
||||
Ok(self.worker_mut()?.retain_startup_feature_cleanup())
|
||||
}
|
||||
|
||||
/// Initialize the real Controller without allocation, canonical restore
|
||||
/// writes, attribution changes, registry reclamation or input consumption.
|
||||
/// MCP discovery and Feature tasks belong to the retained startup cleanup.
|
||||
pub async fn prepare_runtime_controller(
|
||||
mut self,
|
||||
run_dir: &std::path::Path,
|
||||
bindings: RestoredControllerBindings,
|
||||
) -> Result<UnpublishedRestoredController<St>, WorkerBootstrapError> {
|
||||
let worker = self.worker_mut().map_err(WorkerBootstrapError::Worker)?;
|
||||
if let Some(binding) = bindings.backend_job {
|
||||
worker.bind_backend_job(binding).map_err(|error| {
|
||||
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.into()))
|
||||
})?;
|
||||
}
|
||||
worker
|
||||
.workdir_sessions()
|
||||
.close_all()
|
||||
.await
|
||||
.map_err(|error| {
|
||||
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.to_string()))
|
||||
})?;
|
||||
worker.bind_workdir_sessions(bindings.workdir_sessions);
|
||||
worker.bind_worker_observation_provider(bindings.observation_provider);
|
||||
let flow_enabled = worker.manifest().feature.flow.enabled;
|
||||
if flow_enabled {
|
||||
let report = worker
|
||||
.install_runtime_flow_transition_feature()
|
||||
.map_err(|error| {
|
||||
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error))
|
||||
})?;
|
||||
if report.reports.iter().any(|report| !report.installed) {
|
||||
return Err(WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(
|
||||
"Flow transition installation refused".into(),
|
||||
)));
|
||||
}
|
||||
}
|
||||
let (worker, activation) = self
|
||||
.into_controller_parts()
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
let initialized = WorkerController::initialize_restored(worker, run_dir)
|
||||
.await
|
||||
.map_err(|source| WorkerBootstrapError::Controller {
|
||||
source,
|
||||
cleanup_failed: true,
|
||||
})?;
|
||||
if flow_enabled {
|
||||
initialized.handle.shared_state.enable_flow_transition();
|
||||
}
|
||||
let streams = {
|
||||
let _guard = initialized.handle.protocol_snapshot_publish_guard();
|
||||
crate::ipc::protocol_session::subscribe_worker_protocol_session(&initialized.handle)
|
||||
};
|
||||
Ok(UnpublishedRestoredController {
|
||||
initialized: Some(initialized),
|
||||
activation: Some(activation),
|
||||
streams: Some(streams),
|
||||
subjektiv_attached: bindings.subjektiv_attached,
|
||||
effects_applied: false,
|
||||
activation_failed: false,
|
||||
closing: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<St> UnpublishedRestoredController<St>
|
||||
where
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
pub fn snapshot(&self) -> &protocol::Event {
|
||||
&self
|
||||
.streams
|
||||
.as_ref()
|
||||
.expect("unpublished restore streams")
|
||||
.snapshot_event
|
||||
}
|
||||
|
||||
/// Only publication failures after confirmed effects can retry on this
|
||||
/// exact owner. Partial activation must be reconciled, never blindly rerun.
|
||||
pub fn can_retry_publication(&self) -> bool {
|
||||
self.initialized.is_some()
|
||||
&& self.effects_applied
|
||||
&& !self.activation_failed
|
||||
&& !self.closing
|
||||
}
|
||||
|
||||
/// Apply canonical restore effects after the host's durable decision, while
|
||||
/// retaining the same private Controller for subsequent publication.
|
||||
pub async fn apply_effects(&mut self) -> Result<(), WorkerBootstrapError> {
|
||||
let initialized = self
|
||||
.initialized
|
||||
.as_mut()
|
||||
.ok_or_else(|| WorkerBootstrapError::Worker(WorkerError::RestoreCandidateClosing))?;
|
||||
if self.activation_failed || self.closing {
|
||||
return Err(WorkerBootstrapError::Worker(
|
||||
WorkerError::RestoreCandidateClosing,
|
||||
));
|
||||
}
|
||||
// Preparation's read-only authorization is not a publication grant.
|
||||
// Revalidate current Host authority after commit, before effects/input,
|
||||
// including a publication retry on this same private Controller.
|
||||
initialized
|
||||
.worker
|
||||
.validate_subjektiv_body_publication()
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
if !self.effects_applied {
|
||||
self.activation_failed = true;
|
||||
self.activation
|
||||
.as_ref()
|
||||
.expect("restore activation")
|
||||
.activate(&mut initialized.worker)
|
||||
.await
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
initialized
|
||||
.worker
|
||||
.finalize_subjektiv_session_attribution(
|
||||
self.subjektiv_attached,
|
||||
crate::SubjektivSessionAttributionLifecycle::RestoredSession,
|
||||
)
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
self.effects_applied = true;
|
||||
self.activation_failed = false;
|
||||
self.activation = None;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn allocation_cleanup(
|
||||
&self,
|
||||
) -> Option<crate::runtime::worker_allocation::ScopeAllocationCleanup> {
|
||||
self.initialized
|
||||
.as_ref()
|
||||
.and_then(|initialized| initialized.worker.restore_allocation_cleanup_handle())
|
||||
}
|
||||
|
||||
pub async fn activate(&mut self) -> Result<ActivatedRestoredController, WorkerBootstrapError> {
|
||||
self.apply_effects().await?;
|
||||
let initialized = self
|
||||
.initialized
|
||||
.as_mut()
|
||||
.expect("same initialized Controller");
|
||||
let publication = initialized.prepare_publication().await.map_err(|source| {
|
||||
WorkerBootstrapError::Controller {
|
||||
source,
|
||||
cleanup_failed: true,
|
||||
}
|
||||
})?;
|
||||
let initialized = self
|
||||
.initialized
|
||||
.take()
|
||||
.expect("same initialized Controller");
|
||||
let (handle, shutdown, controller_task) = initialized.start(publication);
|
||||
Ok(ActivatedRestoredController {
|
||||
controller: BootstrappedWorker {
|
||||
handle,
|
||||
shutdown,
|
||||
controller_task,
|
||||
},
|
||||
streams: self.streams.take().expect("same paired subscription"),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn discard(&mut self) -> Result<(), WorkerError> {
|
||||
self.closing = true;
|
||||
if let Some(initialized) = self.initialized.as_mut() {
|
||||
initialized
|
||||
.worker
|
||||
.retain_startup_feature_cleanup()
|
||||
.shutdown()
|
||||
.await
|
||||
.map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?;
|
||||
initialized
|
||||
.worker
|
||||
.workdir_sessions()
|
||||
.close_all()
|
||||
.await
|
||||
.map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?;
|
||||
}
|
||||
self.streams = None;
|
||||
self.initialized = None;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
+373
-30
@@ -21,7 +21,7 @@ use crate::shutdown_after_idle::{
|
||||
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
|
||||
take_shutdown_request_after_status,
|
||||
};
|
||||
use crate::spawn::registry::SpawnedWorkerRegistry;
|
||||
use crate::spawn::registry::{PreparedSpawnedWorkerRegistryLoad, SpawnedWorkerRegistry};
|
||||
use crate::spawn::tool::sub_worker_spawn_tool;
|
||||
use crate::worker::{
|
||||
PendingSubmissionHandle, SystemItemCommitter, Worker, WorkerError, WorkerRunResult,
|
||||
@@ -654,7 +654,56 @@ pub type ShutdownReceiver = oneshot::Receiver<()>;
|
||||
|
||||
pub struct WorkerController;
|
||||
|
||||
// Own the real command receiver and all snapshot sources before starting any
|
||||
// protocol listener or controller loop. Fresh startup may materialize the head;
|
||||
// Runtime restore requires an already materialized source and defers canonical
|
||||
// restore effects until its durable decision. Tool discovery can own external
|
||||
// resources, so private initialization is not synonymous with side-effect-free.
|
||||
pub(crate) struct InitializedWorkerController<C: LlmClient, St: Store> {
|
||||
pub(crate) worker: Worker<C, St>,
|
||||
pub(crate) handle: WorkerHandle,
|
||||
method_rx: mpsc::Receiver<Method>,
|
||||
shutdown_tx: oneshot::Sender<()>,
|
||||
shutdown_rx: ShutdownReceiver,
|
||||
registry_reclaim: PreparedSpawnedWorkerRegistryLoad<St>,
|
||||
shutdown_after_idle: ShutdownAfterIdleRequest,
|
||||
command_events: Option<PreparedWorkdirCommandEvents>,
|
||||
}
|
||||
|
||||
pub(crate) struct PreparedControllerPublication {
|
||||
command_observer: Option<tokio::task::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl WorkerController {
|
||||
/// Explicit recovery of an unmaterialized Session head. This installs the
|
||||
/// saved Feature composition needed to produce the seed, but creates no
|
||||
/// Controller, input channel, socket, observer, or queue consumer. The caller
|
||||
/// owns the same operation's Feature/session/allocation cleanup on every exit.
|
||||
pub async fn recover_pending_head<C, St>(
|
||||
worker: &mut Worker<C, St>,
|
||||
runtime_base: &Path,
|
||||
) -> std::io::Result<()>
|
||||
where
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
if worker.needs_initial_session_head_materialization() {
|
||||
worker
|
||||
.validate_subjektiv_body_publication()
|
||||
.map_err(std::io::Error::other)?;
|
||||
let registry = SpawnedWorkerRegistry::new_internal(
|
||||
worker.manifest().worker.name.clone(),
|
||||
worker.scope().clone(),
|
||||
);
|
||||
register_worker_tools(worker, runtime_base.to_path_buf(), registry, None, None).await?;
|
||||
worker
|
||||
.materialize_durable_session_head()
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn spawn<C, St>(
|
||||
worker: Worker<C, St>,
|
||||
runtime_base: &Path,
|
||||
@@ -734,8 +783,12 @@ impl WorkerController {
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
let sessions = worker.workdir_sessions();
|
||||
let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some();
|
||||
let result = Self::spawn_initialized(worker, runtime_base, runtime_run).await;
|
||||
// An opted-in factory must quiesce its captured Feature tasks before
|
||||
// closing their Workdir sessions; this wrapper cannot confirm that order.
|
||||
if result.is_err()
|
||||
&& !factory_owned_cleanup
|
||||
&& let Err(error) = sessions.close_all().await
|
||||
{
|
||||
tracing::warn!(%error, "Workdir sessions close after controller startup failure failed");
|
||||
@@ -744,7 +797,7 @@ impl WorkerController {
|
||||
}
|
||||
|
||||
async fn spawn_initialized<C, St>(
|
||||
mut worker: Worker<C, St>,
|
||||
worker: Worker<C, St>,
|
||||
runtime_base: &Path,
|
||||
runtime_run: Option<&Path>,
|
||||
) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error>
|
||||
@@ -752,6 +805,52 @@ impl WorkerController {
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Self::initialize(worker, runtime_base, runtime_run)
|
||||
.await?
|
||||
.publish()
|
||||
.await
|
||||
}
|
||||
|
||||
async fn initialize<C, St>(
|
||||
worker: Worker<C, St>,
|
||||
runtime_base: &Path,
|
||||
runtime_run: Option<&Path>,
|
||||
) -> Result<InitializedWorkerController<C, St>, std::io::Error>
|
||||
where
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Self::initialize_inner(worker, runtime_base, runtime_run, true).await
|
||||
}
|
||||
|
||||
pub(crate) async fn initialize_restored<St>(
|
||||
worker: Worker<Box<dyn LlmClient>, St>,
|
||||
run_dir: &Path,
|
||||
) -> std::io::Result<InitializedWorkerController<Box<dyn LlmClient>, St>>
|
||||
where
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
let parent = run_dir
|
||||
.parent()
|
||||
.ok_or_else(|| std::io::Error::other("run path has no parent"))?;
|
||||
Self::initialize_inner(worker, parent, Some(run_dir), false).await
|
||||
}
|
||||
|
||||
async fn initialize_inner<C, St>(
|
||||
mut worker: Worker<C, St>,
|
||||
runtime_base: &Path,
|
||||
runtime_run: Option<&Path>,
|
||||
canonical_startup: bool,
|
||||
) -> Result<InitializedWorkerController<C, St>, std::io::Error>
|
||||
where
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
if !canonical_startup && worker.needs_initial_session_head_materialization() {
|
||||
return Err(std::io::Error::other(
|
||||
"restore source requires operation-owned head recovery",
|
||||
));
|
||||
}
|
||||
worker
|
||||
.validate_subjektiv_body_publication()
|
||||
.map_err(std::io::Error::other)?;
|
||||
@@ -773,21 +872,13 @@ impl WorkerController {
|
||||
});
|
||||
|
||||
let spawner_name = worker.manifest().worker.name.clone();
|
||||
let loaded_registry = SpawnedWorkerRegistry::load_from_worker_state_with_reclaim(
|
||||
let registry_reclaim = SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim(
|
||||
runtime_dir.clone(),
|
||||
worker.store().clone(),
|
||||
spawner_name.clone(),
|
||||
Some(worker.scope().clone()),
|
||||
)
|
||||
.await?;
|
||||
let reclaimed_unreachable = loaded_registry.reclaimed_unreachable;
|
||||
let spawned_registry = loaded_registry.registry;
|
||||
if reclaimed_unreachable {
|
||||
worker.push_notify(
|
||||
"Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume."
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
)?;
|
||||
let spawned_registry = registry_reclaim.registry.clone();
|
||||
|
||||
// Hand the alerter to the Worker so internal operations (compaction,
|
||||
// AGENTS.md ingestion during the first turn) can emit user-facing
|
||||
@@ -878,7 +969,6 @@ impl WorkerController {
|
||||
if let Some(fs_for_view) = fs_for_view.as_ref() {
|
||||
shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view.clone()));
|
||||
}
|
||||
|
||||
let artifact_store: Arc<dyn Store> = Arc::new(worker.store().clone());
|
||||
let session_id = worker.session_id();
|
||||
let pending_activations = worker.pending_activation_state();
|
||||
@@ -903,10 +993,71 @@ impl WorkerController {
|
||||
pending_activations,
|
||||
};
|
||||
|
||||
// Start the observer only after fallible head/recovery initialization.
|
||||
let command_observer = fs_for_view
|
||||
let command_events = fs_for_view
|
||||
.as_ref()
|
||||
.and_then(|session| wire_workdir_command_events(session, &in_flight));
|
||||
.map(|session| PreparedWorkdirCommandEvents::new(session, &handle.in_flight));
|
||||
Ok(InitializedWorkerController {
|
||||
worker,
|
||||
handle,
|
||||
method_rx,
|
||||
shutdown_tx,
|
||||
shutdown_rx,
|
||||
registry_reclaim,
|
||||
shutdown_after_idle,
|
||||
command_events,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<C, St> InitializedWorkerController<C, St>
|
||||
where
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
async fn publish(
|
||||
mut self,
|
||||
) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error> {
|
||||
let publication = self.prepare_publication().await?;
|
||||
Ok(self.start(publication))
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_publication(
|
||||
&mut self,
|
||||
) -> std::io::Result<PreparedControllerPublication> {
|
||||
let loaded_registry = self.registry_reclaim.apply().await?;
|
||||
debug_assert!(Arc::ptr_eq(
|
||||
&loaded_registry.registry,
|
||||
&self.handle.spawned_registry
|
||||
));
|
||||
if loaded_registry.reclaimed_unreachable {
|
||||
self.worker.push_notify("Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume.".to_string());
|
||||
}
|
||||
let command_observer = self
|
||||
.command_events
|
||||
.take()
|
||||
.and_then(PreparedWorkdirCommandEvents::start);
|
||||
Ok(PreparedControllerPublication { command_observer })
|
||||
}
|
||||
|
||||
pub(crate) fn start(
|
||||
self,
|
||||
publication: PreparedControllerPublication,
|
||||
) -> (WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>) {
|
||||
let Self {
|
||||
mut worker,
|
||||
handle,
|
||||
method_rx,
|
||||
shutdown_tx,
|
||||
shutdown_rx,
|
||||
shutdown_after_idle,
|
||||
..
|
||||
} = self;
|
||||
let PreparedControllerPublication { command_observer } = publication;
|
||||
let runtime_dir = handle.runtime_dir.clone();
|
||||
let shared_state = handle.shared_state.clone();
|
||||
let working_event_tx = handle.working_event_tx.clone();
|
||||
let spawned_registry = handle.spawned_registry.clone();
|
||||
|
||||
// === 5. controller_loop ===
|
||||
// Clone cancel sender and notification buffer before moving worker
|
||||
// into the controller task so the in-flight turn can be reached
|
||||
@@ -915,6 +1066,7 @@ impl WorkerController {
|
||||
let pause_tx = worker.engine_mut().pause_sender();
|
||||
let notify_buffer = worker.notify_buffer_handle();
|
||||
|
||||
worker.finish_startup_feature_cleanup_capture();
|
||||
let controller_task = tokio::spawn(controller_loop(
|
||||
worker,
|
||||
method_rx,
|
||||
@@ -930,7 +1082,7 @@ impl WorkerController {
|
||||
command_observer,
|
||||
));
|
||||
|
||||
Ok((handle, shutdown_rx, controller_task))
|
||||
(handle, shutdown_rx, controller_task)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -938,15 +1090,35 @@ pub(crate) fn wire_workdir_command_events(
|
||||
session: &Arc<dyn WorkdirSession>,
|
||||
in_flight: &InFlightEvents,
|
||||
) -> Option<tokio::task::JoinHandle<()>> {
|
||||
PreparedWorkdirCommandEvents::new(session, in_flight).start()
|
||||
}
|
||||
|
||||
struct PreparedWorkdirCommandEvents {
|
||||
session: std::sync::Weak<dyn WorkdirSession>,
|
||||
in_flight: InFlightEvents,
|
||||
events: Option<broadcast::Receiver<WorkdirCommandEvent>>,
|
||||
}
|
||||
|
||||
impl PreparedWorkdirCommandEvents {
|
||||
fn new(session: &Arc<dyn WorkdirSession>, in_flight: &InFlightEvents) -> Self {
|
||||
// Subscribe before reading command state; hold the receiver privately
|
||||
// across the durable decision instead of creating a publication gap.
|
||||
let events = session.subscribe_command_events();
|
||||
in_flight.replace_command_snapshot(protocol_command_snapshots(session.as_ref()));
|
||||
let Some(mut events) = session.subscribe_command_events() else {
|
||||
return None;
|
||||
};
|
||||
// Keep only a weak reference in the observer task. Holding the session
|
||||
// strongly here would keep its broadcast sender alive forever and prevent
|
||||
// the receiver from observing closure during Worker teardown.
|
||||
let session = Arc::downgrade(session);
|
||||
let in_flight = in_flight.clone();
|
||||
Self {
|
||||
session: Arc::downgrade(session),
|
||||
in_flight: in_flight.clone(),
|
||||
events,
|
||||
}
|
||||
}
|
||||
|
||||
fn start(self) -> Option<tokio::task::JoinHandle<()>> {
|
||||
let Self {
|
||||
session,
|
||||
in_flight,
|
||||
events,
|
||||
} = self;
|
||||
let mut events = events?;
|
||||
Some(tokio::spawn(async move {
|
||||
loop {
|
||||
match events.recv().await {
|
||||
@@ -963,6 +1135,7 @@ pub(crate) fn wire_workdir_command_events(
|
||||
}
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
fn protocol_command_snapshots(session: &dyn WorkdirSession) -> Vec<ProtocolCommandSnapshot> {
|
||||
session
|
||||
@@ -1687,9 +1860,17 @@ where
|
||||
.add_module(crate::feature::builtin::orchestration::orchestration_feature());
|
||||
}
|
||||
if let Some(workspace_root) = local_workspace_root.as_ref() {
|
||||
if let Some(module) =
|
||||
let module = if let Some(cleanup) = worker.startup_feature_cleanup_handle() {
|
||||
crate::feature::mcp::discover_stdio_tool_feature_with_cleanup(
|
||||
&mcp_config,
|
||||
workspace_root,
|
||||
&cleanup.mcp_discovery(),
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
crate::feature::mcp::discover_stdio_tool_feature(&mcp_config, workspace_root).await
|
||||
{
|
||||
};
|
||||
if let Some(module) = module {
|
||||
feature_registry = feature_registry.with_module(module);
|
||||
}
|
||||
}
|
||||
@@ -3750,6 +3931,159 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// Component connection: real Worker/Store, initialized endpoint, paired
|
||||
// protocol subscription and Controller loop. The model boundary is an
|
||||
// empty non-network script; this test must not request any generation.
|
||||
#[tokio::test]
|
||||
async fn initialized_controller_publishes_the_same_endpoint_and_subscription() {
|
||||
let root = TempDir::new().unwrap();
|
||||
let manifest = manifest::WorkerManifest::from_toml(
|
||||
"[worker]\nname = 'p'\n[model]\nscheme = 'anthropic'\nmodel_id = 'test'\n[engine]\n[scope]\n",
|
||||
)
|
||||
.unwrap();
|
||||
let client = ScriptedClient::new(vec![]);
|
||||
let requests = client.requests.clone();
|
||||
let engine =
|
||||
Engine::<_, agen::state::Mutable, crate::SessionHistoryMetadata>::new_annotated(
|
||||
client,
|
||||
);
|
||||
let store = session_store::CombinedStore::new(
|
||||
session_store::FsStore::new(root.path().join("sessions")).unwrap(),
|
||||
session_store::FsWorkerStore::new(root.path().join("workers")).unwrap(),
|
||||
);
|
||||
let mut worker = Worker::new(
|
||||
manifest,
|
||||
engine,
|
||||
store,
|
||||
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
|
||||
crate::worker::WorkerFilesystemAuthority::None,
|
||||
manifest::Scope::empty(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
worker.set_system_prompt_template(
|
||||
crate::SystemPromptTemplate::parse(
|
||||
"default",
|
||||
crate::PromptCatalogSource::builtins_only(),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
let prepared = WorkerController::initialize(worker, root.path(), None)
|
||||
.await
|
||||
.unwrap();
|
||||
let endpoint = prepared.handle.clone();
|
||||
let mut streams =
|
||||
crate::ipc::protocol_session::subscribe_worker_protocol_session(&endpoint);
|
||||
let entries = endpoint.committed_entries();
|
||||
assert!(matches!(
|
||||
&streams.snapshot_event,
|
||||
Event::Snapshot { greeting, state, session, .. }
|
||||
if !greeting.tools.is_empty()
|
||||
&& state.state == WorkerState::Idle
|
||||
&& session == &session_store::public_snapshot::project_current_session_snapshot(&entries)
|
||||
));
|
||||
assert!(!endpoint.protocol_is_closed());
|
||||
assert!(!endpoint.runtime_dir.path().join("sock").exists());
|
||||
assert!(!endpoint.runtime_dir.path().join("status.json").exists());
|
||||
assert!(!endpoint.runtime_dir.path().join("manifest.toml").exists());
|
||||
assert!(matches!(
|
||||
streams.events.try_recv(),
|
||||
Err(broadcast::error::TryRecvError::Empty)
|
||||
));
|
||||
assert!(requests.lock().unwrap().is_empty());
|
||||
|
||||
let (handle, shutdown, task) = prepared.publish().await.unwrap();
|
||||
assert!(handle.same_controller(&endpoint));
|
||||
assert!(!handle.runtime_dir.path().join("sock").exists());
|
||||
assert!(!handle.runtime_dir.path().join("status.json").exists());
|
||||
assert!(!handle.runtime_dir.path().join("manifest.toml").exists());
|
||||
handle.send(Method::ListPendingSubmissions).await.unwrap();
|
||||
let pending = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if let Event::PendingSubmissionsChanged { pending } =
|
||||
streams.events.recv().await.unwrap()
|
||||
{
|
||||
break pending;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(pending, protocol::PendingSubmissionsSnapshot::default());
|
||||
assert_eq!(
|
||||
serde_json::to_value(handle.committed_entries()).unwrap(),
|
||||
serde_json::to_value(entries).unwrap(),
|
||||
);
|
||||
handle
|
||||
.send(Method::Shutdown {
|
||||
command: WorkerCommandEnvelope { command_id: 1 },
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), shutdown)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), task)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(endpoint.protocol_is_closed());
|
||||
assert!(!endpoint.runtime_dir.path().join("sock").exists());
|
||||
assert!(requests.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tool_initialization_failure_does_not_reclaim_saved_children() {
|
||||
let root = TempDir::new().unwrap();
|
||||
let mut manifest = manifest::WorkerManifest::from_toml(
|
||||
"[worker]\nname = 'p'\n[model]\nscheme = 'anthropic'\nmodel_id = 'test'\n[engine]\n[scope]\n",
|
||||
).unwrap();
|
||||
manifest.feature.ticket.enabled = true;
|
||||
let client = ScriptedClient::new(vec![]);
|
||||
let requests = client.requests.clone();
|
||||
let store = session_store::CombinedStore::new(
|
||||
session_store::FsStore::new(root.path().join("sessions")).unwrap(),
|
||||
session_store::FsWorkerStore::new(root.path().join("workers")).unwrap(),
|
||||
);
|
||||
// New feature activation is refused by the real unavailable host;
|
||||
// legacy child metadata must not be consumed before that refusal.
|
||||
let worker = Worker::new(
|
||||
manifest,
|
||||
Engine::<_, agen::state::Mutable, crate::SessionHistoryMetadata>::new_annotated(
|
||||
client,
|
||||
),
|
||||
store.clone(),
|
||||
crate::worker::WorkerWorkspaceContext::no_workspace(),
|
||||
crate::worker::WorkerFilesystemAuthority::None,
|
||||
manifest::Scope::empty(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut metadata = session_store::WorkerMetadata::new(
|
||||
"p",
|
||||
Some(session_store::WorkerActiveSegmentRef {
|
||||
session_id: worker.session_id(),
|
||||
segment_id: Some(worker.segment_id()),
|
||||
}),
|
||||
);
|
||||
metadata
|
||||
.spawned_children
|
||||
.push(session_store::WorkerSpawnedChild {
|
||||
worker_name: "legacy".into(),
|
||||
socket_path: root.path().join("legacy.sock"),
|
||||
callback_address: root.path().join("p/sock"),
|
||||
scope_delegated: Vec::new(),
|
||||
});
|
||||
store.write(&metadata).unwrap();
|
||||
let before = store.read_by_name("p").unwrap();
|
||||
let result = WorkerController::initialize(worker, root.path(), None).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(store.read_by_name("p").unwrap(), before);
|
||||
assert!(!root.path().join("p/spawned_workers.json").exists());
|
||||
assert!(requests.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DriveWorkspaceHost {
|
||||
available: bool,
|
||||
@@ -4077,10 +4411,19 @@ mod tests {
|
||||
assert!(feature_install < materialize_gate);
|
||||
assert!(materialize_gate < materialize);
|
||||
assert!(materialize < exposure);
|
||||
let observer = startup.find("let command_observer = fs_for_view").unwrap();
|
||||
let observer_preparation = startup
|
||||
.find("PreparedWorkdirCommandEvents::new(session")
|
||||
.unwrap();
|
||||
let recover = startup.find(".recover_unfinished_compaction()").unwrap();
|
||||
let reclaim = startup
|
||||
.find("self.registry_reclaim.apply().await?")
|
||||
.unwrap();
|
||||
let observer_start = startup
|
||||
.find(".and_then(PreparedWorkdirCommandEvents::start)")
|
||||
.unwrap();
|
||||
let loop_start = startup.find("tokio::spawn(controller_loop(").unwrap();
|
||||
assert!(recover < observer && observer < loop_start);
|
||||
assert!(recover < observer_preparation && observer_preparation < reclaim);
|
||||
assert!(reclaim < observer_start && observer_start < loop_start);
|
||||
assert!(
|
||||
startup[materialize_gate..materialize]
|
||||
.contains("needs_initial_session_head_materialization")
|
||||
|
||||
@@ -26,7 +26,7 @@ use flow::{
|
||||
use manifest::{Scope, WorkerManifest};
|
||||
use schemars::JsonSchema;
|
||||
use serde::Deserialize;
|
||||
use session_store::{SegmentId, SessionId, Store, collect_state};
|
||||
use session_store::{SegmentId, SessionId, Store};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::feature::builtin::session_explore::{SessionExploreFeature, SessionExploreState};
|
||||
@@ -469,10 +469,15 @@ where
|
||||
.store
|
||||
.read_all(self.session_id, self.segment_id)
|
||||
.map_err(|error| format!("read committed parent session: {error}"))?;
|
||||
let restored = collect_state(&entries);
|
||||
Ok(SessionCapture::new(
|
||||
let history = crate::session_history::restore_history_entries(
|
||||
self.session_id,
|
||||
self.segment_id,
|
||||
&entries,
|
||||
)?;
|
||||
Ok(SessionCapture::from_complete_history(
|
||||
self.session_id.to_string(),
|
||||
self.segment_id.to_string(),
|
||||
restored.history,
|
||||
history,
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -573,7 +578,8 @@ where
|
||||
"flow:{}:{}",
|
||||
attempt.instance_id, attempt.attempt_id
|
||||
)),
|
||||
max_turns: Some(12),
|
||||
// Evidence exploration is not bounded by an arbitrary model-turn count.
|
||||
max_turns: None,
|
||||
compaction_budget: None,
|
||||
engine_configurator: None,
|
||||
features,
|
||||
@@ -877,14 +883,18 @@ mod tests {
|
||||
#[derive(Clone)]
|
||||
struct FinishVerifierClient {
|
||||
calls: Arc<AtomicUsize>,
|
||||
exploration_turns: usize,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmClient for FinishVerifierClient {
|
||||
async fn stream(&self, _request: Request) -> Result<ResponseStream, ClientError> {
|
||||
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
if call == 0 {
|
||||
let input = serde_json::json!({
|
||||
if call <= self.exploration_turns {
|
||||
let input = if call < self.exploration_turns {
|
||||
"{}".to_string()
|
||||
} else {
|
||||
serde_json::json!({
|
||||
"results": [
|
||||
{
|
||||
"transition_id": "done",
|
||||
@@ -898,14 +908,20 @@ mod tests {
|
||||
}
|
||||
]
|
||||
})
|
||||
.to_string();
|
||||
.to_string()
|
||||
};
|
||||
Ok(Box::pin(stream::iter(vec![
|
||||
Ok(LlmEvent::BlockStart(BlockStart {
|
||||
index: 0,
|
||||
block_type: BlockType::ToolUse,
|
||||
metadata: BlockMetadata::ToolUse {
|
||||
id: "finish-flow".to_string(),
|
||||
name: "FinishFlowVerification".to_string(),
|
||||
id: format!("flow-evidence-{call}"),
|
||||
name: if call < self.exploration_turns {
|
||||
"ShowOverview"
|
||||
} else {
|
||||
"FinishFlowVerification"
|
||||
}
|
||||
.to_string(),
|
||||
},
|
||||
})),
|
||||
Ok(LlmEvent::BlockDelta(BlockDelta {
|
||||
@@ -973,6 +989,15 @@ permission = "read"
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_backed_verifier_requires_structured_finish_result() {
|
||||
assert_worker_backed_verifier_completes(0).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_backed_verifier_can_finish_after_more_than_twelve_turns() {
|
||||
assert_worker_backed_verifier_completes(16).await;
|
||||
}
|
||||
|
||||
async fn assert_worker_backed_verifier_completes(exploration_turns: usize) {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let store = session_store::FsStore::new(directory.path()).unwrap();
|
||||
let session_id: SessionId = Uuid::now_v7();
|
||||
@@ -993,11 +1018,16 @@ permission = "read"
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let mut manifest = verifier_manifest();
|
||||
// A parent's own turn limit must not cap the private verifier either.
|
||||
manifest.engine.max_turns = std::num::NonZeroU32::new(2);
|
||||
let verifier = WorkerBackedFlowVerifier::new(
|
||||
FinishVerifierClient {
|
||||
calls: Arc::new(AtomicUsize::new(0)),
|
||||
calls: calls.clone(),
|
||||
exploration_turns,
|
||||
},
|
||||
verifier_manifest(),
|
||||
manifest,
|
||||
StoreFlowParentCapture::new(store, session_id, segment_id),
|
||||
Vec::new(),
|
||||
);
|
||||
@@ -1008,6 +1038,7 @@ permission = "read"
|
||||
assert_eq!(results.len(), 2);
|
||||
assert_eq!(results[0].verdict, ConditionVerdict::Met);
|
||||
assert_eq!(results[1].verdict, ConditionVerdict::NotMet);
|
||||
assert!(calls.load(Ordering::SeqCst) > exploration_turns);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -702,31 +702,46 @@ impl WorkspaceHttpWorkdirBackend {
|
||||
ATTACH_TOOL,
|
||||
"working_directory_id",
|
||||
)?;
|
||||
let alias_key = workdir::WorkdirAttachmentAlias::new(alias)
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
if self.session_router.session(&alias_key).is_some() {
|
||||
return Err(ToolError::InvalidArgument(format!(
|
||||
"attachment alias `{alias}` already exists"
|
||||
)));
|
||||
}
|
||||
let response = self.attach_response(alias, working_directory_id)?;
|
||||
if let Err(error) = self.session_router.attach(
|
||||
alias_key.clone(),
|
||||
if !response.attachment.attached
|
||||
|| response.attachment.working_directory_id != working_directory_id
|
||||
{
|
||||
return Err(ToolError::ExecutionFailed(
|
||||
"Workdir attachment response does not match the target".into(),
|
||||
));
|
||||
}
|
||||
// Backend ownership decides the name. A requested alias cannot rename
|
||||
// an existing attachment or create a second route to the same Workdir.
|
||||
let attached_alias = &response.attachment.alias;
|
||||
let alias_key = workdir::WorkdirAttachmentAlias::new(attached_alias.clone())
|
||||
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
|
||||
if let Some(session) = self.session_router.session(&alias_key) {
|
||||
if session.workdir().id().as_str() != working_directory_id {
|
||||
return Err(ToolError::ExecutionFailed("Local Workdir attachment differs from Backend; refresh attachments before retrying".into()));
|
||||
}
|
||||
} else if let Err(error) = self.session_router.attach(
|
||||
alias_key,
|
||||
WorkspaceAttachedWorkdirSession::handle_for_workdir_with_environment(
|
||||
self.client.clone(),
|
||||
alias,
|
||||
attached_alias,
|
||||
working_directory_id,
|
||||
response.attachment.capabilities,
|
||||
response.environment.clone(),
|
||||
),
|
||||
) {
|
||||
let _ = self.detach(alias);
|
||||
if !response.attachment.already_attached {
|
||||
let _ = self.detach(attached_alias);
|
||||
}
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"attach Workdir session route: {error}"
|
||||
)));
|
||||
}
|
||||
workdir_output(
|
||||
format!("Attached Workdir {working_directory_id} as {alias}"),
|
||||
if response.attachment.already_attached {
|
||||
format!("Workdir {working_directory_id} is already attached as {attached_alias}")
|
||||
} else {
|
||||
format!("Attached Workdir {working_directory_id} as {attached_alias}")
|
||||
},
|
||||
&response,
|
||||
)
|
||||
}
|
||||
@@ -737,7 +752,9 @@ impl WorkspaceHttpWorkdirBackend {
|
||||
working_directory_id: &str,
|
||||
) -> Result<WorkdirSessionAttachmentResponse, ToolError> {
|
||||
let workspace_id = encode_path_segment(self.workspace_id()?);
|
||||
self.execute_json::<WorkdirSessionAttachmentResponse>(WorkspaceRequest::json(
|
||||
let response = self
|
||||
.client
|
||||
.execute(WorkspaceRequest::json(
|
||||
WorkspaceRequestMethod::Post,
|
||||
format!("/api/w/{workspace_id}/workers/self/workdir-attachments"),
|
||||
serde_json::to_string(&WorkdirAttachRequest {
|
||||
@@ -746,6 +763,20 @@ impl WorkspaceHttpWorkdirBackend {
|
||||
})
|
||||
.map_err(decode_error)?,
|
||||
))
|
||||
.map_err(|error| {
|
||||
ToolError::ExecutionFailed(format!("Workspace Workdir request failed: {error}"))
|
||||
})?;
|
||||
if response.status == 409
|
||||
&& let Ok(error) =
|
||||
serde_json::from_str::<server_api::RepositoryApiError>(&response.body)
|
||||
&& error.error == "workdir_occupied"
|
||||
{
|
||||
return Err(ToolError::StructuredConflict {
|
||||
code: "workdir_occupied".into(),
|
||||
message: "Workdir is occupied by another Worker".into(),
|
||||
});
|
||||
}
|
||||
decode_response(response)
|
||||
}
|
||||
|
||||
fn detach(&self, alias: &str) -> Result<ToolOutput, ToolError> {
|
||||
@@ -778,7 +809,11 @@ impl WorkspaceHttpWorkdirBackend {
|
||||
)
|
||||
}
|
||||
|
||||
fn delete(&self, input: WorkdirDeleteInput) -> Result<ToolOutput, ToolError> {
|
||||
fn delete(
|
||||
&self,
|
||||
input: WorkdirDeleteInput,
|
||||
operation_id: String,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let workdir_id = validate_identity(
|
||||
&input.working_directory_id,
|
||||
DELETE_TOOL,
|
||||
@@ -790,6 +825,7 @@ impl WorkspaceHttpWorkdirBackend {
|
||||
WorkspaceRequestMethod::Delete,
|
||||
format!("/api/w/{workspace_id}/working-directories/{workdir_path}"),
|
||||
serde_json::to_string(&WorkdirRemovalRequest {
|
||||
operation_id,
|
||||
reason: validate_delete_reason(&input.reason)?.to_string(),
|
||||
})
|
||||
.map_err(decode_error)?,
|
||||
@@ -800,7 +836,7 @@ impl WorkspaceHttpWorkdirBackend {
|
||||
response.disposition,
|
||||
removal_guidance(response.failure_category.as_deref()),
|
||||
if response.retryable {
|
||||
" After resolving the cause, repeat this WorkdirDelete with the same id and reason; current safety conditions will be rechecked."
|
||||
" After resolving the cause, a new explicit WorkdirDelete request will recheck current safety conditions. This failed request will not be automatically re-executed."
|
||||
} else {
|
||||
""
|
||||
}
|
||||
@@ -954,9 +990,10 @@ impl Tool for WorkspaceHttpWorkdirTool {
|
||||
self.backend.detach_managed(&input.alias, None).await
|
||||
}
|
||||
|
||||
WorkdirOperation::Delete => self
|
||||
.backend
|
||||
.delete(parse_input::<WorkdirDeleteInput>(input_json)?),
|
||||
WorkdirOperation::Delete => self.backend.delete(
|
||||
parse_input::<WorkdirDeleteInput>(input_json)?,
|
||||
ctx.call_id.to_string(),
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1018,8 +1055,14 @@ fn removal_guidance(category: Option<&str>) -> &'static str {
|
||||
Some("runtime_unavailable" | "provider_unavailable") => {
|
||||
"Restore the Runtime connection; the registry remains until deletion is confirmed."
|
||||
}
|
||||
Some("provider_observation_unknown" | "provider_cleanup_outcome_unknown") => {
|
||||
"Deletion is not confirmed. Re-observe with normal removal after restoring provider access; the registry is retained."
|
||||
Some("provider_cleanup_outcome_unknown") => {
|
||||
"Deletion outcome is unknown. Restore provider access and reconcile the existing request; do not submit a fresh deletion intent. Recovery only observes the target and never repeats deletion."
|
||||
}
|
||||
Some("provider_observation_unknown") => {
|
||||
"The target could not be observed before deletion. Restore provider access before a new request."
|
||||
}
|
||||
Some("request_interrupted") => {
|
||||
"The request was interrupted before execution. A new explicit request is required."
|
||||
}
|
||||
Some("unsupported_target") => {
|
||||
"Ask the owner for the provider's supported cleanup operation."
|
||||
@@ -1219,6 +1262,8 @@ struct WorkdirAttachmentResponse {
|
||||
working_directory_id: String,
|
||||
capabilities: WorkdirSessionCapabilities,
|
||||
attached: bool,
|
||||
#[serde(default)]
|
||||
already_attached: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -1337,8 +1382,44 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_call_identity_distinguishes_new_requests_from_exact_resends() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(
|
||||
(0..3)
|
||||
.map(|_| {
|
||||
response(json!({
|
||||
"working_directory_id": "wd-request",
|
||||
"disposition": "attention_required",
|
||||
"retryable": true,
|
||||
"failure_category": "resource_busy"
|
||||
}))
|
||||
})
|
||||
.collect(),
|
||||
));
|
||||
let tool = WorkspaceHttpWorkdirTool {
|
||||
backend: WorkspaceHttpWorkdirBackend::new(client.clone()),
|
||||
operation: WorkdirOperation::Delete,
|
||||
};
|
||||
let input = r#"{"working_directory_id":"wd-request","reason":"remove unused Workdir"}"#;
|
||||
for id in ["delete-1", "delete-1", "delete-2"] {
|
||||
tool.execute(input, ToolExecutionContext::new(id, "batch", 0))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let bodies: Vec<serde_json::Value> = client
|
||||
.requests()
|
||||
.iter()
|
||||
.map(|r| serde_json::from_str(r.body.as_deref().unwrap()).unwrap())
|
||||
.collect();
|
||||
assert_eq!(bodies[0], bodies[1]);
|
||||
assert_eq!(bodies[0]["operation_id"], "delete-1");
|
||||
assert_eq!(bodies[2]["operation_id"], "delete-2");
|
||||
assert_eq!(bodies[0]["reason"], bodies[2]["reason"]);
|
||||
assert!(delete_schema()["properties"].get("operation_id").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_workdir_delete_reports_safe_cause_and_exact_retry_guidance() {
|
||||
fn ordinary_workdir_delete_reports_safe_cause_and_new_request_guidance() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
||||
"working_directory_id": "wd-retry",
|
||||
"disposition": "attention_required",
|
||||
@@ -1347,10 +1428,13 @@ mod tests {
|
||||
}))]));
|
||||
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
||||
let output = backend
|
||||
.delete(WorkdirDeleteInput {
|
||||
.delete(
|
||||
WorkdirDeleteInput {
|
||||
working_directory_id: "wd-retry".into(),
|
||||
reason: "remove unused Workdir".into(),
|
||||
})
|
||||
},
|
||||
"delete-call-1".to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(output.summary.contains("Confirm its owner"));
|
||||
assert!(
|
||||
@@ -1358,7 +1442,11 @@ mod tests {
|
||||
.summary
|
||||
.contains("Do not recursively delete mounted data")
|
||||
);
|
||||
assert!(output.summary.contains("same id and reason"));
|
||||
assert!(
|
||||
output
|
||||
.summary
|
||||
.contains("new explicit WorkdirDelete request")
|
||||
);
|
||||
assert!(output.summary.len() <= 600);
|
||||
let content: serde_json::Value =
|
||||
serde_json::from_str(output.content.as_deref().unwrap()).unwrap();
|
||||
@@ -1596,10 +1684,13 @@ mod tests {
|
||||
.unwrap();
|
||||
backend.detach("checkout").unwrap();
|
||||
backend
|
||||
.delete(WorkdirDeleteInput {
|
||||
.delete(
|
||||
WorkdirDeleteInput {
|
||||
working_directory_id: "wd-created".to_string(),
|
||||
reason: "remove stale Workdir".to_string(),
|
||||
})
|
||||
},
|
||||
"delete-call-1".to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let requests = client.requests();
|
||||
@@ -1637,7 +1728,102 @@ mod tests {
|
||||
assert_eq!(requests[4].method, WorkspaceRequestMethod::Delete);
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_str(requests[4].body.as_deref().unwrap()).unwrap();
|
||||
assert_eq!(body, json!({"reason": "remove stale Workdir"}));
|
||||
assert_eq!(
|
||||
body,
|
||||
json!({"reason": "remove stale Workdir", "operation_id": "delete-call-1"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attach_reports_other_worker_occupancy_without_exposing_backend_details() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![WorkspaceResponse {
|
||||
status: 409,
|
||||
body: serde_json::to_string(&server_api::RepositoryApiError::new(
|
||||
409,
|
||||
"workdir_occupied",
|
||||
"private backend detail",
|
||||
Vec::new(),
|
||||
))
|
||||
.unwrap(),
|
||||
}]));
|
||||
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
||||
let error = backend
|
||||
.attach(WorkdirAttachInput {
|
||||
alias: "work".into(),
|
||||
working_directory_id: "occupied".into(),
|
||||
})
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
matches!(error, ToolError::StructuredConflict { code, message } if code == "workdir_occupied" && message == "Workdir is occupied by another Worker")
|
||||
);
|
||||
assert_eq!(client.requests().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attach_reuses_backend_alias_without_duplicate_routes_or_detach() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new((0..3).map(|_| response(json!({
|
||||
"workspace_id": "workspace/test", "alias": "existing", "working_directory_id": "wd-existing",
|
||||
"capabilities": {"bits": 63}, "attached": true, "already_attached": true,
|
||||
"environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL)
|
||||
}))).collect()));
|
||||
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
||||
let occupied_alias = workdir::WorkdirAttachmentAlias::new("other").unwrap();
|
||||
backend
|
||||
.session_router
|
||||
.attach(
|
||||
occupied_alias.clone(),
|
||||
WorkspaceAttachedWorkdirSession::handle_for_workdir(
|
||||
client.clone(),
|
||||
"other",
|
||||
"wd-other",
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
for alias in ["new-name", "existing", "other"] {
|
||||
let output = backend
|
||||
.attach(WorkdirAttachInput {
|
||||
alias: alias.into(),
|
||||
working_directory_id: "wd-existing".into(),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
output.summary,
|
||||
"Workdir wd-existing is already attached as existing"
|
||||
);
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_str(output.content.as_deref().unwrap()).unwrap();
|
||||
assert_eq!(body["alias"], "existing");
|
||||
assert_eq!(body["already_attached"], true);
|
||||
}
|
||||
assert!(
|
||||
backend
|
||||
.session_router
|
||||
.session(&workdir::WorkdirAttachmentAlias::new("new-name").unwrap())
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.session_router
|
||||
.session(&occupied_alias)
|
||||
.unwrap()
|
||||
.workdir()
|
||||
.id()
|
||||
.as_str(),
|
||||
"wd-other"
|
||||
);
|
||||
assert!(
|
||||
backend
|
||||
.session_router
|
||||
.session(&workdir::WorkdirAttachmentAlias::new("existing").unwrap())
|
||||
.is_some()
|
||||
);
|
||||
assert_eq!(client.requests().len(), 3);
|
||||
assert!(
|
||||
client
|
||||
.requests()
|
||||
.iter()
|
||||
.all(|request| request.method == WorkspaceRequestMethod::Post)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -456,10 +456,13 @@ impl MemoryLifecycleTask {
|
||||
.await;
|
||||
return Ok(());
|
||||
}
|
||||
let view = SessionCapture::from_history_entries(
|
||||
let view = SessionCapture::from_complete_history(
|
||||
capture.session_id.clone(),
|
||||
capture.segment_id.clone(),
|
||||
capture.history[history_start..history_end].to_vec(),
|
||||
);
|
||||
capture.history.clone(),
|
||||
)
|
||||
.select_order_range(history_start as u64..history_end as u64)
|
||||
.map_err(hook_internal)?;
|
||||
let start_entry = pointer
|
||||
.as_ref()
|
||||
.map(|pointer| pointer.processed_through_entry + 1)
|
||||
@@ -1556,8 +1559,11 @@ permission = "write"
|
||||
#[tokio::test]
|
||||
async fn run_committed_background_task_stages_non_empty_extraction_and_commits_pointer() {
|
||||
let source = capture(2, 250);
|
||||
let entry_ref =
|
||||
SessionCapture::from_history_entries(source.segment_id.clone(), source.history.clone())
|
||||
let entry_ref = SessionCapture::from_complete_history(
|
||||
source.session_id.clone(),
|
||||
source.segment_id.clone(),
|
||||
source.history.clone(),
|
||||
)
|
||||
.overview()[0]
|
||||
.id
|
||||
.to_string();
|
||||
@@ -1606,11 +1612,99 @@ permission = "write"
|
||||
);
|
||||
}
|
||||
|
||||
// Real background extraction -> common fixed selection -> StageMemoryCandidate
|
||||
// -> typed backend operation and pointer write. Only model/backend transport
|
||||
// are replaced; the second extraction uses the first run's saved pointer.
|
||||
#[tokio::test]
|
||||
async fn second_extraction_preserves_original_evidence_positions_after_nonzero_pointer() {
|
||||
let first = capture(2, 250);
|
||||
let mut second = capture(4, 750);
|
||||
second.history[..2].clone_from_slice(&first.history);
|
||||
second
|
||||
.usage_history
|
||||
.insert(0, first.usage_history[0].clone());
|
||||
let selected_ref =
|
||||
crate::SessionEntryRef::from_history_entry_id(&second.history[2].annotation.entry_id)
|
||||
.to_string();
|
||||
let extension_writes = Arc::new(Mutex::new(Vec::new()));
|
||||
let workspace_client = Arc::new(RecordingWorkspaceClient::default());
|
||||
let (event_tx, _) = broadcast::channel(64);
|
||||
run_background_task(test_task(
|
||||
first,
|
||||
Box::new(ScriptClient::new(vec![
|
||||
finish_empty_events("first-empty"),
|
||||
completed_events(),
|
||||
])),
|
||||
extension_writes.clone(),
|
||||
event_tx.clone(),
|
||||
workspace_client.clone(),
|
||||
))
|
||||
.await;
|
||||
{
|
||||
let writes = extension_writes.lock().unwrap();
|
||||
assert_eq!(writes.len(), 1);
|
||||
let pointer: memory::ExtractPointerPayload =
|
||||
serde_json::from_value(writes[0].2.clone()).unwrap();
|
||||
assert_eq!(pointer.processed_through_history_len, 2);
|
||||
second
|
||||
.extensions
|
||||
.push((writes[0].1.clone(), writes[0].2.clone()));
|
||||
}
|
||||
run_background_task(test_task(
|
||||
second,
|
||||
Box::new(ScriptClient::new(vec![
|
||||
stage_candidate_events("second-stage", &selected_ref),
|
||||
finish_events("second-finish", 1),
|
||||
completed_events(),
|
||||
])),
|
||||
extension_writes.clone(),
|
||||
event_tx,
|
||||
workspace_client.clone(),
|
||||
))
|
||||
.await;
|
||||
let operations = workspace_client
|
||||
.requests
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter_map(|request| {
|
||||
request.body.as_deref().and_then(|body| {
|
||||
serde_json::from_str::<memory::backend::MemoryBackendOperation>(body).ok()
|
||||
})
|
||||
})
|
||||
.filter_map(|operation| match operation {
|
||||
memory::backend::MemoryBackendOperation::StageCandidate(operation) => {
|
||||
Some(operation)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(operations.len(), 1);
|
||||
let operation = &operations[0];
|
||||
assert_eq!(operation.source.range, [2, 3]);
|
||||
assert_eq!(operation.evidence[0].id, selected_ref);
|
||||
assert_eq!(operation.evidence[0].entry_range, Some([2, 2]));
|
||||
assert_eq!(
|
||||
operation.source_refs[0].segment_id.as_deref(),
|
||||
Some("segment-1")
|
||||
);
|
||||
assert_eq!(operation.source_refs[0].entry_range, Some([2, 2]));
|
||||
let writes = extension_writes.lock().unwrap();
|
||||
assert_eq!(writes.len(), 2);
|
||||
let pointer: memory::ExtractPointerPayload =
|
||||
serde_json::from_value(writes[1].2.clone()).unwrap();
|
||||
assert_eq!(pointer.processed_through_history_len, 4);
|
||||
assert_eq!(pointer.processed_through_entry, 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subjektiv_extraction_stages_once_and_commits_its_independent_pointer() {
|
||||
let source = capture(2, 250);
|
||||
let entry_ref =
|
||||
SessionCapture::from_history_entries(source.segment_id.clone(), source.history.clone())
|
||||
let entry_ref = SessionCapture::from_complete_history(
|
||||
source.session_id.clone(),
|
||||
source.segment_id.clone(),
|
||||
source.history.clone(),
|
||||
)
|
||||
.overview()[0]
|
||||
.id
|
||||
.to_string();
|
||||
@@ -1715,8 +1809,11 @@ permission = "write"
|
||||
#[tokio::test]
|
||||
async fn subjektiv_candidate_remains_staged_when_later_extraction_fails_without_pointer() {
|
||||
let source = capture(2, 250);
|
||||
let entry_ref =
|
||||
SessionCapture::from_history_entries(source.segment_id.clone(), source.history.clone())
|
||||
let entry_ref = SessionCapture::from_complete_history(
|
||||
source.session_id.clone(),
|
||||
source.segment_id.clone(),
|
||||
source.history.clone(),
|
||||
)
|
||||
.overview()[0]
|
||||
.id
|
||||
.to_string();
|
||||
|
||||
@@ -510,7 +510,11 @@ mod tests {
|
||||
|
||||
fn state() -> MemoryStagingOutputState {
|
||||
MemoryStagingOutputState::new(
|
||||
SessionCapture::new("segment-1", vec![Item::user_message("durable decision")]),
|
||||
SessionCapture::new(
|
||||
"session",
|
||||
"segment-1",
|
||||
vec![Item::user_message("durable decision")],
|
||||
),
|
||||
crate::worker::marker_workspace_client(None, "test-backend"),
|
||||
SourceRef {
|
||||
segment_id: "segment-1".to_string(),
|
||||
@@ -537,6 +541,7 @@ mod tests {
|
||||
#[test]
|
||||
fn render_input_uses_session_entry_refs_and_new_tool_names() {
|
||||
let view = SessionCapture::new(
|
||||
"session",
|
||||
"segment-1",
|
||||
vec![
|
||||
Item::user_message("preference"),
|
||||
@@ -586,7 +591,11 @@ mod tests {
|
||||
async fn subjektiv_destination_sends_committed_session_without_subject_scope() {
|
||||
let client = Arc::new(RecordingSubjektivClient::default());
|
||||
let state = MemoryStagingOutputState::new_subjektiv(
|
||||
SessionCapture::new("segment-1", vec![Item::user_message("durable decision")]),
|
||||
SessionCapture::new(
|
||||
"session",
|
||||
"segment-1",
|
||||
vec![Item::user_message("durable decision")],
|
||||
),
|
||||
{
|
||||
let mut host = crate::subjektiv::test_connection(client.clone());
|
||||
host.context.session_id = "session-committed".into();
|
||||
@@ -630,4 +639,53 @@ mod tests {
|
||||
.unwrap_err();
|
||||
assert!(format!("{error:?}").contains("unknown SessionEntryRef"));
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn staging_roundtripped_mixed_sources_uses_original_evidence_ranges_not_view_order() {
|
||||
let client = Arc::new(RecordingSubjektivClient::default());
|
||||
let mut dto = SessionCapture::new(
|
||||
"session",
|
||||
"view",
|
||||
vec![
|
||||
Item::user_message("first"),
|
||||
Item::assistant_message("second"),
|
||||
],
|
||||
)
|
||||
.to_dto();
|
||||
dto.entries[0].source.segment_id = "original-a".into();
|
||||
dto.entries[0].source.entry_index = 80;
|
||||
dto.entries[1].source.segment_id = "original-b".into();
|
||||
dto.entries[1].source.entry_index = 3;
|
||||
let limits = crate::session_capture::CaptureLimits::default();
|
||||
let view = SessionCapture::decode(&dto.encode(limits).unwrap(), limits).unwrap();
|
||||
let state = MemoryStagingOutputState::new_subjektiv(
|
||||
view,
|
||||
crate::subjektiv::test_connection(client.clone()),
|
||||
SourceRef {
|
||||
segment_id: "view".into(),
|
||||
range: [0, 1],
|
||||
},
|
||||
"run".into(),
|
||||
);
|
||||
StageMemoryCandidateTool { state }.execute(
|
||||
r#"{"kind":"decision","claim":"claim","why_useful":"useful","entry_refs":["E00000000","E00000001"]}"#,
|
||||
agen::tool::ToolExecutionContext::direct(),
|
||||
).await.unwrap();
|
||||
let requests = client.requests.lock().unwrap();
|
||||
let request: server_api::SubjektivStageCandidateRequest =
|
||||
serde_json::from_str(requests[0].body.as_deref().unwrap()).unwrap();
|
||||
for (index, segment, position) in [(0, "original-a", 80), (1, "original-b", 3)] {
|
||||
assert_eq!(
|
||||
request.operation.evidence[index].entry_range,
|
||||
Some([position; 2])
|
||||
);
|
||||
assert_eq!(
|
||||
request.operation.source_refs[index].segment_id.as_deref(),
|
||||
Some(segment)
|
||||
);
|
||||
assert_eq!(
|
||||
request.operation.source_refs[index].entry_range,
|
||||
Some([position; 2])
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,13 +91,22 @@ struct MergeRequestInput {
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
struct ShowMergeRequestInput {
|
||||
merge_request_id: String,
|
||||
/// summary (default) omits review bodies/history; detail returns exact paged events.
|
||||
/// For compatibility, supplying after/limit without view selects detail.
|
||||
view: Option<MergeRequestReadView>,
|
||||
/// Return thread events strictly after this sequence.
|
||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
||||
after: Option<u64>,
|
||||
/// Bounded thread page size. The Backend also enforces its authoritative cap.
|
||||
/// Exact detail/history page size (default 20, maximum 200). Continue with event_page.next_after.
|
||||
#[schemars(range(min = 1, max = 200))]
|
||||
limit: Option<usize>,
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum MergeRequestReadView {
|
||||
Summary,
|
||||
Detail,
|
||||
}
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
struct OpenMergeRequestInput {
|
||||
ticket: String,
|
||||
@@ -203,7 +212,27 @@ impl Tool for MergeRequestTool {
|
||||
"limit must be between 1 and 200".into(),
|
||||
));
|
||||
}
|
||||
return self.show_merge_request(ws, &value.merge_request_id, value.after, value.limit);
|
||||
let view = value.view.unwrap_or_else(|| {
|
||||
if value.after.is_some() || value.limit.is_some() {
|
||||
MergeRequestReadView::Detail
|
||||
} else {
|
||||
MergeRequestReadView::Summary
|
||||
}
|
||||
});
|
||||
if matches!(view, MergeRequestReadView::Summary)
|
||||
&& (value.after.is_some() || value.limit.is_some())
|
||||
{
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"after/limit require view=detail".into(),
|
||||
));
|
||||
}
|
||||
return self.show_merge_request(
|
||||
ws,
|
||||
&value.merge_request_id,
|
||||
view,
|
||||
value.after,
|
||||
value.limit,
|
||||
);
|
||||
}
|
||||
let (method, path, body) = match self.kind {
|
||||
Kind::Readiness => {
|
||||
@@ -290,12 +319,19 @@ impl MergeRequestTool {
|
||||
&self,
|
||||
workspace_id: &str,
|
||||
merge_request_id: &str,
|
||||
view: MergeRequestReadView,
|
||||
after: Option<u64>,
|
||||
limit: Option<usize>,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let merge_request_id = encode_path_segment(merge_request_id);
|
||||
let mut path = format!("/api/w/{workspace_id}/merge-requests/{merge_request_id}");
|
||||
let mut query = Vec::new();
|
||||
let mut query = vec![format!(
|
||||
"view={}",
|
||||
match view {
|
||||
MergeRequestReadView::Summary => "summary",
|
||||
MergeRequestReadView::Detail => "detail",
|
||||
}
|
||||
)];
|
||||
if let Some(after) = after {
|
||||
query.push(format!("after={after}"));
|
||||
}
|
||||
@@ -1099,7 +1135,7 @@ fn merge_request_operation_not_found() -> WipOperationError {
|
||||
pub fn description(n: &str) -> Option<&'static str> {
|
||||
match n {
|
||||
"ShowMergeRequest" => Some(
|
||||
"Read the selector-based Merge Request, append-only thread, source-review freshness, and target-integration evidence before review, fix, or handoff decisions.",
|
||||
"Read an authoritative compact MR summary by default: source/target observations, full-evidence review status/readiness and stable event references, without review bodies/history. Use view=detail with bounded after/limit for exact reviews/findings and history; continue with event_page.next_after. Status is never inferred from a history page. Summary is not approval or integration authority.",
|
||||
),
|
||||
"CheckMergeRequestReadiness" => Some(
|
||||
"Resolve current provider refs and derive readiness from exact-source review evidence; source movement requires fresh review while target-only movement preserves unchanged-source approval.",
|
||||
@@ -1222,7 +1258,10 @@ mod tests {
|
||||
let requests = client.requests.lock().expect("request lock");
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert_eq!(requests[0].method, WorkspaceRequestMethod::Get);
|
||||
assert_eq!(requests[0].path, "/api/w/ws/merge-requests/MR%2F1");
|
||||
assert_eq!(
|
||||
requests[0].path,
|
||||
"/api/w/ws/merge-requests/MR%2F1?view=summary"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1243,7 +1282,7 @@ mod tests {
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
client.requests.lock().expect("request lock")[0].path,
|
||||
"/api/w/ws/merge-requests/MR-1?after=42&limit=200"
|
||||
"/api/w/ws/merge-requests/MR-1?view=detail&after=42&limit=200"
|
||||
);
|
||||
|
||||
let rejected = tool
|
||||
@@ -1257,6 +1296,64 @@ mod tests {
|
||||
assert_eq!(client.requests.lock().expect("request lock").len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_read_forwards_explicit_projection_and_rejects_ambiguous_summary_paging() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
||||
"merge_request_id": "MR-1", "view": "detail", "thread": [],
|
||||
"summary": {"review_status": "approved"},
|
||||
"event_page": {"omitted": false, "has_more": true, "next_after": 9, "limit": 1}
|
||||
}))]));
|
||||
let tool = native_merge_request_tools(
|
||||
client.clone(),
|
||||
MergeRequestFeatureConfig {
|
||||
show: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.remove(0);
|
||||
let input = native_merge_request_input(
|
||||
&BTreeMap::from([
|
||||
("view".into(), WipValue::String("detail".into())),
|
||||
("after".into(), json_to_wip(&json!(8)).unwrap()),
|
||||
("limit".into(), json_to_wip(&json!(1)).unwrap()),
|
||||
]),
|
||||
Some("MR-1"),
|
||||
Some("merge_request_id"),
|
||||
)
|
||||
.unwrap_or_else(|_| panic!("native read arguments should bind"));
|
||||
let output = execute_native_merge_request_tool(
|
||||
&tool,
|
||||
&None,
|
||||
input,
|
||||
Some("MR-1"),
|
||||
WipCallContext {
|
||||
execution: ToolExecutionContext::direct(),
|
||||
security_context: "reader".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| panic!("native detail read should succeed"));
|
||||
let output = merge_request_tool_output_json(output)
|
||||
.unwrap_or_else(|_| panic!("valid JSON response"));
|
||||
assert_eq!(output["summary"]["review_status"], "approved");
|
||||
assert_eq!(output["event_page"]["next_after"], 9);
|
||||
assert_eq!(
|
||||
client.requests.lock().unwrap()[0].path,
|
||||
"/api/w/ws/merge-requests/MR-1?view=detail&after=8&limit=1"
|
||||
);
|
||||
let rejected = tool
|
||||
.tool
|
||||
.execute(
|
||||
r#"{"merge_request_id":"MR-1","view":"summary","limit":1}"#,
|
||||
ToolExecutionContext::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(rejected.to_string().contains("require view=detail"));
|
||||
assert_eq!(client.requests.lock().unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn show_requires_explicit_merge_request_identity() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![]));
|
||||
|
||||
@@ -15,7 +15,7 @@ struct ModelTicketQueryItem {
|
||||
title: String,
|
||||
state: ticket::TicketState,
|
||||
closure_operation_key: Option<String>,
|
||||
current_closure: Option<ticket::TicketClosureJudgment>,
|
||||
current_closure: Option<Value>,
|
||||
legacy_handoff: Option<ticket::TicketLegacyWorkflowSnapshot>,
|
||||
assignment_tracking: Vec<ModelAssignmentTracking>,
|
||||
tags: Vec<ticket::WorkspaceTag>,
|
||||
@@ -34,7 +34,6 @@ struct ModelTicketQueryItem {
|
||||
unresolved_review_count: usize,
|
||||
evidence: Option<ModelTicketEvidence>,
|
||||
merge_requests: Vec<ModelMergeRequest>,
|
||||
merge_request: Option<ModelMergeRequest>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize, Serialize)]
|
||||
@@ -50,14 +49,12 @@ pub(super) struct ModelTicketDetail {
|
||||
ticket: String,
|
||||
content_digest: String,
|
||||
title: String,
|
||||
body: String,
|
||||
body_truncated: bool,
|
||||
initial_post: ModelTicketEvent,
|
||||
targets: Vec<ModelTicketTarget>,
|
||||
created_by: Option<String>,
|
||||
state: ticket::TicketState,
|
||||
closure_operation_key: Option<String>,
|
||||
current_closure: Option<ticket::TicketClosureJudgment>,
|
||||
current_closure: Option<Value>,
|
||||
legacy_handoff: Option<ticket::TicketLegacyWorkflowSnapshot>,
|
||||
assignment_tracking: Vec<ModelAssignmentTracking>,
|
||||
tags: Vec<ticket::WorkspaceTag>,
|
||||
@@ -65,7 +62,7 @@ pub(super) struct ModelTicketDetail {
|
||||
priority: Option<String>,
|
||||
created_at: Option<String>,
|
||||
updated_at: Option<String>,
|
||||
closure_history: Vec<ticket::TicketCaseHistoryEntry>,
|
||||
closure_history: Vec<Value>,
|
||||
thread: Vec<ModelTicketEvent>,
|
||||
relations: ModelTicketRelations,
|
||||
linked_objectives: Vec<ModelObjectiveSummary>,
|
||||
@@ -73,7 +70,6 @@ pub(super) struct ModelTicketDetail {
|
||||
current_worker: Option<ModelWorkerSummary>,
|
||||
implementation_reports: Vec<ModelEvidenceEvent>,
|
||||
merge_requests: Vec<ModelMergeRequest>,
|
||||
merge_request: Option<ModelMergeRequest>,
|
||||
evidence: Option<ModelTicketEvidence>,
|
||||
actions: Option<ModelTicketActions>,
|
||||
event_page: Option<ModelEventPage>,
|
||||
@@ -280,6 +276,21 @@ fn typed_field<T: serde::de::DeserializeOwned>(
|
||||
serde_json::from_value(value.clone()).map_err(|error| format!("Invalid Ticket {key}: {error}"))
|
||||
}
|
||||
|
||||
fn closure_metadata<T: Serialize>(value: T) -> Result<Value, String> {
|
||||
let mut value = serde_json::to_value(value).map_err(|error| error.to_string())?;
|
||||
if let Some(object) = value.as_object_mut() {
|
||||
// The full explanation is available in thread history/post, not repeated
|
||||
// in current judgment and closure-index metadata as well.
|
||||
object.remove("explanation");
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn project_current_closure(root: &Map<String, Value>) -> Result<Option<Value>, String> {
|
||||
let closure: Option<ticket::TicketClosureJudgment> = typed_field(root, "current_closure")?;
|
||||
closure.map(closure_metadata).transpose()
|
||||
}
|
||||
|
||||
pub(super) fn project_ticket_query(value: Value) -> Result<ModelTicketQueryResponse, String> {
|
||||
let root = object(&value, "Ticket query response")?;
|
||||
let page = object_field(root, "page")?;
|
||||
@@ -301,7 +312,7 @@ fn project_ticket_query_item(value: &Value) -> Result<ModelTicketQueryItem, Stri
|
||||
title: string_field(item, "title")?,
|
||||
state: typed_field(item, "state")?,
|
||||
closure_operation_key: typed_field(item, "closure_operation_key")?,
|
||||
current_closure: typed_field(item, "current_closure")?,
|
||||
current_closure: project_current_closure(item)?,
|
||||
legacy_handoff: typed_field(item, "legacy_handoff")?,
|
||||
assignment_tracking: project_assignment_tracking(item)?,
|
||||
tags: project_tags(item)?,
|
||||
@@ -330,14 +341,84 @@ fn project_ticket_query_item(value: &Value) -> Result<ModelTicketQueryItem, Stri
|
||||
.iter()
|
||||
.map(project_merge_request)
|
||||
.collect::<Result<Vec<_>, _>>()?,
|
||||
merge_request: item
|
||||
.get("merge_request")
|
||||
.filter(|value| !value.is_null())
|
||||
.map(project_merge_request)
|
||||
.transpose()?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Status is deliberately projected from authority fields, not from serialized detail:
|
||||
/// prose/history size and history parsing have no effect on this response.
|
||||
pub(super) fn project_ticket_status(value: Value) -> Result<Value, String> {
|
||||
use serde_json::json;
|
||||
let root = object(&value, "Ticket summary response")?;
|
||||
let state: ticket::TicketState = typed_field(root, "state")?;
|
||||
let closure_key: Option<String> = typed_field(root, "closure_operation_key")?;
|
||||
let closure: Option<ticket::TicketClosureJudgment> = typed_field(root, "current_closure")?;
|
||||
let targets: Vec<ModelTicketTarget> = typed_field(root, "targets")?;
|
||||
let tracking = project_assignment_tracking(root)?.into_iter().map(|row| json!({
|
||||
"assignment_id": row.assignment_id, "role": row.role, "principal": row.principal,
|
||||
"assigned_by": row.assigned_by, "assigned_at": row.assigned_at, "current": row.current,
|
||||
"status": row.tracking.map(|tracking| tracking.status),
|
||||
"last_event_id": row.last_event_id, "updated_at": row.updated_at,
|
||||
})).collect::<Vec<_>>();
|
||||
let merges = array_field(root, "merge_requests")?
|
||||
.iter()
|
||||
.map(|value| {
|
||||
let mut merge = serde_json::to_value(project_merge_request(value)?)
|
||||
.map_err(|error| error.to_string())?;
|
||||
merge
|
||||
.as_object_mut()
|
||||
.expect("typed merge object")
|
||||
.remove("review_excerpt");
|
||||
Ok(merge)
|
||||
})
|
||||
.collect::<Result<Vec<Value>, String>>()?;
|
||||
let latest_event_ref = array_field(root, "events")?
|
||||
.last()
|
||||
.map(|event| string_field(object(event, "Ticket event")?, "event_ref"))
|
||||
.transpose()?;
|
||||
Ok(json!({
|
||||
"ticket": resource_ref(root, "resource_key", "T-")?,
|
||||
"title": string_field(root, "title")?,
|
||||
"content_digest": string_field(root, "content_digest")?,
|
||||
"state": state, "closure_operation_key": closure_key,
|
||||
"current_closure": closure.map(|j| json!({"operation_key": j.operation_key, "reason": j.reason, "author": j.author, "at": j.at})),
|
||||
"targets": targets, "tags": project_tags(root)?,
|
||||
"readiness": optional_string(root, "readiness")?,
|
||||
"priority": optional_string(root, "priority")?,
|
||||
"updated_at": optional_string(root, "updated_at")?,
|
||||
"assignment_tracking": tracking,
|
||||
"current_worker": root.get("current_worker").filter(|value| !value.is_null()).map(project_worker).transpose()?,
|
||||
"merge_requests": merges,
|
||||
"evidence": root.get("evidence").map(project_evidence).transpose()?,
|
||||
"actions": root.get("action_eligibility").filter(|value| !value.is_null()).map(project_actions).transpose()?,
|
||||
"latest_event_ref": latest_event_ref,
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) fn project_ticket_history(value: Value) -> Result<Value, String> {
|
||||
let root = object(&value, "Ticket history response")?;
|
||||
let thread = array_field(root, "events")?
|
||||
.iter()
|
||||
.filter(|event| event.get("is_initial") != Some(&Value::Bool(true)))
|
||||
.map(project_ticket_event)
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(serde_json::json!({
|
||||
"ticket": resource_ref(root, "resource_key", "T-")?,
|
||||
"thread": thread,
|
||||
"event_page": project_event_page(root.get("event_page").ok_or("missing event_page")?)?,
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) fn project_ticket_post(value: Value) -> Result<Value, String> {
|
||||
let post = project_ticket_event(&value)?;
|
||||
if post.body_truncated || post.body.is_none() {
|
||||
return Err(
|
||||
"Full-post read returned an absent or truncated body; do not use it for edit CAS"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
serde_json::to_value(post).map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
pub(super) fn project_ticket_detail(value: Value) -> Result<ModelTicketDetail, String> {
|
||||
let root = object(&value, "Ticket detail response")?;
|
||||
let current_worker = root
|
||||
@@ -354,8 +435,6 @@ pub(super) fn project_ticket_detail(value: Value) -> Result<ModelTicketDetail, S
|
||||
ticket: resource_ref(root, "resource_key", "T-")?,
|
||||
content_digest: string_field(root, "content_digest")?,
|
||||
title: string_field(root, "title")?,
|
||||
body: string_field(root, "body")?,
|
||||
body_truncated: bool_field(root, "body_truncated")?,
|
||||
initial_post: project_ticket_event(
|
||||
root.get("initial_post")
|
||||
.ok_or("Ticket detail response is missing initial_post")?,
|
||||
@@ -364,9 +443,15 @@ pub(super) fn project_ticket_detail(value: Value) -> Result<ModelTicketDetail, S
|
||||
created_by: optional_string(root, "created_by")?,
|
||||
state: typed_field(root, "state")?,
|
||||
closure_operation_key: typed_field(root, "closure_operation_key")?,
|
||||
current_closure: typed_field(root, "current_closure")?,
|
||||
current_closure: project_current_closure(root)?,
|
||||
legacy_handoff: typed_field(root, "legacy_handoff")?,
|
||||
closure_history: typed_field(root, "closure_history")?,
|
||||
closure_history: typed_field::<Vec<ticket::TicketCaseHistoryEntry>>(
|
||||
root,
|
||||
"closure_history",
|
||||
)?
|
||||
.into_iter()
|
||||
.map(closure_metadata)
|
||||
.collect::<Result<Vec<_>, _>>()?,
|
||||
assignment_tracking: project_assignment_tracking(root)?,
|
||||
tags: project_tags(root)?,
|
||||
readiness: optional_string(root, "readiness")?,
|
||||
@@ -375,6 +460,7 @@ pub(super) fn project_ticket_detail(value: Value) -> Result<ModelTicketDetail, S
|
||||
updated_at: optional_string(root, "updated_at")?,
|
||||
thread: array_field(root, "events")?
|
||||
.iter()
|
||||
.filter(|event| event.get("is_initial") != Some(&Value::Bool(true)))
|
||||
.map(project_ticket_event)
|
||||
.collect::<Result<Vec<_>, _>>()?,
|
||||
relations: project_relations(root.get("relations"))?,
|
||||
@@ -392,11 +478,6 @@ pub(super) fn project_ticket_detail(value: Value) -> Result<ModelTicketDetail, S
|
||||
.iter()
|
||||
.map(project_merge_request)
|
||||
.collect::<Result<Vec<_>, _>>()?,
|
||||
merge_request: root
|
||||
.get("merge_request")
|
||||
.filter(|value| !value.is_null())
|
||||
.map(project_merge_request)
|
||||
.transpose()?,
|
||||
evidence: root.get("evidence").map(project_evidence).transpose()?,
|
||||
actions: root
|
||||
.get("action_eligibility")
|
||||
@@ -919,7 +1000,18 @@ mod tests {
|
||||
assert_eq!(projected["state"], "open");
|
||||
assert_eq!(projected["closure_operation_key"], "close-1");
|
||||
assert_eq!(projected["current_closure"], Value::Null);
|
||||
assert_eq!(projected["closure_history"], history);
|
||||
assert_eq!(
|
||||
projected["closure_history"],
|
||||
json!(
|
||||
history
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(closure_metadata)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.unwrap()
|
||||
)
|
||||
);
|
||||
assert_eq!(projected["legacy_handoff"], legacy);
|
||||
assert_eq!(projected["tags"], response["tags"]);
|
||||
assert_eq!(projected["content_digest"], "content-digest");
|
||||
@@ -1191,7 +1283,8 @@ mod tests {
|
||||
.expect("source evidence must project");
|
||||
let projected = serde_json::to_value(projected).expect("serialize Ticket detail");
|
||||
|
||||
for merge in [&projected["merge_requests"][0], &projected["merge_request"]] {
|
||||
assert!(projected.get("merge_request").is_none());
|
||||
for merge in [&projected["merge_requests"][0]] {
|
||||
assert_eq!(merge["subject_ref"], json!(subject_ref));
|
||||
assert_eq!(merge["source_ref_observation"], observation);
|
||||
assert_eq!(
|
||||
@@ -1378,9 +1471,14 @@ mod tests {
|
||||
"current_closure",
|
||||
"legacy_handoff",
|
||||
] {
|
||||
assert_eq!(projected_detail[field], detail[field], "detail {field}");
|
||||
let expected = if field == "current_closure" {
|
||||
closure_metadata(&detail[field]).unwrap()
|
||||
} else {
|
||||
detail[field].clone()
|
||||
};
|
||||
assert_eq!(projected_detail[field], expected, "detail {field}");
|
||||
assert_eq!(
|
||||
projected_query["tickets"][0][field], detail[field],
|
||||
projected_query["tickets"][0][field], expected,
|
||||
"query {field}"
|
||||
);
|
||||
}
|
||||
@@ -1392,10 +1490,22 @@ mod tests {
|
||||
assert!(actual.get("workspace_id").is_none());
|
||||
assert!(actual.get("ticket_id").is_none());
|
||||
}
|
||||
assert_eq!(projected_detail["closure_history"], history);
|
||||
assert_eq!(
|
||||
projected_detail["closure_history"],
|
||||
json!(
|
||||
history
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(closure_metadata)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.unwrap()
|
||||
)
|
||||
);
|
||||
assert_eq!(projected_detail["content_digest"], "content-1");
|
||||
assert_eq!(projected_detail["created_by"], "workspace-user:creator");
|
||||
assert_eq!(projected_detail["body_truncated"], false);
|
||||
assert_eq!(projected_detail["initial_post"]["body_truncated"], false);
|
||||
assert!(projected_detail.get("body").is_none());
|
||||
for initial in [true, false] {
|
||||
let input = post_fixture(initial, "The same post contract");
|
||||
let output = serde_json::to_value(project_ticket_event(&input).unwrap()).unwrap();
|
||||
@@ -1488,6 +1598,93 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ticket_views_keep_prose_once_and_summary_independent_of_prose_and_history() {
|
||||
let body = "unique-long-requirement-".repeat(600);
|
||||
let post = post_fixture(true, &body);
|
||||
let closure = json!({"operation_key":"close", "reason":"completed", "explanation":"closure prose", "references":[], "author":"user", "at":"today"});
|
||||
let tracking = json!({"workspace_id":"ws", "ticket_id":"internal", "assignment_id":"assignment", "role":"orchestrator", "principal":{"kind":"workspace_agent","agent_key":"workspace-orchestrator"}, "assigned_by":"user", "assigned_at":"today", "release":null, "current":true, "last_event_id":"handling-event", "updated_at":"today", "migration_diagnostic":null, "tracking":{"status":"waiting", "waiting_reason":"prose", "progress":null, "next_action":"prose", "result":null}});
|
||||
let merge = json!({"merge_request_id":"MR-1", "repository_key":"main", "state":"open", "selector_from":"topic", "selector_to":"main", "review_status":"request_changes", "current_subject_ref":"sha1", "source_ref_observation":{"status":"observed"}, "integration_evidence_error":null, "review_excerpt":"review prose"});
|
||||
let mut input = json!({
|
||||
"resource_key":"T-848", "title":"Views", "body":body, "body_truncated":false, "initial_post":post,
|
||||
"content_digest":"content", "state":"closed", "closure_operation_key":"close", "current_closure":closure,
|
||||
"closure_history":[], "legacy_handoff":null, "targets":[], "tags":[], "assignment_tracking":[tracking],
|
||||
"events":[post, post_fixture(false, "discussion")], "assignments":[], "linked_objectives":[],
|
||||
"implementation_reports":[], "merge_requests":[merge.clone()], "merge_request":merge,
|
||||
"event_page":{"next_cursor":"stable-cursor", "has_more":true}
|
||||
});
|
||||
let detail = serde_json::to_value(project_ticket_detail(input.clone()).unwrap()).unwrap();
|
||||
assert_eq!(detail.to_string().matches(&body).count(), 1);
|
||||
assert!(detail.get("body").is_none());
|
||||
assert!(detail.get("merge_request").is_none());
|
||||
assert_eq!(detail["thread"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(detail["initial_post"]["event_ref"], "post-initial");
|
||||
assert_eq!(detail["initial_post"]["body_digest"], "a".repeat(64));
|
||||
assert!(detail["current_closure"].get("explanation").is_none());
|
||||
let history = project_ticket_history(input.clone()).unwrap();
|
||||
assert_eq!(history.as_object().unwrap().len(), 3);
|
||||
assert_eq!(history["event_page"]["next_cursor"], "stable-cursor");
|
||||
let summary = project_ticket_status(input.clone()).unwrap();
|
||||
// Reconstruct the removed model-facing aliases on this same fixture.
|
||||
let mut duplicated = detail.clone();
|
||||
duplicated["body"] = detail["initial_post"]["body"].clone();
|
||||
duplicated["body_truncated"] = detail["initial_post"]["body_truncated"].clone();
|
||||
duplicated["merge_request"] = detail["merge_requests"][0].clone();
|
||||
duplicated["thread"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
.insert(0, detail["initial_post"].clone());
|
||||
assert_eq!(duplicated.to_string().matches(&body).count(), 3);
|
||||
assert!(duplicated.to_string().len() > detail.to_string().len());
|
||||
let mut changed = input.clone();
|
||||
changed["content_digest"] = json!("new-requirements-digest");
|
||||
changed["merge_requests"][0]["current_subject_ref"] = json!("sha2");
|
||||
changed["merge_requests"][0]["review_status"] = json!("pending");
|
||||
let changed = project_ticket_status(changed).unwrap();
|
||||
assert_ne!(summary["content_digest"], changed["content_digest"]);
|
||||
assert_ne!(
|
||||
summary["merge_requests"][0]["subject_ref"],
|
||||
changed["merge_requests"][0]["subject_ref"]
|
||||
);
|
||||
assert_ne!(
|
||||
summary["merge_requests"][0]["review_status"],
|
||||
changed["merge_requests"][0]["review_status"]
|
||||
);
|
||||
assert_eq!(summary["content_digest"], "content");
|
||||
assert_eq!(summary["closure_operation_key"], "close");
|
||||
assert_eq!(summary["assignment_tracking"][0]["status"], "waiting");
|
||||
assert_eq!(
|
||||
summary["assignment_tracking"][0]["last_event_id"],
|
||||
"handling-event"
|
||||
);
|
||||
assert_eq!(summary["merge_requests"][0]["subject_ref"], "sha1");
|
||||
assert_eq!(
|
||||
summary["merge_requests"][0]["review_status"],
|
||||
"request_changes"
|
||||
);
|
||||
let huge = "全文🦀".repeat(100_000);
|
||||
input["body"] = json!(huge);
|
||||
input["initial_post"]["body"] = json!(huge);
|
||||
input["current_closure"]["explanation"] = json!(huge);
|
||||
input["assignment_tracking"][0]["tracking"]["waiting_reason"] = json!(huge);
|
||||
input["merge_requests"][0]["review_excerpt"] = json!(huge);
|
||||
input["closure_history"] = json!(vec![json!({"explanation":huge}); 20]);
|
||||
input["implementation_reports"] = json!(vec![json!({"excerpt":huge}); 20]);
|
||||
input["events"] = json!(vec![post_fixture(false, "history prose"); 1000]);
|
||||
let larger = project_ticket_status(input).unwrap();
|
||||
assert_eq!(summary, larger);
|
||||
eprintln!(
|
||||
"Ticket workspace bytes: duplicated fixture={}, detail={}, summary={} (identical with large prose and 1000 events)",
|
||||
duplicated.to_string().len(),
|
||||
detail.to_string().len(),
|
||||
summary.to_string().len()
|
||||
);
|
||||
let mut full = post_fixture(false, &huge);
|
||||
assert_eq!(project_ticket_post(full.clone()).unwrap()["body"], huge);
|
||||
full["body_truncated"] = json!(true);
|
||||
assert!(project_ticket_post(full).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ticket_tag_history_projection_preserves_observed_names_and_identities() {
|
||||
let tag = serde_json::json!({"workspace_id":"ws","id":"tag-1","name":"Old name","last_operation_key":"create","deleted":false});
|
||||
|
||||
@@ -17,14 +17,14 @@ use crate::feature::{
|
||||
ToolDeclaration,
|
||||
};
|
||||
use crate::session_capture::{
|
||||
ReadDetail, ReadOptions, ReadSelector, ReferenceKind, SearchOptions, SessionCapture,
|
||||
SessionEntryRef, ToolPart,
|
||||
CapturePurpose, RawField, ReadDetail, ReadOptions, ReadSelector, ReferenceKind, SearchOptions,
|
||||
SessionCapture, SessionEntryRef, ToolPart,
|
||||
};
|
||||
|
||||
const SHOW_OVERVIEW_DESCRIPTION: &str =
|
||||
"Show a sparse, bounded index of real user and assistant session entries.";
|
||||
const SEARCH_ENTRIES_DESCRIPTION: &str = "Search or compactly list a bounded range of committed session entries. Reasoning entries are never exposed.";
|
||||
const READ_ENTRY_DESCRIPTION: &str = "Read one committed session entry by stable SessionEntryRef. Compact mode is the default; full mode includes bounded tool input or output.";
|
||||
const READ_ENTRY_DESCRIPTION: &str = "Read one committed session entry by stable SessionEntryRef. Compact mode is the default; full mode is a bounded rendering, not raw source. Raw mode requires raw_field and supports UTF-8 byte_offset/next_offset without omission; use capture_id to bind continuation to this exact capture.";
|
||||
const DEFAULT_PAGE_LIMIT: usize = 20;
|
||||
const MAX_PAGE_LIMIT: usize = 100;
|
||||
const DEFAULT_READ_ITEMS: usize = 10;
|
||||
@@ -173,6 +173,38 @@ struct ReadEntryParams {
|
||||
mode: String,
|
||||
#[serde(default)]
|
||||
max_items: Option<usize>,
|
||||
#[serde(default)]
|
||||
raw_field: Option<RawEntryField>,
|
||||
#[serde(default)]
|
||||
part_index: Option<usize>,
|
||||
#[serde(default)]
|
||||
byte_offset: Option<usize>,
|
||||
#[serde(default)]
|
||||
max_bytes: Option<usize>,
|
||||
#[serde(default)]
|
||||
capture_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum RawEntryField {
|
||||
MessagePart,
|
||||
PublicSystem,
|
||||
ToolArguments,
|
||||
ToolSummary,
|
||||
ToolContent,
|
||||
}
|
||||
|
||||
impl RawEntryField {
|
||||
fn into_common(self, part_index: usize) -> RawField {
|
||||
match self {
|
||||
Self::MessagePart => RawField::MessagePart(part_index),
|
||||
Self::PublicSystem => RawField::PublicSystem,
|
||||
Self::ToolArguments => RawField::ToolArguments,
|
||||
Self::ToolSummary => RawField::ToolSummary,
|
||||
Self::ToolContent => RawField::ToolContent,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_read_mode() -> String {
|
||||
@@ -243,15 +275,9 @@ impl Tool for SearchEntriesTool {
|
||||
let from = params.from.as_deref().map(parse_entry_ref).transpose()?;
|
||||
let through = params.through.as_deref().map(parse_entry_ref).transpose()?;
|
||||
let view = self.state.view();
|
||||
if let (Some(from), Some(through)) = (&from, &through) {
|
||||
if view.source_index_for_ref(from) > view.source_index_for_ref(through) {
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"SearchEntries from must not be after through".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let limit = bounded_limit(params.limit, DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT);
|
||||
let hits = view.search(&SearchOptions {
|
||||
let hits = view
|
||||
.search_checked(&SearchOptions {
|
||||
query: params.query,
|
||||
kind,
|
||||
tool_part,
|
||||
@@ -261,7 +287,8 @@ impl Tool for SearchEntriesTool {
|
||||
from,
|
||||
through,
|
||||
offset: params.offset,
|
||||
});
|
||||
})
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
let entries = hits
|
||||
.iter()
|
||||
.map(|hit| {
|
||||
@@ -272,6 +299,7 @@ impl Tool for SearchEntriesTool {
|
||||
"tool_name": hit.tool_name,
|
||||
"label": hit.label,
|
||||
"text": hit.summary,
|
||||
"text_kind": "preview",
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
@@ -299,12 +327,62 @@ impl Tool for ReadEntryTool {
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let params: ReadEntryParams = parse_input("ReadEntry", input_json)?;
|
||||
let entry_ref = parse_entry_ref(¶ms.entry_ref)?;
|
||||
let view = self.state.view();
|
||||
if params
|
||||
.capture_id
|
||||
.as_deref()
|
||||
.is_some_and(|id| id != view.capture_id())
|
||||
{
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"capture identity does not match".into(),
|
||||
));
|
||||
}
|
||||
if params.mode == "raw" {
|
||||
let field = params
|
||||
.raw_field
|
||||
.ok_or_else(|| ToolError::InvalidArgument("raw mode requires raw_field".into()))?;
|
||||
if params.part_index.is_some() && !matches!(field, RawEntryField::MessagePart) {
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"part_index is only valid for message_part".into(),
|
||||
));
|
||||
}
|
||||
let reference = view
|
||||
.scoped_ref(&entry_ref)
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
let raw = view
|
||||
.read_raw(
|
||||
&reference,
|
||||
CapturePurpose::Observation,
|
||||
field.into_common(params.part_index.unwrap_or(0)),
|
||||
params.byte_offset.unwrap_or(0),
|
||||
params
|
||||
.max_bytes
|
||||
.unwrap_or(MAX_READ_BYTES)
|
||||
.clamp(1, MAX_READ_BYTES),
|
||||
)
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
return json_output(
|
||||
"Read raw session source bytes.".into(),
|
||||
serde_json::json!({
|
||||
"text_kind": "raw", "raw": raw,
|
||||
}),
|
||||
);
|
||||
}
|
||||
if params.raw_field.is_some()
|
||||
|| params.part_index.is_some()
|
||||
|| params.byte_offset.is_some()
|
||||
|| params.max_bytes.is_some()
|
||||
{
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"raw fields require raw mode".into(),
|
||||
));
|
||||
}
|
||||
let detail = match params.mode.as_str() {
|
||||
"compact" => ReadDetail::Compact,
|
||||
"full" => ReadDetail::Full,
|
||||
other => {
|
||||
return Err(ToolError::InvalidArgument(format!(
|
||||
"invalid ReadEntry mode {other:?}; expected compact or full"
|
||||
"invalid ReadEntry mode {other:?}; expected compact, full, or raw"
|
||||
)));
|
||||
}
|
||||
};
|
||||
@@ -347,6 +425,7 @@ impl Tool for ReadEntryTool {
|
||||
serde_json::json!({
|
||||
"entries": entries,
|
||||
"truncated": read.truncated,
|
||||
"text_kind": "rendered",
|
||||
}),
|
||||
)
|
||||
}
|
||||
@@ -408,6 +487,7 @@ mod tests {
|
||||
|
||||
fn state() -> SessionExploreState {
|
||||
SessionExploreState::new(SessionCapture::new(
|
||||
"session",
|
||||
"segment-1",
|
||||
vec![
|
||||
Item::user_message("first"),
|
||||
@@ -479,4 +559,64 @@ mod tests {
|
||||
.unwrap();
|
||||
assert!(output.content.unwrap().contains("second"));
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn search_rejects_unknown_or_reversed_capture_endpoints() {
|
||||
let search = search_entries_definition(state())().1;
|
||||
for input in [
|
||||
r#"{"from":"E99999999"}"#,
|
||||
r#"{"through":"E99999999"}"#,
|
||||
r#"{"from":"E00000003","through":"E00000000"}"#,
|
||||
] {
|
||||
assert!(
|
||||
search
|
||||
.execute(input, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raw_tool_arguments_reassemble_without_preview_or_utf8_loss() {
|
||||
let arguments = "日本語\n\"\\".repeat(100);
|
||||
let state = SessionExploreState::new(SessionCapture::new(
|
||||
"session",
|
||||
"segment",
|
||||
vec![Item::tool_call("call", "Read", arguments.clone())],
|
||||
));
|
||||
let capture_id = state.view().capture_id().to_owned();
|
||||
let tool = read_entry_definition(state)().1;
|
||||
let mut offset = 0;
|
||||
let mut assembled = String::new();
|
||||
loop {
|
||||
let input = serde_json::json!({
|
||||
"entry_ref": "E00000000", "mode": "raw", "raw_field": "tool_arguments",
|
||||
"byte_offset": offset, "max_bytes": 17, "capture_id": capture_id,
|
||||
});
|
||||
let output = tool
|
||||
.execute(
|
||||
&input.to_string(),
|
||||
agen::tool::ToolExecutionContext::direct(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
||||
assert_eq!(value["text_kind"], "raw");
|
||||
let raw = &value["raw"];
|
||||
assert_eq!(raw["offset"], offset);
|
||||
let text = raw["text"].as_str().unwrap();
|
||||
assert!(text.len() <= 17);
|
||||
assembled.push_str(text);
|
||||
match raw["next_offset"].as_u64() {
|
||||
Some(next) => {
|
||||
assert!(next > offset as u64);
|
||||
offset = next as usize;
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
assert_eq!(assembled, arguments);
|
||||
assert!(tool.execute(r#"{"entry_ref":"E00000000","mode":"raw","raw_field":"tool_arguments","capture_id":"other"}"#,
|
||||
agen::tool::ToolExecutionContext::direct()).await.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,13 +39,66 @@ use crate::hook::{
|
||||
HookErrorCategory, HookExecutionPolicy, HookFailurePolicy, HookPreRequestAction, PreLlmRequest,
|
||||
PreRequestContext, RunCommitted, RunCommittedContext,
|
||||
};
|
||||
use crate::session_capture::{SessionCapture, SessionEntryEvidence};
|
||||
use crate::session_capture::{CaptureBody, SessionCapture, SessionEntryEvidence};
|
||||
use crate::subjektiv::SubjektivHostConnection;
|
||||
#[cfg(test)]
|
||||
use crate::worker::WorkspaceClient;
|
||||
#[cfg(test)]
|
||||
use server_api::SubjektivMemoryBackendRequest;
|
||||
|
||||
// Pending-commit receipt interpretation belongs to Memory, not the shared capture.
|
||||
struct CommittedMemoryCall {
|
||||
call_id: String,
|
||||
name: String,
|
||||
arguments: String,
|
||||
evidence: SessionEntryEvidence,
|
||||
}
|
||||
|
||||
fn committed_pending_tool_calls(view: &SessionCapture, names: &[&str]) -> Vec<CommittedMemoryCall> {
|
||||
let pending: HashSet<_> = view
|
||||
.entries()
|
||||
.filter_map(|entry| {
|
||||
let CaptureBody::ToolResult {
|
||||
call_id,
|
||||
content: Some(content),
|
||||
is_error: false,
|
||||
..
|
||||
} = &entry.body
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
(serde_json::from_str::<serde_json::Value>(content)
|
||||
.ok()
|
||||
.and_then(|value| value.get("status").cloned())
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_str)
|
||||
== Some("pending_commit"))
|
||||
.then_some(call_id.as_str())
|
||||
})
|
||||
.collect();
|
||||
view.entries()
|
||||
.filter_map(|entry| {
|
||||
let CaptureBody::ToolCall {
|
||||
call_id,
|
||||
name,
|
||||
arguments,
|
||||
} = &entry.body
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
if !names.contains(&name.as_str()) || !pending.contains(call_id.as_str()) {
|
||||
return None;
|
||||
}
|
||||
Some(CommittedMemoryCall {
|
||||
call_id: call_id.clone(),
|
||||
name: name.clone(),
|
||||
arguments: arguments.clone(),
|
||||
evidence: view.evidence_for_tool_call(call_id)?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
const QUERY_TOOL: &str = "SubjektivMemoryQuery";
|
||||
const READ_TOOL: &str = "SubjektivMemoryRead";
|
||||
const LIST_CHANGES_TOOL: &str = "SubjektivMemoryListChanges";
|
||||
@@ -518,9 +571,13 @@ impl SubjektivMemoryState {
|
||||
.capture()
|
||||
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
|
||||
let session_id = capture.session_id.clone();
|
||||
let view = SessionCapture::from_history_entries(capture.segment_id, capture.history);
|
||||
let committed = view
|
||||
.committed_pending_tool_calls(&[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL])
|
||||
let view = SessionCapture::from_complete_history(
|
||||
capture.session_id.clone(),
|
||||
capture.segment_id,
|
||||
capture.history,
|
||||
);
|
||||
let committed =
|
||||
committed_pending_tool_calls(&view, &[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL])
|
||||
.into_iter()
|
||||
.any(|call| {
|
||||
receipt_id(&session_id, &call.call_id)
|
||||
@@ -623,10 +680,13 @@ fn replay_pending_committed_memory_with_policy(
|
||||
.capture
|
||||
.capture()
|
||||
.map_err(|error| HookError::new(HookErrorCategory::Dependency, error.to_string()))?;
|
||||
let view =
|
||||
SessionCapture::from_history_entries(capture.segment_id.clone(), capture.history.clone());
|
||||
let view = SessionCapture::from_complete_history(
|
||||
capture.session_id.clone(),
|
||||
capture.segment_id.clone(),
|
||||
capture.history.clone(),
|
||||
);
|
||||
let mut failures = Vec::new();
|
||||
for call in view.committed_pending_tool_calls(&[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL]) {
|
||||
for call in committed_pending_tool_calls(&view, &[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL]) {
|
||||
let receipt_id = match receipt_id(&capture.session_id, &call.call_id) {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
@@ -755,8 +815,11 @@ fn resolve_entry_refs(
|
||||
capture: &crate::feature::session::CommittedSessionCapture,
|
||||
entry_refs: &[String],
|
||||
) -> Result<Vec<SessionEntryEvidence>, ToolError> {
|
||||
let view =
|
||||
SessionCapture::from_history_entries(capture.segment_id.clone(), capture.history.clone());
|
||||
let view = SessionCapture::from_complete_history(
|
||||
capture.session_id.clone(),
|
||||
capture.segment_id.clone(),
|
||||
capture.history.clone(),
|
||||
);
|
||||
let mut seen = HashSet::new();
|
||||
entry_refs
|
||||
.iter()
|
||||
|
||||
@@ -80,7 +80,11 @@ use crate::wip::{
|
||||
use crate::worker::{WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod};
|
||||
use agen::tool::{Tool, ToolError, ToolExecutionContext, ToolMeta, ToolOutput};
|
||||
|
||||
use super::resource_projection::{project_ticket_detail, project_ticket_query};
|
||||
use super::resource_projection::{
|
||||
project_ticket_detail, project_ticket_history, project_ticket_post, project_ticket_query,
|
||||
project_ticket_status,
|
||||
};
|
||||
use ticket::tool::ShowTicketView;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum WorkspaceTicketReadKind {
|
||||
@@ -102,7 +106,7 @@ impl WorkspaceTicketReadKind {
|
||||
"Query authoritative Workspace Tickets with bounded typed filters, stable snippets, evidence summaries, and cursor metadata."
|
||||
}
|
||||
Self::Show => {
|
||||
"Show one authoritative Workspace Ticket with its ordered implementation targets (repository_key, ref_selector, access), content digest, paged thread, links, historical implementation reports, and current Merge Request readiness evidence. Read targets here before choosing a work repository; a null selector remains unspecified, not an inferred default."
|
||||
"Read one authoritative Workspace Ticket. Use view=summary for current state/CAS, targets, delegation identities/status and MR evidence without prose/history; detail (default) for canonical initial_post and bounded discussion; history with event_cursor for only thread/page metadata; post with an observed event_ref for the exact full post. Never use truncated text for edit CAS or guess event IDs. Read targets before choosing a repository; a null selector remains unspecified. Summary alone is not approval or completion authority."
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -227,9 +231,14 @@ struct WorkspaceQueryTicketInput {
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, JsonSchema)]
|
||||
struct WorkspaceShowTicketInput {
|
||||
/// summary omits prose/history; detail is default; history returns only a thread page; post returns the exact full post up to 1 MiB (larger posts fail, never truncate).
|
||||
#[serde(default)]
|
||||
view: ShowTicketView,
|
||||
/// Exact observed event_ref, required only for view=post. Never a display ordinal.
|
||||
event_ref: Option<String>,
|
||||
/// Ticket reference. Prefer `T-*`; canonical internal ids remain accepted for compatibility.
|
||||
id: String,
|
||||
/// Most-recent thread entries to return, bounded by the Backend to 1..=50.
|
||||
/// Most-recent thread entries to return, bounded by the Backend to 1..=100.
|
||||
event_limit: Option<usize>,
|
||||
/// Opaque event cursor returned by a prior ShowTicket page.
|
||||
event_cursor: Option<String>,
|
||||
@@ -251,6 +260,8 @@ impl Tool for WorkspaceTicketReadTool {
|
||||
let workspace_id = self.client.workspace_id().ok_or_else(|| {
|
||||
ToolError::InvalidArgument("Workspace Ticket reads require workspace identity".into())
|
||||
})?;
|
||||
let mut view = ShowTicketView::Detail;
|
||||
let mut method = WorkspaceRequestMethod::Post;
|
||||
let (path, body) = match self.kind {
|
||||
WorkspaceTicketReadKind::Query => {
|
||||
let input: WorkspaceQueryTicketInput = serde_json::from_str(&input)
|
||||
@@ -276,22 +287,41 @@ impl Tool for WorkspaceTicketReadTool {
|
||||
"ShowTicket.id must not be empty".into(),
|
||||
));
|
||||
}
|
||||
let path = format!("/api/w/{workspace_id}/tickets/{}/show", input.id.trim());
|
||||
view = input.view;
|
||||
view.validate(input.event_cursor.as_deref(), input.event_ref.as_deref())
|
||||
.map_err(ToolError::InvalidArgument)?;
|
||||
let id =
|
||||
WorkspaceHttpTicketBackend::ticket_path(&TicketIdOrSlug::from(input.id.trim()));
|
||||
if let Some(event_ref) = input.event_ref {
|
||||
method = WorkspaceRequestMethod::Get;
|
||||
let event_ref =
|
||||
WorkspaceHttpTicketBackend::ticket_path(&TicketIdOrSlug::from(event_ref));
|
||||
(
|
||||
format!("/api/w/{workspace_id}/tickets/{id}/posts/{event_ref}"),
|
||||
Value::Null,
|
||||
)
|
||||
} else {
|
||||
let path = format!("/api/w/{workspace_id}/tickets/{id}/show");
|
||||
let body = json!({
|
||||
"event_limit": input.event_limit,
|
||||
"event_limit": if view == ShowTicketView::Summary { Some(1) } else { input.event_limit },
|
||||
"event_cursor": input.event_cursor,
|
||||
});
|
||||
(path, body)
|
||||
}
|
||||
}
|
||||
};
|
||||
let response = self
|
||||
.client
|
||||
.execute(WorkspaceRequest::json(
|
||||
WorkspaceRequestMethod::Post,
|
||||
.execute(if method == WorkspaceRequestMethod::Get {
|
||||
WorkspaceRequest::get(path)
|
||||
} else {
|
||||
WorkspaceRequest::json(
|
||||
method,
|
||||
path,
|
||||
serde_json::to_string(&body)
|
||||
.map_err(|error| ToolError::Internal(error.to_string()))?,
|
||||
))
|
||||
)
|
||||
})
|
||||
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
|
||||
if !response.is_success() {
|
||||
return Err(ToolError::ExecutionFailed(ticket_api_error_message(
|
||||
@@ -308,9 +338,20 @@ impl Tool for WorkspaceTicketReadTool {
|
||||
WorkspaceTicketReadKind::Query => serde_json::to_string(
|
||||
&project_ticket_query(response_value).map_err(ToolError::ExecutionFailed)?,
|
||||
),
|
||||
WorkspaceTicketReadKind::Show => serde_json::to_string(
|
||||
WorkspaceTicketReadKind::Show => match view {
|
||||
ShowTicketView::Detail => serde_json::to_string(
|
||||
&project_ticket_detail(response_value).map_err(ToolError::ExecutionFailed)?,
|
||||
),
|
||||
ShowTicketView::Summary => serde_json::to_string(
|
||||
&project_ticket_status(response_value).map_err(ToolError::ExecutionFailed)?,
|
||||
),
|
||||
ShowTicketView::History => serde_json::to_string(
|
||||
&project_ticket_history(response_value).map_err(ToolError::ExecutionFailed)?,
|
||||
),
|
||||
ShowTicketView::Post => serde_json::to_string(
|
||||
&json!({"post": project_ticket_post(response_value).map_err(ToolError::ExecutionFailed)?}),
|
||||
),
|
||||
},
|
||||
}
|
||||
.map_err(|error| ToolError::Internal(error.to_string()))?;
|
||||
Ok(ToolOutput {
|
||||
@@ -2894,6 +2935,127 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn show_views_share_tool_and_route_bound_wip_wiring() {
|
||||
for view in ["summary", "detail", "history", "post"] {
|
||||
let mut detail = target_detail_fixture(
|
||||
json!([{"repository_key":"main", "ref_selector":null, "access":"read_write"}]),
|
||||
);
|
||||
let unicode = "全文🦀é\n".repeat(20_000);
|
||||
detail["event_page"] = json!({"next_cursor":"opaque-before", "has_more":true});
|
||||
let mut post = detail["initial_post"].clone();
|
||||
post["event_ref"] = json!("persisted:id/with unicode界");
|
||||
post["body"] = json!(unicode);
|
||||
let response = if view == "post" { post.clone() } else { detail };
|
||||
let client = Arc::new(DecisionClient {
|
||||
requests: Mutex::new(Vec::new()),
|
||||
response: crate::worker::WorkspaceResponse {
|
||||
status: 200,
|
||||
body: response.to_string(),
|
||||
},
|
||||
});
|
||||
let (meta, tool) =
|
||||
workspace_ticket_read_definition(client.clone(), WorkspaceTicketReadKind::Show)();
|
||||
assert!(meta.input_schema["properties"]["view"].is_object());
|
||||
assert!(meta.input_schema["properties"]["event_ref"].is_object());
|
||||
let mut input = json!({"id":"T-739", "view":view});
|
||||
if view == "post" {
|
||||
input["event_ref"] = post["event_ref"].clone();
|
||||
}
|
||||
if view == "history" {
|
||||
input["event_cursor"] = json!("opaque-before");
|
||||
}
|
||||
let direct = ticket_tool_output_json(
|
||||
tool.execute(&input.to_string(), ToolExecutionContext::direct())
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap_or_else(|_| panic!("Ticket {view} returned invalid JSON"));
|
||||
let tools =
|
||||
native_ticket_tools(client.clone(), TicketFeatureAccess::read_only()).unwrap();
|
||||
let descriptor = ticket_descriptor(&tools, "Ticket", "bound").unwrap();
|
||||
let read = descriptor
|
||||
.operations
|
||||
.iter()
|
||||
.find(|op| op.name == "read")
|
||||
.unwrap();
|
||||
assert!(read.parameters.iter().any(|param| param.name == "view"));
|
||||
assert!(
|
||||
read.parameters
|
||||
.iter()
|
||||
.any(|param| param.name == "event_ref")
|
||||
);
|
||||
assert!(!read.parameters.iter().any(|param| param.name == "id"));
|
||||
let resolver = TicketItemResolver {
|
||||
tools: operation_map(tools),
|
||||
permissions: None,
|
||||
collection_route: "/tickets".into(),
|
||||
observations: Arc::new(Mutex::new(TicketObservationState::default())),
|
||||
};
|
||||
input.as_object_mut().unwrap().remove("id");
|
||||
let arguments = input
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|(key, value)| (key.clone(), json_to_wip(value).unwrap()))
|
||||
.collect();
|
||||
let bound = resolver.resolve("T-739").unwrap();
|
||||
let output = bound
|
||||
.handler
|
||||
.call(
|
||||
"read",
|
||||
&arguments,
|
||||
WipCallContext {
|
||||
execution: ToolExecutionContext::direct(),
|
||||
security_context: "reader".into(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| panic!("WIP {view} failed"));
|
||||
assert_eq!(wip_to_json(&output.value).unwrap(), direct);
|
||||
match view {
|
||||
"post" => {
|
||||
assert_eq!(direct["post"]["body"], unicode);
|
||||
assert_eq!(direct["post"]["body_truncated"], false);
|
||||
}
|
||||
"history" => {
|
||||
assert_eq!(direct.as_object().unwrap().len(), 3);
|
||||
}
|
||||
"summary" => {
|
||||
assert!(direct.get("initial_post").is_none());
|
||||
assert_eq!(direct["content_digest"], "observed-content");
|
||||
}
|
||||
_ => {
|
||||
assert_eq!(direct["initial_post"]["body"], "Requirements");
|
||||
}
|
||||
}
|
||||
let requests = client.requests.lock().unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert_eq!(requests[0].method, requests[1].method);
|
||||
assert_eq!(requests[0].path, requests[1].path);
|
||||
assert_eq!(requests[0].body, requests[1].body);
|
||||
if view == "post" {
|
||||
assert_eq!(requests[0].method, WorkspaceRequestMethod::Get);
|
||||
assert_eq!(
|
||||
requests[0].path,
|
||||
"/api/w/workspace/tickets/T-739/posts/persisted%3Aid%2Fwith%20unicode%E7%95%8C"
|
||||
);
|
||||
assert!(requests[0].body.is_none());
|
||||
} else {
|
||||
let body: Value =
|
||||
serde_json::from_str(requests[0].body.as_deref().unwrap()).unwrap();
|
||||
assert!(body.get("view").is_none());
|
||||
assert!(body.get("event_ref").is_none());
|
||||
if view == "summary" {
|
||||
assert_eq!(body["event_limit"], 1);
|
||||
}
|
||||
if view == "history" {
|
||||
assert_eq!(body["event_cursor"], "opaque-before");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn show_rejects_missing_or_malformed_targets_instead_of_reporting_unassigned() {
|
||||
for malformed in [
|
||||
|
||||
@@ -570,15 +570,8 @@ impl Tool for SearchSessionEntriesTool {
|
||||
let view = latest_view(&*self.provider, ¶ms.subject).await?;
|
||||
let from = params.from.as_deref().map(parse_entry_ref).transpose()?;
|
||||
let through = params.through.as_deref().map(parse_entry_ref).transpose()?;
|
||||
if let (Some(from), Some(through)) = (&from, &through) {
|
||||
if view.source_index_for_ref(from) > view.source_index_for_ref(through) {
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"SearchSessionEntries from must not be after through".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let entries = view
|
||||
.search(&SearchOptions {
|
||||
.search_checked(&SearchOptions {
|
||||
query: params.query,
|
||||
kind: params.kind.as_deref().map(parse_kind).transpose()?,
|
||||
tool_part: params
|
||||
@@ -593,6 +586,7 @@ impl Tool for SearchSessionEntriesTool {
|
||||
through,
|
||||
offset: params.offset,
|
||||
})
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?
|
||||
.into_iter()
|
||||
.map(|entry| {
|
||||
serde_json::json!({
|
||||
@@ -604,6 +598,7 @@ impl Tool for SearchSessionEntriesTool {
|
||||
"tool_name": entry.tool_name,
|
||||
"label": entry.label,
|
||||
"text": entry.summary,
|
||||
"text_kind": "preview",
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
@@ -687,7 +682,10 @@ async fn latest_view(
|
||||
.capture_worker_session(subject)
|
||||
.await
|
||||
.map_err(tool_error)?;
|
||||
// Observation providers expose a stable subject scope rather than a raw
|
||||
// Session handle. This scope is never used to select recording material.
|
||||
Ok(SessionCapture::from_session_snapshot(
|
||||
format!("observation:{}", capture.segment_id),
|
||||
capture.segment_id,
|
||||
capture.session,
|
||||
))
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
//! the regular registry decides Tool-name collisions.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -25,6 +26,7 @@ use mcp::stdio::{
|
||||
};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use super::{
|
||||
FeatureDescriptor, FeatureDiagnostic, FeatureInstallContext, FeatureInstallError,
|
||||
@@ -52,8 +54,88 @@ const MAX_LIST_ITEMS: usize = 128;
|
||||
const MAX_RESOURCE_CONTENTS: usize = 16;
|
||||
const MAX_PROMPT_MESSAGES: usize = 32;
|
||||
|
||||
/// Owns discovery clients until process reaping is confirmed. Retain a clone in
|
||||
/// startup cleanup *before* awaiting discovery. Stop/cancel discovery before
|
||||
/// calling `shutdown`; the shared lock serializes discovery and cleanup, not
|
||||
/// cancellation of an in-flight discovery task.
|
||||
///
|
||||
/// Error or cancellation leaves the actual clients here for another shutdown
|
||||
/// attempt. Dropping the final owner only provides best-effort process killing.
|
||||
#[derive(Clone, Default)]
|
||||
pub struct McpDiscoveryCleanup {
|
||||
clients: Arc<Mutex<Vec<McpStdioClient>>>,
|
||||
}
|
||||
|
||||
impl McpDiscoveryCleanup {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Try every retained client once, removing only confirmed shutdowns. An
|
||||
/// error is not a cleanup receipt; retry this same owner, not discovery.
|
||||
pub async fn shutdown(&self) -> io::Result<()> {
|
||||
let mut clients = self.clients.lock().await;
|
||||
let mut first_error = None;
|
||||
let mut index = 0;
|
||||
while index < clients.len() {
|
||||
match clients[index].shutdown().await {
|
||||
Ok(_) => {
|
||||
clients.remove(index);
|
||||
}
|
||||
Err(err) => {
|
||||
first_error.get_or_insert_with(|| discovery_shutdown_error(&err));
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
match first_error {
|
||||
Some(err) => Err(err),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn discovery_shutdown_error(err: &McpClientError) -> io::Error {
|
||||
io::Error::other(bounded_diagnostic(format!(
|
||||
"MCP discovery shutdown unconfirmed; client retained for cleanup retry: {err}"
|
||||
)))
|
||||
}
|
||||
|
||||
/// Restore preparation must fail if discovery cannot confirm process shutdown.
|
||||
/// The caller retains `cleanup` before this future is polled, including during
|
||||
/// initialization. Ordinary discovery failures remain isolated contributions
|
||||
/// only after their client has been reaped.
|
||||
pub async fn discover_stdio_tool_feature_with_cleanup(
|
||||
config: &McpConfig,
|
||||
workspace_root: &Path,
|
||||
cleanup: &McpDiscoveryCleanup,
|
||||
) -> Result<Option<McpStdioToolFeature>, io::Error> {
|
||||
if config.stdio_servers.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut feature = McpStdioToolFeature::new();
|
||||
for server in &config.stdio_servers {
|
||||
match resolve_stdio_server(server, workspace_root, None) {
|
||||
Ok(spec) => {
|
||||
feature.add_contribution(
|
||||
discover_server_tools_with_cleanup(spec, cleanup, McpStdioLimits::default())
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
feature.add_diagnostic(FeatureDiagnostic::error(bounded_diagnostic(format!(
|
||||
"failed to resolve MCP stdio server `{}`: {err}",
|
||||
server.name
|
||||
))))
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Some(feature))
|
||||
}
|
||||
|
||||
/// Discover enabled MCP stdio server tools and return a single feature module
|
||||
/// containing startup contributions for normal ToolRegistry installation.
|
||||
/// Callers needing retryable startup cleanup must use the cleanup-aware API.
|
||||
pub async fn discover_stdio_tool_feature(
|
||||
config: &McpConfig,
|
||||
workspace_root: &Path,
|
||||
@@ -81,21 +163,66 @@ pub async fn discover_stdio_tool_feature(
|
||||
}
|
||||
|
||||
async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderContribution {
|
||||
let declaration = provider_declaration(&spec.name, None);
|
||||
let cleanup = McpDiscoveryCleanup::new();
|
||||
match discover_server_tools_with_cleanup(spec, &cleanup, McpStdioLimits::default()).await {
|
||||
Ok(contribution) => contribution,
|
||||
Err(err) => {
|
||||
ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn discover_server_tools_with_cleanup(
|
||||
spec: McpStdioServerSpec,
|
||||
cleanup: &McpDiscoveryCleanup,
|
||||
limits: McpStdioLimits,
|
||||
) -> io::Result<ProtocolProviderContribution> {
|
||||
let declaration = provider_declaration(&spec.name, None);
|
||||
let mut clients = cleanup.clients.lock().await;
|
||||
if !clients.is_empty() {
|
||||
return Err(io::Error::other(
|
||||
"MCP discovery cleanup is pending; shut down the retained owner before retrying discovery",
|
||||
));
|
||||
}
|
||||
let client = match McpStdioClient::spawn_uninitialized(spec.clone(), limits) {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
// The synchronous spawn API only errors before a process exists;
|
||||
// pipe/handshake failures retain a client and use the shutdown path.
|
||||
return Ok(ProtocolProviderContribution::failed(
|
||||
declaration,
|
||||
bounded_diagnostic(err.to_string()),
|
||||
));
|
||||
}
|
||||
};
|
||||
// Spawn is synchronous: the owner holds the client before the first await
|
||||
// that can leave a running process on initialization failure/cancellation.
|
||||
clients.push(client);
|
||||
let client = clients.last_mut().expect("retained discovery client");
|
||||
let contribution = match client.initialize_with_timeout().await {
|
||||
Ok(()) => discover_connected_server_tools(spec, client).await,
|
||||
Err(err) => {
|
||||
ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string()))
|
||||
}
|
||||
};
|
||||
client
|
||||
.shutdown()
|
||||
.await
|
||||
.map_err(|err| discovery_shutdown_error(&err))?;
|
||||
drop(clients.pop());
|
||||
Ok(contribution)
|
||||
}
|
||||
|
||||
async fn discover_connected_server_tools(
|
||||
spec: McpStdioServerSpec,
|
||||
client: &mut McpStdioClient,
|
||||
) -> ProtocolProviderContribution {
|
||||
let declaration = provider_declaration(&spec.name, None);
|
||||
let mut contribution = ProtocolProviderContribution::ready(declaration.clone());
|
||||
let server_namespace = sanitize_segment(&spec.name);
|
||||
let execution_spec = spec.clone();
|
||||
|
||||
let mut client = match McpStdioClient::connect(spec, McpStdioLimits::default()).await {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
return ProtocolProviderContribution::failed(
|
||||
declaration,
|
||||
bounded_diagnostic(err.to_string()),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
let (server_version, has_tools, has_resources, has_prompts) = client
|
||||
.initialize_result()
|
||||
.map(|result| {
|
||||
@@ -133,18 +260,10 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
|
||||
{
|
||||
Ok(list) => list,
|
||||
Err(err) => {
|
||||
let mut failed = ProtocolProviderContribution::failed(
|
||||
return ProtocolProviderContribution::failed(
|
||||
declaration,
|
||||
bounded_diagnostic(err.to_string()),
|
||||
);
|
||||
if let Err(shutdown_err) = client.shutdown().await {
|
||||
failed = failed.with_diagnostic(FeatureDiagnostic::warning(
|
||||
bounded_diagnostic(format!(
|
||||
"MCP server shutdown after discovery failure failed: {shutdown_err}"
|
||||
)),
|
||||
));
|
||||
}
|
||||
return failed;
|
||||
}
|
||||
};
|
||||
let list_changes = client.snapshot_list_changes().await;
|
||||
@@ -166,20 +285,12 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
|
||||
{
|
||||
Ok(list) => list,
|
||||
Err(err) => {
|
||||
let mut failed = ProtocolProviderContribution::failed(
|
||||
return ProtocolProviderContribution::failed(
|
||||
declaration,
|
||||
bounded_diagnostic(format!(
|
||||
"MCP server sent notifications/tools/list_changed during tool discovery and refresh failed: {err}"
|
||||
)),
|
||||
);
|
||||
if let Err(shutdown_err) = client.shutdown().await {
|
||||
failed = failed.with_diagnostic(FeatureDiagnostic::warning(
|
||||
bounded_diagnostic(format!(
|
||||
"MCP server shutdown after discovery refresh failure failed: {shutdown_err}"
|
||||
)),
|
||||
));
|
||||
}
|
||||
return failed;
|
||||
}
|
||||
};
|
||||
let refresh_changes = client.snapshot_list_changes().await;
|
||||
@@ -197,13 +308,6 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let shutdown_result = client.shutdown().await;
|
||||
if let Err(err) = shutdown_result {
|
||||
contribution =
|
||||
contribution.with_diagnostic(FeatureDiagnostic::warning(bounded_diagnostic(format!(
|
||||
"MCP server shutdown after capability discovery failed: {err}"
|
||||
))));
|
||||
}
|
||||
|
||||
if let Some(list) = list {
|
||||
contribution = normalize_listed_tools(
|
||||
@@ -1819,6 +1923,140 @@ read exit_notification || true
|
||||
McpStdioServerSpec::new("shell-mock", "/bin/sh").args(["-c".to_string(), script])
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn discovery_config(spec: McpStdioServerSpec) -> McpConfig {
|
||||
McpConfig {
|
||||
stdio_servers: vec![manifest::McpStdioServerConfig {
|
||||
name: spec.name,
|
||||
command: spec.command,
|
||||
args: spec.args,
|
||||
cwd: None,
|
||||
env: Default::default(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn cancelled_discovery_retains_initializing_client_in_cleanup_clone() {
|
||||
use std::future::{Future, poll_fn};
|
||||
use std::task::Poll;
|
||||
use std::time::Duration;
|
||||
|
||||
let config = discovery_config(
|
||||
McpStdioServerSpec::new("initializing", "/bin/sh")
|
||||
.args(["-c", "while read line; do :; done"]),
|
||||
);
|
||||
let cleanup = McpDiscoveryCleanup::new();
|
||||
let retained = cleanup.clone();
|
||||
let mut discovery = Box::pin(discover_stdio_tool_feature_with_cleanup(
|
||||
&config,
|
||||
Path::new("."),
|
||||
&cleanup,
|
||||
));
|
||||
poll_fn(|cx| {
|
||||
assert!(discovery.as_mut().poll(cx).is_pending());
|
||||
Poll::Ready(())
|
||||
})
|
||||
.await;
|
||||
drop(discovery);
|
||||
drop(cleanup);
|
||||
assert_eq!(retained.clients.lock().await.len(), 1);
|
||||
let retry_error =
|
||||
discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &retained)
|
||||
.await
|
||||
.err()
|
||||
.expect("discovery must not replace an unreaped client");
|
||||
assert!(retry_error.to_string().contains("cleanup is pending"));
|
||||
assert_eq!(retained.clients.lock().await.len(), 1);
|
||||
tokio::time::timeout(Duration::from_secs(3), retained.shutdown())
|
||||
.await
|
||||
.expect("cleanup bounded")
|
||||
.expect("initializing process reaped");
|
||||
assert!(retained.clients.lock().await.is_empty());
|
||||
retained
|
||||
.shutdown()
|
||||
.await
|
||||
.expect("confirmed cleanup repeatable");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn cancelled_owner_shutdown_keeps_client_for_retry() {
|
||||
use std::future::{Future, poll_fn};
|
||||
use std::task::Poll;
|
||||
use std::time::Duration;
|
||||
|
||||
let cleanup = McpDiscoveryCleanup::new();
|
||||
cleanup.clients.lock().await.push(
|
||||
McpStdioClient::spawn_uninitialized(
|
||||
McpStdioServerSpec::new("shutdown-pending", "/bin/sh")
|
||||
.args(["-c", "while :; do :; done"]),
|
||||
McpStdioLimits {
|
||||
shutdown_timeout: Duration::from_millis(50),
|
||||
kill_timeout: Duration::from_millis(100),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect("local process spawned"),
|
||||
);
|
||||
let mut shutdown = Box::pin(cleanup.shutdown());
|
||||
poll_fn(|cx| {
|
||||
assert!(shutdown.as_mut().poll(cx).is_pending());
|
||||
Poll::Ready(())
|
||||
})
|
||||
.await;
|
||||
drop(shutdown);
|
||||
assert_eq!(cleanup.clients.lock().await.len(), 1);
|
||||
tokio::time::timeout(Duration::from_secs(2), cleanup.shutdown())
|
||||
.await
|
||||
.expect("retry bounded")
|
||||
.expect("retained client shutdown");
|
||||
assert!(cleanup.clients.lock().await.is_empty());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn cleanup_aware_discovery_reaps_success_and_initialize_failure() {
|
||||
let cleanup = McpDiscoveryCleanup::new();
|
||||
let config = discovery_config(shell_capability_server(r#"{"resources":{}}"#));
|
||||
let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup)
|
||||
.await
|
||||
.expect("discovery cleanup confirmed")
|
||||
.expect("configured feature");
|
||||
assert!(
|
||||
feature.contributions[0]
|
||||
.tools
|
||||
.iter()
|
||||
.any(|tool| { tool.name() == "Mcp_shell_mock_resources_read" })
|
||||
);
|
||||
assert!(cleanup.clients.lock().await.is_empty());
|
||||
|
||||
let spec = McpStdioServerSpec::new("init-rejected", "/bin/sh").args([
|
||||
"-c",
|
||||
r#"read init || exit 1
|
||||
printf '%s\n' '{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"initialization rejected"}}'
|
||||
while read line; do :; done
|
||||
"#,
|
||||
]);
|
||||
let config = discovery_config(spec);
|
||||
let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup)
|
||||
.await
|
||||
.expect("failure isolated only after cleanup")
|
||||
.expect("failed contribution preserved");
|
||||
assert!(feature.contributions[0].tools.is_empty());
|
||||
assert!(
|
||||
feature.contributions[0]
|
||||
.diagnostics
|
||||
.iter()
|
||||
.any(|diagnostic| {
|
||||
diagnostic.severity == FeatureDiagnosticSeverity::Error
|
||||
&& diagnostic.message.contains("initialization rejected")
|
||||
})
|
||||
);
|
||||
assert!(cleanup.clients.lock().await.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mcp_resource_prompt_operation_tools_are_namespaced_and_untrusted() {
|
||||
let declaration = provider_declaration("demo", Some("1.0.0"));
|
||||
|
||||
@@ -179,11 +179,11 @@ async fn builtin_journal_profile_installs_only_bound_tools_and_finalizes_saved_s
|
||||
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":{}})),
|
||||
prose(),
|
||||
]);
|
||||
// This runner/Feature fixture has no selected sources; descriptor delivery gates
|
||||
// for nonempty ranges are owned and tested by the Subjektiv service.
|
||||
// This runner/Feature fixture has no selected sources; actual body-bearing input,
|
||||
// trusted delivery and checkpoint gates are covered by the Host integration suites.
|
||||
let mut journal_request = request("builtin:subjektiv-journal-recording");
|
||||
journal_request.input = json!({"selected_ranges":[],"origins":[]});
|
||||
journal_request.instruction = "Read immutable selected_ranges through ReadJournalText Source before recording; descriptors are not source bodies.".into();
|
||||
journal_request.input = json!({"selected_ranges":[],"capture":{"entries":[]}});
|
||||
journal_request.instruction = "Record only the immutable canonical bodies in this Job input. Do not read original Sessions through ReadJournalText Source.".into();
|
||||
let job = prepare_job_from_registry_with_grant(
|
||||
&journal_request,
|
||||
root.path(),
|
||||
@@ -249,7 +249,7 @@ async fn builtin_journal_profile_installs_only_bound_tools_and_finalizes_saved_s
|
||||
"ReadJournalText",
|
||||
"Source",
|
||||
"content_token",
|
||||
"Conflict",
|
||||
"input digest",
|
||||
] {
|
||||
assert!(
|
||||
prompt.contains(contract),
|
||||
|
||||
@@ -19,6 +19,9 @@ pub mod model_client;
|
||||
mod paste_artifact_tool;
|
||||
pub mod profile_manifest;
|
||||
pub mod prompt;
|
||||
#[cfg(test)]
|
||||
#[path = "../../../tests/support/restore_environment.rs"]
|
||||
mod restore_test_environment;
|
||||
pub mod runtime;
|
||||
pub mod segment_log_sink;
|
||||
mod session_capture;
|
||||
@@ -69,8 +72,9 @@ pub use session_history::{
|
||||
};
|
||||
pub use shared_state::WorkerSharedState;
|
||||
pub use worker::{
|
||||
LocalWorkingDirectory, SubjektivSessionAttributionLifecycle, Worker, WorkerError,
|
||||
WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, WorkspaceBinaryRequest,
|
||||
LocalWorkingDirectory, ReviewerChildWorkspaceClient, ReviewerContext,
|
||||
SubjektivSessionAttributionLifecycle, Worker, WorkerError, WorkerFilesystemAuthority,
|
||||
WorkerRunResult, WorkerStartupFeatureCleanup, WorkerWorkspaceContext, WorkspaceBinaryRequest,
|
||||
WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
|
||||
WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse,
|
||||
WorkspaceServerOperation, WorkspaceWorkerDiscoveryRequest, apply_worker_manifest,
|
||||
|
||||
@@ -823,6 +823,20 @@ mod tests {
|
||||
assert!(prompt.contains("SubmitBackendJobResult"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flow_verifier_reuses_inspected_evidence_without_claiming_live_read_authority() {
|
||||
let prompt = PromptCatalog::builtins_only()
|
||||
.unwrap()
|
||||
.flow_verifier_system()
|
||||
.unwrap();
|
||||
assert!(prompt.contains("stable entry references"));
|
||||
assert!(prompt.contains("unchanged tree under equivalent conditions"));
|
||||
assert!(prompt.contains("never transfers an old approval or authority"));
|
||||
assert!(prompt.contains("cannot perform live MR or other-session reads"));
|
||||
assert!(prompt.contains("return `indeterminate`"));
|
||||
assert!(prompt.contains("FinishFlowVerification"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_request_prompts_keep_recording_and_reviewer_contracts() {
|
||||
let catalog = PromptCatalog::builtins_only().unwrap();
|
||||
@@ -834,7 +848,10 @@ mod tests {
|
||||
assert!(common.contains("target-only movement preserves approval"));
|
||||
assert!(reviewer.contains("complete linked MR/result snapshot"));
|
||||
assert!(reviewer.contains("Submit `ReviewMergeRequest` once"));
|
||||
assert!(reviewer.contains("without modifying implementation files"));
|
||||
assert!(reviewer.contains("Bash"));
|
||||
assert!(reviewer.contains("Git"));
|
||||
assert!(!reviewer.contains("without modifying"));
|
||||
assert!(!reviewer.contains("read-only"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -21,8 +21,9 @@ pub use error::ScopeLockError;
|
||||
#[cfg(test)]
|
||||
pub(crate) use lifecycle::install_top_level_at_for_test;
|
||||
pub use lifecycle::{
|
||||
ScopeAllocationGuard, SegmentActivationCommitError, SegmentActivationGuard, SegmentLockInfo,
|
||||
adopt_allocation, install_top_level, install_top_level_with_deny, lookup_segment,
|
||||
ScopeAllocationCleanup, ScopeAllocationGuard, SegmentActivationCommitError,
|
||||
SegmentActivationGuard, SegmentLockInfo, adopt_allocation, install_top_level,
|
||||
install_top_level_with_deny, lookup_segment,
|
||||
};
|
||||
pub use mutate::{
|
||||
delegate_scope, reclaim_delegated_scope, reclaim_stale, reclaim_stale_with, register_worker,
|
||||
|
||||
@@ -11,6 +11,14 @@ use session_store::SegmentId;
|
||||
pub enum ScopeLockError {
|
||||
#[error("I/O error on workers.json: {0}")]
|
||||
Io(#[from] io::Error),
|
||||
#[error("worker `{worker_name}` allocation still has a live guard")]
|
||||
AllocationGuardAttached { worker_name: String },
|
||||
#[error("allocation acquisition was published but its save failed: {source}")]
|
||||
AcquisitionSaveFailed {
|
||||
#[source]
|
||||
source: io::Error,
|
||||
cleanup: super::lifecycle::ScopeAllocationCleanup,
|
||||
},
|
||||
#[error("worker `{0}` is already allocated")]
|
||||
DuplicateWorkerName(String),
|
||||
#[error("requested scope `{}` conflicts with worker allocation `{competitor}` rule `{}`", .rule.target.display(), .competitor_rule.target.display())]
|
||||
@@ -46,3 +54,15 @@ pub enum ScopeLockError {
|
||||
actual: Option<SegmentId>,
|
||||
},
|
||||
}
|
||||
|
||||
impl ScopeLockError {
|
||||
/// Retain the release authority for an owned acquisition whose table rename
|
||||
/// succeeded before save returned an error. Refusals and unpublished saves
|
||||
/// do not confer ownership of any preexisting allocation.
|
||||
pub fn allocation_cleanup_handle(&self) -> Option<super::lifecycle::ScopeAllocationCleanup> {
|
||||
match self {
|
||||
Self::AcquisitionSaveFailed { cleanup, .. } => Some(cleanup.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
//! up on drop.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use manifest::ScopeRule;
|
||||
use session_store::SegmentId;
|
||||
@@ -11,13 +12,82 @@ use super::error::ScopeLockError;
|
||||
use super::mutate::release_worker;
|
||||
use super::table::{LockFileGuard, default_allocation_path};
|
||||
|
||||
/// Owned allocation: on drop, opens the lock file and releases this
|
||||
/// Worker's entry. The guard keeps only the name + lock-file path; it
|
||||
/// does not hold the `flock` for the Worker's lifetime.
|
||||
/// Owned allocation shared with any retained cleanup handles. Only the final
|
||||
/// owner's drop attempts best-effort release; no `flock` is held for its lifetime.
|
||||
#[derive(Debug)]
|
||||
pub struct ScopeAllocationGuard {
|
||||
owner: Arc<ScopeAllocationOwner>,
|
||||
}
|
||||
|
||||
/// Retains an allocation's release authority even after its Worker is consumed
|
||||
/// or dropped. A successful controller transfer should drop this handle without
|
||||
/// explicitly releasing the live Worker's allocation.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScopeAllocationCleanup {
|
||||
owner: Arc<ScopeAllocationOwner>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ScopeAllocationOwner {
|
||||
worker_name: String,
|
||||
lock_path: PathBuf,
|
||||
state: Mutex<ScopeAllocationOwnerState>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ScopeAllocationOwnerState {
|
||||
guard_attached: bool,
|
||||
released: bool,
|
||||
}
|
||||
|
||||
impl ScopeAllocationOwner {
|
||||
fn release(&self) -> Result<(), ScopeLockError> {
|
||||
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if state.guard_attached {
|
||||
return Err(ScopeLockError::AllocationGuardAttached {
|
||||
worker_name: self.worker_name.clone(),
|
||||
});
|
||||
}
|
||||
if state.released {
|
||||
return Ok(());
|
||||
}
|
||||
let mut guard = LockFileGuard::open(&self.lock_path)?;
|
||||
if guard.data().find(&self.worker_name).is_some() {
|
||||
release_worker(&mut guard, &self.worker_name)?;
|
||||
} else {
|
||||
// A previous save may have removed the entry but failed after rename.
|
||||
// Complete the durable save before reporting release as confirmed.
|
||||
guard.save()?;
|
||||
}
|
||||
state.released = true;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScopeAllocationOwner {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.release();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScopeAllocationGuard {
|
||||
fn drop(&mut self) {
|
||||
self.owner
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.guard_attached = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl ScopeAllocationCleanup {
|
||||
/// Release only after the factory has returned an error and no live
|
||||
/// Controller can still use this allocation. An attached guard rejects
|
||||
/// release. Success is idempotent across all clones; failure retains the
|
||||
/// authority for an explicit retry.
|
||||
pub fn release(&self) -> Result<(), ScopeLockError> {
|
||||
self.owner.release()
|
||||
}
|
||||
}
|
||||
|
||||
/// Exclusive authority to activate a replacement Segment for one allocated Worker.
|
||||
@@ -56,12 +126,58 @@ impl SegmentActivationCommitError {
|
||||
}
|
||||
|
||||
impl ScopeAllocationGuard {
|
||||
fn new(worker_name: String, lock_path: PathBuf) -> Self {
|
||||
Self {
|
||||
owner: Arc::new(ScopeAllocationOwner {
|
||||
worker_name,
|
||||
lock_path,
|
||||
state: Mutex::new(ScopeAllocationOwnerState {
|
||||
guard_attached: true,
|
||||
released: false,
|
||||
}),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn commit_acquisition(self, table: &mut LockFileGuard) -> Result<Self, ScopeLockError> {
|
||||
let mut published = false;
|
||||
match table.save_with_publication(&mut published) {
|
||||
Ok(()) => Ok(self),
|
||||
Err(source) => {
|
||||
let cleanup = if published {
|
||||
Some(self.cleanup_handle())
|
||||
} else {
|
||||
// A staged entry is not ours until rename. Disarm before
|
||||
// detaching so Drop cannot remove a preexisting allocation
|
||||
// (or reopen the table lock still held by this caller).
|
||||
self.owner
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.released = true;
|
||||
None
|
||||
};
|
||||
drop(self);
|
||||
match cleanup {
|
||||
Some(cleanup) => Err(ScopeLockError::AcquisitionSaveFailed { source, cleanup }),
|
||||
None => Err(ScopeLockError::Io(source)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn worker_name(&self) -> &str {
|
||||
&self.worker_name
|
||||
&self.owner.worker_name
|
||||
}
|
||||
|
||||
pub fn lock_path(&self) -> &Path {
|
||||
&self.lock_path
|
||||
&self.owner.lock_path
|
||||
}
|
||||
|
||||
pub fn cleanup_handle(&self) -> ScopeAllocationCleanup {
|
||||
ScopeAllocationCleanup {
|
||||
owner: Arc::clone(&self.owner),
|
||||
}
|
||||
}
|
||||
|
||||
/// Lock the machine-wide table and validate a Segment replacement before
|
||||
@@ -71,24 +187,25 @@ impl ScopeAllocationGuard {
|
||||
expected: SegmentId,
|
||||
replacement: SegmentId,
|
||||
) -> Result<SegmentActivationGuard, ScopeLockError> {
|
||||
let guard = LockFileGuard::open(&self.lock_path)?;
|
||||
let guard = LockFileGuard::open(self.lock_path())?;
|
||||
let actual = guard
|
||||
.data()
|
||||
.allocations
|
||||
.iter()
|
||||
.find(|allocation| allocation.worker_name == self.worker_name)
|
||||
.find(|allocation| allocation.worker_name == self.worker_name())
|
||||
.and_then(|allocation| allocation.segment_id);
|
||||
|
||||
if actual != Some(expected) {
|
||||
return Err(ScopeLockError::SegmentChanged {
|
||||
worker_name: self.worker_name.clone(),
|
||||
worker_name: self.worker_name().to_owned(),
|
||||
expected,
|
||||
actual,
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(existing) = guard.data().allocations.iter().find(|allocation| {
|
||||
allocation.segment_id == Some(replacement) && allocation.worker_name != self.worker_name
|
||||
allocation.segment_id == Some(replacement)
|
||||
&& allocation.worker_name != self.worker_name()
|
||||
}) {
|
||||
return Err(ScopeLockError::SegmentConflict {
|
||||
segment_id: replacement,
|
||||
@@ -99,7 +216,7 @@ impl ScopeAllocationGuard {
|
||||
|
||||
Ok(SegmentActivationGuard {
|
||||
guard,
|
||||
worker_name: self.worker_name.clone(),
|
||||
worker_name: self.worker_name().to_owned(),
|
||||
expected,
|
||||
replacement,
|
||||
})
|
||||
@@ -145,14 +262,6 @@ impl SegmentActivationGuard {
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScopeAllocationGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Ok(mut guard) = LockFileGuard::open(&self.lock_path) {
|
||||
let _ = release_worker(&mut guard, &self.worker_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Open the default lock file, register a top-level Worker, and return a
|
||||
/// guard that will release the allocation on drop.
|
||||
pub fn install_top_level(
|
||||
@@ -185,7 +294,7 @@ pub fn install_top_level_with_deny(
|
||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||
let lock_path = default_allocation_path()?;
|
||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||
super::mutate::register_worker_with_deny(
|
||||
super::mutate::prepare_worker_with_deny(
|
||||
&mut guard,
|
||||
worker_name.clone(),
|
||||
pid,
|
||||
@@ -194,10 +303,7 @@ pub fn install_top_level_with_deny(
|
||||
scope_deny,
|
||||
segment_id,
|
||||
)?;
|
||||
Ok(ScopeAllocationGuard {
|
||||
worker_name,
|
||||
lock_path,
|
||||
})
|
||||
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -210,7 +316,7 @@ pub(crate) fn install_top_level_at_for_test(
|
||||
segment_id: SegmentId,
|
||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||
super::mutate::register_worker_with_deny(
|
||||
super::mutate::prepare_worker_with_deny(
|
||||
&mut guard,
|
||||
worker_name.clone(),
|
||||
pid,
|
||||
@@ -219,10 +325,7 @@ pub(crate) fn install_top_level_at_for_test(
|
||||
Vec::new(),
|
||||
segment_id,
|
||||
)?;
|
||||
Ok(ScopeAllocationGuard {
|
||||
worker_name,
|
||||
lock_path,
|
||||
})
|
||||
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||
}
|
||||
|
||||
/// Take ownership of an existing allocation that was pre-registered by
|
||||
@@ -240,6 +343,15 @@ pub fn adopt_allocation(
|
||||
segment_id: SegmentId,
|
||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||
let lock_path = default_allocation_path()?;
|
||||
adopt_allocation_at(lock_path, worker_name, new_pid, segment_id)
|
||||
}
|
||||
|
||||
fn adopt_allocation_at(
|
||||
lock_path: PathBuf,
|
||||
worker_name: String,
|
||||
new_pid: u32,
|
||||
segment_id: SegmentId,
|
||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||
let alloc = guard
|
||||
.data_mut()
|
||||
@@ -247,11 +359,7 @@ pub fn adopt_allocation(
|
||||
.ok_or_else(|| ScopeLockError::UnknownWorker(worker_name.clone()))?;
|
||||
alloc.pid = new_pid;
|
||||
alloc.segment_id = Some(segment_id);
|
||||
guard.save()?;
|
||||
Ok(ScopeAllocationGuard {
|
||||
worker_name,
|
||||
lock_path,
|
||||
})
|
||||
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||
}
|
||||
|
||||
/// Information about a Worker that currently holds an allocation for a
|
||||
@@ -329,6 +437,388 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retained_cleanup_handle_defers_release_until_final_owner_drop() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let guard = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"retained".into(),
|
||||
std::process::id(),
|
||||
sock("retained"),
|
||||
Vec::new(),
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
let cleanup = guard.cleanup_handle();
|
||||
let retained = cleanup.clone();
|
||||
|
||||
drop(guard);
|
||||
drop(cleanup);
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("retained").is_some());
|
||||
}
|
||||
drop(retained);
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("retained").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dropping_cleanup_handle_after_transfer_preserves_live_guard() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let segment = sid();
|
||||
let guard = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"transferred".into(),
|
||||
std::process::id(),
|
||||
sock("transferred"),
|
||||
Vec::new(),
|
||||
segment,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
drop(guard.cleanup_handle());
|
||||
// A successful controller transfer leaves the guard usable, including
|
||||
// the existing activation protocol, without retaining a cleanup handle.
|
||||
let replacement = sid();
|
||||
guard
|
||||
.begin_segment_activation(segment, replacement)
|
||||
.unwrap()
|
||||
.commit()
|
||||
.unwrap();
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert_eq!(
|
||||
table.data().find("transferred").unwrap().segment_id,
|
||||
Some(replacement)
|
||||
);
|
||||
}
|
||||
drop(guard);
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("transferred").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cleanup_rejects_live_guard_then_releases_only_detached_allocation() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let guard = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"live".into(),
|
||||
std::process::id(),
|
||||
sock("live"),
|
||||
Vec::new(),
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
let _competitor = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"competitor".into(),
|
||||
std::process::id(),
|
||||
sock("competitor"),
|
||||
Vec::new(),
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
let cleanup = guard.cleanup_handle();
|
||||
assert!(matches!(
|
||||
cleanup.release(),
|
||||
Err(ScopeLockError::AllocationGuardAttached { worker_name }) if worker_name == "live"
|
||||
));
|
||||
drop(cleanup.clone());
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("live").is_some());
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
|
||||
drop(guard);
|
||||
cleanup.release().unwrap();
|
||||
cleanup.clone().release().unwrap();
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("live").is_none());
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn published_install_failure_retains_cleanup_across_error_drop_and_retry() {
|
||||
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let _competitor = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"competitor".into(),
|
||||
std::process::id(),
|
||||
sock("competitor"),
|
||||
Vec::new(),
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
let segment = sid();
|
||||
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||
let error = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"failed".into(),
|
||||
std::process::id(),
|
||||
sock("failed"),
|
||||
Vec::new(),
|
||||
segment,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
&error,
|
||||
ScopeLockError::AcquisitionSaveFailed { .. }
|
||||
));
|
||||
let cleanup = error.allocation_cleanup_handle().unwrap();
|
||||
let retry = error.allocation_cleanup_handle().unwrap();
|
||||
drop(error);
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert_eq!(
|
||||
table.data().find("failed").unwrap().segment_id,
|
||||
Some(segment)
|
||||
);
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
|
||||
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||
drop(cleanup);
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("failed").is_some());
|
||||
}
|
||||
retry.release().unwrap();
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("failed").is_none());
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refused_install_has_no_cleanup_authority_over_existing_allocation() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let segment = sid();
|
||||
let _competitor = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"competitor".into(),
|
||||
std::process::id(),
|
||||
sock("competitor"),
|
||||
vec![write_rule("/shared", true)],
|
||||
segment,
|
||||
)
|
||||
.unwrap();
|
||||
let cases = [
|
||||
("competitor", sid(), Vec::new()),
|
||||
("other", segment, Vec::new()),
|
||||
("other", sid(), vec![write_rule("/shared", true)]),
|
||||
];
|
||||
for (name, requested_segment, scope) in cases {
|
||||
let error = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
name.into(),
|
||||
std::process::id(),
|
||||
sock("refused"),
|
||||
scope,
|
||||
requested_segment,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
&error,
|
||||
ScopeLockError::DuplicateWorkerName(_)
|
||||
| ScopeLockError::SegmentConflict { .. }
|
||||
| ScopeLockError::WriteConflict { .. }
|
||||
));
|
||||
assert!(error.allocation_cleanup_handle().is_none());
|
||||
drop(error);
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert_eq!(table.data().allocations.len(), 1);
|
||||
assert_eq!(
|
||||
table.data().find("competitor").unwrap().socket,
|
||||
sock("competitor")
|
||||
);
|
||||
assert_eq!(
|
||||
table.data().find("competitor").unwrap().segment_id,
|
||||
Some(segment)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unpublished_install_failure_has_no_cleanup_authority() {
|
||||
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let _competitor = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"competitor".into(),
|
||||
std::process::id(),
|
||||
sock("competitor"),
|
||||
Vec::new(),
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||
let error = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"failed".into(),
|
||||
std::process::id(),
|
||||
sock("failed"),
|
||||
Vec::new(),
|
||||
sid(),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(&error, ScopeLockError::Io(_)));
|
||||
assert!(error.allocation_cleanup_handle().is_none());
|
||||
drop(error);
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("failed").is_none());
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn published_adoption_failure_retains_cleanup_for_retry() {
|
||||
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
{
|
||||
let mut table = LockFileGuard::open(&lock_path).unwrap();
|
||||
delegate_placeholder(&mut table, "child", std::process::id());
|
||||
delegate_placeholder(&mut table, "competitor", std::process::id());
|
||||
}
|
||||
let segment = sid();
|
||||
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||
let error = adopt_allocation_at(
|
||||
lock_path.clone(),
|
||||
"child".into(),
|
||||
std::process::id(),
|
||||
segment,
|
||||
)
|
||||
.unwrap_err();
|
||||
let cleanup = error.allocation_cleanup_handle().unwrap();
|
||||
drop(error);
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert_eq!(
|
||||
table.data().find("child").unwrap().segment_id,
|
||||
Some(segment)
|
||||
);
|
||||
}
|
||||
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("child").is_none());
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
cleanup.release().unwrap();
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("child").is_none());
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unpublished_adoption_failure_preserves_placeholder_without_cleanup_authority() {
|
||||
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let placeholder_pid = std::process::id();
|
||||
{
|
||||
let mut table = LockFileGuard::open(&lock_path).unwrap();
|
||||
delegate_placeholder(&mut table, "child", placeholder_pid);
|
||||
}
|
||||
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||
let error = adopt_allocation_at(
|
||||
lock_path.clone(),
|
||||
"child".into(),
|
||||
placeholder_pid.wrapping_add(1),
|
||||
sid(),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(&error, ScopeLockError::Io(_)));
|
||||
assert!(error.allocation_cleanup_handle().is_none());
|
||||
drop(error);
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
let allocation = table.data().find("child").unwrap();
|
||||
assert_eq!(allocation.pid, placeholder_pid);
|
||||
assert_eq!(allocation.segment_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_cleanup_release_retains_authority_for_retry() {
|
||||
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||
|
||||
for point in [SaveFaultPoint::BeforeRename, SaveFaultPoint::AfterRename] {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let guard = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"failed".into(),
|
||||
std::process::id(),
|
||||
sock("failed"),
|
||||
vec![read_rule("/shared", true)],
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
let _other = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"other".into(),
|
||||
std::process::id(),
|
||||
sock("other"),
|
||||
vec![read_rule("/shared", true)],
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
let cleanup = guard.cleanup_handle();
|
||||
let retry = cleanup.clone();
|
||||
drop(guard);
|
||||
fail_next_save_at_for_test(&lock_path, point);
|
||||
|
||||
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||
drop(cleanup);
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert_eq!(
|
||||
table.data().find("failed").is_some(),
|
||||
point == SaveFaultPoint::BeforeRename
|
||||
);
|
||||
assert!(table.data().find("other").is_some());
|
||||
}
|
||||
retry.release().unwrap();
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("failed").is_none());
|
||||
assert!(table.data().find("other").is_some());
|
||||
}
|
||||
|
||||
// Success disarms every clone and final Drop, even if the name is
|
||||
// subsequently allocated to a different Worker.
|
||||
let _replacement = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"failed".into(),
|
||||
std::process::id(),
|
||||
sock("replacement"),
|
||||
vec![read_rule("/shared", true)],
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
retry.clone().release().unwrap();
|
||||
drop(retry);
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert_eq!(
|
||||
table.data().find("failed").unwrap().socket,
|
||||
sock("replacement")
|
||||
);
|
||||
assert!(table.data().find("other").is_some());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adopt_allocation_rewrites_pid_and_releases_on_drop() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
@@ -358,6 +848,7 @@ mod tests {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let _sandbox = RuntimeDirSandbox::new(dir.path());
|
||||
let err = adopt_allocation("ghost".into(), 42, sid()).unwrap_err();
|
||||
assert!(err.allocation_cleanup_handle().is_none());
|
||||
assert!(matches!(err, ScopeLockError::UnknownWorker(ref n) if n == "ghost"));
|
||||
}
|
||||
|
||||
|
||||
@@ -57,6 +57,30 @@ pub fn register_worker_with_deny(
|
||||
scope_allow: Vec<ScopeRule>,
|
||||
scope_deny: Vec<ScopeRule>,
|
||||
segment_id: SegmentId,
|
||||
) -> Result<(), ScopeLockError> {
|
||||
prepare_worker_with_deny(
|
||||
guard,
|
||||
worker_name,
|
||||
pid,
|
||||
socket,
|
||||
scope_allow,
|
||||
scope_deny,
|
||||
segment_id,
|
||||
)?;
|
||||
guard.save()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Admit and stage a registration without publishing it. Owned installation
|
||||
/// constructs its cleanup authority after admission and before committing.
|
||||
pub(super) fn prepare_worker_with_deny(
|
||||
guard: &mut LockFileGuard,
|
||||
worker_name: String,
|
||||
pid: u32,
|
||||
socket: PathBuf,
|
||||
scope_allow: Vec<ScopeRule>,
|
||||
scope_deny: Vec<ScopeRule>,
|
||||
segment_id: SegmentId,
|
||||
) -> Result<(), ScopeLockError> {
|
||||
reclaim_stale(guard);
|
||||
if guard.data().find(&worker_name).is_some() {
|
||||
@@ -101,7 +125,6 @@ pub fn register_worker_with_deny(
|
||||
delegated_from: None,
|
||||
segment_id: Some(segment_id),
|
||||
});
|
||||
guard.save()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -246,6 +246,14 @@ impl LockFileGuard {
|
||||
/// Persist with atomic replacement while the separate allocation lock stays
|
||||
/// held. A crash exposes either the old complete table or the new one.
|
||||
pub fn save(&mut self) -> io::Result<()> {
|
||||
let mut published = false;
|
||||
self.save_with_publication(&mut published)
|
||||
}
|
||||
|
||||
/// Report this save's rename while the caller still holds the table lock.
|
||||
/// In-memory mutation alone is not evidence that an acquisition took place.
|
||||
pub(super) fn save_with_publication(&mut self, published: &mut bool) -> io::Result<()> {
|
||||
*published = false;
|
||||
let json = serde_json::to_vec_pretty(&self.data).map_err(io::Error::other)?;
|
||||
let mut temp_os = self.data_path.as_os_str().to_owned();
|
||||
temp_os.push(".tmp");
|
||||
@@ -265,6 +273,7 @@ impl LockFileGuard {
|
||||
));
|
||||
}
|
||||
fs::rename(&temp_path, &self.data_path)?;
|
||||
*published = true;
|
||||
#[cfg(test)]
|
||||
if take_save_fault(&self.data_path, SaveFaultPoint::AfterRename) {
|
||||
return Err(io::Error::other(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -340,6 +340,84 @@ pub struct SpawnedWorkerRegistryLoad {
|
||||
pub reclaimed_unreachable: bool,
|
||||
}
|
||||
|
||||
/// Read-only restore preparation using the registry that will be published by the Host.
|
||||
/// The Host must retain its lifecycle exclusion from preparation through application;
|
||||
/// the metadata reread detects source drift, but is not an atomic store transaction.
|
||||
/// Dropping an unapplied plan does not reclaim legacy child state.
|
||||
pub(crate) struct PreparedSpawnedWorkerRegistryLoad<St> {
|
||||
pub(crate) registry: Arc<SpawnedWorkerRegistry>,
|
||||
/// Projected outcome, not evidence that reclamation has already happened.
|
||||
pub(crate) reclaimed_unreachable: bool,
|
||||
runtime_dir: Arc<RuntimeDir>,
|
||||
store: St,
|
||||
worker_name: String,
|
||||
parent_scope: Option<SharedScope>,
|
||||
persisted_children: Vec<WorkerSpawnedChild>,
|
||||
valid_records: Vec<SpawnedWorkerRecord>,
|
||||
legacy_projection_exists: bool,
|
||||
}
|
||||
|
||||
impl<St> PreparedSpawnedWorkerRegistryLoad<St>
|
||||
where
|
||||
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
/// Apply only after all other fallible tool initialization, before socket publication.
|
||||
/// Effects retain their original order: projection, canonical reclaim history, then
|
||||
/// allocation/scope. An error may follow committed effects; no rollback is attempted.
|
||||
pub(crate) async fn apply(&mut self) -> io::Result<SpawnedWorkerRegistryLoad> {
|
||||
let metadata = self
|
||||
.store
|
||||
.read_by_name(&self.worker_name)
|
||||
.map_err(store_error_to_io)?;
|
||||
let reclaimed: Vec<_> = self
|
||||
.persisted_children
|
||||
.iter()
|
||||
.map(reclaimed_child_from_metadata)
|
||||
.collect();
|
||||
// A prior apply may have committed the canonical reclaim and then failed
|
||||
// in allocation/scope cleanup. Under the host's same lifecycle lock,
|
||||
// confirm its exact retained facts rather than appending them again.
|
||||
let already_reclaimed = !reclaimed.is_empty()
|
||||
&& metadata.as_ref().is_some_and(|metadata| {
|
||||
metadata.spawned_children.is_empty()
|
||||
&& metadata.reclaimed_children.ends_with(&reclaimed)
|
||||
});
|
||||
if !already_reclaimed
|
||||
&& metadata
|
||||
.as_ref()
|
||||
.map(|metadata| metadata.spawned_children.as_slice())
|
||||
.unwrap_or_default()
|
||||
!= self.persisted_children.as_slice()
|
||||
{
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"spawned-child metadata changed after registry preparation",
|
||||
));
|
||||
}
|
||||
|
||||
// Runtime projection is migration input only; the normal Internal registry is never
|
||||
// materialized into spawned_workers.json. Only existence was captured, not its contents.
|
||||
if !self.persisted_children.is_empty() || self.legacy_projection_exists {
|
||||
self.runtime_dir.write_spawned_workers(&[]).await?;
|
||||
}
|
||||
if !reclaimed.is_empty() && !already_reclaimed {
|
||||
self.store
|
||||
.reclaim_spawned_children(&self.worker_name, reclaimed)
|
||||
.map_err(store_error_to_io)?;
|
||||
}
|
||||
for record in &self.valid_records {
|
||||
reclaim_record(&self.worker_name, self.parent_scope.as_ref(), record)?;
|
||||
}
|
||||
|
||||
let reclaimed_unreachable = self.reclaimed_unreachable;
|
||||
self.reclaimed_unreachable = false;
|
||||
Ok(SpawnedWorkerRegistryLoad {
|
||||
registry: self.registry.clone(),
|
||||
reclaimed_unreachable,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl SpawnedWorkerRegistry {
|
||||
pub(crate) fn bash_output(&self) -> tools::bash_output::BashOutputView {
|
||||
self.bash_output.lock().unwrap().clone()
|
||||
@@ -426,6 +504,23 @@ impl SpawnedWorkerRegistry {
|
||||
worker_name: String,
|
||||
parent_scope: Option<SharedScope>,
|
||||
) -> io::Result<SpawnedWorkerRegistryLoad>
|
||||
where
|
||||
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Self::prepare_from_worker_state_with_reclaim(runtime_dir, store, worker_name, parent_scope)?
|
||||
.apply()
|
||||
.await
|
||||
}
|
||||
|
||||
/// Capture legacy child facts without starting children or changing metadata,
|
||||
/// projection, allocation, or scope. Tool initialization must clone this registry,
|
||||
/// not construct a replacement, and defer reclamation to the owned plan's `apply`.
|
||||
pub(crate) fn prepare_from_worker_state_with_reclaim<St>(
|
||||
runtime_dir: Arc<RuntimeDir>,
|
||||
store: St,
|
||||
worker_name: String,
|
||||
parent_scope: Option<SharedScope>,
|
||||
) -> io::Result<PreparedSpawnedWorkerRegistryLoad<St>>
|
||||
where
|
||||
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
@@ -442,54 +537,33 @@ impl SpawnedWorkerRegistry {
|
||||
Ok(record) => {
|
||||
warn!(
|
||||
worker = %record.worker_name,
|
||||
"reclaiming legacy persisted process SubWorker during Internal session restore"
|
||||
"captured legacy persisted process SubWorker for reclamation during Internal session restore"
|
||||
);
|
||||
valid_records.push(record);
|
||||
}
|
||||
Err(error) => warn!(
|
||||
error = %error,
|
||||
worker = %child.worker_name,
|
||||
"clearing corrupt legacy persisted process SubWorker record"
|
||||
"captured corrupt legacy persisted process SubWorker record for clearing"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// Runtime projection is migration input only; the normal Internal registry is never
|
||||
// materialized into spawned_workers.json.
|
||||
let legacy_projection_exists = runtime_dir.path().join("spawned_workers.json").exists();
|
||||
if !persisted_children.is_empty() || legacy_projection_exists {
|
||||
runtime_dir.write_spawned_workers(&[]).await?;
|
||||
}
|
||||
if !persisted_children.is_empty() {
|
||||
let reclaimed = persisted_children
|
||||
.iter()
|
||||
.map(reclaimed_child_from_metadata)
|
||||
.collect();
|
||||
store
|
||||
.reclaim_spawned_children(&worker_name, reclaimed)
|
||||
.map_err(store_error_to_io)?;
|
||||
}
|
||||
for record in &valid_records {
|
||||
reclaim_record(&worker_name, parent_scope.as_ref(), record)?;
|
||||
}
|
||||
|
||||
Ok(SpawnedWorkerRegistryLoad {
|
||||
registry: Arc::new(Self {
|
||||
internal_records: std::sync::Mutex::new(Vec::new()),
|
||||
service_records: std::sync::Mutex::new(Vec::new()),
|
||||
#[cfg(test)]
|
||||
fail_service_stops: std::sync::Mutex::new(HashMap::new()),
|
||||
internal_names: std::sync::Mutex::new(HashSet::new()),
|
||||
internal_shutting_down: AtomicBool::new(false),
|
||||
pending_internal_spawns: AtomicUsize::new(0),
|
||||
pending_internal_notify: Notify::new(),
|
||||
internal_spawn_cleanup_failed: AtomicBool::new(false),
|
||||
parent_scope,
|
||||
bash_output: Mutex::new(tools::bash_output::BashOutputView::new()),
|
||||
parent_protocol: Mutex::new(None),
|
||||
protocol_publish_lock: Arc::new(Mutex::new(())),
|
||||
}),
|
||||
let registry = match &parent_scope {
|
||||
Some(scope) => Self::new_internal(worker_name.clone(), scope.clone()),
|
||||
None => Self::new(runtime_dir.clone()),
|
||||
};
|
||||
Ok(PreparedSpawnedWorkerRegistryLoad {
|
||||
registry,
|
||||
reclaimed_unreachable: !persisted_children.is_empty(),
|
||||
runtime_dir,
|
||||
store,
|
||||
worker_name,
|
||||
parent_scope,
|
||||
persisted_children,
|
||||
valid_records,
|
||||
legacy_projection_exists,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1140,6 +1214,10 @@ impl Drop for SpawnedWorkerRegistry {
|
||||
.filter(|record| !record.scope_reclaimed.load(Ordering::Acquire))
|
||||
.flat_map(delegated_write_rules)
|
||||
.collect::<Vec<_>>();
|
||||
// Empty prepared registries own no delegations, including when abandoned or rejected.
|
||||
if write_rules.is_empty() {
|
||||
return;
|
||||
}
|
||||
let _ = parent_scope.update(|current| current.with_removed_deny_rules(write_rules));
|
||||
}
|
||||
}
|
||||
@@ -1165,8 +1243,8 @@ fn reclaim_record(
|
||||
parent_scope: Option<&SharedScope>,
|
||||
record: &SpawnedWorkerRecord,
|
||||
) -> io::Result<()> {
|
||||
if let Ok(path) = worker_allocation::default_allocation_path() {
|
||||
if let Ok(mut guard) = worker_allocation::LockFileGuard::open(&path) {
|
||||
let path = worker_allocation::default_allocation_path().map_err(io::Error::other)?;
|
||||
let mut guard = worker_allocation::LockFileGuard::open(&path).map_err(io::Error::other)?;
|
||||
match worker_allocation::reclaim_delegated_scope(
|
||||
&mut guard,
|
||||
parent_name,
|
||||
@@ -1176,8 +1254,7 @@ fn reclaim_record(
|
||||
Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {}
|
||||
Err(error) => return Err(io::Error::other(error)),
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(guard);
|
||||
if let Some(parent_scope) = parent_scope {
|
||||
let write_rules = record
|
||||
.scope_delegated
|
||||
@@ -1240,10 +1317,18 @@ fn store_error_to_io(error: WorkerStoreError) -> io::Error {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
//! Legacy restore cases connect the registry to canonical aggregate storage,
|
||||
//! runtime projections, and temporary SharedScope rules. Successful apply uses
|
||||
//! a corrupt legacy record to exercise durable clearing without consulting the
|
||||
//! process-global legacy allocation file; valid-record scope reclaim is also
|
||||
//! covered by the existing external restore test.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use manifest::{Scope, ScopeConfig};
|
||||
use session_store::LogEntry;
|
||||
use session_store::{LogEntry, WorkerAggregateStore, WorkerMetadata, WorkerSpawnedScopeRule};
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
use crate::internal_worker::{
|
||||
@@ -1251,6 +1336,249 @@ mod tests {
|
||||
test_internal_worker_session_with_greeting,
|
||||
};
|
||||
|
||||
struct LegacyRestoreFixture {
|
||||
runtime_dir: Arc<RuntimeDir>,
|
||||
store: WorkerAggregateStore,
|
||||
parent_scope: SharedScope,
|
||||
scope_root: PathBuf,
|
||||
metadata_path: PathBuf,
|
||||
// Keep all storage and scope paths alive until the other fixture fields are dropped.
|
||||
_root: TempDir,
|
||||
}
|
||||
|
||||
impl LegacyRestoreFixture {
|
||||
async fn new(permission: &str) -> Self {
|
||||
let root = TempDir::new().unwrap();
|
||||
let scope_root = root.path().join("scope");
|
||||
std::fs::create_dir(&scope_root).unwrap();
|
||||
let aggregate = root.path().join("aggregate");
|
||||
let store = WorkerAggregateStore::new(&aggregate, "parent").unwrap();
|
||||
let runtime_dir = Arc::new(
|
||||
RuntimeDir::create(&root.path().join("runtime"), "parent")
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let mut metadata =
|
||||
WorkerMetadata::new("parent", None).with_workspace_id("restore-test-workspace");
|
||||
metadata.spawned_children.push(WorkerSpawnedChild {
|
||||
worker_name: "legacy-child".into(),
|
||||
socket_path: root.path().join("legacy.sock"),
|
||||
callback_address: runtime_dir.path().join("sock"),
|
||||
scope_delegated: vec![WorkerSpawnedScopeRule {
|
||||
target: scope_root.clone(),
|
||||
permission: permission.into(),
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
}],
|
||||
});
|
||||
store.write(&metadata).unwrap();
|
||||
let write_rule = ScopeRule {
|
||||
target: scope_root.clone(),
|
||||
permission: Permission::Write,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
};
|
||||
let parent_scope = SharedScope::new(
|
||||
Scope::from_config(&ScopeConfig {
|
||||
allow: vec![write_rule.clone()],
|
||||
deny: vec![write_rule],
|
||||
})
|
||||
.unwrap(),
|
||||
);
|
||||
Self {
|
||||
runtime_dir,
|
||||
store,
|
||||
parent_scope,
|
||||
scope_root,
|
||||
metadata_path: aggregate.join("metadata.json"),
|
||||
_root: root,
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare(&self) -> PreparedSpawnedWorkerRegistryLoad<WorkerAggregateStore> {
|
||||
SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim(
|
||||
self.runtime_dir.clone(),
|
||||
self.store.clone(),
|
||||
"parent".into(),
|
||||
Some(self.parent_scope.clone()),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn projection_path(&self) -> PathBuf {
|
||||
self.runtime_dir.path().join("spawned_workers.json")
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registry_preparation_and_abandonment_leave_legacy_state_unchanged() {
|
||||
for projection_exists in [false, true] {
|
||||
let fixture = LegacyRestoreFixture::new("write").await;
|
||||
if projection_exists {
|
||||
// Projection contents are deliberately not decoded during preparation.
|
||||
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||
}
|
||||
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
|
||||
let projection_before = std::fs::read(fixture.projection_path()).ok();
|
||||
let scope_before = fixture.parent_scope.snapshot();
|
||||
assert!(!scope_before.is_writable(&fixture.scope_root));
|
||||
|
||||
let prepared = fixture.prepare();
|
||||
|
||||
assert!(prepared.reclaimed_unreachable);
|
||||
assert!(prepared.registry.list_internal().is_empty());
|
||||
assert!(prepared.registry.internal_worker_snapshots().is_empty());
|
||||
assert_eq!(
|
||||
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||
metadata_before
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(fixture.projection_path()).ok(),
|
||||
projection_before
|
||||
);
|
||||
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||
assert!(!fixture.runtime_dir.path().join("sock").exists());
|
||||
drop(prepared);
|
||||
assert_eq!(
|
||||
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||
metadata_before
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(fixture.projection_path()).ok(),
|
||||
projection_before
|
||||
);
|
||||
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_apply_clears_canonical_children_and_projection_on_same_registry() {
|
||||
let fixture = LegacyRestoreFixture::new("unsupported-legacy-permission").await;
|
||||
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||
let scope_before = fixture.parent_scope.snapshot();
|
||||
let mut prepared = fixture.prepare();
|
||||
let shared_registry = prepared.registry.clone();
|
||||
// Unrelated metadata may advance while tools are initialized.
|
||||
fixture.store.add_peer("parent", "new-peer").unwrap();
|
||||
let mut expected = fixture.store.read_by_name("parent").unwrap().unwrap();
|
||||
expected.reclaimed_children.extend(
|
||||
expected
|
||||
.spawned_children
|
||||
.iter()
|
||||
.map(reclaimed_child_from_metadata),
|
||||
);
|
||||
expected.spawned_children.clear();
|
||||
|
||||
let loaded = prepared.apply().await.unwrap();
|
||||
|
||||
assert!(loaded.reclaimed_unreachable);
|
||||
assert!(Arc::ptr_eq(&shared_registry, &loaded.registry));
|
||||
assert!(shared_registry.list_internal().is_empty());
|
||||
assert_eq!(
|
||||
fixture.store.read_by_name("parent").unwrap(),
|
||||
Some(expected)
|
||||
);
|
||||
let projection: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap();
|
||||
assert_eq!(projection, serde_json::json!([]));
|
||||
// Corrupt records are audited and cleared, not interpreted as scope grants.
|
||||
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||
assert!(
|
||||
!fixture
|
||||
.parent_scope
|
||||
.snapshot()
|
||||
.is_writable(&fixture.scope_root)
|
||||
);
|
||||
assert!(!fixture.runtime_dir.path().join("sock").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_apply_rejects_child_source_drift_before_any_effects() {
|
||||
for drift in ["scope", "socket", "removal", "metadata-deletion"] {
|
||||
let fixture = LegacyRestoreFixture::new("write").await;
|
||||
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||
let scope_before = fixture.parent_scope.snapshot();
|
||||
let mut prepared = fixture.prepare();
|
||||
if drift == "metadata-deletion" {
|
||||
fixture.store.delete_by_name("parent").unwrap();
|
||||
} else {
|
||||
fixture
|
||||
.store
|
||||
.update_by_name("parent", |metadata| match drift {
|
||||
"scope" => {
|
||||
metadata.spawned_children[0].scope_delegated[0].recursive = false
|
||||
}
|
||||
"socket" => {
|
||||
metadata.spawned_children[0].socket_path =
|
||||
fixture.runtime_dir.path().join("replacement.sock");
|
||||
}
|
||||
"removal" => metadata.spawned_children.clear(),
|
||||
_ => unreachable!(),
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
let metadata_before = std::fs::read(&fixture.metadata_path).ok();
|
||||
let projection_before = std::fs::read(fixture.projection_path()).unwrap();
|
||||
|
||||
let error = prepared
|
||||
.apply()
|
||||
.await
|
||||
.err()
|
||||
.expect("source drift must fail");
|
||||
|
||||
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
|
||||
assert_eq!(std::fs::read(&fixture.metadata_path).ok(), metadata_before);
|
||||
assert_eq!(
|
||||
std::fs::read(fixture.projection_path()).unwrap(),
|
||||
projection_before
|
||||
);
|
||||
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||
assert!(!fixture.runtime_dir.path().join("sock").exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_apply_clears_projection_only_without_reporting_child_reclaim() {
|
||||
let fixture = LegacyRestoreFixture::new("write").await;
|
||||
fixture
|
||||
.store
|
||||
.set_spawned_children("parent", Vec::new())
|
||||
.unwrap();
|
||||
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
|
||||
let mut prepared = fixture.prepare();
|
||||
assert!(!prepared.reclaimed_unreachable);
|
||||
|
||||
let loaded = prepared.apply().await.unwrap();
|
||||
|
||||
assert!(!loaded.reclaimed_unreachable);
|
||||
assert_eq!(
|
||||
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||
metadata_before
|
||||
);
|
||||
let projection: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap();
|
||||
assert_eq!(projection, serde_json::json!([]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_apply_projection_failure_keeps_canonical_children_and_scope() {
|
||||
let fixture = LegacyRestoreFixture::new("write").await;
|
||||
std::fs::create_dir(fixture.projection_path()).unwrap();
|
||||
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
|
||||
let scope_before = fixture.parent_scope.snapshot();
|
||||
let mut prepared = fixture.prepare();
|
||||
|
||||
assert!(prepared.apply().await.is_err());
|
||||
|
||||
assert_eq!(
|
||||
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||
metadata_before
|
||||
);
|
||||
assert!(fixture.projection_path().is_dir());
|
||||
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||
}
|
||||
|
||||
fn registry() -> Arc<SpawnedWorkerRegistry> {
|
||||
let scope = Scope::from_config(&ScopeConfig {
|
||||
allow: vec![ScopeRule {
|
||||
|
||||
@@ -70,7 +70,8 @@ struct SubWorkerSpawnInput {
|
||||
/// Explicitly grant command execution through the parent-owned Workdir tool broker.
|
||||
#[serde(default)]
|
||||
command: bool,
|
||||
/// Binds an actual read-only builtin Reviewer child to the current Merge Request candidate.
|
||||
/// Binds an actual builtin Reviewer child to the current Merge Request candidate.
|
||||
/// Bash may be enabled for Git and other commands.
|
||||
/// Review capability material is generated by the trusted spawn layer.
|
||||
#[serde(default)]
|
||||
review: Option<ReviewerHandoffInput>,
|
||||
@@ -1176,6 +1177,17 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reviewer_handoff_schema_allows_bash_for_git_commands() {
|
||||
let schema = serde_json::to_value(schemars::schema_for!(SubWorkerSpawnInput)).unwrap();
|
||||
let description = schema["properties"]["review"]["description"]
|
||||
.as_str()
|
||||
.expect("review handoff description");
|
||||
assert!(description.contains("Bash"));
|
||||
assert!(description.contains("Git"));
|
||||
assert!(!description.contains("read-only"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reviewer_handoff_requires_explicit_builtin_profile_and_writable_scope() {
|
||||
let valid: SubWorkerSpawnInput = serde_json::from_value(serde_json::json!({
|
||||
|
||||
+1946
-62
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,180 @@
|
||||
// Construction coverage only, not model-behavior measurement or a restored prompt update.
|
||||
use super::*;
|
||||
use crate::feature::builtin::ticket::{TicketFeature, TicketFeatureAccess};
|
||||
|
||||
#[derive(Debug)]
|
||||
struct GuidanceHost;
|
||||
impl WorkspaceClient for GuidanceHost {
|
||||
fn workspace_id(&self) -> Option<&str> {
|
||||
Some("workspace-test")
|
||||
}
|
||||
fn kind(&self) -> &str {
|
||||
"guidance-fixture"
|
||||
}
|
||||
fn is_available(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn authorize_feature_activation(&self, _feature: &str) -> Result<(), WorkspaceClientError> {
|
||||
// This fixture tests construction, not Backend authorization policy.
|
||||
Ok(())
|
||||
}
|
||||
fn execute(
|
||||
&self,
|
||||
request: WorkspaceRequest,
|
||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
if request.path.ends_with("/authorization/check") {
|
||||
return Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: "{}".into(),
|
||||
});
|
||||
}
|
||||
assert_eq!(request.path, "/api/w/workspace-test/flows/resolve");
|
||||
let definition = flow::builtin_flow_source("coder-review")
|
||||
.unwrap()
|
||||
.compile()
|
||||
.unwrap();
|
||||
let source = flow::ResolvedFlowSource {
|
||||
selector: "builtin:coder-review".parse().unwrap(),
|
||||
workspace_id: "workspace-test".into(),
|
||||
flow_id: "guidance-fixture".into(),
|
||||
content_digest: definition.content_digest.clone(),
|
||||
definition,
|
||||
};
|
||||
Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: serde_json::to_string(&source).unwrap(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fresh_builtin_profiles_materialize_validation_guidance() {
|
||||
for (profile, guidance) in [
|
||||
("coder", "own diagnosis, test selection, and fix iterations"),
|
||||
("orchestrator", "Same-named files on separate branches"),
|
||||
("reviewer", "Responsibility for the complete snapshot"),
|
||||
] {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let manifest = manifest::ProfileResolver::new()
|
||||
.with_workspace_base(dir.path())
|
||||
.resolve(
|
||||
&manifest::ProfileSelector::parse_cli(&format!("builtin:{profile}")),
|
||||
manifest::ProfileResolveOptions::with_worker_name("prompt-wiring"),
|
||||
)
|
||||
.unwrap()
|
||||
.manifest;
|
||||
let instruction = manifest.engine.instruction.clone();
|
||||
let config = manifest.feature.merge_request;
|
||||
let ticket = &manifest.feature.ticket;
|
||||
let access = TicketFeatureAccess {
|
||||
authoring: ticket.authoring,
|
||||
thread: ticket.thread,
|
||||
intake: ticket.intake,
|
||||
workflow: ticket.workflow,
|
||||
};
|
||||
let orchestration = manifest.feature.sub_worker.enabled || manifest.feature.worker.enabled;
|
||||
assert!(config.any() && ticket.enabled);
|
||||
let cwd = dir.path().join("workspace");
|
||||
std::fs::create_dir_all(&cwd).unwrap();
|
||||
let mut worker = Worker::new(
|
||||
manifest,
|
||||
Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient),
|
||||
session_store::FsStore::new(dir.path().join("sessions")).unwrap(),
|
||||
WorkerWorkspaceContext::with_client(
|
||||
Some(WorkspaceId::new("workspace-test").unwrap()),
|
||||
Arc::new(GuidanceHost),
|
||||
),
|
||||
WorkerFilesystemAuthority::local(cwd.clone(), cwd.clone()),
|
||||
Scope::writable(&cwd).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let feature = crate::feature::builtin::merge_request::MergeRequestFeature::new(
|
||||
worker.workspace_client_handle(),
|
||||
config,
|
||||
);
|
||||
let report = worker.install_features(
|
||||
FeatureRegistryBuilder::new()
|
||||
.with_module(feature)
|
||||
.with_module(TicketFeature::new(worker.workspace_client_handle(), access)),
|
||||
);
|
||||
assert!(
|
||||
!report.has_errors(),
|
||||
"{profile}: {}",
|
||||
report.error_message()
|
||||
);
|
||||
if orchestration {
|
||||
worker.register_worker_orchestration_instruction();
|
||||
}
|
||||
worker.set_system_prompt_template(
|
||||
SystemPromptTemplate::parse(
|
||||
&instruction,
|
||||
crate::prompt::source::PromptCatalogSource::builtins_only(),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
worker.ensure_system_prompt_materialized().await.unwrap();
|
||||
let prompt = worker.engine().get_system_prompt().unwrap();
|
||||
assert!(prompt.contains(guidance), "{profile}");
|
||||
assert_eq!(
|
||||
prompt.matches("Record validation evidence once").count(),
|
||||
1,
|
||||
"{profile}"
|
||||
);
|
||||
assert_eq!(
|
||||
prompt.matches("Record a Ticket dependency when").count(),
|
||||
1,
|
||||
"{profile}"
|
||||
);
|
||||
assert!(prompt.contains("rerun affected or uncertain checks"));
|
||||
assert!(prompt.contains("Changed source or requirements need fresh attestation"));
|
||||
if orchestration {
|
||||
assert!(prompt.contains("SubWorkerSpawn"));
|
||||
assert!(prompt.contains("delegated authority"));
|
||||
}
|
||||
if profile == "coder" {
|
||||
worker.ensure_segment_head().await.unwrap();
|
||||
let report = worker.install_runtime_flow_transition_feature().unwrap();
|
||||
assert!(!report.has_errors(), "{}", report.error_message());
|
||||
let (segments, state, projection) = worker
|
||||
.prepare_flow_input(vec![Segment::Flow {
|
||||
selector: "builtin:coder-review".into(),
|
||||
}])
|
||||
.unwrap();
|
||||
let state = state.expect("new Flow");
|
||||
let builtin = flow::builtin_flow_source("coder-review")
|
||||
.unwrap()
|
||||
.compile()
|
||||
.unwrap();
|
||||
let entry = worker.projected_input_history(
|
||||
&segments,
|
||||
projection.as_ref(),
|
||||
&[SessionHistoryEntryId::new()],
|
||||
&WorkerHistoryProvenance::HumanInput {
|
||||
account_id: "fixture".into(),
|
||||
},
|
||||
);
|
||||
assert_eq!(entry.len(), 1);
|
||||
assert!(matches!(
|
||||
entry[0].annotation.origin,
|
||||
WorkerHistoryProvenance::FlowInstruction { .. }
|
||||
));
|
||||
assert_eq!(
|
||||
entry[0].item.as_text().as_deref(),
|
||||
Some(
|
||||
builtin
|
||||
.state(&builtin.initial)
|
||||
.unwrap()
|
||||
.instructions
|
||||
.as_str()
|
||||
)
|
||||
);
|
||||
// The resolved immutable Flow, rather than a hard-coded test recipe, was installed.
|
||||
assert!(
|
||||
serde_json::to_string(&state)
|
||||
.unwrap()
|
||||
.contains(&builtin.content_digest)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
@@ -150,6 +151,19 @@ impl LlmClient for MockClient {
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[path = "../../../tests/support/restore_environment.rs"]
|
||||
mod restore_test_environment;
|
||||
use restore_test_environment::require_restore_test_environment;
|
||||
|
||||
fn owned_relative_runtime_base() -> (tempfile::TempDir, PathBuf) {
|
||||
require_restore_test_environment();
|
||||
let temp = tempfile::tempdir_in(".").unwrap();
|
||||
// The relative reference and the TempDir name the same owned tree, without
|
||||
// putting the checkout prefix into sockaddr_un or changing process cwd.
|
||||
let runtime_base = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
||||
(temp, runtime_base)
|
||||
}
|
||||
|
||||
fn simple_text_events() -> Vec<LlmEvent> {
|
||||
vec![
|
||||
LlmEvent::text_block_start(0),
|
||||
@@ -295,6 +309,7 @@ async fn make_worker_with_manifest_and_workspace_context(
|
||||
manifest: WorkerManifest,
|
||||
workspace_context: WorkerWorkspaceContext,
|
||||
) -> (Worker<MockClient, TestStore>, std::path::PathBuf) {
|
||||
require_restore_test_environment();
|
||||
let store_tmp = tempfile::tempdir().unwrap();
|
||||
let store = CombinedStore::new(
|
||||
FsStore::new(store_tmp.path()).unwrap(),
|
||||
@@ -320,18 +335,21 @@ async fn make_worker_with_manifest_and_workspace_context(
|
||||
}
|
||||
|
||||
async fn spawn_controller(worker: Worker<MockClient, TestStore>) -> WorkerHandle {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let runtime_base = tmp.path().to_owned();
|
||||
std::mem::forget(tmp);
|
||||
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||
let (tmp, runtime_base) = owned_relative_runtime_base();
|
||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||
.await
|
||||
.unwrap();
|
||||
// Retain ownership until controller shutdown (or test-runtime teardown).
|
||||
tokio::spawn(async move {
|
||||
let _ = shutdown_rx.await;
|
||||
drop(tmp);
|
||||
});
|
||||
handle
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (root, runtime_base) = owned_relative_runtime_base();
|
||||
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
||||
let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap();
|
||||
let store = CombinedStore::new(
|
||||
@@ -353,7 +371,7 @@ async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, &root.path().join("runtime"))
|
||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut rx = handle.subscribe();
|
||||
@@ -762,6 +780,227 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
|
||||
));
|
||||
}
|
||||
|
||||
mod registry_startup_failure {
|
||||
//! Connect real controller preparation, registry source fences and session cleanup.
|
||||
//! The Workdir telemetry boundary substitutes a concurrent metadata change at subscription;
|
||||
//! the real registry must reject drift before starting its prepared observer.
|
||||
use super::*;
|
||||
use session_store::WorkerMetadataStore;
|
||||
use tokio::sync::broadcast;
|
||||
use workdir::WorkdirSession;
|
||||
|
||||
struct ObservedSession {
|
||||
local: LocalWorkdirSession,
|
||||
metadata_store: TestStore,
|
||||
worker_name: String,
|
||||
events: broadcast::Sender<workdir::CommandEvent>,
|
||||
subscriptions: AtomicUsize,
|
||||
receivers_at_close: Mutex<Vec<usize>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ObservedSession {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("ObservedSession")
|
||||
.field("worker_name", &self.worker_name)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl WorkdirSession for ObservedSession {
|
||||
fn workdir(&self) -> &Workdir {
|
||||
self.local.workdir()
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
||||
self.local.capabilities()
|
||||
}
|
||||
|
||||
async fn stat(
|
||||
&self,
|
||||
request: workdir::StatRequest,
|
||||
) -> Result<workdir::StatResult, WorkdirError> {
|
||||
self.local.stat(request).await
|
||||
}
|
||||
|
||||
async fn read(
|
||||
&self,
|
||||
request: workdir::ReadRequest,
|
||||
) -> Result<workdir::ReadResult, WorkdirError> {
|
||||
self.local.read(request).await
|
||||
}
|
||||
|
||||
async fn write(
|
||||
&self,
|
||||
request: workdir::WriteRequest,
|
||||
) -> Result<workdir::WriteResult, WorkdirError> {
|
||||
self.local.write(request).await
|
||||
}
|
||||
|
||||
async fn edit(
|
||||
&self,
|
||||
request: workdir::EditRequest,
|
||||
) -> Result<workdir::EditResult, WorkdirError> {
|
||||
self.local.edit(request).await
|
||||
}
|
||||
|
||||
async fn list(
|
||||
&self,
|
||||
request: workdir::ListRequest,
|
||||
) -> Result<workdir::ListResult, WorkdirError> {
|
||||
self.local.list(request).await
|
||||
}
|
||||
|
||||
async fn glob(
|
||||
&self,
|
||||
request: workdir::GlobRequest,
|
||||
) -> Result<workdir::GlobResult, WorkdirError> {
|
||||
self.local.glob(request).await
|
||||
}
|
||||
|
||||
async fn grep(
|
||||
&self,
|
||||
request: workdir::GrepRequest,
|
||||
) -> Result<workdir::GrepResult, WorkdirError> {
|
||||
self.local.grep(request).await
|
||||
}
|
||||
|
||||
async fn start_command(
|
||||
&self,
|
||||
request: CommandRequest,
|
||||
) -> Result<workdir::CommandHandle, WorkdirError> {
|
||||
self.local.start_command(request).await
|
||||
}
|
||||
|
||||
async fn command_status(
|
||||
&self,
|
||||
handle: workdir::CommandHandle,
|
||||
) -> Result<workdir::CommandStatus, WorkdirError> {
|
||||
self.local.command_status(handle).await
|
||||
}
|
||||
|
||||
async fn command_output(
|
||||
&self,
|
||||
request: CommandOutputRequest,
|
||||
) -> Result<workdir::CommandOutput, WorkdirError> {
|
||||
self.local.command_output(request).await
|
||||
}
|
||||
|
||||
async fn cancel_command(&self, handle: workdir::CommandHandle) -> Result<(), WorkdirError> {
|
||||
self.local.cancel_command(handle).await
|
||||
}
|
||||
|
||||
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<workdir::CommandEvent>> {
|
||||
// Interleave a canonical change after read-only registry preparation.
|
||||
let mut metadata = self
|
||||
.metadata_store
|
||||
.read_by_name(&self.worker_name)
|
||||
.unwrap()
|
||||
.unwrap_or_else(|| session_store::WorkerMetadata::new(&self.worker_name, None));
|
||||
metadata
|
||||
.spawned_children
|
||||
.push(session_store::WorkerSpawnedChild {
|
||||
worker_name: "legacy-drift".into(),
|
||||
socket_path: PathBuf::new(),
|
||||
callback_address: PathBuf::new(),
|
||||
scope_delegated: vec![],
|
||||
});
|
||||
self.metadata_store.write(&metadata).unwrap();
|
||||
self.subscriptions.fetch_add(1, Ordering::SeqCst);
|
||||
Some(self.events.subscribe())
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<(), WorkdirError> {
|
||||
// Record before any await: aborting without joining must not pass by getting a
|
||||
// scheduler turn during the provider's asynchronous close. The sender stays alive
|
||||
// even after close, so channel closure cannot accidentally dispose of the observer.
|
||||
self.receivers_at_close
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(self.events.receiver_count());
|
||||
self.local.close().await
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn registry_drift_drops_prepared_observer_before_closing_session() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let run_dir = root.path().join("runs/1");
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
||||
let store = CombinedStore::new(
|
||||
FsStore::new(root.path().join("sessions")).unwrap(),
|
||||
FsWorkerStore::new(root.path().join("workers")).unwrap(),
|
||||
);
|
||||
let engine =
|
||||
Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
||||
client.clone(),
|
||||
);
|
||||
let mut worker = Worker::new(
|
||||
manifest,
|
||||
engine,
|
||||
store.clone(),
|
||||
WorkerWorkspaceContext::local_filesystem(None),
|
||||
WorkerFilesystemAuthority::local(root.path().into(), root.path().into()),
|
||||
manifest::Scope::writable(root.path()).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (events, unused_receiver) = broadcast::channel(8);
|
||||
drop(unused_receiver);
|
||||
let session = Arc::new(ObservedSession {
|
||||
local: LocalWorkdirSession::materialized_bound(
|
||||
Workdir::new("registry-startup-failure"),
|
||||
root.path().into(),
|
||||
root.path().into(),
|
||||
worker.scope().clone(),
|
||||
WorkdirSessionCapabilities::ALL,
|
||||
),
|
||||
metadata_store: store,
|
||||
worker_name: worker.manifest().worker.name.clone(),
|
||||
events,
|
||||
subscriptions: AtomicUsize::new(0),
|
||||
receivers_at_close: Mutex::new(Vec::new()),
|
||||
});
|
||||
worker.bind_single_workdir_session(Some(session.clone()));
|
||||
let sessions = worker.workdir_sessions();
|
||||
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
WorkerController::spawn_runtime_managed_run(worker, &run_dir),
|
||||
)
|
||||
.await
|
||||
.expect("registry rejection and cleanup must settle");
|
||||
let error = match result {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("registry source drift must reject controller startup"),
|
||||
};
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(session.subscriptions.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(*session.receivers_at_close.lock().unwrap(), vec![0]);
|
||||
assert_eq!(session.events.receiver_count(), 0);
|
||||
assert!(sessions.aliases().is_empty());
|
||||
// No controller retains the Worker/Engine after the failed startup.
|
||||
assert_eq!(Arc::strong_count(&client.call_count), 1);
|
||||
assert_eq!(client.call_count.load(Ordering::SeqCst), 0);
|
||||
assert!(!run_dir.join("worker.sock").exists());
|
||||
assert!(!run_dir.join("status.json").exists());
|
||||
assert!(!run_dir.join("manifest.toml").exists());
|
||||
assert!(matches!(
|
||||
session
|
||||
.start_command(CommandRequest {
|
||||
command: "printf unreachable".to_owned(),
|
||||
timeout_secs: 5,
|
||||
transport_byte_budget: workdir::MAX_COMMAND_TRANSPORT_BYTES,
|
||||
cwd: workdir::WorkdirPath::root(),
|
||||
tool_call_id: None,
|
||||
})
|
||||
.await,
|
||||
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||
loop {
|
||||
@@ -1561,11 +1800,12 @@ permission = "write"
|
||||
"#;
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let result = WorkerController::spawn(worker, tmp.path()).await;
|
||||
let (_tmp, runtime_base) = owned_relative_runtime_base();
|
||||
let result = WorkerController::spawn(worker, &runtime_base).await;
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"feature exposure must not imply delegation authority"
|
||||
"feature exposure must not imply delegation authority: {:?}",
|
||||
result.as_ref().err()
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,10 @@ autobins = false
|
||||
name = "yoi-server"
|
||||
path = "src/main.rs"
|
||||
|
||||
[features]
|
||||
# Offline fixture preparation only; build product binaries without this feature.
|
||||
test-support = []
|
||||
|
||||
[dependencies]
|
||||
api-macros = { workspace = true, features = ["axum"] }
|
||||
async-trait.workspace = true
|
||||
|
||||
@@ -476,7 +476,9 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
||||
(99, "Explicit hosting administrators and self-registration policy".into()),
|
||||
(100, "Provisional users and scoped passkey activation".into()),
|
||||
(101, "Durable explicit Subject body release".into()),
|
||||
(102, "Durable failed Subject creation recovery".into())
|
||||
(102, "Durable failed Subject creation recovery".into()),
|
||||
(103, "Passkey metadata and atomic addition results".into()),
|
||||
(104, "Converge Passkey results and Job input retention".into())
|
||||
]
|
||||
);
|
||||
for (table, rows) in retained.iter().zip(original.iter()) {
|
||||
|
||||
@@ -228,13 +228,21 @@ fn resolve_whoami_sync(
|
||||
use server_api::{BrowserSessionState, WhoamiResponse};
|
||||
let anonymous = || WhoamiResponse {
|
||||
actor: None,
|
||||
authentication_reference: None,
|
||||
session: BrowserSessionState::Anonymous {},
|
||||
};
|
||||
// Preserve the existing API-token precedence and fallback rules, including expired tokens.
|
||||
if let Some((actor, expires_at)) =
|
||||
resolve_credentials_sync(store, bearer, session_token.clone())?
|
||||
resolve_credentials_sync(store, bearer.clone(), session_token.clone())?
|
||||
{
|
||||
let credential = match actor.auth_method {
|
||||
ActorAuthMethod::ApiToken => bearer.as_deref(),
|
||||
ActorAuthMethod::BrowserSession => session_token.as_deref(),
|
||||
}
|
||||
.ok_or_else(|| auth_error("auth_required", "authentication binding missing"))?;
|
||||
let authentication_reference = Some(authentication_reference(&token_hash(credential)));
|
||||
return Ok(WhoamiResponse {
|
||||
authentication_reference,
|
||||
actor: Some(actor),
|
||||
session: BrowserSessionState::Authenticated { expires_at },
|
||||
});
|
||||
@@ -255,6 +263,7 @@ fn resolve_whoami_sync(
|
||||
if within_session_expiry_grace(&session.expires_at, Utc::now()) {
|
||||
return Ok(WhoamiResponse {
|
||||
actor: None,
|
||||
authentication_reference: None,
|
||||
session: BrowserSessionState::Expired {},
|
||||
});
|
||||
}
|
||||
@@ -262,6 +271,13 @@ fn resolve_whoami_sync(
|
||||
Ok(anonymous())
|
||||
}
|
||||
|
||||
/// Domain-separated public comparison value; never accepted as a credential.
|
||||
pub(crate) fn authentication_reference(credential_hash: &str) -> String {
|
||||
token_hash(&format!(
|
||||
"passkey-authentication-reference:{credential_hash}"
|
||||
))
|
||||
}
|
||||
|
||||
fn within_session_expiry_grace(expires_at: &str, now: DateTime<Utc>) -> bool {
|
||||
DateTime::parse_from_rfc3339(expires_at).is_ok_and(|expires_at| {
|
||||
expires_at <= now && now < expires_at + Duration::seconds(SESSION_EXPIRY_GRACE_SECONDS)
|
||||
|
||||
@@ -463,10 +463,52 @@ impl BackendJobRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// Availability of the immutable input body, independent of execution outcome.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum JobInputDisposition {
|
||||
Available,
|
||||
Reclaimed,
|
||||
Erased,
|
||||
Invalidated,
|
||||
}
|
||||
|
||||
impl JobInputDisposition {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Available => "available",
|
||||
Self::Reclaimed => "reclaimed",
|
||||
Self::Erased => "erased",
|
||||
Self::Invalidated => "invalidated",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Host-authorized retirement; erasure and legacy invalidation do not change outcomes.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum JobInputRetirementReason {
|
||||
Completed,
|
||||
ExplicitErasure,
|
||||
LegacyInvalidation,
|
||||
}
|
||||
|
||||
impl JobInputRetirementReason {
|
||||
pub fn disposition(self) -> JobInputDisposition {
|
||||
match self {
|
||||
Self::Completed => JobInputDisposition::Reclaimed,
|
||||
Self::ExplicitErasure => JobInputDisposition::Erased,
|
||||
Self::LegacyInvalidation => JobInputDisposition::Invalidated,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct BackendJobRecord {
|
||||
pub workspace_id: String,
|
||||
/// Receipt metadata. After retirement `input` is null; use the verified loader to execute.
|
||||
pub request: BackendJobRequest,
|
||||
pub input_digest: String,
|
||||
pub input_disposition: JobInputDisposition,
|
||||
pub intent_fingerprint: String,
|
||||
pub state: BackendJobState,
|
||||
pub current_attempt: u8,
|
||||
@@ -479,6 +521,28 @@ pub struct BackendJobRecord {
|
||||
pub completed_at: Option<String>,
|
||||
}
|
||||
|
||||
impl BackendJobRecord {
|
||||
/// Verify the original immutable body and envelope before any execution or retry.
|
||||
/// No Session read or replay request may repair unavailable/corrupt input.
|
||||
pub fn load_executable_request(&self) -> Result<&BackendJobRequest> {
|
||||
if self.input_disposition != JobInputDisposition::Available {
|
||||
return Err(Error::InvalidInput(format!(
|
||||
"Backend Job input is {}",
|
||||
self.input_disposition.as_str()
|
||||
)));
|
||||
}
|
||||
self.request.validate()?;
|
||||
if self.request.input_digest()? != self.input_digest
|
||||
|| self.request.fingerprint()? != self.intent_fingerprint
|
||||
{
|
||||
return Err(Error::Store(
|
||||
"Backend Job immutable input or intent fingerprint mismatch".into(),
|
||||
));
|
||||
}
|
||||
Ok(&self.request)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum BackendJobWorkerCleanupState {
|
||||
@@ -665,6 +729,8 @@ mod tests {
|
||||
"workspace_id": "workspace-a",
|
||||
"request": request(),
|
||||
"intent_fingerprint": request().fingerprint().unwrap(),
|
||||
"input_digest": request().input_digest().unwrap(),
|
||||
"input_disposition": "available",
|
||||
"state": "pending",
|
||||
"current_attempt": 2,
|
||||
"created_at": "created",
|
||||
@@ -1124,3 +1190,17 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const INPUT_RETENTION_TRIGGERS: &str = r#"
|
||||
CREATE TRIGGER backend_job_identity_immutable BEFORE UPDATE OF
|
||||
workspace_id, job_id, purpose, input_digest, input_ref, resource_key, intent_fingerprint ON backend_jobs
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Backend Job identity'); END;
|
||||
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE OF request_json, input_disposition ON backend_jobs
|
||||
WHEN (
|
||||
((OLD.input_disposition = 'available' AND NEW.input_disposition IN ('reclaimed','erased','invalidated'))
|
||||
OR (OLD.input_disposition IN ('reclaimed','invalidated') AND NEW.input_disposition = 'erased'))
|
||||
AND json_remove(NEW.request_json,'$.input') = json_remove(OLD.request_json,'$.input')
|
||||
AND json_type(NEW.request_json,'$.input') = 'null'
|
||||
) IS NOT TRUE
|
||||
BEGIN SELECT RAISE(ABORT, 'immutable Backend Job input retirement'); END;
|
||||
"#;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user